pipelex-api 0.87.0__tar.gz → 0.89.0__tar.gz

{pipelex_api-0.87.0 → pipelex_api-0.89.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: pipelex-api
-Version: 0.87.0
+Version: 0.89.0
 Summary: Pipelex API
 Project-URL: Homepage, https://pipelex.com
 Project-URL: Repository, https://github.com/Pipelex/pipelex
@@ -16,7 +16,7 @@ Classifier: Programming Language :: Python :: 3.13
 Classifier: Programming Language :: Python :: 3.14
 Requires-Python: <3.15,>=3.11
 Requires-Dist: fastapi<0.142,>=0.118.0
-Requires-Dist: pipelex[anthropic,bedrock,fal,google,google-genai,mistralai]==0.87.0
+Requires-Dist: pipelex[anthropic,bedrock,fal,google,google-genai,mistralai]==0.89.0
 Requires-Dist: pyjwt>=2.10.1
 Requires-Dist: uvicorn>=0.37.0
 Provides-Extra: dev

{pipelex_api-0.87.0 → pipelex_api-0.89.0}/pipelex_api/blob_store.py

@@ -17,18 +17,27 @@ full, and never by an upload: `PUT /v1/blobs/{sha256}` writes under `blobs/` alo
 be a well-formed listing proves nothing. A listing present means the whole tree is present, which lets
 `POST /v1/blobs/missing` answer a tree's digest with one lookup.
 
+Every write is create-only and checksummed, as the hosted platform writes the same keys: a blob or a listing is created
+only when its key holds nothing (`If-None-Match: *` on S3, generation 0 on GCS, a hard link on the local filesystem),
+so nothing written there is ever replaced, and with its SHA-256 as its checksum, which S3 checks against the body it
+receives. A key already holding an object is the object already held, never an error. The checksum of a blob is its
+hash, and that of a listing the hexadecimal part of its tree's digest, which is the listing's own hash, so every object
+under `blobs/<hex>` or `trees/<hex>` holds bytes whose SHA-256 is that `<hex>`, and a bucket refusing every
+unconditional write under those prefixes refuses none of this runner's.
+
 Every object under `blobs/<sha256>` holds, whole, the bytes whose SHA-256 is that hash: the key is the hash this runner
 computes from the bytes it writes, the runner's own `PUT /v1/blobs/{sha256}` writes only bytes it checked against the
-path's hash, and every built-in storage provider publishes a write whole or not at all (the local one through a
-temporary file renamed onto the key). So an object's stored size, read from its metadata without reading its bytes, is
-the size of that hash's bytes, and nothing deletes or rewrites a blob. A listing, written once each of its files was
-held at its size, therefore stays true.
+path's hash, and every built-in storage provider publishes a write whole or not at all. So an object's stored size,
+read from its metadata without reading its bytes, is the size of that hash's bytes, and nothing deletes or rewrites a
+blob. A listing, written once each of its files was held at its size, therefore stays true.
 
 From this the blob store answers without reading what it does not need:
 
 - `POST /v1/blobs/missing` lists a hash this runner does not hold at the size the sender states, an object held at
-  another size included, which only a storage that does not publish writes whole could leave short, and which the
-  upload then writes over.
+  another size included. The upload then tells the two cases apart, holding the bytes: bytes of the size held are the
+  ones held, and nothing is written, while bytes of another size mean the object held is damaged, which only a storage
+  that does not publish writes whole could leave, a 500 `BlobStoreCorrupted`, since nothing writes over a blob and its
+  operator must remove the object.
 - A run looks up each hash's stored size first. A hash held at another size than an entry states is a 422
   `BlobMismatch`, never `MissingBlobs`, since uploading those bytes again would change nothing: the entry is wrong.
   Only a blob held at the size its entries state is read, so the bytes read are bounded by the sizes the request
@@ -101,10 +110,6 @@ class RunnerBlobStore:
         except StorageFileNotFoundError:
             return None
 
-    async def holds_blob(self, *, sha256: str, size: int) -> bool:
-        """Whether the bytes whose SHA-256 is `sha256` are held at `size`, an object of another size being no blob."""
-        return await self.stored_size(sha256=sha256) == size
-
     async def stored_sizes(self, *, sha256s: Iterable[str]) -> dict[str, int | None]:
         """Each hash's stored size, None for a hash not held, looked up a bounded number at a time."""
         hashes = list(sha256s)
@@ -138,16 +143,33 @@ class RunnerBlobStore:
         return InMemoryBlobStore(blobs)
 
     async def store_blob(self, *, data: bytes) -> str:
-        """Hold `data` under its SHA-256, writing it unless it is held at its size already, and return the hash.
+        """Hold `data` under its SHA-256, creating the blob unless it is held already, and return the hash.
+
+        The blob is looked up first, so bytes already held are not sent again, then created only if its key is still
+        free, with its hash as its checksum: a writer that created it meanwhile wrote the same bytes, and the key
+        holding an object is the blob held.
 
-        An object under the hash at another size, which only a storage that does not publish writes whole could leave,
-        is written over.
+        Raises (as a problem document): a 500 `BlobStoreCorrupted` when an object of another size is held under the
+        hash, which only damaged storage could hold and which this runner never writes over.
         """
         sha256 = sha256_hex(data)
-        if not await self.holds_blob(sha256=sha256, size=len(data)):
-            await self._storage.store(data, key=self._blob_key(sha256=sha256), content_type=_BLOB_CONTENT_TYPE)
+        await self._store_hashed_blob(sha256=sha256, data=data)
         return sha256
 
+    async def _store_hashed_blob(self, *, sha256: str, data: bytes) -> None:
+        """Hold `data`, whose SHA-256 is `sha256`, as `store_blob` does, for a caller that hashed it already."""
+        held_size = await self.stored_size(sha256=sha256)
+        if held_size is None:
+            await self._storage.store_if_absent(
+                data=data, key=self._blob_key(sha256=sha256), content_type=_BLOB_CONTENT_TYPE, checksum_sha256_hex=sha256
+            )
+        elif held_size != len(data):
+            msg = (
+                f"This runner's blob store holds {held_size} bytes under {sha256}, whose bytes are {len(data)} long: its "
+                "storage is damaged, and its operator must remove that object for the blob to be uploaded again."
+            )
+            raise_internal_server_error(message=msg, error_type=ErrorType.BLOB_STORE_CORRUPTED)
+
     async def holds_tree(self, *, tree_digest: str) -> bool:
         """Whether the listing of the tree `tree_digest` names is held, which means every file of the tree is."""
         return await self._holds_key(key=self._tree_key(tree_digest=tree_digest))
@@ -173,16 +195,29 @@ class RunnerBlobStore:
         """Hold every file of a tree read in full, then its listing, so a later request can send it by hash.
 
         The listing is written last, once every file it names is held, so a listing present always means its whole
-        tree is, and a tree whose listing is held already is left as it is, without a lookup per file.
+        tree is, and a tree whose listing is held already is left as it is, without a lookup per file. The listing is
+        created only if its key is free, with the hexadecimal part of the tree's digest as its checksum, which is the
+        listing's own hash: a listing held already, whoever wrote it, is the same bytes.
 
         Args:
             tree: The tree, as a map of file entries made it.
-            contents: Each file's bytes by tree path.
+            contents: Each file's bytes by tree path. Files of the same bytes are held as one blob, written once.
         """
         if await self.holds_tree(tree_digest=tree.digest):
             return
-        await gather_bounded([partial(self.store_blob, data=data) for data in contents.values()], max_concurrency=MAX_CONCURRENT_BLOB_TRANSFERS)
-        await self._storage.store(tree_listing(tree.files), key=self._tree_key(tree_digest=tree.digest), content_type=_LISTING_CONTENT_TYPE)
+        # One write per distinct blob: two files of the same bytes, such as two empty files, name one key, and two
+        # concurrent creates of one key would race each other for nothing.
+        blobs_by_hash = {sha256_hex(data): data for data in contents.values()}
+        await gather_bounded(
+            [partial(self._store_hashed_blob, sha256=sha256, data=data) for sha256, data in blobs_by_hash.items()],
+            max_concurrency=MAX_CONCURRENT_BLOB_TRANSFERS,
+        )
+        await self._storage.store_if_absent(
+            data=tree_listing(tree.files),
+            key=self._tree_key(tree_digest=tree.digest),
+            content_type=_LISTING_CONTENT_TYPE,
+            checksum_sha256_hex=tree.digest.removeprefix(DIGEST_PREFIX),
+        )
 
     async def missing_blobs(self, *, sizes: Mapping[str, int]) -> list[str]:
         """The hashes of `sizes` not held at their size, in the mapping's order."""

{pipelex_api-0.87.0 → pipelex_api-0.89.0}/pipelex_api/error_types.py

@@ -48,6 +48,9 @@ class ErrorType(StrEnum):
     # A run request's `read_scope` extra is not one to three path-safe segments, or the run's storage
     # scope does not lie under it, so the run could not read back what it writes.
     INVALID_READ_SCOPE = "InvalidReadScope"
+    # A caller authenticated by its own token (`AUTH_MODE=jwt`) sent a `read_scope` that does not lie under its own id.
+    # A 403: the scope is well formed, but this caller may not read under it.
+    READ_SCOPE_FORBIDDEN = "ReadScopeForbidden"
     # A run request's `analytics_groups` extra is not a mapping of group type to group key within
     # the runtime's charset and entry cap (`pipelex.system.run_extras`).
     INVALID_ANALYTICS_GROUPS = "InvalidAnalyticsGroups"

{pipelex_api-0.87.0 → pipelex_api-0.89.0}/pipelex_api/method_source.py

@@ -19,16 +19,10 @@ The packages a package ships under its own `.mthds/methods/<name>/`, the layout
 its tree (a dot directory), and each becomes a dependency tree of its own, keyed by the address its manifest declares,
 as a dependency a request sends in `dependencies` is.
 
-The routes use it two ways:
-
-- The run routes and `/validate` use :func:`fetched_method`: the tree and its dependency trees, read in full, which the
-  routes write out and load as a request's `files` are (`pipelex_api.method_tree_request`), so a fetched package and a
-  sent method run the same path, the files a step reads, such as a `PipeDocGen` template, included.
-- The tooling routes (`/resolve`, `/codegen`, `/pipe-io`) use :func:`fetch_method_mthds_files`: the tree's `.mthds`
-  files, as `files[]` items labelled with their tree paths, paired with the manifest's `main_pipe` so the per-pipe route
-  defaults its selector as a run does. It reads only what the membership rule needs to tell which `.mthds` files are
-  the tree's, never the shipped packages, so it neither stages nor hashes the rest of the package. No Python ever loads
-  there, so the execution-locus gate does not apply.
+Every route taking a `method_ref` — the run routes, `/validate`, and the crate routes `/resolve`, `/codegen` and
+`/pipe-io` — uses :func:`fetched_method`: the tree and its dependency trees, read in full, which the routes write out
+and load as a request's `files` are (`pipelex_api.method_tree_request`), so a fetched package and a sent method run the
+same path, the files a step reads, such as a `PipeDocGen` template, included.
 
 The security gate (packaging invariant 7 — execution locus decides) is the request tree's
 (`pipelex_api.method_tree_request.ensure_python_allowed`): on a deployment that is not sandbox-hosted, a package
@@ -44,21 +38,17 @@ import os
 import shutil
 import subprocess
 import tempfile
-from collections.abc import Callable, Generator
+from collections.abc import Generator
 from contextlib import contextmanager
 from pathlib import Path
 from typing import NamedTuple
 
-from mthds.method_tree.discovery import BUNDLE_SUFFIX, GITIGNORE_NAME, MANIFEST_NAME
 from pipelex import log
 from pipelex.libraries.method_trees import list_package_method_tree
 from pipelex.methods.exceptions import MethodFetchError
 from pipelex.methods.fetching import FetchedMethodPackage, MethodProvenance
 from pipelex.methods.method_ref import parse_method_ref
 
-from pipelex_api.error_types import ErrorType
-from pipelex_api.errors import raise_validation_error
-from pipelex_api.limits import MAX_MTHDS_FILE_BYTES
 from pipelex_api.method_cache import get_method_clone_cache
 from pipelex_api.method_tree_request import (
     RequestMethod,
@@ -69,7 +59,6 @@ from pipelex_api.method_tree_request import (
     ensure_tree_within_limits,
     is_package_dir_name,
 )
-from pipelex_api.schemas.models import MthdsFileItem
 
 GIT_TIMEOUT_SECONDS = 60
 # The git modes of a tree entry this reader treats apart from a file: an executable file, a symbolic link, and a submodule's commit.
@@ -78,9 +67,6 @@ _GIT_MODE_SYMLINK = "120000"
 _GIT_MODE_SUBMODULE = "160000"
 # Where a package ships the method packages it calls, the layout of the local method cache.
 _VENDORED_METHODS_DIR = Path(".mthds") / "methods"
-# The files the membership rule reads to judge an entry beside the entry's own name: the patterns of a `.gitignore` file
-# and of the manifest, a manifest marking a nested package, and a `pyvenv.cfg` marking a virtual environment.
-_MEMBERSHIP_RULE_NAMES = frozenset({GITIGNORE_NAME, MANIFEST_NAME, "pyvenv.cfg"})
 
 
 class FetchedMethod(NamedTuple):
@@ -96,16 +82,6 @@ class FetchedMethod(NamedTuple):
         return f"'{self.package.full_address}'"
 
 
-class FetchedMthdsFiles(NamedTuple):
-    """A fetched package shaped for the tooling path: its `.mthds` files plus the manifest's entry pipe."""
-
-    files: list[MthdsFileItem]
-    """The package's `.mthds` files as `files[]` items, each labelled with its tree path."""
-
-    main_pipe: str | None
-    """The manifest's declared `main_pipe` (a bare pipe code), or None when the manifest declares none."""
-
-
 class _GitTreeEntry(NamedTuple):
     mode: str
     object_sha: str
@@ -200,18 +176,13 @@ def _read_git_objects(*, package: FetchedMethodPackage, object_shas: list[str])
     return contents
 
 
-def _stage_package(*, package: FetchedMethodPackage, staging_dir: Path, keeps: Callable[[str], bool] | None = None) -> None:
+def _stage_package(*, package: FetchedMethodPackage, staging_dir: Path) -> None:
     """Write the package's directory as its commit stores it into `staging_dir`, from the clone's git objects.
 
     A symbolic link is written as one, for the tree's lister to refuse unless the membership rule excludes it, and a
     submodule contributes no file.
-
-    Args:
-        package: The fetched package.
-        staging_dir: The directory to write it into.
-        keeps: Which paths, relative to the package's directory, to write; every path when None.
     """
-    entries = [entry for entry in _package_tree_entries(package=package) if keeps is None or keeps(entry.path)]
+    entries = _package_tree_entries(package=package)
     blobs = _read_git_objects(package=package, object_shas=[entry.object_sha for entry in entries if entry.mode != _GIT_MODE_SUBMODULE])
     for entry in entries:
         if entry.mode == _GIT_MODE_SUBMODULE:
@@ -241,7 +212,15 @@ def _read_listed_tree(*, root: Path, package_address: str, label: str) -> Reques
 
 
 def _read_package_method(*, package: FetchedMethodPackage) -> RequestMethod:
-    """The package's tree and the dependency trees it ships, read from the commit's git objects."""
+    """The package's tree and the dependency trees it ships, read from the commit's git objects.
+
+    The staging directory is read into memory and removed here, and a route loading the method writes the tree out a
+    second time (`written_method`), rather than loading the staged directory in place. The staged copy is the commit
+    as git holds it: files the membership rule leaves out of the tree, symlinks, and entries under `.mthds/methods/`
+    that are no package all stay in it, so loading it would load more than the tree. The tree is needed in memory
+    anyway, for the Python policy, the limits, the `include_files` echo and the blob store, and every method a route
+    loads, sent or fetched, is then written by the one `written_method`, which owns its teardown.
+    """
     staging_dir = Path(tempfile.mkdtemp(prefix="pipelex-package-"))
     try:
         package_root = staging_dir / "package"
@@ -292,27 +271,6 @@ def _fetched_package(method_ref: str) -> Generator[FetchedMethodPackage, None, N
         yield package
 
 
-def _keeps_for_bundle_listing(path: str) -> bool:
-    """Whether a package's file is one the tooling read writes: a `.mthds` file, or one the membership rule reads."""
-    name = path.rsplit("/", 1)[-1]
-    return path.endswith(BUNDLE_SUFFIX) or name in _MEMBERSHIP_RULE_NAMES
-
-
-def _read_package_bundles(*, package: FetchedMethodPackage) -> dict[str, bytes]:
-    """The `.mthds` files of the package's tree by tree path, read without staging or hashing the rest of the package.
-
-    Only the `.mthds` files and the files the membership rule reads are written out, so the lister keeps exactly the
-    `.mthds` files the whole tree would keep. The packages the package ships stay out, being under a dot directory.
-    """
-    staging_dir = Path(tempfile.mkdtemp(prefix="pipelex-package-bundles-"))
-    try:
-        _stage_package(package=package, staging_dir=staging_dir, keeps=_keeps_for_bundle_listing)
-        tree = list_package_method_tree(root=staging_dir, package_address=package.full_address)
-        return {tree_path: tree.file_location(tree_path).read_bytes() for tree_path in tree.tree_paths if tree_path.endswith(BUNDLE_SUFFIX)}
-    finally:
-        shutil.rmtree(staging_dir, ignore_errors=True)
-
-
 def fetched_method(method_ref: str) -> FetchedMethod:
     """Fetch a `method_ref`'s package and read it as a method tree, with the dependency trees it ships.
 
@@ -329,37 +287,3 @@ def fetched_method(method_ref: str) -> FetchedMethod:
         request_method = _read_package_method(package=package)
     provenance = package.provenance.model_copy(update={"tree_digest": request_method.tree.tree.digest})
     return FetchedMethod(request_method=request_method, package=package, provenance=provenance)
-
-
-def fetch_method_mthds_files(method_ref: str) -> FetchedMthdsFiles:
-    """Fetch a `method_ref`'s package and return its tree's `.mthds` files as `files[]` items, plus its `main_pipe`.
-
-    The tooling-route shape: each item pairs the file's content with its tree path as `source`, so crate provenance and
-    diagnostics carry true per-file labels. Only the package's own `.mthds` files travel, never those of a package it
-    ships under its `.mthds/methods/`, since these routes resolve no address-based dependency. The manifest's `main_pipe`
-    rides beside the files so a per-pipe projection can default its selector the way a run does.
-
-    Args:
-        method_ref: The raw `method_ref` string from the request.
-
-    Raises:
-        MethodRefError subclasses: parse, fetch, location, and bounds failures.
-        MethodTreeLoadError: The standard refused the package's tree.
-        ApiError: 422 for a package with no `.mthds` file or one this server cannot accept.
-    """
-    with _fetched_package(method_ref) as package:
-        bundles = _read_package_bundles(package=package)
-    label = f"'{package.full_address}'"
-    items: list[MthdsFileItem] = []
-    for tree_path, data in bundles.items():
-        try:
-            content = data.decode("utf-8")
-        except UnicodeDecodeError:
-            raise_validation_error(message=f"File '{tree_path}' in method package {label} is not valid UTF-8.")
-        if len(data) > MAX_MTHDS_FILE_BYTES:
-            msg = f"File '{tree_path}' in method package {label} exceeds the {MAX_MTHDS_FILE_BYTES // 1024} KiB per-file limit."
-            raise_validation_error(message=msg)
-        items.append(MthdsFileItem(content=content, source=tree_path))
-    if not items:
-        raise_validation_error(message=f"Method package {label} contains no {BUNDLE_SUFFIX} file.", error_type=ErrorType.INVALID_BUNDLE)
-    return FetchedMthdsFiles(files=items, main_pipe=package.manifest.main_pipe)

{pipelex_api-0.87.0 → pipelex_api-0.89.0}/pipelex_api/method_tree_request.py

@@ -100,6 +100,17 @@ class RequestTree(NamedTuple):
     def bundle_paths(self) -> list[str]:
         return [tree_path for tree_path in self.tree.tree_paths if tree_path.endswith(BUNDLE_SUFFIX)]
 
+    @property
+    def bundle_texts(self) -> dict[str, str]:
+        """The text of each `.mthds` bundle of the tree by its tree path, in tree path order, as a valid verdict echoes them.
+
+        Decoded strictly, so it is read only once the bundles loaded: a bundle that is not UTF-8 never loads.
+
+        Raises:
+            UnicodeDecodeError: A bundle is not UTF-8 text, which a tree that loaded never holds.
+        """
+        return {tree_path: self.contents[tree_path].decode("utf-8") for tree_path in self.bundle_paths}
+
     @classmethod
     def make_from_contents(cls, *, contents: dict[str, bytes]) -> "RequestTree":
         """The tree these files make, each at its tree path, already judged as a tree's files."""
@@ -687,7 +698,7 @@ def select_tree_entry(*, tree: RequestTree, method_label: str) -> MethodEntry:
             raise MethodEntryError(msg) from exc
         with translate_to_validate_bundle_error():
             blueprint = MthdsParser.make_pipelex_bundle_blueprint(mthds_content=content, mthds_source=tree_path)
-        candidates.append(EntryCandidate(tree_path=tree_path, blueprint=blueprint))
+        candidates.append(EntryCandidate.of_blueprint(tree_path=tree_path, blueprint=blueprint))
     manifest = manifest_of(tree=tree, label=f"the method {method_label}")
     return select_entry(candidates=candidates, manifest=manifest, method_label=method_label)
 

{pipelex_api-0.87.0 → pipelex_api-0.89.0}/pipelex_api/openapi_responses.py

@@ -124,7 +124,22 @@ PROBLEM_403_RUN_POLICY: dict[str, Any] = _problem(
     "`MethodStructuresRefusedError` — the method's Python declares structure classes (`StructuredContent` "
     "subclasses), whether it arrived as a bundle (`files` or `bundle_b64`) or as a fetched `method_ref` package: "
     "a sandbox-hosted deployment imports no caller Python into its own process, so it refuses them before "
-    "loading anything (express the types as MTHDS concepts instead).",
+    "loading anything (express the types as MTHDS concepts instead); or `ReadScopeForbidden` — on a `jwt` deployment, "
+    "the request sent a `read_scope` that does not lie under the caller's own `user_id`.",
+)
+
+PROBLEM_403_METHOD_PYTHON_POLICY: dict[str, Any] = _problem(
+    "The run routes' Python policy, applied to the method's tree: `CustomCodeRequiresSandbox` — the method (sent in "
+    "`files`, or a fetched `method_ref` package) ships custom Python and this deployment is not sandbox-hosted; or "
+    "`MethodStructuresRefusedError` — the method's Python declares structure classes (`StructuredContent` subclasses), "
+    "which a sandbox-hosted deployment never imports into its own process (express the types as MTHDS concepts instead). "
+    "Also `ReadScopeForbidden` — on a `jwt` deployment, the request sent a `read_scope` that does not lie under the "
+    "caller's own `user_id`.",
+)
+
+PROBLEM_403_READ_SCOPE: dict[str, Any] = _problem(
+    "`ReadScopeForbidden` — on a `jwt` deployment, the request sent a `read_scope` that does not lie under the caller's "
+    "own `user_id`, the only scope its token may read.",
 )
 
 PROBLEM_409_DUPLICATE_RUN: dict[str, Any] = _problem(
@@ -174,9 +189,9 @@ PROBLEM_403_BLOB_UPLOAD: dict[str, Any] = _problem(
 )
 
 PROBLEM_501_METHOD_REF: dict[str, Any] = _problem(
-    "`MethodRefNotSupported` — the request selected its closure by a **registry-form** `method_ref` (not a "
+    "`MethodRefNotSupported` — the request selected its method by a **registry-form** `method_ref` (not a "
     "`github.com/...` address), and no server-side method registry resolves those yet. Use an address-form reference "
-    "or submit inline `files[]` instead.",
+    "or send the method's tree in `files` instead.",
 )
 
 

{pipelex_api-0.87.0 → pipelex_api-0.89.0}/pipelex_api/routes/blob_upload.py

@@ -3,10 +3,11 @@
 This router mounts OUTSIDE the authenticated composite `/v1` router, like `/v1/version`: the upload is authorized by
 the signed query of its target (`pipelex_api.blob_upload`), which only a caller who passed the deployment's
 authentication on `POST /v1/blobs/missing` holds, so a sender uploads with exactly the headers the target names. The
-bytes are written under the scope the signature binds, whole: a blob is never changed, so uploading bytes the runner
-already holds at their size succeeds and changes nothing, and only bytes held short, which a storage that does not
-publish writes whole could leave, are written over. Only `blobs/` is ever written here, never a tree's listing under `trees/`,
-so an uploaded file that happens to read as a listing proves nothing.
+bytes are written under the scope the signature binds, whole and create-only (`pipelex_api.blob_store`): a blob is never
+changed, so uploading bytes the runner already holds succeeds and changes nothing, and bytes of another size than the
+object held under their hash, which only damaged storage could hold, are a 500 `BlobStoreCorrupted` rather than written
+over it. Only `blobs/` is ever written here, never a tree's listing under `trees/`, so an uploaded file that happens to
+read as a listing proves nothing.
 """
 
 from typing import Annotated
@@ -55,7 +56,8 @@ async def put_blob(
     403 `BlobUploadNotAuthorized`, as is a query that is not one this runner signs, such as an expiry that is not a
     decimal integer; a body past `MAX_METHOD_TREE_FILE_KIB` a 422 `MethodTreeTooLarge`; and a body whose SHA-256 is not
     the path's a 422 `BlobMismatch`. A blob is never changed: uploading bytes the runner already holds answers 204 and
-    changes nothing.
+    changes nothing, and uploading bytes whose hash names an object of another size, which only damaged storage could
+    hold, is a 500 `BlobStoreCorrupted` that leaves the object as it is.
     """
     if not is_upload_authorized(scope=scope, sha256=sha256, expires=expires, signature=signature) or scope is None:
         msg = "This upload carries no valid target: ask POST /v1/blobs/missing for one, and upload to the URL it names before it expires."

{pipelex_api-0.87.0 → pipelex_api-0.89.0}/pipelex_api/routes/pipelex/blobs.py

@@ -4,7 +4,8 @@ A sender about to send a method by hash first asks which of its files' bytes thi
 missing ones to the targets the answer names (`pipelex_api.routes.blob_upload`), then sends the request with hash
 entries. The blobs live in the runner's blob store (`pipelex_api.blob_store`), under the caller's scope: the request's
 `read_scope` query parameter when a multi-tenant host sends one for the organization it serves, else the caller's own
-id, the scope a run's hash entries are resolved in.
+id, the scope a run's hash entries are resolved in. A caller holding its own token (`AUTH_MODE=jwt`) may name only a
+scope under its own id (`bound_read_scope`).
 """
 
 from typing import Annotated
@@ -19,6 +20,7 @@ from pipelex_api.error_types import ErrorType
 from pipelex_api.errors import raise_validation_error
 from pipelex_api.json_body import JsonBodyRoute
 from pipelex_api.method_tree_request import ensure_within_tree_limits
+from pipelex_api.openapi_responses import PROBLEM_403_READ_SCOPE
 from pipelex_api.routes.pipelex.pipeline import resolve_blob_scope
 
 router = APIRouter(tags=["blobs"], route_class=JsonBodyRoute)
@@ -89,6 +91,7 @@ def _unique_blobs(*, blobs: list[BlobRef]) -> dict[str, int]:
     "/blobs/missing",
     summary="Which of these blobs the runner lacks, and where to upload each.",
     openapi_extra={"x-mthds-protocol": True},
+    responses={403: PROBLEM_403_READ_SCOPE},
 )
 async def find_missing_blobs(
     request: Request,
@@ -105,8 +108,10 @@ async def find_missing_blobs(
 ) -> MissingBlobsResult:
     """List every blob of the request this runner does not hold, each with a signed upload target (MTHDS Protocol).
 
-    A blob it holds at the stated size is left out, and an empty list means there is nothing to upload; bytes held
-    under a hash at another size are listed, for the upload to write over. When the request names the tree the blobs
+    A blob it holds at the stated size is left out, and an empty list means there is nothing to upload; a hash held
+    at another size than the one stated is listed too, and its upload tells the two cases apart: bytes of the size held
+    change nothing, and bytes of another size mean the object held is damaged, a 500 `BlobStoreCorrupted`, since nothing
+    writes over a blob. When the request names the tree the blobs
     make up by its `digest`, this runner has recorded that tree as held in full, which it does only after reading every
     file of a tree a request sent, and every listed blob is a file of that tree, the answer is an empty list at once.
     A hash given twice with different sizes is a 422 `BlobMismatch`, and a blob or a set of blobs past this runner's

{pipelex_api-0.87.0 → pipelex_api-0.89.0}/pipelex_api/routes/pipelex/codegen.py

@@ -1,13 +1,13 @@
 from enum import StrEnum
 from typing import Annotated, Literal, Self, Union
 
-from fastapi import APIRouter
+from fastapi import APIRouter, Request
 from fastapi.responses import JSONResponse
+from pipelex.base_exceptions import ErrorReport
 from pipelex.codegen.emission import build_stamped_projection
 from pipelex.codegen.emitters.target import CodegenKind, CodegenTarget
 from pipelex.codegen.emitters.types_emitter import emit_types
 from pipelex.codegen.lock import CODEGEN_LOCK_FILENAME
-from pipelex.pipeline.exceptions import ValidateBundleError
 from pipelex.pipeline.usage_stats_reporting import CodeGenerationReport, usage_stats_codegen_target
 from pipelex.system.usage_stats.usage_stats_contract import UsageStatsCodegenArtifact
 from pipelex.tools.misc.package_utils import get_package_version
@@ -15,15 +15,15 @@ from pipelex.tools.typing.pydantic_utils import empty_list_factory_of
 from pydantic import BaseModel, Field, model_validator
 
 from pipelex_api.json_body import JsonBodyRoute
-from pipelex_api.openapi_responses import PROBLEM_404_METHOD_PACKAGE, PROBLEM_501_METHOD_REF
+from pipelex_api.openapi_responses import PROBLEM_403_METHOD_PYTHON_POLICY, PROBLEM_404_METHOD_PACKAGE, PROBLEM_501_METHOD_REF
 from pipelex_api.routes.pipelex.crate_ops import (
     CrateInvalidReport,
     GeneratedArtifact,
+    ResolvedMethod,
     invalid_crate_report_response,
-    resolve_requested_crate,
-    teardown_current_library,
+    project_resolved_method,
 )
-from pipelex_api.schemas.models import MthdsFilesRequest
+from pipelex_api.schemas.models import MethodTreeRequest
 
 router = APIRouter(tags=["codegen"], route_class=JsonBodyRoute)
 
@@ -50,8 +50,8 @@ class CodegenRouteKind(StrEnum):
                 return CodegenKind.TYPES
 
 
-class CodegenRequest(MthdsFilesRequest):
-    """The codegen request: the shared closure selector plus the two explicit projection axes."""
+class CodegenRequest(MethodTreeRequest):
+    """The codegen request: the crate routes' method selector plus the two explicit projection axes."""
 
     kind: CodegenRouteKind = Field(..., description="What to project. `types` projects the crate's concept set into typed models.")
     target: CodegenTarget = Field(
@@ -112,17 +112,17 @@ CodegenResponse = Annotated[Union[CodegenValidReport, CrateInvalidReport], Field
 @router.post(
     "/codegen",
     response_model=CodegenResponse,
-    # On top of the composite router's shared 401/413/422/500: the `method_ref` closure selector
-    # the envelope accepts but no server-side method registry resolves yet (shared with `/resolve`).
-    responses={404: PROBLEM_404_METHOD_PACKAGE, 501: PROBLEM_501_METHOD_REF},
+    # On top of the composite router's shared 401/413/422/500: the Python policy's 403s and the
+    # `method_ref` fetch outcomes, shared with `/resolve`.
+    responses={403: PROBLEM_403_METHOD_PYTHON_POLICY, 404: PROBLEM_404_METHOD_PACKAGE, 501: PROBLEM_501_METHOD_REF},
     # NOT tagged `x-mthds-protocol` — a Pipelex API extension, like `/resolve`. The MTHDS standard
     # specifies the crate this reads (the Library Crate Format); it specifies no type projection, so
     # every `target` here — `ts-zod` and `python-pydantic` no less than `python-structures` — is ours.
 )
-async def codegen_mthds(request_data: CodegenRequest) -> JSONResponse:
-    """Generate typed artifacts from a library closure (Pipelex API extension).
+async def codegen_mthds(request: Request, request_data: CodegenRequest) -> JSONResponse:
+    """Generate typed artifacts from a method (Pipelex API extension).
 
-    Resolves the closure to its normalized crate (exactly like `POST /resolve`), then projects it
+    Resolves the method to its normalized crate (exactly like `POST /resolve`), then projects it
     through the requested `kind`/`target` axes and returns the **stamped** artifact set plus its
     `codegen.lock` — everything a client needs to materialize a byte-identical local projection
     and run the offline drift check. There is deliberately **no** server-side check route: the
@@ -134,27 +134,25 @@ async def codegen_mthds(request_data: CodegenRequest) -> JSONResponse:
     - **Invalid verdict (200, `is_valid: false`):** the library could not be parsed, loaded, or
       validated — `validation_errors[]` from pipelex's one shared builder; no artifacts exist.
     - **No verdict (non-2xx):** an unknown projection `kind`/`target`, a `pipe_ref` on a
-      concept-set-wide kind, or a malformed closure selector is a request-shape 422 problem+json;
-      `method_ref` is a 501 until server-side method registry resolution exists; auth is 401/403;
-      server fault is 5xx.
+      concept-set-wide kind, or a malformed method selector is a request-shape 422 problem+json; the
+      method's tree is refused as `/resolve` refuses it (422 `InvalidMethodFiles` or `MissingBlobs`,
+      the Python policy's 403s, the `method_ref` fetch outcomes, 501 for a registry-form
+      `method_ref`); auth is 401/403; server fault is 5xx.
     """
     with CodeGenerationReport(
         artifact=UsageStatsCodegenArtifact.TYPES, target=usage_stats_codegen_target(target=request_data.target)
     ) as generation_report:
-        return await _generate_codegen_response(request_data=request_data, generation_report=generation_report)
+        return await _generate_codegen_response(request=request, request_data=request_data, generation_report=generation_report)
 
 
-async def _generate_codegen_response(*, request_data: CodegenRequest, generation_report: CodeGenerationReport) -> JSONResponse:
+async def _generate_codegen_response(*, request: Request, request_data: CodegenRequest, generation_report: CodeGenerationReport) -> JSONResponse:
     """The body of `codegen_mthds`, timed by `generation_report`, which an invalid verdict declares failed."""
-    try:
-        # `types` is concept-set-wide: no pipe is selected, so the manifest's `main_pipe` beside the crate is unused.
-        # A future per-pipe kind takes it from the `ResolvedClosure` and hands it to `resolve_requested_pipe`.
-        crate = (await resolve_requested_crate(request_data)).crate
-    except ValidateBundleError as validate_error:
-        generation_report.declare_failed()
-        return invalid_crate_report_response(validate_error.to_error_report())
-    generation_report.describe(crate=crate)
-    try:
+
+    def _projection_response(resolution: ResolvedMethod) -> JSONResponse:
+        # `types` is concept-set-wide: no pipe is selected, so the method's manifest `main_pipe` is unused. A future
+        # per-pipe kind reads it off the resolution and hands it to `resolve_requested_pipe`.
+        crate = resolution.crate
+        generation_report.describe(crate=crate)
         emitted = emit_types(crate, target=request_data.target)
         projection = build_stamped_projection(
             emitted,
@@ -172,5 +170,9 @@ async def _generate_codegen_response(*, request_data: CodegenRequest, generation
             lock=projection.lock_content,
         )
         return JSONResponse(content=report.model_dump(mode="json", by_alias=True))
-    finally:
-        teardown_current_library()
+
+    outcome = await project_resolved_method(request=request, request_data=request_data, project=_projection_response)
+    if isinstance(outcome, ErrorReport):
+        generation_report.declare_failed()
+        return invalid_crate_report_response(outcome)
+    return outcome