pipelex-api 0.92.0__tar.gz → 0.92.2__tar.gz

{pipelex_api-0.92.0 → pipelex_api-0.92.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: pipelex-api
-Version: 0.92.0
+Version: 0.92.2
 Summary: Pipelex API
 Project-URL: Homepage, https://pipelex.com
 Project-URL: Repository, https://github.com/Pipelex/pipelex
@@ -16,7 +16,7 @@ Classifier: Programming Language :: Python :: 3.13
 Classifier: Programming Language :: Python :: 3.14
 Requires-Python: <3.15,>=3.11
 Requires-Dist: fastapi<0.142,>=0.118.0
-Requires-Dist: pipelex[anthropic,bedrock,fal,google,google-genai,mistralai]==0.92.0
+Requires-Dist: pipelex[anthropic,bedrock,fal,google,google-genai,mistralai]==0.92.2
 Requires-Dist: pyjwt>=2.10.1
 Requires-Dist: uvicorn>=0.37.0
 Provides-Extra: dev

{pipelex_api-0.92.0 → pipelex_api-0.92.2}/pipelex_api/blob_store.py

@@ -38,21 +38,24 @@ From this the blob store answers without reading what it does not need:
   ones held, and nothing is written, while bytes of another size mean the object held is damaged, which only a storage
   that does not publish writes whole could leave, a 500 `pipelex.storage.damaged`, since nothing writes over a blob
   and its operator must remove the object.
-- A run looks up each hash's stored size first. A hash held at another size than an entry states is a 422
-  `request.invalid`, never `request.blobs_missing`, since uploading those bytes again would change nothing: the entry
-  is wrong.
-  Only a blob held at the size its entries state is read, so the bytes read are bounded by the sizes the request
-  states, which its census already held to the tree's limits, and each is checked against its hash once read. Bytes
-  that do not hash to their key mean the object stored under it is damaged, which no caller can repair and no retry
-  cures: a 500 `pipelex.storage.damaged` naming the kind of storage, whose verdict is not to retry, and an error
-  record, "Stored blob bytes do not match their hash", naming the hash for the operator to remove the object, after
-  which the blob is uploaded again.
+- A run reads each hash's bytes in one request, a read of at most one byte more than the largest size its entries
+  state, so the bytes read are bounded by the sizes the request states, which its census already held to the tree's
+  limits, whatever size the object is held at. A hash held at another size than an entry states, which the length read
+  shows, is a 422 `request.invalid`, never `request.blobs_missing`, since uploading those bytes again would change
+  nothing: the entry is wrong. Only that refusal looks the size held up, for the operator's message, when the read
+  stopped short of the object's end.
+  Bytes read at the size their entries state are checked against their hash. Bytes that do not hash to their key mean
+  the object stored under it is damaged, which no caller can repair and no retry cures: a 500
+  `pipelex.storage.damaged` naming the kind of storage, whose verdict is not to retry, and an error record, "Stored blob
+  bytes do not match their hash", naming the hash for the operator to remove the object, after which the blob is
+  uploaded again.
 
 Lookups and reads run a bounded number at a time, so one request naming many blobs never opens them all at once.
 """
 
 from collections.abc import Iterable, Mapping
 from functools import partial
+from typing import NamedTuple
 
 from mthds.method_tree.digest import DIGEST_PREFIX, sha256_hex, tree_listing
 from mthds.method_tree.file_entry import BlobStore, InMemoryBlobStore
@@ -73,9 +76,11 @@ BLOBS_PREFIX = "blobs"
 TREES_PREFIX = "trees"
 _BLOB_CONTENT_TYPE = "application/octet-stream"
 _LISTING_CONTENT_TYPE = "text/plain; charset=utf-8"
-# How many lookups of a blob's size, which read metadata alone, and how many reads or writes of a blob's bytes, one
-# request runs at a time.
+# How many lookups of a blob's size, which read metadata alone, how many reads of the blobs a request names, each
+# bounded by the size the request states for it, and how many writes of a blob's bytes one request runs at a time. The
+# S3 provider's pool of connections holds at least as many as the first two (`S3_MAX_POOL_CONNECTIONS`).
 MAX_CONCURRENT_BLOB_LOOKUPS = 16
+MAX_CONCURRENT_BLOB_READS = 16
 MAX_CONCURRENT_BLOB_TRANSFERS = 4
 # The message of the error record a blob read whose bytes do not match their hash writes, for the operator to search.
 BLOB_DAMAGED_MESSAGE = "Stored blob bytes do not match their hash"
@@ -104,6 +109,16 @@ def problem_scheme_of(*, storage_method: str) -> str:
             return PLUGIN_STORAGE_SCHEME
 
 
+class StatedBlobs(NamedTuple):
+    """The blobs a map of file entries names by hash, as the blob store holds them."""
+
+    held: BlobStore
+    """The bytes of each hash held at the size its entries state, checked against the hash."""
+
+    mismatched: dict[str, int]
+    """Each hash held at another size than an entry states, with the size it is held at, in hash order."""
+
+
 class RunnerBlobStore:
     """The blobs and tree listings of one caller's scope, over the deployment's storage provider."""
 
@@ -155,37 +170,68 @@ class RunnerBlobStore:
         sizes = await gather_bounded([partial(self.stored_size, sha256=sha256) for sha256 in hashes], max_concurrency=MAX_CONCURRENT_BLOB_LOOKUPS)
         return dict(zip(hashes, sizes, strict=True))
 
-    async def _read_verified(self, *, sha256: str, size: int) -> bytes:
-        """The stored bytes of `sha256`, held at `size`, checked against their hash.
+    async def _read_bounded(self, *, sha256: str, nb_bytes: int) -> bytes | None:
+        """At most `nb_bytes` leading bytes of the blob `sha256`, read in one request, or None when the hash is not held."""
+        try:
+            return await self._storage.load_head(f"{PIPELEX_STORAGE_SCHEME}{self._blob_key(sha256=sha256)}", nb_bytes=nb_bytes)
+        except StorageFileNotFoundError:
+            return None
+
+    async def read_stated(self, *, stated_sizes: Mapping[str, set[int]]) -> StatedBlobs:
+        """Read the blobs a map of file entries names, each in one request bounded by the sizes its entries state.
+
+        Each hash is read for at most one byte more than the largest size its entries state, so a hash held at a larger
+        size shows it without its bytes being read, and the bytes read stay bounded by the sizes stated. A hash not
+        held is left out, for the reader to report among the blobs this runner does not hold.
 
-        Raises (as a problem document): a 500 `pipelex.storage.damaged` naming the kind of storage, for bytes that
-        are not `size` bytes hashing to `sha256`, after an error record naming the hash, the one place the operator
-        reads which damaged object to remove.
+        Args:
+            stated_sizes: Each hash the map names, with every size its entries state for it, which the map's census
+                held to the tree's limits.
+
+        Raises (as a problem document): a 500 `pipelex.storage.damaged` naming the kind of storage, when no hash is
+        held at another size and bytes read at the size their entries state do not hash to their key, after an error
+        record naming each such hash, the one place the operator reads which damaged object to remove.
         """
-        data = await self._storage.load(f"{PIPELEX_STORAGE_SCHEME}{self._blob_key(sha256=sha256)}")
-        if len(data) != size or sha256_hex(data) != sha256:
+        hashes = sorted(stated_sizes)
+        reads = await gather_bounded(
+            [partial(self._read_bounded, sha256=sha256, nb_bytes=max(stated_sizes[sha256]) + 1) for sha256 in hashes],
+            max_concurrency=MAX_CONCURRENT_BLOB_READS,
+        )
+        blobs: dict[str, bytes] = {}
+        mismatched: dict[str, int] = {}
+        for sha256, data in zip(hashes, reads, strict=True):
+            if data is None:
+                continue
+            sizes = stated_sizes[sha256]
+            if sizes == {len(data)}:
+                blobs[sha256] = data
+            elif len(data) <= max(sizes):
+                mismatched[sha256] = len(data)
+            else:
+                # The read stopped a byte past the largest size stated: the object is larger, and its size is looked up.
+                held_size = await self.stored_size(sha256=sha256)
+                if held_size is not None:
+                    mismatched[sha256] = held_size
+        if mismatched:
+            return StatedBlobs(held=InMemoryBlobStore(), mismatched=mismatched)
+        damaged = [sha256 for sha256, data in blobs.items() if sha256_hex(data) != sha256]
+        for sha256 in damaged:
             log.error(
                 BLOB_DAMAGED_MESSAGE,
-                fields={"blob_sha256": sha256, "blob_size_bytes": size, "read_size_bytes": len(data), "storage_scheme": self._scheme},
+                fields={
+                    "blob_sha256": sha256,
+                    "blob_size_bytes": len(blobs[sha256]),
+                    "read_size_bytes": len(blobs[sha256]),
+                    "storage_scheme": self._scheme,
+                },
             )
+        if damaged:
             msg = (
-                f"The bytes this runner's blob store holds under {sha256} do not hash to it: the storage holds a damaged "
+                f"The bytes this runner's blob store holds under {damaged[0]} do not hash to it: the storage holds a damaged "
                 "object, which its operator removes for the blob to be uploaded again."
             )
             raise BlobStoreDamagedError(msg, scheme=self._scheme)
-        return data
-
-    async def read_held(self, *, sizes: Mapping[str, int]) -> BlobStore:
-        """Read these held blobs, each at its stored size, as the synchronous blob store a map of file entries is read with.
-
-        Args:
-            sizes: Each hash to read, with the size it is held at, which the caller bounded before asking.
-        """
-        blobs = await gather_bounded(
-            [partial(self._read_verified, sha256=sha256, size=size) for sha256, size in sizes.items()],
-            max_concurrency=MAX_CONCURRENT_BLOB_TRANSFERS,
-        )
-        return InMemoryBlobStore(blobs)
+        return StatedBlobs(held=InMemoryBlobStore(list(blobs.values())), mismatched={})
 
     async def store_blob(self, *, data: bytes) -> str:
         """Hold `data` under its SHA-256, creating the blob unless it is held already, and return the hash.

{pipelex_api-0.92.0 → pipelex_api-0.92.2}/pipelex_api/method_source.py

@@ -13,7 +13,8 @@ commit, read from the commit's git objects as stored, with no checkout filter, l
 `export-ignore`, so its digest is the same on every machine, and filtered by the membership rule read with that
 commit's own `.gitignore` files and the package's own `exclude` patterns. A submodule contributes no file, and a
 symbolic link committed in the package is refused unless the rule excludes it. The tree's digest is recorded in the
-run's provenance (`method_provenance.tree_digest`), beside the commit.
+run's provenance (`method_provenance.tree_digest`), beside the commit and the `display_name` the package's manifest
+declares, which the fetched package's own provenance already carries.
 
 The packages a package ships under its own `.mthds/methods/<name>/`, the layout of the local method cache, are out of
 its tree (a dot directory), and each becomes a dependency tree of its own, keyed by the address its manifest declares,
@@ -291,5 +292,7 @@ def fetched_method(method_ref: str) -> FetchedMethod:
     """
     with _fetched_package(method_ref) as package:
         request_method = _read_package_method(package=package)
+    # The package's own provenance already holds the address, the tag, the commit and the manifest's display name;
+    # only the digest of the tree this runner read is added here.
     provenance = package.provenance.model_copy(update={"tree_digest": request_method.tree.tree.digest})
     return FetchedMethod(request_method=request_method, package=package, provenance=provenance)

{pipelex_api-0.92.0 → pipelex_api-0.92.2}/pipelex_api/method_tree_request.py

@@ -419,26 +419,27 @@ def _refuse_map(*, refusal_error: MethodTreeRefusedError, label: str, field: str
 
 
 async def _held_blobs(*, blob_store: RunnerBlobStore, stated_sizes: dict[str, set[int]], label: str, field: str) -> BlobStore:
-    """The bytes a map's hash entries name that the blob store holds, looked up by size before any is read.
+    """The bytes a map's hash entries name that the blob store holds, each read in one request bounded by its stated size.
 
     A hash held at another size than one of its entries states is refused, since uploading its bytes again would not
-    change the size the store holds them at. Every other held hash is held at the one size its entries state, which the
-    census bounded, so reading them reads no more than the census allowed. A hash not held is left out, for the reader
-    to report among the blobs this runner does not hold.
-
-    Raises (as a problem document): a 422 `request.invalid` naming the field the map rides; its message, in the operator's
-    diagnostics, lists each hash held at another size, with that size, which no parameter of the code carries.
+    change the size the store holds them at. Every hash is read for at most one byte past the largest size its entries
+    state, which the census bounded, so reading them reads no more than the census allowed. A hash not held is left
+    out, for the reader to report among the blobs this runner does not hold.
+
+    Raises (as problem documents): a 422 `request.invalid` naming the field the map rides, whose message, in the
+    operator's diagnostics, lists each hash held at another size, with that size, which no parameter of the code
+    carries; and, when no hash is, the 500 `pipelex.storage.damaged` of the blob store for bytes that do not hash to
+    their key.
     """
-    stored = await blob_store.stored_sizes(sha256s=sorted(stated_sizes))
-    mismatched = {sha256: size for sha256, size in stored.items() if size is not None and stated_sizes[sha256] != {size}}
-    if mismatched:
-        described = ", ".join(f"{sha256} (held at {size} bytes)" for sha256, size in mismatched.items())
+    stated = await blob_store.read_stated(stated_sizes=stated_sizes)
+    if stated.mismatched:
+        described = ", ".join(f"{sha256} (held at {size} bytes)" for sha256, size in stated.mismatched.items())
         msg = (
             f"{label} states a size for a hash other than the size of the bytes this runner holds under it: {described}. "
             "A hash names one set of bytes, so state the size of those bytes."
         )
         raise RunnerRequestError(msg, field=field)
-    return await blob_store.read_held(sizes={sha256: size for sha256, size in stored.items() if size is not None})
+    return stated.held
 
 
 async def read_request_tree(

{pipelex_api-0.92.0 → pipelex_api-0.92.2}/pipelex_api/routes/pipelex/pipeline.py

@@ -1043,8 +1043,9 @@ async def execute(request: Request) -> JSONResponse:
     # `TokensUsageRecord` wire shape instead (pipelex owns the shape authority).
     response_dump = response.model_dump(mode="json", serialize_as_any=True, by_alias=True)
     apply_tokens_usage_wire_shape(response_dump, pipe_output=response.pipe_output)
-    # Provenance of a `method_ref` run (address, tag, resolved commit SHA) — attached only when
-    # the run was selected by reference, so the classic response stays byte-identical.
+    # Provenance of a `method_ref` run (address, tag, resolved commit SHA, tree digest, the manifest's
+    # display name) — attached only when the run was selected by reference, so the classic response
+    # stays byte-identical.
     if source.method_provenance is not None:
         response_dump["method_provenance"] = source.method_provenance.model_dump(mode="json")
     return JSONResponse(content=response_dump)
@@ -1140,9 +1141,10 @@ async def start(
             requested_orchestration_mode=extras.orchestration_mode,
             callback_problem_view=problem_view_of(request),
         )
-    # The ack plus this server's provenance extension: `(address, tag, commit_sha)` for a
-    # `method_ref` run, null otherwise — the caller (and, on hosted, the platform's Run row)
-    # learns exactly what was fetched and at which commit.
+    # The ack plus this server's provenance extension: `(address, tag, commit_sha, tree_digest,
+    # display_name)` for a `method_ref` run, null otherwise — the caller (and, on hosted, the
+    # platform's Run row) learns exactly what was fetched, at which commit, and what the package
+    # calls itself.
     return PipelexApiStartResponse(
         **dict(start_result),
         method_provenance=source.method_provenance,

{pipelex_api-0.92.0 → pipelex_api-0.92.2}/pipelex_api/schemas/models.py

@@ -615,9 +615,10 @@ class PipeOutputWire(PipeOutput):
 
 _METHOD_PROVENANCE_DESCRIPTION = (
     "PIPELEX-API EXTENSION (not part of the MTHDS Protocol) — provenance of a `method_ref` run: the package's "
-    "resolved full address, the requested tag (null for a bare address), and the commit SHA that was actually "
-    "fetched. The SHA is what keeps the run explainable when a tag moves. Absent (or null) for runs from inline "
-    "source or a bundle."
+    "resolved full address, the requested tag (null for a bare address), the commit SHA that was actually "
+    "fetched, the digest of the method tree the run loaded, and the `display_name` the package's manifest declares "
+    "(null when it declares none). The SHA is what keeps the run explainable when a tag moves. Absent (or null) for "
+    "runs from inline source or a bundle."
 )