pipelex-api 0.91.0__tar.gz → 0.92.1__tar.gz

{pipelex_api-0.91.0 → pipelex_api-0.92.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: pipelex-api
-Version: 0.91.0
+Version: 0.92.1
 Summary: Pipelex API
 Project-URL: Homepage, https://pipelex.com
 Project-URL: Repository, https://github.com/Pipelex/pipelex
@@ -16,7 +16,7 @@ Classifier: Programming Language :: Python :: 3.13
 Classifier: Programming Language :: Python :: 3.14
 Requires-Python: <3.15,>=3.11
 Requires-Dist: fastapi<0.142,>=0.118.0
-Requires-Dist: pipelex[anthropic,bedrock,fal,google,google-genai,mistralai]==0.91.0
+Requires-Dist: pipelex[anthropic,bedrock,fal,google,google-genai,mistralai]==0.92.1
 Requires-Dist: pyjwt>=2.10.1
 Requires-Dist: uvicorn>=0.37.0
 Provides-Extra: dev

{pipelex_api-0.91.0 → pipelex_api-0.92.1}/pipelex_api/blob_store.py

@@ -38,21 +38,24 @@ From this the blob store answers without reading what it does not need:
   ones held, and nothing is written, while bytes of another size mean the object held is damaged, which only a storage
   that does not publish writes whole could leave, a 500 `pipelex.storage.damaged`, since nothing writes over a blob
   and its operator must remove the object.
-- A run looks up each hash's stored size first. A hash held at another size than an entry states is a 422
-  `request.invalid`, never `request.blobs_missing`, since uploading those bytes again would change nothing: the entry
-  is wrong.
-  Only a blob held at the size its entries state is read, so the bytes read are bounded by the sizes the request
-  states, which its census already held to the tree's limits, and each is checked against its hash once read. Bytes
-  that do not hash to their key mean the object stored under it is damaged, which no caller can repair and no retry
-  cures: a 500 `pipelex.storage.damaged` naming the kind of storage, whose verdict is not to retry, and an error
-  record, "Stored blob bytes do not match their hash", naming the hash for the operator to remove the object, after
-  which the blob is uploaded again.
+- A run reads each hash's bytes in one request, a read of at most one byte more than the largest size its entries
+  state, so the bytes read are bounded by the sizes the request states, which its census already held to the tree's
+  limits, whatever size the object is held at. A hash held at another size than an entry states, which the length read
+  shows, is a 422 `request.invalid`, never `request.blobs_missing`, since uploading those bytes again would change
+  nothing: the entry is wrong. Only that refusal looks the size held up, for the operator's message, when the read
+  stopped short of the object's end.
+  Bytes read at the size their entries state are checked against their hash. Bytes that do not hash to their key mean
+  the object stored under it is damaged, which no caller can repair and no retry cures: a 500
+  `pipelex.storage.damaged` naming the kind of storage, whose verdict is not to retry, and an error record, "Stored blob
+  bytes do not match their hash", naming the hash for the operator to remove the object, after which the blob is
+  uploaded again.
 
 Lookups and reads run a bounded number at a time, so one request naming many blobs never opens them all at once.
 """
 
 from collections.abc import Iterable, Mapping
 from functools import partial
+from typing import NamedTuple
 
 from mthds.method_tree.digest import DIGEST_PREFIX, sha256_hex, tree_listing
 from mthds.method_tree.file_entry import BlobStore, InMemoryBlobStore
@@ -73,9 +76,11 @@ BLOBS_PREFIX = "blobs"
 TREES_PREFIX = "trees"
 _BLOB_CONTENT_TYPE = "application/octet-stream"
 _LISTING_CONTENT_TYPE = "text/plain; charset=utf-8"
-# How many lookups of a blob's size, which read metadata alone, and how many reads or writes of a blob's bytes, one
-# request runs at a time.
+# How many lookups of a blob's size, which read metadata alone, how many reads of the blobs a request names, each
+# bounded by the size the request states for it, and how many writes of a blob's bytes one request runs at a time. The
+# S3 provider's pool of connections holds at least as many as the first two (`S3_MAX_POOL_CONNECTIONS`).
 MAX_CONCURRENT_BLOB_LOOKUPS = 16
+MAX_CONCURRENT_BLOB_READS = 16
 MAX_CONCURRENT_BLOB_TRANSFERS = 4
 # The message of the error record a blob read whose bytes do not match their hash writes, for the operator to search.
 BLOB_DAMAGED_MESSAGE = "Stored blob bytes do not match their hash"
@@ -104,6 +109,16 @@ def problem_scheme_of(*, storage_method: str) -> str:
             return PLUGIN_STORAGE_SCHEME
 
 
+class StatedBlobs(NamedTuple):
+    """The blobs a map of file entries names by hash, as the blob store holds them."""
+
+    held: BlobStore
+    """The bytes of each hash held at the size its entries state, checked against the hash."""
+
+    mismatched: dict[str, int]
+    """Each hash held at another size than an entry states, with the size it is held at, in hash order."""
+
+
 class RunnerBlobStore:
     """The blobs and tree listings of one caller's scope, over the deployment's storage provider."""
 
@@ -155,37 +170,68 @@ class RunnerBlobStore:
         sizes = await gather_bounded([partial(self.stored_size, sha256=sha256) for sha256 in hashes], max_concurrency=MAX_CONCURRENT_BLOB_LOOKUPS)
         return dict(zip(hashes, sizes, strict=True))
 
-    async def _read_verified(self, *, sha256: str, size: int) -> bytes:
-        """The stored bytes of `sha256`, held at `size`, checked against their hash.
+    async def _read_bounded(self, *, sha256: str, nb_bytes: int) -> bytes | None:
+        """At most `nb_bytes` leading bytes of the blob `sha256`, read in one request, or None when the hash is not held."""
+        try:
+            return await self._storage.load_head(f"{PIPELEX_STORAGE_SCHEME}{self._blob_key(sha256=sha256)}", nb_bytes=nb_bytes)
+        except StorageFileNotFoundError:
+            return None
+
+    async def read_stated(self, *, stated_sizes: Mapping[str, set[int]]) -> StatedBlobs:
+        """Read the blobs a map of file entries names, each in one request bounded by the sizes its entries state.
+
+        Each hash is read for at most one byte more than the largest size its entries state, so a hash held at a larger
+        size shows it without its bytes being read, and the bytes read stay bounded by the sizes stated. A hash not
+        held is left out, for the reader to report among the blobs this runner does not hold.
 
-        Raises (as a problem document): a 500 `pipelex.storage.damaged` naming the kind of storage, for bytes that
-        are not `size` bytes hashing to `sha256`, after an error record naming the hash, the one place the operator
-        reads which damaged object to remove.
+        Args:
+            stated_sizes: Each hash the map names, with every size its entries state for it, which the map's census
+                held to the tree's limits.
+
+        Raises (as a problem document): a 500 `pipelex.storage.damaged` naming the kind of storage, when no hash is
+        held at another size and bytes read at the size their entries state do not hash to their key, after an error
+        record naming each such hash, the one place the operator reads which damaged object to remove.
         """
-        data = await self._storage.load(f"{PIPELEX_STORAGE_SCHEME}{self._blob_key(sha256=sha256)}")
-        if len(data) != size or sha256_hex(data) != sha256:
+        hashes = sorted(stated_sizes)
+        reads = await gather_bounded(
+            [partial(self._read_bounded, sha256=sha256, nb_bytes=max(stated_sizes[sha256]) + 1) for sha256 in hashes],
+            max_concurrency=MAX_CONCURRENT_BLOB_READS,
+        )
+        blobs: dict[str, bytes] = {}
+        mismatched: dict[str, int] = {}
+        for sha256, data in zip(hashes, reads, strict=True):
+            if data is None:
+                continue
+            sizes = stated_sizes[sha256]
+            if sizes == {len(data)}:
+                blobs[sha256] = data
+            elif len(data) <= max(sizes):
+                mismatched[sha256] = len(data)
+            else:
+                # The read stopped a byte past the largest size stated: the object is larger, and its size is looked up.
+                held_size = await self.stored_size(sha256=sha256)
+                if held_size is not None:
+                    mismatched[sha256] = held_size
+        if mismatched:
+            return StatedBlobs(held=InMemoryBlobStore(), mismatched=mismatched)
+        damaged = [sha256 for sha256, data in blobs.items() if sha256_hex(data) != sha256]
+        for sha256 in damaged:
             log.error(
                 BLOB_DAMAGED_MESSAGE,
-                fields={"blob_sha256": sha256, "blob_size_bytes": size, "read_size_bytes": len(data), "storage_scheme": self._scheme},
+                fields={
+                    "blob_sha256": sha256,
+                    "blob_size_bytes": len(blobs[sha256]),
+                    "read_size_bytes": len(blobs[sha256]),
+                    "storage_scheme": self._scheme,
+                },
             )
+        if damaged:
             msg = (
-                f"The bytes this runner's blob store holds under {sha256} do not hash to it: the storage holds a damaged "
+                f"The bytes this runner's blob store holds under {damaged[0]} do not hash to it: the storage holds a damaged "
                 "object, which its operator removes for the blob to be uploaded again."
             )
             raise BlobStoreDamagedError(msg, scheme=self._scheme)
-        return data
-
-    async def read_held(self, *, sizes: Mapping[str, int]) -> BlobStore:
-        """Read these held blobs, each at its stored size, as the synchronous blob store a map of file entries is read with.
-
-        Args:
-            sizes: Each hash to read, with the size it is held at, which the caller bounded before asking.
-        """
-        blobs = await gather_bounded(
-            [partial(self._read_verified, sha256=sha256, size=size) for sha256, size in sizes.items()],
-            max_concurrency=MAX_CONCURRENT_BLOB_TRANSFERS,
-        )
-        return InMemoryBlobStore(blobs)
+        return StatedBlobs(held=InMemoryBlobStore(list(blobs.values())), mismatched={})
 
     async def store_blob(self, *, data: bytes) -> str:
         """Hold `data` under its SHA-256, creating the blob unless it is held already, and return the hash.

{pipelex_api-0.91.0 → pipelex_api-0.92.1}/pipelex_api/method_tree_request.py

@@ -419,26 +419,27 @@ def _refuse_map(*, refusal_error: MethodTreeRefusedError, label: str, field: str
 
 
 async def _held_blobs(*, blob_store: RunnerBlobStore, stated_sizes: dict[str, set[int]], label: str, field: str) -> BlobStore:
-    """The bytes a map's hash entries name that the blob store holds, looked up by size before any is read.
+    """The bytes a map's hash entries name that the blob store holds, each read in one request bounded by its stated size.
 
     A hash held at another size than one of its entries states is refused, since uploading its bytes again would not
-    change the size the store holds them at. Every other held hash is held at the one size its entries state, which the
-    census bounded, so reading them reads no more than the census allowed. A hash not held is left out, for the reader
-    to report among the blobs this runner does not hold.
-
-    Raises (as a problem document): a 422 `request.invalid` naming the field the map rides; its message, in the operator's
-    diagnostics, lists each hash held at another size, with that size, which no parameter of the code carries.
+    change the size the store holds them at. Every hash is read for at most one byte past the largest size its entries
+    state, which the census bounded, so reading them reads no more than the census allowed. A hash not held is left
+    out, for the reader to report among the blobs this runner does not hold.
+
+    Raises (as problem documents): a 422 `request.invalid` naming the field the map rides, whose message, in the
+    operator's diagnostics, lists each hash held at another size, with that size, which no parameter of the code
+    carries; and, when no hash is, the 500 `pipelex.storage.damaged` of the blob store for bytes that do not hash to
+    their key.
     """
-    stored = await blob_store.stored_sizes(sha256s=sorted(stated_sizes))
-    mismatched = {sha256: size for sha256, size in stored.items() if size is not None and stated_sizes[sha256] != {size}}
-    if mismatched:
-        described = ", ".join(f"{sha256} (held at {size} bytes)" for sha256, size in mismatched.items())
+    stated = await blob_store.read_stated(stated_sizes=stated_sizes)
+    if stated.mismatched:
+        described = ", ".join(f"{sha256} (held at {size} bytes)" for sha256, size in stated.mismatched.items())
         msg = (
             f"{label} states a size for a hash other than the size of the bytes this runner holds under it: {described}. "
             "A hash names one set of bytes, so state the size of those bytes."
         )
         raise RunnerRequestError(msg, field=field)
-    return await blob_store.read_held(sizes={sha256: size for sha256, size in stored.items() if size is not None})
+    return stated.held
 
 
 async def read_request_tree(

{pipelex_api-0.91.0 → pipelex_api-0.92.1}/pipelex_api/routes/pipelex/pipeline.py

@@ -590,9 +590,14 @@ class ApiRunner(PipelexMTHDSProtocol):
         and it is made through `tree_listing`, which the caller hands to the request's load, so the load reads the trees
         checked here without listing them again.
 
+        A refusal of the method is the invalid verdict the load of the same directories gives, through the one translation
+        the run path's load uses (`translate_to_validate_bundle_error`): a `method.invalid` problem whose items each keep
+        their own code, a `method.dependency_missing` or a `method.package_invalid` located at `requirements.txt` and at the
+        line refused. A run answers it as a 422, and a validation as its invalid verdict.
+
         Raises:
-            MethodRequirementsFileError: The method's requirements.txt holds a line the standard refuses.
-            MethodRequirementsNotMetError: The PipeFunc environment does not meet the method's requirements.
+            ValidateBundleError: The method's requirements.txt holds a line the standard refuses, or the PipeFunc
+                environment does not meet the method's requirements, or the deployment refuses them with its own reason.
             HostLibraryTreeError: A directory of the request could not be listed as a method tree.
             PipeFuncPackageSetUnstatedError: The deployment states no package set for its PipeFunc sandbox.
         """
@@ -601,7 +606,9 @@ class ApiRunner(PipelexMTHDSProtocol):
         library_dirs = [Path(library_dir) for library_dir in self.library_dirs]
         # In the method-load pool, whose cancellation waits for the check to end: the directories it lists are removed
         # when the request's scope exits, which a client disconnect can make happen while the check still hashes them.
-        await run_method_load(ensure_served_method_requirements_met, library_dirs=library_dirs, tree_listing=tree_listing)
+        # The directories are the caller's (`library_dirs_are_callers`), so their refusal is translated as their load's is.
+        with translate_to_validate_bundle_error():
+            await run_method_load(ensure_served_method_requirements_met, library_dirs=library_dirs, tree_listing=tree_listing)
 
     async def validate_verdict(
         self,
@@ -668,11 +675,10 @@ class ApiRunner(PipelexMTHDSProtocol):
         # its PipeFunc steps run with, before any dispatch, so a refusal reaches the caller whichever validator runs. A
         # worker a method tree's validation is dispatched to never reads these directories: it receives the method's
         # trees through its method file store and judges their requirements again as it loads them. A refusal is the
-        # method's invalid verdict, as a run of it is refused before its files load.
+        # method's invalid verdict, the same one a run of it is refused with before its files load.
         tree_listing = MethodTreeListing()
         try:
-            with translate_to_validate_bundle_error():
-                await self._ensure_served_method_requirements_met(tree_listing=tree_listing)
+            await self._ensure_served_method_requirements_met(tree_listing=tree_listing)
         except ValidateBundleError as invalid_method:
             return BundleValidationFailure(
                 problem=invalid_method.to_problem(view=ProblemView.OPERATOR),