pipelex-api 0.87.0__tar.gz → 0.88.0__tar.gz

{pipelex_api-0.87.0 → pipelex_api-0.88.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: pipelex-api
-Version: 0.87.0
+Version: 0.88.0
 Summary: Pipelex API
 Project-URL: Homepage, https://pipelex.com
 Project-URL: Repository, https://github.com/Pipelex/pipelex
@@ -16,7 +16,7 @@ Classifier: Programming Language :: Python :: 3.13
 Classifier: Programming Language :: Python :: 3.14
 Requires-Python: <3.15,>=3.11
 Requires-Dist: fastapi<0.142,>=0.118.0
-Requires-Dist: pipelex[anthropic,bedrock,fal,google,google-genai,mistralai]==0.87.0
+Requires-Dist: pipelex[anthropic,bedrock,fal,google,google-genai,mistralai]==0.88.0
 Requires-Dist: pyjwt>=2.10.1
 Requires-Dist: uvicorn>=0.37.0
 Provides-Extra: dev

{pipelex_api-0.87.0 → pipelex_api-0.88.0}/pipelex_api/blob_store.py

@@ -17,18 +17,27 @@ full, and never by an upload: `PUT /v1/blobs/{sha256}` writes under `blobs/` alo
 be a well-formed listing proves nothing. A listing present means the whole tree is present, which lets
 `POST /v1/blobs/missing` answer a tree's digest with one lookup.
 
+Every write is create-only and checksummed, as the hosted platform writes the same keys: a blob or a listing is created
+only when its key holds nothing (`If-None-Match: *` on S3, generation 0 on GCS, a hard link on the local filesystem),
+so nothing written there is ever replaced, and with its SHA-256 as its checksum, which S3 checks against the body it
+receives. A key already holding an object is the object already held, never an error. The checksum of a blob is its
+hash, and that of a listing the hexadecimal part of its tree's digest, which is the listing's own hash, so every object
+under `blobs/<hex>` or `trees/<hex>` holds bytes whose SHA-256 is that `<hex>`, and a bucket refusing every
+unconditional write under those prefixes refuses none of this runner's.
+
 Every object under `blobs/<sha256>` holds, whole, the bytes whose SHA-256 is that hash: the key is the hash this runner
 computes from the bytes it writes, the runner's own `PUT /v1/blobs/{sha256}` writes only bytes it checked against the
-path's hash, and every built-in storage provider publishes a write whole or not at all (the local one through a
-temporary file renamed onto the key). So an object's stored size, read from its metadata without reading its bytes, is
-the size of that hash's bytes, and nothing deletes or rewrites a blob. A listing, written once each of its files was
-held at its size, therefore stays true.
+path's hash, and every built-in storage provider publishes a write whole or not at all. So an object's stored size,
+read from its metadata without reading its bytes, is the size of that hash's bytes, and nothing deletes or rewrites a
+blob. A listing, written once each of its files was held at its size, therefore stays true.
 
 From this the blob store answers without reading what it does not need:
 
 - `POST /v1/blobs/missing` lists a hash this runner does not hold at the size the sender states, an object held at
-  another size included, which only a storage that does not publish writes whole could leave short, and which the
-  upload then writes over.
+  another size included. The upload then tells the two cases apart, holding the bytes: bytes of the size held are the
+  ones held, and nothing is written, while bytes of another size mean the object held is damaged, which only a storage
+  that does not publish writes whole could leave, a 500 `BlobStoreCorrupted`, since nothing writes over a blob and its
+  operator must remove the object.
 - A run looks up each hash's stored size first. A hash held at another size than an entry states is a 422
   `BlobMismatch`, never `MissingBlobs`, since uploading those bytes again would change nothing: the entry is wrong.
   Only a blob held at the size its entries state is read, so the bytes read are bounded by the sizes the request
@@ -101,10 +110,6 @@ class RunnerBlobStore:
         except StorageFileNotFoundError:
             return None
 
-    async def holds_blob(self, *, sha256: str, size: int) -> bool:
-        """Whether the bytes whose SHA-256 is `sha256` are held at `size`, an object of another size being no blob."""
-        return await self.stored_size(sha256=sha256) == size
-
     async def stored_sizes(self, *, sha256s: Iterable[str]) -> dict[str, int | None]:
         """Each hash's stored size, None for a hash not held, looked up a bounded number at a time."""
         hashes = list(sha256s)
@@ -138,16 +143,33 @@ class RunnerBlobStore:
         return InMemoryBlobStore(blobs)
 
     async def store_blob(self, *, data: bytes) -> str:
-        """Hold `data` under its SHA-256, writing it unless it is held at its size already, and return the hash.
+        """Hold `data` under its SHA-256, creating the blob unless it is held already, and return the hash.
+
+        The blob is looked up first, so bytes already held are not sent again, then created only if its key is still
+        free, with its hash as its checksum: a writer that created it meanwhile wrote the same bytes, and the key
+        holding an object is the blob held.
 
-        An object under the hash at another size, which only a storage that does not publish writes whole could leave,
-        is written over.
+        Raises (as a problem document): a 500 `BlobStoreCorrupted` when an object of another size is held under the
+        hash, which only damaged storage could hold and which this runner never writes over.
         """
         sha256 = sha256_hex(data)
-        if not await self.holds_blob(sha256=sha256, size=len(data)):
-            await self._storage.store(data, key=self._blob_key(sha256=sha256), content_type=_BLOB_CONTENT_TYPE)
+        await self._store_hashed_blob(sha256=sha256, data=data)
         return sha256
 
+    async def _store_hashed_blob(self, *, sha256: str, data: bytes) -> None:
+        """Hold `data`, whose SHA-256 is `sha256`, as `store_blob` does, for a caller that hashed it already."""
+        held_size = await self.stored_size(sha256=sha256)
+        if held_size is None:
+            await self._storage.store_if_absent(
+                data=data, key=self._blob_key(sha256=sha256), content_type=_BLOB_CONTENT_TYPE, checksum_sha256_hex=sha256
+            )
+        elif held_size != len(data):
+            msg = (
+                f"This runner's blob store holds {held_size} bytes under {sha256}, whose bytes are {len(data)} long: its "
+                "storage is damaged, and its operator must remove that object for the blob to be uploaded again."
+            )
+            raise_internal_server_error(message=msg, error_type=ErrorType.BLOB_STORE_CORRUPTED)
+
     async def holds_tree(self, *, tree_digest: str) -> bool:
         """Whether the listing of the tree `tree_digest` names is held, which means every file of the tree is."""
         return await self._holds_key(key=self._tree_key(tree_digest=tree_digest))
@@ -173,16 +195,29 @@ class RunnerBlobStore:
         """Hold every file of a tree read in full, then its listing, so a later request can send it by hash.
 
         The listing is written last, once every file it names is held, so a listing present always means its whole
-        tree is, and a tree whose listing is held already is left as it is, without a lookup per file.
+        tree is, and a tree whose listing is held already is left as it is, without a lookup per file. The listing is
+        created only if its key is free, with the hexadecimal part of the tree's digest as its checksum, which is the
+        listing's own hash: a listing held already, whoever wrote it, is the same bytes.
 
         Args:
             tree: The tree, as a map of file entries made it.
-            contents: Each file's bytes by tree path.
+            contents: Each file's bytes by tree path. Files of the same bytes are held as one blob, written once.
         """
         if await self.holds_tree(tree_digest=tree.digest):
             return
-        await gather_bounded([partial(self.store_blob, data=data) for data in contents.values()], max_concurrency=MAX_CONCURRENT_BLOB_TRANSFERS)
-        await self._storage.store(tree_listing(tree.files), key=self._tree_key(tree_digest=tree.digest), content_type=_LISTING_CONTENT_TYPE)
+        # One write per distinct blob: two files of the same bytes, such as two empty files, name one key, and two
+        # concurrent creates of one key would race each other for nothing.
+        blobs_by_hash = {sha256_hex(data): data for data in contents.values()}
+        await gather_bounded(
+            [partial(self._store_hashed_blob, sha256=sha256, data=data) for sha256, data in blobs_by_hash.items()],
+            max_concurrency=MAX_CONCURRENT_BLOB_TRANSFERS,
+        )
+        await self._storage.store_if_absent(
+            data=tree_listing(tree.files),
+            key=self._tree_key(tree_digest=tree.digest),
+            content_type=_LISTING_CONTENT_TYPE,
+            checksum_sha256_hex=tree.digest.removeprefix(DIGEST_PREFIX),
+        )
 
     async def missing_blobs(self, *, sizes: Mapping[str, int]) -> list[str]:
         """The hashes of `sizes` not held at their size, in the mapping's order."""

{pipelex_api-0.87.0 → pipelex_api-0.88.0}/pipelex_api/routes/blob_upload.py

@@ -3,10 +3,11 @@
 This router mounts OUTSIDE the authenticated composite `/v1` router, like `/v1/version`: the upload is authorized by
 the signed query of its target (`pipelex_api.blob_upload`), which only a caller who passed the deployment's
 authentication on `POST /v1/blobs/missing` holds, so a sender uploads with exactly the headers the target names. The
-bytes are written under the scope the signature binds, whole: a blob is never changed, so uploading bytes the runner
-already holds at their size succeeds and changes nothing, and only bytes held short, which a storage that does not
-publish writes whole could leave, are written over. Only `blobs/` is ever written here, never a tree's listing under `trees/`,
-so an uploaded file that happens to read as a listing proves nothing.
+bytes are written under the scope the signature binds, whole and create-only (`pipelex_api.blob_store`): a blob is never
+changed, so uploading bytes the runner already holds succeeds and changes nothing, and bytes of another size than the
+object held under their hash, which only damaged storage could hold, are a 500 `BlobStoreCorrupted` rather than written
+over it. Only `blobs/` is ever written here, never a tree's listing under `trees/`, so an uploaded file that happens to
+read as a listing proves nothing.
 """
 
 from typing import Annotated
@@ -55,7 +56,8 @@ async def put_blob(
     403 `BlobUploadNotAuthorized`, as is a query that is not one this runner signs, such as an expiry that is not a
     decimal integer; a body past `MAX_METHOD_TREE_FILE_KIB` a 422 `MethodTreeTooLarge`; and a body whose SHA-256 is not
     the path's a 422 `BlobMismatch`. A blob is never changed: uploading bytes the runner already holds answers 204 and
-    changes nothing.
+    changes nothing, and uploading bytes whose hash names an object of another size, which only damaged storage could
+    hold, is a 500 `BlobStoreCorrupted` that leaves the object as it is.
     """
     if not is_upload_authorized(scope=scope, sha256=sha256, expires=expires, signature=signature) or scope is None:
         msg = "This upload carries no valid target: ask POST /v1/blobs/missing for one, and upload to the URL it names before it expires."

{pipelex_api-0.87.0 → pipelex_api-0.88.0}/pipelex_api/routes/pipelex/blobs.py

@@ -105,8 +105,10 @@ async def find_missing_blobs(
 ) -> MissingBlobsResult:
     """List every blob of the request this runner does not hold, each with a signed upload target (MTHDS Protocol).
 
-    A blob it holds at the stated size is left out, and an empty list means there is nothing to upload; bytes held
-    under a hash at another size are listed, for the upload to write over. When the request names the tree the blobs
+    A blob it holds at the stated size is left out, and an empty list means there is nothing to upload; a hash held
+    at another size than the one stated is listed too, and its upload tells the two cases apart: bytes of the size held
+    change nothing, and bytes of another size mean the object held is damaged, a 500 `BlobStoreCorrupted`, since nothing
+    writes over a blob. When the request names the tree the blobs
     make up by its `digest`, this runner has recorded that tree as held in full, which it does only after reading every
     file of a tree a request sent, and every listed blob is a file of that tree, the answer is an empty list at once.
     A hash given twice with different sizes is a 422 `BlobMismatch`, and a blob or a set of blobs past this runner's

{pipelex_api-0.87.0 → pipelex_api-0.88.0}/pipelex_api/routes/pipelex/pipeline.py

@@ -489,6 +489,8 @@ class ApiRunner(PipelexMTHDSProtocol):
             created_at=created_at,
             state=RunState.STARTED,
             workflow_id=dispatch_ack.workflow_id,
+            # The crate the job was built from, which a host records beside the run's tree digest.
+            crate_fingerprint=pipe_job.job_metadata.run_metadata.crate_fingerprint,
         )
 
     async def _ensure_served_method_requirements_met(self) -> None:

{pipelex_api-0.87.0 → pipelex_api-0.88.0}/pipelex_api/routes/pipelex/validate.py

@@ -140,7 +140,8 @@ class ValidateRequest(MthdsContentsRequest, CallerAnalyticsGroupsMixin):
         description=(
             "PIPELEX-API EXTENSION (not part of the MTHDS Protocol) — the scope of the blob store a hash entry of `files` "
             "resolves in, as a run's `read_scope`: a multi-tenant host sends the one of the organization it serves. Omit it "
-            "and the caller's own id is the scope."
+            "and the caller's own id is the scope. Only a validation carrying it records the tree it read in the blob "
+            "store, so a dry run without one leaves nothing stored."
         ),
     )
 
@@ -419,7 +420,10 @@ async def validate_mthds(request: Request, request_data: ValidateRequest) -> JSO
             method_label = REQUEST_METHOD_LABEL
             ensure_method_holds_a_bundle(request_method=request_method, label="This method")
             ensure_python_allowed(request_method=request_method, label="This method")
-            await record_request_method(request_method=request_method, blob_store=blob_store)
+            # A dry run records the tree only under the scope a host named: without one, the fallback scope is the
+            # caller's own id, a prefix no host lists or erases, and a validation must leave nothing stored there.
+            if request_data.read_scope is not None:
+                await record_request_method(request_method=request_method, blob_store=blob_store)
         validated_contents = _bundle_texts(request_method=request_method)
         # The entry pipe of the tree, the one a selector-less run executes, is the graph's pipe and the default the
         # verdict reports; a tree with none still validates.