tensorfs 0.3.44__tar.gz → 0.3.45__tar.gz

{tensorfs-0.3.44 → tensorfs-0.3.45}/Cargo.lock

@@ -421,7 +421,7 @@ checksum = "adb6935a6f5c20170eeceb1a3835a49e12e19d792f6dd344ccc76a985ca5a6ca"
 
 [[package]]
 name = "tensorfs-core"
-version = "0.3.44"
+version = "0.3.45"
 dependencies = [
  "fs2",
  "libc",
@@ -435,7 +435,7 @@ dependencies = [
 
 [[package]]
 name = "tensorfs-py"
-version = "0.3.44"
+version = "0.3.45"
 dependencies = [
  "pyo3",
  "tensorfs-core",

{tensorfs-0.3.44 → tensorfs-0.3.45}/Cargo.toml

@@ -3,5 +3,5 @@ resolver = "2"
 members = ["crates/tensorfs-core", "crates/tensorfs-py"]
 
 [workspace.package]
-version = "0.3.44"
+version = "0.3.45"
 edition = "2021"

{tensorfs-0.3.44 → tensorfs-0.3.45}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: tensorfs
-Version: 0.3.44
+Version: 0.3.45
 Classifier: Programming Language :: Python :: 3
 Classifier: Programming Language :: Python :: Implementation :: CPython
 Classifier: Programming Language :: Rust

tensorfs-0.3.45/crates/tensorfs-core/src/asset_update.rs

@@ -0,0 +1,293 @@
+//! Metadata-only model-asset updates.
+//!
+//! An asset update rewrites only the CozyTensors header and manifest.  Tensor part
+//! ObjectRefs are copied verbatim; asset source objects are selected from an existing
+//! manifest and are never opened or rehashed by this operation.  Payload verification
+//! remains the normal `read_asset` responsibility (or the repository admission gate).
+
+use std::collections::BTreeMap;
+
+use crate::checkpoint;
+use crate::err::{refuse, Code, Refusal, Result};
+use crate::header::Asset;
+use crate::ids::ObjectRef;
+use crate::manifest::{Entry, Manifest};
+use crate::store::Store;
+
+#[derive(Debug, Clone, PartialEq, Eq)]
+pub enum Edit {
+    /// Add or replace one named asset from an exact ordinary-file entry in the source
+    /// manifest.  The source ObjectRef becomes the sole segment; no payload is read.
+    Copy {
+        name: String,
+        source_path: String,
+        media_type: String,
+    },
+    /// Remove an asset by its exact header name.  Missing names are harmless, making
+    /// retries deterministic.
+    Drop { name: String },
+}
+
+#[derive(Debug, Clone, PartialEq, Eq)]
+pub struct ResultSnapshot {
+    pub header: ObjectRef,
+    pub manifest: ObjectRef,
+}
+
+fn source_object(manifest: &Manifest, path: &str) -> Result<ObjectRef> {
+    manifest
+        .entries()
+        .iter()
+        .find_map(|(candidate, entry)| {
+            (candidate == path).then(|| match entry {
+                Entry::File(object) => Ok(object.clone()),
+                Entry::CozyTensors(_) => Err(Refusal {
+                    code: Code::WRONG_TYPE,
+                    detail: format!("asset source {path:?} is a CozyTensors header, not a file"),
+                }),
+            })
+        })
+        .unwrap_or_else(|| {
+            Err(Refusal {
+                code: Code::OBJECT_ABSENT,
+                detail: format!("asset source {path:?} is absent from the source manifest"),
+            })
+        })
+}
+
+/// Apply add/replace/drop edits to one existing checkpoint without reading tensor payloads.
+///
+/// The source manifest and header are read, and each selected source object is checked only
+/// through its existing verification record and length.  The new header and manifest are the
+/// only bytes written.  Callers that need a payload guarantee should subsequently use
+/// [`crate::checkpoint::read_asset`], which hashes the reassembled asset once.
+pub fn update(
+    store: &Store,
+    source_manifest: &ObjectRef,
+    edits: &[Edit],
+) -> Result<ResultSnapshot> {
+    let source = checkpoint::load_manifest(store, source_manifest)?;
+    let header_ref = source.header().ok_or_else(|| Refusal {
+        code: Code::ATTACHMENT_CARDINALITY,
+        detail: "asset update source has no CozyTensors header".into(),
+    })?;
+    let mut header = checkpoint::load_header(store, header_ref)?;
+    let closure = checkpoint::load_closure(store, &header)?;
+
+    let mut assets: BTreeMap<String, Asset> = header.assets.drain(..).collect();
+    for edit in edits {
+        match edit {
+            Edit::Drop { name } => {
+                assets.remove(name);
+            }
+            Edit::Copy {
+                name,
+                source_path,
+                media_type,
+            } => {
+                if name.is_empty() || media_type.is_empty() {
+                    return refuse(
+                        Code::MISSING_FIELD,
+                        "asset name and media_type are required",
+                    );
+                }
+                let object = source_object(&source, source_path)?;
+                // This consults only the durable record's inode/length facts.  It does not
+                // open or hash the payload, which is the defining property of this operation.
+                let record = store
+                    .record_valid(&object.sha256)
+                    .map_err(|detail| Refusal {
+                        code: Code::OBJECT_CORRUPT,
+                        detail: format!("asset source {} is not verified: {detail}", object.id()),
+                    })?;
+                if record.length != object.length {
+                    return refuse(
+                        Code::LENGTH_MISMATCH,
+                        format!("asset source {} record length differs", object.id()),
+                    );
+                }
+                assets.insert(
+                    name.clone(),
+                    Asset {
+                        logical_sha256: object.sha256.clone(),
+                        logical_length: object.length,
+                        media_type: media_type.clone(),
+                        segments: vec![object],
+                    },
+                );
+            }
+        }
+    }
+    header.assets = assets.into_iter().collect();
+    header.validate(&closure)?;
+    let header_object = checkpoint::put_doc(store, &header)?;
+    let manifest = checkpoint::build_manifest(&header, &header_object, &closure)?;
+    let manifest_object = store.put_manifest(&manifest)?.obj;
+    Ok(ResultSnapshot {
+        header: header_object,
+        manifest: manifest_object,
+    })
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use crate::checkpoint;
+    use crate::header::{Body, Header, Part, Tensor};
+    use crate::manifest::Draft;
+    use crate::store::Fault;
+    use crate::{dtype::Dtype, registry};
+
+    fn temp() -> std::path::PathBuf {
+        std::env::temp_dir().join(format!(
+            "tensorfs-asset-update-{}",
+            crate::meta::now_nanos_unique()
+        ))
+    }
+
+    fn put(store: &Store, bytes: &[u8]) -> ObjectRef {
+        let expected = ObjectRef::of(bytes);
+        let mut input = bytes;
+        store
+            .put_stream(&mut input, Some(&expected), &Fault::default())
+            .unwrap()
+            .obj
+    }
+
+    #[test]
+    fn copies_asset_refs_without_changing_tensor_refs_and_read_asset_verifies_bytes() {
+        let root = temp();
+        let store = Store::init(&root).unwrap();
+        let spec = registry::seeds()
+            .into_iter()
+            .find(|seed| seed.alias == "plain/1")
+            .unwrap()
+            .spec;
+        let header = Header {
+            configs: vec![],
+            assets: vec![],
+            encodings: vec![spec.clone()],
+            components: vec![(
+                "model".into(),
+                vec![(
+                    "weight".into(),
+                    Tensor {
+                        dtype: Dtype::U8,
+                        shape: vec![3],
+                        encoding: spec.object_id(),
+                        parts: vec![(
+                            "value".into(),
+                            Part {
+                                dtype: Dtype::U8,
+                                shape: vec![3],
+                                body: Body::Inline(vec![1, 2, 3]),
+                            },
+                        )],
+                    },
+                )],
+            )],
+        };
+        let header_ref = put(&store, &header.canonical_bytes().unwrap());
+        let asset_bytes = b"tokenizer bytes";
+        let asset_ref = put(&store, asset_bytes);
+        let source = Draft {
+            entries: vec![
+                ("model.cozytensors".into(), Entry::CozyTensors(header_ref)),
+                ("tokenizer/vocab.txt".into(), Entry::File(asset_ref.clone())),
+            ],
+        }
+        .seal()
+        .unwrap();
+        let source_ref = store.put_manifest(&source).unwrap().obj;
+
+        let result = update(
+            &store,
+            &source_ref,
+            &[Edit::Copy {
+                name: "tokenizer/vocab.txt".into(),
+                source_path: "tokenizer/vocab.txt".into(),
+                media_type: "text/plain".into(),
+            }],
+        )
+        .unwrap();
+        let output = checkpoint::load_header(&store, &result.header).unwrap();
+        assert_eq!(output.tensors().count(), 1);
+        assert_eq!(
+            output.tensors().next().unwrap().2.parts[0].1.body,
+            Body::Inline(vec![1, 2, 3])
+        );
+        let asset = &output.assets[0].1;
+        assert_eq!(asset.segments, vec![asset_ref]);
+        assert_eq!(asset.logical_length, asset_bytes.len() as u64);
+        let mut verified = Vec::new();
+        checkpoint::read_asset(&store, "tokenizer/vocab.txt", asset, &mut verified).unwrap();
+        assert_eq!(verified, asset_bytes);
+        std::fs::remove_dir_all(root).unwrap();
+    }
+
+    #[test]
+    fn drops_are_deterministic_and_do_not_touch_payloads() {
+        let root = temp();
+        let store = Store::init(&root).unwrap();
+        let spec = registry::seeds().into_iter().next().unwrap().spec;
+        let payload = put(&store, b"asset");
+        let header = Header {
+            configs: vec![],
+            assets: vec![(
+                "old".into(),
+                Asset {
+                    logical_sha256: payload.sha256.clone(),
+                    logical_length: payload.length,
+                    media_type: "text/plain".into(),
+                    segments: vec![payload.clone()],
+                },
+            )],
+            encodings: vec![spec.clone()],
+            components: vec![(
+                "model".into(),
+                vec![(
+                    "w".into(),
+                    Tensor {
+                        dtype: Dtype::U8,
+                        shape: vec![1],
+                        encoding: spec.object_id(),
+                        parts: vec![(
+                            "value".into(),
+                            Part {
+                                dtype: Dtype::U8,
+                                shape: vec![1],
+                                body: Body::Inline(vec![0]),
+                            },
+                        )],
+                    },
+                )],
+            )],
+        };
+        let href = put(&store, &header.canonical_bytes().unwrap());
+        let source = Draft {
+            entries: vec![("model.cozytensors".into(), Entry::CozyTensors(href))],
+        }
+        .seal()
+        .unwrap();
+        let source_ref = store.put_manifest(&source).unwrap().obj;
+        let one = update(
+            &store,
+            &source_ref,
+            &[Edit::Drop {
+                name: "missing".into(),
+            }],
+        )
+        .unwrap();
+        let two = update(
+            &store,
+            &source_ref,
+            &[Edit::Drop {
+                name: "missing".into(),
+            }],
+        )
+        .unwrap();
+        assert_eq!(one.header, two.header);
+        assert_eq!(one.manifest, two.manifest);
+        std::fs::remove_dir_all(root).unwrap();
+    }
+}

{tensorfs-0.3.44 → tensorfs-0.3.45}/crates/tensorfs-core/src/lib.rs

@@ -8,6 +8,7 @@
 // crates/tensorfs-core/src is the check that it stays that way.
 #![deny(unsafe_code)]
 
+pub mod asset_update;
 pub mod b64;
 mod cache_roots;
 pub mod canon;

{tensorfs-0.3.44 → tensorfs-0.3.45}/crates/tensorfs-py/src/store.rs

@@ -591,6 +591,43 @@ impl PyStore {
         tensorfs_core::IMPLEMENTATION_SHA256
     }
 
+    /// Rewrite only a CozyTensors header/manifest to add, replace, or drop model assets.
+    /// `copies` are `(name, source_path, media_type)` rows selected from the exact source
+    /// manifest; TensorFS reuses their ObjectRefs without reading payload bytes.
+    #[pyo3(signature = (manifest_id, manifest_length, copies=Vec::new(), drops=Vec::new()))]
+    fn update_model_assets<'p>(
+        &self,
+        py: Python<'p>,
+        manifest_id: &str,
+        manifest_length: u64,
+        copies: Vec<(String, String, String)>,
+        drops: Vec<String>,
+    ) -> PyResult<Bound<'p, PyDict>> {
+        let source = obj_ref(py, manifest_id, manifest_length)?;
+        let mut edits = Vec::with_capacity(copies.len() + drops.len());
+        edits.extend(copies.into_iter().map(|(name, source_path, media_type)| {
+            tensorfs_core::asset_update::Edit::Copy {
+                name,
+                source_path,
+                media_type,
+            }
+        }));
+        edits.extend(
+            drops
+                .into_iter()
+                .map(|name| tensorfs_core::asset_update::Edit::Drop { name }),
+        );
+        let result = py
+            .detach(|| tensorfs_core::asset_update::update(&self.store, &source, &edits))
+            .or_refuse(py)?;
+        let out = PyDict::new(py);
+        out.set_item("header_id", result.header.id())?;
+        out.set_item("header_length", result.header.length)?;
+        out.set_item("manifest_id", result.manifest.id())?;
+        out.set_item("manifest_length", result.manifest.length)?;
+        Ok(out)
+    }
+
     fn verify_checkpoint_source(
         &self,
         py: Python<'_>,

{tensorfs-0.3.44 → tensorfs-0.3.45}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "maturin"
 
 [project]
 name = "tensorfs"
-version = "0.3.44"
+version = "0.3.45"
 description = "TensorFS — canonical tensor artifact substrate"
 requires-python = ">=3.12"
 classifiers = [

{tensorfs-0.3.44 → tensorfs-0.3.45}/python/tensorfs/_ext.pyi

@@ -484,6 +484,13 @@ class Store:
     def prepare_readers(self) -> None: ...
     def complete_cozytensors_manifests(self) -> list[str]: ...
     def contains(self, hex: str) -> bool: ...
+    def update_model_assets(
+        self,
+        manifest_id: str,
+        manifest_length: int,
+        copies: Sequence[tuple[str, str, str]] = ...,
+        drops: Sequence[str] = ...,
+    ) -> dict[str, str | int]: ...
     def objects(self) -> list[str]: ...
     def reap_holds(self) -> list[str]: ...