tensorfs 0.3.61__tar.gz → 0.3.63__tar.gz

{tensorfs-0.3.61 → tensorfs-0.3.63}/Cargo.lock

@@ -439,7 +439,7 @@ checksum = "adb6935a6f5c20170eeceb1a3835a49e12e19d792f6dd344ccc76a985ca5a6ca"
 
 [[package]]
 name = "tensorfs-core"
-version = "0.3.61"
+version = "0.3.63"
 dependencies = [
  "fs2",
  "libc",
@@ -454,7 +454,7 @@ dependencies = [
 
 [[package]]
 name = "tensorfs-py"
-version = "0.3.61"
+version = "0.3.63"
 dependencies = [
  "pyo3",
  "tensorfs-core",

{tensorfs-0.3.61 → tensorfs-0.3.63}/Cargo.toml

@@ -3,5 +3,5 @@ resolver = "2"
 members = ["crates/tensorfs-core", "crates/tensorfs-py"]
 
 [workspace.package]
-version = "0.3.61"
+version = "0.3.63"
 edition = "2021"

{tensorfs-0.3.61 → tensorfs-0.3.63}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: tensorfs
-Version: 0.3.61
+Version: 0.3.63
 Classifier: Programming Language :: Python :: 3
 Classifier: Programming Language :: Python :: Implementation :: CPython
 Classifier: Programming Language :: Rust

{tensorfs-0.3.61 → tensorfs-0.3.63}/crates/tensorfs-core/src/asset_update.rs

@@ -11,7 +11,7 @@ use crate::checkpoint;
 use crate::err::{refuse, Code, Refusal, Result};
 use crate::header::Asset;
 use crate::ids::ObjectRef;
-use crate::manifest::{Entry, Manifest};
+use crate::manifest::Manifest;
 use crate::store::Store;
 
 #[derive(Debug, Clone, PartialEq, Eq)]
@@ -42,13 +42,7 @@ fn source_object(manifest: &Manifest, path: &str) -> Result<ObjectRef> {
         .entries()
         .iter()
         .find_map(|(candidate, entry)| {
-            (candidate == path).then(|| match entry {
-                Entry::File(object) => Ok(object.clone()),
-                Entry::CozyTensors(_) => Err(Refusal {
-                    code: Code::WRONG_TYPE,
-                    detail: format!("asset source {path:?} is a CozyTensors header, not a file"),
-                }),
-            })
+            (candidate == path).then(|| entry.materializable(path).cloned())
         })
         .unwrap_or_else(|| {
             Err(Refusal {
@@ -187,6 +181,7 @@ mod tests {
     use crate::checkpoint;
     use crate::header::{Body, Header, Part, Tensor};
     use crate::manifest::Draft;
+    use crate::manifest::Entry;
     use crate::store::Fault;
     use crate::{dtype::Dtype, registry};
 

{tensorfs-0.3.61 → tensorfs-0.3.63}/crates/tensorfs-core/src/bin/tfs/main.rs

@@ -2207,16 +2207,17 @@ fn build_corpus() -> TfsResult<(Vec<Case>, Vec<EncodingSpec>)> {
         "carrier no longer equals the logical dtype",
     ));
 
-    // --- location-typed Manifest reds
+    // A kind a newer TensorFS wrote is opaque here: it parses, and only materializing refuses.
     let manifest_bytes = snap2.canonical_bytes();
-    cases.push(red_case(
-        "manifest-red",
-        "entry-kind-symlink",
+    cases.push(ok_case(
+        "manifest",
+        "entry-kind-future",
         "manifest",
         sub(&manifest_bytes, "\"kind\":\"file\"", "\"kind\":\"symlink\""),
-        "UNKNOWN_FIELD",
-        "only file and cozytensors entries exist",
+        "an entry kind from a newer TensorFS is carried opaquely",
     ));
+
+    // --- location-typed Manifest reds
     let mut draft = Draft::of(&snap);
     draft.entries.clear();
     cases.push(red_case(
@@ -2259,6 +2260,7 @@ fn build_corpus() -> TfsResult<(Vec<Case>, Vec<EncodingSpec>)> {
     // --- the border's own documents (tfs-003)
     let g = "ingest";
     let session = tensorfs_core::ingest::transaction::IngestSession {
+        tensorfs: None,
         session: "e3b0c44298fc1c14".to_string(),
         tenant: "acme".to_string(),
         candidates: vec![ObjectRef::of(b"a candidate header")],

{tensorfs-0.3.61 → tensorfs-0.3.63}/crates/tensorfs-core/src/bin/tfs/repo.rs

@@ -9,7 +9,7 @@ use tensorfs_core::err::{Code, Refusal};
 use tensorfs_core::gc;
 use tensorfs_core::header::{Body, Header};
 use tensorfs_core::ids::{hex64, prefixed, Doc, ObjectRef};
-use tensorfs_core::manifest::{Entry, Manifest};
+use tensorfs_core::manifest::Manifest;
 use tensorfs_core::reclaim;
 use tensorfs_core::repository::{self, Mutation, Repository};
 use tensorfs_core::storage::{
@@ -738,13 +738,7 @@ pub fn cmd_manifest_inspect(path: &Path, flags: &Flags) -> ExitCode {
         .iter()
         .map(|(path, entry)| {
             tensorfs_core::canon::write(&Value::obj(vec![
-                (
-                    "kind",
-                    Value::str(match entry {
-                        Entry::File(_) => "file",
-                        Entry::CozyTensors(_) => "cozytensors",
-                    }),
-                ),
+                ("kind", Value::str(entry.kind())),
                 ("length", Value::uint(entry.blob().length)),
                 ("path", Value::str(path.clone())),
                 ("sha256", Value::str(entry.blob().sha256.clone())),
@@ -855,7 +849,7 @@ fn closure_refs(
     // Runtime roles precede ordinary snapshot paths so an aliased README/sample path cannot
     // relabel a required config, model asset, or tensor part as an ignorable file.
     for (_, entry) in manifest.entries() {
-        if let Entry::File(object) = entry {
+        if let Some(object) = entry.content() {
             insert(&mut refs, "file", object)?;
         }
     }
@@ -1206,7 +1200,7 @@ pub fn cmd_reclaim_run(root: &Path, flags: &Flags) -> ExitCode {
 mod tests {
     use super::*;
     use std::collections::HashSet;
-    use tensorfs_core::manifest::Draft;
+    use tensorfs_core::manifest::{Draft, Entry};
 
     #[test]
     fn inspect_closure_preserves_runtime_roles_over_aliased_files() {
@@ -1246,10 +1240,7 @@ mod tests {
         let file_ids: HashSet<&str> = manifest
             .entries()
             .iter()
-            .filter_map(|(_, entry)| match entry {
-                Entry::File(object) => Some(object.sha256.as_str()),
-                Entry::CozyTensors(_) => None,
-            })
+            .filter_map(|(_, entry)| entry.content().map(|object| object.sha256.as_str()))
             .collect();
 
         assert!(header.assets.iter().any(|(_, asset)| asset
@@ -1268,11 +1259,11 @@ mod tests {
             .filter(|(kind, _)| *kind != "file")
             .map(|(_, object)| object.sha256.as_str())
             .collect();
-        assert!(manifest.entries().iter().any(|(_, entry)| match entry {
-            Entry::File(object) => {
+        assert!(manifest
+            .entries()
+            .iter()
+            .any(|(_, entry)| entry.content().is_some_and(|object| {
                 !runtime_ids.contains(object.sha256.as_str()) && refs[&object.sha256].0 == "file"
-            }
-            Entry::CozyTensors(_) => false,
-        }));
+            })));
     }
 }

{tensorfs-0.3.61 → tensorfs-0.3.63}/crates/tensorfs-core/src/bin/tfs/snap.rs

@@ -104,7 +104,7 @@ fn referenced_blobs(header: &Header, draft: &Draft) -> Result<BTreeMap<String, u
         }
     }
     for (_, entry) in &draft.entries {
-        if let Entry::File(object) = entry {
+        if let Some(object) = entry.content() {
             add_ref(&mut refs, object, "manifest file")?;
         }
     }
@@ -257,6 +257,9 @@ pub fn cmd_show(root: &Path, hex: &str) -> ExitCode {
                 blob.id(),
                 blob.length
             ),
+            Entry::Other { kind, blob } => {
+                println!("    {kind:<12} {path}  {} ({} B)", blob.id(), blob.length)
+            }
         }
     }
     ExitCode::SUCCESS

{tensorfs-0.3.61 → tensorfs-0.3.63}/crates/tensorfs-core/src/canon.rs

@@ -456,6 +456,31 @@ impl<'a> Fields<'a> {
         }
     }
 
+    /// End a record whose unread field could keep bytes alive (a GC root): refuse, naming the
+    /// newer TensorFS that wrote it, rather than drop what that version may rely on.
+    pub fn done_written_by(self, writer: Option<&str>) -> Result<()> {
+        let Some((k, _)) = self.left.first() else {
+            return Ok(());
+        };
+        let by = writer.map_or_else(
+            || "a newer TensorFS".to_string(),
+            |writer| format!("TensorFS {writer}"),
+        );
+        let upgrade = writer.map_or_else(
+            || "upgrade TensorFS".to_string(),
+            |writer| format!("upgrade to tensorfs>={writer}"),
+        );
+        refuse(
+            Code::UNKNOWN_FIELD,
+            format!(
+                "{}: field {k:?} was written by {by}; this is TensorFS {} — {upgrade}. It is \
+                 refused rather than dropped because it may keep bytes alive",
+                self.what,
+                crate::VERSION
+            ),
+        )
+    }
+
     /// End a document that tolerates additive fields, keeping them for re-emission.
     pub fn rest(self) -> Extra {
         Extra(

{tensorfs-0.3.61 → tensorfs-0.3.63}/crates/tensorfs-core/src/checkpoint.rs

@@ -523,17 +523,14 @@ pub fn walk(store: &Store, snap: &Manifest) -> Result<Walk> {
         .map(|r| (r.obj.sha256.clone(), r.obj.length))
         .collect();
     for (path, e) in snap.entries() {
-        match e {
-            crate::manifest::Entry::File(o) => {
-                if !accounted.contains(&(o.sha256.clone(), o.length)) {
-                    w.siblings.push(path.clone());
-                    w.objects.push(Reached {
-                        kind: "asset",
-                        obj: o.clone(),
-                    });
-                }
+        if let Some(o) = e.content() {
+            if !accounted.contains(&(o.sha256.clone(), o.length)) {
+                w.siblings.push(path.clone());
+                w.objects.push(Reached {
+                    kind: "asset",
+                    obj: o.clone(),
+                });
             }
-            crate::manifest::Entry::CozyTensors(_) => {}
         }
     }
     Ok(w)

{tensorfs-0.3.61 → tensorfs-0.3.63}/crates/tensorfs-core/src/checkpoint_root.rs

@@ -22,6 +22,8 @@ pub struct CheckpointRoot {
     pub repository: RepositoryName,
     pub manifest: ObjectRef,
     pub released: bool,
+    /// The newest TensorFS that wrote this root.
+    pub tensorfs: Option<String>,
 }
 
 impl Plain for CheckpointRoot {
@@ -36,11 +38,18 @@ impl Plain for CheckpointRoot {
             Value::Bool(value) => *value,
             _ => return refuse(Code::WRONG_TYPE, "checkpoint root release must be boolean"),
         };
-        fields.done()?;
+        let tensorfs = fields
+            .opt("tensorfs")
+            .map(|value| {
+                crate::canon::as_str("CheckpointRoot", "tensorfs", value).map(str::to_string)
+            })
+            .transpose()?;
+        fields.done_written_by(tensorfs.as_deref())?;
         if manifest.length == 0 || manifest.length > crate::limits::INT_MAX as u64 {
             return refuse(Code::LENGTH_MISMATCH, "checkpoint root manifest is empty");
         }
         Ok(Self {
+            tensorfs,
             owner,
             repository,
             manifest,
@@ -49,7 +58,7 @@ impl Plain for CheckpointRoot {
     }
 
     fn to_value(&self) -> Value {
-        Value::obj(vec![
+        let mut fields = vec![
             ("owner", Value::str(self.owner.clone())),
             (
                 "repository",
@@ -57,7 +66,11 @@ impl Plain for CheckpointRoot {
             ),
             ("manifest", self.manifest.to_value()),
             ("released", Value::Bool(self.released)),
-        ])
+        ];
+        if let Some(tensorfs) = &self.tensorfs {
+            fields.push(("tensorfs", Value::str(tensorfs.clone())));
+        }
+        Value::obj(fields)
     }
 }
 
@@ -126,6 +139,10 @@ fn lock(store: &Store, owner: &str) -> Result<File> {
 }
 
 fn write(store: &Store, root: &CheckpointRoot) -> Result<()> {
+    let root = &CheckpointRoot {
+        tensorfs: Some(crate::VERSION.into()),
+        ..root.clone()
+    };
     let path = path(store, &root.owner)?;
     let temporary = path.with_extension(format!("tmp-{}", crate::meta::now_nanos_unique()));
     let result = (|| {
@@ -159,6 +176,7 @@ fn identity(owner: &str, source: &str, manifest: ObjectRef) -> Result<Checkpoint
         return refuse(Code::LENGTH_MISMATCH, "checkpoint root manifest is empty");
     }
     Ok(CheckpointRoot {
+        tensorfs: Some(crate::VERSION.into()),
         owner: prefixed("checkpoint root owner", owner)?,
         repository: repository(source)?,
         manifest,

{tensorfs-0.3.61 → tensorfs-0.3.63}/crates/tensorfs-core/src/derived/companion_tests.rs

@@ -191,7 +191,7 @@ fn ordinary_companions_survive_derivation_without_copying_tensor_backing_files()
         .push(("LICENSE".into(), b"explicit licence".to_vec()));
     assert_eq!(
         companion_files(&d, &sources).unwrap()["LICENSE"],
-        ObjectRef::of(b"explicit licence")
+        Entry::File(ObjectRef::of(b"explicit licence"))
     );
     std::fs::remove_dir_all(store.root()).unwrap();
 }

tensorfs-0.3.63/crates/tensorfs-core/src/derived/part_rows_tests.rs

@@ -0,0 +1,169 @@
+//! Accepted parts are rows of their own: accepting one never reads or rewrites the others.
+
+use super::*;
+use rusqlite::params;
+
+fn fixture(label: &str) -> (Store, Meta, String, Declaration) {
+    let root = std::env::temp_dir().join(format!(
+        "tensorfs-part-rows-{label}-{}",
+        crate::meta::now_nanos_unique()
+    ));
+    let store = Store::init(&root).unwrap();
+    let meta = Meta::open(&store).unwrap();
+    let mut declaration = super::tests::created_declaration();
+    let prototype = declaration.components[0].add[0].clone();
+    declaration.components[0].add = (0..3)
+        .map(|index| {
+            let mut tensor = prototype.clone();
+            tensor.key = format!("weight_{index}");
+            tensor.shape = vec![128];
+            tensor.parts[0].shape = vec![128];
+            tensor
+        })
+        .collect();
+    declaration.order = (0..3)
+        .map(|index| ("model".into(), format!("weight_{index}")))
+        .collect();
+    declaration.max_new_bytes = 3 * 512;
+    (
+        store,
+        meta,
+        format!("sha256:{}", "7a".repeat(32)),
+        declaration,
+    )
+}
+
+fn catalog(store: &Store) -> rusqlite::Connection {
+    rusqlite::Connection::open(crate::catalog::Catalog::path(store.root())).unwrap()
+}
+
+fn add(store: &Store, meta: &Meta, id: &str, writer: &Begin, key: &str, byte: u8) -> Result<()> {
+    add_indexed_part(
+        store,
+        meta,
+        id,
+        1,
+        &writer.additions,
+        ("model", key, "value"),
+        &mut [byte; 512].as_slice(),
+    )
+    .map(|_| ())
+}
+
+fn stored_rows(store: &Store, id: &str) -> (usize, Option<i64>, i64) {
+    let connection = catalog(store);
+    let (bytes, open): (Vec<u8>, Option<i64>) = connection
+        .query_row(
+            "SELECT bytes,open_session FROM tensorfs_derived_transactions WHERE id=?1",
+            [id],
+            |row| Ok((row.get(0)?, row.get(1)?)),
+        )
+        .unwrap();
+    let value = crate::canon::parse_canonical(&bytes, limits::DOC_MAX_BYTES).unwrap();
+    let inline = TransactionRow::from_value(&value)
+        .unwrap()
+        .added_parts
+        .len();
+    let parts = connection
+        .query_row(
+            "SELECT count(*) FROM tensorfs_derived_parts WHERE id=?1",
+            [id],
+            |row| row.get(0),
+        )
+        .unwrap();
+    (inline, open, parts)
+}
+
+#[test]
+fn parts_are_rows_that_resume_refuse_conflicts_and_stay_retained() {
+    let (store, meta, id, declaration) = fixture("rows");
+    let writer = begin(&store, &meta, &id, 1, declaration.clone(), None).unwrap();
+    add(&store, &meta, &id, &writer, "weight_0", 1).unwrap();
+    add(&store, &meta, &id, &writer, "weight_1", 2).unwrap();
+    // The row itself names no part; each part is one row; the writer's session is indexed.
+    assert_eq!(stored_rows(&store, &id), (0, Some(1), 2));
+    // The same bytes again are the same accepted part; other bytes conflict.
+    add(&store, &meta, &id, &writer, "weight_0", 1).unwrap();
+    assert_eq!(stored_rows(&store, &id).2, 2);
+    assert_eq!(
+        add(&store, &meta, &id, &writer, "weight_0", 9)
+            .unwrap_err()
+            .code,
+        Code::TRANSACTION_CONFLICT
+    );
+    // A fenced writer cannot add. Once it is gone, GC keeps its accepted bytes through
+    // the append-only journal alone, and its successor resumes with both parts.
+    fence(&meta, &id, 1).unwrap();
+    assert_eq!(
+        add(&store, &meta, &id, &writer, "weight_2", 3)
+            .unwrap_err()
+            .code,
+        Code::WRITER_FENCED
+    );
+    release_guards(&meta, Some(writer.writer_hold), writer.source_leases);
+    crate::gc::collect(store.root(), false).unwrap();
+    assert!(store.contains(&ObjectRef::of(&[1u8; 512]).sha256));
+    assert!(store.contains(&ObjectRef::of(&[2u8; 512]).sha256));
+    let resumed = begin(&store, &meta, &id, 2, declaration, None).unwrap();
+    assert_eq!(completed(&meta, &id, 2).unwrap().0.len(), 2);
+    add_indexed_part(
+        &store,
+        &meta,
+        &id,
+        2,
+        &resumed.additions,
+        ("model", "weight_2", "value"),
+        &mut [3u8; 512].as_slice(),
+    )
+    .unwrap();
+    let receipt = commit(&store, &meta, &id, 2).unwrap();
+    assert_eq!(receipt.added_objects.len(), 3);
+    assert_eq!(stored_rows(&store, &id).1, None);
+    fence(&meta, &id, 2).unwrap();
+    release_guards(&meta, Some(resumed.writer_hold), resumed.source_leases);
+    std::fs::remove_dir_all(store.root()).unwrap();
+}
+
+#[test]
+fn an_older_builds_inline_row_and_a_torn_journal_line_are_read() {
+    let (store, meta, id, declaration) = fixture("legacy");
+    let writer = begin(&store, &meta, &id, 1, declaration, None).unwrap();
+    add(&store, &meta, &id, &writer, "weight_0", 1).unwrap();
+    // An older build rewrites the row with its parts inline and no open session.
+    let row = meta.derived_row(&id).unwrap().unwrap();
+    let connection = catalog(&store);
+    connection
+        .execute("DELETE FROM tensorfs_derived_parts WHERE id=?1", [&id])
+        .unwrap();
+    connection
+        .execute(
+            "UPDATE tensorfs_derived_transactions SET bytes=?2, open_session=NULL WHERE id=?1",
+            params![id, crate::canon::write(&row.to_value())],
+        )
+        .unwrap();
+    // A writer that died mid-append left a torn journal line.
+    let journal = store
+        .root()
+        .join("roots/derived")
+        .join(format!("{}.parts", id.trim_start_matches("sha256:")));
+    let mut torn = std::fs::read(&journal).unwrap();
+    torn.extend_from_slice(b"[{\"length\":512,\"sha");
+    std::fs::write(&journal, torn).unwrap();
+
+    assert_eq!(
+        add(&store, &meta, &id, &writer, "weight_0", 9)
+            .unwrap_err()
+            .code,
+        Code::TRANSACTION_CONFLICT
+    );
+    add(&store, &meta, &id, &writer, "weight_1", 2).unwrap();
+    assert_eq!(completed(&meta, &id, 1).unwrap().0.len(), 2);
+    fence(&meta, &id, 1).unwrap();
+    release_guards(&meta, Some(writer.writer_hold), writer.source_leases);
+    crate::gc::collect(store.root(), false).unwrap();
+    assert!(store.contains(&ObjectRef::of(&[1u8; 512]).sha256));
+    assert!(store.contains(&ObjectRef::of(&[2u8; 512]).sha256));
+    abandon(&store, &meta, &id).unwrap();
+    assert!(!journal.exists());
+    std::fs::remove_dir_all(store.root()).unwrap();
+}

{tensorfs-0.3.61 → tensorfs-0.3.63}/crates/tensorfs-core/src/derived/resume_tests.rs

@@ -984,11 +984,12 @@ fn profile_derived_transaction_scaling() {
         for index in 0..writes {
             let key = format!("weight_{index:04}");
             let started = Instant::now();
-            add_part(
+            add_indexed_part(
                 &store,
                 &meta,
                 &id,
                 1,
+                &writer.additions,
                 ("model", &key, "value"),
                 &mut [index as u8; 16].as_slice(),
             )

{tensorfs-0.3.61 → tensorfs-0.3.63}/crates/tensorfs-core/src/derived/retention.rs

@@ -22,6 +22,8 @@ pub struct RetainedResult {
     pub receipt_digest: String,
     pub manifest: Option<ObjectRef>,
     pub released: bool,
+    /// The newest TensorFS that wrote this root.
+    pub tensorfs: Option<String>,
 }
 
 impl Plain for RetainedResult {
@@ -40,11 +42,18 @@ impl Plain for RetainedResult {
             Value::Bool(value) => *value,
             _ => return refuse(Code::WRONG_TYPE, "retained result release must be boolean"),
         };
-        f.done()?;
+        let tensorfs = f
+            .opt("tensorfs")
+            .map(|value| {
+                crate::canon::as_str("RetainedResult", "tensorfs", value).map(str::to_string)
+            })
+            .transpose()?;
+        f.done_written_by(tensorfs.as_deref())?;
         if !released && manifest.is_none() {
             return refuse(Code::MISSING_FIELD, "held result has no manifest");
         }
         Ok(Self {
+            tensorfs,
             retention_id,
             transaction_id,
             receipt_digest,
@@ -63,6 +72,9 @@ impl Plain for RetainedResult {
         if let Some(manifest) = &self.manifest {
             fields.push(("manifest", manifest.to_value()));
         }
+        if let Some(tensorfs) = &self.tensorfs {
+            fields.push(("tensorfs", Value::str(tensorfs.clone())));
+        }
         Value::obj(fields)
     }
 }
@@ -104,6 +116,10 @@ fn read(store: &Store, id: &str) -> Result<Option<RetainedResult>> {
 }
 
 fn write(store: &Store, root: &RetainedResult) -> Result<()> {
+    let root = &RetainedResult {
+        tensorfs: Some(crate::VERSION.into()),
+        ..root.clone()
+    };
     let path = path(store, &root.retention_id)?;
     fs::create_dir_all(directory(store)).map_err(io)?;
     let temp = path.with_extension(format!("tmp-{}", crate::meta::now_nanos_unique()));
@@ -128,6 +144,7 @@ fn write(store: &Store, root: &RetainedResult) -> Result<()> {
 
 fn identity(transaction: &str, receipt: &str, id: &str) -> Result<RetainedResult> {
     Ok(RetainedResult {
+        tensorfs: Some(crate::VERSION.into()),
         retention_id: prefixed("retention", id)?,
         transaction_id: prefixed("transaction", transaction)?,
         receipt_digest: prefixed("receipt", receipt)?,