tensorfs 0.3.9__tar.gz → 0.3.10__tar.gz

{tensorfs-0.3.9 → tensorfs-0.3.10}/Cargo.lock

@@ -392,7 +392,7 @@ checksum = "adb6935a6f5c20170eeceb1a3835a49e12e19d792f6dd344ccc76a985ca5a6ca"
 
 [[package]]
 name = "tensorfs-core"
-version = "0.3.9"
+version = "0.3.10"
 dependencies = [
  "fs2",
  "minicbor",
@@ -404,7 +404,7 @@ dependencies = [
 
 [[package]]
 name = "tensorfs-py"
-version = "0.3.9"
+version = "0.3.10"
 dependencies = [
  "pyo3",
  "tensorfs-core",

{tensorfs-0.3.9 → tensorfs-0.3.10}/Cargo.toml

@@ -3,5 +3,5 @@ resolver = "2"
 members = ["crates/tensorfs-core", "crates/tensorfs-py"]
 
 [workspace.package]
-version = "0.3.9"
+version = "0.3.10"
 edition = "2021"

{tensorfs-0.3.9 → tensorfs-0.3.10}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: tensorfs
-Version: 0.3.9
+Version: 0.3.10
 Classifier: Programming Language :: Python :: 3
 Classifier: Programming Language :: Python :: Implementation :: CPython
 Classifier: Programming Language :: Rust

{tensorfs-0.3.9 → tensorfs-0.3.10}/crates/tensorfs-core/src/derived.rs

@@ -1338,22 +1338,28 @@ fn require_payload_record(store: &Store, object: &ObjectRef, what: &str) -> Resu
     }
 }
 
-fn resolve_sources(
-    store: &Store,
-    meta: &Meta,
-    declaration: &Declaration,
-) -> Result<Vec<SourceData>> {
-    let mut out = Vec::new();
+fn require_source_roots(store: &Store, declaration: &Declaration) -> Result<()> {
+    if declaration.sources.is_empty() {
+        return Ok(());
+    }
+    let retained = crate::gc::retained_manifests(store)?;
     for declared in &declaration.sources {
-        if !meta.manifest_is_released(&declared.manifest.sha256)? {
+        if !retained.contains(&declared.manifest) {
             return refuse(
                 Code::ROOT_ABSENT,
                 format!(
-                    "source {} is not referenced by a repository release",
+                    "source {} has no exact retained filesystem root",
                     declared.manifest.id()
                 ),
             );
         }
+    }
+    Ok(())
+}
+
+fn load_sources(store: &Store, declaration: &Declaration) -> Result<Vec<SourceData>> {
+    let mut out = Vec::new();
+    for declared in &declaration.sources {
         let manifest = checkpoint::load_manifest(store, &declared.manifest)?;
         if manifest.manifest_id() != declared.manifest.id() {
             return refuse(
@@ -1791,6 +1797,18 @@ pub fn inspect_source(
     manifest: ObjectRef,
     component_names: Vec<String>,
     config_names: Vec<String>,
+) -> Result<SourceInspection> {
+    let hold = meta.acquire_hold("derived-source-inspect")?;
+    let result = inspect_retained_source(store, manifest, component_names, config_names);
+    let released = hold.release(meta);
+    result.and_then(|inspection| released.map(|()| inspection))
+}
+
+fn inspect_retained_source(
+    store: &Store,
+    manifest: ObjectRef,
+    component_names: Vec<String>,
+    config_names: Vec<String>,
 ) -> Result<SourceInspection> {
     let alias = "source".to_string();
     let mut declaration = Declaration {
@@ -1820,7 +1838,8 @@ pub fn inspect_source(
         max_new_bytes: 0,
     };
     declaration.normalize_and_validate()?;
-    let mut sources = resolve_sources(store, meta, &declaration)?;
+    require_source_roots(store, &declaration)?;
+    let mut sources = load_sources(store, &declaration)?;
     let source = sources.pop().ok_or_else(|| Refusal {
         code: Code::ROOT_ABSENT,
         detail: "source inspection resolved no source".into(),
@@ -1882,7 +1901,9 @@ pub fn begin(
     // The writer hold is registered before source metadata is touched. It is the live
     // process proof that keeps exclusive GC out even for a source-free create.
     let writer_hold = meta.acquire_hold("derived-writer")?;
-    let sources = match resolve_sources(store, meta, &declaration) {
+    let sources = match require_source_roots(store, &declaration)
+        .and_then(|()| load_sources(store, &declaration))
+    {
         Ok(sources) => sources,
         Err(error) => {
             release_guards(meta, Some(writer_hold), Vec::new());
@@ -2330,6 +2351,9 @@ fn receipt_from(_store: &Store, row: &TransactionRow) -> Result<ReceiptFacts> {
     })
 }
 
+/// The caller retains Begin's writer hold and source leases through this call.
+/// Root membership was admitted at begin; the live hold now protects those exact
+/// bytes even if their original repository/session/private root is released.
 pub fn commit(
     store: &Store,
     meta: &Meta,
@@ -2342,7 +2366,7 @@ pub fn commit(
         open_row(&mut txn.rows, &transaction, writer_session)?.clone()
     };
     let declaration = declaration(&row_before)?;
-    let sources = resolve_sources(store, meta, &declaration)?;
+    let sources = load_sources(store, &declaration)?;
     let (header, closure, registry_adoptions) = build_header(
         &declaration,
         &sources,
@@ -2700,6 +2724,235 @@ mod tests {
         }
     }
 
+    fn inherited_declaration(manifest: ObjectRef) -> Declaration {
+        Declaration {
+            sources: vec![Source {
+                alias: "source".into(),
+                manifest,
+            }],
+            components: vec![ComponentDeclaration {
+                target: "inherited".into(),
+                source: Some("source".into()),
+                source_component: Some("model".into()),
+                drop: Vec::new(),
+                add: Vec::new(),
+            }],
+            configs: Vec::new(),
+            order: vec![("inherited".into(), "weight".into())],
+            max_new_bytes: 0,
+        }
+    }
+
+    #[test]
+    fn retained_source_admission_and_writer_hold_share_the_gc_roots() {
+        let root = std::env::temp_dir().join(format!(
+            "tensorfs-retained-source-{}",
+            crate::meta::now_nanos_unique()
+        ));
+        let store = Store::init(&root).unwrap();
+        let meta = Meta::open(&store).unwrap();
+        let source_id = format!("sha256:{}", "81".repeat(32));
+        let source_writer = begin(&store, &meta, &source_id, 1, created_declaration()).unwrap();
+        let bytes = vec![0x21; 2048];
+        add_part(
+            &store,
+            &meta,
+            &source_id,
+            1,
+            ("model", "weight", "value"),
+            &mut bytes.as_slice(),
+        )
+        .unwrap();
+        let source = commit(&store, &meta, &source_id, 1).unwrap();
+        release_guards(
+            &meta,
+            Some(source_writer.writer_hold),
+            source_writer.source_leases,
+        );
+        let preview = || {
+            inspect_source(
+                &store,
+                &meta,
+                source.manifest.clone(),
+                vec!["model".into()],
+                Vec::new(),
+            )
+        };
+
+        // Committed pending output and owner-adopted output have real filesystem roots.
+        preview().unwrap();
+        adopt(&store, &meta, &source_id, "source-private").unwrap();
+        preview().unwrap();
+        dispose(&store, &meta, &source_id).unwrap();
+        assert_eq!(preview().unwrap_err().code, Code::ROOT_ABSENT);
+        let target_id = format!("sha256:{}", "82".repeat(32));
+        assert_eq!(
+            begin(
+                &store,
+                &meta,
+                &target_id,
+                1,
+                inherited_declaration(source.manifest.clone())
+            )
+            .err()
+            .unwrap()
+            .code,
+            Code::ROOT_ABSENT
+        );
+        assert!(matches!(
+            lookup(&store, &meta, &target_id).unwrap(),
+            Lookup::Absent
+        ));
+
+        // The coordinator's actual prepared-source lifecycle names a private candidate.
+        let operation = "prepared-source";
+        let request_digest = format!("sha256:{}", "83".repeat(32));
+        let catalog = crate::catalog::Catalog::open(&root).unwrap();
+        let crate::catalog::SourcePreparation::Open(source_guard) = catalog
+            .begin_source_preparation(operation, &request_digest)
+            .unwrap()
+        else {
+            panic!("source operation already prepared")
+        };
+        crate::ingest::transaction::ensure_session_root(&root, operation, "_tensorfs").unwrap();
+        crate::ingest::transaction::name_candidates(
+            &root,
+            operation,
+            std::slice::from_ref(&source.manifest),
+        )
+        .unwrap();
+        // An actual candidate is retained even before the rebuildable result row is written.
+        preview().unwrap();
+        catalog
+            .commit_source_preparation(
+                operation,
+                &request_digest,
+                &[crate::catalog::SourcePreparationResult {
+                    slot: "source".into(),
+                    profile: "proof".into(),
+                    manifest_sha256: source.manifest.sha256.clone(),
+                    manifest_length: source.manifest.length,
+                }],
+            )
+            .unwrap();
+        drop(source_guard);
+        crate::gc::collect(&root, false).unwrap();
+        preview().unwrap();
+        let mut wrong = inherited_declaration(source.manifest.clone());
+        wrong.sources[0].manifest.length += 1;
+        assert_eq!(
+            begin(&store, &meta, &target_id, 1, wrong)
+                .err()
+                .unwrap()
+                .code,
+            Code::ROOT_ABSENT
+        );
+
+        let writer = begin(
+            &store,
+            &meta,
+            &target_id,
+            1,
+            inherited_declaration(source.manifest.clone()),
+        )
+        .unwrap();
+        crate::ingest::source::release_model_source(&store, operation).unwrap();
+        assert_eq!(preview().unwrap_err().code, Code::ROOT_ABSENT);
+        writer.writer_hold.still_registered(&meta).unwrap();
+        assert_eq!(
+            crate::gc::collect(&root, false).unwrap_err().code,
+            Code::STORE_BUSY
+        );
+        let inherited = commit(&store, &meta, &target_id, 1).unwrap();
+        assert_eq!(inherited.inherited_payload_bytes, 2048);
+        assert!(inherited.added_objects.is_empty());
+        release_guards(&meta, Some(writer.writer_hold), writer.source_leases);
+        crate::gc::collect(&root, false).unwrap();
+        store.read_manifest(&inherited.manifest).unwrap();
+        inspect_source(
+            &store,
+            &meta,
+            inherited.manifest.clone(),
+            vec!["inherited".into()],
+            Vec::new(),
+        )
+        .unwrap();
+        dispose(&store, &meta, &target_id).unwrap();
+        assert_eq!(
+            inspect_source(
+                &store,
+                &meta,
+                inherited.manifest,
+                vec!["inherited".into()],
+                Vec::new()
+            )
+            .unwrap_err()
+            .code,
+            Code::ROOT_ABSENT
+        );
+        let _ = std::fs::remove_dir_all(root);
+    }
+
+    #[test]
+    fn retained_source_still_refuses_corrupt_payload_and_abandoned_writer() {
+        let root = std::env::temp_dir().join(format!(
+            "tensorfs-retained-source-refusal-{}",
+            crate::meta::now_nanos_unique()
+        ));
+        let store = Store::init(&root).unwrap();
+        let meta = Meta::open(&store).unwrap();
+        let source_id = format!("sha256:{}", "84".repeat(32));
+        let source_writer = begin(&store, &meta, &source_id, 1, created_declaration()).unwrap();
+        let bytes = vec![0x31; 2048];
+        let part = add_part(
+            &store,
+            &meta,
+            &source_id,
+            1,
+            ("model", "weight", "value"),
+            &mut bytes.as_slice(),
+        )
+        .unwrap();
+        let source = commit(&store, &meta, &source_id, 1).unwrap();
+        release_guards(
+            &meta,
+            Some(source_writer.writer_hold),
+            source_writer.source_leases,
+        );
+        let target_id = format!("sha256:{}", "85".repeat(32));
+        let writer = begin(
+            &store,
+            &meta,
+            &target_id,
+            1,
+            inherited_declaration(source.manifest.clone()),
+        )
+        .unwrap();
+        assert!(fence(&meta, &target_id, 1).unwrap());
+        abandon(&store, &meta, &target_id).unwrap();
+        assert_eq!(
+            commit(&store, &meta, &target_id, 1).unwrap_err().code,
+            Code::TRANSACTION_CLOSED
+        );
+        release_guards(&meta, Some(writer.writer_hold), writer.source_leases);
+        let path = store.blob_path(&part.part.segments()[0].sha256);
+        std::fs::remove_file(&path).unwrap();
+        std::fs::write(path, vec![0x32; 2048]).unwrap();
+        assert_eq!(
+            inspect_source(
+                &store,
+                &meta,
+                source.manifest,
+                vec!["model".into()],
+                Vec::new()
+            )
+            .unwrap_err()
+            .code,
+            Code::OBJECT_CORRUPT
+        );
+        let _ = std::fs::remove_dir_all(root);
+    }
+
     #[test]
     fn private_adoption_retains_real_payload_until_dispose_without_a_repository() {
         let root = std::env::temp_dir().join(format!(
@@ -2946,6 +3199,15 @@ mod tests {
             )
             .unwrap()
             .unwrap();
+        // An owner-retained checkpoint is a GC root before any public release exists.
+        inspect_source(
+            &store,
+            &meta,
+            manifest_ref.clone(),
+            vec!["model".into()],
+            Vec::new(),
+        )
+        .unwrap();
         store
             .apply_repository(
                 Some(&checkpointed.canonical_bytes()),

{tensorfs-0.3.9 → tensorfs-0.3.10}/crates/tensorfs-core/src/gc.rs

@@ -154,6 +154,32 @@ pub fn session_holds(store: &Store) -> Result<(Vec<HeldKey>, Vec<String>)> {
     Ok((holds, sessions))
 }
 
+/// The same exact manifest roots used by reclamation, without a payload census.
+/// Membership is retention, not caller authorization. A consumer must acquire its
+/// live hold before checking roots so GC cannot remove bytes during admission.
+pub(crate) fn retained_manifests(store: &Store) -> Result<Vec<crate::ids::ObjectRef>> {
+    let mut roots = Vec::new();
+    for (repository, _) in storage::read_repositories(store.root())? {
+        roots.extend(
+            repository
+                .checkpoints
+                .into_iter()
+                .map(|checkpoint| checkpoint.manifest),
+        );
+    }
+    let (holds, _) = session_holds(store)?;
+    roots.extend(
+        holds
+            .into_iter()
+            .filter(|hold| hold.kind == "manifest")
+            .map(|hold| crate::ids::ObjectRef {
+                sha256: hold.sha256,
+                length: hold.length,
+            }),
+    );
+    Ok(roots)
+}
+
 /// One complete pass: exclusivity, orphan temps, census, plan, delete, catalog repair.
 pub fn collect(root: &Path, dry_run: bool) -> Result<Report> {
     let store = Store::open(root)?;

{tensorfs-0.3.9 → tensorfs-0.3.10}/crates/tensorfs-core/src/meta.rs

@@ -11,7 +11,7 @@ use fs2::FileExt;
 use rusqlite::{params, Connection};
 
 use crate::err::{refuse, Code, Refusal, Result};
-use crate::ids::{ascii_name, hex64};
+use crate::ids::ascii_name;
 use crate::limits;
 use crate::store::Store;
 
@@ -82,19 +82,6 @@ impl Meta {
         self.lease_dir.join(format!("{id}.lease"))
     }
 
-    pub fn manifest_is_released(&self, digest: &str) -> Result<bool> {
-        hex64("manifest", digest)?;
-        let count: u64 = self
-            .connection()?
-            .query_row(
-                "SELECT count(*) FROM tensorfs_released_manifests WHERE manifest_sha256=?1",
-                [digest],
-                |row| row.get(0),
-            )
-            .map_err(|error| sql("query released manifest", error))?;
-        Ok(count != 0)
-    }
-
     pub(crate) fn txn(&self) -> Result<Txn> {
         let connection = self.connection()?;
         connection

{tensorfs-0.3.9 → tensorfs-0.3.10}/crates/tensorfs-core/src/storage.rs

@@ -1223,7 +1223,7 @@ impl GcDelete {
     }
 }
 
-fn read_repositories(root: &Path) -> Result<Vec<(Repository, Vec<u8>)>> {
+pub(crate) fn read_repositories(root: &Path) -> Result<Vec<(Repository, Vec<u8>)>> {
     let mut result = Vec::new();
     for org_path in read_dir(&root.join("repos"))? {
         if !org_path.is_dir() {

{tensorfs-0.3.9 → tensorfs-0.3.10}/crates/tensorfs-py/src/store.rs

@@ -1892,6 +1892,12 @@ impl PyDerivedWriter {
     fn commit<'p>(&self, py: Python<'p>) -> PyResult<Bound<'p, PyAny>> {
         let value = py
             .detach(|| {
+                let guards = self.guards.lock().expect("writer hold mutex poisoned");
+                let guards = guards.as_ref().ok_or_else(|| Refusal {
+                    code: Code::LEASE_REVOKED,
+                    detail: "derived writer holds were released".into(),
+                })?;
+                guards.writer_hold.still_registered(&self.meta)?;
                 derived::commit(
                     &self.store,
                     &self.meta,

{tensorfs-0.3.9 → tensorfs-0.3.10}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "maturin"
 
 [project]
 name = "tensorfs"
-version = "0.3.9"
+version = "0.3.10"
 description = "TensorFS — canonical tensor artifact substrate"
 requires-python = ">=3.12"
 classifiers = [