tensorfs 0.3.36__tar.gz → 0.3.38__tar.gz

{tensorfs-0.3.36 → tensorfs-0.3.38}/Cargo.lock

@@ -48,6 +48,16 @@ version = "0.5.8"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "7cd812cc2bc1d69d4764bd80df88b4317eaef9e773c75226407d9bc0876b211c"
 
+[[package]]
+name = "errno"
+version = "0.3.14"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb"
+dependencies = [
+ "libc",
+ "windows-sys",
+]
+
 [[package]]
 name = "fallible-iterator"
 version = "0.3.0"
@@ -128,6 +138,12 @@ dependencies = [
  "vcpkg",
 ]
 
+[[package]]
+name = "linux-raw-sys"
+version = "0.12.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "32a66949e030da00e8c7d4434b251670a91556f4144941d37452769c25d58a53"
+
 [[package]]
 name = "minicbor"
 version = "2.3.0"
@@ -290,6 +306,19 @@ dependencies = [
  "smallvec",
 ]
 
+[[package]]
+name = "rustix"
+version = "1.1.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b6fe4565b9518b83ef4f91bb47ce29620ca828bd32cb7e408f0062e9930ba190"
+dependencies = [
+ "bitflags",
+ "errno",
+ "libc",
+ "linux-raw-sys",
+ "windows-sys",
+]
+
 [[package]]
 name = "rustls"
 version = "0.23.43"
@@ -392,19 +421,20 @@ checksum = "adb6935a6f5c20170eeceb1a3835a49e12e19d792f6dd344ccc76a985ca5a6ca"
 
 [[package]]
 name = "tensorfs-core"
-version = "0.3.36"
+version = "0.3.38"
 dependencies = [
  "fs2",
  "minicbor",
  "rcgen",
  "rusqlite",
+ "rustix",
  "rustls",
  "webpki-roots 0.26.11",
 ]
 
 [[package]]
 name = "tensorfs-py"
-version = "0.3.36"
+version = "0.3.38"
 dependencies = [
  "pyo3",
  "tensorfs-core",

{tensorfs-0.3.36 → tensorfs-0.3.38}/Cargo.toml

@@ -3,5 +3,5 @@ resolver = "2"
 members = ["crates/tensorfs-core", "crates/tensorfs-py"]
 
 [workspace.package]
-version = "0.3.36"
+version = "0.3.38"
 edition = "2021"

{tensorfs-0.3.36 → tensorfs-0.3.38}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: tensorfs
-Version: 0.3.36
+Version: 0.3.38
 Classifier: Programming Language :: Python :: 3
 Classifier: Programming Language :: Python :: Implementation :: CPython
 Classifier: Programming Language :: Rust

{tensorfs-0.3.36 → tensorfs-0.3.38}/crates/tensorfs-core/Cargo.toml

@@ -11,6 +11,7 @@ path = "src/bin/tfs/main.rs"
 fs2 = "0.4"
 minicbor = { version = "2.3.0", default-features = false, features = ["alloc"] }
 rusqlite = { version = "0.32", features = ["bundled"] }
+rustix = { version = "1.1.4", default-features = false, features = ["fs", "std"] }
 # tfs-049: the ONE transport (src/transport/). rustls is the one TLS stack, used only
 # there; scripts/fence.py refuses it anywhere else so a second mover cannot grow.
 rustls = { version = "0.23", default-features = false, features = ["ring", "std", "tls12"] }

{tensorfs-0.3.36 → tensorfs-0.3.38}/crates/tensorfs-core/src/derived/adoption.rs

@@ -250,6 +250,146 @@ fn export(
 #[cfg(test)]
 mod tests {
     use super::*;
+    use std::collections::BTreeSet;
+    use std::sync::atomic::{AtomicU64, Ordering};
+    use std::sync::Arc;
+
+    #[derive(Default)]
+    struct MeasuredIO {
+        moved: AtomicU64,
+        admitted: AtomicU64,
+    }
+
+    impl crate::store::Progress for MeasuredIO {
+        fn moved(&self, bytes: u64) {
+            self.moved.fetch_add(bytes, Ordering::Relaxed);
+        }
+        fn admitted(&self) {
+            self.admitted.fetch_add(1, Ordering::Relaxed);
+        }
+    }
+
+    impl MeasuredIO {
+        fn take(&self) -> (u64, u64) {
+            (
+                self.moved.swap(0, Ordering::Relaxed),
+                self.admitted.swap(0, Ordering::Relaxed),
+            )
+        }
+    }
+
+    #[test]
+    fn multiple_checkpoint_groups_separate_rehash_from_adoption_metadata_io() {
+        let (store, meta, mut declaration, source, target) = fixture();
+        let mut third = declaration.components[0].add[0].clone();
+        third.key = "third".into();
+        declaration.components[0].add.push(third);
+        declaration.order.push(("model".into(), "third".into()));
+        declaration.max_new_bytes += 2048;
+        let raw = declaration.canonical_work_bytes().unwrap();
+        let writer = begin(&store, &meta, &source, 1, declaration.clone(), None).unwrap();
+        let first = write(&store, &meta, &source, "weight", 3);
+        checkpoint_progress(&store, &meta, &source, 1, "source", "model", None).unwrap();
+        let second = write(&store, &meta, &source, "next", 4);
+        let (_, checkpoint) =
+            checkpoint_progress(&store, &meta, &source, 1, "source", "model", None).unwrap();
+        fence(&meta, &source, 1).unwrap();
+        release_guards(&meta, Some(writer.writer_hold), writer.source_leases);
+
+        let progress = Arc::new(MeasuredIO::default());
+        let store = store.observed(progress.clone());
+        eprintln!("CR108_IO_ROOT {}", store.root().display());
+        eprintln!("CR108_IO_PAYLOAD {} {}", first.sha256, second.sha256);
+        // Lose only the verification receipts in this fixture, not the actual
+        // bytes. The real verifier must rehash them before the warm adoption arm.
+        let catalog = crate::catalog::Catalog::open(store.root()).unwrap();
+        catalog.drop_verification(&first.sha256);
+        catalog.drop_verification(&second.sha256);
+        drop(catalog);
+        eprintln!("CR108_IO_PHASE verification");
+        assert!(matches!(
+            store.verify(&first.sha256).unwrap(),
+            crate::store::Verdict::Invalidated { .. }
+        ));
+        assert!(matches!(
+            store.verify(&second.sha256).unwrap(),
+            crate::store::Verdict::Invalidated { .. }
+        ));
+        let verification = progress.take();
+        assert_eq!(verification, (4096, 0));
+
+        eprintln!("CR108_IO_PHASE inspect_before");
+        let before: BTreeSet<_> = store.objects().unwrap().into_iter().collect();
+        let before_payloads: Vec<_> = [&first, &second]
+            .iter()
+            .map(|object| {
+                let path = store.blob_path(&object.sha256);
+                (
+                    std::fs::read(&path).unwrap(),
+                    std::fs::metadata(path).unwrap().modified().unwrap(),
+                )
+            })
+            .collect();
+        eprintln!("CR108_IO_PHASE adoption");
+        let (_, adopted) = adopt_checkpoint(
+            &store,
+            &meta,
+            &target,
+            &source,
+            &raw,
+            checkpoint.head.as_ref().unwrap(),
+            "target",
+            "model",
+        )
+        .unwrap();
+        let adoption = progress.take();
+        eprintln!("CR108_IO_PHASE inspect_after");
+        let fresh: Vec<_> = store
+            .objects()
+            .unwrap()
+            .into_iter()
+            .filter(|id| !before.contains(id))
+            .collect();
+        let metadata_bytes: u64 = fresh
+            .iter()
+            .map(|id| std::fs::metadata(store.blob_path(id)).unwrap().len())
+            .sum();
+        assert!(metadata_bytes > 0);
+        assert_eq!(adoption, (metadata_bytes, fresh.len() as u64));
+        for (object, prior) in [&first, &second].iter().zip(before_payloads) {
+            let path = store.blob_path(&object.sha256);
+            assert_eq!(std::fs::read(&path).unwrap(), prior.0);
+            assert_eq!(
+                std::fs::metadata(path).unwrap().modified().unwrap(),
+                prior.1
+            );
+        }
+        eprintln!("CR108_IO_PHASE restore");
+        let next = begin(
+            &store,
+            &meta,
+            &target,
+            1,
+            declaration,
+            adopted.head.as_ref(),
+        )
+        .unwrap();
+        assert_eq!(
+            completed(&meta, &target, 1).unwrap().0,
+            vec![
+                ("model".into(), "next".into(), "value".into()),
+                ("model".into(), "weight".into(), "value".into())
+            ]
+        );
+        let restore = progress.take();
+        assert_eq!(restore, (0, 0));
+        eprintln!("CR108_IO_PHASE finish");
+        write(&store, &meta, &target, "third", 5);
+        commit(&store, &meta, &target, 1).unwrap();
+        release_guards(&meta, Some(next.writer_hold), next.source_leases);
+        eprintln!("CR108_IO_PHASE done");
+        eprintln!("CR108_IO_COUNTS verification={verification:?} adoption={adoption:?} restore={restore:?} adoption_metadata_bytes={metadata_bytes}");
+    }
 
     fn fixture() -> (Store, Meta, Declaration, String, String) {
         let path = std::env::temp_dir().join(format!(

{tensorfs-0.3.36 → tensorfs-0.3.38}/crates/tensorfs-core/src/reclaim.rs

@@ -498,22 +498,11 @@ pub(crate) fn plan_managed(
 
 // ------------------------------------------------------------------ the act (tfs-066)
 
-/// How a plan's bytes are proven to be gettable again before they are let go.
-#[derive(Debug, Clone, Copy, PartialEq, Eq)]
-pub enum Recovery {
-    /// The object is already on the bound repo cache at its exact length.
-    Cache,
-    /// This run put it there.
-    Demoted,
-    /// The caller declared the roots recoverable from the hub. Their word, like free space.
-    Declared,
-}
-
 /// What `run` was asked to do beyond the plan itself.
 #[derive(Debug, Clone, Copy, Default)]
 pub struct RunOptions {
-    /// Publish each object to the bound repo cache before dropping it, so the refetch is a
-    /// volume read rather than an R2 pull.
+    /// Best-effort cache warming before reclaim. This never authorizes deletion;
+    /// assume_upstream is still required because the cache is disposable.
     pub demote: bool,
     /// The caller declares these roots recoverable from the hub. Same class of caller-input
     /// as free space (decisions.md 241): TensorFS cannot see the hub, the caller can.
@@ -573,87 +562,23 @@ impl RunReport {
     }
 }
 
-/// Is this object gettable again without the pod's own history?
-///
-/// decisions.md 842 set the DURABLE-UPSTREAM INVARIANT — *"pressure eviction may target
-/// only objects with a recorded durable upstream; local-only artifacts are never
-/// pressure-evicted"* — with the falsifier *"a pressure sweep evicting a local-only
-/// artifact"*. tfs-020 then deleted the `hub_published` boolean along with the pressure
-/// plane it gated, so the invariant has had no mechanism since.
-///
-/// It comes back CONSTRUCTIVELY rather than as a flag someone must remember to set: the
-/// cache is checked, or this run puts the bytes there, or the caller says so. Demotion is
-/// the interesting one, because it does not FIND the invariant true — it MAKES it true, and
-/// it makes it true against the `/workspace` volume, which measurably survives the
-/// container-disk reset that destroys the Store (th-152, 2026-09-04).
-fn recovery(
-    store: &Store,
-    row: &crate::storage::GcDelete,
-    options: RunOptions,
-    report: &mut RunReport,
-) -> Option<Recovery> {
-    if options.assume_upstream {
-        return Some(Recovery::Declared);
-    }
-    let cache = store.repo_cache()?;
-    let kind = match row.kind {
-        "manifest" => CacheKind::Manifest,
-        _ => CacheKind::Blob,
-    };
-    let want = ObjectRef {
-        sha256: row.sha256.clone(),
-        length: row.length,
-    };
-    // Present at the exact length is the cheap check and the honest one: the cache's own
-    // admission path re-verifies the digest on the way back in, so a byte-level check here
-    // would pay for a promise something else already keeps.
-    if let Ok(path) = cache.path(kind, &row.sha256) {
-        if std::fs::metadata(&path).is_ok_and(|meta| meta.is_file() && meta.len() == row.length) {
-            report.already_cached_objects += 1;
-            return Some(Recovery::Cache);
-        }
-    }
-    if !options.demote {
-        return None;
-    }
-    match cache.backfill_store(store, kind, &want) {
-        Ok(CacheWrite::Stored) => {
-            report.demoted_objects += 1;
-            report.demoted_bytes = report.demoted_bytes.saturating_add(row.length);
-            Some(Recovery::Demoted)
-        }
-        Ok(CacheWrite::Present) => {
-            report.already_cached_objects += 1;
-            Some(Recovery::Cache)
-        }
-        // A cache that cannot take the bytes is weather everywhere else in this crate and
-        // stays weather here — but for an EVICTION the consequence is different. On a fetch
-        // the authority is still R2; on an eviction this pod may hold the last copy. So a
-        // failed demote does not become a silent delete, it becomes a refusal.
-        Ok(CacheWrite::Unavailable) | Err(_) => None,
-    }
-}
-
-/// Demote, tombstone, collect — in that order, and the order is the argument.
-///
-/// 1. **Demote** while the objects are still live and still reachable. Nothing is deleted.
-/// 2. **Tombstone** — `DeleteRepository` per victim, through `apply_repository`'s
-///    compare-and-set. This is the *explicit act* decisions.md 1135 requires: bytes become
-///    collectable because somebody removed the thing that named them, never because a sweep
-///    inferred they were garbage.
-/// 3. **Collect** — the existing `gc::collect`, unchanged. It recomputes reachability from
-///    `repos/`, `manifests/` and `blobs/` alone.
-///
-/// **Step 3 is why a wrong plan cannot corrupt the Store.** The plan chooses which ROOTS
-/// stop being named; the census decides which OBJECTS that actually frees. A caller that
-/// names every root still cannot delete an object a hold protects or a retained manifest
-/// needs.
+/// Reclaim named roots only after the caller confirms durable upstream custody.
+/// Optional cache warming precedes repository tombstones and ordinary GC; cache
+/// presence never authorizes either step. Retained roots and active holds still
+/// protect their objects through the existing reachability census.
 pub fn run(store: &Store, drop: &[RepositoryName], options: RunOptions) -> Result<RunReport> {
     permitted(store)?;
     let mut report = RunReport::default();
     if drop.is_empty() {
         return Ok(report);
     }
+    if !options.assume_upstream {
+        return refuse(
+            Code::DURABILITY_UNPROVEN,
+            "named local roots require --assume-upstream after confirming durable upstream \
+             custody; an optional cache or --demote cannot preserve the only copy",
+        );
+    }
 
     // EXCLUSIVITY IS PROVED BEFORE ANYTHING IS DONE, not discovered at the end.
     //
@@ -698,26 +623,29 @@ pub fn run(store: &Store, drop: &[RepositoryName], options: RunOptions) -> Resul
         .filter(|row| !already.contains(&row.key))
         .collect();
 
-    let mut unrecoverable = Vec::new();
-    for row in &condemned {
-        if recovery(store, row, options, &mut report).is_none() {
-            unrecoverable.push(format!("{}:{}", row.kind, row.sha256));
+    if options.demote {
+        if let Some(cache) = store.repo_cache() {
+            for row in &condemned {
+                let kind = if row.kind == "manifest" {
+                    CacheKind::Manifest
+                } else {
+                    CacheKind::Blob
+                };
+                let object = ObjectRef {
+                    sha256: row.sha256.clone(),
+                    length: row.length,
+                };
+                match cache.backfill_store(store, kind, &object) {
+                    Ok(CacheWrite::Stored) => {
+                        report.demoted_objects += 1;
+                        report.demoted_bytes = report.demoted_bytes.saturating_add(row.length);
+                    }
+                    Ok(CacheWrite::Present) => report.already_cached_objects += 1,
+                    Ok(CacheWrite::Unavailable) | Err(_) => {}
+                }
+            }
         }
     }
-    if !unrecoverable.is_empty() {
-        let shown: Vec<&str> = unrecoverable.iter().take(3).map(String::as_str).collect();
-        return refuse(
-            Code::DURABILITY_UNPROVEN,
-            format!(
-                "{} of {} objects cannot be got back if they go — {} — so nothing was \
-                 dropped; bind a repo cache and pass --demote, or declare the roots \
-                 recoverable with --assume-upstream",
-                unrecoverable.len(),
-                condemned.len(),
-                shown.join(", "),
-            ),
-        );
-    }
 
     let fault = crate::store::Fault::default();
     for repo in drop {