tensorfs 0.3.59__tar.gz → 0.3.60__tar.gz

{tensorfs-0.3.59 → tensorfs-0.3.60}/Cargo.lock

@@ -439,7 +439,7 @@ checksum = "adb6935a6f5c20170eeceb1a3835a49e12e19d792f6dd344ccc76a985ca5a6ca"
 
 [[package]]
 name = "tensorfs-core"
-version = "0.3.59"
+version = "0.3.60"
 dependencies = [
  "fs2",
  "libc",
@@ -454,7 +454,7 @@ dependencies = [
 
 [[package]]
 name = "tensorfs-py"
-version = "0.3.59"
+version = "0.3.60"
 dependencies = [
  "pyo3",
  "tensorfs-core",

{tensorfs-0.3.59 → tensorfs-0.3.60}/Cargo.toml

@@ -3,5 +3,5 @@ resolver = "2"
 members = ["crates/tensorfs-core", "crates/tensorfs-py"]
 
 [workspace.package]
-version = "0.3.59"
+version = "0.3.60"
 edition = "2021"

{tensorfs-0.3.59 → tensorfs-0.3.60}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: tensorfs
-Version: 0.3.59
+Version: 0.3.60
 Classifier: Programming Language :: Python :: 3
 Classifier: Programming Language :: Python :: Implementation :: CPython
 Classifier: Programming Language :: Rust

{tensorfs-0.3.59 → tensorfs-0.3.60}/crates/tensorfs-core/src/bin/tfs/main.rs

@@ -505,7 +505,7 @@ fn cmd_repo_cache_restore(store: &Path, object: &str, length: &str, flags: &Flag
 /// The chain head names links that live on the cache; each is read at its exact digest, and
 /// the chain is DISCARDED WHOLE unless every link was written under the plan `--plan` names
 /// — the plan a resuming run re-derives from the source headers alone, for zero tensor
-/// bytes. That is what makes a stale chain harmless rather than dangerous.
+/// bytes. A discarded chain restores nothing and succeeds: the run converts afresh.
 ///
 /// The objects come back exactly as a checkpoint's closure does. The conversion journal the
 /// newest link points at is installed beside them under `--session`, so the next
@@ -548,7 +548,15 @@ fn cmd_repo_cache_resume(store: &Path, object: &str, length: &str, flags: &Flags
     };
     let set = match durability::fold(&chain, plan) {
         Ok(set) => set,
-        Err(error) => return bail(error),
+        Err(error) if error.code == tensorfs_core::err::Code::IO_FAILED => return bail(error),
+        Err(error) => {
+            println!(
+                "chain        DISCARDED {}: {} — nothing restored; the conversion starts a new chain",
+                error.code.as_str(),
+                error.detail
+            );
+            return ExitCode::SUCCESS;
+        }
     };
     println!(
         "chain        {} link(s) over operation {} — {} B declared durable",

{tensorfs-0.3.59 → tensorfs-0.3.60}/crates/tensorfs-core/src/checkpoint.rs

@@ -418,6 +418,28 @@ impl Walk {
         }
         Ok(())
     }
+
+    /// Every reached object present, or held by a live source operation's publication
+    /// custodian. A complete checkpoint for publication, never a readable one.
+    pub fn require_held(
+        &self,
+        store: &Store,
+        custodied: &std::collections::BTreeSet<String>,
+    ) -> Result<()> {
+        for r in self.distinct() {
+            if !store.contains(&r.sha256) && !custodied.contains(&r.sha256) {
+                return refuse(
+                    Code::OBJECT_ABSENT,
+                    format!(
+                        "{} is committed by this manifest but neither resident nor held by \
+                         a live publication custodian",
+                        r.id()
+                    ),
+                );
+            }
+        }
+        Ok(())
+    }
 }
 
 /// Manifest → CozyTensorsHeader → contained tensor rules → storage-part object runs.

{tensorfs-0.3.59 → tensorfs-0.3.60}/crates/tensorfs-core/src/derived/resume_tests.rs

@@ -351,7 +351,7 @@ fn missing_corrupt_and_changed_work_cannot_be_reused() {
 }
 
 #[test]
-fn recovery_refuses_old_writer_authority_and_malformed_part_geometry() {
+fn recovery_discards_old_writer_authority_and_malformed_part_geometry() {
     let (store, meta, id, declaration) = fixture("authority");
     let writer = begin(&store, &meta, &id, 9, declaration.clone(), None).unwrap();
     data(&store, &meta, &id, 9, 0x31);
@@ -359,7 +359,8 @@ fn recovery_refuses_old_writer_authority_and_malformed_part_geometry() {
     let plan = progress_plan(&row);
     let mut geometry = progress_row(&row);
     geometry.added_parts[0].part.shape = vec![16, 31];
-    for snapshot in [row, geometry] {
+    let mut writers = vec![writer];
+    for (epoch, snapshot) in [(10, row), (11, geometry)] {
         let bytes = crate::canon::write(&snapshot.to_value());
         let head = crate::durability::checkpoint_bytes(
             &store,
@@ -378,11 +379,20 @@ fn recovery_refuses_old_writer_authority_and_malformed_part_geometry() {
         .head
         .head
         .unwrap();
-        assert!(begin(&store, &meta, &id, 10, declaration.clone(), Some(&head)).is_err());
-        check_writer(&meta, &id, 9).unwrap();
+        // The checkpoint cannot be continued: it is discarded, nothing of it is imported,
+        // and the new writer proceeds from the work this Store already holds.
+        writers.push(begin(&store, &meta, &id, epoch, declaration.clone(), Some(&head)).unwrap());
+        check_writer(&meta, &id, epoch).unwrap();
+        let current = meta.derived_rows().unwrap().remove(0);
+        assert!(current
+            .added_parts
+            .iter()
+            .all(|added| added.part.shape != vec![16, 31]));
+    }
+    fence(&meta, &id, 11).unwrap();
+    for writer in writers {
+        release(&meta, writer);
     }
-    fence(&meta, &id, 9).unwrap();
-    release(&meta, writer);
     abandon(&store, &meta, &id).unwrap();
     std::fs::remove_dir_all(store.root()).unwrap();
 }
@@ -489,7 +499,7 @@ fn source_identity_is_locked_and_source_closure_stays_rooted_after_writer_death(
 }
 
 #[test]
-fn wrong_document_lengths_refuse_before_parsing_or_importing_work() {
+fn wrong_document_lengths_refuse_to_validate_and_are_discarded_on_resume() {
     let (store, meta, id, declaration) = fixture("lengths");
     let writer = begin(&store, &meta, &id, 1, declaration.clone(), None).unwrap();
     data(&store, &meta, &id, 1, 0x31);
@@ -528,16 +538,13 @@ fn wrong_document_lengths_refuse_before_parsing_or_importing_work() {
             .code,
         Code::LENGTH_MISMATCH
     );
-    assert_eq!(
-        begin(&store, &meta, &id, 2, declaration, Some(&forged))
-            .err()
-            .unwrap()
-            .code,
-        Code::LENGTH_MISMATCH
-    );
-    check_writer(&meta, &id, 1).unwrap();
-    fence(&meta, &id, 1).unwrap();
+    // A forged checkpoint is discarded, not imported; this begin then meets the live writer.
+    // Resuming from it discards it instead of refusing: the next writer proceeds.
+    let next = begin(&store, &meta, &id, 2, declaration, Some(&forged)).unwrap();
+    check_writer(&meta, &id, 2).unwrap();
+    fence(&meta, &id, 2).unwrap();
     release(&meta, writer);
+    release(&meta, next);
     abandon(&store, &meta, &id).unwrap();
     std::fs::remove_dir_all(store.root()).unwrap();
 }

{tensorfs-0.3.59 → tensorfs-0.3.60}/crates/tensorfs-core/src/derived.rs

@@ -1601,6 +1601,9 @@ fn require_source_roots(store: &Store, declaration: &Declaration) -> Result<()>
 }
 
 fn load_sources(store: &Store, declaration: &Declaration) -> Result<Vec<SourceData>> {
+    // A prepared model whose output a live source operation's publication custodian holds
+    // derives by reference without its bytes; the result is publishable, never readable.
+    let custodied = crate::ingest::custody::live(store.root())?;
     let mut out = Vec::new();
     for declared in &declaration.sources {
         let manifest = checkpoint::load_manifest(store, &declared.manifest)?;
@@ -1625,11 +1628,13 @@ fn load_sources(store: &Store, declaration: &Declaration) -> Result<Vec<SourceDa
         let closure = checkpoint::load_closure(store, &header)?;
         header.validate(&closure)?;
         let walk = checkpoint::walk_cozytensors(store, &manifest)?;
-        walk.require_resident(store)?;
+        walk.require_held(store, &custodied)?;
         // This is the zero-read/zero-hash inherit gate: the verification record proves the
         // exact installed inode/length/generation without opening or hashing payload bytes.
         for reached in &walk.objects {
-            if reached.kind == "part" || reached.kind == "config" {
+            if (reached.kind == "part" || reached.kind == "config")
+                && !custodied.contains(&reached.obj.sha256)
+            {
                 require_payload_record(store, &reached.obj, reached.kind)?;
             }
         }
@@ -2308,9 +2313,35 @@ pub fn begin(
         release_guards(meta, Some(writer_hold), Vec::new());
         return Err(error);
     }
+    // Pin every resident source object. A custodied object has no local bytes to pin: a
+    // derivation over it carries it by reference and never reads it.
+    let custodied = match crate::ingest::custody::live(store.root()) {
+        Ok(custodied) => custodied,
+        Err(error) => {
+            release_guards(meta, Some(writer_hold), Vec::new());
+            return Err(error);
+        }
+    };
     let mut source_leases = Vec::new();
     for source in &sources {
-        match read::acquire_cozytensors(store, meta, &source.manifest) {
+        let pinned = if custodied.is_empty() {
+            read::acquire_cozytensors(store, meta, &source.manifest)
+        } else {
+            checkpoint::walk_cozytensors(store, &source.manifest).and_then(|walk| {
+                let id = source.manifest.manifest_id();
+                read::acquire(
+                    store,
+                    meta,
+                    id.strip_prefix("sha256:").unwrap_or(&id),
+                    walk.distinct()
+                        .into_iter()
+                        .filter(|object| !custodied.contains(&object.sha256))
+                        .cloned()
+                        .collect(),
+                )
+            })
+        };
+        match pinned {
             Ok((lease, receipts)) if receipts.0.is_empty() => {
                 source_leases.push((source.fact.alias.clone(), lease));
             }
@@ -2342,15 +2373,22 @@ pub fn begin(
         })
         .collect();
 
+    // A checkpoint this work cannot continue (another plan or work context, or no longer
+    // whole) is discarded: the work re-derives what it lost instead of refusing.
+    let mut checkpoint = checkpoint;
     let incoming = match checkpoint
         .map(|head| read_progress(store, head, &transaction, &declaration_text, &source_facts))
-        .transpose()
     {
-        Ok(incoming) => incoming,
-        Err(error) => {
+        None => None,
+        Some(Ok(row)) => Some(row),
+        Some(Err(error)) if error.code == Code::IO_FAILED => {
             release_guards(meta, Some(writer_hold), source_leases);
             return Err(error);
         }
+        Some(Err(_)) => {
+            checkpoint = None;
+            None
+        }
     };
     // Byte verification can update the verification catalog. Keep it outside the SQL
     // writer transaction, then compare the observed row before assigning this epoch.
@@ -3269,7 +3307,7 @@ pub fn commit(
     let manifest_ref = store.put_manifest(&manifest)?.obj;
     store.read_manifest(&manifest_ref)?;
     let walk = checkpoint::walk(store, &manifest)?;
-    walk.require_resident(store)?;
+    walk.require_held(store, &crate::ingest::custody::live(store.root())?)?;
     let (inherited_payload_objects, inherited_payload_bytes) =
         inherit_observation(&header, &row_before)?;
 

{tensorfs-0.3.59 → tensorfs-0.3.60}/crates/tensorfs-core/src/durability.rs

@@ -289,6 +289,23 @@ pub fn local_chain(store: &Store, head: &ObjectRef, plan: &str) -> Result<Set> {
     fold(&links, plan)
 }
 
+/// A supplied chain this conversion can continue, or `None` when it was written under another
+/// plan or slot or is no longer whole in this Store. Such a chain is discarded and the work
+/// reconverts into a new chain: losing a checkpoint costs time, never a refusal.
+pub fn continuable(
+    store: &Store,
+    head: &ObjectRef,
+    plan: &str,
+    chain: &str,
+) -> Result<Option<Set>> {
+    match local_chain(store, head, plan) {
+        Ok(set) if set.operation == chain => Ok(Some(set)),
+        Ok(_) => Ok(None),
+        Err(error) if error.code == Code::IO_FAILED => Err(error),
+        Err(_) => Ok(None),
+    }
+}
+
 /// Export locally complete progress. This does not assert remote durability: the RecordOwner
 /// publishes the bounded chain and advances its durable pointer only after upload acceptance.
 /// NFS is not consulted and cannot delay this checkpoint.

{tensorfs-0.3.59 → tensorfs-0.3.60}/crates/tensorfs-core/src/gc.rs

@@ -108,8 +108,12 @@ pub fn session_holds(store: &Store) -> Result<(Vec<HeldKey>, Vec<String>)> {
     let mut sessions = Vec::new();
     for session in tx::sessions(store) {
         let root = tx::read_session(store.root(), &session)?;
-        let journalled =
+        // An object the operation's publication custodian already holds is not in flight:
+        // the journal stops naming it, so the local copy is ordinary unreferenced bytes.
+        let custody = crate::ingest::custody::read_session(store.root(), &session)?;
+        let mut journalled =
             crate::ingest::journal::session_objects(&tx::candidate_dir(store.root(), &session))?;
+        journalled.retain(|object| !custody.contains(object));
         if root.candidates.is_empty() && journalled.is_empty() {
             continue;
         }

tensorfs-0.3.60/crates/tensorfs-core/src/ingest/custody.rs

@@ -0,0 +1,249 @@
+//! Remote custody of one model-source operation's converted output.
+//!
+//! A bounded-disk ingest publishes converted objects while it converts, so its output never
+//! has to fit on the pod. Once the publication custodian has verified an object, the caller
+//! records that fact here. From then on the operation's journal stops holding that object
+//! against GC, a resumed pass does not re-convert it, and finalize or a derivation over the
+//! prepared model accept it without local bytes. Nothing READS a custodied object: it is not
+//! resident, and a reader still has to fetch it back through the ordinary verified door.
+//!
+//! The record lives in the ingest session root and is released with it, so an attestation
+//! never outlives the operation that made it. The custodian is named by opaque evidence the
+//! caller supplies (its publication identity); TensorFS carries it and never interprets it.
+
+use std::collections::{BTreeMap, BTreeSet};
+use std::fs::{self, OpenOptions};
+use std::io::{Read, Write};
+use std::path::Path;
+
+use crate::canon::{as_arr, Fields, Value};
+use crate::err::{refuse, Code, Refusal, Result};
+use crate::header::Part;
+use crate::ids::{ObjectRef, Plain};
+use crate::limits;
+use crate::store::Store;
+
+use super::transaction::candidate_dir;
+
+const FILE: &str = "custody.json";
+const FORMAT: &str = "tensorfs.source-custody/1";
+const EVIDENCE_MAX_BYTES: usize = 1024;
+
+fn io(what: &str, error: std::io::Error) -> Refusal {
+    Refusal {
+        code: Code::IO_FAILED,
+        detail: format!("source custody {what}: {error}"),
+    }
+}
+
+/// Every object this operation's custodian holds, with the evidence that says where.
+#[derive(Debug, Clone, Default, PartialEq, Eq)]
+pub struct Custody {
+    objects: BTreeMap<String, (u64, String)>,
+}
+
+impl Plain for Custody {
+    const MAX_BYTES: usize = limits::DOC_MAX_BYTES;
+
+    fn from_value(value: &Value) -> Result<Self> {
+        let mut fields = Fields::new("Custody", value)?;
+        if fields.req_str("format")? != FORMAT {
+            return refuse(Code::WRONG_TYPE, "source custody has an unknown format");
+        }
+        let mut objects = BTreeMap::new();
+        for row in as_arr("Custody", "objects", fields.req("objects")?)? {
+            let mut row = Fields::new("Custody.objects", row)?;
+            let evidence = row.req_str("evidence")?.to_string();
+            let object = ObjectRef::from_value("Custody.object", row.req("object")?)?;
+            row.done()?;
+            check_evidence(&evidence)?;
+            if objects
+                .insert(object.sha256, (object.length, evidence))
+                .is_some()
+            {
+                return refuse(Code::DUPLICATE_KEY, "source custody repeats an object");
+            }
+        }
+        fields.done()?;
+        Ok(Self { objects })
+    }
+
+    fn to_value(&self) -> Value {
+        Value::obj(vec![
+            ("format", Value::str(FORMAT)),
+            (
+                "objects",
+                Value::arr(
+                    self.objects
+                        .iter()
+                        .map(|(sha256, (length, evidence))| {
+                            Value::obj(vec![
+                                ("evidence", Value::str(evidence.clone())),
+                                (
+                                    "object",
+                                    ObjectRef {
+                                        sha256: sha256.clone(),
+                                        length: *length,
+                                    }
+                                    .to_value(),
+                                ),
+                            ])
+                        })
+                        .collect(),
+                ),
+            ),
+        ])
+    }
+}
+
+impl Custody {
+    pub fn contains(&self, object: &ObjectRef) -> bool {
+        self.objects
+            .get(&object.sha256)
+            .is_some_and(|(length, _)| *length == object.length)
+    }
+
+    /// A part stands on custody alone only when it has segments and the custodian holds
+    /// every one of them. An inline part carries its bytes in the journal line itself.
+    pub fn holds_part(&self, part: &Part) -> bool {
+        !part.segments().is_empty() && part.segments().iter().all(|s| self.contains(s))
+    }
+
+    pub fn digests(&self) -> BTreeSet<String> {
+        self.objects.keys().cloned().collect()
+    }
+
+    pub fn rows(&self) -> Vec<(ObjectRef, String)> {
+        self.objects
+            .iter()
+            .map(|(sha256, (length, evidence))| {
+                (
+                    ObjectRef {
+                        sha256: sha256.clone(),
+                        length: *length,
+                    },
+                    evidence.clone(),
+                )
+            })
+            .collect()
+    }
+
+    pub fn is_empty(&self) -> bool {
+        self.objects.is_empty()
+    }
+}
+
+fn check_evidence(evidence: &str) -> Result<()> {
+    if evidence.is_empty()
+        || evidence.len() > EVIDENCE_MAX_BYTES
+        || !evidence.bytes().all(|b| (0x21..0x7f).contains(&b))
+    {
+        return refuse(
+            Code::KEY_GRAMMAR,
+            "custody evidence is 1..=1024 printable ASCII bytes without spaces",
+        );
+    }
+    Ok(())
+}
+
+/// The operation's record, empty when it has none.
+pub fn read(store_root: &Path, operation: &str) -> Result<Custody> {
+    crate::catalog::source_operation_id(operation)?;
+    read_session(store_root, operation)
+}
+
+/// Every object the operation's conversion journals name, custodied or not.
+pub fn journalled(store_root: &Path, operation: &str) -> Result<Vec<ObjectRef>> {
+    crate::catalog::source_operation_id(operation)?;
+    super::journal::session_objects(&candidate_dir(store_root, operation))
+}
+
+/// A session directory GC already enumerated; its name came from the filesystem.
+pub(crate) fn read_session(store_root: &Path, session: &str) -> Result<Custody> {
+    let path = candidate_dir(store_root, session).join(FILE);
+    let file = match fs::File::open(&path) {
+        Ok(file) => file,
+        Err(error) if error.kind() == std::io::ErrorKind::NotFound => return Ok(Custody::default()),
+        Err(error) => return Err(io("open", error)),
+    };
+    let mut bytes = Vec::new();
+    file.take(Custody::MAX_BYTES as u64 + 1)
+        .read_to_end(&mut bytes)
+        .map_err(|error| io("read", error))?;
+    Custody::parse(&bytes)
+}
+
+/// Every live session's custody, for the readers that walk more than one operation:
+/// GC's census and a derivation over a prepared model.
+pub fn live(store_root: &Path) -> Result<BTreeSet<String>> {
+    let base = store_root.join("tmp").join("ingest");
+    let entries = match fs::read_dir(&base) {
+        Ok(entries) => entries,
+        Err(error) if error.kind() == std::io::ErrorKind::NotFound => return Ok(BTreeSet::new()),
+        Err(error) => return Err(io("list sessions", error)),
+    };
+    let mut out = BTreeSet::new();
+    for entry in entries {
+        let entry = entry.map_err(|error| io("list sessions", error))?;
+        if !entry.path().is_dir() {
+            continue;
+        }
+        let session = entry.file_name().to_string_lossy().to_string();
+        out.extend(read_session(store_root, &session)?.digests());
+    }
+    Ok(out)
+}
+
+/// Record that the custodian named by `evidence` holds these journalled objects. Only
+/// objects this operation's own conversion journal names may be recorded; the record is
+/// rewritten whole, atomically, so a crash leaves either the old set or the new one.
+pub fn record(store: &Store, operation: &str, objects: &[ObjectRef], evidence: &str) -> Result<()> {
+    crate::catalog::source_operation_id(operation)?;
+    check_evidence(evidence)?;
+    let dir = candidate_dir(store.root(), operation);
+    if !dir.is_dir() {
+        return refuse(Code::ROOT_ABSENT, "source custody names no live session");
+    }
+    let journalled: BTreeMap<String, u64> = journalled(store.root(), operation)?
+        .into_iter()
+        .map(|object| (object.sha256, object.length))
+        .collect();
+    let mut custody = read(store.root(), operation)?;
+    for object in objects {
+        if journalled.get(&object.sha256) != Some(&object.length) {
+            return refuse(
+                Code::OBJECT_ID_MISMATCH,
+                format!(
+                    "{} is not an object this operation's conversion journal names",
+                    object.id()
+                ),
+            );
+        }
+        custody
+            .objects
+            .entry(object.sha256.clone())
+            .or_insert_with(|| (object.length, evidence.to_string()));
+    }
+    let bytes = custody.canonical_bytes();
+    if bytes.len() > Custody::MAX_BYTES {
+        return refuse(Code::SIZE_CAP, "source custody exceeds its document bound");
+    }
+    let path = dir.join(FILE);
+    let temporary = dir.join(format!(".{FILE}.{}", crate::meta::now_nanos_unique()));
+    let written = (|| {
+        let mut file = OpenOptions::new()
+            .write(true)
+            .create_new(true)
+            .open(&temporary)
+            .map_err(|error| io("create", error))?;
+        file.write_all(&bytes)
+            .and_then(|()| file.sync_all())
+            .map_err(|error| io("write", error))?;
+        fs::rename(&temporary, &path).map_err(|error| io("rename", error))?;
+        crate::store::fsync_dir(&dir)
+    })();
+    if written.is_err() {
+        let _ = fs::remove_file(&temporary);
+    }
+    written
+}

{tensorfs-0.3.59 → tensorfs-0.3.60}/crates/tensorfs-core/src/ingest/journal.rs

@@ -441,6 +441,19 @@ impl Journal {
 /// makes no part true either: `convert` re-checks each one against `segments_still_stand`
 /// before reusing it, so a journal restored beside objects that did not survive re-converts
 /// exactly those ops.
+/// Remove one plan's journal and cursor: work that can no longer be continued.
+pub fn discard(dir: &Path, plan_digest: &str) -> Result<()> {
+    let journal = journal_path(dir, plan_digest);
+    for path in [journal.with_extension("head.json"), journal] {
+        match fs::remove_file(&path) {
+            Ok(()) => {}
+            Err(error) if error.kind() == std::io::ErrorKind::NotFound => {}
+            Err(error) => return Err(io("discard conversion journal", error)),
+        }
+    }
+    Ok(())
+}
+
 pub fn install(dir: &Path, plan_digest: &str, bytes: &[u8]) -> Result<()> {
     hex64("conversion journal plan", plan_digest)?;
     for line in bytes.split(|byte| *byte == b'\n') {

{tensorfs-0.3.59 → tensorfs-0.3.60}/crates/tensorfs-core/src/ingest/mod.rs

@@ -17,6 +17,7 @@
 
 pub mod carrier;
 pub mod convert;
+pub mod custody;
 pub mod fingerprint;
 pub mod gguf;
 mod h3_lora;