tensorfs 0.3.8__tar.gz → 0.3.9__tar.gz

{tensorfs-0.3.8 → tensorfs-0.3.9}/Cargo.lock

@@ -392,7 +392,7 @@ checksum = "adb6935a6f5c20170eeceb1a3835a49e12e19d792f6dd344ccc76a985ca5a6ca"
 
 [[package]]
 name = "tensorfs-core"
-version = "0.3.8"
+version = "0.3.9"
 dependencies = [
  "fs2",
  "minicbor",
@@ -404,7 +404,7 @@ dependencies = [
 
 [[package]]
 name = "tensorfs-py"
-version = "0.3.8"
+version = "0.3.9"
 dependencies = [
  "pyo3",
  "tensorfs-core",

{tensorfs-0.3.8 → tensorfs-0.3.9}/Cargo.toml

@@ -3,5 +3,5 @@ resolver = "2"
 members = ["crates/tensorfs-core", "crates/tensorfs-py"]
 
 [workspace.package]
-version = "0.3.8"
+version = "0.3.9"
 edition = "2021"

{tensorfs-0.3.8 → tensorfs-0.3.9}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: tensorfs
-Version: 0.3.8
+Version: 0.3.9
 Classifier: Programming Language :: Python :: 3
 Classifier: Programming Language :: Python :: Implementation :: CPython
 Classifier: Programming Language :: Rust

{tensorfs-0.3.8 → tensorfs-0.3.9}/crates/tensorfs-core/src/derived.rs

@@ -143,6 +143,20 @@ pub struct Declaration {
 }
 
 impl Declaration {
+    /// The native declaration encoder shared by Runtime intents and writers.
+    pub fn canonical_bytes(&mut self) -> Result<Vec<u8>> {
+        self.normalize_and_validate()?;
+        self.require_order()?;
+        let bytes = crate::canon::write(&self.to_value());
+        if bytes.len() > limits::DOC_MAX_BYTES / 2 {
+            return refuse(
+                Code::SIZE_CAP,
+                "derived declaration is over its metadata budget",
+            );
+        }
+        Ok(bytes)
+    }
+
     fn require_order(&self) -> Result<()> {
         if self.order.is_empty() {
             return refuse(
@@ -1862,15 +1876,8 @@ pub fn begin(
             "WriterSessionId must be positive and inside the canonical integer range",
         );
     }
-    declaration.normalize_and_validate()?;
-    declaration.require_order()?;
-    let declaration_text = declaration.canonical_text();
-    if declaration_text.len() > limits::DOC_MAX_BYTES / 2 {
-        return refuse(
-            Code::SIZE_CAP,
-            "derived declaration is over its metadata budget",
-        );
-    }
+    let declaration_text =
+        String::from_utf8(declaration.canonical_bytes()?).expect("canonical JSON is UTF-8");
 
     // The writer hold is registered before source metadata is touched. It is the live
     // process proof that keeps exclusive GC out even for a source-free create.

{tensorfs-0.3.8 → tensorfs-0.3.9}/crates/tensorfs-core/src/durability.rs

@@ -185,21 +185,34 @@ pub fn canonical_refs(mut refs: Vec<ObjectRef>) -> Vec<ObjectRef> {
 /// Read an admitted link from the local Store. Unlike `chain`, this does not require an
 /// NFS binding: the ordinary transfer plane may have obtained it from the hub.
 pub fn local_link(store: &Store, object: &ObjectRef) -> Result<Link> {
-    if object.length > Link::MAX_BYTES as u64 {
+    Link::parse(&read_document(store, object, Link::MAX_BYTES)?)
+}
+
+/// Bound the actual pinned file before allocation, not only the untrusted claimed length.
+/// Source and derived progress use the same exact ObjectRef admission contract as links.
+pub(crate) fn read_document(store: &Store, object: &ObjectRef, cap: usize) -> Result<Vec<u8>> {
+    if object.length > cap as u64 {
+        return refuse(Code::COUNT_CAP, "checkpoint document exceeds its byte cap");
+    }
+    let mut file = store.open_verified(&object.sha256)?;
+    if file.len() != object.length {
         return refuse(
-            Code::COUNT_CAP,
-            "checkpoint link exceeds the document byte cap",
+            Code::LENGTH_MISMATCH,
+            "checkpoint document differs from its ObjectRef length",
         );
     }
-    let mut bytes = Vec::with_capacity(object.length as usize);
-    store.read_into(&object.sha256, &mut bytes)?;
+    let mut bytes = vec![0; object.length as usize];
+    std::io::Read::read_exact(&mut file, &mut bytes).map_err(|error| crate::err::Refusal {
+        code: Code::IO_FAILED,
+        detail: format!("read checkpoint document: {error}"),
+    })?;
     if ObjectRef::of(&bytes) != *object {
         return refuse(
             Code::OBJECT_ID_MISMATCH,
-            "checkpoint link differs from its ObjectRef",
+            "checkpoint document differs from its ObjectRef",
         );
     }
-    Link::parse(&bytes)
+    Ok(bytes)
 }
 
 /// Each call reads one link and returns a bounded window, so a caller can fetch links and
@@ -1030,6 +1043,40 @@ pub fn restore_conversion(
 mod tests {
     use super::*;
 
+    #[test]
+    fn checkpoint_documents_refuse_wrong_lengths_before_reading_into_memory() {
+        let root = std::env::temp_dir().join(format!(
+            "tensorfs-document-bound-{}",
+            crate::meta::now_nanos_unique()
+        ));
+        let store = Store::init(&root).unwrap();
+        let bytes = b"a verified resident object";
+        let object = store
+            .put_stream(
+                &mut bytes.as_slice(),
+                Some(&ObjectRef::of(bytes)),
+                &Default::default(),
+            )
+            .unwrap()
+            .obj;
+        let mut short = object.clone();
+        short.length = 1;
+        assert_eq!(
+            local_link(&store, &short).unwrap_err().code,
+            Code::LENGTH_MISMATCH
+        );
+        assert_eq!(
+            read_document(&store, &short, 1).unwrap_err().code,
+            Code::LENGTH_MISMATCH
+        );
+        assert_eq!(
+            read_document(&store, &object, 1).unwrap_err().code,
+            Code::COUNT_CAP
+        );
+        assert_eq!(read_document(&store, &object, bytes.len()).unwrap(), bytes);
+        std::fs::remove_dir_all(root).unwrap();
+    }
+
     fn object(byte: &str, length: u64) -> ObjectRef {
         ObjectRef {
             sha256: byte.repeat(32),

{tensorfs-0.3.8 → tensorfs-0.3.9}/crates/tensorfs-core/src/ingest/source.rs

@@ -1207,11 +1207,51 @@ fn prepare_model_source_with_registry(
             for source in &sources {
                 validate_prepared_result(store, source)?;
             }
+            // The final response can be lost after its catalog commit. Its existing local
+            // cursors must remain discoverable without converting or reading source bodies.
+            let mut checkpoints = Vec::new();
+            let dir = transaction::candidate_dir(store.root(), &request.operation_id);
+            for (requested, (plan_hex, _)) in request.profiles.iter().zip(&plans) {
+                let plan_digest = format!("sha256:{plan_hex}");
+                let chain = format!("{}/{}", request.operation_id, requested.slot);
+                let journal = super::journal::Journal::open(&dir, plan_hex)?;
+                let local = journal.local_head()?.ok_or_else(|| Refusal {
+                    code: Code::DURABILITY_UNPROVEN,
+                    detail: "completed source has no local checkpoint cursor".into(),
+                })?;
+                let supplied = request
+                    .checkpoints
+                    .iter()
+                    .find(|(slot, _)| slot == &requested.slot)
+                    .map(|(_, head)| head);
+                let head = durability::checkpoint_predecessor(
+                    store,
+                    &durability::Policy {
+                        chain: &chain,
+                        operation: &request.operation_id,
+                        plan: &plan_digest,
+                        interval: durability::INTERVAL_BYTES,
+                    },
+                    Some(&local),
+                    supplied,
+                )?
+                .expect("a local cursor was supplied");
+                let held = durability::local_chain(store, &head, &plan_digest)?;
+                checkpoints.push(SourceCheckpoint {
+                    slot: requested.slot.clone(),
+                    plan_digest,
+                    head: durability::Head {
+                        head: Some(head),
+                        links: held.links,
+                        bytes: held.bytes,
+                    },
+                });
+            }
             return Ok(PreparedModelSources {
                 replayed: true,
                 complete: true,
                 sources,
-                checkpoints: Vec::new(),
+                checkpoints,
                 spent: request
                     .roster
                     .iter()
@@ -1257,20 +1297,8 @@ fn prepare_model_source_with_registry(
             durability::retain_local_chain(store, head, &request.operation_id)?;
             if journal.is_empty() {
                 if let Some(progress) = held.progress.last() {
-                    if progress.length > crate::limits::DOC_MAX_BYTES as u64 {
-                        return refuse(
-                            Code::COUNT_CAP,
-                            "checkpoint progress exceeds the document byte cap",
-                        );
-                    }
-                    let mut bytes = Vec::with_capacity(progress.length as usize);
-                    store.read_into(&progress.sha256, &mut bytes)?;
-                    if ObjectRef::of(&bytes) != *progress {
-                        return refuse(
-                            Code::OBJECT_ID_MISMATCH,
-                            "checkpoint progress differs from its ObjectRef",
-                        );
-                    }
+                    let bytes =
+                        durability::read_document(store, progress, crate::limits::DOC_MAX_BYTES)?;
                     drop(journal);
                     super::journal::install(&dir, &plan_hex, &bytes)?;
                     journal = super::journal::Journal::open(&dir, &plan_hex)?;
@@ -1583,6 +1611,19 @@ mod tests {
                 .unwrap();
         assert!(replay.replayed);
         assert_eq!(replay.sources, first.sources);
+        assert_eq!(replay.checkpoints, first.checkpoints);
+        let mut bodies_gone = request.clone();
+        bodies_gone.landed.clear();
+        for row in &bodies_gone.roster {
+            fs::remove_file(&row.path).unwrap();
+        }
+        let recovered_heads =
+            prepare_model_source_with_registry(&store, &bodies_gone, "fixture:registry", &registry)
+                .unwrap();
+        assert_eq!(recovered_heads.checkpoints, first.checkpoints);
+        // Remaining arms need their original carriers again.
+        carrier(&request.roster[0].path, "weight", [1, 2, 3, 4]);
+        carrier(&request.roster[1].path, "weight", [5, 6, 7, 8]);
 
         let mut changed = request.clone();
         changed.roster[0].object_id = format!("sha256:{}", "33".repeat(32));
@@ -2239,6 +2280,10 @@ mod tests {
         )
         .unwrap();
         assert!(replay.replayed && replay.complete);
+        assert_eq!(
+            replay.checkpoints, resumed.checkpoints,
+            "a lost final response replays even with an old ACK"
+        );
         let mut unrelated = registry.clone();
         unrelated.entries[0]
             .provenance

{tensorfs-0.3.8 → tensorfs-0.3.9}/crates/tensorfs-py/src/store.rs

@@ -1437,6 +1437,24 @@ impl PyStore {
         Ok(out)
     }
 
+    /// Canonical TensorFS declaration bytes for the current producer intent.
+    #[allow(clippy::too_many_arguments)]
+    #[pyo3(signature = (sources, targets, configs, order, max_new_bytes))]
+    fn derived_declaration<'p>(
+        &self,
+        py: Python<'p>,
+        sources: &Bound<'_, PyDict>,
+        targets: &Bound<'_, PyDict>,
+        configs: &Bound<'_, PyDict>,
+        order: Vec<(String, String)>,
+        max_new_bytes: u64,
+    ) -> PyResult<Bound<'p, PyBytes>> {
+        let mut declaration =
+            parse_declaration(py, sources, targets, configs, order, max_new_bytes)?;
+        let bytes = declaration.canonical_bytes().or_refuse(py)?;
+        Ok(PyBytes::new(py, &bytes))
+    }
+
     /// Open one exact tfs-019 transaction. Python supplies typed plain values; canonical
     /// declaration bytes are constructed and locked only by the compiled core.
     // PyO3 exposes these as six named boundary fields. Packing them into an opaque dict

{tensorfs-0.3.8 → tensorfs-0.3.9}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "maturin"
 
 [project]
 name = "tensorfs"
-version = "0.3.8"
+version = "0.3.9"
 description = "TensorFS — canonical tensor artifact substrate"
 requires-python = ">=3.12"
 classifiers = [

{tensorfs-0.3.8 → tensorfs-0.3.9}/python/tensorfs/_ext.pyi

@@ -481,6 +481,10 @@ class Store:
         components: Sequence[str],
         configs: Sequence[str],
     ) -> dict[str, Any]: ...
+    def derived_declaration(
+        self, sources: dict[str, Any], targets: dict[str, Any], configs: dict[str, Any],
+        order: Sequence[tuple[str, str]], max_new_bytes: int,
+    ) -> bytes: ...
     def begin_derived(
         self,
         transaction_id: str,