tensorfs 0.3.38__tar.gz → 0.3.40__tar.gz

{tensorfs-0.3.38 → tensorfs-0.3.40}/Cargo.lock

@@ -421,7 +421,7 @@ checksum = "adb6935a6f5c20170eeceb1a3835a49e12e19d792f6dd344ccc76a985ca5a6ca"
 
 [[package]]
 name = "tensorfs-core"
-version = "0.3.38"
+version = "0.3.40"
 dependencies = [
  "fs2",
  "minicbor",
@@ -434,7 +434,7 @@ dependencies = [
 
 [[package]]
 name = "tensorfs-py"
-version = "0.3.38"
+version = "0.3.40"
 dependencies = [
  "pyo3",
  "tensorfs-core",

{tensorfs-0.3.38 → tensorfs-0.3.40}/Cargo.toml

@@ -3,5 +3,5 @@ resolver = "2"
 members = ["crates/tensorfs-core", "crates/tensorfs-py"]
 
 [workspace.package]
-version = "0.3.38"
+version = "0.3.40"
 edition = "2021"

{tensorfs-0.3.38 → tensorfs-0.3.40}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: tensorfs
-Version: 0.3.38
+Version: 0.3.40
 Classifier: Programming Language :: Python :: 3
 Classifier: Programming Language :: Python :: Implementation :: CPython
 Classifier: Programming Language :: Rust

tensorfs-0.3.40/crates/tensorfs-core/src/derived/estimate.rs

@@ -0,0 +1,303 @@
+//! Remaining added-role geometry without opening or fencing a native writer.
+
+use super::*;
+
+#[derive(Debug, Default, PartialEq, Eq)]
+pub struct WriteEstimate {
+    pub payload_bytes: u64,
+    pub largest_part_bytes: u64,
+    pub parts: u64,
+}
+
+/// Inspect exactly the progress that begin would adopt. The observation is not a
+/// reservation or receipt: begin revalidates it, and per-role admission still
+/// guards every actual write. All inspection holds are released before return.
+pub fn estimate_write(
+    store: &Store,
+    meta: &Meta,
+    transaction: &str,
+    writer_session: u64,
+    mut declaration: Declaration,
+    checkpoint: Option<&ObjectRef>,
+) -> Result<WriteEstimate> {
+    let transaction = transaction_id(transaction)?;
+    if writer_session == 0 || writer_session > limits::INT_MAX as u64 {
+        return refuse(
+            Code::NUMBER_RANGE,
+            "WriterSessionId must be positive and inside the canonical integer range",
+        );
+    }
+    let text =
+        String::from_utf8(declaration.canonical_work_bytes()?).expect("canonical JSON is UTF-8");
+    let hold = meta.acquire_hold("derived-write-estimate")?;
+    let estimate = (|| {
+        require_source_roots(store, &declaration)?;
+        let sources = load_sources(store, &declaration)?;
+        let facts: Vec<_> = sources.iter().map(|source| source.fact.clone()).collect();
+        let incoming = checkpoint
+            .map(|head| read_progress(store, head, &transaction, &text, &facts))
+            .transpose()?;
+        let initial = meta
+            .derived_rows()?
+            .into_iter()
+            .find(|row| row.id == transaction);
+        let mut row = planned_row(
+            initial.as_ref(),
+            &transaction,
+            writer_session,
+            &text,
+            &facts,
+        )?;
+        merge_progress(store, &mut row, incoming.as_ref())?;
+        build_header(
+            &declaration,
+            &sources,
+            &row.added_parts,
+            &row.added_configs,
+            true,
+        )?;
+        if roots::read(store, &transaction)?
+            .and_then(|root| root.restore_epoch)
+            .is_some_and(|epoch| epoch > writer_session)
+        {
+            return refuse(
+                Code::WRITER_FENCED,
+                "writer is behind the retained restore epoch",
+            );
+        }
+        let completed: HashSet<_> = row
+            .added_parts
+            .iter()
+            .map(|part| {
+                (
+                    part.component.as_str(),
+                    part.key.as_str(),
+                    part.role.as_str(),
+                )
+            })
+            .collect();
+        let mut result = WriteEstimate::default();
+        for component in &declaration.components {
+            for tensor in &component.add {
+                for part in &tensor.parts {
+                    if part.source.is_some()
+                        || completed.contains(&(
+                            component.target.as_str(),
+                            tensor.key.as_str(),
+                            part.role.as_str(),
+                        ))
+                    {
+                        continue;
+                    }
+                    let length = checked_bytes("remaining derived part", &part.shape, part.dtype)?;
+                    result.payload_bytes = result
+                        .payload_bytes
+                        .checked_add(length)
+                        .ok_or_else(|| arithmetic("remaining derived payload overflows"))?;
+                    result.largest_part_bytes = result.largest_part_bytes.max(length);
+                    result.parts += 1;
+                }
+            }
+        }
+        Ok(result)
+    })();
+    let released = hold.release(meta);
+    estimate.and_then(|value| released.map(|()| value))
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    fn fixture(label: &str) -> (Store, Meta, String, Declaration) {
+        let path = std::env::temp_dir().join(format!(
+            "tensorfs-derived-estimate-{label}-{}",
+            crate::meta::now_nanos_unique()
+        ));
+        let store = Store::init(&path).unwrap();
+        let meta = Meta::open(&store).unwrap();
+        (
+            store,
+            meta,
+            format!("sha256:{}", "78".repeat(32)),
+            super::super::tests::created_declaration(),
+        )
+    }
+
+    fn release(meta: &Meta, begun: Begin) {
+        release_guards(meta, Some(begun.writer_hold), begun.source_leases);
+    }
+
+    #[test]
+    fn fresh_estimate_does_not_claim_writer_or_block_collection() {
+        let (store, meta, id, declaration) = fixture("fresh");
+        let before = meta.derived_rows().unwrap();
+        assert_eq!(
+            estimate_write(&store, &meta, &id, 1, declaration.clone(), None).unwrap(),
+            WriteEstimate {
+                payload_bytes: 2048,
+                largest_part_bytes: 2048,
+                parts: 1
+            }
+        );
+        assert_eq!(meta.derived_rows().unwrap(), before);
+        assert!(matches!(
+            lookup(&store, &meta, &id).unwrap(),
+            Lookup::Absent
+        ));
+        crate::gc::collect(store.root(), false).unwrap();
+        let writer = begin(&store, &meta, &id, 1, declaration, None).unwrap();
+        fence(&meta, &id, 1).unwrap();
+        release(&meta, writer);
+        std::fs::remove_dir_all(store.root()).unwrap();
+    }
+
+    #[test]
+    fn interrupted_large_writer_only_charges_its_small_remaining_tail() {
+        let (store, meta, id, mut declaration) = fixture("large-partial");
+        const LARGE: u64 = 64 << 20;
+        let mut tail = declaration.components[0].add[0].clone();
+        tail.key = "tail".into();
+        tail.shape = vec![1];
+        tail.parts[0].shape = vec![1];
+        declaration.components[0].add[0].shape = vec![LARGE / 4];
+        declaration.components[0].add[0].parts[0].shape = vec![LARGE / 4];
+        declaration.components[0].add.push(tail);
+        declaration.order.push(("model".into(), "tail".into()));
+        declaration.max_new_bytes = LARGE + 4;
+        let writer = begin(&store, &meta, &id, 1, declaration.clone(), None).unwrap();
+        let written = add_part(
+            &store,
+            &meta,
+            &id,
+            1,
+            ("model", "weight", "value"),
+            &mut std::io::repeat(0x31).take(LARGE),
+        )
+        .unwrap();
+        let (_, checkpoint) =
+            checkpoint_progress(&store, &meta, &id, 1, "request", "model", None).unwrap();
+        assert!(checkpoint.head.is_some());
+        fence(&meta, &id, 1).unwrap();
+        release(&meta, writer);
+        let before = meta.derived_rows().unwrap();
+        assert_eq!(
+            estimate_write(
+                &store,
+                &meta,
+                &id,
+                1,
+                declaration.clone(),
+                checkpoint.head.as_ref()
+            )
+            .unwrap_err()
+            .code,
+            Code::WRITER_FENCED
+        );
+        assert_eq!(
+            estimate_write(
+                &store,
+                &meta,
+                &id,
+                2,
+                declaration.clone(),
+                checkpoint.head.as_ref()
+            )
+            .unwrap(),
+            WriteEstimate {
+                payload_bytes: 4,
+                largest_part_bytes: 4,
+                parts: 1
+            }
+        );
+        assert_eq!(meta.derived_rows().unwrap(), before);
+        crate::gc::collect(store.root(), false).unwrap();
+        assert!(written
+            .part
+            .segments()
+            .iter()
+            .all(|part| store.contains(&part.sha256)));
+        let resumed = begin(&store, &meta, &id, 2, declaration, checkpoint.head.as_ref()).unwrap();
+        assert_eq!(
+            completed(&meta, &id, 2).unwrap().0,
+            vec![("model".into(), "weight".into(), "value".into())]
+        );
+        add_part(
+            &store,
+            &meta,
+            &id,
+            2,
+            ("model", "tail", "value"),
+            &mut [0u8; 4].as_slice(),
+        )
+        .unwrap();
+        commit(&store, &meta, &id, 2).unwrap();
+        release(&meta, resumed);
+        std::fs::remove_dir_all(store.root()).unwrap();
+    }
+
+    #[test]
+    fn grafts_charge_no_new_payload_and_preserve_source_custody() {
+        let (store, meta, id, mut declaration) = fixture("graft");
+        let writer = begin(&store, &meta, &id, 1, declaration.clone(), None).unwrap();
+        add_part(
+            &store,
+            &meta,
+            &id,
+            1,
+            ("model", "weight", "value"),
+            &mut [0x31u8; 2048].as_slice(),
+        )
+        .unwrap();
+        let original = commit(&store, &meta, &id, 1).unwrap();
+        release(&meta, writer);
+        declaration.sources.push(Source {
+            alias: "original".into(),
+            manifest: original.manifest.clone(),
+        });
+        declaration.components[0].add[0].parts[0].source = Some(PartSource {
+            source: "original".into(),
+            component: "model".into(),
+            tensor: "weight".into(),
+            role: "value".into(),
+        });
+        declaration.max_new_bytes = 0;
+        let target = format!("sha256:{}", "79".repeat(32));
+        let before = meta.derived_rows().unwrap();
+        assert_eq!(
+            estimate_write(&store, &meta, &target, 1, declaration.clone(), None).unwrap(),
+            WriteEstimate::default()
+        );
+        assert_eq!(meta.derived_rows().unwrap(), before);
+        crate::gc::collect(store.root(), false).unwrap();
+        let target_writer = begin(&store, &meta, &target, 1, declaration, None).unwrap();
+        assert_eq!(
+            commit(&store, &meta, &target, 1).unwrap().manifest,
+            original.manifest
+        );
+        release(&meta, target_writer);
+        std::fs::remove_dir_all(store.root()).unwrap();
+    }
+
+    #[test]
+    fn changed_work_and_missing_checkpoint_refuse_without_mutating_progress() {
+        let (store, meta, id, declaration) = fixture("refuse");
+        let writer = begin(&store, &meta, &id, 1, declaration.clone(), None).unwrap();
+        fence(&meta, &id, 1).unwrap();
+        release(&meta, writer);
+        let before = meta.derived_rows().unwrap();
+        let mut changed = declaration.clone();
+        changed.work_fingerprint = Some(format!("sha256:{}", "65".repeat(32)));
+        assert_eq!(
+            estimate_write(&store, &meta, &id, 2, changed, None)
+                .unwrap_err()
+                .code,
+            Code::TRANSACTION_CONFLICT
+        );
+        let absent = ObjectRef::of(b"not a checkpoint");
+        assert!(estimate_write(&store, &meta, &id, 2, declaration, Some(&absent)).is_err());
+        assert_eq!(meta.derived_rows().unwrap(), before);
+        crate::gc::collect(store.root(), false).unwrap();
+        std::fs::remove_dir_all(store.root()).unwrap();
+    }
+}

{tensorfs-0.3.38 → tensorfs-0.3.40}/crates/tensorfs-core/src/derived/resume_tests.rs

@@ -918,3 +918,153 @@ fn source_free_restore_holds_survive_gc_until_begin_or_explicit_abandon() {
     release(&source_meta, writer);
     std::fs::remove_dir_all(source.root()).unwrap();
 }
+
+/// Opt-in generated-data profile. Payloads are tiny so metadata scaling is visible.
+/// Run in release mode; elapsed time is evidence, never a flaky unit-test threshold.
+#[test]
+#[ignore]
+fn profile_derived_transaction_scaling() {
+    use std::time::Instant;
+    let scenarios = [
+        (219, 1, 1),
+        (219, 1, 10),
+        (219, 4, 10),
+        (219, 32, 10),
+        (219, 128, 10),
+        (3968, 1, 10),
+        (3968, 4, 10),
+        (219, 1, 219),
+    ];
+    for (keys, histories, writes) in scenarios {
+        let (store, meta, id, _) = fixture("row-profile");
+        let mut declaration = super::tests::created_declaration();
+        let prototype = declaration.components[0].add[0].clone();
+        declaration.components[0].add = (0..keys)
+            .map(|index| {
+                let mut tensor = prototype.clone();
+                tensor.key = format!("weight_{index:04}");
+                tensor.shape = vec![4];
+                tensor.parts[0].shape = vec![4];
+                tensor
+            })
+            .collect();
+        declaration.order = declaration.components[0]
+            .add
+            .iter()
+            .map(|tensor| ("model".into(), tensor.key.clone()))
+            .collect();
+        declaration.max_new_bytes = (keys * 16) as u64;
+        let writer = begin(&store, &meta, &id, 1, declaration, None).unwrap();
+        // Populate representative unrelated transactions without timing fixture creation.
+        let mut txn = meta.txn().unwrap();
+        let prototype = txn.rows[0].clone();
+        let row_bytes = crate::canon::write(&prototype.to_value()).len();
+        for index in 1..histories {
+            let mut row = prototype.clone();
+            row.id = ObjectRef::of(format!("history-{index}").as_bytes()).id();
+            txn.rows.push(row);
+        }
+        txn.commit().unwrap();
+        let started = Instant::now();
+        for _ in 0..20 {
+            check_writer(&meta, &id, 1).unwrap();
+        }
+        let fence_ms = started.elapsed().as_secs_f64() * 1000.0;
+        let mut part_ms = 0.0;
+        let mut checkpoint_ms = 0.0;
+        let mut prior = None;
+        for index in 0..writes {
+            let key = format!("weight_{index:04}");
+            let started = Instant::now();
+            add_part(
+                &store,
+                &meta,
+                &id,
+                1,
+                ("model", &key, "value"),
+                &mut [index as u8; 16].as_slice(),
+            )
+            .unwrap();
+            part_ms += started.elapsed().as_secs_f64() * 1000.0;
+            let started = Instant::now();
+            let (_, head) =
+                checkpoint_progress(&store, &meta, &id, 1, "profile", "model", prior.as_ref())
+                    .unwrap();
+            prior = head.head;
+            checkpoint_ms += started.elapsed().as_secs_f64() * 1000.0;
+        }
+        eprintln!("PROFILE {{\"keys\":{keys},\"histories\":{histories},\"writes\":{writes},\"row_bytes\":{row_bytes},\"fence20_ms\":{fence_ms:.3},\"add_ms\":{part_ms:.3},\"checkpoint_ms\":{checkpoint_ms:.3}}}");
+        fence(&meta, &id, 1).unwrap();
+        release(&meta, writer);
+        std::fs::remove_dir_all(store.root()).unwrap();
+    }
+}
+
+#[test]
+fn named_writer_operations_leave_other_rows_and_writer_lock_alone() {
+    use rusqlite::{params, Connection};
+    let (store, meta, first_id, declaration) = fixture("scoped-row");
+    let second_id = ObjectRef::of(b"independent writer").id();
+    let first = begin(&store, &meta, &first_id, 1, declaration.clone(), None).unwrap();
+    let second = begin(&store, &meta, &second_id, 1, declaration.clone(), None).unwrap();
+    let connection = Connection::open(crate::catalog::Catalog::path(store.root())).unwrap();
+    let untouched: Vec<u8> = connection
+        .query_row(
+            "SELECT bytes FROM tensorfs_derived_transactions WHERE id=?1",
+            [&second_id],
+            |row| row.get(0),
+        )
+        .unwrap();
+    // Corrupt unrelated history makes an accidental whole-table read fail, while
+    // a scoped mutation must preserve it byte-for-byte and mutate only this writer.
+    let damaged = ObjectRef::of(b"unrelated damaged history").id();
+    connection
+        .execute(
+            "INSERT INTO tensorfs_derived_transactions(id,bytes) VALUES(?1,?2)",
+            params![damaged, b"broken".as_slice()],
+        )
+        .unwrap();
+    data(&store, &meta, &first_id, 1, 0x31);
+    let after: Vec<u8> = connection
+        .query_row(
+            "SELECT bytes FROM tensorfs_derived_transactions WHERE id=?1",
+            [&second_id],
+            |row| row.get(0),
+        )
+        .unwrap();
+    assert_eq!(untouched, after);
+    // A read fence must succeed while an unrelated SQLite writer is active. The old
+    // BEGIN IMMEDIATE observation instead hit the five-second busy timeout.
+    connection.execute_batch("BEGIN IMMEDIATE").unwrap();
+    check_writer(&meta, &second_id, 1).unwrap();
+    assert!(completed(&meta, &second_id, 1).unwrap().0.is_empty());
+    connection.execute_batch("ROLLBACK").unwrap();
+    // A named operation does not parse unrelated history. Global inventory still
+    // reports the damaged row rather than silently skipping it.
+    check_writer(&meta, &first_id, 1).unwrap();
+    assert!(meta.derived_rows().is_err());
+    connection
+        .execute(
+            "DELETE FROM tensorfs_derived_transactions WHERE id=?1",
+            [&damaged],
+        )
+        .unwrap();
+    assert!(fence(&meta, &first_id, 1).unwrap());
+    assert_eq!(
+        check_writer(&meta, &first_id, 1).unwrap_err().code,
+        Code::WRITER_FENCED
+    );
+    check_writer(&meta, &second_id, 1).unwrap();
+    release(&meta, first);
+    let resumed = begin(&store, &meta, &first_id, 2, declaration, None).unwrap();
+    assert_eq!(completed(&meta, &first_id, 2).unwrap().0.len(), 1);
+    assert_eq!(
+        check_writer(&meta, &first_id, 1).unwrap_err().code,
+        Code::WRITER_FENCED
+    );
+    fence(&meta, &first_id, 2).unwrap();
+    fence(&meta, &second_id, 1).unwrap();
+    release(&meta, resumed);
+    release(&meta, second);
+    std::fs::remove_dir_all(store.root()).unwrap();
+}