tensorfs 0.3.70__tar.gz → 0.3.73__tar.gz

{tensorfs-0.3.70 → tensorfs-0.3.73}/Cargo.lock

@@ -439,7 +439,7 @@ checksum = "adb6935a6f5c20170eeceb1a3835a49e12e19d792f6dd344ccc76a985ca5a6ca"
 
 [[package]]
 name = "tensorfs-core"
-version = "0.3.70"
+version = "0.3.73"
 dependencies = [
  "fs2",
  "libc",
@@ -454,7 +454,7 @@ dependencies = [
 
 [[package]]
 name = "tensorfs-py"
-version = "0.3.70"
+version = "0.3.73"
 dependencies = [
  "pyo3",
  "tensorfs-core",

{tensorfs-0.3.70 → tensorfs-0.3.73}/Cargo.toml

@@ -3,5 +3,5 @@ resolver = "2"
 members = ["crates/tensorfs-core", "crates/tensorfs-py"]
 
 [workspace.package]
-version = "0.3.70"
+version = "0.3.73"
 edition = "2021"

{tensorfs-0.3.70 → tensorfs-0.3.73}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: tensorfs
-Version: 0.3.70
+Version: 0.3.73
 Classifier: Programming Language :: Python :: 3
 Classifier: Programming Language :: Python :: Implementation :: CPython
 Classifier: Programming Language :: Rust

{tensorfs-0.3.70 → tensorfs-0.3.73}/crates/tensorfs-core/src/bin/tfs/ingest.rs

@@ -445,7 +445,7 @@ pub fn cmd_bank(dialect: &str, converter: &str, f: &Flags) -> ExitCode {
     let mut reviewed = Vec::new();
     for (component, path) in &srcs {
         let (full, raw) = ok!(source_core::read_carrier_at(path));
-        let h = if conv.name == "single_file.identity/1" {
+        let h = if conv.dialect == "safetensors.single_file" {
             ok!(convert::single_file_component(component, &full))
         } else {
             full
@@ -584,11 +584,11 @@ pub fn cmd_bank(dialect: &str, converter: &str, f: &Flags) -> ExitCode {
             .collect::<Vec<_>>()
             .join(",");
         let variant = flag(f, "spec-variant").and_then(|value| value.parse().ok());
-        let mut prepared = match source_core::prepare_with_registry(
+        let mut prepared = match source_core::prepare(
             &target,
             &unnamed(&srcs),
             &CarrierSet::default(),
-            &reg,
+            Some(&reg),
             variant,
         ) {
             Ok(prepared) => prepared,
@@ -849,17 +849,22 @@ fn prepare_closed_source_plan(
     if closed.registry != BUILTIN_REGISTRY {
         flags.push(("registry".to_string(), closed.registry.clone()));
     }
+    let as_is = closed.converter == convert::IDENTITY;
+    if as_is {
+        flags.push(("source-profile".to_string(), source_core::AS_IS.to_string()));
+    }
     for (component, source) in &closed.sources {
         flags.push((
             "source".to_string(),
             format!("{component}={}", source.path.to_string_lossy()),
         ));
     }
-    let mut prepared = source_core::prepare_with_registry(
+    // The plan was re-derived above, so its converter is the planner's: as-is has no registry.
+    let mut prepared = source_core::prepare(
         &closed.target,
         &closed.sources,
         &closed.set,
-        &registry,
+        (!as_is).then_some(&registry),
         None,
     )
     .map_err(bail)?;
@@ -989,15 +994,17 @@ pub(crate) fn construction_order(f: &Flags) -> Result<Vec<(String, String)>, Exi
 
 /// Headers in, plan out. The one function `plan`, `run` and the worker all go through, so
 /// a preview and a run cannot disagree about what was going to happen.
+/// `--source-profile as-is/1` stores the sources as they are, unclassified.
 fn prepare(f: &Flags, target: &str) -> Result<Prepared, ExitCode> {
     let sources = sources(f)?;
     let registry = load_registry(f)?;
     let variant = flag(f, "spec-variant").and_then(|value| value.parse().ok());
-    source_core::prepare_with_registry(
+    let as_is = flag(f, "source-profile") == Some(source_core::AS_IS);
+    source_core::prepare(
         target,
         &unnamed(&sources),
         &CarrierSet::default(),
-        &registry,
+        (!as_is).then_some(&registry),
         variant,
     )
     .map_err(bail)
@@ -1206,6 +1213,7 @@ pub fn cmd_run(root: &Path, target: &str, f: &Flags) -> ExitCode {
                 | "tenant"
                 | "isolation-tier"
                 | "checkpoint-bytes"
+                | "source-profile"
         ) {
             argv.push(format!("--{k}"));
             argv.push(v.clone());

{tensorfs-0.3.70 → tensorfs-0.3.73}/crates/tensorfs-core/src/bin/tfs/main.rs

@@ -22,7 +22,6 @@
 //!   tfs store prepare-readers [<root>]
 //!   tfs store complete-cozytensors [<root>]
 //!   tfs home                     print TENSORFS_HOME or ~/.tensorfs
-//!   tfs home migrate <old-root> [--to <new-root>]
 //!   tfs store rebuild <census> --rows <jsonl> [--sqlite <tensorfs.sqlite>]
 //!   tfs store admit-file <store> blob|manifest <id> <length> <source>
 //!   tfs repo-cache admit <cache> <store> blob|manifest <id> <length>
@@ -208,7 +207,6 @@ fn main() -> ExitCode {
     match a.as_slice() {
         ["version"] => cmd_version(),
         ["home"] => cmd_home(None),
-        ["home", "migrate", source] => cmd_home_migrate(Path::new(source), &flags),
         ["home", r] => cmd_home(Some(Path::new(r))),
         ["store", "init"] => cmd_store_default("init", &flags),
         ["store", "info"] => cmd_store_default("info", &flags),
@@ -817,17 +815,6 @@ fn cmd_home(explicit: Option<&Path>) -> ExitCode {
     ExitCode::SUCCESS
 }
 
-fn cmd_home_migrate(source: &Path, flags: &Flags) -> ExitCode {
-    let destination = flag(flags, "to").map(Path::new);
-    let report = ok!(tensorfs_core::home::move_store(source, destination));
-    println!(
-        "moved {} -> {} (device/inode/mtime preserved by same-filesystem rename)",
-        report.source.display(),
-        report.destination.display(),
-    );
-    ExitCode::SUCCESS
-}
-
 /// The no-root spelling of a `store` verb. It uses `TENSORFS_HOME`, or an existing
 /// `$HOME/.tensorfs` — never a Store it would have to invent (tfs-053).
 fn cmd_store_default(verb: &str, flags: &Flags) -> ExitCode {

{tensorfs-0.3.70 → tensorfs-0.3.73}/crates/tensorfs-core/src/derived/companion_tests.rs

@@ -196,6 +196,50 @@ fn ordinary_companions_survive_derivation_without_copying_tensor_backing_files()
     std::fs::remove_dir_all(store.root()).unwrap();
 }
 
+#[test]
+fn a_verified_store_object_is_a_companion_by_reference_and_never_a_new_byte() {
+    let (store, meta, id) = fixture();
+    let vocab = vec![0x7b; 256 << 10]; // over the inline bound: references have none
+    let object = store
+        .put_stream(
+            &mut vocab.as_slice(),
+            Some(&ObjectRef::of(&vocab)),
+            &Fault::default(),
+        )
+        .unwrap()
+        .obj;
+    let mut d = super::tests::created_declaration();
+    d.objects = vec![("tokenizer/vocab.json".into(), object.clone())];
+    let encoded = d.canonical_work_bytes().unwrap();
+    assert_eq!(
+        Declaration::parse_text(std::str::from_utf8(&encoded).unwrap()).unwrap(),
+        d
+    );
+    let writer = begin(&store, &meta, &id, 1, d.clone(), None).unwrap();
+    write_weight(&store, &meta, &id);
+    let receipt = commit(&store, &meta, &id, 1).unwrap();
+    release(&meta, writer);
+    assert!(!receipt.added_objects.contains(&object));
+    let header = checkpoint::load_header(&store, &receipt.header).unwrap();
+    let (name, asset) = &header.assets[0];
+    assert_eq!(
+        (name.as_str(), &asset.segments),
+        ("tokenizer/vocab.json", &vec![object])
+    );
+    let mut read = Vec::new();
+    checkpoint::read_asset(&store, name, asset, &mut read).unwrap();
+    assert_eq!(read, vocab);
+
+    // An object this Store does not hold is refused at commit, never published.
+    let next = format!("sha256:{}", "c3".repeat(32));
+    d.objects = vec![("tokenizer/vocab.json".into(), ObjectRef::of(b"absent"))];
+    let writer = begin(&store, &meta, &next, 1, d, None).unwrap();
+    write_weight(&store, &meta, &next);
+    assert!(commit(&store, &meta, &next, 1).is_err());
+    release(&meta, writer);
+    std::fs::remove_dir_all(store.root()).unwrap();
+}
+
 #[test]
 fn companion_paths_size_count_and_quotas_are_native_refusals() {
     for path in [

tensorfs-0.3.73/crates/tensorfs-core/src/derived/history_cost_tests.rs

@@ -0,0 +1,149 @@
+//! A derivation costs the same however many derivations the Store already retains (finding
+//! 50): proving its source rooted walked every retained checkpoint chain, so each lane on a
+//! conversion pod took longer than the one before.
+
+use std::time::{Duration, Instant};
+
+use super::tests::created_declaration;
+use super::*;
+use crate::repository::{Mutation, RepositoryName};
+use crate::store::Fault;
+
+const PARTS: usize = 32;
+const CONVERSIONS: usize = 60;
+const WINDOW: usize = 10;
+
+/// A source-inheriting derivation that adds PARTS tensors and checkpoints after each, as a
+/// quantization lane does: its source inspection, writer, chain and pending root.
+fn convert(store: &Store, meta: &Meta, index: usize, source: &ObjectRef) -> Duration {
+    let started = Instant::now();
+    inspect_source(
+        store,
+        meta,
+        source.clone(),
+        vec!["model".into()],
+        Vec::new(),
+    )
+    .unwrap();
+    let prototype = created_declaration().components.remove(0).add.remove(0);
+    let add: Vec<_> = (0..PARTS)
+        .map(|part| TensorDeclaration {
+            key: format!("added_{part}"),
+            ..prototype.clone()
+        })
+        .collect();
+    let mut order = vec![("model".to_string(), "weight".to_string())];
+    order.extend(
+        add.iter()
+            .map(|tensor| ("model".to_string(), tensor.key.clone())),
+    );
+    let declaration = Declaration {
+        sources: vec![Source {
+            alias: "source".into(),
+            manifest: source.clone(),
+        }],
+        components: vec![ComponentDeclaration {
+            target: "model".into(),
+            source: Some("source".into()),
+            source_component: Some("model".into()),
+            drop: Vec::new(),
+            add,
+        }],
+        order,
+        max_new_bytes: (PARTS * 2048) as u64,
+        ..created_declaration()
+    };
+    let transaction = format!("sha256:{:064x}", index + 1);
+    let writer = begin(store, meta, &transaction, 1, declaration, None).unwrap();
+    let (mut cursor, mut head) = (None, None);
+    for part in 0..PARTS {
+        add_indexed_part(
+            store,
+            meta,
+            &transaction,
+            1,
+            &writer.additions,
+            cursor.as_mut(),
+            ("model", &format!("added_{part}"), "value"),
+            &mut [(index * PARTS + part) as u8; 2048].as_slice(),
+        )
+        .unwrap();
+        let (_, next) = checkpoint_indexed(
+            store,
+            meta,
+            &transaction,
+            1,
+            "lane",
+            "model",
+            head.as_ref(),
+            &mut cursor,
+        )
+        .unwrap();
+        head = next.head;
+    }
+    commit(store, meta, &transaction, 1).unwrap();
+    release_guards(meta, Some(writer.writer_hold), writer.source_leases);
+    started.elapsed()
+}
+
+#[test]
+fn a_conversion_costs_the_same_after_many_retained_conversions() {
+    let scratch = std::path::Path::new("/dev/shm");
+    let root = if scratch.is_dir() {
+        scratch.to_path_buf()
+    } else {
+        std::env::temp_dir()
+    }
+    .join(format!(
+        "tensorfs-history-cost-{}-{}",
+        std::process::id(),
+        crate::meta::now_nanos_unique()
+    ));
+    let store = Store::init(&root).unwrap();
+    let meta = Meta::open(&store).unwrap();
+
+    // The source is a model checkpoint, rooted by its repository as a pulled model is.
+    let source_id = format!("sha256:{}", "5a".repeat(32));
+    let writer = begin(&store, &meta, &source_id, 1, created_declaration(), None).unwrap();
+    add_part(
+        &store,
+        &meta,
+        &source_id,
+        1,
+        ("model", "weight", "value"),
+        &mut [0x21; 2048].as_slice(),
+    )
+    .unwrap();
+    let source = commit(&store, &meta, &source_id, 1).unwrap().manifest;
+    release_guards(&meta, Some(writer.writer_hold), writer.source_leases);
+    store
+        .apply_repository(
+            None,
+            &Mutation::PutCheckpoint {
+                repo: RepositoryName::new("org", "source").unwrap(),
+                manifest: source.clone(),
+            },
+            &Fault::default(),
+        )
+        .unwrap();
+
+    let costs: Vec<Duration> = (0..CONVERSIONS)
+        .map(|index| convert(&store, &meta, index, &source))
+        .collect();
+    let median = |window: &[Duration]| {
+        let mut sorted = window.to_vec();
+        sorted.sort();
+        sorted[sorted.len() / 2]
+    };
+    let (first, last) = (
+        median(&costs[..WINDOW]),
+        median(&costs[CONVERSIONS - WINDOW..]),
+    );
+    let _ = std::fs::remove_dir_all(&root);
+    // Retained history grew six-fold between the windows; the cost of one conversion may
+    // not follow it. Before the fix the last window cost several times the first.
+    assert!(
+        last.as_secs_f64() < 2.0 * first.as_secs_f64(),
+        "conversion cost grew with retained history: first {first:?}, last {last:?}"
+    );
+}

{tensorfs-0.3.70 → tensorfs-0.3.73}/crates/tensorfs-core/src/derived/part_rows_tests.rs

@@ -333,40 +333,23 @@ fn a_digest_receipt_names_the_declaration_the_intent_bound() {
     for (index, key) in ["weight_0", "weight_1", "weight_2"].into_iter().enumerate() {
         add(&store, &meta, &id, &writer, key, index as u8 + 1).unwrap();
     }
-    let receipt = commit(&store, &meta, &id, 1).unwrap();
-    let inline = receipt.value(ReceiptDeclaration::Inline);
-    let digest = receipt.value(ReceiptDeclaration::Digest);
-    assert_eq!(inline, receipt.to_value());
-    let field = |value: &Value, name: &str| {
-        as_obj("receipt", "facts", value)
+    let receipt = commit(&store, &meta, &id, 1).unwrap().to_value();
+    let field = |name: &str| {
+        as_obj("receipt", "facts", &receipt)
             .unwrap()
             .iter()
             .find(|(key, _)| key == name)
             .map(|(_, value)| value.clone())
     };
+    assert!(field("declaration").is_none());
     assert_eq!(
-        crate::canon::write(&field(&inline, "declaration").unwrap()),
-        intent
-    );
-    assert!(field(&digest, "declaration").is_none());
-    assert_eq!(
-        field(&digest, "declaration_digest"),
+        field("declaration_digest"),
         Some(Value::str(ObjectRef::of(&intent).id()))
     );
     assert_eq!(
-        field(&digest, "declaration_length"),
+        field("declaration_length"),
         Some(Value::uint(intent.len() as u64))
     );
-    // Every other fact is identical in both forms.
-    let others = |value: &Value| {
-        as_obj("receipt", "facts", value)
-            .unwrap()
-            .iter()
-            .filter(|(key, _)| !key.starts_with("declaration"))
-            .cloned()
-            .collect::<Vec<_>>()
-    };
-    assert_eq!(others(&inline), others(&digest));
     let _ = fence(&meta, &id, 1);
     release_guards(&meta, Some(writer.writer_hold), writer.source_leases);
     std::fs::remove_dir_all(store.root()).unwrap();

{tensorfs-0.3.70 → tensorfs-0.3.73}/crates/tensorfs-core/src/derived/retention.rs

@@ -5,8 +5,7 @@ use std::io::{Read, Write};
 use std::path::PathBuf;
 
 use super::{
-    checkpoint, receipt_from, require_payload_record, roots, Disposition, ReceiptDeclaration,
-    TransactionState,
+    checkpoint, receipt_from, require_payload_record, roots, Disposition, TransactionState,
 };
 use crate::canon::{Fields, Value};
 use crate::catalog::WriterGuard;
@@ -218,12 +217,7 @@ pub fn retain_result(
         );
     }
     let facts = receipt_from(store, row)?;
-    // A receipt's identity is the digest of whichever form its caller was handed.
-    let forms = [ReceiptDeclaration::Inline, ReceiptDeclaration::Digest];
-    if !forms
-        .iter()
-        .any(|form| object_id(&crate::canon::write(&facts.value(*form))) == receipt)
-    {
+    if object_id(&crate::canon::write(&facts.to_value())) != receipt {
         return refuse(
             Code::OBJECT_ID_MISMATCH,
             "retained result receipt differs from committed computation",
@@ -376,20 +370,9 @@ mod tests {
     }
 
     #[test]
-    fn a_result_is_retained_under_either_receipt_form_and_nothing_else() {
+    fn a_result_is_retained_under_its_receipt_and_nothing_else() {
         let f = fixture();
-        let facts = derived::lookup(&f.store, &f.meta, &f.transaction).unwrap();
-        let derived::Lookup::Committed(committed) = facts else {
-            panic!("committed result expected")
-        };
-        let digest_form = object_id(&crate::canon::write(
-            &committed.receipt().value(ReceiptDeclaration::Digest),
-        ));
-        assert_ne!(digest_form, f.receipt);
-        let retained =
-            retain_result(&f.store, &f.meta, &f.transaction, &digest_form, &id("b1")).unwrap();
-        assert_eq!(retained.receipt_digest, digest_form);
-        retain_result(&f.store, &f.meta, &f.transaction, &f.receipt, &id("b2")).unwrap();
+        retain_result(&f.store, &f.meta, &f.transaction, &f.receipt, &id("b1")).unwrap();
         assert_eq!(
             retain_result(&f.store, &f.meta, &f.transaction, &id("99"), &id("b3"))
                 .unwrap_err()