tensorfs 0.3.0__tar.gz → 0.3.1__tar.gz

{tensorfs-0.3.0 → tensorfs-0.3.1}/Cargo.lock

@@ -249,7 +249,7 @@ checksum = "adb6935a6f5c20170eeceb1a3835a49e12e19d792f6dd344ccc76a985ca5a6ca"
 
 [[package]]
 name = "tensorfs-core"
-version = "0.3.0"
+version = "0.3.1"
 dependencies = [
  "fs2",
  "minicbor",
@@ -258,7 +258,7 @@ dependencies = [
 
 [[package]]
 name = "tensorfs-py"
-version = "0.3.0"
+version = "0.3.1"
 dependencies = [
  "pyo3",
  "tensorfs-core",

{tensorfs-0.3.0 → tensorfs-0.3.1}/Cargo.toml

@@ -3,5 +3,5 @@ resolver = "2"
 members = ["crates/tensorfs-core", "crates/tensorfs-py"]
 
 [workspace.package]
-version = "0.3.0"
+version = "0.3.1"
 edition = "2021"

{tensorfs-0.3.0 → tensorfs-0.3.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: tensorfs
-Version: 0.3.0
+Version: 0.3.1
 Classifier: Programming Language :: Python :: 3
 Classifier: Programming Language :: Python :: Implementation :: CPython
 Classifier: Programming Language :: Rust

{tensorfs-0.3.0 → tensorfs-0.3.1}/crates/tensorfs-core/src/bin/tfs/ingest.rs

@@ -1282,7 +1282,7 @@ pub fn cmd_worker(root: &Path, target: &str, f: &Flags) -> ExitCode {
     }
     let session = session_of(f, &p);
     let tenant = flag(f, "tenant").unwrap_or("dev").to_string();
-    let mut store = ok!(Store::init(root));
+    let mut store = ok!(Store::open(root));
     if flag_on(f, "progress") {
         store = store.observed(Arc::new(Reporter::default()));
     }

{tensorfs-0.3.0 → tensorfs-0.3.1}/crates/tensorfs-core/src/bin/tfs/main.rs

@@ -1,5 +1,6 @@
 //! tfs — TensorFS store and CozyTensors inspection tool.
 //!
+//!   tfs version                  release version + sha256 of this binary, the skew observable
 //!   tfs registry                 print the seeded platform namespace (alias -> digest)
 //!   tfs id <file>                object id (+ derived ids) of a stored document
 //!   tfs cbor encode|decode|check <file>  header conformance seam; JSON is diagnostic only
@@ -10,7 +11,15 @@
 //! --size <bytes> --small <n>, plus the dev-only kill points --fault <stage> --ready <path>.
 //!
 //!   tfs store init|info <root>
+//!   tfs store ensure [<root>]
+//!   tfs store prepare-readers [<root>]
+//!   tfs store complete-cozytensors [<root>]
+//!   tfs home                     print TENSORFS_HOME or ~/.tensorfs
+//!   tfs home migrate <old-root> [--to <new-root>]
 //!   tfs store rebuild <census> --rows <jsonl> [--sqlite <tensorfs.sqlite>]
+//!   tfs store admit-file <store> blob|manifest <id> <length> <source>
+//!   tfs repo-cache admit <cache> <store> blob|manifest <id> <length>
+//!   tfs repo-cache backfill <cache> <store> blob|manifest <id> <length>
 //!   tfs put <root> <file>        stream one object in (any shape)
 //!   tfs get <root> <hex>         verified read, whole or ranged
 //!   tfs contains <root> <hex>    the presence HINT, and what it is worth
@@ -94,6 +103,7 @@ use tensorfs_core::ingest::stamp::Stamp;
 use tensorfs_core::ingest::{Evidence, IngestProfile, IngestSubject, IngestVerificationReceipt};
 use tensorfs_core::manifest::{Draft, Entry, Manifest};
 use tensorfs_core::registry;
+use tensorfs_core::repo_cache::{CacheKind, RepoObjectCache};
 use tensorfs_core::spec::EncodingSpec;
 use tensorfs_core::store::{Fault, Store, Verdict, FAULT_STAGES};
 
@@ -123,9 +133,30 @@ fn main() -> ExitCode {
     let args = accept_printed_ids(args);
     let a: Vec<&str> = args.iter().map(|s| s.as_str()).collect();
     match a.as_slice() {
+        ["version"] => cmd_version(),
+        ["home"] => cmd_home(None),
+        ["home", "migrate", source] => cmd_home_migrate(Path::new(source), &flags),
+        ["home", r] => cmd_home(Some(Path::new(r))),
+        ["store", "init"] => cmd_store_default("init"),
+        ["store", "info"] => cmd_store_default("info"),
+        ["store", "ensure"] => cmd_store_default("ensure"),
+        ["store", "prepare-readers"] => cmd_store_prepare_readers(None),
+        ["store", "complete-cozytensors"] => cmd_store_complete_cozytensors(None),
         ["store", "init", r] => cmd_store(Path::new(r), "init"),
         ["store", "info", r] => cmd_store(Path::new(r), "info"),
+        ["store", "ensure", r] => cmd_store(Path::new(r), "ensure"),
+        ["store", "prepare-readers", r] => cmd_store_prepare_readers(Some(Path::new(r))),
+        ["store", "complete-cozytensors", r] => cmd_store_complete_cozytensors(Some(Path::new(r))),
         ["store", "rebuild", r] => repo::cmd_rebuild(Path::new(r), &flags),
+        ["store", "admit-file", store, kind, object, length, source] => {
+            cmd_store_admit_file(Path::new(store), kind, object, length, Path::new(source))
+        }
+        ["repo-cache", "admit", cache, store, kind, object, length] => {
+            cmd_repo_cache_admit(Path::new(cache), Path::new(store), kind, object, length)
+        }
+        ["repo-cache", "backfill", cache, store, kind, object, length] => {
+            cmd_repo_cache_backfill(Path::new(cache), Path::new(store), kind, object, length)
+        }
         ["put", r, f] => cmd_put(Path::new(r), Path::new(f), &flags),
         ["get", r, h] => cmd_get(Path::new(r), h, &flags),
         ["contains", r, h] => cmd_contains(Path::new(r), h),
@@ -255,6 +286,157 @@ fn main() -> ExitCode {
     }
 }
 
+/// `tfs <version> sha256:<digest of this binary's own bytes>` — one line, machine-parsed by
+/// tensorhub and cozy-creator at startup (tfs-051). Byte-identical builds print the same
+/// digest; two builds stamped with one version cannot hide behind it.
+fn cmd_version() -> ExitCode {
+    let bytes = match std::env::current_exe().and_then(fs::read) {
+        Ok(bytes) => bytes,
+        Err(error) => {
+            eprintln!("REFUSED IO_FAILED: read own binary: {error}");
+            return ExitCode::from(1);
+        }
+    };
+    println!(
+        "tfs {} sha256:{}",
+        env!("CARGO_PKG_VERSION"),
+        tensorfs_core::sha256::hex_digest(&bytes)
+    );
+    ExitCode::SUCCESS
+}
+
+fn cache_args(kind: &str, object: &str, length: &str) -> TfsResult<(CacheKind, ObjectRef)> {
+    let kind = CacheKind::parse(kind).ok_or_else(|| Refusal {
+        code: tensorfs_core::err::Code::WRONG_TYPE,
+        detail: format!("repo cache kind {kind:?} is not blob or manifest"),
+    })?;
+    let sha256 = tensorfs_core::ids::hex64("repo cache object", object)?;
+    let length = length.parse::<u64>().map_err(|_| Refusal {
+        code: tensorfs_core::err::Code::NUMBER_RANGE,
+        detail: format!("repo cache length {length:?} is not an unsigned integer"),
+    })?;
+    Ok((kind, ObjectRef { sha256, length }))
+}
+
+fn cmd_repo_cache_admit(
+    cache: &Path,
+    store: &Path,
+    kind: &str,
+    object: &str,
+    length: &str,
+) -> ExitCode {
+    let (kind, object) = match cache_args(kind, object, length) {
+        Ok(value) => value,
+        Err(error) => return bail(error),
+    };
+    let store = match Store::open(store) {
+        Ok(value) => value,
+        Err(error) => return bail(error),
+    };
+    let status = match RepoObjectCache::new(cache).admit(&store, kind, &object) {
+        Ok(value) => value,
+        Err(error) => return bail(error),
+    };
+    println!("{{\"status\":\"{}\"}}", status.as_str());
+    ExitCode::SUCCESS
+}
+
+fn cmd_repo_cache_backfill(
+    cache: &Path,
+    store: &Path,
+    kind: &str,
+    object: &str,
+    length: &str,
+) -> ExitCode {
+    let (kind, object) = match cache_args(kind, object, length) {
+        Ok(value) => value,
+        Err(error) => return bail(error),
+    };
+    let store = match Store::open(store) {
+        Ok(value) => value,
+        Err(error) => return bail(error),
+    };
+    let status = match RepoObjectCache::new(cache).backfill_store(&store, kind, &object) {
+        Ok(value) => value,
+        Err(error) => return bail(error),
+    };
+    println!("{{\"status\":\"{}\"}}", status.as_str());
+    ExitCode::SUCCESS
+}
+
+fn cmd_store_admit_file(
+    store: &Path,
+    kind: &str,
+    object: &str,
+    length: &str,
+    source: &Path,
+) -> ExitCode {
+    let (kind, object) = match cache_args(kind, object, length) {
+        Ok(value) => value,
+        Err(error) => return bail(error),
+    };
+    let store = match Store::open(store) {
+        Ok(value) => value,
+        Err(error) => return bail(error),
+    };
+    let admitted = match kind {
+        CacheKind::Blob => match store.put_file(source, Some(&object), &Fault::default()) {
+            Ok(value) => value.admitted,
+            Err(error) => return bail(error),
+        },
+        CacheKind::Manifest => {
+            let metadata = match fs::metadata(source) {
+                Ok(metadata) => metadata,
+                Err(error) => {
+                    return bail(Refusal {
+                        code: tensorfs_core::err::Code::IO_FAILED,
+                        detail: format!("stat {}: {error}", source.display()),
+                    })
+                }
+            };
+            if metadata.len() != object.length || object.length > Manifest::MAX_BYTES as u64 {
+                return bail(Refusal {
+                    code: tensorfs_core::err::Code::LENGTH_MISMATCH,
+                    detail: format!(
+                        "{} is {} bytes, expected {}",
+                        source.display(),
+                        metadata.len(),
+                        object.length
+                    ),
+                });
+            }
+            let bytes = match fs::read(source) {
+                Ok(bytes) => bytes,
+                Err(error) => {
+                    return bail(Refusal {
+                        code: tensorfs_core::err::Code::IO_FAILED,
+                        detail: format!("read {}: {error}", source.display()),
+                    })
+                }
+            };
+            let manifest = match Manifest::parse(&bytes) {
+                Ok(value) => value,
+                Err(error) => return bail(error),
+            };
+            if ObjectRef::of(&bytes) != object {
+                return bail(Refusal {
+                    code: tensorfs_core::err::Code::OBJECT_ID_MISMATCH,
+                    detail: format!("{} bytes disagree with {}", source.display(), object.id()),
+                });
+            }
+            match store.put_manifest(&manifest) {
+                Ok(value) => value.admitted,
+                Err(error) => return bail(error),
+            }
+        }
+    };
+    println!(
+        "{{\"status\":\"{}\"}}",
+        if admitted { "stored" } else { "present" }
+    );
+    ExitCode::SUCCESS
+}
+
 fn cmd_config_canonicalize(path: &Path, flags: &Flags) -> ExitCode {
     let Some(output) = flag(flags, "out") else {
         eprintln!("--out <canonical.json> is required");
@@ -336,9 +518,30 @@ mod repo;
 mod snap;
 mod watch;
 
+fn cmd_home(explicit: Option<&Path>) -> ExitCode {
+    println!("{}", ok!(tensorfs_core::home::resolve(explicit)).display());
+    ExitCode::SUCCESS
+}
+
+fn cmd_home_migrate(source: &Path, flags: &Flags) -> ExitCode {
+    let destination = flag(flags, "to").map(Path::new);
+    let report = ok!(tensorfs_core::home::move_store(source, destination));
+    println!(
+        "moved {} -> {} (device/inode/mtime preserved by same-filesystem rename)",
+        report.source.display(),
+        report.destination.display(),
+    );
+    ExitCode::SUCCESS
+}
+
+fn cmd_store_default(verb: &str) -> ExitCode {
+    cmd_store(&ok!(tensorfs_core::home::resolve(None)), verb)
+}
+
 fn cmd_store(root: &Path, verb: &str) -> ExitCode {
     let s = match verb {
         "init" => ok!(Store::init(root)),
+        "ensure" => ok!(Store::ensure(root)),
         _ => ok!(Store::open(root)),
     };
     println!("root:       {}", s.root().display());
@@ -346,6 +549,24 @@ fn cmd_store(root: &Path, verb: &str) -> ExitCode {
     ExitCode::SUCCESS
 }
 
+fn cmd_store_prepare_readers(explicit: Option<&Path>) -> ExitCode {
+    let root = ok!(tensorfs_core::home::resolve(explicit));
+    let store = ok!(Store::open(&root));
+    ok!(store.prepare_readers());
+    println!("root:       {}", store.root().display());
+    println!("readers:    prepared");
+    ExitCode::SUCCESS
+}
+
+fn cmd_store_complete_cozytensors(explicit: Option<&Path>) -> ExitCode {
+    let root = ok!(tensorfs_core::home::resolve(explicit));
+    let store = ok!(Store::open(&root));
+    for manifest in ok!(store.complete_cozytensors_manifests()) {
+        println!("{manifest}");
+    }
+    ExitCode::SUCCESS
+}
+
 fn fault_of(f: &Flags) -> Fault {
     Fault {
         stage: flag(f, "fault").map(|s| s.to_string()),

tensorfs-0.3.1/crates/tensorfs-core/src/home.rs

@@ -0,0 +1,195 @@
+//! TensorFS-owned default Store location.
+//!
+//! Core Store operations remain explicit-root. This resolver exists for user-facing entrypoints
+//! and never consults another product's home or configuration.
+
+use std::fs;
+use std::os::unix::fs::MetadataExt;
+use std::path::{Path, PathBuf};
+
+use crate::catalog::WriterGuard;
+use crate::err::{refuse, Code, Result};
+use crate::store::{self, Store};
+
+pub const ENV: &str = "TENSORFS_HOME";
+
+#[derive(Debug, Clone, PartialEq, Eq)]
+pub struct MoveReport {
+    pub source: PathBuf,
+    pub destination: PathBuf,
+}
+
+/// Resolve an optional caller-supplied Store root.
+///
+/// Precedence is explicit argument, `TENSORFS_HOME`, then `$HOME/.tensorfs`.
+pub fn resolve(explicit: Option<&Path>) -> Result<PathBuf> {
+    resolve_values(
+        explicit,
+        std::env::var_os(ENV).map(PathBuf::from).as_deref(),
+        std::env::var_os("HOME").map(PathBuf::from).as_deref(),
+    )
+}
+
+fn resolve_values(
+    explicit: Option<&Path>,
+    tensorfs_home: Option<&Path>,
+    user_home: Option<&Path>,
+) -> Result<PathBuf> {
+    if let Some(root) = explicit {
+        return nonempty("explicit TensorFS root", root);
+    }
+    if let Some(root) = tensorfs_home {
+        return nonempty(ENV, root);
+    }
+    match user_home {
+        Some(root) if !root.as_os_str().is_empty() => Ok(root.join(".tensorfs")),
+        _ => refuse(
+            Code::IO_FAILED,
+            "TensorFS root is absent: pass one explicitly, set TENSORFS_HOME, or set HOME",
+        ),
+    }
+}
+
+fn nonempty(source: &str, root: &Path) -> Result<PathBuf> {
+    if root.as_os_str().is_empty() {
+        return refuse(Code::IO_FAILED, format!("{source} is empty"));
+    }
+    Ok(root.to_path_buf())
+}
+
+/// Move one stopped Store into the resolved TensorFS home without copying or rehashing it.
+///
+/// The destination must not exist and both parents must be on the same filesystem. The Store's
+/// recovery lock refuses already-active readers and writers. This is still an offline operator
+/// command: possible Store openers must be stopped so none can recreate the retired pathname
+/// after the rename. No redirect or reservation is left at the old path.
+pub fn move_store(source: &Path, explicit_destination: Option<&Path>) -> Result<MoveReport> {
+    let source_metadata = fs::symlink_metadata(source).map_err(|error| io("stat source", error))?;
+    if source_metadata.file_type().is_symlink() || !source_metadata.is_dir() {
+        return refuse(
+            Code::STORE_ERA,
+            format!("{} is not a TensorFS Store directory", source.display()),
+        );
+    }
+    let source = fs::canonicalize(source).map_err(|error| io("resolve source", error))?;
+    Store::open(&source)?;
+
+    let requested_destination = resolve(explicit_destination)?;
+    match fs::symlink_metadata(&requested_destination) {
+        Ok(_) => {
+            return refuse(
+                Code::IO_FAILED,
+                format!(
+                    "destination {} already exists; remove the conflict or choose another root",
+                    requested_destination.display()
+                ),
+            )
+        }
+        Err(error) if error.kind() == std::io::ErrorKind::NotFound => {}
+        Err(error) => return Err(io("stat destination", error)),
+    }
+    let destination_name =
+        requested_destination
+            .file_name()
+            .ok_or_else(|| crate::err::Refusal {
+                code: Code::IO_FAILED,
+                detail: format!(
+                    "destination {} has no final path component",
+                    requested_destination.display()
+                ),
+            })?;
+    let destination_parent = requested_destination
+        .parent()
+        .filter(|path| !path.as_os_str().is_empty())
+        .unwrap_or_else(|| Path::new("."));
+    let destination_parent = fs::canonicalize(destination_parent)
+        .map_err(|error| io("resolve destination parent", error))?;
+    let destination = destination_parent.join(destination_name);
+    if source == destination {
+        return refuse(Code::IO_FAILED, "source and destination are the same Store");
+    }
+    let source_parent = source.parent().ok_or_else(|| crate::err::Refusal {
+        code: Code::IO_FAILED,
+        detail: format!("source {} has no parent", source.display()),
+    })?;
+    let source_device = fs::metadata(&source)
+        .map_err(|error| io("stat source", error))?
+        .dev();
+    let destination_device = fs::metadata(&destination_parent)
+        .map_err(|error| io("stat destination parent", error))?
+        .dev();
+    if source_device != destination_device {
+        return refuse(
+            Code::IO_FAILED,
+            format!(
+                "source {} and destination {} are on different filesystems; a hardcut requires an atomic rename",
+                source.display(),
+                destination.display()
+            ),
+        );
+    }
+
+    let _exclusive = WriterGuard::lock_rebuild(&source)?;
+    fs::rename(&source, &destination).map_err(|error| io("rename Store", error))?;
+    store::fsync_dir(&destination_parent)?;
+    if source_parent != destination_parent {
+        store::fsync_dir(source_parent)?;
+    }
+    Store::open(&destination)?;
+    Ok(MoveReport {
+        source,
+        destination,
+    })
+}
+
+fn io(what: &str, error: std::io::Error) -> crate::err::Refusal {
+    crate::err::Refusal {
+        code: Code::IO_FAILED,
+        detail: format!("{what}: {error}"),
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn precedence_is_explicit_then_tensorfs_home_then_user_home() {
+        assert_eq!(
+            resolve_values(
+                Some(Path::new("/explicit")),
+                Some(Path::new("/environment")),
+                Some(Path::new("/user")),
+            )
+            .unwrap(),
+            Path::new("/explicit"),
+        );
+        assert_eq!(
+            resolve_values(
+                None,
+                Some(Path::new("/environment")),
+                Some(Path::new("/user"))
+            )
+            .unwrap(),
+            Path::new("/environment"),
+        );
+        assert_eq!(
+            resolve_values(None, None, Some(Path::new("/user"))).unwrap(),
+            Path::new("/user/.tensorfs"),
+        );
+    }
+
+    #[test]
+    fn an_empty_selected_value_refuses_instead_of_falling_through() {
+        assert_eq!(
+            resolve_values(None, Some(Path::new("")), Some(Path::new("/user")))
+                .unwrap_err()
+                .code,
+            Code::IO_FAILED,
+        );
+        assert_eq!(
+            resolve_values(None, None, None).unwrap_err().code,
+            Code::IO_FAILED,
+        );
+    }
+}

{tensorfs-0.3.0 → tensorfs-0.3.1}/crates/tensorfs-core/src/lib.rs

@@ -18,6 +18,7 @@ pub mod fill;
 pub mod fit;
 pub mod gc;
 pub mod header;
+pub mod home;
 pub mod ids;
 pub mod ingest;
 pub mod jcs;
@@ -30,6 +31,7 @@ pub mod read;
 pub mod receipt;
 pub mod registry;
 pub mod relation;
+pub mod repo_cache;
 pub mod repository;
 pub mod sha256;
 pub mod spec;