tensorfs 0.3.71__tar.gz → 0.3.74__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {tensorfs-0.3.71 → tensorfs-0.3.74}/Cargo.lock +2 -2
- {tensorfs-0.3.71 → tensorfs-0.3.74}/Cargo.toml +1 -1
- {tensorfs-0.3.71 → tensorfs-0.3.74}/PKG-INFO +1 -1
- {tensorfs-0.3.71 → tensorfs-0.3.74}/crates/tensorfs-core/src/bin/tfs/fetch.rs +2 -3
- {tensorfs-0.3.71 → tensorfs-0.3.74}/crates/tensorfs-core/src/bin/tfs/main.rs +2 -15
- tensorfs-0.3.74/crates/tensorfs-core/src/derived/history_cost_tests.rs +149 -0
- {tensorfs-0.3.71 → tensorfs-0.3.74}/crates/tensorfs-core/src/derived/part_rows_tests.rs +6 -23
- {tensorfs-0.3.71 → tensorfs-0.3.74}/crates/tensorfs-core/src/derived/retention.rs +4 -21
- {tensorfs-0.3.71 → tensorfs-0.3.74}/crates/tensorfs-core/src/derived.rs +47 -44
- {tensorfs-0.3.71 → tensorfs-0.3.74}/crates/tensorfs-core/src/gc.rs +39 -20
- {tensorfs-0.3.71 → tensorfs-0.3.74}/crates/tensorfs-core/src/home.rs +0 -102
- {tensorfs-0.3.71 → tensorfs-0.3.74}/crates/tensorfs-core/src/store.rs +2 -8
- {tensorfs-0.3.71 → tensorfs-0.3.74}/crates/tensorfs-core/src/transport/hub.rs +54 -81
- {tensorfs-0.3.71 → tensorfs-0.3.74}/crates/tensorfs-core/src/transport/pull.rs +9 -32
- {tensorfs-0.3.71 → tensorfs-0.3.74}/crates/tensorfs-core/src/transport/tests.rs +95 -42
- {tensorfs-0.3.71 → tensorfs-0.3.74}/crates/tensorfs-core/tests/home.rs +3 -121
- {tensorfs-0.3.71 → tensorfs-0.3.74}/crates/tensorfs-py/src/store.rs +19 -50
- {tensorfs-0.3.71 → tensorfs-0.3.74}/pyproject.toml +1 -1
- {tensorfs-0.3.71 → tensorfs-0.3.74}/python/tensorfs/_ext.pyi +6 -12
- {tensorfs-0.3.71 → tensorfs-0.3.74}/python/tensorfs/errors.py +1 -10
- {tensorfs-0.3.71 → tensorfs-0.3.74}/crates/tensorfs-core/Cargo.toml +0 -0
- {tensorfs-0.3.71 → tensorfs-0.3.74}/crates/tensorfs-core/build.rs +0 -0
- {tensorfs-0.3.71 → tensorfs-0.3.74}/crates/tensorfs-core/examples/h3-swiglu-layout.md +0 -0
- {tensorfs-0.3.71 → tensorfs-0.3.74}/crates/tensorfs-core/examples/optional-cache-backfill.md +0 -0
- {tensorfs-0.3.71 → tensorfs-0.3.74}/crates/tensorfs-core/examples/resident-conversion.md +0 -0
- {tensorfs-0.3.71 → tensorfs-0.3.74}/crates/tensorfs-core/examples/resident_conversion_profile.rs +0 -0
- {tensorfs-0.3.71 → tensorfs-0.3.74}/crates/tensorfs-core/src/asset_update.rs +0 -0
- {tensorfs-0.3.71 → tensorfs-0.3.74}/crates/tensorfs-core/src/b64.rs +0 -0
- {tensorfs-0.3.71 → tensorfs-0.3.74}/crates/tensorfs-core/src/bin/tfs/ckpt.rs +0 -0
- {tensorfs-0.3.71 → tensorfs-0.3.74}/crates/tensorfs-core/src/bin/tfs/conform.rs +0 -0
- {tensorfs-0.3.71 → tensorfs-0.3.74}/crates/tensorfs-core/src/bin/tfs/ingest.rs +0 -0
- {tensorfs-0.3.71 → tensorfs-0.3.74}/crates/tensorfs-core/src/bin/tfs/rd.rs +0 -0
- {tensorfs-0.3.71 → tensorfs-0.3.74}/crates/tensorfs-core/src/bin/tfs/repo.rs +0 -0
- {tensorfs-0.3.71 → tensorfs-0.3.74}/crates/tensorfs-core/src/bin/tfs/snap.rs +0 -0
- {tensorfs-0.3.71 → tensorfs-0.3.74}/crates/tensorfs-core/src/bin/tfs/watch.rs +0 -0
- {tensorfs-0.3.71 → tensorfs-0.3.74}/crates/tensorfs-core/src/cache_roots.rs +0 -0
- {tensorfs-0.3.71 → tensorfs-0.3.74}/crates/tensorfs-core/src/canon.rs +0 -0
- {tensorfs-0.3.71 → tensorfs-0.3.74}/crates/tensorfs-core/src/capability.rs +0 -0
- {tensorfs-0.3.71 → tensorfs-0.3.74}/crates/tensorfs-core/src/catalog/lock.rs +0 -0
- {tensorfs-0.3.71 → tensorfs-0.3.74}/crates/tensorfs-core/src/catalog.rs +0 -0
- {tensorfs-0.3.71 → tensorfs-0.3.74}/crates/tensorfs-core/src/cbor.rs +0 -0
- {tensorfs-0.3.71 → tensorfs-0.3.74}/crates/tensorfs-core/src/checkpoint.rs +0 -0
- {tensorfs-0.3.71 → tensorfs-0.3.74}/crates/tensorfs-core/src/checkpoint_root.rs +0 -0
- {tensorfs-0.3.71 → tensorfs-0.3.74}/crates/tensorfs-core/src/corpus.rs +0 -0
- {tensorfs-0.3.71 → tensorfs-0.3.74}/crates/tensorfs-core/src/derived/adoption.rs +0 -0
- {tensorfs-0.3.71 → tensorfs-0.3.74}/crates/tensorfs-core/src/derived/companion_tests.rs +0 -0
- {tensorfs-0.3.71 → tensorfs-0.3.74}/crates/tensorfs-core/src/derived/estimate.rs +0 -0
- {tensorfs-0.3.71 → tensorfs-0.3.74}/crates/tensorfs-core/src/derived/graft_tests.rs +0 -0
- {tensorfs-0.3.71 → tensorfs-0.3.74}/crates/tensorfs-core/src/derived/group_commit_tests.rs +0 -0
- {tensorfs-0.3.71 → tensorfs-0.3.74}/crates/tensorfs-core/src/derived/progress.rs +0 -0
- {tensorfs-0.3.71 → tensorfs-0.3.74}/crates/tensorfs-core/src/derived/resume_tests.rs +0 -0
- {tensorfs-0.3.71 → tensorfs-0.3.74}/crates/tensorfs-core/src/derived/roots.rs +0 -0
- {tensorfs-0.3.71 → tensorfs-0.3.74}/crates/tensorfs-core/src/descriptors.rs +0 -0
- {tensorfs-0.3.71 → tensorfs-0.3.74}/crates/tensorfs-core/src/disk.rs +0 -0
- {tensorfs-0.3.71 → tensorfs-0.3.74}/crates/tensorfs-core/src/dtype.rs +0 -0
- {tensorfs-0.3.71 → tensorfs-0.3.74}/crates/tensorfs-core/src/durability.rs +0 -0
- {tensorfs-0.3.71 → tensorfs-0.3.74}/crates/tensorfs-core/src/err.rs +0 -0
- {tensorfs-0.3.71 → tensorfs-0.3.74}/crates/tensorfs-core/src/fetch.rs +0 -0
- {tensorfs-0.3.71 → tensorfs-0.3.74}/crates/tensorfs-core/src/fill.rs +0 -0
- {tensorfs-0.3.71 → tensorfs-0.3.74}/crates/tensorfs-core/src/fit.rs +0 -0
- {tensorfs-0.3.71 → tensorfs-0.3.74}/crates/tensorfs-core/src/header.rs +0 -0
- {tensorfs-0.3.71 → tensorfs-0.3.74}/crates/tensorfs-core/src/ids.rs +0 -0
- {tensorfs-0.3.71 → tensorfs-0.3.74}/crates/tensorfs-core/src/ingest/carrier.rs +0 -0
- {tensorfs-0.3.71 → tensorfs-0.3.74}/crates/tensorfs-core/src/ingest/convert.rs +0 -0
- {tensorfs-0.3.71 → tensorfs-0.3.74}/crates/tensorfs-core/src/ingest/custody.rs +0 -0
- {tensorfs-0.3.71 → tensorfs-0.3.74}/crates/tensorfs-core/src/ingest/fingerprint.rs +0 -0
- {tensorfs-0.3.71 → tensorfs-0.3.74}/crates/tensorfs-core/src/ingest/gguf.rs +0 -0
- {tensorfs-0.3.71 → tensorfs-0.3.74}/crates/tensorfs-core/src/ingest/h3_lora.rs +0 -0
- {tensorfs-0.3.71 → tensorfs-0.3.74}/crates/tensorfs-core/src/ingest/journal.rs +0 -0
- {tensorfs-0.3.71 → tensorfs-0.3.74}/crates/tensorfs-core/src/ingest/mod.rs +0 -0
- {tensorfs-0.3.71 → tensorfs-0.3.74}/crates/tensorfs-core/src/ingest/narrow.rs +0 -0
- {tensorfs-0.3.71 → tensorfs-0.3.74}/crates/tensorfs-core/src/ingest/preflight.rs +0 -0
- {tensorfs-0.3.71 → tensorfs-0.3.74}/crates/tensorfs-core/src/ingest/routes/sdxl.routes.json +0 -0
- {tensorfs-0.3.71 → tensorfs-0.3.74}/crates/tensorfs-core/src/ingest/routes/sdxl.spec.json +0 -0
- {tensorfs-0.3.71 → tensorfs-0.3.74}/crates/tensorfs-core/src/ingest/routes.rs +0 -0
- {tensorfs-0.3.71 → tensorfs-0.3.74}/crates/tensorfs-core/src/ingest/source.rs +0 -0
- {tensorfs-0.3.71 → tensorfs-0.3.74}/crates/tensorfs-core/src/ingest/stamp.rs +0 -0
- {tensorfs-0.3.71 → tensorfs-0.3.74}/crates/tensorfs-core/src/ingest/transaction.rs +0 -0
- {tensorfs-0.3.71 → tensorfs-0.3.74}/crates/tensorfs-core/src/jcs.rs +0 -0
- {tensorfs-0.3.71 → tensorfs-0.3.74}/crates/tensorfs-core/src/lib.rs +0 -0
- {tensorfs-0.3.71 → tensorfs-0.3.74}/crates/tensorfs-core/src/limits.rs +0 -0
- {tensorfs-0.3.71 → tensorfs-0.3.74}/crates/tensorfs-core/src/machine.rs +0 -0
- {tensorfs-0.3.71 → tensorfs-0.3.74}/crates/tensorfs-core/src/manifest.rs +0 -0
- {tensorfs-0.3.71 → tensorfs-0.3.74}/crates/tensorfs-core/src/meta.rs +0 -0
- {tensorfs-0.3.71 → tensorfs-0.3.74}/crates/tensorfs-core/src/project.rs +0 -0
- {tensorfs-0.3.71 → tensorfs-0.3.74}/crates/tensorfs-core/src/providers.rs +0 -0
- {tensorfs-0.3.71 → tensorfs-0.3.74}/crates/tensorfs-core/src/read.rs +0 -0
- {tensorfs-0.3.71 → tensorfs-0.3.74}/crates/tensorfs-core/src/receipt.rs +0 -0
- {tensorfs-0.3.71 → tensorfs-0.3.74}/crates/tensorfs-core/src/reclaim.rs +0 -0
- {tensorfs-0.3.71 → tensorfs-0.3.74}/crates/tensorfs-core/src/registry.rs +0 -0
- {tensorfs-0.3.71 → tensorfs-0.3.74}/crates/tensorfs-core/src/relation.rs +0 -0
- {tensorfs-0.3.71 → tensorfs-0.3.74}/crates/tensorfs-core/src/repo_cache/identity.rs +0 -0
- {tensorfs-0.3.71 → tensorfs-0.3.74}/crates/tensorfs-core/src/repo_cache.rs +0 -0
- {tensorfs-0.3.71 → tensorfs-0.3.74}/crates/tensorfs-core/src/repository.rs +0 -0
- {tensorfs-0.3.71 → tensorfs-0.3.74}/crates/tensorfs-core/src/sha256.rs +0 -0
- {tensorfs-0.3.71 → tensorfs-0.3.74}/crates/tensorfs-core/src/source_artifact.rs +0 -0
- {tensorfs-0.3.71 → tensorfs-0.3.74}/crates/tensorfs-core/src/spec.rs +0 -0
- {tensorfs-0.3.71 → tensorfs-0.3.74}/crates/tensorfs-core/src/staging.rs +0 -0
- {tensorfs-0.3.71 → tensorfs-0.3.74}/crates/tensorfs-core/src/stats.rs +0 -0
- {tensorfs-0.3.71 → tensorfs-0.3.74}/crates/tensorfs-core/src/storage.rs +0 -0
- {tensorfs-0.3.71 → tensorfs-0.3.74}/crates/tensorfs-core/src/transport/api.rs +0 -0
- {tensorfs-0.3.71 → tensorfs-0.3.74}/crates/tensorfs-core/src/transport/http.rs +0 -0
- {tensorfs-0.3.71 → tensorfs-0.3.74}/crates/tensorfs-core/src/transport/http_reuse_tests.rs +0 -0
- {tensorfs-0.3.71 → tensorfs-0.3.74}/crates/tensorfs-core/src/transport/ledger.rs +0 -0
- {tensorfs-0.3.71 → tensorfs-0.3.74}/crates/tensorfs-core/src/transport/memory.rs +0 -0
- {tensorfs-0.3.71 → tensorfs-0.3.74}/crates/tensorfs-core/src/transport/mod.rs +0 -0
- {tensorfs-0.3.71 → tensorfs-0.3.74}/crates/tensorfs-core/src/transport/pace.rs +0 -0
- {tensorfs-0.3.71 → tensorfs-0.3.74}/crates/tensorfs-core/src/transport/policy.rs +0 -0
- {tensorfs-0.3.71 → tensorfs-0.3.74}/crates/tensorfs-core/src/transport/prefix.rs +0 -0
- {tensorfs-0.3.71 → tensorfs-0.3.74}/crates/tensorfs-core/src/transport/push.rs +0 -0
- {tensorfs-0.3.71 → tensorfs-0.3.74}/crates/tensorfs-core/src/transport/ranged.rs +0 -0
- {tensorfs-0.3.71 → tensorfs-0.3.74}/crates/tensorfs-core/src/transport/speculate.rs +0 -0
- {tensorfs-0.3.71 → tensorfs-0.3.74}/crates/tensorfs-core/src/unsynced.rs +0 -0
- {tensorfs-0.3.71 → tensorfs-0.3.74}/crates/tensorfs-core/src/vectors.rs +0 -0
- {tensorfs-0.3.71 → tensorfs-0.3.74}/crates/tensorfs-core/tests/additive_fields.rs +0 -0
- {tensorfs-0.3.71 → tensorfs-0.3.74}/crates/tensorfs-core/tests/anima_hf_profile.rs +0 -0
- {tensorfs-0.3.71 → tensorfs-0.3.74}/crates/tensorfs-core/tests/as_is.rs +0 -0
- {tensorfs-0.3.71 → tensorfs-0.3.74}/crates/tensorfs-core/tests/checkpoint_roots.rs +0 -0
- {tensorfs-0.3.71 → tensorfs-0.3.74}/crates/tensorfs-core/tests/civitai_sdxl_profile.rs +0 -0
- {tensorfs-0.3.71 → tensorfs-0.3.74}/crates/tensorfs-core/tests/fd_budget.rs +0 -0
- {tensorfs-0.3.71 → tensorfs-0.3.74}/crates/tensorfs-core/tests/fd_headroom.rs +0 -0
- {tensorfs-0.3.71 → tensorfs-0.3.74}/crates/tensorfs-core/tests/fetch_url_progress.rs +0 -0
- {tensorfs-0.3.71 → tensorfs-0.3.74}/crates/tensorfs-core/tests/fit.rs +0 -0
- {tensorfs-0.3.71 → tensorfs-0.3.74}/crates/tensorfs-core/tests/fixtures/repo_apply/blobs.jsonl +0 -0
- {tensorfs-0.3.71 → tensorfs-0.3.74}/crates/tensorfs-core/tests/fixtures/repo_apply/expected_list.jsonl +0 -0
- {tensorfs-0.3.71 → tensorfs-0.3.74}/crates/tensorfs-core/tests/fixtures/repo_apply/expected_repository.json +0 -0
- {tensorfs-0.3.71 → tensorfs-0.3.74}/crates/tensorfs-core/tests/fixtures/repo_apply/expected_rows.jsonl +0 -0
- {tensorfs-0.3.71 → tensorfs-0.3.74}/crates/tensorfs-core/tests/fixtures/repo_apply/header.cbor +0 -0
- {tensorfs-0.3.71 → tensorfs-0.3.74}/crates/tensorfs-core/tests/fixtures/repo_apply/manifest.json +0 -0
- {tensorfs-0.3.71 → tensorfs-0.3.74}/crates/tensorfs-core/tests/fixtures/repo_apply/mutation.json +0 -0
- {tensorfs-0.3.71 → tensorfs-0.3.74}/crates/tensorfs-core/tests/foreign_json.rs +0 -0
- {tensorfs-0.3.71 → tensorfs-0.3.74}/crates/tensorfs-core/tests/h3_lora.rs +0 -0
- {tensorfs-0.3.71 → tensorfs-0.3.74}/crates/tensorfs-core/tests/h3_swiglu.rs +0 -0
- {tensorfs-0.3.71 → tensorfs-0.3.74}/crates/tensorfs-core/tests/import_tree.rs +0 -0
- {tensorfs-0.3.71 → tensorfs-0.3.74}/crates/tensorfs-core/tests/instrument_profiles.rs +0 -0
- {tensorfs-0.3.71 → tensorfs-0.3.74}/crates/tensorfs-core/tests/liveness.rs +0 -0
- {tensorfs-0.3.71 → tensorfs-0.3.74}/crates/tensorfs-core/tests/qwen_image21_profile.rs +0 -0
- {tensorfs-0.3.71 → tensorfs-0.3.74}/crates/tensorfs-core/tests/read_plan.rs +0 -0
- {tensorfs-0.3.71 → tensorfs-0.3.74}/crates/tensorfs-core/tests/reclaim.rs +0 -0
- {tensorfs-0.3.71 → tensorfs-0.3.74}/crates/tensorfs-core/tests/reclaim_run.rs +0 -0
- {tensorfs-0.3.71 → tensorfs-0.3.74}/crates/tensorfs-core/tests/repo_cache.rs +0 -0
- {tensorfs-0.3.71 → tensorfs-0.3.74}/crates/tensorfs-core/tests/repo_store.rs +0 -0
- {tensorfs-0.3.71 → tensorfs-0.3.74}/crates/tensorfs-core/tests/resident_admission.rs +0 -0
- {tensorfs-0.3.71 → tensorfs-0.3.74}/crates/tensorfs-core/tests/routed_single_file.rs +0 -0
- {tensorfs-0.3.71 → tensorfs-0.3.74}/crates/tensorfs-core/tests/sha256_backends.rs +0 -0
- {tensorfs-0.3.71 → tensorfs-0.3.74}/crates/tensorfs-core/tests/sharded_cas.rs +0 -0
- {tensorfs-0.3.71 → tensorfs-0.3.74}/crates/tensorfs-core/tests/source_artifacts.rs +0 -0
- {tensorfs-0.3.71 → tensorfs-0.3.74}/crates/tensorfs-core/tests/source_download.rs +0 -0
- {tensorfs-0.3.71 → tensorfs-0.3.74}/crates/tensorfs-core/tests/source_plan.rs +0 -0
- {tensorfs-0.3.71 → tensorfs-0.3.74}/crates/tensorfs-core/tests/source_prefixes.rs +0 -0
- {tensorfs-0.3.71 → tensorfs-0.3.74}/crates/tensorfs-core/tests/source_preflight.rs +0 -0
- {tensorfs-0.3.71 → tensorfs-0.3.74}/crates/tensorfs-core/tests/staging.rs +0 -0
- {tensorfs-0.3.71 → tensorfs-0.3.74}/crates/tensorfs-core/tests/store_boundaries.rs +0 -0
- {tensorfs-0.3.71 → tensorfs-0.3.74}/crates/tensorfs-core/tests/tensor_schema.rs +0 -0
- {tensorfs-0.3.71 → tensorfs-0.3.74}/crates/tensorfs-core/tests/transfer_memory.rs +0 -0
- {tensorfs-0.3.71 → tensorfs-0.3.74}/crates/tensorfs-core/tests/trust_index.rs +0 -0
- {tensorfs-0.3.71 → tensorfs-0.3.74}/crates/tensorfs-core/tests/version.rs +0 -0
- {tensorfs-0.3.71 → tensorfs-0.3.74}/crates/tensorfs-core/tests/write_through.rs +0 -0
- {tensorfs-0.3.71 → tensorfs-0.3.74}/crates/tensorfs-py/Cargo.toml +0 -0
- {tensorfs-0.3.71 → tensorfs-0.3.74}/crates/tensorfs-py/src/docs.rs +0 -0
- {tensorfs-0.3.71 → tensorfs-0.3.74}/crates/tensorfs-py/src/errors.rs +0 -0
- {tensorfs-0.3.71 → tensorfs-0.3.74}/crates/tensorfs-py/src/ingest.rs +0 -0
- {tensorfs-0.3.71 → tensorfs-0.3.74}/crates/tensorfs-py/src/lib.rs +0 -0
- {tensorfs-0.3.71 → tensorfs-0.3.74}/python/tensorfs/__init__.py +0 -0
- {tensorfs-0.3.71 → tensorfs-0.3.74}/python/tensorfs/_derived_channel.py +0 -0
- {tensorfs-0.3.71 → tensorfs-0.3.74}/python/tensorfs/_source_channel.py +0 -0
- {tensorfs-0.3.71 → tensorfs-0.3.74}/python/tensorfs/_tensors.py +0 -0
- {tensorfs-0.3.71 → tensorfs-0.3.74}/python/tensorfs/derived.py +0 -0
- {tensorfs-0.3.71 → tensorfs-0.3.74}/python/tensorfs/py.typed +0 -0
|
@@ -439,7 +439,7 @@ checksum = "adb6935a6f5c20170eeceb1a3835a49e12e19d792f6dd344ccc76a985ca5a6ca"
|
|
|
439
439
|
|
|
440
440
|
[[package]]
|
|
441
441
|
name = "tensorfs-core"
|
|
442
|
-
version = "0.3.
|
|
442
|
+
version = "0.3.74"
|
|
443
443
|
dependencies = [
|
|
444
444
|
"fs2",
|
|
445
445
|
"libc",
|
|
@@ -454,7 +454,7 @@ dependencies = [
|
|
|
454
454
|
|
|
455
455
|
[[package]]
|
|
456
456
|
name = "tensorfs-py"
|
|
457
|
-
version = "0.3.
|
|
457
|
+
version = "0.3.74"
|
|
458
458
|
dependencies = [
|
|
459
459
|
"pyo3",
|
|
460
460
|
"tensorfs-core",
|
|
@@ -192,9 +192,8 @@ fn deadline_of(flags: &Flags) -> Deadline {
|
|
|
192
192
|
|
|
193
193
|
/// The ONE credential spelling, reaching the child explicitly (tfs-050): `--credential-file
|
|
194
194
|
/// <path>` or the deliberately-set `TFS_CREDENTIAL` — never argv (world-readable in /proc),
|
|
195
|
-
/// never the inherited environment at large. The value is `bearer <token
|
|
196
|
-
/// exactly the named hosts, or empty for anonymous
|
|
197
|
-
/// the hub's closure and presign routes taking no credential at all.
|
|
195
|
+
/// never the inherited environment at large. The value is `bearer <token>` or
|
|
196
|
+
/// `worker <id> <token>`, scoped to exactly the named hosts, or empty for anonymous.
|
|
198
197
|
fn credential_of(
|
|
199
198
|
flags: &Flags,
|
|
200
199
|
hosts: Vec<String>,
|
|
@@ -22,7 +22,6 @@
|
|
|
22
22
|
//! tfs store prepare-readers [<root>]
|
|
23
23
|
//! tfs store complete-cozytensors [<root>]
|
|
24
24
|
//! tfs home print TENSORFS_HOME or ~/.tensorfs
|
|
25
|
-
//! tfs home migrate <old-root> [--to <new-root>]
|
|
26
25
|
//! tfs store rebuild <census> --rows <jsonl> [--sqlite <tensorfs.sqlite>]
|
|
27
26
|
//! tfs store admit-file <store> blob|manifest <id> <length> <source>
|
|
28
27
|
//! tfs repo-cache admit <cache> <store> blob|manifest <id> <length>
|
|
@@ -40,8 +39,8 @@
|
|
|
40
39
|
//! tfs fill <root> <plan> resumable fill: skip only verified objects
|
|
41
40
|
//!
|
|
42
41
|
//! The ONE transport (tfs-048/tfs-049/tfs-050). The credential is `--credential-file` or
|
|
43
|
-
//! TFS_CREDENTIAL (`bearer <token>`, or empty for anonymous), never
|
|
44
|
-
//! URL of `-` arrives on stdin. Flags: --hub <base> --lane --session --streams <n>
|
|
42
|
+
//! TFS_CREDENTIAL (`bearer <token>`, `worker <id> <token>`, or empty for anonymous), never
|
|
43
|
+
//! bare argv; a URL of `-` arrives on stdin. Flags: --hub <base> --lane --session --streams <n>
|
|
45
44
|
//! --streams-start <n> --timeout <s> --allow-hosts a,b,.c --allow-local --max-redirects <n>.
|
|
46
45
|
//! `--streams` is the most objects in flight (default 128, memory-bounded); the walk starts
|
|
47
46
|
//! at `--streams-start` (default 16) and doubles while measured goodput follows.
|
|
@@ -208,7 +207,6 @@ fn main() -> ExitCode {
|
|
|
208
207
|
match a.as_slice() {
|
|
209
208
|
["version"] => cmd_version(),
|
|
210
209
|
["home"] => cmd_home(None),
|
|
211
|
-
["home", "migrate", source] => cmd_home_migrate(Path::new(source), &flags),
|
|
212
210
|
["home", r] => cmd_home(Some(Path::new(r))),
|
|
213
211
|
["store", "init"] => cmd_store_default("init", &flags),
|
|
214
212
|
["store", "info"] => cmd_store_default("info", &flags),
|
|
@@ -817,17 +815,6 @@ fn cmd_home(explicit: Option<&Path>) -> ExitCode {
|
|
|
817
815
|
ExitCode::SUCCESS
|
|
818
816
|
}
|
|
819
817
|
|
|
820
|
-
fn cmd_home_migrate(source: &Path, flags: &Flags) -> ExitCode {
|
|
821
|
-
let destination = flag(flags, "to").map(Path::new);
|
|
822
|
-
let report = ok!(tensorfs_core::home::move_store(source, destination));
|
|
823
|
-
println!(
|
|
824
|
-
"moved {} -> {} (device/inode/mtime preserved by same-filesystem rename)",
|
|
825
|
-
report.source.display(),
|
|
826
|
-
report.destination.display(),
|
|
827
|
-
);
|
|
828
|
-
ExitCode::SUCCESS
|
|
829
|
-
}
|
|
830
|
-
|
|
831
818
|
/// The no-root spelling of a `store` verb. It uses `TENSORFS_HOME`, or an existing
|
|
832
819
|
/// `$HOME/.tensorfs` — never a Store it would have to invent (tfs-053).
|
|
833
820
|
fn cmd_store_default(verb: &str, flags: &Flags) -> ExitCode {
|
|
@@ -0,0 +1,149 @@
|
|
|
1
|
+
//! A derivation costs the same however many derivations the Store already retains (finding
|
|
2
|
+
//! 50): proving its source rooted walked every retained checkpoint chain, so each lane on a
|
|
3
|
+
//! conversion pod took longer than the one before.
|
|
4
|
+
|
|
5
|
+
use std::time::{Duration, Instant};
|
|
6
|
+
|
|
7
|
+
use super::tests::created_declaration;
|
|
8
|
+
use super::*;
|
|
9
|
+
use crate::repository::{Mutation, RepositoryName};
|
|
10
|
+
use crate::store::Fault;
|
|
11
|
+
|
|
12
|
+
const PARTS: usize = 32;
|
|
13
|
+
const CONVERSIONS: usize = 60;
|
|
14
|
+
const WINDOW: usize = 10;
|
|
15
|
+
|
|
16
|
+
/// A source-inheriting derivation that adds PARTS tensors and checkpoints after each, as a
|
|
17
|
+
/// quantization lane does: its source inspection, writer, chain and pending root.
|
|
18
|
+
fn convert(store: &Store, meta: &Meta, index: usize, source: &ObjectRef) -> Duration {
|
|
19
|
+
let started = Instant::now();
|
|
20
|
+
inspect_source(
|
|
21
|
+
store,
|
|
22
|
+
meta,
|
|
23
|
+
source.clone(),
|
|
24
|
+
vec!["model".into()],
|
|
25
|
+
Vec::new(),
|
|
26
|
+
)
|
|
27
|
+
.unwrap();
|
|
28
|
+
let prototype = created_declaration().components.remove(0).add.remove(0);
|
|
29
|
+
let add: Vec<_> = (0..PARTS)
|
|
30
|
+
.map(|part| TensorDeclaration {
|
|
31
|
+
key: format!("added_{part}"),
|
|
32
|
+
..prototype.clone()
|
|
33
|
+
})
|
|
34
|
+
.collect();
|
|
35
|
+
let mut order = vec![("model".to_string(), "weight".to_string())];
|
|
36
|
+
order.extend(
|
|
37
|
+
add.iter()
|
|
38
|
+
.map(|tensor| ("model".to_string(), tensor.key.clone())),
|
|
39
|
+
);
|
|
40
|
+
let declaration = Declaration {
|
|
41
|
+
sources: vec![Source {
|
|
42
|
+
alias: "source".into(),
|
|
43
|
+
manifest: source.clone(),
|
|
44
|
+
}],
|
|
45
|
+
components: vec![ComponentDeclaration {
|
|
46
|
+
target: "model".into(),
|
|
47
|
+
source: Some("source".into()),
|
|
48
|
+
source_component: Some("model".into()),
|
|
49
|
+
drop: Vec::new(),
|
|
50
|
+
add,
|
|
51
|
+
}],
|
|
52
|
+
order,
|
|
53
|
+
max_new_bytes: (PARTS * 2048) as u64,
|
|
54
|
+
..created_declaration()
|
|
55
|
+
};
|
|
56
|
+
let transaction = format!("sha256:{:064x}", index + 1);
|
|
57
|
+
let writer = begin(store, meta, &transaction, 1, declaration, None).unwrap();
|
|
58
|
+
let (mut cursor, mut head) = (None, None);
|
|
59
|
+
for part in 0..PARTS {
|
|
60
|
+
add_indexed_part(
|
|
61
|
+
store,
|
|
62
|
+
meta,
|
|
63
|
+
&transaction,
|
|
64
|
+
1,
|
|
65
|
+
&writer.additions,
|
|
66
|
+
cursor.as_mut(),
|
|
67
|
+
("model", &format!("added_{part}"), "value"),
|
|
68
|
+
&mut [(index * PARTS + part) as u8; 2048].as_slice(),
|
|
69
|
+
)
|
|
70
|
+
.unwrap();
|
|
71
|
+
let (_, next) = checkpoint_indexed(
|
|
72
|
+
store,
|
|
73
|
+
meta,
|
|
74
|
+
&transaction,
|
|
75
|
+
1,
|
|
76
|
+
"lane",
|
|
77
|
+
"model",
|
|
78
|
+
head.as_ref(),
|
|
79
|
+
&mut cursor,
|
|
80
|
+
)
|
|
81
|
+
.unwrap();
|
|
82
|
+
head = next.head;
|
|
83
|
+
}
|
|
84
|
+
commit(store, meta, &transaction, 1).unwrap();
|
|
85
|
+
release_guards(meta, Some(writer.writer_hold), writer.source_leases);
|
|
86
|
+
started.elapsed()
|
|
87
|
+
}
|
|
88
|
+
|
|
89
|
+
#[test]
|
|
90
|
+
fn a_conversion_costs_the_same_after_many_retained_conversions() {
|
|
91
|
+
let scratch = std::path::Path::new("/dev/shm");
|
|
92
|
+
let root = if scratch.is_dir() {
|
|
93
|
+
scratch.to_path_buf()
|
|
94
|
+
} else {
|
|
95
|
+
std::env::temp_dir()
|
|
96
|
+
}
|
|
97
|
+
.join(format!(
|
|
98
|
+
"tensorfs-history-cost-{}-{}",
|
|
99
|
+
std::process::id(),
|
|
100
|
+
crate::meta::now_nanos_unique()
|
|
101
|
+
));
|
|
102
|
+
let store = Store::init(&root).unwrap();
|
|
103
|
+
let meta = Meta::open(&store).unwrap();
|
|
104
|
+
|
|
105
|
+
// The source is a model checkpoint, rooted by its repository as a pulled model is.
|
|
106
|
+
let source_id = format!("sha256:{}", "5a".repeat(32));
|
|
107
|
+
let writer = begin(&store, &meta, &source_id, 1, created_declaration(), None).unwrap();
|
|
108
|
+
add_part(
|
|
109
|
+
&store,
|
|
110
|
+
&meta,
|
|
111
|
+
&source_id,
|
|
112
|
+
1,
|
|
113
|
+
("model", "weight", "value"),
|
|
114
|
+
&mut [0x21; 2048].as_slice(),
|
|
115
|
+
)
|
|
116
|
+
.unwrap();
|
|
117
|
+
let source = commit(&store, &meta, &source_id, 1).unwrap().manifest;
|
|
118
|
+
release_guards(&meta, Some(writer.writer_hold), writer.source_leases);
|
|
119
|
+
store
|
|
120
|
+
.apply_repository(
|
|
121
|
+
None,
|
|
122
|
+
&Mutation::PutCheckpoint {
|
|
123
|
+
repo: RepositoryName::new("org", "source").unwrap(),
|
|
124
|
+
manifest: source.clone(),
|
|
125
|
+
},
|
|
126
|
+
&Fault::default(),
|
|
127
|
+
)
|
|
128
|
+
.unwrap();
|
|
129
|
+
|
|
130
|
+
let costs: Vec<Duration> = (0..CONVERSIONS)
|
|
131
|
+
.map(|index| convert(&store, &meta, index, &source))
|
|
132
|
+
.collect();
|
|
133
|
+
let median = |window: &[Duration]| {
|
|
134
|
+
let mut sorted = window.to_vec();
|
|
135
|
+
sorted.sort();
|
|
136
|
+
sorted[sorted.len() / 2]
|
|
137
|
+
};
|
|
138
|
+
let (first, last) = (
|
|
139
|
+
median(&costs[..WINDOW]),
|
|
140
|
+
median(&costs[CONVERSIONS - WINDOW..]),
|
|
141
|
+
);
|
|
142
|
+
let _ = std::fs::remove_dir_all(&root);
|
|
143
|
+
// Retained history grew six-fold between the windows; the cost of one conversion may
|
|
144
|
+
// not follow it. Before the fix the last window cost several times the first.
|
|
145
|
+
assert!(
|
|
146
|
+
last.as_secs_f64() < 2.0 * first.as_secs_f64(),
|
|
147
|
+
"conversion cost grew with retained history: first {first:?}, last {last:?}"
|
|
148
|
+
);
|
|
149
|
+
}
|
|
@@ -333,40 +333,23 @@ fn a_digest_receipt_names_the_declaration_the_intent_bound() {
|
|
|
333
333
|
for (index, key) in ["weight_0", "weight_1", "weight_2"].into_iter().enumerate() {
|
|
334
334
|
add(&store, &meta, &id, &writer, key, index as u8 + 1).unwrap();
|
|
335
335
|
}
|
|
336
|
-
let receipt = commit(&store, &meta, &id, 1).unwrap();
|
|
337
|
-
let
|
|
338
|
-
|
|
339
|
-
assert_eq!(inline, receipt.to_value());
|
|
340
|
-
let field = |value: &Value, name: &str| {
|
|
341
|
-
as_obj("receipt", "facts", value)
|
|
336
|
+
let receipt = commit(&store, &meta, &id, 1).unwrap().to_value();
|
|
337
|
+
let field = |name: &str| {
|
|
338
|
+
as_obj("receipt", "facts", &receipt)
|
|
342
339
|
.unwrap()
|
|
343
340
|
.iter()
|
|
344
341
|
.find(|(key, _)| key == name)
|
|
345
342
|
.map(|(_, value)| value.clone())
|
|
346
343
|
};
|
|
344
|
+
assert!(field("declaration").is_none());
|
|
347
345
|
assert_eq!(
|
|
348
|
-
|
|
349
|
-
intent
|
|
350
|
-
);
|
|
351
|
-
assert!(field(&digest, "declaration").is_none());
|
|
352
|
-
assert_eq!(
|
|
353
|
-
field(&digest, "declaration_digest"),
|
|
346
|
+
field("declaration_digest"),
|
|
354
347
|
Some(Value::str(ObjectRef::of(&intent).id()))
|
|
355
348
|
);
|
|
356
349
|
assert_eq!(
|
|
357
|
-
field(
|
|
350
|
+
field("declaration_length"),
|
|
358
351
|
Some(Value::uint(intent.len() as u64))
|
|
359
352
|
);
|
|
360
|
-
// Every other fact is identical in both forms.
|
|
361
|
-
let others = |value: &Value| {
|
|
362
|
-
as_obj("receipt", "facts", value)
|
|
363
|
-
.unwrap()
|
|
364
|
-
.iter()
|
|
365
|
-
.filter(|(key, _)| !key.starts_with("declaration"))
|
|
366
|
-
.cloned()
|
|
367
|
-
.collect::<Vec<_>>()
|
|
368
|
-
};
|
|
369
|
-
assert_eq!(others(&inline), others(&digest));
|
|
370
353
|
let _ = fence(&meta, &id, 1);
|
|
371
354
|
release_guards(&meta, Some(writer.writer_hold), writer.source_leases);
|
|
372
355
|
std::fs::remove_dir_all(store.root()).unwrap();
|
|
@@ -5,8 +5,7 @@ use std::io::{Read, Write};
|
|
|
5
5
|
use std::path::PathBuf;
|
|
6
6
|
|
|
7
7
|
use super::{
|
|
8
|
-
checkpoint, receipt_from, require_payload_record, roots, Disposition,
|
|
9
|
-
TransactionState,
|
|
8
|
+
checkpoint, receipt_from, require_payload_record, roots, Disposition, TransactionState,
|
|
10
9
|
};
|
|
11
10
|
use crate::canon::{Fields, Value};
|
|
12
11
|
use crate::catalog::WriterGuard;
|
|
@@ -218,12 +217,7 @@ pub fn retain_result(
|
|
|
218
217
|
);
|
|
219
218
|
}
|
|
220
219
|
let facts = receipt_from(store, row)?;
|
|
221
|
-
|
|
222
|
-
let forms = [ReceiptDeclaration::Inline, ReceiptDeclaration::Digest];
|
|
223
|
-
if !forms
|
|
224
|
-
.iter()
|
|
225
|
-
.any(|form| object_id(&crate::canon::write(&facts.value(*form))) == receipt)
|
|
226
|
-
{
|
|
220
|
+
if object_id(&crate::canon::write(&facts.to_value())) != receipt {
|
|
227
221
|
return refuse(
|
|
228
222
|
Code::OBJECT_ID_MISMATCH,
|
|
229
223
|
"retained result receipt differs from committed computation",
|
|
@@ -376,20 +370,9 @@ mod tests {
|
|
|
376
370
|
}
|
|
377
371
|
|
|
378
372
|
#[test]
|
|
379
|
-
fn
|
|
373
|
+
fn a_result_is_retained_under_its_receipt_and_nothing_else() {
|
|
380
374
|
let f = fixture();
|
|
381
|
-
|
|
382
|
-
let derived::Lookup::Committed(committed) = facts else {
|
|
383
|
-
panic!("committed result expected")
|
|
384
|
-
};
|
|
385
|
-
let digest_form = object_id(&crate::canon::write(
|
|
386
|
-
&committed.receipt().value(ReceiptDeclaration::Digest),
|
|
387
|
-
));
|
|
388
|
-
assert_ne!(digest_form, f.receipt);
|
|
389
|
-
let retained =
|
|
390
|
-
retain_result(&f.store, &f.meta, &f.transaction, &digest_form, &id("b1")).unwrap();
|
|
391
|
-
assert_eq!(retained.receipt_digest, digest_form);
|
|
392
|
-
retain_result(&f.store, &f.meta, &f.transaction, &f.receipt, &id("b2")).unwrap();
|
|
375
|
+
retain_result(&f.store, &f.meta, &f.transaction, &f.receipt, &id("b1")).unwrap();
|
|
393
376
|
assert_eq!(
|
|
394
377
|
retain_result(&f.store, &f.meta, &f.transaction, &id("99"), &id("b3"))
|
|
395
378
|
.unwrap_err()
|
|
@@ -1690,22 +1690,21 @@ fn require_payload_record(store: &Store, object: &ObjectRef, what: &str) -> Resu
|
|
|
1690
1690
|
}
|
|
1691
1691
|
|
|
1692
1692
|
fn require_source_roots(store: &Store, declaration: &Declaration) -> Result<()> {
|
|
1693
|
-
|
|
1694
|
-
|
|
1695
|
-
|
|
1696
|
-
|
|
1697
|
-
|
|
1698
|
-
|
|
1699
|
-
|
|
1700
|
-
|
|
1701
|
-
|
|
1702
|
-
|
|
1703
|
-
|
|
1704
|
-
|
|
1705
|
-
|
|
1706
|
-
|
|
1693
|
+
let wanted: Vec<ObjectRef> = declaration
|
|
1694
|
+
.sources
|
|
1695
|
+
.iter()
|
|
1696
|
+
.map(|source| source.manifest.clone())
|
|
1697
|
+
.collect();
|
|
1698
|
+
match crate::gc::unretained_manifests(store, &wanted)?.first() {
|
|
1699
|
+
Some(missing) => refuse(
|
|
1700
|
+
Code::ROOT_ABSENT,
|
|
1701
|
+
format!(
|
|
1702
|
+
"source {} has no exact retained filesystem root",
|
|
1703
|
+
missing.id()
|
|
1704
|
+
),
|
|
1705
|
+
),
|
|
1706
|
+
None => Ok(()),
|
|
1707
1707
|
}
|
|
1708
|
-
Ok(())
|
|
1709
1708
|
}
|
|
1710
1709
|
|
|
1711
1710
|
fn load_sources(store: &Store, declaration: &Declaration) -> Result<Vec<SourceData>> {
|
|
@@ -3126,6 +3125,22 @@ pub(crate) fn retain_restore(
|
|
|
3126
3125
|
roots::write(store, &root)
|
|
3127
3126
|
}
|
|
3128
3127
|
|
|
3128
|
+
/// The manifests private roots name themselves, without walking their checkpoint chains.
|
|
3129
|
+
pub(crate) fn rooted_manifests(store: &Store) -> Result<Vec<ObjectRef>> {
|
|
3130
|
+
let mut named: Vec<ObjectRef> = retention::held_objects(store)?
|
|
3131
|
+
.into_iter()
|
|
3132
|
+
.map(|hold| ObjectRef {
|
|
3133
|
+
sha256: hold.sha256,
|
|
3134
|
+
length: hold.length,
|
|
3135
|
+
})
|
|
3136
|
+
.collect();
|
|
3137
|
+
for root in roots::all(store)? {
|
|
3138
|
+
named.extend(root.sources);
|
|
3139
|
+
named.extend(root.manifest);
|
|
3140
|
+
}
|
|
3141
|
+
Ok(named)
|
|
3142
|
+
}
|
|
3143
|
+
|
|
3129
3144
|
pub(crate) fn retained_objects(store: &Store) -> Result<Vec<crate::storage::HeldKey>> {
|
|
3130
3145
|
let mut holds = retention::held_objects(store)?;
|
|
3131
3146
|
for object in roots::journaled(store)? {
|
|
@@ -3482,44 +3497,29 @@ pub struct ReceiptFacts {
|
|
|
3482
3497
|
pub inherited_payload_bytes: u64,
|
|
3483
3498
|
}
|
|
3484
3499
|
|
|
3485
|
-
/// How a receipt names its declaration: the whole document, or its digest and length.
|
|
3486
|
-
/// The whole document grows with the tensors declared; a digest keeps receipts constant.
|
|
3487
|
-
#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
|
|
3488
|
-
pub enum ReceiptDeclaration {
|
|
3489
|
-
#[default]
|
|
3490
|
-
Inline,
|
|
3491
|
-
Digest,
|
|
3492
|
-
}
|
|
3493
|
-
|
|
3494
3500
|
impl ReceiptFacts {
|
|
3501
|
+
/// The receipt names its declaration by digest and length: the declaration grows with the
|
|
3502
|
+
/// tensors declared, and the receipt stays constant.
|
|
3495
3503
|
pub fn to_value(&self) -> Value {
|
|
3496
|
-
self.
|
|
3497
|
-
|
|
3498
|
-
|
|
3499
|
-
|
|
3500
|
-
|
|
3501
|
-
|
|
3502
|
-
|
|
3503
|
-
|
|
3504
|
-
|
|
3505
|
-
|
|
3506
|
-
|
|
3507
|
-
Value::str(ObjectRef::of(&bytes).id()),
|
|
3508
|
-
));
|
|
3509
|
-
fields.push(("declaration_length".into(), Value::uint(bytes.len() as u64)));
|
|
3510
|
-
fields.sort_by(|left, right| left.0.cmp(&right.0));
|
|
3511
|
-
}
|
|
3512
|
-
}
|
|
3513
|
-
value
|
|
3504
|
+
let bytes = crate::canon::write(&self.declaration.to_value());
|
|
3505
|
+
let Value::Obj(mut fields) = self.facts() else {
|
|
3506
|
+
unreachable!("receipt facts are an object")
|
|
3507
|
+
};
|
|
3508
|
+
fields.push((
|
|
3509
|
+
"declaration_digest".into(),
|
|
3510
|
+
Value::str(ObjectRef::of(&bytes).id()),
|
|
3511
|
+
));
|
|
3512
|
+
fields.push(("declaration_length".into(), Value::uint(bytes.len() as u64)));
|
|
3513
|
+
fields.sort_by(|left, right| left.0.cmp(&right.0));
|
|
3514
|
+
Value::Obj(fields)
|
|
3514
3515
|
}
|
|
3515
3516
|
|
|
3516
|
-
fn
|
|
3517
|
+
fn facts(&self) -> Value {
|
|
3517
3518
|
Value::obj(vec![
|
|
3518
3519
|
(
|
|
3519
3520
|
"added_objects",
|
|
3520
3521
|
Value::arr(self.added_objects.iter().map(ObjectRef::to_value).collect()),
|
|
3521
3522
|
),
|
|
3522
|
-
("declaration", self.declaration.to_value()),
|
|
3523
3523
|
("header", self.header.to_value()),
|
|
3524
3524
|
(
|
|
3525
3525
|
"inherit_observation",
|
|
@@ -4666,3 +4666,6 @@ mod part_rows_tests;
|
|
|
4666
4666
|
|
|
4667
4667
|
#[cfg(test)]
|
|
4668
4668
|
mod group_commit_tests;
|
|
4669
|
+
|
|
4670
|
+
#[cfg(test)]
|
|
4671
|
+
mod history_cost_tests;
|
|
@@ -29,6 +29,7 @@ use std::path::Path;
|
|
|
29
29
|
|
|
30
30
|
use crate::catalog::{Catalog, WriterGuard};
|
|
31
31
|
use crate::err::{Code, Refusal, Result};
|
|
32
|
+
use crate::ids::ObjectRef;
|
|
32
33
|
use crate::ingest::transaction as tx;
|
|
33
34
|
use crate::storage::{self, Census, GcDelete, HeldKey};
|
|
34
35
|
use crate::store::Store;
|
|
@@ -158,30 +159,48 @@ pub fn session_holds(store: &Store) -> Result<(Vec<HeldKey>, Vec<String>)> {
|
|
|
158
159
|
Ok((holds, sessions))
|
|
159
160
|
}
|
|
160
161
|
|
|
161
|
-
/// The
|
|
162
|
-
///
|
|
163
|
-
///
|
|
164
|
-
|
|
165
|
-
|
|
166
|
-
|
|
167
|
-
|
|
168
|
-
|
|
169
|
-
.checkpoints
|
|
170
|
-
.into_iter()
|
|
171
|
-
.map(|checkpoint| checkpoint.manifest),
|
|
172
|
-
);
|
|
173
|
-
}
|
|
174
|
-
let (holds, _) = session_holds(store)?;
|
|
175
|
-
roots.extend(
|
|
162
|
+
/// The `wanted` manifests that no root reclamation keeps names. Roots are read cheapest
|
|
163
|
+
/// first and the search stops once every wanted manifest is found, so a rooted source costs
|
|
164
|
+
/// the roots that name it rather than every checkpoint chain and parts journal the Store
|
|
165
|
+
/// has retained, which grow with its history (finding 50). Membership is retention, not
|
|
166
|
+
/// caller authorization. A consumer must acquire its live hold before checking roots so
|
|
167
|
+
/// GC cannot remove bytes during admission.
|
|
168
|
+
pub(crate) fn unretained_manifests(store: &Store, wanted: &[ObjectRef]) -> Result<Vec<ObjectRef>> {
|
|
169
|
+
let manifests = |holds: Vec<HeldKey>| {
|
|
176
170
|
holds
|
|
177
171
|
.into_iter()
|
|
178
|
-
.filter(
|
|
179
|
-
.map(|hold|
|
|
172
|
+
.filter(HeldKey::is_manifest)
|
|
173
|
+
.map(|hold| ObjectRef {
|
|
180
174
|
sha256: hold.sha256,
|
|
181
175
|
length: hold.length,
|
|
182
|
-
})
|
|
183
|
-
|
|
184
|
-
|
|
176
|
+
})
|
|
177
|
+
};
|
|
178
|
+
let repositories = || -> Result<Vec<ObjectRef>> {
|
|
179
|
+
Ok(storage::read_repositories(store.root())?
|
|
180
|
+
.into_iter()
|
|
181
|
+
.flat_map(|(repository, _)| repository.checkpoints)
|
|
182
|
+
.map(|checkpoint| checkpoint.manifest)
|
|
183
|
+
.collect())
|
|
184
|
+
};
|
|
185
|
+
let named_roots = || -> Result<Vec<ObjectRef>> {
|
|
186
|
+
let mut named = crate::derived::rooted_manifests(store)?;
|
|
187
|
+
named.extend(manifests(crate::source_artifact::held_objects(store)?));
|
|
188
|
+
named.extend(manifests(crate::checkpoint_root::held_objects(store)?));
|
|
189
|
+
Ok(named)
|
|
190
|
+
};
|
|
191
|
+
let everything =
|
|
192
|
+
|| -> Result<Vec<ObjectRef>> { Ok(manifests(session_holds(store)?.0).collect()) };
|
|
193
|
+
let passes: [&dyn Fn() -> Result<Vec<ObjectRef>>; 3] =
|
|
194
|
+
[&repositories, &named_roots, &everything];
|
|
195
|
+
let mut missing = wanted.to_vec();
|
|
196
|
+
for pass in passes {
|
|
197
|
+
if missing.is_empty() {
|
|
198
|
+
break;
|
|
199
|
+
}
|
|
200
|
+
let named = pass()?;
|
|
201
|
+
missing.retain(|manifest| !named.contains(manifest));
|
|
202
|
+
}
|
|
203
|
+
Ok(missing)
|
|
185
204
|
}
|
|
186
205
|
|
|
187
206
|
/// One complete pass: exclusivity, orphan temps, census, plan, delete, catalog repair.
|
|
@@ -9,22 +9,12 @@
|
|
|
9
9
|
//! (tfs-053). Creation stays with the verbs that say so, `tfs store init` and `tfs store ensure`,
|
|
10
10
|
//! over a root that was named.
|
|
11
11
|
|
|
12
|
-
use std::fs;
|
|
13
|
-
use std::os::unix::fs::MetadataExt;
|
|
14
12
|
use std::path::{Path, PathBuf};
|
|
15
13
|
|
|
16
|
-
use crate::catalog::WriterGuard;
|
|
17
14
|
use crate::err::{refuse, Code, Result};
|
|
18
|
-
use crate::store::{self, Store};
|
|
19
15
|
|
|
20
16
|
pub const ENV: &str = "TENSORFS_HOME";
|
|
21
17
|
|
|
22
|
-
#[derive(Debug, Clone, PartialEq, Eq)]
|
|
23
|
-
pub struct MoveReport {
|
|
24
|
-
pub source: PathBuf,
|
|
25
|
-
pub destination: PathBuf,
|
|
26
|
-
}
|
|
27
|
-
|
|
28
18
|
/// Where a resolved root came from. `Named` roots were asked for; `Fallback` is a guess.
|
|
29
19
|
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
|
30
20
|
enum Origin {
|
|
@@ -98,98 +88,6 @@ fn nonempty(source: &str, root: &Path) -> Result<PathBuf> {
|
|
|
98
88
|
Ok(root.to_path_buf())
|
|
99
89
|
}
|
|
100
90
|
|
|
101
|
-
/// Move one stopped Store into the resolved TensorFS home without copying or rehashing it.
|
|
102
|
-
///
|
|
103
|
-
/// The destination must not exist and both parents must be on the same filesystem. The Store's
|
|
104
|
-
/// recovery lock refuses already-active readers and writers. This is still an offline operator
|
|
105
|
-
/// command: possible Store openers must be stopped so none can recreate the retired pathname
|
|
106
|
-
/// after the rename. No redirect or reservation is left at the old path.
|
|
107
|
-
pub fn move_store(source: &Path, explicit_destination: Option<&Path>) -> Result<MoveReport> {
|
|
108
|
-
let source_metadata = fs::symlink_metadata(source).map_err(|error| io("stat source", error))?;
|
|
109
|
-
if source_metadata.file_type().is_symlink() || !source_metadata.is_dir() {
|
|
110
|
-
return refuse(
|
|
111
|
-
Code::STORE_ERA,
|
|
112
|
-
format!("{} is not a TensorFS Store directory", source.display()),
|
|
113
|
-
);
|
|
114
|
-
}
|
|
115
|
-
let source = fs::canonicalize(source).map_err(|error| io("resolve source", error))?;
|
|
116
|
-
Store::open(&source)?;
|
|
117
|
-
|
|
118
|
-
let requested_destination = resolve(explicit_destination)?;
|
|
119
|
-
match fs::symlink_metadata(&requested_destination) {
|
|
120
|
-
Ok(_) => {
|
|
121
|
-
return refuse(
|
|
122
|
-
Code::IO_FAILED,
|
|
123
|
-
format!(
|
|
124
|
-
"destination {} already exists; remove the conflict or choose another root",
|
|
125
|
-
requested_destination.display()
|
|
126
|
-
),
|
|
127
|
-
)
|
|
128
|
-
}
|
|
129
|
-
Err(error) if error.kind() == std::io::ErrorKind::NotFound => {}
|
|
130
|
-
Err(error) => return Err(io("stat destination", error)),
|
|
131
|
-
}
|
|
132
|
-
let destination_name =
|
|
133
|
-
requested_destination
|
|
134
|
-
.file_name()
|
|
135
|
-
.ok_or_else(|| crate::err::Refusal {
|
|
136
|
-
code: Code::IO_FAILED,
|
|
137
|
-
detail: format!(
|
|
138
|
-
"destination {} has no final path component",
|
|
139
|
-
requested_destination.display()
|
|
140
|
-
),
|
|
141
|
-
})?;
|
|
142
|
-
let destination_parent = requested_destination
|
|
143
|
-
.parent()
|
|
144
|
-
.filter(|path| !path.as_os_str().is_empty())
|
|
145
|
-
.unwrap_or_else(|| Path::new("."));
|
|
146
|
-
let destination_parent = fs::canonicalize(destination_parent)
|
|
147
|
-
.map_err(|error| io("resolve destination parent", error))?;
|
|
148
|
-
let destination = destination_parent.join(destination_name);
|
|
149
|
-
if source == destination {
|
|
150
|
-
return refuse(Code::IO_FAILED, "source and destination are the same Store");
|
|
151
|
-
}
|
|
152
|
-
let source_parent = source.parent().ok_or_else(|| crate::err::Refusal {
|
|
153
|
-
code: Code::IO_FAILED,
|
|
154
|
-
detail: format!("source {} has no parent", source.display()),
|
|
155
|
-
})?;
|
|
156
|
-
let source_device = fs::metadata(&source)
|
|
157
|
-
.map_err(|error| io("stat source", error))?
|
|
158
|
-
.dev();
|
|
159
|
-
let destination_device = fs::metadata(&destination_parent)
|
|
160
|
-
.map_err(|error| io("stat destination parent", error))?
|
|
161
|
-
.dev();
|
|
162
|
-
if source_device != destination_device {
|
|
163
|
-
return refuse(
|
|
164
|
-
Code::IO_FAILED,
|
|
165
|
-
format!(
|
|
166
|
-
"source {} and destination {} are on different filesystems; a hardcut requires an atomic rename",
|
|
167
|
-
source.display(),
|
|
168
|
-
destination.display()
|
|
169
|
-
),
|
|
170
|
-
);
|
|
171
|
-
}
|
|
172
|
-
|
|
173
|
-
let _exclusive = WriterGuard::lock_rebuild(&source)?;
|
|
174
|
-
fs::rename(&source, &destination).map_err(|error| io("rename Store", error))?;
|
|
175
|
-
store::fsync_dir(&destination_parent)?;
|
|
176
|
-
if source_parent != destination_parent {
|
|
177
|
-
store::fsync_dir(source_parent)?;
|
|
178
|
-
}
|
|
179
|
-
Store::open(&destination)?;
|
|
180
|
-
Ok(MoveReport {
|
|
181
|
-
source,
|
|
182
|
-
destination,
|
|
183
|
-
})
|
|
184
|
-
}
|
|
185
|
-
|
|
186
|
-
fn io(what: &str, error: std::io::Error) -> crate::err::Refusal {
|
|
187
|
-
crate::err::Refusal {
|
|
188
|
-
code: Code::IO_FAILED,
|
|
189
|
-
detail: format!("{what}: {error}"),
|
|
190
|
-
}
|
|
191
|
-
}
|
|
192
|
-
|
|
193
91
|
#[cfg(test)]
|
|
194
92
|
mod tests {
|
|
195
93
|
use super::*;
|