tensorfs 0.3.58__tar.gz → 0.3.60__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {tensorfs-0.3.58 → tensorfs-0.3.60}/Cargo.lock +2 -2
- {tensorfs-0.3.58 → tensorfs-0.3.60}/Cargo.toml +1 -1
- {tensorfs-0.3.58 → tensorfs-0.3.60}/PKG-INFO +1 -1
- {tensorfs-0.3.58 → tensorfs-0.3.60}/crates/tensorfs-core/src/bin/tfs/main.rs +10 -2
- {tensorfs-0.3.58 → tensorfs-0.3.60}/crates/tensorfs-core/src/catalog.rs +17 -1
- {tensorfs-0.3.58 → tensorfs-0.3.60}/crates/tensorfs-core/src/checkpoint.rs +22 -0
- {tensorfs-0.3.58 → tensorfs-0.3.60}/crates/tensorfs-core/src/derived/resume_tests.rs +23 -16
- {tensorfs-0.3.58 → tensorfs-0.3.60}/crates/tensorfs-core/src/derived.rs +45 -7
- {tensorfs-0.3.58 → tensorfs-0.3.60}/crates/tensorfs-core/src/durability.rs +17 -0
- {tensorfs-0.3.58 → tensorfs-0.3.60}/crates/tensorfs-core/src/gc.rs +5 -1
- tensorfs-0.3.60/crates/tensorfs-core/src/ingest/custody.rs +249 -0
- {tensorfs-0.3.58 → tensorfs-0.3.60}/crates/tensorfs-core/src/ingest/journal.rs +13 -0
- {tensorfs-0.3.58 → tensorfs-0.3.60}/crates/tensorfs-core/src/ingest/mod.rs +1 -0
- {tensorfs-0.3.58 → tensorfs-0.3.60}/crates/tensorfs-core/src/ingest/source.rs +385 -100
- {tensorfs-0.3.58 → tensorfs-0.3.60}/crates/tensorfs-core/src/ingest/transaction.rs +38 -7
- {tensorfs-0.3.58 → tensorfs-0.3.60}/crates/tensorfs-core/src/storage.rs +20 -10
- {tensorfs-0.3.58 → tensorfs-0.3.60}/crates/tensorfs-core/src/transport/http.rs +13 -2
- tensorfs-0.3.60/crates/tensorfs-core/src/transport/memory.rs +92 -0
- {tensorfs-0.3.58 → tensorfs-0.3.60}/crates/tensorfs-core/src/transport/mod.rs +2 -0
- {tensorfs-0.3.58 → tensorfs-0.3.60}/crates/tensorfs-core/src/transport/prefix.rs +1 -1
- {tensorfs-0.3.58 → tensorfs-0.3.60}/crates/tensorfs-core/src/transport/pull.rs +1 -1
- {tensorfs-0.3.58 → tensorfs-0.3.60}/crates/tensorfs-core/src/transport/push.rs +9 -9
- {tensorfs-0.3.58 → tensorfs-0.3.60}/crates/tensorfs-core/tests/source_download.rs +256 -5
- {tensorfs-0.3.58 → tensorfs-0.3.60}/crates/tensorfs-core/tests/write_through.rs +15 -14
- tensorfs-0.3.60/crates/tensorfs-py/src/ingest.rs +223 -0
- {tensorfs-0.3.58 → tensorfs-0.3.60}/crates/tensorfs-py/src/lib.rs +7 -0
- {tensorfs-0.3.58 → tensorfs-0.3.60}/crates/tensorfs-py/src/store.rs +52 -15
- {tensorfs-0.3.58 → tensorfs-0.3.60}/pyproject.toml +1 -1
- {tensorfs-0.3.58 → tensorfs-0.3.60}/python/tensorfs/__init__.py +12 -0
- {tensorfs-0.3.58 → tensorfs-0.3.60}/python/tensorfs/_ext.pyi +33 -0
- {tensorfs-0.3.58 → tensorfs-0.3.60}/crates/tensorfs-core/Cargo.toml +0 -0
- {tensorfs-0.3.58 → tensorfs-0.3.60}/crates/tensorfs-core/build.rs +0 -0
- {tensorfs-0.3.58 → tensorfs-0.3.60}/crates/tensorfs-core/examples/h3-swiglu-layout.md +0 -0
- {tensorfs-0.3.58 → tensorfs-0.3.60}/crates/tensorfs-core/examples/optional-cache-backfill.md +0 -0
- {tensorfs-0.3.58 → tensorfs-0.3.60}/crates/tensorfs-core/examples/resident-conversion.md +0 -0
- {tensorfs-0.3.58 → tensorfs-0.3.60}/crates/tensorfs-core/examples/resident_conversion_profile.rs +0 -0
- {tensorfs-0.3.58 → tensorfs-0.3.60}/crates/tensorfs-core/src/asset_update.rs +0 -0
- {tensorfs-0.3.58 → tensorfs-0.3.60}/crates/tensorfs-core/src/b64.rs +0 -0
- {tensorfs-0.3.58 → tensorfs-0.3.60}/crates/tensorfs-core/src/bin/tfs/ckpt.rs +0 -0
- {tensorfs-0.3.58 → tensorfs-0.3.60}/crates/tensorfs-core/src/bin/tfs/conform.rs +0 -0
- {tensorfs-0.3.58 → tensorfs-0.3.60}/crates/tensorfs-core/src/bin/tfs/fetch.rs +0 -0
- {tensorfs-0.3.58 → tensorfs-0.3.60}/crates/tensorfs-core/src/bin/tfs/ingest.rs +0 -0
- {tensorfs-0.3.58 → tensorfs-0.3.60}/crates/tensorfs-core/src/bin/tfs/rd.rs +0 -0
- {tensorfs-0.3.58 → tensorfs-0.3.60}/crates/tensorfs-core/src/bin/tfs/repo.rs +0 -0
- {tensorfs-0.3.58 → tensorfs-0.3.60}/crates/tensorfs-core/src/bin/tfs/snap.rs +0 -0
- {tensorfs-0.3.58 → tensorfs-0.3.60}/crates/tensorfs-core/src/bin/tfs/watch.rs +0 -0
- {tensorfs-0.3.58 → tensorfs-0.3.60}/crates/tensorfs-core/src/cache_roots.rs +0 -0
- {tensorfs-0.3.58 → tensorfs-0.3.60}/crates/tensorfs-core/src/canon.rs +0 -0
- {tensorfs-0.3.58 → tensorfs-0.3.60}/crates/tensorfs-core/src/capability.rs +0 -0
- {tensorfs-0.3.58 → tensorfs-0.3.60}/crates/tensorfs-core/src/cbor.rs +0 -0
- {tensorfs-0.3.58 → tensorfs-0.3.60}/crates/tensorfs-core/src/checkpoint_root.rs +0 -0
- {tensorfs-0.3.58 → tensorfs-0.3.60}/crates/tensorfs-core/src/corpus.rs +0 -0
- {tensorfs-0.3.58 → tensorfs-0.3.60}/crates/tensorfs-core/src/derived/adoption.rs +0 -0
- {tensorfs-0.3.58 → tensorfs-0.3.60}/crates/tensorfs-core/src/derived/companion_tests.rs +0 -0
- {tensorfs-0.3.58 → tensorfs-0.3.60}/crates/tensorfs-core/src/derived/estimate.rs +0 -0
- {tensorfs-0.3.58 → tensorfs-0.3.60}/crates/tensorfs-core/src/derived/graft_tests.rs +0 -0
- {tensorfs-0.3.58 → tensorfs-0.3.60}/crates/tensorfs-core/src/derived/progress.rs +0 -0
- {tensorfs-0.3.58 → tensorfs-0.3.60}/crates/tensorfs-core/src/derived/retention.rs +0 -0
- {tensorfs-0.3.58 → tensorfs-0.3.60}/crates/tensorfs-core/src/derived/roots.rs +0 -0
- {tensorfs-0.3.58 → tensorfs-0.3.60}/crates/tensorfs-core/src/dtype.rs +0 -0
- {tensorfs-0.3.58 → tensorfs-0.3.60}/crates/tensorfs-core/src/err.rs +0 -0
- {tensorfs-0.3.58 → tensorfs-0.3.60}/crates/tensorfs-core/src/fetch.rs +0 -0
- {tensorfs-0.3.58 → tensorfs-0.3.60}/crates/tensorfs-core/src/fill.rs +0 -0
- {tensorfs-0.3.58 → tensorfs-0.3.60}/crates/tensorfs-core/src/fit.rs +0 -0
- {tensorfs-0.3.58 → tensorfs-0.3.60}/crates/tensorfs-core/src/header.rs +0 -0
- {tensorfs-0.3.58 → tensorfs-0.3.60}/crates/tensorfs-core/src/home.rs +0 -0
- {tensorfs-0.3.58 → tensorfs-0.3.60}/crates/tensorfs-core/src/ids.rs +0 -0
- {tensorfs-0.3.58 → tensorfs-0.3.60}/crates/tensorfs-core/src/ingest/carrier.rs +0 -0
- {tensorfs-0.3.58 → tensorfs-0.3.60}/crates/tensorfs-core/src/ingest/convert.rs +0 -0
- {tensorfs-0.3.58 → tensorfs-0.3.60}/crates/tensorfs-core/src/ingest/fingerprint.rs +0 -0
- {tensorfs-0.3.58 → tensorfs-0.3.60}/crates/tensorfs-core/src/ingest/gguf.rs +0 -0
- {tensorfs-0.3.58 → tensorfs-0.3.60}/crates/tensorfs-core/src/ingest/h3_lora.rs +0 -0
- {tensorfs-0.3.58 → tensorfs-0.3.60}/crates/tensorfs-core/src/ingest/preflight.rs +0 -0
- {tensorfs-0.3.58 → tensorfs-0.3.60}/crates/tensorfs-core/src/ingest/stamp.rs +0 -0
- {tensorfs-0.3.58 → tensorfs-0.3.60}/crates/tensorfs-core/src/jcs.rs +0 -0
- {tensorfs-0.3.58 → tensorfs-0.3.60}/crates/tensorfs-core/src/lib.rs +0 -0
- {tensorfs-0.3.58 → tensorfs-0.3.60}/crates/tensorfs-core/src/limits.rs +0 -0
- {tensorfs-0.3.58 → tensorfs-0.3.60}/crates/tensorfs-core/src/machine.rs +0 -0
- {tensorfs-0.3.58 → tensorfs-0.3.60}/crates/tensorfs-core/src/manifest.rs +0 -0
- {tensorfs-0.3.58 → tensorfs-0.3.60}/crates/tensorfs-core/src/meta.rs +0 -0
- {tensorfs-0.3.58 → tensorfs-0.3.60}/crates/tensorfs-core/src/project.rs +0 -0
- {tensorfs-0.3.58 → tensorfs-0.3.60}/crates/tensorfs-core/src/providers.rs +0 -0
- {tensorfs-0.3.58 → tensorfs-0.3.60}/crates/tensorfs-core/src/read.rs +0 -0
- {tensorfs-0.3.58 → tensorfs-0.3.60}/crates/tensorfs-core/src/receipt.rs +0 -0
- {tensorfs-0.3.58 → tensorfs-0.3.60}/crates/tensorfs-core/src/reclaim.rs +0 -0
- {tensorfs-0.3.58 → tensorfs-0.3.60}/crates/tensorfs-core/src/registry.rs +0 -0
- {tensorfs-0.3.58 → tensorfs-0.3.60}/crates/tensorfs-core/src/relation.rs +0 -0
- {tensorfs-0.3.58 → tensorfs-0.3.60}/crates/tensorfs-core/src/repo_cache/identity.rs +0 -0
- {tensorfs-0.3.58 → tensorfs-0.3.60}/crates/tensorfs-core/src/repo_cache.rs +0 -0
- {tensorfs-0.3.58 → tensorfs-0.3.60}/crates/tensorfs-core/src/repository.rs +0 -0
- {tensorfs-0.3.58 → tensorfs-0.3.60}/crates/tensorfs-core/src/sha256.rs +0 -0
- {tensorfs-0.3.58 → tensorfs-0.3.60}/crates/tensorfs-core/src/source_artifact.rs +0 -0
- {tensorfs-0.3.58 → tensorfs-0.3.60}/crates/tensorfs-core/src/spec.rs +0 -0
- {tensorfs-0.3.58 → tensorfs-0.3.60}/crates/tensorfs-core/src/staging.rs +0 -0
- {tensorfs-0.3.58 → tensorfs-0.3.60}/crates/tensorfs-core/src/stats.rs +0 -0
- {tensorfs-0.3.58 → tensorfs-0.3.60}/crates/tensorfs-core/src/store.rs +0 -0
- {tensorfs-0.3.58 → tensorfs-0.3.60}/crates/tensorfs-core/src/transport/api.rs +0 -0
- {tensorfs-0.3.58 → tensorfs-0.3.60}/crates/tensorfs-core/src/transport/http_reuse_tests.rs +0 -0
- {tensorfs-0.3.58 → tensorfs-0.3.60}/crates/tensorfs-core/src/transport/hub.rs +0 -0
- {tensorfs-0.3.58 → tensorfs-0.3.60}/crates/tensorfs-core/src/transport/ledger.rs +0 -0
- {tensorfs-0.3.58 → tensorfs-0.3.60}/crates/tensorfs-core/src/transport/policy.rs +0 -0
- {tensorfs-0.3.58 → tensorfs-0.3.60}/crates/tensorfs-core/src/transport/ranged.rs +0 -0
- {tensorfs-0.3.58 → tensorfs-0.3.60}/crates/tensorfs-core/src/transport/speculate.rs +0 -0
- {tensorfs-0.3.58 → tensorfs-0.3.60}/crates/tensorfs-core/src/transport/tests.rs +0 -0
- {tensorfs-0.3.58 → tensorfs-0.3.60}/crates/tensorfs-core/src/vectors.rs +0 -0
- {tensorfs-0.3.58 → tensorfs-0.3.60}/crates/tensorfs-core/tests/additive_fields.rs +0 -0
- {tensorfs-0.3.58 → tensorfs-0.3.60}/crates/tensorfs-core/tests/checkpoint_roots.rs +0 -0
- {tensorfs-0.3.58 → tensorfs-0.3.60}/crates/tensorfs-core/tests/fd_headroom.rs +0 -0
- {tensorfs-0.3.58 → tensorfs-0.3.60}/crates/tensorfs-core/tests/fetch_url_progress.rs +0 -0
- {tensorfs-0.3.58 → tensorfs-0.3.60}/crates/tensorfs-core/tests/fit.rs +0 -0
- {tensorfs-0.3.58 → tensorfs-0.3.60}/crates/tensorfs-core/tests/fixtures/repo_apply/blobs.jsonl +0 -0
- {tensorfs-0.3.58 → tensorfs-0.3.60}/crates/tensorfs-core/tests/fixtures/repo_apply/expected_list.jsonl +0 -0
- {tensorfs-0.3.58 → tensorfs-0.3.60}/crates/tensorfs-core/tests/fixtures/repo_apply/expected_repository.json +0 -0
- {tensorfs-0.3.58 → tensorfs-0.3.60}/crates/tensorfs-core/tests/fixtures/repo_apply/expected_rows.jsonl +0 -0
- {tensorfs-0.3.58 → tensorfs-0.3.60}/crates/tensorfs-core/tests/fixtures/repo_apply/header.cbor +0 -0
- {tensorfs-0.3.58 → tensorfs-0.3.60}/crates/tensorfs-core/tests/fixtures/repo_apply/manifest.json +0 -0
- {tensorfs-0.3.58 → tensorfs-0.3.60}/crates/tensorfs-core/tests/fixtures/repo_apply/mutation.json +0 -0
- {tensorfs-0.3.58 → tensorfs-0.3.60}/crates/tensorfs-core/tests/foreign_json.rs +0 -0
- {tensorfs-0.3.58 → tensorfs-0.3.60}/crates/tensorfs-core/tests/h3_lora.rs +0 -0
- {tensorfs-0.3.58 → tensorfs-0.3.60}/crates/tensorfs-core/tests/h3_swiglu.rs +0 -0
- {tensorfs-0.3.58 → tensorfs-0.3.60}/crates/tensorfs-core/tests/home.rs +0 -0
- {tensorfs-0.3.58 → tensorfs-0.3.60}/crates/tensorfs-core/tests/import_tree.rs +0 -0
- {tensorfs-0.3.58 → tensorfs-0.3.60}/crates/tensorfs-core/tests/instrument_profiles.rs +0 -0
- {tensorfs-0.3.58 → tensorfs-0.3.60}/crates/tensorfs-core/tests/liveness.rs +0 -0
- {tensorfs-0.3.58 → tensorfs-0.3.60}/crates/tensorfs-core/tests/qwen_image21_profile.rs +0 -0
- {tensorfs-0.3.58 → tensorfs-0.3.60}/crates/tensorfs-core/tests/read_plan.rs +0 -0
- {tensorfs-0.3.58 → tensorfs-0.3.60}/crates/tensorfs-core/tests/reclaim.rs +0 -0
- {tensorfs-0.3.58 → tensorfs-0.3.60}/crates/tensorfs-core/tests/reclaim_run.rs +0 -0
- {tensorfs-0.3.58 → tensorfs-0.3.60}/crates/tensorfs-core/tests/repo_cache.rs +0 -0
- {tensorfs-0.3.58 → tensorfs-0.3.60}/crates/tensorfs-core/tests/repo_store.rs +0 -0
- {tensorfs-0.3.58 → tensorfs-0.3.60}/crates/tensorfs-core/tests/resident_admission.rs +0 -0
- {tensorfs-0.3.58 → tensorfs-0.3.60}/crates/tensorfs-core/tests/sha256_backends.rs +0 -0
- {tensorfs-0.3.58 → tensorfs-0.3.60}/crates/tensorfs-core/tests/sharded_cas.rs +0 -0
- {tensorfs-0.3.58 → tensorfs-0.3.60}/crates/tensorfs-core/tests/source_artifacts.rs +0 -0
- {tensorfs-0.3.58 → tensorfs-0.3.60}/crates/tensorfs-core/tests/source_plan.rs +0 -0
- {tensorfs-0.3.58 → tensorfs-0.3.60}/crates/tensorfs-core/tests/source_prefixes.rs +0 -0
- {tensorfs-0.3.58 → tensorfs-0.3.60}/crates/tensorfs-core/tests/source_preflight.rs +0 -0
- {tensorfs-0.3.58 → tensorfs-0.3.60}/crates/tensorfs-core/tests/staging.rs +0 -0
- {tensorfs-0.3.58 → tensorfs-0.3.60}/crates/tensorfs-core/tests/store_boundaries.rs +0 -0
- {tensorfs-0.3.58 → tensorfs-0.3.60}/crates/tensorfs-core/tests/tensor_schema.rs +0 -0
- {tensorfs-0.3.58 → tensorfs-0.3.60}/crates/tensorfs-core/tests/transfer_memory.rs +0 -0
- {tensorfs-0.3.58 → tensorfs-0.3.60}/crates/tensorfs-core/tests/trust_index.rs +0 -0
- {tensorfs-0.3.58 → tensorfs-0.3.60}/crates/tensorfs-core/tests/version.rs +0 -0
- {tensorfs-0.3.58 → tensorfs-0.3.60}/crates/tensorfs-py/Cargo.toml +0 -0
- {tensorfs-0.3.58 → tensorfs-0.3.60}/crates/tensorfs-py/src/docs.rs +0 -0
- {tensorfs-0.3.58 → tensorfs-0.3.60}/crates/tensorfs-py/src/errors.rs +0 -0
- {tensorfs-0.3.58 → tensorfs-0.3.60}/python/tensorfs/_derived_channel.py +0 -0
- {tensorfs-0.3.58 → tensorfs-0.3.60}/python/tensorfs/_source_channel.py +0 -0
- {tensorfs-0.3.58 → tensorfs-0.3.60}/python/tensorfs/_tensors.py +0 -0
- {tensorfs-0.3.58 → tensorfs-0.3.60}/python/tensorfs/derived.py +0 -0
- {tensorfs-0.3.58 → tensorfs-0.3.60}/python/tensorfs/errors.py +0 -0
- {tensorfs-0.3.58 → tensorfs-0.3.60}/python/tensorfs/py.typed +0 -0
|
@@ -439,7 +439,7 @@ checksum = "adb6935a6f5c20170eeceb1a3835a49e12e19d792f6dd344ccc76a985ca5a6ca"
|
|
|
439
439
|
|
|
440
440
|
[[package]]
|
|
441
441
|
name = "tensorfs-core"
|
|
442
|
-
version = "0.3.
|
|
442
|
+
version = "0.3.60"
|
|
443
443
|
dependencies = [
|
|
444
444
|
"fs2",
|
|
445
445
|
"libc",
|
|
@@ -454,7 +454,7 @@ dependencies = [
|
|
|
454
454
|
|
|
455
455
|
[[package]]
|
|
456
456
|
name = "tensorfs-py"
|
|
457
|
-
version = "0.3.
|
|
457
|
+
version = "0.3.60"
|
|
458
458
|
dependencies = [
|
|
459
459
|
"pyo3",
|
|
460
460
|
"tensorfs-core",
|
|
@@ -505,7 +505,7 @@ fn cmd_repo_cache_restore(store: &Path, object: &str, length: &str, flags: &Flag
|
|
|
505
505
|
/// The chain head names links that live on the cache; each is read at its exact digest, and
|
|
506
506
|
/// the chain is DISCARDED WHOLE unless every link was written under the plan `--plan` names
|
|
507
507
|
/// — the plan a resuming run re-derives from the source headers alone, for zero tensor
|
|
508
|
-
/// bytes.
|
|
508
|
+
/// bytes. A discarded chain restores nothing and succeeds: the run converts afresh.
|
|
509
509
|
///
|
|
510
510
|
/// The objects come back exactly as a checkpoint's closure does. The conversion journal the
|
|
511
511
|
/// newest link points at is installed beside them under `--session`, so the next
|
|
@@ -548,7 +548,15 @@ fn cmd_repo_cache_resume(store: &Path, object: &str, length: &str, flags: &Flags
|
|
|
548
548
|
};
|
|
549
549
|
let set = match durability::fold(&chain, plan) {
|
|
550
550
|
Ok(set) => set,
|
|
551
|
-
Err(error) => return bail(error),
|
|
551
|
+
Err(error) if error.code == tensorfs_core::err::Code::IO_FAILED => return bail(error),
|
|
552
|
+
Err(error) => {
|
|
553
|
+
println!(
|
|
554
|
+
"chain DISCARDED {}: {} — nothing restored; the conversion starts a new chain",
|
|
555
|
+
error.code.as_str(),
|
|
556
|
+
error.detail
|
|
557
|
+
);
|
|
558
|
+
return ExitCode::SUCCESS;
|
|
559
|
+
}
|
|
552
560
|
};
|
|
553
561
|
println!(
|
|
554
562
|
"chain {} link(s) over operation {} — {} B declared durable",
|
|
@@ -911,9 +911,14 @@ impl Catalog {
|
|
|
911
911
|
object: &crate::ids::ObjectRef,
|
|
912
912
|
publish: impl FnOnce() -> Result<Option<crate::store::VRecord>>,
|
|
913
913
|
) -> Result<bool> {
|
|
914
|
+
// One admission at a time per catalog in this process, holding the write lock from
|
|
915
|
+
// before publication: streams landing objects together queue here instead of in
|
|
916
|
+
// SQLite's busy timeout, where a lost race refused bytes already verified.
|
|
917
|
+
let turn = admissions(&self.path);
|
|
918
|
+
let _turn = turn.lock().unwrap_or_else(|error| error.into_inner());
|
|
914
919
|
let mut connection = self.connection()?;
|
|
915
920
|
let transaction = connection
|
|
916
|
-
.
|
|
921
|
+
.transaction_with_behavior(rusqlite::TransactionBehavior::Immediate)
|
|
917
922
|
.map_err(|error| sql("begin blob admission", error))?;
|
|
918
923
|
if let Some(operation) = operation {
|
|
919
924
|
hold_on(
|
|
@@ -1079,6 +1084,17 @@ impl Catalog {
|
|
|
1079
1084
|
}
|
|
1080
1085
|
}
|
|
1081
1086
|
|
|
1087
|
+
fn admissions(path: &Path) -> Arc<Mutex<()>> {
|
|
1088
|
+
static TURNS: OnceLock<Mutex<HashMap<PathBuf, Arc<Mutex<()>>>>> = OnceLock::new();
|
|
1089
|
+
TURNS
|
|
1090
|
+
.get_or_init(Default::default)
|
|
1091
|
+
.lock()
|
|
1092
|
+
.unwrap_or_else(|error| error.into_inner())
|
|
1093
|
+
.entry(path.to_path_buf())
|
|
1094
|
+
.or_default()
|
|
1095
|
+
.clone()
|
|
1096
|
+
}
|
|
1097
|
+
|
|
1082
1098
|
fn hold_on(connection: &Connection, operation: &str, key: &str, length: u64) -> Result<()> {
|
|
1083
1099
|
let kind = if key.starts_with("blobs/") {
|
|
1084
1100
|
"blob"
|
|
@@ -418,6 +418,28 @@ impl Walk {
|
|
|
418
418
|
}
|
|
419
419
|
Ok(())
|
|
420
420
|
}
|
|
421
|
+
|
|
422
|
+
/// Every reached object present, or held by a live source operation's publication
|
|
423
|
+
/// custodian. A complete checkpoint for publication, never a readable one.
|
|
424
|
+
pub fn require_held(
|
|
425
|
+
&self,
|
|
426
|
+
store: &Store,
|
|
427
|
+
custodied: &std::collections::BTreeSet<String>,
|
|
428
|
+
) -> Result<()> {
|
|
429
|
+
for r in self.distinct() {
|
|
430
|
+
if !store.contains(&r.sha256) && !custodied.contains(&r.sha256) {
|
|
431
|
+
return refuse(
|
|
432
|
+
Code::OBJECT_ABSENT,
|
|
433
|
+
format!(
|
|
434
|
+
"{} is committed by this manifest but neither resident nor held by \
|
|
435
|
+
a live publication custodian",
|
|
436
|
+
r.id()
|
|
437
|
+
),
|
|
438
|
+
);
|
|
439
|
+
}
|
|
440
|
+
}
|
|
441
|
+
Ok(())
|
|
442
|
+
}
|
|
421
443
|
}
|
|
422
444
|
|
|
423
445
|
/// Manifest → CozyTensorsHeader → contained tensor rules → storage-part object runs.
|
|
@@ -351,7 +351,7 @@ fn missing_corrupt_and_changed_work_cannot_be_reused() {
|
|
|
351
351
|
}
|
|
352
352
|
|
|
353
353
|
#[test]
|
|
354
|
-
fn
|
|
354
|
+
fn recovery_discards_old_writer_authority_and_malformed_part_geometry() {
|
|
355
355
|
let (store, meta, id, declaration) = fixture("authority");
|
|
356
356
|
let writer = begin(&store, &meta, &id, 9, declaration.clone(), None).unwrap();
|
|
357
357
|
data(&store, &meta, &id, 9, 0x31);
|
|
@@ -359,7 +359,8 @@ fn recovery_refuses_old_writer_authority_and_malformed_part_geometry() {
|
|
|
359
359
|
let plan = progress_plan(&row);
|
|
360
360
|
let mut geometry = progress_row(&row);
|
|
361
361
|
geometry.added_parts[0].part.shape = vec![16, 31];
|
|
362
|
-
|
|
362
|
+
let mut writers = vec![writer];
|
|
363
|
+
for (epoch, snapshot) in [(10, row), (11, geometry)] {
|
|
363
364
|
let bytes = crate::canon::write(&snapshot.to_value());
|
|
364
365
|
let head = crate::durability::checkpoint_bytes(
|
|
365
366
|
&store,
|
|
@@ -378,11 +379,20 @@ fn recovery_refuses_old_writer_authority_and_malformed_part_geometry() {
|
|
|
378
379
|
.head
|
|
379
380
|
.head
|
|
380
381
|
.unwrap();
|
|
381
|
-
|
|
382
|
-
|
|
382
|
+
// The checkpoint cannot be continued: it is discarded, nothing of it is imported,
|
|
383
|
+
// and the new writer proceeds from the work this Store already holds.
|
|
384
|
+
writers.push(begin(&store, &meta, &id, epoch, declaration.clone(), Some(&head)).unwrap());
|
|
385
|
+
check_writer(&meta, &id, epoch).unwrap();
|
|
386
|
+
let current = meta.derived_rows().unwrap().remove(0);
|
|
387
|
+
assert!(current
|
|
388
|
+
.added_parts
|
|
389
|
+
.iter()
|
|
390
|
+
.all(|added| added.part.shape != vec![16, 31]));
|
|
391
|
+
}
|
|
392
|
+
fence(&meta, &id, 11).unwrap();
|
|
393
|
+
for writer in writers {
|
|
394
|
+
release(&meta, writer);
|
|
383
395
|
}
|
|
384
|
-
fence(&meta, &id, 9).unwrap();
|
|
385
|
-
release(&meta, writer);
|
|
386
396
|
abandon(&store, &meta, &id).unwrap();
|
|
387
397
|
std::fs::remove_dir_all(store.root()).unwrap();
|
|
388
398
|
}
|
|
@@ -489,7 +499,7 @@ fn source_identity_is_locked_and_source_closure_stays_rooted_after_writer_death(
|
|
|
489
499
|
}
|
|
490
500
|
|
|
491
501
|
#[test]
|
|
492
|
-
fn
|
|
502
|
+
fn wrong_document_lengths_refuse_to_validate_and_are_discarded_on_resume() {
|
|
493
503
|
let (store, meta, id, declaration) = fixture("lengths");
|
|
494
504
|
let writer = begin(&store, &meta, &id, 1, declaration.clone(), None).unwrap();
|
|
495
505
|
data(&store, &meta, &id, 1, 0x31);
|
|
@@ -528,16 +538,13 @@ fn wrong_document_lengths_refuse_before_parsing_or_importing_work() {
|
|
|
528
538
|
.code,
|
|
529
539
|
Code::LENGTH_MISMATCH
|
|
530
540
|
);
|
|
531
|
-
|
|
532
|
-
|
|
533
|
-
|
|
534
|
-
|
|
535
|
-
|
|
536
|
-
Code::LENGTH_MISMATCH
|
|
537
|
-
);
|
|
538
|
-
check_writer(&meta, &id, 1).unwrap();
|
|
539
|
-
fence(&meta, &id, 1).unwrap();
|
|
541
|
+
// A forged checkpoint is discarded, not imported; this begin then meets the live writer.
|
|
542
|
+
// Resuming from it discards it instead of refusing: the next writer proceeds.
|
|
543
|
+
let next = begin(&store, &meta, &id, 2, declaration, Some(&forged)).unwrap();
|
|
544
|
+
check_writer(&meta, &id, 2).unwrap();
|
|
545
|
+
fence(&meta, &id, 2).unwrap();
|
|
540
546
|
release(&meta, writer);
|
|
547
|
+
release(&meta, next);
|
|
541
548
|
abandon(&store, &meta, &id).unwrap();
|
|
542
549
|
std::fs::remove_dir_all(store.root()).unwrap();
|
|
543
550
|
}
|
|
@@ -1601,6 +1601,9 @@ fn require_source_roots(store: &Store, declaration: &Declaration) -> Result<()>
|
|
|
1601
1601
|
}
|
|
1602
1602
|
|
|
1603
1603
|
fn load_sources(store: &Store, declaration: &Declaration) -> Result<Vec<SourceData>> {
|
|
1604
|
+
// A prepared model whose output a live source operation's publication custodian holds
|
|
1605
|
+
// derives by reference without its bytes; the result is publishable, never readable.
|
|
1606
|
+
let custodied = crate::ingest::custody::live(store.root())?;
|
|
1604
1607
|
let mut out = Vec::new();
|
|
1605
1608
|
for declared in &declaration.sources {
|
|
1606
1609
|
let manifest = checkpoint::load_manifest(store, &declared.manifest)?;
|
|
@@ -1625,11 +1628,13 @@ fn load_sources(store: &Store, declaration: &Declaration) -> Result<Vec<SourceDa
|
|
|
1625
1628
|
let closure = checkpoint::load_closure(store, &header)?;
|
|
1626
1629
|
header.validate(&closure)?;
|
|
1627
1630
|
let walk = checkpoint::walk_cozytensors(store, &manifest)?;
|
|
1628
|
-
walk.
|
|
1631
|
+
walk.require_held(store, &custodied)?;
|
|
1629
1632
|
// This is the zero-read/zero-hash inherit gate: the verification record proves the
|
|
1630
1633
|
// exact installed inode/length/generation without opening or hashing payload bytes.
|
|
1631
1634
|
for reached in &walk.objects {
|
|
1632
|
-
if reached.kind == "part" || reached.kind == "config"
|
|
1635
|
+
if (reached.kind == "part" || reached.kind == "config")
|
|
1636
|
+
&& !custodied.contains(&reached.obj.sha256)
|
|
1637
|
+
{
|
|
1633
1638
|
require_payload_record(store, &reached.obj, reached.kind)?;
|
|
1634
1639
|
}
|
|
1635
1640
|
}
|
|
@@ -2308,9 +2313,35 @@ pub fn begin(
|
|
|
2308
2313
|
release_guards(meta, Some(writer_hold), Vec::new());
|
|
2309
2314
|
return Err(error);
|
|
2310
2315
|
}
|
|
2316
|
+
// Pin every resident source object. A custodied object has no local bytes to pin: a
|
|
2317
|
+
// derivation over it carries it by reference and never reads it.
|
|
2318
|
+
let custodied = match crate::ingest::custody::live(store.root()) {
|
|
2319
|
+
Ok(custodied) => custodied,
|
|
2320
|
+
Err(error) => {
|
|
2321
|
+
release_guards(meta, Some(writer_hold), Vec::new());
|
|
2322
|
+
return Err(error);
|
|
2323
|
+
}
|
|
2324
|
+
};
|
|
2311
2325
|
let mut source_leases = Vec::new();
|
|
2312
2326
|
for source in &sources {
|
|
2313
|
-
|
|
2327
|
+
let pinned = if custodied.is_empty() {
|
|
2328
|
+
read::acquire_cozytensors(store, meta, &source.manifest)
|
|
2329
|
+
} else {
|
|
2330
|
+
checkpoint::walk_cozytensors(store, &source.manifest).and_then(|walk| {
|
|
2331
|
+
let id = source.manifest.manifest_id();
|
|
2332
|
+
read::acquire(
|
|
2333
|
+
store,
|
|
2334
|
+
meta,
|
|
2335
|
+
id.strip_prefix("sha256:").unwrap_or(&id),
|
|
2336
|
+
walk.distinct()
|
|
2337
|
+
.into_iter()
|
|
2338
|
+
.filter(|object| !custodied.contains(&object.sha256))
|
|
2339
|
+
.cloned()
|
|
2340
|
+
.collect(),
|
|
2341
|
+
)
|
|
2342
|
+
})
|
|
2343
|
+
};
|
|
2344
|
+
match pinned {
|
|
2314
2345
|
Ok((lease, receipts)) if receipts.0.is_empty() => {
|
|
2315
2346
|
source_leases.push((source.fact.alias.clone(), lease));
|
|
2316
2347
|
}
|
|
@@ -2342,15 +2373,22 @@ pub fn begin(
|
|
|
2342
2373
|
})
|
|
2343
2374
|
.collect();
|
|
2344
2375
|
|
|
2376
|
+
// A checkpoint this work cannot continue (another plan or work context, or no longer
|
|
2377
|
+
// whole) is discarded: the work re-derives what it lost instead of refusing.
|
|
2378
|
+
let mut checkpoint = checkpoint;
|
|
2345
2379
|
let incoming = match checkpoint
|
|
2346
2380
|
.map(|head| read_progress(store, head, &transaction, &declaration_text, &source_facts))
|
|
2347
|
-
.transpose()
|
|
2348
2381
|
{
|
|
2349
|
-
|
|
2350
|
-
|
|
2382
|
+
None => None,
|
|
2383
|
+
Some(Ok(row)) => Some(row),
|
|
2384
|
+
Some(Err(error)) if error.code == Code::IO_FAILED => {
|
|
2351
2385
|
release_guards(meta, Some(writer_hold), source_leases);
|
|
2352
2386
|
return Err(error);
|
|
2353
2387
|
}
|
|
2388
|
+
Some(Err(_)) => {
|
|
2389
|
+
checkpoint = None;
|
|
2390
|
+
None
|
|
2391
|
+
}
|
|
2354
2392
|
};
|
|
2355
2393
|
// Byte verification can update the verification catalog. Keep it outside the SQL
|
|
2356
2394
|
// writer transaction, then compare the observed row before assigning this epoch.
|
|
@@ -3269,7 +3307,7 @@ pub fn commit(
|
|
|
3269
3307
|
let manifest_ref = store.put_manifest(&manifest)?.obj;
|
|
3270
3308
|
store.read_manifest(&manifest_ref)?;
|
|
3271
3309
|
let walk = checkpoint::walk(store, &manifest)?;
|
|
3272
|
-
walk.
|
|
3310
|
+
walk.require_held(store, &crate::ingest::custody::live(store.root())?)?;
|
|
3273
3311
|
let (inherited_payload_objects, inherited_payload_bytes) =
|
|
3274
3312
|
inherit_observation(&header, &row_before)?;
|
|
3275
3313
|
|
|
@@ -289,6 +289,23 @@ pub fn local_chain(store: &Store, head: &ObjectRef, plan: &str) -> Result<Set> {
|
|
|
289
289
|
fold(&links, plan)
|
|
290
290
|
}
|
|
291
291
|
|
|
292
|
+
/// A supplied chain this conversion can continue, or `None` when it was written under another
|
|
293
|
+
/// plan or slot or is no longer whole in this Store. Such a chain is discarded and the work
|
|
294
|
+
/// reconverts into a new chain: losing a checkpoint costs time, never a refusal.
|
|
295
|
+
pub fn continuable(
|
|
296
|
+
store: &Store,
|
|
297
|
+
head: &ObjectRef,
|
|
298
|
+
plan: &str,
|
|
299
|
+
chain: &str,
|
|
300
|
+
) -> Result<Option<Set>> {
|
|
301
|
+
match local_chain(store, head, plan) {
|
|
302
|
+
Ok(set) if set.operation == chain => Ok(Some(set)),
|
|
303
|
+
Ok(_) => Ok(None),
|
|
304
|
+
Err(error) if error.code == Code::IO_FAILED => Err(error),
|
|
305
|
+
Err(_) => Ok(None),
|
|
306
|
+
}
|
|
307
|
+
}
|
|
308
|
+
|
|
292
309
|
/// Export locally complete progress. This does not assert remote durability: the RecordOwner
|
|
293
310
|
/// publishes the bounded chain and advances its durable pointer only after upload acceptance.
|
|
294
311
|
/// NFS is not consulted and cannot delay this checkpoint.
|
|
@@ -108,8 +108,12 @@ pub fn session_holds(store: &Store) -> Result<(Vec<HeldKey>, Vec<String>)> {
|
|
|
108
108
|
let mut sessions = Vec::new();
|
|
109
109
|
for session in tx::sessions(store) {
|
|
110
110
|
let root = tx::read_session(store.root(), &session)?;
|
|
111
|
-
|
|
111
|
+
// An object the operation's publication custodian already holds is not in flight:
|
|
112
|
+
// the journal stops naming it, so the local copy is ordinary unreferenced bytes.
|
|
113
|
+
let custody = crate::ingest::custody::read_session(store.root(), &session)?;
|
|
114
|
+
let mut journalled =
|
|
112
115
|
crate::ingest::journal::session_objects(&tx::candidate_dir(store.root(), &session))?;
|
|
116
|
+
journalled.retain(|object| !custody.contains(object));
|
|
113
117
|
if root.candidates.is_empty() && journalled.is_empty() {
|
|
114
118
|
continue;
|
|
115
119
|
}
|
|
@@ -0,0 +1,249 @@
|
|
|
1
|
+
//! Remote custody of one model-source operation's converted output.
|
|
2
|
+
//!
|
|
3
|
+
//! A bounded-disk ingest publishes converted objects while it converts, so its output never
|
|
4
|
+
//! has to fit on the pod. Once the publication custodian has verified an object, the caller
|
|
5
|
+
//! records that fact here. From then on the operation's journal stops holding that object
|
|
6
|
+
//! against GC, a resumed pass does not re-convert it, and finalize or a derivation over the
|
|
7
|
+
//! prepared model accept it without local bytes. Nothing READS a custodied object: it is not
|
|
8
|
+
//! resident, and a reader still has to fetch it back through the ordinary verified door.
|
|
9
|
+
//!
|
|
10
|
+
//! The record lives in the ingest session root and is released with it, so an attestation
|
|
11
|
+
//! never outlives the operation that made it. The custodian is named by opaque evidence the
|
|
12
|
+
//! caller supplies (its publication identity); TensorFS carries it and never interprets it.
|
|
13
|
+
|
|
14
|
+
use std::collections::{BTreeMap, BTreeSet};
|
|
15
|
+
use std::fs::{self, OpenOptions};
|
|
16
|
+
use std::io::{Read, Write};
|
|
17
|
+
use std::path::Path;
|
|
18
|
+
|
|
19
|
+
use crate::canon::{as_arr, Fields, Value};
|
|
20
|
+
use crate::err::{refuse, Code, Refusal, Result};
|
|
21
|
+
use crate::header::Part;
|
|
22
|
+
use crate::ids::{ObjectRef, Plain};
|
|
23
|
+
use crate::limits;
|
|
24
|
+
use crate::store::Store;
|
|
25
|
+
|
|
26
|
+
use super::transaction::candidate_dir;
|
|
27
|
+
|
|
28
|
+
const FILE: &str = "custody.json";
|
|
29
|
+
const FORMAT: &str = "tensorfs.source-custody/1";
|
|
30
|
+
const EVIDENCE_MAX_BYTES: usize = 1024;
|
|
31
|
+
|
|
32
|
+
fn io(what: &str, error: std::io::Error) -> Refusal {
|
|
33
|
+
Refusal {
|
|
34
|
+
code: Code::IO_FAILED,
|
|
35
|
+
detail: format!("source custody {what}: {error}"),
|
|
36
|
+
}
|
|
37
|
+
}
|
|
38
|
+
|
|
39
|
+
/// Every object this operation's custodian holds, with the evidence that says where.
|
|
40
|
+
#[derive(Debug, Clone, Default, PartialEq, Eq)]
|
|
41
|
+
pub struct Custody {
|
|
42
|
+
objects: BTreeMap<String, (u64, String)>,
|
|
43
|
+
}
|
|
44
|
+
|
|
45
|
+
impl Plain for Custody {
|
|
46
|
+
const MAX_BYTES: usize = limits::DOC_MAX_BYTES;
|
|
47
|
+
|
|
48
|
+
fn from_value(value: &Value) -> Result<Self> {
|
|
49
|
+
let mut fields = Fields::new("Custody", value)?;
|
|
50
|
+
if fields.req_str("format")? != FORMAT {
|
|
51
|
+
return refuse(Code::WRONG_TYPE, "source custody has an unknown format");
|
|
52
|
+
}
|
|
53
|
+
let mut objects = BTreeMap::new();
|
|
54
|
+
for row in as_arr("Custody", "objects", fields.req("objects")?)? {
|
|
55
|
+
let mut row = Fields::new("Custody.objects", row)?;
|
|
56
|
+
let evidence = row.req_str("evidence")?.to_string();
|
|
57
|
+
let object = ObjectRef::from_value("Custody.object", row.req("object")?)?;
|
|
58
|
+
row.done()?;
|
|
59
|
+
check_evidence(&evidence)?;
|
|
60
|
+
if objects
|
|
61
|
+
.insert(object.sha256, (object.length, evidence))
|
|
62
|
+
.is_some()
|
|
63
|
+
{
|
|
64
|
+
return refuse(Code::DUPLICATE_KEY, "source custody repeats an object");
|
|
65
|
+
}
|
|
66
|
+
}
|
|
67
|
+
fields.done()?;
|
|
68
|
+
Ok(Self { objects })
|
|
69
|
+
}
|
|
70
|
+
|
|
71
|
+
fn to_value(&self) -> Value {
|
|
72
|
+
Value::obj(vec![
|
|
73
|
+
("format", Value::str(FORMAT)),
|
|
74
|
+
(
|
|
75
|
+
"objects",
|
|
76
|
+
Value::arr(
|
|
77
|
+
self.objects
|
|
78
|
+
.iter()
|
|
79
|
+
.map(|(sha256, (length, evidence))| {
|
|
80
|
+
Value::obj(vec![
|
|
81
|
+
("evidence", Value::str(evidence.clone())),
|
|
82
|
+
(
|
|
83
|
+
"object",
|
|
84
|
+
ObjectRef {
|
|
85
|
+
sha256: sha256.clone(),
|
|
86
|
+
length: *length,
|
|
87
|
+
}
|
|
88
|
+
.to_value(),
|
|
89
|
+
),
|
|
90
|
+
])
|
|
91
|
+
})
|
|
92
|
+
.collect(),
|
|
93
|
+
),
|
|
94
|
+
),
|
|
95
|
+
])
|
|
96
|
+
}
|
|
97
|
+
}
|
|
98
|
+
|
|
99
|
+
impl Custody {
|
|
100
|
+
pub fn contains(&self, object: &ObjectRef) -> bool {
|
|
101
|
+
self.objects
|
|
102
|
+
.get(&object.sha256)
|
|
103
|
+
.is_some_and(|(length, _)| *length == object.length)
|
|
104
|
+
}
|
|
105
|
+
|
|
106
|
+
/// A part stands on custody alone only when it has segments and the custodian holds
|
|
107
|
+
/// every one of them. An inline part carries its bytes in the journal line itself.
|
|
108
|
+
pub fn holds_part(&self, part: &Part) -> bool {
|
|
109
|
+
!part.segments().is_empty() && part.segments().iter().all(|s| self.contains(s))
|
|
110
|
+
}
|
|
111
|
+
|
|
112
|
+
pub fn digests(&self) -> BTreeSet<String> {
|
|
113
|
+
self.objects.keys().cloned().collect()
|
|
114
|
+
}
|
|
115
|
+
|
|
116
|
+
pub fn rows(&self) -> Vec<(ObjectRef, String)> {
|
|
117
|
+
self.objects
|
|
118
|
+
.iter()
|
|
119
|
+
.map(|(sha256, (length, evidence))| {
|
|
120
|
+
(
|
|
121
|
+
ObjectRef {
|
|
122
|
+
sha256: sha256.clone(),
|
|
123
|
+
length: *length,
|
|
124
|
+
},
|
|
125
|
+
evidence.clone(),
|
|
126
|
+
)
|
|
127
|
+
})
|
|
128
|
+
.collect()
|
|
129
|
+
}
|
|
130
|
+
|
|
131
|
+
pub fn is_empty(&self) -> bool {
|
|
132
|
+
self.objects.is_empty()
|
|
133
|
+
}
|
|
134
|
+
}
|
|
135
|
+
|
|
136
|
+
fn check_evidence(evidence: &str) -> Result<()> {
|
|
137
|
+
if evidence.is_empty()
|
|
138
|
+
|| evidence.len() > EVIDENCE_MAX_BYTES
|
|
139
|
+
|| !evidence.bytes().all(|b| (0x21..0x7f).contains(&b))
|
|
140
|
+
{
|
|
141
|
+
return refuse(
|
|
142
|
+
Code::KEY_GRAMMAR,
|
|
143
|
+
"custody evidence is 1..=1024 printable ASCII bytes without spaces",
|
|
144
|
+
);
|
|
145
|
+
}
|
|
146
|
+
Ok(())
|
|
147
|
+
}
|
|
148
|
+
|
|
149
|
+
/// The operation's record, empty when it has none.
|
|
150
|
+
pub fn read(store_root: &Path, operation: &str) -> Result<Custody> {
|
|
151
|
+
crate::catalog::source_operation_id(operation)?;
|
|
152
|
+
read_session(store_root, operation)
|
|
153
|
+
}
|
|
154
|
+
|
|
155
|
+
/// Every object the operation's conversion journals name, custodied or not.
|
|
156
|
+
pub fn journalled(store_root: &Path, operation: &str) -> Result<Vec<ObjectRef>> {
|
|
157
|
+
crate::catalog::source_operation_id(operation)?;
|
|
158
|
+
super::journal::session_objects(&candidate_dir(store_root, operation))
|
|
159
|
+
}
|
|
160
|
+
|
|
161
|
+
/// A session directory GC already enumerated; its name came from the filesystem.
|
|
162
|
+
pub(crate) fn read_session(store_root: &Path, session: &str) -> Result<Custody> {
|
|
163
|
+
let path = candidate_dir(store_root, session).join(FILE);
|
|
164
|
+
let file = match fs::File::open(&path) {
|
|
165
|
+
Ok(file) => file,
|
|
166
|
+
Err(error) if error.kind() == std::io::ErrorKind::NotFound => return Ok(Custody::default()),
|
|
167
|
+
Err(error) => return Err(io("open", error)),
|
|
168
|
+
};
|
|
169
|
+
let mut bytes = Vec::new();
|
|
170
|
+
file.take(Custody::MAX_BYTES as u64 + 1)
|
|
171
|
+
.read_to_end(&mut bytes)
|
|
172
|
+
.map_err(|error| io("read", error))?;
|
|
173
|
+
Custody::parse(&bytes)
|
|
174
|
+
}
|
|
175
|
+
|
|
176
|
+
/// Every live session's custody, for the readers that walk more than one operation:
|
|
177
|
+
/// GC's census and a derivation over a prepared model.
|
|
178
|
+
pub fn live(store_root: &Path) -> Result<BTreeSet<String>> {
|
|
179
|
+
let base = store_root.join("tmp").join("ingest");
|
|
180
|
+
let entries = match fs::read_dir(&base) {
|
|
181
|
+
Ok(entries) => entries,
|
|
182
|
+
Err(error) if error.kind() == std::io::ErrorKind::NotFound => return Ok(BTreeSet::new()),
|
|
183
|
+
Err(error) => return Err(io("list sessions", error)),
|
|
184
|
+
};
|
|
185
|
+
let mut out = BTreeSet::new();
|
|
186
|
+
for entry in entries {
|
|
187
|
+
let entry = entry.map_err(|error| io("list sessions", error))?;
|
|
188
|
+
if !entry.path().is_dir() {
|
|
189
|
+
continue;
|
|
190
|
+
}
|
|
191
|
+
let session = entry.file_name().to_string_lossy().to_string();
|
|
192
|
+
out.extend(read_session(store_root, &session)?.digests());
|
|
193
|
+
}
|
|
194
|
+
Ok(out)
|
|
195
|
+
}
|
|
196
|
+
|
|
197
|
+
/// Record that the custodian named by `evidence` holds these journalled objects. Only
|
|
198
|
+
/// objects this operation's own conversion journal names may be recorded; the record is
|
|
199
|
+
/// rewritten whole, atomically, so a crash leaves either the old set or the new one.
|
|
200
|
+
pub fn record(store: &Store, operation: &str, objects: &[ObjectRef], evidence: &str) -> Result<()> {
|
|
201
|
+
crate::catalog::source_operation_id(operation)?;
|
|
202
|
+
check_evidence(evidence)?;
|
|
203
|
+
let dir = candidate_dir(store.root(), operation);
|
|
204
|
+
if !dir.is_dir() {
|
|
205
|
+
return refuse(Code::ROOT_ABSENT, "source custody names no live session");
|
|
206
|
+
}
|
|
207
|
+
let journalled: BTreeMap<String, u64> = journalled(store.root(), operation)?
|
|
208
|
+
.into_iter()
|
|
209
|
+
.map(|object| (object.sha256, object.length))
|
|
210
|
+
.collect();
|
|
211
|
+
let mut custody = read(store.root(), operation)?;
|
|
212
|
+
for object in objects {
|
|
213
|
+
if journalled.get(&object.sha256) != Some(&object.length) {
|
|
214
|
+
return refuse(
|
|
215
|
+
Code::OBJECT_ID_MISMATCH,
|
|
216
|
+
format!(
|
|
217
|
+
"{} is not an object this operation's conversion journal names",
|
|
218
|
+
object.id()
|
|
219
|
+
),
|
|
220
|
+
);
|
|
221
|
+
}
|
|
222
|
+
custody
|
|
223
|
+
.objects
|
|
224
|
+
.entry(object.sha256.clone())
|
|
225
|
+
.or_insert_with(|| (object.length, evidence.to_string()));
|
|
226
|
+
}
|
|
227
|
+
let bytes = custody.canonical_bytes();
|
|
228
|
+
if bytes.len() > Custody::MAX_BYTES {
|
|
229
|
+
return refuse(Code::SIZE_CAP, "source custody exceeds its document bound");
|
|
230
|
+
}
|
|
231
|
+
let path = dir.join(FILE);
|
|
232
|
+
let temporary = dir.join(format!(".{FILE}.{}", crate::meta::now_nanos_unique()));
|
|
233
|
+
let written = (|| {
|
|
234
|
+
let mut file = OpenOptions::new()
|
|
235
|
+
.write(true)
|
|
236
|
+
.create_new(true)
|
|
237
|
+
.open(&temporary)
|
|
238
|
+
.map_err(|error| io("create", error))?;
|
|
239
|
+
file.write_all(&bytes)
|
|
240
|
+
.and_then(|()| file.sync_all())
|
|
241
|
+
.map_err(|error| io("write", error))?;
|
|
242
|
+
fs::rename(&temporary, &path).map_err(|error| io("rename", error))?;
|
|
243
|
+
crate::store::fsync_dir(&dir)
|
|
244
|
+
})();
|
|
245
|
+
if written.is_err() {
|
|
246
|
+
let _ = fs::remove_file(&temporary);
|
|
247
|
+
}
|
|
248
|
+
written
|
|
249
|
+
}
|
|
@@ -441,6 +441,19 @@ impl Journal {
|
|
|
441
441
|
/// makes no part true either: `convert` re-checks each one against `segments_still_stand`
|
|
442
442
|
/// before reusing it, so a journal restored beside objects that did not survive re-converts
|
|
443
443
|
/// exactly those ops.
|
|
444
|
+
/// Remove one plan's journal and cursor: work that can no longer be continued.
|
|
445
|
+
pub fn discard(dir: &Path, plan_digest: &str) -> Result<()> {
|
|
446
|
+
let journal = journal_path(dir, plan_digest);
|
|
447
|
+
for path in [journal.with_extension("head.json"), journal] {
|
|
448
|
+
match fs::remove_file(&path) {
|
|
449
|
+
Ok(()) => {}
|
|
450
|
+
Err(error) if error.kind() == std::io::ErrorKind::NotFound => {}
|
|
451
|
+
Err(error) => return Err(io("discard conversion journal", error)),
|
|
452
|
+
}
|
|
453
|
+
}
|
|
454
|
+
Ok(())
|
|
455
|
+
}
|
|
456
|
+
|
|
444
457
|
pub fn install(dir: &Path, plan_digest: &str, bytes: &[u8]) -> Result<()> {
|
|
445
458
|
hex64("conversion journal plan", plan_digest)?;
|
|
446
459
|
for line in bytes.split(|byte| *byte == b'\n') {
|