tensorfs 0.3.23__tar.gz → 0.3.26__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {tensorfs-0.3.23 → tensorfs-0.3.26}/Cargo.lock +2 -2
- {tensorfs-0.3.23 → tensorfs-0.3.26}/Cargo.toml +1 -1
- {tensorfs-0.3.23 → tensorfs-0.3.26}/PKG-INFO +1 -1
- {tensorfs-0.3.23 → tensorfs-0.3.26}/crates/tensorfs-core/src/checkpoint.rs +5 -1
- {tensorfs-0.3.23 → tensorfs-0.3.26}/crates/tensorfs-core/src/source_artifact.rs +65 -9
- {tensorfs-0.3.23 → tensorfs-0.3.26}/crates/tensorfs-core/src/transport/hub.rs +34 -5
- {tensorfs-0.3.23 → tensorfs-0.3.26}/crates/tensorfs-core/src/transport/pull.rs +41 -10
- {tensorfs-0.3.23 → tensorfs-0.3.26}/crates/tensorfs-core/src/transport/tests.rs +117 -1
- tensorfs-0.3.26/crates/tensorfs-core/tests/import_tree.rs +217 -0
- {tensorfs-0.3.23 → tensorfs-0.3.26}/crates/tensorfs-core/tests/source_artifacts.rs +82 -0
- {tensorfs-0.3.23 → tensorfs-0.3.26}/crates/tensorfs-py/src/store.rs +32 -21
- {tensorfs-0.3.23 → tensorfs-0.3.26}/pyproject.toml +1 -1
- {tensorfs-0.3.23 → tensorfs-0.3.26}/python/tensorfs/_ext.pyi +4 -1
- {tensorfs-0.3.23 → tensorfs-0.3.26}/crates/tensorfs-core/Cargo.toml +0 -0
- {tensorfs-0.3.23 → tensorfs-0.3.26}/crates/tensorfs-core/build.rs +0 -0
- {tensorfs-0.3.23 → tensorfs-0.3.26}/crates/tensorfs-core/examples/h3-swiglu-layout.md +0 -0
- {tensorfs-0.3.23 → tensorfs-0.3.26}/crates/tensorfs-core/examples/optional-cache-backfill.md +0 -0
- {tensorfs-0.3.23 → tensorfs-0.3.26}/crates/tensorfs-core/examples/resident-conversion.md +0 -0
- {tensorfs-0.3.23 → tensorfs-0.3.26}/crates/tensorfs-core/examples/resident_conversion_profile.rs +0 -0
- {tensorfs-0.3.23 → tensorfs-0.3.26}/crates/tensorfs-core/src/b64.rs +0 -0
- {tensorfs-0.3.23 → tensorfs-0.3.26}/crates/tensorfs-core/src/bin/tfs/ckpt.rs +0 -0
- {tensorfs-0.3.23 → tensorfs-0.3.26}/crates/tensorfs-core/src/bin/tfs/conform.rs +0 -0
- {tensorfs-0.3.23 → tensorfs-0.3.26}/crates/tensorfs-core/src/bin/tfs/fetch.rs +0 -0
- {tensorfs-0.3.23 → tensorfs-0.3.26}/crates/tensorfs-core/src/bin/tfs/ingest.rs +0 -0
- {tensorfs-0.3.23 → tensorfs-0.3.26}/crates/tensorfs-core/src/bin/tfs/main.rs +0 -0
- {tensorfs-0.3.23 → tensorfs-0.3.26}/crates/tensorfs-core/src/bin/tfs/rd.rs +0 -0
- {tensorfs-0.3.23 → tensorfs-0.3.26}/crates/tensorfs-core/src/bin/tfs/repo.rs +0 -0
- {tensorfs-0.3.23 → tensorfs-0.3.26}/crates/tensorfs-core/src/bin/tfs/snap.rs +0 -0
- {tensorfs-0.3.23 → tensorfs-0.3.26}/crates/tensorfs-core/src/bin/tfs/watch.rs +0 -0
- {tensorfs-0.3.23 → tensorfs-0.3.26}/crates/tensorfs-core/src/canon.rs +0 -0
- {tensorfs-0.3.23 → tensorfs-0.3.26}/crates/tensorfs-core/src/capability.rs +0 -0
- {tensorfs-0.3.23 → tensorfs-0.3.26}/crates/tensorfs-core/src/catalog.rs +0 -0
- {tensorfs-0.3.23 → tensorfs-0.3.26}/crates/tensorfs-core/src/cbor.rs +0 -0
- {tensorfs-0.3.23 → tensorfs-0.3.26}/crates/tensorfs-core/src/corpus.rs +0 -0
- {tensorfs-0.3.23 → tensorfs-0.3.26}/crates/tensorfs-core/src/derived/adoption.rs +0 -0
- {tensorfs-0.3.23 → tensorfs-0.3.26}/crates/tensorfs-core/src/derived/graft_tests.rs +0 -0
- {tensorfs-0.3.23 → tensorfs-0.3.26}/crates/tensorfs-core/src/derived/progress.rs +0 -0
- {tensorfs-0.3.23 → tensorfs-0.3.26}/crates/tensorfs-core/src/derived/resume_tests.rs +0 -0
- {tensorfs-0.3.23 → tensorfs-0.3.26}/crates/tensorfs-core/src/derived/retention.rs +0 -0
- {tensorfs-0.3.23 → tensorfs-0.3.26}/crates/tensorfs-core/src/derived/roots.rs +0 -0
- {tensorfs-0.3.23 → tensorfs-0.3.26}/crates/tensorfs-core/src/derived.rs +0 -0
- {tensorfs-0.3.23 → tensorfs-0.3.26}/crates/tensorfs-core/src/dtype.rs +0 -0
- {tensorfs-0.3.23 → tensorfs-0.3.26}/crates/tensorfs-core/src/durability.rs +0 -0
- {tensorfs-0.3.23 → tensorfs-0.3.26}/crates/tensorfs-core/src/err.rs +0 -0
- {tensorfs-0.3.23 → tensorfs-0.3.26}/crates/tensorfs-core/src/fetch.rs +0 -0
- {tensorfs-0.3.23 → tensorfs-0.3.26}/crates/tensorfs-core/src/fill.rs +0 -0
- {tensorfs-0.3.23 → tensorfs-0.3.26}/crates/tensorfs-core/src/fit.rs +0 -0
- {tensorfs-0.3.23 → tensorfs-0.3.26}/crates/tensorfs-core/src/gc.rs +0 -0
- {tensorfs-0.3.23 → tensorfs-0.3.26}/crates/tensorfs-core/src/header.rs +0 -0
- {tensorfs-0.3.23 → tensorfs-0.3.26}/crates/tensorfs-core/src/home.rs +0 -0
- {tensorfs-0.3.23 → tensorfs-0.3.26}/crates/tensorfs-core/src/ids.rs +0 -0
- {tensorfs-0.3.23 → tensorfs-0.3.26}/crates/tensorfs-core/src/ingest/carrier.rs +0 -0
- {tensorfs-0.3.23 → tensorfs-0.3.26}/crates/tensorfs-core/src/ingest/convert.rs +0 -0
- {tensorfs-0.3.23 → tensorfs-0.3.26}/crates/tensorfs-core/src/ingest/fingerprint.rs +0 -0
- {tensorfs-0.3.23 → tensorfs-0.3.26}/crates/tensorfs-core/src/ingest/gguf.rs +0 -0
- {tensorfs-0.3.23 → tensorfs-0.3.26}/crates/tensorfs-core/src/ingest/journal.rs +0 -0
- {tensorfs-0.3.23 → tensorfs-0.3.26}/crates/tensorfs-core/src/ingest/mod.rs +0 -0
- {tensorfs-0.3.23 → tensorfs-0.3.26}/crates/tensorfs-core/src/ingest/preflight.rs +0 -0
- {tensorfs-0.3.23 → tensorfs-0.3.26}/crates/tensorfs-core/src/ingest/source.rs +0 -0
- {tensorfs-0.3.23 → tensorfs-0.3.26}/crates/tensorfs-core/src/ingest/stamp.rs +0 -0
- {tensorfs-0.3.23 → tensorfs-0.3.26}/crates/tensorfs-core/src/ingest/transaction.rs +0 -0
- {tensorfs-0.3.23 → tensorfs-0.3.26}/crates/tensorfs-core/src/jcs.rs +0 -0
- {tensorfs-0.3.23 → tensorfs-0.3.26}/crates/tensorfs-core/src/lib.rs +0 -0
- {tensorfs-0.3.23 → tensorfs-0.3.26}/crates/tensorfs-core/src/limits.rs +0 -0
- {tensorfs-0.3.23 → tensorfs-0.3.26}/crates/tensorfs-core/src/machine.rs +0 -0
- {tensorfs-0.3.23 → tensorfs-0.3.26}/crates/tensorfs-core/src/manifest.rs +0 -0
- {tensorfs-0.3.23 → tensorfs-0.3.26}/crates/tensorfs-core/src/meta.rs +0 -0
- {tensorfs-0.3.23 → tensorfs-0.3.26}/crates/tensorfs-core/src/project.rs +0 -0
- {tensorfs-0.3.23 → tensorfs-0.3.26}/crates/tensorfs-core/src/providers.rs +0 -0
- {tensorfs-0.3.23 → tensorfs-0.3.26}/crates/tensorfs-core/src/read.rs +0 -0
- {tensorfs-0.3.23 → tensorfs-0.3.26}/crates/tensorfs-core/src/receipt.rs +0 -0
- {tensorfs-0.3.23 → tensorfs-0.3.26}/crates/tensorfs-core/src/reclaim.rs +0 -0
- {tensorfs-0.3.23 → tensorfs-0.3.26}/crates/tensorfs-core/src/registry.rs +0 -0
- {tensorfs-0.3.23 → tensorfs-0.3.26}/crates/tensorfs-core/src/relation.rs +0 -0
- {tensorfs-0.3.23 → tensorfs-0.3.26}/crates/tensorfs-core/src/repo_cache.rs +0 -0
- {tensorfs-0.3.23 → tensorfs-0.3.26}/crates/tensorfs-core/src/repository.rs +0 -0
- {tensorfs-0.3.23 → tensorfs-0.3.26}/crates/tensorfs-core/src/sha256.rs +0 -0
- {tensorfs-0.3.23 → tensorfs-0.3.26}/crates/tensorfs-core/src/spec.rs +0 -0
- {tensorfs-0.3.23 → tensorfs-0.3.26}/crates/tensorfs-core/src/staging.rs +0 -0
- {tensorfs-0.3.23 → tensorfs-0.3.26}/crates/tensorfs-core/src/storage.rs +0 -0
- {tensorfs-0.3.23 → tensorfs-0.3.26}/crates/tensorfs-core/src/store.rs +0 -0
- {tensorfs-0.3.23 → tensorfs-0.3.26}/crates/tensorfs-core/src/transport/api.rs +0 -0
- {tensorfs-0.3.23 → tensorfs-0.3.26}/crates/tensorfs-core/src/transport/http.rs +0 -0
- {tensorfs-0.3.23 → tensorfs-0.3.26}/crates/tensorfs-core/src/transport/ledger.rs +0 -0
- {tensorfs-0.3.23 → tensorfs-0.3.26}/crates/tensorfs-core/src/transport/mod.rs +0 -0
- {tensorfs-0.3.23 → tensorfs-0.3.26}/crates/tensorfs-core/src/transport/policy.rs +0 -0
- {tensorfs-0.3.23 → tensorfs-0.3.26}/crates/tensorfs-core/src/transport/prefix.rs +0 -0
- {tensorfs-0.3.23 → tensorfs-0.3.26}/crates/tensorfs-core/src/transport/push.rs +0 -0
- {tensorfs-0.3.23 → tensorfs-0.3.26}/crates/tensorfs-core/src/transport/ranged.rs +0 -0
- {tensorfs-0.3.23 → tensorfs-0.3.26}/crates/tensorfs-core/src/vectors.rs +0 -0
- {tensorfs-0.3.23 → tensorfs-0.3.26}/crates/tensorfs-core/tests/fetch_url_progress.rs +0 -0
- {tensorfs-0.3.23 → tensorfs-0.3.26}/crates/tensorfs-core/tests/fit.rs +0 -0
- {tensorfs-0.3.23 → tensorfs-0.3.26}/crates/tensorfs-core/tests/fixtures/repo_apply/blobs.jsonl +0 -0
- {tensorfs-0.3.23 → tensorfs-0.3.26}/crates/tensorfs-core/tests/fixtures/repo_apply/expected_list.jsonl +0 -0
- {tensorfs-0.3.23 → tensorfs-0.3.26}/crates/tensorfs-core/tests/fixtures/repo_apply/expected_repository.json +0 -0
- {tensorfs-0.3.23 → tensorfs-0.3.26}/crates/tensorfs-core/tests/fixtures/repo_apply/expected_rows.jsonl +0 -0
- {tensorfs-0.3.23 → tensorfs-0.3.26}/crates/tensorfs-core/tests/fixtures/repo_apply/header.cbor +0 -0
- {tensorfs-0.3.23 → tensorfs-0.3.26}/crates/tensorfs-core/tests/fixtures/repo_apply/manifest.json +0 -0
- {tensorfs-0.3.23 → tensorfs-0.3.26}/crates/tensorfs-core/tests/fixtures/repo_apply/mutation.json +0 -0
- {tensorfs-0.3.23 → tensorfs-0.3.26}/crates/tensorfs-core/tests/h3_swiglu.rs +0 -0
- {tensorfs-0.3.23 → tensorfs-0.3.26}/crates/tensorfs-core/tests/home.rs +0 -0
- {tensorfs-0.3.23 → tensorfs-0.3.26}/crates/tensorfs-core/tests/liveness.rs +0 -0
- {tensorfs-0.3.23 → tensorfs-0.3.26}/crates/tensorfs-core/tests/read_plan.rs +0 -0
- {tensorfs-0.3.23 → tensorfs-0.3.26}/crates/tensorfs-core/tests/reclaim.rs +0 -0
- {tensorfs-0.3.23 → tensorfs-0.3.26}/crates/tensorfs-core/tests/reclaim_run.rs +0 -0
- {tensorfs-0.3.23 → tensorfs-0.3.26}/crates/tensorfs-core/tests/repo_cache.rs +0 -0
- {tensorfs-0.3.23 → tensorfs-0.3.26}/crates/tensorfs-core/tests/repo_store.rs +0 -0
- {tensorfs-0.3.23 → tensorfs-0.3.26}/crates/tensorfs-core/tests/resident_admission.rs +0 -0
- {tensorfs-0.3.23 → tensorfs-0.3.26}/crates/tensorfs-core/tests/sha256_backends.rs +0 -0
- {tensorfs-0.3.23 → tensorfs-0.3.26}/crates/tensorfs-core/tests/sharded_cas.rs +0 -0
- {tensorfs-0.3.23 → tensorfs-0.3.26}/crates/tensorfs-core/tests/source_plan.rs +0 -0
- {tensorfs-0.3.23 → tensorfs-0.3.26}/crates/tensorfs-core/tests/source_prefixes.rs +0 -0
- {tensorfs-0.3.23 → tensorfs-0.3.26}/crates/tensorfs-core/tests/source_preflight.rs +0 -0
- {tensorfs-0.3.23 → tensorfs-0.3.26}/crates/tensorfs-core/tests/staging.rs +0 -0
- {tensorfs-0.3.23 → tensorfs-0.3.26}/crates/tensorfs-core/tests/store_boundaries.rs +0 -0
- {tensorfs-0.3.23 → tensorfs-0.3.26}/crates/tensorfs-core/tests/tensor_schema.rs +0 -0
- {tensorfs-0.3.23 → tensorfs-0.3.26}/crates/tensorfs-core/tests/version.rs +0 -0
- {tensorfs-0.3.23 → tensorfs-0.3.26}/crates/tensorfs-core/tests/write_through.rs +0 -0
- {tensorfs-0.3.23 → tensorfs-0.3.26}/crates/tensorfs-py/Cargo.toml +0 -0
- {tensorfs-0.3.23 → tensorfs-0.3.26}/crates/tensorfs-py/src/docs.rs +0 -0
- {tensorfs-0.3.23 → tensorfs-0.3.26}/crates/tensorfs-py/src/errors.rs +0 -0
- {tensorfs-0.3.23 → tensorfs-0.3.26}/crates/tensorfs-py/src/lib.rs +0 -0
- {tensorfs-0.3.23 → tensorfs-0.3.26}/python/tensorfs/__init__.py +0 -0
- {tensorfs-0.3.23 → tensorfs-0.3.26}/python/tensorfs/errors.py +0 -0
- {tensorfs-0.3.23 → tensorfs-0.3.26}/python/tensorfs/py.typed +0 -0
|
@@ -392,7 +392,7 @@ checksum = "adb6935a6f5c20170eeceb1a3835a49e12e19d792f6dd344ccc76a985ca5a6ca"
|
|
|
392
392
|
|
|
393
393
|
[[package]]
|
|
394
394
|
name = "tensorfs-core"
|
|
395
|
-
version = "0.3.
|
|
395
|
+
version = "0.3.26"
|
|
396
396
|
dependencies = [
|
|
397
397
|
"fs2",
|
|
398
398
|
"minicbor",
|
|
@@ -404,7 +404,7 @@ dependencies = [
|
|
|
404
404
|
|
|
405
405
|
[[package]]
|
|
406
406
|
name = "tensorfs-py"
|
|
407
|
-
version = "0.3.
|
|
407
|
+
version = "0.3.26"
|
|
408
408
|
dependencies = [
|
|
409
409
|
"pyo3",
|
|
410
410
|
"tensorfs-core",
|
|
@@ -477,7 +477,11 @@ pub fn walk_cozytensors(store: &Store, snap: &Manifest) -> Result<Walk> {
|
|
|
477
477
|
/// Repository completeness, checkout, retention, and accounting use this full walk; workers
|
|
478
478
|
/// use `walk_cozytensors` because an optional README or sample image is not a runtime input.
|
|
479
479
|
pub fn walk(store: &Store, snap: &Manifest) -> Result<Walk> {
|
|
480
|
-
let mut w =
|
|
480
|
+
let mut w = if snap.header().is_some() {
|
|
481
|
+
walk_cozytensors(store, snap)?
|
|
482
|
+
} else {
|
|
483
|
+
Walk::default()
|
|
484
|
+
};
|
|
481
485
|
|
|
482
486
|
// Anything listed that the header does not account for is a sibling asset — with two
|
|
483
487
|
// audit-driven tightenings (external review, folded in at tfs-003):
|
|
@@ -8,12 +8,13 @@ use crate::{
|
|
|
8
8
|
ids::{prefixed, Doc, ObjectRef, Plain},
|
|
9
9
|
manifest::Manifest,
|
|
10
10
|
storage::HeldKey,
|
|
11
|
-
store::Store,
|
|
11
|
+
store::{Fault, Store},
|
|
12
12
|
};
|
|
13
13
|
use fs2::FileExt;
|
|
14
14
|
use std::{
|
|
15
15
|
fs::{self, File, OpenOptions},
|
|
16
16
|
io::{Read, Write},
|
|
17
|
+
os::unix::fs::OpenOptionsExt,
|
|
17
18
|
path::PathBuf,
|
|
18
19
|
};
|
|
19
20
|
|
|
@@ -157,14 +158,16 @@ fn verify(store: &Store, manifest: &ObjectRef) -> Result<Manifest> {
|
|
|
157
158
|
);
|
|
158
159
|
}
|
|
159
160
|
for (_, entry) in tree.entries() {
|
|
160
|
-
|
|
161
|
-
|
|
162
|
-
|
|
163
|
-
|
|
164
|
-
|
|
165
|
-
|
|
166
|
-
|
|
167
|
-
|
|
161
|
+
// Retention asks the same question as reading: are these exact bytes present and
|
|
162
|
+
// verified? Keep the native distinction between an absent member (a cache miss)
|
|
163
|
+
// and corrupt or unreadable bytes. The pinned reader also repairs a stale record
|
|
164
|
+
// by hashing only when its fast metadata check can no longer establish identity.
|
|
165
|
+
let file = store.open_verified(&entry.blob().sha256)?;
|
|
166
|
+
if file.len() != entry.blob().length {
|
|
167
|
+
return refuse(
|
|
168
|
+
Code::LENGTH_MISMATCH,
|
|
169
|
+
"tree member length differs from its manifest",
|
|
170
|
+
);
|
|
168
171
|
}
|
|
169
172
|
}
|
|
170
173
|
Ok(tree)
|
|
@@ -272,6 +275,59 @@ pub fn create(store: &Store, owner: &str, tree: &Manifest) -> Result<TreeRoot> {
|
|
|
272
275
|
}
|
|
273
276
|
writer.finish(tree)
|
|
274
277
|
}
|
|
278
|
+
|
|
279
|
+
/// Import an exact ordinary-file roster under durable custody before admitting bytes.
|
|
280
|
+
/// A retry needs local paths only for members without a standing verification record.
|
|
281
|
+
pub fn import_tree(
|
|
282
|
+
store: &Store,
|
|
283
|
+
owner: &str,
|
|
284
|
+
tree: &Manifest,
|
|
285
|
+
files: &[(String, PathBuf)],
|
|
286
|
+
fault: &Fault,
|
|
287
|
+
) -> Result<TreeRoot> {
|
|
288
|
+
check_roster(owner, tree)?;
|
|
289
|
+
if tree.header().is_some() {
|
|
290
|
+
return refuse(Code::WRONG_TYPE, "import requires an ordinary-file tree");
|
|
291
|
+
}
|
|
292
|
+
let paths: std::collections::BTreeMap<_, _> = files.iter().map(|(k, v)| (k, v)).collect();
|
|
293
|
+
if paths.len() != files.len()
|
|
294
|
+
|| paths.len() != tree.entries().len()
|
|
295
|
+
|| tree
|
|
296
|
+
.entries()
|
|
297
|
+
.iter()
|
|
298
|
+
.any(|(member, _)| !paths.contains_key(member))
|
|
299
|
+
{
|
|
300
|
+
return refuse(
|
|
301
|
+
Code::TRANSACTION_CONFLICT,
|
|
302
|
+
"import paths differ from the exact manifest roster",
|
|
303
|
+
);
|
|
304
|
+
}
|
|
305
|
+
let mut writer = Writer::open(store, owner, tree.object_ref())?;
|
|
306
|
+
if !writer.completed()? {
|
|
307
|
+
for (member, entry) in tree.entries() {
|
|
308
|
+
let object = entry.blob();
|
|
309
|
+
writer.hold_expected(object)?;
|
|
310
|
+
if !matches!(store.record_valid(&object.sha256), Ok(row) if row.length == object.length)
|
|
311
|
+
{
|
|
312
|
+
let path = paths[member];
|
|
313
|
+
if !fs::symlink_metadata(path).map_err(io)?.is_file() {
|
|
314
|
+
return refuse(Code::WRONG_TYPE, "import member must be a regular file");
|
|
315
|
+
}
|
|
316
|
+
let mut file = OpenOptions::new()
|
|
317
|
+
.read(true)
|
|
318
|
+
.custom_flags(crate::store::O_NOFOLLOW)
|
|
319
|
+
.open(path)
|
|
320
|
+
.map_err(io)?;
|
|
321
|
+
if !file.metadata().map_err(io)?.is_file() {
|
|
322
|
+
return refuse(Code::WRONG_TYPE, "import member must be a regular file");
|
|
323
|
+
}
|
|
324
|
+
store.put_stream(&mut file, Some(object), fault)?;
|
|
325
|
+
}
|
|
326
|
+
writer.landed(object)?;
|
|
327
|
+
}
|
|
328
|
+
}
|
|
329
|
+
writer.finish(tree)
|
|
330
|
+
}
|
|
275
331
|
/// Adopt only from a still-retained complete owner, before releasing the source owner.
|
|
276
332
|
pub fn retain(store: &Store, source: &str, owner: &str) -> Result<TreeRoot> {
|
|
277
333
|
let _writer = WriterGuard::acquire(store.root())?;
|
|
@@ -593,7 +593,29 @@ fn call_once(
|
|
|
593
593
|
})
|
|
594
594
|
});
|
|
595
595
|
}
|
|
596
|
-
let refusal = envelope(status, &raw, path)
|
|
596
|
+
let Some(refusal) = envelope(status, &raw, path) else {
|
|
597
|
+
// NOT THE HUB. Every refusal the hub can write carries its typed envelope, so a
|
|
598
|
+
// status that carries none was written by something in front of it — a tunnel
|
|
599
|
+
// whose agent is reconnecting, a load balancer with no backend, a captive
|
|
600
|
+
// gateway. That is weather over an immutable request, exactly like a connection
|
|
601
|
+
// that was never answered, and it is classified as such.
|
|
602
|
+
//
|
|
603
|
+
// Reading it as a verdict cost a paid H100 on 2026-09-08 (tfs-101): the public
|
|
604
|
+
// origin's tunnel agent was seventeen seconds out of a reconnect loop, its edge
|
|
605
|
+
// answered `POST /v1/tensorfs/presign` with its own 404 page, this function
|
|
606
|
+
// called that REF_NOT_FOUND, the pod supervisor's poisoned set made it terminal,
|
|
607
|
+
// and the pod journaled a permanent refusal for a placement set whose every
|
|
608
|
+
// object the hub was holding and would have signed.
|
|
609
|
+
return Err(Retryable {
|
|
610
|
+
wait: ledger.patience(),
|
|
611
|
+
refusal: Refusal {
|
|
612
|
+
code: Code::HUB_UNREACHABLE,
|
|
613
|
+
detail: format!(
|
|
614
|
+
"{base}{path} answered HTTP {status} with no hub envelope, so the hub did not write it; artifacts already in the local store are usable offline"
|
|
615
|
+
),
|
|
616
|
+
},
|
|
617
|
+
});
|
|
618
|
+
};
|
|
597
619
|
if !crate::transport::pull::RETRYABLE_STATUS.contains(&status) {
|
|
598
620
|
return Err(Retryable::from(refusal));
|
|
599
621
|
}
|
|
@@ -626,7 +648,14 @@ fn stated_wait(raw: &[u8]) -> Option<std::time::Duration> {
|
|
|
626
648
|
|
|
627
649
|
/// The hub's own typed envelope, crossed verbatim: its code and remedy are BETTER than
|
|
628
650
|
/// anything invented here — the hub knows why it refused.
|
|
629
|
-
|
|
651
|
+
///
|
|
652
|
+
/// `None` when the answer carried no envelope, and that is the point of the return type.
|
|
653
|
+
/// The STATUS ALONE is not evidence about the catalog: a 404 means "the published catalog
|
|
654
|
+
/// does not name this" only when the hub said so, and the presence of `error.code` is the
|
|
655
|
+
/// only thing that distinguishes the hub's verdict from a proxy's opinion. Mapping status
|
|
656
|
+
/// without that test is how a tunnel's error page came to mean a model does not exist —
|
|
657
|
+
/// permanently, on a pod that had already paid to boot.
|
|
658
|
+
fn envelope(status: u16, raw: &[u8], path: &str) -> Option<Refusal> {
|
|
630
659
|
let mut code = String::new();
|
|
631
660
|
let mut message = String::new();
|
|
632
661
|
let mut remedy = String::new();
|
|
@@ -645,7 +674,7 @@ fn envelope(status: u16, raw: &[u8], path: &str) -> Refusal {
|
|
|
645
674
|
}
|
|
646
675
|
}
|
|
647
676
|
if code.is_empty() {
|
|
648
|
-
|
|
677
|
+
return None;
|
|
649
678
|
}
|
|
650
679
|
if message.is_empty() {
|
|
651
680
|
message = format!("the hub answered HTTP {status} to {path}");
|
|
@@ -655,7 +684,7 @@ fn envelope(status: u16, raw: &[u8], path: &str) -> Refusal {
|
|
|
655
684
|
} else {
|
|
656
685
|
format!("{code}: {message} — {remedy}")
|
|
657
686
|
};
|
|
658
|
-
match status {
|
|
687
|
+
Some(match status {
|
|
659
688
|
404 => Refusal {
|
|
660
689
|
code: Code::REF_NOT_FOUND,
|
|
661
690
|
detail,
|
|
@@ -672,5 +701,5 @@ fn envelope(status: u16, raw: &[u8], path: &str) -> Refusal {
|
|
|
672
701
|
code: Code::HUB_REFUSED,
|
|
673
702
|
detail,
|
|
674
703
|
},
|
|
675
|
-
}
|
|
704
|
+
})
|
|
676
705
|
}
|
|
@@ -629,18 +629,34 @@ impl ObjectUrls for Leases<'_> {
|
|
|
629
629
|
return Ok(url);
|
|
630
630
|
}
|
|
631
631
|
self.mint_from(at)?;
|
|
632
|
-
self.fresh(&object.sha256)
|
|
633
|
-
|
|
634
|
-
|
|
635
|
-
|
|
636
|
-
|
|
637
|
-
|
|
632
|
+
if let Some(url) = self.fresh(&object.sha256) {
|
|
633
|
+
return Ok(url);
|
|
634
|
+
}
|
|
635
|
+
// The mint answered and the URL was STILL not usable. With `minted` stamped on
|
|
636
|
+
// arrival this can only mean the hub granted a life it had already spent — it
|
|
637
|
+
// declared `expires_at_unix` at or before its own `server_time_unix` — so say
|
|
638
|
+
// that, and say the number, instead of leaving the reader to infer which of the
|
|
639
|
+
// two clocks was wrong.
|
|
640
|
+
refuse(
|
|
641
|
+
Code::HUB_REFUSED,
|
|
642
|
+
format!(
|
|
643
|
+
"{}: the mint answered but the URL was unusable on arrival; the hub \
|
|
644
|
+
declared a life of {:?} for it, and a URL whose declared life is already \
|
|
645
|
+
spent when it is granted authorizes nothing",
|
|
646
|
+
object.id(),
|
|
647
|
+
self.declared_lifetime(&object.sha256)
|
|
638
648
|
),
|
|
639
|
-
|
|
649
|
+
)
|
|
640
650
|
}
|
|
641
651
|
}
|
|
642
652
|
|
|
643
653
|
impl Leases<'_> {
|
|
654
|
+
/// What the hub said this digest's URL was good for, for a refusal to quote.
|
|
655
|
+
fn declared_lifetime(&self, digest: &str) -> Option<Duration> {
|
|
656
|
+
let held = self.held.lock().unwrap();
|
|
657
|
+
held.get(digest).and_then(|lease| lease.lifetime)
|
|
658
|
+
}
|
|
659
|
+
|
|
644
660
|
fn fresh(&self, digest: &str) -> Option<String> {
|
|
645
661
|
let held = self.held.lock().unwrap();
|
|
646
662
|
held.get(digest)
|
|
@@ -694,9 +710,6 @@ impl Leases<'_> {
|
|
|
694
710
|
if digests.is_empty() {
|
|
695
711
|
return Ok(());
|
|
696
712
|
}
|
|
697
|
-
// Stamped BEFORE the call, so the call's own latency is spent out of the lease it
|
|
698
|
-
// is fetching rather than granted back to it.
|
|
699
|
-
let minted = Instant::now();
|
|
700
713
|
let answer = hub::presign(
|
|
701
714
|
self.client,
|
|
702
715
|
self.base,
|
|
@@ -706,6 +719,24 @@ impl Leases<'_> {
|
|
|
706
719
|
self.deadline,
|
|
707
720
|
self.ledger,
|
|
708
721
|
)?;
|
|
722
|
+
// Stamped when the ANSWER ARRIVES (tfs-102). It used to be stamped before the
|
|
723
|
+
// call, "so the call's own latency is spent out of the lease it is fetching
|
|
724
|
+
// rather than granted back to it" — but the URL does not exist for any of that
|
|
725
|
+
// time. The hub signs it while handling the call and states, in its own clock,
|
|
726
|
+
// the life it granted; the request leg, the hub's queueing, custody's signing and
|
|
727
|
+
// every retry `call` makes inside `presign` are all time the URL had not begun.
|
|
728
|
+
//
|
|
729
|
+
// Charging them to it made a slow mint POISON ITS OWN ANSWER. The hub's lifetime
|
|
730
|
+
// is ten minutes and a lease is spent at half of it, so a presign that took five
|
|
731
|
+
// minutes — one flaky tunnel, or two of `call`'s own retries — returned URLs that
|
|
732
|
+
// were unusable the instant they arrived, and the walk refused terminally on a
|
|
733
|
+
// paid pod with "a URL whose declared life is already spent when it is granted
|
|
734
|
+
// authorizes nothing". Twice on 2026-09-08, on gayong and weed, each after ~82 GB
|
|
735
|
+
// had already moved at 378 MB/s.
|
|
736
|
+
//
|
|
737
|
+
// What is left uncharged is the return leg alone, which is one hop and bounded by
|
|
738
|
+
// the call's own deadline.
|
|
739
|
+
let minted = Instant::now();
|
|
709
740
|
let mut held = self.held.lock().unwrap();
|
|
710
741
|
for (digest, url) in answer.urls {
|
|
711
742
|
held.insert(
|
|
@@ -1255,13 +1255,56 @@ fn hub_envelope_crosses_verbatim_as_typed_refusals() {
|
|
|
1255
1255
|
assert!(refusal.detail.contains("models.unknown_ref"));
|
|
1256
1256
|
assert!(refusal.detail.contains("check the ref"));
|
|
1257
1257
|
|
|
1258
|
-
let denied: Dispatch = Arc::new(|_: &Request|
|
|
1258
|
+
let denied: Dispatch = Arc::new(|_: &Request| {
|
|
1259
|
+
status(
|
|
1260
|
+
401,
|
|
1261
|
+
br#"{"error":{"code":"auth.credential_required","message":"no credential"}}"#,
|
|
1262
|
+
)
|
|
1263
|
+
});
|
|
1259
1264
|
let origin = self::origin(denied);
|
|
1260
1265
|
let refusal = pull(&request_for(&store, &origin.base, &policy)).unwrap_err();
|
|
1261
1266
|
assert_eq!(refusal.code, Code::CREDENTIAL_REQUIRED);
|
|
1262
1267
|
let _ = std::fs::remove_dir_all(store_root);
|
|
1263
1268
|
}
|
|
1264
1269
|
|
|
1270
|
+
// tfs-101. A status is evidence about the CATALOG only when the hub wrote it. On
|
|
1271
|
+
// 2026-09-08 the public origin's tunnel agent was seconds out of a reconnect loop and its
|
|
1272
|
+
// edge answered `POST /v1/tensorfs/presign` with its own 404 page; the status alone was
|
|
1273
|
+
// read as REF_NOT_FOUND, which the pod supervisor treats as poisoned, and a paid H100
|
|
1274
|
+
// journaled a permanent refusal for a placement set the hub was holding in full.
|
|
1275
|
+
#[test]
|
|
1276
|
+
fn a_status_with_no_hub_envelope_is_the_edge_and_stays_resumable() {
|
|
1277
|
+
let store_root = temporary("unenveloped");
|
|
1278
|
+
let store = Store::init(&store_root).unwrap();
|
|
1279
|
+
let policy = SourcePolicy::default();
|
|
1280
|
+
// Exactly what an ngrok endpoint answers while its agent session is down.
|
|
1281
|
+
let edge_page: Dispatch = Arc::new(|_: &Request| {
|
|
1282
|
+
status(
|
|
1283
|
+
404,
|
|
1284
|
+
b"<!DOCTYPE html><html><body>ERR_NGROK_3200 endpoint offline</body></html>",
|
|
1285
|
+
)
|
|
1286
|
+
});
|
|
1287
|
+
let origin = origin(edge_page);
|
|
1288
|
+
let refusal = pull(&request_for(&store, &origin.base, &policy)).unwrap_err();
|
|
1289
|
+
assert_eq!(refusal.code, Code::HUB_UNREACHABLE);
|
|
1290
|
+
assert!(refusal.detail.contains("no hub envelope"));
|
|
1291
|
+
assert!(refusal.detail.contains("404"));
|
|
1292
|
+
|
|
1293
|
+
// A 401 page from a gateway is not this hub asking for a credential it does not take.
|
|
1294
|
+
let gateway: Dispatch = Arc::new(|_: &Request| status(401, b"<html>Unauthorized</html>"));
|
|
1295
|
+
let origin = self::origin(gateway);
|
|
1296
|
+
let refusal = pull(&request_for(&store, &origin.base, &policy)).unwrap_err();
|
|
1297
|
+
assert_eq!(refusal.code, Code::HUB_UNREACHABLE);
|
|
1298
|
+
|
|
1299
|
+
// And a well-formed JSON body that is simply not the envelope stays weather too: the
|
|
1300
|
+
// test is the hub's own code, not the content type.
|
|
1301
|
+
let shaped: Dispatch = Arc::new(|_: &Request| status(404, br#"{"detail":"not found"}"#));
|
|
1302
|
+
let origin = self::origin(shaped);
|
|
1303
|
+
let refusal = pull(&request_for(&store, &origin.base, &policy)).unwrap_err();
|
|
1304
|
+
assert_eq!(refusal.code, Code::HUB_UNREACHABLE);
|
|
1305
|
+
let _ = std::fs::remove_dir_all(store_root);
|
|
1306
|
+
}
|
|
1307
|
+
|
|
1265
1308
|
#[test]
|
|
1266
1309
|
fn credential_headers_reach_the_hub_and_not_the_object_host() {
|
|
1267
1310
|
let root = temporary("cred");
|
|
@@ -2320,6 +2363,79 @@ fn a_403_is_judged_by_the_leases_age_and_not_by_a_count() {
|
|
|
2320
2363
|
);
|
|
2321
2364
|
}
|
|
2322
2365
|
|
|
2366
|
+
/// **A SLOW MINT MUST NOT POISON ITS OWN ANSWER.**
|
|
2367
|
+
///
|
|
2368
|
+
/// tfs-102, from two paid H100s on 2026-09-08. `gayong` and `weed` each moved ~82 GB of
|
|
2369
|
+
/// `paul/minimax-h3` at 378 MB/s, then stopped dead and refused with "a URL whose declared
|
|
2370
|
+
/// life is already spent when it is granted authorizes nothing". Nothing was wrong with the
|
|
2371
|
+
/// objects — the same closure landed clean on two other pods within the hour.
|
|
2372
|
+
///
|
|
2373
|
+
/// `minted` was stamped BEFORE the presign call, and a lease is spent at half the life the
|
|
2374
|
+
/// hub declared. The hub's life is ten minutes, so a presign that took five — one flaky
|
|
2375
|
+
/// tunnel, or two of `call`'s own retries — returned URLs that were unusable the instant
|
|
2376
|
+
/// they arrived, and the walk refused terminally with 82 GB already on disk.
|
|
2377
|
+
///
|
|
2378
|
+
/// The URL does not exist for the duration of the call that fetches it. The hub signs it
|
|
2379
|
+
/// while handling the request and states the life it granted in its own clock; the request
|
|
2380
|
+
/// leg, the queueing and the signing are time the URL had not begun. Stamping on ARRIVAL
|
|
2381
|
+
/// charges it only for the return leg, which is one hop.
|
|
2382
|
+
///
|
|
2383
|
+
/// Reaching the decision needs a mint slower than half a declared life, so the fixture
|
|
2384
|
+
/// shortens the LIFE rather than lengthening the call. Everything else is production.
|
|
2385
|
+
#[test]
|
|
2386
|
+
fn a_slow_mint_does_not_poison_the_urls_it_returns() {
|
|
2387
|
+
let root = temporary("slow-mint");
|
|
2388
|
+
let store = Store::init(&root).unwrap();
|
|
2389
|
+
let checkpoint = Arc::new(checkpoint(3, 4 * 1024));
|
|
2390
|
+
// The shortest life the wire can express is one second — the hub states its clocks in
|
|
2391
|
+
// whole seconds — so the call need only take past half of it. Stamped before the call,
|
|
2392
|
+
// every URL in the answer is over half its life on arrival and the walk has none it
|
|
2393
|
+
// may use; stamped on arrival, every one is whole.
|
|
2394
|
+
const DECLARED: u64 = 1;
|
|
2395
|
+
let slow = Duration::from_millis(700);
|
|
2396
|
+
let base_slot: Arc<Mutex<String>> = Arc::default();
|
|
2397
|
+
let dispatch_base = Arc::clone(&base_slot);
|
|
2398
|
+
let mints = Arc::new(AtomicUsize::new(0));
|
|
2399
|
+
let minted = Arc::clone(&mints);
|
|
2400
|
+
let served = Arc::clone(&checkpoint);
|
|
2401
|
+
let dispatch: Dispatch = Arc::new(move |request: &Request| {
|
|
2402
|
+
let base = dispatch_base.lock().unwrap().clone();
|
|
2403
|
+
match (request.method.as_str(), request.path.as_str()) {
|
|
2404
|
+
("POST", "/v1/tensorfs/closure") => ok_json(&closure_value(&served, &base)),
|
|
2405
|
+
("POST", "/v1/tensorfs/presign") => {
|
|
2406
|
+
minted.fetch_add(1, Ordering::Relaxed);
|
|
2407
|
+
std::thread::sleep(slow);
|
|
2408
|
+
let now = std::time::SystemTime::now()
|
|
2409
|
+
.duration_since(std::time::UNIX_EPOCH)
|
|
2410
|
+
.unwrap()
|
|
2411
|
+
.as_secs();
|
|
2412
|
+
let Value::Obj(mut fields) = presign_value(&request.body, &base) else {
|
|
2413
|
+
panic!()
|
|
2414
|
+
};
|
|
2415
|
+
fields.push(("expires_at_unix".into(), Value::uint(now + DECLARED)));
|
|
2416
|
+
fields.push(("server_time_unix".into(), Value::uint(now)));
|
|
2417
|
+
fields.sort_by(|a, b| a.0.cmp(&b.0));
|
|
2418
|
+
ok_json(&Value::Obj(fields))
|
|
2419
|
+
}
|
|
2420
|
+
("GET", path) if path.starts_with("/obj/") => {
|
|
2421
|
+
match served.bodies.get(&path["/obj/".len()..]) {
|
|
2422
|
+
Some(body) => status(200, body),
|
|
2423
|
+
None => status(404, b"no such object"),
|
|
2424
|
+
}
|
|
2425
|
+
}
|
|
2426
|
+
_ => status(404, b"no such route"),
|
|
2427
|
+
}
|
|
2428
|
+
});
|
|
2429
|
+
let origin = origin(dispatch);
|
|
2430
|
+
*base_slot.lock().unwrap() = origin.base.clone();
|
|
2431
|
+
let policy = SourcePolicy::default();
|
|
2432
|
+
let report = pull(&request_for(&store, &origin.base, &policy))
|
|
2433
|
+
.expect("a mint slower than half a declared life still returns usable URLs");
|
|
2434
|
+
assert_eq!(report.fetched, checkpoint.objects.len() as u64 + 1);
|
|
2435
|
+
assert!(mints.load(Ordering::Relaxed) >= 1);
|
|
2436
|
+
let _ = std::fs::remove_dir_all(root);
|
|
2437
|
+
}
|
|
2438
|
+
|
|
2323
2439
|
/// **A hub that says "wait" is waited for, and the same request is then answered.**
|
|
2324
2440
|
///
|
|
2325
2441
|
/// A rate limit is the purest resumable refusal in the protocol — the identical request gets
|
|
@@ -0,0 +1,217 @@
|
|
|
1
|
+
use std::{
|
|
2
|
+
fs,
|
|
3
|
+
path::{Path, PathBuf},
|
|
4
|
+
process::{Command, Stdio},
|
|
5
|
+
thread,
|
|
6
|
+
time::Duration,
|
|
7
|
+
};
|
|
8
|
+
use tensorfs_core::{
|
|
9
|
+
err::Code,
|
|
10
|
+
gc,
|
|
11
|
+
ids::{Doc, ObjectRef},
|
|
12
|
+
manifest::{Draft, Entry, Manifest},
|
|
13
|
+
source_artifact,
|
|
14
|
+
store::{Fault, Store},
|
|
15
|
+
};
|
|
16
|
+
|
|
17
|
+
fn directory(label: &str) -> PathBuf {
|
|
18
|
+
std::env::temp_dir().join(format!(
|
|
19
|
+
"tfs-import-{label}-{}-{}",
|
|
20
|
+
std::process::id(),
|
|
21
|
+
tensorfs_core::meta::now_nanos_unique()
|
|
22
|
+
))
|
|
23
|
+
}
|
|
24
|
+
fn owner() -> String {
|
|
25
|
+
ObjectRef::of(b"local import").id()
|
|
26
|
+
}
|
|
27
|
+
fn tree() -> Manifest {
|
|
28
|
+
Manifest::from_files(vec![
|
|
29
|
+
("a.json".into(), ObjectRef::of(b"first member")),
|
|
30
|
+
("nested/b.bin".into(), ObjectRef::of(b"second member")),
|
|
31
|
+
])
|
|
32
|
+
.unwrap()
|
|
33
|
+
}
|
|
34
|
+
fn files(dir: &Path) -> Vec<(String, PathBuf)> {
|
|
35
|
+
vec![
|
|
36
|
+
("a.json".into(), dir.join("a")),
|
|
37
|
+
("nested/b.bin".into(), dir.join("b")),
|
|
38
|
+
]
|
|
39
|
+
}
|
|
40
|
+
|
|
41
|
+
#[test]
|
|
42
|
+
fn refuses_roster_and_typed_header_before_starting_custody() {
|
|
43
|
+
let dir = directory("roster");
|
|
44
|
+
let store = Store::init(&dir).unwrap();
|
|
45
|
+
let paths = files(&dir);
|
|
46
|
+
for bad in [
|
|
47
|
+
paths[..1].to_vec(),
|
|
48
|
+
vec![paths[0].clone(), paths[0].clone()],
|
|
49
|
+
vec![paths[0].clone(), ("extra".into(), dir.join("extra"))],
|
|
50
|
+
] {
|
|
51
|
+
assert_eq!(
|
|
52
|
+
source_artifact::import_tree(&store, &owner(), &tree(), &bad, &Fault::default())
|
|
53
|
+
.unwrap_err()
|
|
54
|
+
.code,
|
|
55
|
+
Code::TRANSACTION_CONFLICT
|
|
56
|
+
);
|
|
57
|
+
assert!(source_artifact::read(&store, &owner()).unwrap().is_none());
|
|
58
|
+
}
|
|
59
|
+
let typed = Draft {
|
|
60
|
+
entries: vec![(
|
|
61
|
+
"model.ct".into(),
|
|
62
|
+
Entry::CozyTensors(ObjectRef::of(b"header")),
|
|
63
|
+
)],
|
|
64
|
+
}
|
|
65
|
+
.seal()
|
|
66
|
+
.unwrap();
|
|
67
|
+
assert_eq!(
|
|
68
|
+
source_artifact::import_tree(&store, &owner(), &typed, &paths, &Fault::default())
|
|
69
|
+
.unwrap_err()
|
|
70
|
+
.code,
|
|
71
|
+
Code::WRONG_TYPE
|
|
72
|
+
);
|
|
73
|
+
assert!(source_artifact::read(&store, &owner()).unwrap().is_none());
|
|
74
|
+
fs::remove_dir_all(dir).unwrap();
|
|
75
|
+
}
|
|
76
|
+
|
|
77
|
+
#[test]
|
|
78
|
+
fn verifies_bytes_and_resumes_only_missing_members() {
|
|
79
|
+
let dir = directory("verify");
|
|
80
|
+
let store = Store::init(&dir).unwrap();
|
|
81
|
+
fs::write(dir.join("a"), b"first member").unwrap();
|
|
82
|
+
fs::write(dir.join("b"), b"short").unwrap();
|
|
83
|
+
assert_eq!(
|
|
84
|
+
source_artifact::import_tree(&store, &owner(), &tree(), &files(&dir), &Fault::default())
|
|
85
|
+
.unwrap_err()
|
|
86
|
+
.code,
|
|
87
|
+
Code::LENGTH_MISMATCH
|
|
88
|
+
);
|
|
89
|
+
fs::write(dir.join("b"), b"wrong! member").unwrap();
|
|
90
|
+
assert_eq!(
|
|
91
|
+
source_artifact::import_tree(&store, &owner(), &tree(), &files(&dir), &Fault::default())
|
|
92
|
+
.unwrap_err()
|
|
93
|
+
.code,
|
|
94
|
+
Code::OBJECT_ID_MISMATCH
|
|
95
|
+
);
|
|
96
|
+
assert!(
|
|
97
|
+
!source_artifact::read(&store, &owner())
|
|
98
|
+
.unwrap()
|
|
99
|
+
.unwrap()
|
|
100
|
+
.complete
|
|
101
|
+
);
|
|
102
|
+
fs::remove_file(dir.join("a")).unwrap();
|
|
103
|
+
gc::collect(&dir, false).unwrap();
|
|
104
|
+
fs::write(dir.join("b"), b"second member").unwrap();
|
|
105
|
+
let result =
|
|
106
|
+
source_artifact::import_tree(&store, &owner(), &tree(), &files(&dir), &Fault::default())
|
|
107
|
+
.unwrap();
|
|
108
|
+
assert!(result.complete);
|
|
109
|
+
assert_eq!(result.manifest, tree().object_ref());
|
|
110
|
+
fs::remove_file(dir.join("b")).unwrap();
|
|
111
|
+
assert_eq!(
|
|
112
|
+
source_artifact::import_tree(&store, &owner(), &tree(), &files(&dir), &Fault::default())
|
|
113
|
+
.unwrap(),
|
|
114
|
+
result
|
|
115
|
+
);
|
|
116
|
+
let changed = Manifest::from_files(vec![("a.json".into(), ObjectRef::of(b"other"))]).unwrap();
|
|
117
|
+
assert_eq!(
|
|
118
|
+
source_artifact::import_tree(
|
|
119
|
+
&store,
|
|
120
|
+
&owner(),
|
|
121
|
+
&changed,
|
|
122
|
+
&files(&dir)[..1],
|
|
123
|
+
&Fault::default()
|
|
124
|
+
)
|
|
125
|
+
.unwrap_err()
|
|
126
|
+
.code,
|
|
127
|
+
Code::TRANSACTION_CONFLICT
|
|
128
|
+
);
|
|
129
|
+
let recipient = ObjectRef::of(b"recipient").id();
|
|
130
|
+
source_artifact::retain(&store, &owner(), &recipient).unwrap();
|
|
131
|
+
source_artifact::release(&store, &owner()).unwrap();
|
|
132
|
+
gc::collect(&dir, false).unwrap();
|
|
133
|
+
assert_eq!(store.read_manifest(&result.manifest).unwrap(), tree());
|
|
134
|
+
for (_, entry) in tree().entries() {
|
|
135
|
+
assert!(store.contains(&entry.blob().sha256));
|
|
136
|
+
}
|
|
137
|
+
assert_eq!(
|
|
138
|
+
source_artifact::import_tree(&store, &owner(), &tree(), &files(&dir), &Fault::default())
|
|
139
|
+
.unwrap_err()
|
|
140
|
+
.code,
|
|
141
|
+
Code::TRANSACTION_CLOSED
|
|
142
|
+
);
|
|
143
|
+
fs::remove_dir_all(dir).unwrap();
|
|
144
|
+
}
|
|
145
|
+
|
|
146
|
+
#[test]
|
|
147
|
+
fn import_child() {
|
|
148
|
+
let Some(dir) = std::env::var_os("TFS_IMPORT_TREE_CHILD") else {
|
|
149
|
+
return;
|
|
150
|
+
};
|
|
151
|
+
let dir = PathBuf::from(dir);
|
|
152
|
+
let store = Store::open(&dir).unwrap();
|
|
153
|
+
source_artifact::import_tree(
|
|
154
|
+
&store,
|
|
155
|
+
&owner(),
|
|
156
|
+
&tree(),
|
|
157
|
+
&files(&dir),
|
|
158
|
+
&Fault {
|
|
159
|
+
stage: Some("record".into()),
|
|
160
|
+
ready: Some(dir.join("ready")),
|
|
161
|
+
},
|
|
162
|
+
)
|
|
163
|
+
.unwrap();
|
|
164
|
+
panic!("fault should wait for the parent to kill this process");
|
|
165
|
+
}
|
|
166
|
+
|
|
167
|
+
#[test]
|
|
168
|
+
fn symlink_member_is_refused_before_reading_its_target() {
|
|
169
|
+
let dir = directory("symlink");
|
|
170
|
+
let store = Store::init(&dir).unwrap();
|
|
171
|
+
fs::write(dir.join("target"), b"first member").unwrap();
|
|
172
|
+
std::os::unix::fs::symlink(dir.join("target"), dir.join("a")).unwrap();
|
|
173
|
+
assert_eq!(
|
|
174
|
+
source_artifact::import_tree(&store, &owner(), &tree(), &files(&dir), &Fault::default())
|
|
175
|
+
.unwrap_err()
|
|
176
|
+
.code,
|
|
177
|
+
Code::WRONG_TYPE
|
|
178
|
+
);
|
|
179
|
+
assert!(!store.contains(&ObjectRef::of(b"first member").sha256));
|
|
180
|
+
fs::remove_dir_all(dir).unwrap();
|
|
181
|
+
}
|
|
182
|
+
|
|
183
|
+
#[test]
|
|
184
|
+
fn killed_after_admission_before_landed_keeps_member_live_for_gc_and_resume() {
|
|
185
|
+
let dir = directory("kill");
|
|
186
|
+
let store = Store::init(&dir).unwrap();
|
|
187
|
+
fs::write(dir.join("a"), b"first member").unwrap();
|
|
188
|
+
fs::write(dir.join("b"), b"second member").unwrap();
|
|
189
|
+
let mut child = Command::new(std::env::current_exe().unwrap())
|
|
190
|
+
.args(["--exact", "import_child", "--nocapture"])
|
|
191
|
+
.env("TFS_IMPORT_TREE_CHILD", &dir)
|
|
192
|
+
.stdout(Stdio::null())
|
|
193
|
+
.spawn()
|
|
194
|
+
.unwrap();
|
|
195
|
+
for _ in 0..1000 {
|
|
196
|
+
if dir.join("ready").is_file() {
|
|
197
|
+
break;
|
|
198
|
+
}
|
|
199
|
+
assert!(child.try_wait().unwrap().is_none());
|
|
200
|
+
thread::sleep(Duration::from_millis(5));
|
|
201
|
+
}
|
|
202
|
+
assert!(dir.join("ready").is_file());
|
|
203
|
+
child.kill().unwrap();
|
|
204
|
+
child.wait().unwrap();
|
|
205
|
+
let partial = source_artifact::read(&store, &owner()).unwrap().unwrap();
|
|
206
|
+
assert!(!partial.complete);
|
|
207
|
+
assert_eq!(partial.objects, vec![ObjectRef::of(b"first member")]);
|
|
208
|
+
gc::collect(&dir, false).unwrap();
|
|
209
|
+
fs::remove_file(dir.join("a")).unwrap();
|
|
210
|
+
let completed =
|
|
211
|
+
source_artifact::import_tree(&store, &owner(), &tree(), &files(&dir), &Fault::default())
|
|
212
|
+
.unwrap();
|
|
213
|
+
assert!(completed.complete);
|
|
214
|
+
gc::collect(&dir, false).unwrap();
|
|
215
|
+
assert_eq!(store.read_manifest(&completed.manifest).unwrap(), tree());
|
|
216
|
+
fs::remove_dir_all(dir).unwrap();
|
|
217
|
+
}
|