tensorfs 0.3.2__tar.gz → 0.3.3__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {tensorfs-0.3.2 → tensorfs-0.3.3}/Cargo.lock +2 -2
- {tensorfs-0.3.2 → tensorfs-0.3.3}/Cargo.toml +1 -1
- {tensorfs-0.3.2 → tensorfs-0.3.3}/PKG-INFO +1 -1
- {tensorfs-0.3.2 → tensorfs-0.3.3}/crates/tensorfs-core/src/bin/tfs/fetch.rs +63 -20
- {tensorfs-0.3.2 → tensorfs-0.3.3}/crates/tensorfs-core/src/bin/tfs/main.rs +13 -4
- {tensorfs-0.3.2 → tensorfs-0.3.3}/crates/tensorfs-core/src/catalog.rs +31 -4
- {tensorfs-0.3.2 → tensorfs-0.3.3}/crates/tensorfs-core/src/err.rs +1 -0
- {tensorfs-0.3.2 → tensorfs-0.3.3}/crates/tensorfs-core/src/home.rs +49 -8
- {tensorfs-0.3.2 → tensorfs-0.3.3}/crates/tensorfs-core/src/transport/mod.rs +1 -1
- {tensorfs-0.3.2 → tensorfs-0.3.3}/crates/tensorfs-core/src/transport/push.rs +104 -3
- {tensorfs-0.3.2 → tensorfs-0.3.3}/crates/tensorfs-core/src/transport/tests.rs +140 -5
- {tensorfs-0.3.2 → tensorfs-0.3.3}/crates/tensorfs-core/tests/home.rs +92 -0
- {tensorfs-0.3.2 → tensorfs-0.3.3}/crates/tensorfs-py/src/store.rs +3 -3
- {tensorfs-0.3.2 → tensorfs-0.3.3}/pyproject.toml +1 -1
- {tensorfs-0.3.2 → tensorfs-0.3.3}/crates/tensorfs-core/Cargo.toml +0 -0
- {tensorfs-0.3.2 → tensorfs-0.3.3}/crates/tensorfs-core/src/b64.rs +0 -0
- {tensorfs-0.3.2 → tensorfs-0.3.3}/crates/tensorfs-core/src/bin/tfs/ckpt.rs +0 -0
- {tensorfs-0.3.2 → tensorfs-0.3.3}/crates/tensorfs-core/src/bin/tfs/conform.rs +0 -0
- {tensorfs-0.3.2 → tensorfs-0.3.3}/crates/tensorfs-core/src/bin/tfs/ingest.rs +0 -0
- {tensorfs-0.3.2 → tensorfs-0.3.3}/crates/tensorfs-core/src/bin/tfs/rd.rs +0 -0
- {tensorfs-0.3.2 → tensorfs-0.3.3}/crates/tensorfs-core/src/bin/tfs/repo.rs +0 -0
- {tensorfs-0.3.2 → tensorfs-0.3.3}/crates/tensorfs-core/src/bin/tfs/snap.rs +0 -0
- {tensorfs-0.3.2 → tensorfs-0.3.3}/crates/tensorfs-core/src/bin/tfs/watch.rs +0 -0
- {tensorfs-0.3.2 → tensorfs-0.3.3}/crates/tensorfs-core/src/canon.rs +0 -0
- {tensorfs-0.3.2 → tensorfs-0.3.3}/crates/tensorfs-core/src/capability.rs +0 -0
- {tensorfs-0.3.2 → tensorfs-0.3.3}/crates/tensorfs-core/src/cbor.rs +0 -0
- {tensorfs-0.3.2 → tensorfs-0.3.3}/crates/tensorfs-core/src/checkpoint.rs +0 -0
- {tensorfs-0.3.2 → tensorfs-0.3.3}/crates/tensorfs-core/src/corpus.rs +0 -0
- {tensorfs-0.3.2 → tensorfs-0.3.3}/crates/tensorfs-core/src/derived.rs +0 -0
- {tensorfs-0.3.2 → tensorfs-0.3.3}/crates/tensorfs-core/src/dtype.rs +0 -0
- {tensorfs-0.3.2 → tensorfs-0.3.3}/crates/tensorfs-core/src/fetch.rs +0 -0
- {tensorfs-0.3.2 → tensorfs-0.3.3}/crates/tensorfs-core/src/fill.rs +0 -0
- {tensorfs-0.3.2 → tensorfs-0.3.3}/crates/tensorfs-core/src/fit.rs +0 -0
- {tensorfs-0.3.2 → tensorfs-0.3.3}/crates/tensorfs-core/src/gc.rs +0 -0
- {tensorfs-0.3.2 → tensorfs-0.3.3}/crates/tensorfs-core/src/header.rs +0 -0
- {tensorfs-0.3.2 → tensorfs-0.3.3}/crates/tensorfs-core/src/ids.rs +0 -0
- {tensorfs-0.3.2 → tensorfs-0.3.3}/crates/tensorfs-core/src/ingest/carrier.rs +0 -0
- {tensorfs-0.3.2 → tensorfs-0.3.3}/crates/tensorfs-core/src/ingest/convert.rs +0 -0
- {tensorfs-0.3.2 → tensorfs-0.3.3}/crates/tensorfs-core/src/ingest/fingerprint.rs +0 -0
- {tensorfs-0.3.2 → tensorfs-0.3.3}/crates/tensorfs-core/src/ingest/gguf.rs +0 -0
- {tensorfs-0.3.2 → tensorfs-0.3.3}/crates/tensorfs-core/src/ingest/mod.rs +0 -0
- {tensorfs-0.3.2 → tensorfs-0.3.3}/crates/tensorfs-core/src/ingest/source.rs +0 -0
- {tensorfs-0.3.2 → tensorfs-0.3.3}/crates/tensorfs-core/src/ingest/stamp.rs +0 -0
- {tensorfs-0.3.2 → tensorfs-0.3.3}/crates/tensorfs-core/src/ingest/transaction.rs +0 -0
- {tensorfs-0.3.2 → tensorfs-0.3.3}/crates/tensorfs-core/src/jcs.rs +0 -0
- {tensorfs-0.3.2 → tensorfs-0.3.3}/crates/tensorfs-core/src/lib.rs +0 -0
- {tensorfs-0.3.2 → tensorfs-0.3.3}/crates/tensorfs-core/src/limits.rs +0 -0
- {tensorfs-0.3.2 → tensorfs-0.3.3}/crates/tensorfs-core/src/machine.rs +0 -0
- {tensorfs-0.3.2 → tensorfs-0.3.3}/crates/tensorfs-core/src/manifest.rs +0 -0
- {tensorfs-0.3.2 → tensorfs-0.3.3}/crates/tensorfs-core/src/meta.rs +0 -0
- {tensorfs-0.3.2 → tensorfs-0.3.3}/crates/tensorfs-core/src/project.rs +0 -0
- {tensorfs-0.3.2 → tensorfs-0.3.3}/crates/tensorfs-core/src/providers.rs +0 -0
- {tensorfs-0.3.2 → tensorfs-0.3.3}/crates/tensorfs-core/src/read.rs +0 -0
- {tensorfs-0.3.2 → tensorfs-0.3.3}/crates/tensorfs-core/src/receipt.rs +0 -0
- {tensorfs-0.3.2 → tensorfs-0.3.3}/crates/tensorfs-core/src/registry.rs +0 -0
- {tensorfs-0.3.2 → tensorfs-0.3.3}/crates/tensorfs-core/src/relation.rs +0 -0
- {tensorfs-0.3.2 → tensorfs-0.3.3}/crates/tensorfs-core/src/repo_cache.rs +0 -0
- {tensorfs-0.3.2 → tensorfs-0.3.3}/crates/tensorfs-core/src/repository.rs +0 -0
- {tensorfs-0.3.2 → tensorfs-0.3.3}/crates/tensorfs-core/src/sha256.rs +0 -0
- {tensorfs-0.3.2 → tensorfs-0.3.3}/crates/tensorfs-core/src/spec.rs +0 -0
- {tensorfs-0.3.2 → tensorfs-0.3.3}/crates/tensorfs-core/src/storage.rs +0 -0
- {tensorfs-0.3.2 → tensorfs-0.3.3}/crates/tensorfs-core/src/store.rs +0 -0
- {tensorfs-0.3.2 → tensorfs-0.3.3}/crates/tensorfs-core/src/transport/api.rs +0 -0
- {tensorfs-0.3.2 → tensorfs-0.3.3}/crates/tensorfs-core/src/transport/http.rs +0 -0
- {tensorfs-0.3.2 → tensorfs-0.3.3}/crates/tensorfs-core/src/transport/hub.rs +0 -0
- {tensorfs-0.3.2 → tensorfs-0.3.3}/crates/tensorfs-core/src/transport/ledger.rs +0 -0
- {tensorfs-0.3.2 → tensorfs-0.3.3}/crates/tensorfs-core/src/transport/policy.rs +0 -0
- {tensorfs-0.3.2 → tensorfs-0.3.3}/crates/tensorfs-core/src/transport/pull.rs +0 -0
- {tensorfs-0.3.2 → tensorfs-0.3.3}/crates/tensorfs-core/src/vectors.rs +0 -0
- {tensorfs-0.3.2 → tensorfs-0.3.3}/crates/tensorfs-core/tests/fit.rs +0 -0
- {tensorfs-0.3.2 → tensorfs-0.3.3}/crates/tensorfs-core/tests/fixtures/repo_apply/blobs.jsonl +0 -0
- {tensorfs-0.3.2 → tensorfs-0.3.3}/crates/tensorfs-core/tests/fixtures/repo_apply/expected_list.jsonl +0 -0
- {tensorfs-0.3.2 → tensorfs-0.3.3}/crates/tensorfs-core/tests/fixtures/repo_apply/expected_repository.json +0 -0
- {tensorfs-0.3.2 → tensorfs-0.3.3}/crates/tensorfs-core/tests/fixtures/repo_apply/expected_rows.jsonl +0 -0
- {tensorfs-0.3.2 → tensorfs-0.3.3}/crates/tensorfs-core/tests/fixtures/repo_apply/header.cbor +0 -0
- {tensorfs-0.3.2 → tensorfs-0.3.3}/crates/tensorfs-core/tests/fixtures/repo_apply/manifest.json +0 -0
- {tensorfs-0.3.2 → tensorfs-0.3.3}/crates/tensorfs-core/tests/fixtures/repo_apply/mutation.json +0 -0
- {tensorfs-0.3.2 → tensorfs-0.3.3}/crates/tensorfs-core/tests/liveness.rs +0 -0
- {tensorfs-0.3.2 → tensorfs-0.3.3}/crates/tensorfs-core/tests/repo_cache.rs +0 -0
- {tensorfs-0.3.2 → tensorfs-0.3.3}/crates/tensorfs-core/tests/repo_store.rs +0 -0
- {tensorfs-0.3.2 → tensorfs-0.3.3}/crates/tensorfs-core/tests/source_plan.rs +0 -0
- {tensorfs-0.3.2 → tensorfs-0.3.3}/crates/tensorfs-core/tests/store_boundaries.rs +0 -0
- {tensorfs-0.3.2 → tensorfs-0.3.3}/crates/tensorfs-core/tests/tensor_schema.rs +0 -0
- {tensorfs-0.3.2 → tensorfs-0.3.3}/crates/tensorfs-core/tests/version.rs +0 -0
- {tensorfs-0.3.2 → tensorfs-0.3.3}/crates/tensorfs-py/Cargo.toml +0 -0
- {tensorfs-0.3.2 → tensorfs-0.3.3}/crates/tensorfs-py/src/docs.rs +0 -0
- {tensorfs-0.3.2 → tensorfs-0.3.3}/crates/tensorfs-py/src/errors.rs +0 -0
- {tensorfs-0.3.2 → tensorfs-0.3.3}/crates/tensorfs-py/src/lib.rs +0 -0
- {tensorfs-0.3.2 → tensorfs-0.3.3}/python/tensorfs/__init__.py +0 -0
- {tensorfs-0.3.2 → tensorfs-0.3.3}/python/tensorfs/_ext.pyi +0 -0
- {tensorfs-0.3.2 → tensorfs-0.3.3}/python/tensorfs/errors.py +0 -0
- {tensorfs-0.3.2 → tensorfs-0.3.3}/python/tensorfs/py.typed +0 -0
|
@@ -392,7 +392,7 @@ checksum = "adb6935a6f5c20170eeceb1a3835a49e12e19d792f6dd344ccc76a985ca5a6ca"
|
|
|
392
392
|
|
|
393
393
|
[[package]]
|
|
394
394
|
name = "tensorfs-core"
|
|
395
|
-
version = "0.3.
|
|
395
|
+
version = "0.3.3"
|
|
396
396
|
dependencies = [
|
|
397
397
|
"fs2",
|
|
398
398
|
"minicbor",
|
|
@@ -404,7 +404,7 @@ dependencies = [
|
|
|
404
404
|
|
|
405
405
|
[[package]]
|
|
406
406
|
name = "tensorfs-py"
|
|
407
|
-
version = "0.3.
|
|
407
|
+
version = "0.3.3"
|
|
408
408
|
dependencies = [
|
|
409
409
|
"pyo3",
|
|
410
410
|
"tensorfs-core",
|
|
@@ -336,35 +336,78 @@ pub fn cmd_url(root: &Path, id: &str, length: &str, url: &str, flags: &Flags) ->
|
|
|
336
336
|
ExitCode::SUCCESS
|
|
337
337
|
}
|
|
338
338
|
|
|
339
|
-
///
|
|
340
|
-
///
|
|
341
|
-
///
|
|
342
|
-
|
|
339
|
+
/// The push destination, reaching the child exactly the way the credential does (tfs-050) —
|
|
340
|
+
/// never argv, which is world-readable in `/proc`, and a presigned PUT is a bearer secret.
|
|
341
|
+
/// A minted grant is a destination AND the headers its signature covers, so the two travel
|
|
342
|
+
/// as ONE document and there is no spelling that spends the URL without its conditions:
|
|
343
|
+
/// `--grant-file <path>` or one deliberately-set `TFS_PUSH_GRANT`, the URL on the first
|
|
344
|
+
/// line then one `name: value` per signed header. `--url <presigned|->` remains for a bare
|
|
345
|
+
/// destination that signs nothing, `-` reading it from stdin. Exactly one of the three.
|
|
346
|
+
fn grant_of(flags: &Flags) -> Result<transport::UploadGrant, ExitCode> {
|
|
343
347
|
use std::io::BufRead;
|
|
348
|
+
let file = flag(flags, "grant-file");
|
|
349
|
+
let environment = std::env::var("TFS_PUSH_GRANT").unwrap_or_default();
|
|
350
|
+
let url = flag(flags, "url");
|
|
351
|
+
let given = [
|
|
352
|
+
file.is_some(),
|
|
353
|
+
!environment.trim().is_empty(),
|
|
354
|
+
url.is_some(),
|
|
355
|
+
];
|
|
356
|
+
if given.iter().filter(|named| **named).count() != 1 {
|
|
357
|
+
eprintln!(
|
|
358
|
+
"exactly one of --grant-file <path>, TFS_PUSH_GRANT, or --url <presigned|-> \
|
|
359
|
+
names the destination"
|
|
360
|
+
);
|
|
361
|
+
return Err(ExitCode::from(2));
|
|
362
|
+
}
|
|
363
|
+
let document = match file {
|
|
364
|
+
Some(path) => match fs::read_to_string(path) {
|
|
365
|
+
Ok(text) => Some(text),
|
|
366
|
+
Err(error) => {
|
|
367
|
+
eprintln!("REFUSED IO_FAILED: read {path}: {error}");
|
|
368
|
+
return Err(ExitCode::FAILURE);
|
|
369
|
+
}
|
|
370
|
+
},
|
|
371
|
+
None if !environment.trim().is_empty() => Some(environment),
|
|
372
|
+
None => None,
|
|
373
|
+
};
|
|
374
|
+
if let Some(document) = document {
|
|
375
|
+
return match transport::UploadGrant::parse(&document) {
|
|
376
|
+
Ok(grant) => Ok(grant),
|
|
377
|
+
Err(refusal) => {
|
|
378
|
+
eprintln!("REFUSED {refusal}");
|
|
379
|
+
Err(ExitCode::FAILURE)
|
|
380
|
+
}
|
|
381
|
+
};
|
|
382
|
+
}
|
|
383
|
+
let url = url.unwrap_or_default();
|
|
384
|
+
if url != "-" {
|
|
385
|
+
return Ok(transport::UploadGrant::to(url));
|
|
386
|
+
}
|
|
387
|
+
let mut line = String::new();
|
|
388
|
+
if io::stdin().lock().read_line(&mut line).is_err() {
|
|
389
|
+
eprintln!("REFUSED SOURCE_NOT_ALLOWED: could not read the URL from stdin");
|
|
390
|
+
return Err(ExitCode::FAILURE);
|
|
391
|
+
}
|
|
392
|
+
Ok(transport::UploadGrant::to(line.trim()))
|
|
393
|
+
}
|
|
394
|
+
|
|
395
|
+
/// `tfs push <root> <sha256> --grant-file <path> [--manifest]` — the upload half: one
|
|
396
|
+
/// verified store object to the one destination a grant names, under the exact headers
|
|
397
|
+
/// its signature covers, 412 a success under `If-None-Match: *`.
|
|
398
|
+
pub fn cmd_push(root: &Path, id: &str, flags: &Flags) -> ExitCode {
|
|
344
399
|
let store = ok!(Store::open(root));
|
|
345
400
|
let sha256 = ok!(tensorfs_core::ids::hex64("push object", id));
|
|
346
|
-
let
|
|
347
|
-
|
|
348
|
-
return
|
|
349
|
-
};
|
|
350
|
-
let stdin_url;
|
|
351
|
-
let url = if url_flag == "-" {
|
|
352
|
-
let mut line = String::new();
|
|
353
|
-
if io::stdin().lock().read_line(&mut line).is_err() {
|
|
354
|
-
eprintln!("REFUSED SOURCE_NOT_ALLOWED: could not read the URL from stdin");
|
|
355
|
-
return ExitCode::FAILURE;
|
|
356
|
-
}
|
|
357
|
-
stdin_url = line.trim().to_string();
|
|
358
|
-
stdin_url.as_str()
|
|
359
|
-
} else {
|
|
360
|
-
url_flag
|
|
401
|
+
let grant = match grant_of(flags) {
|
|
402
|
+
Ok(grant) => grant,
|
|
403
|
+
Err(code) => return code,
|
|
361
404
|
};
|
|
362
405
|
let ledger = transport::Ledger::new();
|
|
363
406
|
let pushed = ok!(transport::push_object(
|
|
364
407
|
&store,
|
|
365
408
|
&sha256,
|
|
366
409
|
crate::flag_on(flags, "manifest"),
|
|
367
|
-
|
|
410
|
+
&grant,
|
|
368
411
|
&policy_of(flags),
|
|
369
412
|
&Anonymous,
|
|
370
413
|
deadline_of(flags),
|
|
@@ -10,6 +10,10 @@
|
|
|
10
10
|
//! --range <off>:<len> --rate <f> --pass <n> --slow-ms <n>
|
|
11
11
|
//! --size <bytes> --small <n>, plus the dev-only kill points --fault <stage> --ready <path>.
|
|
12
12
|
//!
|
|
13
|
+
//! A Store is created only where someone said to: `tfs store init|ensure <root>`, or the same
|
|
14
|
+
//! verbs over `TENSORFS_HOME`. Every other command REFUSES a root that does not exist, and the
|
|
15
|
+
//! `~/.tensorfs` fallback is opened, never conjured (tfs-053).
|
|
16
|
+
//!
|
|
13
17
|
//! tfs store init|info <root>
|
|
14
18
|
//! tfs store ensure [<root>]
|
|
15
19
|
//! tfs store prepare-readers [<root>]
|
|
@@ -36,7 +40,10 @@
|
|
|
36
40
|
//! tfs fetch <root> <ref> closure -> plan -> presign(wanted) -> N-way walk -> proof
|
|
37
41
|
//! tfs fetch plan-objects <root> <session> --refs <jsonl> HELD/WANTED, no manifest
|
|
38
42
|
//! tfs fetch url <root> <sha256> <length> <url|-> one granted object
|
|
39
|
-
//! tfs push <root> <sha256> --
|
|
43
|
+
//! tfs push <root> <sha256> --grant-file <path> [--manifest] [--content-type ct]
|
|
44
|
+
//! the grant is the destination on line one, then one "name: value" per header
|
|
45
|
+
//! its signature covers; TFS_PUSH_GRANT carries the same text, and
|
|
46
|
+
//! --url <presigned|-> is a bare destination that signs no conditions
|
|
40
47
|
//! tfs source list <hf://org/repo@rev | civitai://id> [--api <base>] provider listing
|
|
41
48
|
//! tfs source pull <root> <uri> [--api <base>] [--allow-local] tfs-052 handlers
|
|
42
49
|
//! tfs bench-store <root> put/read/range/rehash throughput + peak RSS
|
|
@@ -558,8 +565,10 @@ fn cmd_home_migrate(source: &Path, flags: &Flags) -> ExitCode {
|
|
|
558
565
|
ExitCode::SUCCESS
|
|
559
566
|
}
|
|
560
567
|
|
|
568
|
+
/// The no-root spelling of a `store` verb. It uses `TENSORFS_HOME`, or an existing
|
|
569
|
+
/// `$HOME/.tensorfs` — never a Store it would have to invent (tfs-053).
|
|
561
570
|
fn cmd_store_default(verb: &str) -> ExitCode {
|
|
562
|
-
cmd_store(&ok!(tensorfs_core::home::
|
|
571
|
+
cmd_store(&ok!(tensorfs_core::home::store_root(None)), verb)
|
|
563
572
|
}
|
|
564
573
|
|
|
565
574
|
fn cmd_store(root: &Path, verb: &str) -> ExitCode {
|
|
@@ -574,7 +583,7 @@ fn cmd_store(root: &Path, verb: &str) -> ExitCode {
|
|
|
574
583
|
}
|
|
575
584
|
|
|
576
585
|
fn cmd_store_prepare_readers(explicit: Option<&Path>) -> ExitCode {
|
|
577
|
-
let root = ok!(tensorfs_core::home::
|
|
586
|
+
let root = ok!(tensorfs_core::home::store_root(explicit));
|
|
578
587
|
let store = ok!(Store::open(&root));
|
|
579
588
|
ok!(store.prepare_readers());
|
|
580
589
|
println!("root: {}", store.root().display());
|
|
@@ -583,7 +592,7 @@ fn cmd_store_prepare_readers(explicit: Option<&Path>) -> ExitCode {
|
|
|
583
592
|
}
|
|
584
593
|
|
|
585
594
|
fn cmd_store_complete_cozytensors(explicit: Option<&Path>) -> ExitCode {
|
|
586
|
-
let root = ok!(tensorfs_core::home::
|
|
595
|
+
let root = ok!(tensorfs_core::home::store_root(explicit));
|
|
587
596
|
let store = ok!(Store::open(&root));
|
|
588
597
|
for manifest in ok!(store.complete_cozytensors_manifests()) {
|
|
589
598
|
println!("{manifest}");
|
|
@@ -861,10 +861,38 @@ pub struct WriterGuard {
|
|
|
861
861
|
_recovery: File,
|
|
862
862
|
}
|
|
863
863
|
|
|
864
|
+
/// `tmp/writers` inside a root that already exists.
|
|
865
|
+
///
|
|
866
|
+
/// A writer marker is never the thing that brings a root into being: pointed at an absent path
|
|
867
|
+
/// this used to lay down a directory tree wherever the caller happened to be (tfs-053).
|
|
868
|
+
fn writer_markers(root: &Path) -> Result<PathBuf> {
|
|
869
|
+
match fs::symlink_metadata(root) {
|
|
870
|
+
Ok(metadata) if metadata.is_dir() && !metadata.file_type().is_symlink() => {}
|
|
871
|
+
Ok(_) => {
|
|
872
|
+
return refuse(
|
|
873
|
+
Code::STORE_ERA,
|
|
874
|
+
format!("{} is not a real Store directory", root.display()),
|
|
875
|
+
)
|
|
876
|
+
}
|
|
877
|
+
Err(error) if error.kind() == std::io::ErrorKind::NotFound => {
|
|
878
|
+
return refuse(
|
|
879
|
+
Code::STORE_ROOT_ABSENT,
|
|
880
|
+
format!(
|
|
881
|
+
"{} does not exist; create the root before writing into it",
|
|
882
|
+
root.display()
|
|
883
|
+
),
|
|
884
|
+
)
|
|
885
|
+
}
|
|
886
|
+
Err(error) => return Err(io(format!("stat root {}", root.display()), error)),
|
|
887
|
+
}
|
|
888
|
+
let directory = root.join("tmp/writers");
|
|
889
|
+
fs::create_dir_all(&directory).map_err(|error| io("mkdir writer markers", error))?;
|
|
890
|
+
Ok(directory)
|
|
891
|
+
}
|
|
892
|
+
|
|
864
893
|
impl WriterGuard {
|
|
865
894
|
pub fn acquire(root: &Path) -> Result<Self> {
|
|
866
|
-
let directory = root
|
|
867
|
-
fs::create_dir_all(&directory).map_err(|error| io("mkdir writer markers", error))?;
|
|
895
|
+
let directory = writer_markers(root)?;
|
|
868
896
|
let recovery = File::options()
|
|
869
897
|
.create(true)
|
|
870
898
|
.truncate(false)
|
|
@@ -894,8 +922,7 @@ impl WriterGuard {
|
|
|
894
922
|
}
|
|
895
923
|
|
|
896
924
|
pub fn lock_rebuild(root: &Path) -> Result<RebuildGuard> {
|
|
897
|
-
let directory = root
|
|
898
|
-
fs::create_dir_all(&directory).map_err(|error| io("mkdir writer markers", error))?;
|
|
925
|
+
let directory = writer_markers(root)?;
|
|
899
926
|
let recovery = File::options()
|
|
900
927
|
.create(true)
|
|
901
928
|
.truncate(false)
|
|
@@ -2,6 +2,12 @@
|
|
|
2
2
|
//!
|
|
3
3
|
//! Core Store operations remain explicit-root. This resolver exists for user-facing entrypoints
|
|
4
4
|
//! and never consults another product's home or configuration.
|
|
5
|
+
//!
|
|
6
|
+
//! A root the operator NAMED — a positional argument or `TENSORFS_HOME` — is a place they asked
|
|
7
|
+
//! for. `$HOME/.tensorfs` is a guess, so it is only ever OPENED: no command conjures a Store
|
|
8
|
+
//! there, and nothing creates a Store in whatever directory a process happens to be run from
|
|
9
|
+
//! (tfs-053). Creation stays with the verbs that say so, `tfs store init` and `tfs store ensure`,
|
|
10
|
+
//! over a root that was named.
|
|
5
11
|
|
|
6
12
|
use std::fs;
|
|
7
13
|
use std::os::unix::fs::MetadataExt;
|
|
@@ -19,10 +25,43 @@ pub struct MoveReport {
|
|
|
19
25
|
pub destination: PathBuf,
|
|
20
26
|
}
|
|
21
27
|
|
|
28
|
+
/// Where a resolved root came from. `Named` roots were asked for; `Fallback` is a guess.
|
|
29
|
+
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
|
30
|
+
enum Origin {
|
|
31
|
+
Named,
|
|
32
|
+
Fallback,
|
|
33
|
+
}
|
|
34
|
+
|
|
22
35
|
/// Resolve an optional caller-supplied Store root.
|
|
23
36
|
///
|
|
24
|
-
/// Precedence is explicit argument, `TENSORFS_HOME`, then `$HOME/.tensorfs`.
|
|
37
|
+
/// Precedence is explicit argument, `TENSORFS_HOME`, then `$HOME/.tensorfs`. This answers WHERE
|
|
38
|
+
/// a Store would live; it neither requires nor creates one. `tfs home` prints exactly this.
|
|
25
39
|
pub fn resolve(explicit: Option<&Path>) -> Result<PathBuf> {
|
|
40
|
+
Ok(resolve_env(explicit)?.0)
|
|
41
|
+
}
|
|
42
|
+
|
|
43
|
+
/// The Store root of a command that needs a Store.
|
|
44
|
+
///
|
|
45
|
+
/// A named root — argument or `TENSORFS_HOME` — is used as given, and `tfs store init`/`ensure`
|
|
46
|
+
/// may bring it into existence. The `$HOME/.tensorfs` fallback is accepted only for a Store that
|
|
47
|
+
/// ALREADY exists: with nothing named and nothing there, every command refuses instead of
|
|
48
|
+
/// materializing a database nobody asked for.
|
|
49
|
+
pub fn store_root(explicit: Option<&Path>) -> Result<PathBuf> {
|
|
50
|
+
let (root, origin) = resolve_env(explicit)?;
|
|
51
|
+
if origin == Origin::Named || root.is_dir() {
|
|
52
|
+
return Ok(root);
|
|
53
|
+
}
|
|
54
|
+
refuse(
|
|
55
|
+
Code::STORE_ROOT_ABSENT,
|
|
56
|
+
format!(
|
|
57
|
+
"no Store root was given and the default {} does not exist: name a root, set {ENV}, \
|
|
58
|
+
or create one with `tfs store init <root>`",
|
|
59
|
+
root.display()
|
|
60
|
+
),
|
|
61
|
+
)
|
|
62
|
+
}
|
|
63
|
+
|
|
64
|
+
fn resolve_env(explicit: Option<&Path>) -> Result<(PathBuf, Origin)> {
|
|
26
65
|
resolve_values(
|
|
27
66
|
explicit,
|
|
28
67
|
std::env::var_os(ENV).map(PathBuf::from).as_deref(),
|
|
@@ -34,15 +73,17 @@ fn resolve_values(
|
|
|
34
73
|
explicit: Option<&Path>,
|
|
35
74
|
tensorfs_home: Option<&Path>,
|
|
36
75
|
user_home: Option<&Path>,
|
|
37
|
-
) -> Result<PathBuf> {
|
|
76
|
+
) -> Result<(PathBuf, Origin)> {
|
|
38
77
|
if let Some(root) = explicit {
|
|
39
|
-
return nonempty("explicit TensorFS root", root);
|
|
78
|
+
return Ok((nonempty("explicit TensorFS root", root)?, Origin::Named));
|
|
40
79
|
}
|
|
41
80
|
if let Some(root) = tensorfs_home {
|
|
42
|
-
return nonempty(ENV, root);
|
|
81
|
+
return Ok((nonempty(ENV, root)?, Origin::Named));
|
|
43
82
|
}
|
|
44
83
|
match user_home {
|
|
45
|
-
Some(root) if !root.as_os_str().is_empty() =>
|
|
84
|
+
Some(root) if !root.as_os_str().is_empty() => {
|
|
85
|
+
Ok((root.join(".tensorfs"), Origin::Fallback))
|
|
86
|
+
}
|
|
46
87
|
_ => refuse(
|
|
47
88
|
Code::IO_FAILED,
|
|
48
89
|
"TensorFS root is absent: pass one explicitly, set TENSORFS_HOME, or set HOME",
|
|
@@ -162,7 +203,7 @@ mod tests {
|
|
|
162
203
|
Some(Path::new("/user")),
|
|
163
204
|
)
|
|
164
205
|
.unwrap(),
|
|
165
|
-
|
|
206
|
+
(PathBuf::from("/explicit"), Origin::Named),
|
|
166
207
|
);
|
|
167
208
|
assert_eq!(
|
|
168
209
|
resolve_values(
|
|
@@ -171,11 +212,11 @@ mod tests {
|
|
|
171
212
|
Some(Path::new("/user"))
|
|
172
213
|
)
|
|
173
214
|
.unwrap(),
|
|
174
|
-
|
|
215
|
+
(PathBuf::from("/environment"), Origin::Named),
|
|
175
216
|
);
|
|
176
217
|
assert_eq!(
|
|
177
218
|
resolve_values(None, None, Some(Path::new("/user"))).unwrap(),
|
|
178
|
-
|
|
219
|
+
(PathBuf::from("/user/.tensorfs"), Origin::Fallback),
|
|
179
220
|
);
|
|
180
221
|
}
|
|
181
222
|
|
|
@@ -53,4 +53,4 @@ pub use policy::{base_host, blocked, is_loopback_origin, resolve, CheckedUrl, So
|
|
|
53
53
|
pub use pull::{
|
|
54
54
|
fetch_url, fetch_wanted, pull, Fetched, PullReport, PullRequest, WalkOutcome, RETRYABLE_STATUS,
|
|
55
55
|
};
|
|
56
|
-
pub use push::{push_object, Pushed, PUT_ATTEMPTS};
|
|
56
|
+
pub use push::{push_object, Pushed, UploadGrant, MAX_GRANT_BYTES, PUT_ATTEMPTS};
|
|
@@ -14,6 +14,16 @@
|
|
|
14
14
|
//! silence bound is the ledger's floor — the same derived noise floor `egress.py` used —
|
|
15
15
|
//! never an invented constant. Authority stays where it was: `podweights` (or whoever
|
|
16
16
|
//! minted the grant) decided WHETHER these bytes may go; this module only moves them.
|
|
17
|
+
//!
|
|
18
|
+
//! **A grant is a URL AND the headers its signature covers, together.** tensorhub signs
|
|
19
|
+
//! `if-none-match: *` and `x-amz-checksum-sha256: <base64 of the raw digest>` into every
|
|
20
|
+
//! upload grant it mints (`podweights.validateGrant` demands exactly those two), and a
|
|
21
|
+
//! SigV4 store answers 400 to a PUT that omits a signed header. A mover that carried the
|
|
22
|
+
//! URL and dropped the conditions therefore could not spend ANY minted grant — which is
|
|
23
|
+
//! what th-132 found. So the destination and its conditions are ONE value, `UploadGrant`,
|
|
24
|
+
//! and no spelling reaches this function with one and not the other. The headers cross the
|
|
25
|
+
//! wire verbatim, exactly once each, on every attempt, and they win over this module's own
|
|
26
|
+
//! defaults and over the credential's: the grant's spelling is what the signature covers.
|
|
17
27
|
|
|
18
28
|
use crate::err::{refuse, Code, Refusal, Result};
|
|
19
29
|
use crate::ids::{Doc, ObjectRef};
|
|
@@ -30,6 +40,88 @@ pub const PUT_ATTEMPTS: u32 = 4;
|
|
|
30
40
|
|
|
31
41
|
const HTTP_PRECONDITION_FAILED: u16 = 412;
|
|
32
42
|
|
|
43
|
+
/// One upload grant: the exact destination and the exact headers its signature covers.
|
|
44
|
+
/// The pair is indivisible — a presigned PUT spent without its signed conditions is not a
|
|
45
|
+
/// degraded push, it is a 400 — so this is the only way to name a push destination.
|
|
46
|
+
#[derive(Debug, Clone, Default, PartialEq, Eq)]
|
|
47
|
+
pub struct UploadGrant {
|
|
48
|
+
pub url: String,
|
|
49
|
+
/// Lowercase names in the grant's own order, each sent verbatim exactly once.
|
|
50
|
+
pub headers: Vec<(String, String)>,
|
|
51
|
+
}
|
|
52
|
+
|
|
53
|
+
/// A grant is a URL, a few signed headers and nothing else; anything larger is not one.
|
|
54
|
+
pub const MAX_GRANT_BYTES: usize = 64 * 1024;
|
|
55
|
+
|
|
56
|
+
impl UploadGrant {
|
|
57
|
+
/// A destination that signs no conditions — a plain PUT target, not a minted grant.
|
|
58
|
+
pub fn to(url: &str) -> UploadGrant {
|
|
59
|
+
UploadGrant {
|
|
60
|
+
url: url.to_string(),
|
|
61
|
+
headers: Vec::new(),
|
|
62
|
+
}
|
|
63
|
+
}
|
|
64
|
+
|
|
65
|
+
/// The grant as its minter writes it, in the line grammar the credential file already
|
|
66
|
+
/// uses: the destination on the first line, then one `name: value` per line for each
|
|
67
|
+
/// header the signature covers. No escaping, no canonical-bytes question — a presigned
|
|
68
|
+
/// URL is full of `&` and a base64 checksum of `+/=`, and a grammar that can mangle
|
|
69
|
+
/// either is a grammar that loses the signature.
|
|
70
|
+
///
|
|
71
|
+
/// ```text
|
|
72
|
+
/// https://bucket.s3.amazonaws.com/o/<key>?X-Amz-Algorithm=...&X-Amz-Signature=...
|
|
73
|
+
/// if-none-match: *
|
|
74
|
+
/// x-amz-checksum-sha256: 47DEQpj8HBSa+/TImW+5JCeuQeRkm5NMpJWZG3hSuFU=
|
|
75
|
+
/// ```
|
|
76
|
+
pub fn parse(text: &str) -> Result<UploadGrant> {
|
|
77
|
+
if text.len() > MAX_GRANT_BYTES {
|
|
78
|
+
return refuse(
|
|
79
|
+
Code::SIZE_CAP,
|
|
80
|
+
format!(
|
|
81
|
+
"{} bytes is not an upload grant — one is a URL and a few headers",
|
|
82
|
+
text.len()
|
|
83
|
+
),
|
|
84
|
+
);
|
|
85
|
+
}
|
|
86
|
+
let mut lines = text.lines().map(str::trim).filter(|line| !line.is_empty());
|
|
87
|
+
let Some(url) = lines.next() else {
|
|
88
|
+
return refuse(
|
|
89
|
+
Code::SOURCE_NOT_ALLOWED,
|
|
90
|
+
"an upload grant is a destination on the first line, then one \
|
|
91
|
+
`name: value` per signed header",
|
|
92
|
+
);
|
|
93
|
+
};
|
|
94
|
+
let mut grant = UploadGrant::to(url);
|
|
95
|
+
for line in lines {
|
|
96
|
+
let Some((name, value)) = line.split_once(':') else {
|
|
97
|
+
return refuse(
|
|
98
|
+
Code::SOURCE_NOT_ALLOWED,
|
|
99
|
+
format!("upload grant line {line:?} is not `name: value`"),
|
|
100
|
+
);
|
|
101
|
+
};
|
|
102
|
+
let (name, value) = (name.trim(), value.trim());
|
|
103
|
+
if name.is_empty() || name != name.to_ascii_lowercase() || value.is_empty() {
|
|
104
|
+
return refuse(
|
|
105
|
+
Code::SOURCE_NOT_ALLOWED,
|
|
106
|
+
format!(
|
|
107
|
+
"upload grant header {name:?} is not one canonical lowercase name \
|
|
108
|
+
with a value — send the grant's headers exactly as its signature \
|
|
109
|
+
spells them"
|
|
110
|
+
),
|
|
111
|
+
);
|
|
112
|
+
}
|
|
113
|
+
if grant.headers.iter().any(|(held, _)| held == name) {
|
|
114
|
+
return refuse(
|
|
115
|
+
Code::SOURCE_NOT_ALLOWED,
|
|
116
|
+
format!("upload grant repeats the signed header {name:?}"),
|
|
117
|
+
);
|
|
118
|
+
}
|
|
119
|
+
grant.headers.push((name.to_string(), value.to_string()));
|
|
120
|
+
}
|
|
121
|
+
Ok(grant)
|
|
122
|
+
}
|
|
123
|
+
}
|
|
124
|
+
|
|
33
125
|
fn shared_client() -> &'static Client {
|
|
34
126
|
static CLIENT: OnceLock<Client> = OnceLock::new();
|
|
35
127
|
CLIENT.get_or_init(Client::new)
|
|
@@ -88,13 +180,14 @@ fn verified_bytes(store: &Store, sha256: &str, manifest: bool) -> Result<(Vec<u8
|
|
|
88
180
|
}
|
|
89
181
|
}
|
|
90
182
|
|
|
91
|
-
/// Stream one verified store object to one
|
|
183
|
+
/// Stream one verified store object to the one destination a grant names, under the exact
|
|
184
|
+
/// headers that grant's signature covers.
|
|
92
185
|
#[allow(clippy::too_many_arguments)]
|
|
93
186
|
pub fn push_object(
|
|
94
187
|
store: &Store,
|
|
95
188
|
sha256: &str,
|
|
96
189
|
manifest: bool,
|
|
97
|
-
|
|
190
|
+
grant: &UploadGrant,
|
|
98
191
|
policy: &SourcePolicy,
|
|
99
192
|
credential: &dyn CredentialProvider,
|
|
100
193
|
deadline: Deadline,
|
|
@@ -107,7 +200,7 @@ pub fn push_object(
|
|
|
107
200
|
let mut last_refusal: Option<Refusal> = None;
|
|
108
201
|
for attempt in 1..=PUT_ATTEMPTS {
|
|
109
202
|
let last = attempt == PUT_ATTEMPTS;
|
|
110
|
-
let checked = policy.check(url)?;
|
|
203
|
+
let checked = policy.check(&grant.url)?;
|
|
111
204
|
let mut headers = vec![
|
|
112
205
|
(
|
|
113
206
|
"content-type".to_string(),
|
|
@@ -120,6 +213,14 @@ pub fn push_object(
|
|
|
120
213
|
("if-none-match".to_string(), "*".to_string()),
|
|
121
214
|
];
|
|
122
215
|
headers.extend(credential.headers(&checked.host));
|
|
216
|
+
// The grant's own spelling is the one the signature covers, so it replaces a
|
|
217
|
+
// default or a credential header of the same name rather than joining it.
|
|
218
|
+
for (name, value) in &grant.headers {
|
|
219
|
+
match headers.iter_mut().find(|(held, _)| held == name) {
|
|
220
|
+
Some(slot) => slot.1 = value.clone(),
|
|
221
|
+
None => headers.push((name.clone(), value.clone())),
|
|
222
|
+
}
|
|
223
|
+
}
|
|
123
224
|
let sent = http::request(
|
|
124
225
|
client,
|
|
125
226
|
&checked,
|
|
@@ -7,7 +7,7 @@ use super::hub::{Anonymous, HostToken};
|
|
|
7
7
|
use super::ledger::{Deadline, Ledger};
|
|
8
8
|
use super::policy::{blocked, SourcePolicy};
|
|
9
9
|
use super::pull::{fetch_url, pull, PullRequest};
|
|
10
|
-
use super::push::push_object;
|
|
10
|
+
use super::push::{push_object, UploadGrant};
|
|
11
11
|
use crate::canon::Value;
|
|
12
12
|
use crate::dtype::Dtype;
|
|
13
13
|
use crate::err::Code;
|
|
@@ -933,7 +933,7 @@ fn push_streams_a_verified_object_and_412_is_the_object_being_durable() {
|
|
|
933
933
|
&store,
|
|
934
934
|
&object.sha256,
|
|
935
935
|
false,
|
|
936
|
-
&format!("{}/put/{}", origin.base, object.sha256),
|
|
936
|
+
&UploadGrant::to(&format!("{}/put/{}", origin.base, object.sha256)),
|
|
937
937
|
&local_policy(),
|
|
938
938
|
&Anonymous,
|
|
939
939
|
Deadline::none(),
|
|
@@ -967,7 +967,7 @@ fn push_streams_a_verified_object_and_412_is_the_object_being_durable() {
|
|
|
967
967
|
&store,
|
|
968
968
|
&object.sha256,
|
|
969
969
|
false,
|
|
970
|
-
&format!("{}/put/x", origin.base),
|
|
970
|
+
&UploadGrant::to(&format!("{}/put/x", origin.base)),
|
|
971
971
|
&local_policy(),
|
|
972
972
|
&Anonymous,
|
|
973
973
|
Deadline::none(),
|
|
@@ -985,7 +985,7 @@ fn push_streams_a_verified_object_and_412_is_the_object_being_durable() {
|
|
|
985
985
|
&store,
|
|
986
986
|
&object.sha256,
|
|
987
987
|
false,
|
|
988
|
-
&format!("{}/put/x", origin.base),
|
|
988
|
+
&UploadGrant::to(&format!("{}/put/x", origin.base)),
|
|
989
989
|
&local_policy(),
|
|
990
990
|
&Anonymous,
|
|
991
991
|
Deadline::none(),
|
|
@@ -1007,7 +1007,7 @@ fn push_streams_a_verified_object_and_412_is_the_object_being_durable() {
|
|
|
1007
1007
|
&store,
|
|
1008
1008
|
&object.sha256,
|
|
1009
1009
|
false,
|
|
1010
|
-
&format!("{}/put/x", origin.base),
|
|
1010
|
+
&UploadGrant::to(&format!("{}/put/x", origin.base)),
|
|
1011
1011
|
&local_policy(),
|
|
1012
1012
|
&Anonymous,
|
|
1013
1013
|
Deadline::none(),
|
|
@@ -1019,6 +1019,141 @@ fn push_streams_a_verified_object_and_412_is_the_object_being_durable() {
|
|
|
1019
1019
|
let _ = std::fs::remove_dir_all(root);
|
|
1020
1020
|
}
|
|
1021
1021
|
|
|
1022
|
+
/// th-132's other half, proven at the socket: a tensorhub upload grant is a presigned URL
|
|
1023
|
+
/// AND the exact headers its signature covers — `if-none-match: *` and
|
|
1024
|
+
/// `x-amz-checksum-sha256` (podweights.validateGrant refuses a grant without them, and a
|
|
1025
|
+
/// SigV4 store refuses a PUT without them) — so the mover sends them verbatim, once each,
|
|
1026
|
+
/// on every attempt. The origin here DEMANDS them: without the checksum header it answers
|
|
1027
|
+
/// 400 and the push fails, which is exactly what made an upload grant unspendable.
|
|
1028
|
+
#[test]
|
|
1029
|
+
fn push_spends_a_grant_the_origin_refuses_without_its_signed_headers() {
|
|
1030
|
+
let root = temporary("push-grant-headers");
|
|
1031
|
+
let store = Store::init(&root).unwrap();
|
|
1032
|
+
let body = b"granted bytes".to_vec();
|
|
1033
|
+
let object = ObjectRef::of(&body);
|
|
1034
|
+
store
|
|
1035
|
+
.put_stream(&mut body.as_slice(), Some(&object), &Default::default())
|
|
1036
|
+
.unwrap();
|
|
1037
|
+
let raw: Vec<u8> = object
|
|
1038
|
+
.sha256
|
|
1039
|
+
.as_bytes()
|
|
1040
|
+
.chunks(2)
|
|
1041
|
+
.map(|pair| u8::from_str_radix(std::str::from_utf8(pair).unwrap(), 16).unwrap())
|
|
1042
|
+
.collect();
|
|
1043
|
+
let checksum = crate::b64::encode(&raw);
|
|
1044
|
+
|
|
1045
|
+
type SeenHeaders = Arc<Mutex<Vec<Vec<(String, String)>>>>;
|
|
1046
|
+
let seen: SeenHeaders = Arc::default();
|
|
1047
|
+
let seen_for = Arc::clone(&seen);
|
|
1048
|
+
let checksum_for = checksum.clone();
|
|
1049
|
+
let demanding: Dispatch = Arc::new(move |request: &Request| {
|
|
1050
|
+
seen_for.lock().unwrap().push(request.headers.clone());
|
|
1051
|
+
// The store's side of the contract, exactly as S3 enforces it: no signed
|
|
1052
|
+
// checksum, no PUT.
|
|
1053
|
+
if request.header("x-amz-checksum-sha256") != Some(checksum_for.as_str())
|
|
1054
|
+
|| request.header("if-none-match") != Some("*")
|
|
1055
|
+
{
|
|
1056
|
+
return status(400, b"InvalidRequest: the signed headers are required");
|
|
1057
|
+
}
|
|
1058
|
+
status(200, b"")
|
|
1059
|
+
});
|
|
1060
|
+
let origin = origin(demanding);
|
|
1061
|
+
let ledger = Ledger::with_resolution(Duration::from_millis(25));
|
|
1062
|
+
let destination = format!("{}/put/{}", origin.base, object.sha256);
|
|
1063
|
+
|
|
1064
|
+
// Without the grant's headers this origin refuses — the gap th-132 recorded.
|
|
1065
|
+
let refusal = push_object(
|
|
1066
|
+
&store,
|
|
1067
|
+
&object.sha256,
|
|
1068
|
+
false,
|
|
1069
|
+
&UploadGrant::to(&destination),
|
|
1070
|
+
&local_policy(),
|
|
1071
|
+
&Anonymous,
|
|
1072
|
+
Deadline::none(),
|
|
1073
|
+
&ledger,
|
|
1074
|
+
"",
|
|
1075
|
+
)
|
|
1076
|
+
.unwrap_err();
|
|
1077
|
+
assert_eq!(refusal.code, Code::TRANSFER_FAILED);
|
|
1078
|
+
assert!(refusal.detail.contains("400"), "{refusal}");
|
|
1079
|
+
|
|
1080
|
+
// The grant as its minter writes it — url and headers in one document, so the URL
|
|
1081
|
+
// cannot be spent without its conditions.
|
|
1082
|
+
let document = format!("{destination}\nif-none-match: *\nx-amz-checksum-sha256: {checksum}\n");
|
|
1083
|
+
let grant = UploadGrant::parse(&document).unwrap();
|
|
1084
|
+
assert_eq!(grant.url, destination);
|
|
1085
|
+
assert_eq!(grant.headers.len(), 2);
|
|
1086
|
+
seen.lock().unwrap().clear();
|
|
1087
|
+
let pushed = push_object(
|
|
1088
|
+
&store,
|
|
1089
|
+
&object.sha256,
|
|
1090
|
+
false,
|
|
1091
|
+
&grant,
|
|
1092
|
+
&local_policy(),
|
|
1093
|
+
&Anonymous,
|
|
1094
|
+
Deadline::none(),
|
|
1095
|
+
&ledger,
|
|
1096
|
+
"",
|
|
1097
|
+
)
|
|
1098
|
+
.unwrap();
|
|
1099
|
+
assert_eq!(pushed.http_status, 200);
|
|
1100
|
+
assert_eq!(pushed.bytes_sent, object.length);
|
|
1101
|
+
{
|
|
1102
|
+
let seen = seen.lock().unwrap();
|
|
1103
|
+
assert_eq!(seen.len(), 1);
|
|
1104
|
+
let of = |name: &str| -> Vec<&str> {
|
|
1105
|
+
seen[0]
|
|
1106
|
+
.iter()
|
|
1107
|
+
.filter(|(held, _)| held == name)
|
|
1108
|
+
.map(|(_, value)| value.as_str())
|
|
1109
|
+
.collect()
|
|
1110
|
+
};
|
|
1111
|
+
// The grant's own if-none-match replaced the built-in — sent exactly once.
|
|
1112
|
+
assert_eq!(of("if-none-match"), vec!["*"]);
|
|
1113
|
+
assert_eq!(of("x-amz-checksum-sha256"), vec![checksum.as_str()]);
|
|
1114
|
+
}
|
|
1115
|
+
let _ = std::fs::remove_dir_all(root);
|
|
1116
|
+
}
|
|
1117
|
+
|
|
1118
|
+
/// A grant document is refused for the first thing it gets wrong, before any byte moves —
|
|
1119
|
+
/// and the grammar survives what a real grant actually contains: a presigned URL full of
|
|
1120
|
+
/// `&` and `=`, and a base64 checksum ending in `=`.
|
|
1121
|
+
#[test]
|
|
1122
|
+
fn a_grant_document_is_refused_for_the_first_disagreeing_thing() {
|
|
1123
|
+
let presigned = "https://bucket.s3.amazonaws.com/o/k?X-Amz-Algorithm=AWS4-HMAC-SHA256\
|
|
1124
|
+
&X-Amz-Signature=deadbeef&X-Amz-SignedHeaders=host%3Bif-none-match";
|
|
1125
|
+
let checksum = "47DEQpj8HBSa+/TImW+5JCeuQeRkm5NMpJWZG3hSuFU=";
|
|
1126
|
+
let grant = UploadGrant::parse(&format!(
|
|
1127
|
+
"{presigned}\n\nif-none-match: *\nx-amz-checksum-sha256: {checksum}\n"
|
|
1128
|
+
))
|
|
1129
|
+
.unwrap();
|
|
1130
|
+
assert_eq!(grant.url, presigned);
|
|
1131
|
+
assert_eq!(
|
|
1132
|
+
grant.headers,
|
|
1133
|
+
vec![
|
|
1134
|
+
("if-none-match".to_string(), "*".to_string()),
|
|
1135
|
+
("x-amz-checksum-sha256".to_string(), checksum.to_string()),
|
|
1136
|
+
]
|
|
1137
|
+
);
|
|
1138
|
+
|
|
1139
|
+
// A name that is not its own canonical lowercase self is not the name the signature
|
|
1140
|
+
// covers; an empty value, a line that is not `name: value`, a repeat, and an empty
|
|
1141
|
+
// document are each the grant saying something this mover will not guess at.
|
|
1142
|
+
for bad in [
|
|
1143
|
+
"https://s3.example/put\nX-Amz-Checksum-Sha256: abc",
|
|
1144
|
+
"https://s3.example/put\nif-none-match:",
|
|
1145
|
+
"https://s3.example/put\nif-none-match",
|
|
1146
|
+
"https://s3.example/put\nif-none-match: *\nif-none-match: *",
|
|
1147
|
+
" \n\n",
|
|
1148
|
+
] {
|
|
1149
|
+
let refusal = UploadGrant::parse(bad).unwrap_err();
|
|
1150
|
+
assert_eq!(refusal.code, Code::SOURCE_NOT_ALLOWED, "{bad:?}");
|
|
1151
|
+
}
|
|
1152
|
+
// The cap is the parser's, so every door gets it — file, environment, and any caller.
|
|
1153
|
+
let refusal = UploadGrant::parse(&"x".repeat(super::MAX_GRANT_BYTES + 1)).unwrap_err();
|
|
1154
|
+
assert_eq!(refusal.code, Code::SIZE_CAP);
|
|
1155
|
+
}
|
|
1156
|
+
|
|
1022
1157
|
// ---------------------------------------------------------------- TLS
|
|
1023
1158
|
|
|
1024
1159
|
#[test]
|
|
@@ -193,3 +193,95 @@ fn migration_refuses_a_source_symlink_without_following_it() {
|
|
|
193
193
|
assert!(!destination.exists());
|
|
194
194
|
fs::remove_dir_all(root).unwrap();
|
|
195
195
|
}
|
|
196
|
+
|
|
197
|
+
/// tfs-053. An empty skeleton Store once materialized inside a source checkout because a
|
|
198
|
+
/// process ran a store verb with no root from its own working directory. A verb that needs a
|
|
199
|
+
/// Store now refuses when nothing was named and nothing is there, and writes nothing at all.
|
|
200
|
+
fn refuse_without_a_named_root(name: &str, args: &[&str]) {
|
|
201
|
+
let root = temporary(name);
|
|
202
|
+
let home = root.join("home");
|
|
203
|
+
let cwd = root.join("cwd");
|
|
204
|
+
fs::create_dir_all(&home).unwrap();
|
|
205
|
+
fs::create_dir_all(&cwd).unwrap();
|
|
206
|
+
|
|
207
|
+
let output = Command::new(env!("CARGO_BIN_EXE_tfs"))
|
|
208
|
+
.args(args)
|
|
209
|
+
.current_dir(&cwd)
|
|
210
|
+
.env("HOME", &home)
|
|
211
|
+
.env_remove("TENSORFS_HOME")
|
|
212
|
+
.output()
|
|
213
|
+
.unwrap();
|
|
214
|
+
|
|
215
|
+
let stderr = String::from_utf8(output.stderr).unwrap();
|
|
216
|
+
assert!(!output.status.success(), "tfs {args:?} did not refuse");
|
|
217
|
+
assert!(stderr.contains("STORE_ROOT_ABSENT"), "{stderr}");
|
|
218
|
+
assert_eq!(
|
|
219
|
+
fs::read_dir(&cwd).unwrap().count(),
|
|
220
|
+
0,
|
|
221
|
+
"tfs {args:?} wrote into its working directory"
|
|
222
|
+
);
|
|
223
|
+
assert!(
|
|
224
|
+
!home.join(".tensorfs").exists(),
|
|
225
|
+
"tfs {args:?} conjured a home Store"
|
|
226
|
+
);
|
|
227
|
+
fs::remove_dir_all(root).unwrap();
|
|
228
|
+
}
|
|
229
|
+
|
|
230
|
+
#[test]
|
|
231
|
+
fn store_verbs_refuse_and_write_nothing_when_no_root_is_named() {
|
|
232
|
+
for (name, args) in [
|
|
233
|
+
("no-root-init", &["store", "init"][..]),
|
|
234
|
+
("no-root-ensure", &["store", "ensure"][..]),
|
|
235
|
+
("no-root-info", &["store", "info"][..]),
|
|
236
|
+
("no-root-readers", &["store", "prepare-readers"][..]),
|
|
237
|
+
("no-root-complete", &["store", "complete-cozytensors"][..]),
|
|
238
|
+
] {
|
|
239
|
+
refuse_without_a_named_root(name, args);
|
|
240
|
+
}
|
|
241
|
+
}
|
|
242
|
+
|
|
243
|
+
#[test]
|
|
244
|
+
fn rebuild_refuses_an_absent_census_root_instead_of_creating_one() {
|
|
245
|
+
refuse_without_a_named_root(
|
|
246
|
+
"absent-census",
|
|
247
|
+
&["store", "rebuild", "census", "--rows", "rows.jsonl"],
|
|
248
|
+
);
|
|
249
|
+
}
|
|
250
|
+
|
|
251
|
+
#[test]
|
|
252
|
+
fn a_named_root_still_creates_and_store_ensure_stays_idempotent() {
|
|
253
|
+
let root = temporary("named-root");
|
|
254
|
+
let named = root.join("named");
|
|
255
|
+
let environment = root.join("environment");
|
|
256
|
+
let home = root.join("home");
|
|
257
|
+
fs::create_dir_all(&home).unwrap();
|
|
258
|
+
|
|
259
|
+
let run = |args: &[&str], tensorfs_home: Option<&Path>| {
|
|
260
|
+
let mut command = Command::new(env!("CARGO_BIN_EXE_tfs"));
|
|
261
|
+
command.args(args).env("HOME", &home);
|
|
262
|
+
match tensorfs_home {
|
|
263
|
+
Some(value) => command.env("TENSORFS_HOME", value),
|
|
264
|
+
None => command.env_remove("TENSORFS_HOME"),
|
|
265
|
+
};
|
|
266
|
+
let output = command.output().unwrap();
|
|
267
|
+
assert!(
|
|
268
|
+
output.status.success(),
|
|
269
|
+
"{}",
|
|
270
|
+
String::from_utf8_lossy(&output.stderr)
|
|
271
|
+
);
|
|
272
|
+
};
|
|
273
|
+
|
|
274
|
+
run(&["store", "init", named.to_str().unwrap()], None);
|
|
275
|
+
assert!(named.join("blobs").is_dir());
|
|
276
|
+
run(&["store", "ensure"], Some(&environment));
|
|
277
|
+
run(&["store", "ensure"], Some(&environment));
|
|
278
|
+
assert!(environment.join("blobs").is_dir());
|
|
279
|
+
|
|
280
|
+
// The `$HOME/.tensorfs` fallback is opened once it exists; only creating it needs a name.
|
|
281
|
+
run(
|
|
282
|
+
&["store", "init", home.join(".tensorfs").to_str().unwrap()],
|
|
283
|
+
None,
|
|
284
|
+
);
|
|
285
|
+
run(&["store", "info"], None);
|
|
286
|
+
fs::remove_dir_all(root).unwrap();
|
|
287
|
+
}
|
|
@@ -500,21 +500,21 @@ impl PyStore {
|
|
|
500
500
|
#[staticmethod]
|
|
501
501
|
#[pyo3(signature = (root = None))]
|
|
502
502
|
fn ensure(py: Python<'_>, root: Option<PathBuf>) -> PyResult<PyStore> {
|
|
503
|
-
let root = tensorfs_core::home::
|
|
503
|
+
let root = tensorfs_core::home::store_root(root.as_deref()).or_refuse(py)?;
|
|
504
504
|
open_store(py, root, CoreStore::ensure)
|
|
505
505
|
}
|
|
506
506
|
|
|
507
507
|
#[staticmethod]
|
|
508
508
|
#[pyo3(signature = (root = None))]
|
|
509
509
|
fn open(py: Python<'_>, root: Option<PathBuf>) -> PyResult<PyStore> {
|
|
510
|
-
let root = tensorfs_core::home::
|
|
510
|
+
let root = tensorfs_core::home::store_root(root.as_deref()).or_refuse(py)?;
|
|
511
511
|
open_store(py, root, CoreStore::open)
|
|
512
512
|
}
|
|
513
513
|
|
|
514
514
|
#[staticmethod]
|
|
515
515
|
#[pyo3(signature = (root = None))]
|
|
516
516
|
fn init(py: Python<'_>, root: Option<PathBuf>) -> PyResult<PyStore> {
|
|
517
|
-
let root = tensorfs_core::home::
|
|
517
|
+
let root = tensorfs_core::home::store_root(root.as_deref()).or_refuse(py)?;
|
|
518
518
|
open_store(py, root, CoreStore::init)
|
|
519
519
|
}
|
|
520
520
|
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
{tensorfs-0.3.2 → tensorfs-0.3.3}/crates/tensorfs-core/tests/fixtures/repo_apply/blobs.jsonl
RENAMED
|
File without changes
|
{tensorfs-0.3.2 → tensorfs-0.3.3}/crates/tensorfs-core/tests/fixtures/repo_apply/expected_list.jsonl
RENAMED
|
File without changes
|
|
File without changes
|
{tensorfs-0.3.2 → tensorfs-0.3.3}/crates/tensorfs-core/tests/fixtures/repo_apply/expected_rows.jsonl
RENAMED
|
File without changes
|
{tensorfs-0.3.2 → tensorfs-0.3.3}/crates/tensorfs-core/tests/fixtures/repo_apply/header.cbor
RENAMED
|
File without changes
|
{tensorfs-0.3.2 → tensorfs-0.3.3}/crates/tensorfs-core/tests/fixtures/repo_apply/manifest.json
RENAMED
|
File without changes
|
{tensorfs-0.3.2 → tensorfs-0.3.3}/crates/tensorfs-core/tests/fixtures/repo_apply/mutation.json
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|