covalence 0.1.0.dev4__tar.gz → 0.1.0.dev5__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {covalence-0.1.0.dev4 → covalence-0.1.0.dev5}/Cargo.lock +19 -19
- {covalence-0.1.0.dev4 → covalence-0.1.0.dev5}/Cargo.toml +17 -17
- {covalence-0.1.0.dev4 → covalence-0.1.0.dev5}/PKG-INFO +25 -8
- {covalence-0.1.0.dev4 → covalence-0.1.0.dev5}/README.md +24 -7
- {covalence-0.1.0.dev4 → covalence-0.1.0.dev5}/crates/kv/s3/src/lib.rs +106 -6
- {covalence-0.1.0.dev4 → covalence-0.1.0.dev5}/crates/kv/s3/src/write.rs +92 -8
- {covalence-0.1.0.dev4 → covalence-0.1.0.dev5}/crates/kv/s3/tests/kv.rs +391 -29
- {covalence-0.1.0.dev4 → covalence-0.1.0.dev5}/crates/python/README.md +24 -7
- {covalence-0.1.0.dev4 → covalence-0.1.0.dev5}/crates/python/src/lib.rs +19 -4
- {covalence-0.1.0.dev4 → covalence-0.1.0.dev5}/crates/python/tests/test_api.py +38 -0
- {covalence-0.1.0.dev4 → covalence-0.1.0.dev5}/python/covalence/THIRD-PARTY-LICENSES.md +17 -17
- {covalence-0.1.0.dev4 → covalence-0.1.0.dev5}/python/covalence/_native.pyi +14 -4
- {covalence-0.1.0.dev4 → covalence-0.1.0.dev5}/LICENSE-APACHE +0 -0
- {covalence-0.1.0.dev4 → covalence-0.1.0.dev5}/LICENSE-CC0 +0 -0
- {covalence-0.1.0.dev4 → covalence-0.1.0.dev5}/LICENSE-MIT +0 -0
- {covalence-0.1.0.dev4 → covalence-0.1.0.dev5}/crates/blob/trait/Cargo.toml +0 -0
- {covalence-0.1.0.dev4 → covalence-0.1.0.dev5}/crates/blob/trait/LICENSE-APACHE +0 -0
- {covalence-0.1.0.dev4 → covalence-0.1.0.dev5}/crates/blob/trait/LICENSE-CC0 +0 -0
- {covalence-0.1.0.dev4 → covalence-0.1.0.dev5}/crates/blob/trait/LICENSE-MIT +0 -0
- {covalence-0.1.0.dev4 → covalence-0.1.0.dev5}/crates/blob/trait/README.md +0 -0
- {covalence-0.1.0.dev4 → covalence-0.1.0.dev5}/crates/blob/trait/src/lib.rs +0 -0
- {covalence-0.1.0.dev4 → covalence-0.1.0.dev5}/crates/blob/trait/tests/blob.rs +0 -0
- {covalence-0.1.0.dev4 → covalence-0.1.0.dev5}/crates/blob/trait/tests/common/mod.rs +0 -0
- {covalence-0.1.0.dev4 → covalence-0.1.0.dev5}/crates/blob/trait/tests/fact.rs +0 -0
- {covalence-0.1.0.dev4 → covalence-0.1.0.dev5}/crates/cas/kv/Cargo.toml +0 -0
- {covalence-0.1.0.dev4 → covalence-0.1.0.dev5}/crates/cas/kv/LICENSE-APACHE +0 -0
- {covalence-0.1.0.dev4 → covalence-0.1.0.dev5}/crates/cas/kv/LICENSE-CC0 +0 -0
- {covalence-0.1.0.dev4 → covalence-0.1.0.dev5}/crates/cas/kv/LICENSE-MIT +0 -0
- {covalence-0.1.0.dev4 → covalence-0.1.0.dev5}/crates/cas/kv/src/lib.rs +0 -0
- {covalence-0.1.0.dev4 → covalence-0.1.0.dev5}/crates/cas/kv/tests/kv.rs +0 -0
- {covalence-0.1.0.dev4 → covalence-0.1.0.dev5}/crates/cas/memory/Cargo.toml +0 -0
- {covalence-0.1.0.dev4 → covalence-0.1.0.dev5}/crates/cas/memory/LICENSE-APACHE +0 -0
- {covalence-0.1.0.dev4 → covalence-0.1.0.dev5}/crates/cas/memory/LICENSE-CC0 +0 -0
- {covalence-0.1.0.dev4 → covalence-0.1.0.dev5}/crates/cas/memory/LICENSE-MIT +0 -0
- {covalence-0.1.0.dev4 → covalence-0.1.0.dev5}/crates/cas/memory/src/lib.rs +0 -0
- {covalence-0.1.0.dev4 → covalence-0.1.0.dev5}/crates/cas/memory/tests/memory.rs +0 -0
- {covalence-0.1.0.dev4 → covalence-0.1.0.dev5}/crates/cas/sqlite/Cargo.toml +0 -0
- {covalence-0.1.0.dev4 → covalence-0.1.0.dev5}/crates/cas/sqlite/LICENSE-APACHE +0 -0
- {covalence-0.1.0.dev4 → covalence-0.1.0.dev5}/crates/cas/sqlite/LICENSE-CC0 +0 -0
- {covalence-0.1.0.dev4 → covalence-0.1.0.dev5}/crates/cas/sqlite/LICENSE-MIT +0 -0
- {covalence-0.1.0.dev4 → covalence-0.1.0.dev5}/crates/cas/sqlite/src/lib.rs +0 -0
- {covalence-0.1.0.dev4 → covalence-0.1.0.dev5}/crates/cas/sqlite/tests/sqlite.rs +0 -0
- {covalence-0.1.0.dev4 → covalence-0.1.0.dev5}/crates/cas/trait/Cargo.toml +0 -0
- {covalence-0.1.0.dev4 → covalence-0.1.0.dev5}/crates/cas/trait/LICENSE-APACHE +0 -0
- {covalence-0.1.0.dev4 → covalence-0.1.0.dev5}/crates/cas/trait/LICENSE-CC0 +0 -0
- {covalence-0.1.0.dev4 → covalence-0.1.0.dev5}/crates/cas/trait/LICENSE-MIT +0 -0
- {covalence-0.1.0.dev4 → covalence-0.1.0.dev5}/crates/cas/trait/src/lib.rs +0 -0
- {covalence-0.1.0.dev4 → covalence-0.1.0.dev5}/crates/cas/trait/tests/trait.rs +0 -0
- {covalence-0.1.0.dev4 → covalence-0.1.0.dev5}/crates/kv/memory/Cargo.toml +0 -0
- {covalence-0.1.0.dev4 → covalence-0.1.0.dev5}/crates/kv/memory/LICENSE-APACHE +0 -0
- {covalence-0.1.0.dev4 → covalence-0.1.0.dev5}/crates/kv/memory/LICENSE-CC0 +0 -0
- {covalence-0.1.0.dev4 → covalence-0.1.0.dev5}/crates/kv/memory/LICENSE-MIT +0 -0
- {covalence-0.1.0.dev4 → covalence-0.1.0.dev5}/crates/kv/memory/src/lib.rs +0 -0
- {covalence-0.1.0.dev4 → covalence-0.1.0.dev5}/crates/kv/memory/tests/memory.rs +0 -0
- {covalence-0.1.0.dev4 → covalence-0.1.0.dev5}/crates/kv/s3/Cargo.toml +0 -0
- {covalence-0.1.0.dev4 → covalence-0.1.0.dev5}/crates/kv/s3/LICENSE-APACHE +0 -0
- {covalence-0.1.0.dev4 → covalence-0.1.0.dev5}/crates/kv/s3/LICENSE-CC0 +0 -0
- {covalence-0.1.0.dev4 → covalence-0.1.0.dev5}/crates/kv/s3/LICENSE-MIT +0 -0
- {covalence-0.1.0.dev4 → covalence-0.1.0.dev5}/crates/kv/s3/src/read.rs +0 -0
- {covalence-0.1.0.dev4 → covalence-0.1.0.dev5}/crates/kv/s3/tests/aws.rs +0 -0
- {covalence-0.1.0.dev4 → covalence-0.1.0.dev5}/crates/kv/trait/Cargo.toml +0 -0
- {covalence-0.1.0.dev4 → covalence-0.1.0.dev5}/crates/kv/trait/LICENSE-APACHE +0 -0
- {covalence-0.1.0.dev4 → covalence-0.1.0.dev5}/crates/kv/trait/LICENSE-CC0 +0 -0
- {covalence-0.1.0.dev4 → covalence-0.1.0.dev5}/crates/kv/trait/LICENSE-MIT +0 -0
- {covalence-0.1.0.dev4 → covalence-0.1.0.dev5}/crates/kv/trait/src/lib.rs +0 -0
- {covalence-0.1.0.dev4 → covalence-0.1.0.dev5}/crates/kv/trait/tests/atomic.rs +0 -0
- {covalence-0.1.0.dev4 → covalence-0.1.0.dev5}/crates/lib/cid/Cargo.toml +0 -0
- {covalence-0.1.0.dev4 → covalence-0.1.0.dev5}/crates/lib/cid/LICENSE-APACHE +0 -0
- {covalence-0.1.0.dev4 → covalence-0.1.0.dev5}/crates/lib/cid/LICENSE-CC0 +0 -0
- {covalence-0.1.0.dev4 → covalence-0.1.0.dev5}/crates/lib/cid/LICENSE-MIT +0 -0
- {covalence-0.1.0.dev4 → covalence-0.1.0.dev5}/crates/lib/cid/README.md +0 -0
- {covalence-0.1.0.dev4 → covalence-0.1.0.dev5}/crates/lib/cid/src/dasl.rs +0 -0
- {covalence-0.1.0.dev4 → covalence-0.1.0.dev5}/crates/lib/cid/src/fact.rs +0 -0
- {covalence-0.1.0.dev4 → covalence-0.1.0.dev5}/crates/lib/cid/src/git.rs +0 -0
- {covalence-0.1.0.dev4 → covalence-0.1.0.dev5}/crates/lib/cid/src/lib.rs +0 -0
- {covalence-0.1.0.dev4 → covalence-0.1.0.dev5}/crates/lib/cid/src/multicodec.rs +0 -0
- {covalence-0.1.0.dev4 → covalence-0.1.0.dev5}/crates/lib/cid/src/o256.rs +0 -0
- {covalence-0.1.0.dev4 → covalence-0.1.0.dev5}/crates/lib/cid/src/storage.rs +0 -0
- {covalence-0.1.0.dev4 → covalence-0.1.0.dev5}/crates/lib/cid/tests/cid.rs +0 -0
- {covalence-0.1.0.dev4 → covalence-0.1.0.dev5}/crates/lib/cid/tests/fact.rs +0 -0
- {covalence-0.1.0.dev4 → covalence-0.1.0.dev5}/crates/lib/cid/tests/git.rs +0 -0
- {covalence-0.1.0.dev4 → covalence-0.1.0.dev5}/crates/lib/cid/tests/o256.rs +0 -0
- {covalence-0.1.0.dev4 → covalence-0.1.0.dev5}/crates/lib/cid/tests/shattered/1.bin +0 -0
- {covalence-0.1.0.dev4 → covalence-0.1.0.dev5}/crates/lib/cid/tests/shattered/2.bin +0 -0
- {covalence-0.1.0.dev4 → covalence-0.1.0.dev5}/crates/lib/cid/tests/shattered/README.md +0 -0
- {covalence-0.1.0.dev4 → covalence-0.1.0.dev5}/crates/lib/dasl/Cargo.toml +0 -0
- {covalence-0.1.0.dev4 → covalence-0.1.0.dev5}/crates/lib/dasl/LICENSE-APACHE +0 -0
- {covalence-0.1.0.dev4 → covalence-0.1.0.dev5}/crates/lib/dasl/LICENSE-CC0 +0 -0
- {covalence-0.1.0.dev4 → covalence-0.1.0.dev5}/crates/lib/dasl/LICENSE-MIT +0 -0
- {covalence-0.1.0.dev4 → covalence-0.1.0.dev5}/crates/lib/dasl/README.md +0 -0
- {covalence-0.1.0.dev4 → covalence-0.1.0.dev5}/crates/lib/dasl/src/drisl.rs +0 -0
- {covalence-0.1.0.dev4 → covalence-0.1.0.dev5}/crates/lib/dasl/src/event.rs +0 -0
- {covalence-0.1.0.dev4 → covalence-0.1.0.dev5}/crates/lib/dasl/src/json.rs +0 -0
- {covalence-0.1.0.dev4 → covalence-0.1.0.dev5}/crates/lib/dasl/src/lib.rs +0 -0
- {covalence-0.1.0.dev4 → covalence-0.1.0.dev5}/crates/lib/dasl/tests/codec.rs +0 -0
- {covalence-0.1.0.dev4 → covalence-0.1.0.dev5}/crates/lib/dasl/tests/events.rs +0 -0
- {covalence-0.1.0.dev4 → covalence-0.1.0.dev5}/crates/lib/dasl/tests/fact.rs +0 -0
- {covalence-0.1.0.dev4 → covalence-0.1.0.dev5}/crates/lib/dasl/tests/fixtures/LICENSE +0 -0
- {covalence-0.1.0.dev4 → covalence-0.1.0.dev5}/crates/lib/dasl/tests/fixtures/README.md +0 -0
- {covalence-0.1.0.dev4 → covalence-0.1.0.dev5}/crates/lib/dasl/tests/fixtures/cid.json +0 -0
- {covalence-0.1.0.dev4 → covalence-0.1.0.dev5}/crates/lib/dasl/tests/fixtures/concat.json +0 -0
- {covalence-0.1.0.dev4 → covalence-0.1.0.dev5}/crates/lib/dasl/tests/fixtures/floats.json +0 -0
- {covalence-0.1.0.dev4 → covalence-0.1.0.dev5}/crates/lib/dasl/tests/fixtures/indefinite.json +0 -0
- {covalence-0.1.0.dev4 → covalence-0.1.0.dev5}/crates/lib/dasl/tests/fixtures/integer_range.json +0 -0
- {covalence-0.1.0.dev4 → covalence-0.1.0.dev5}/crates/lib/dasl/tests/fixtures/map_keys.json +0 -0
- {covalence-0.1.0.dev4 → covalence-0.1.0.dev5}/crates/lib/dasl/tests/fixtures/numeric_reduction.json +0 -0
- {covalence-0.1.0.dev4 → covalence-0.1.0.dev5}/crates/lib/dasl/tests/fixtures/short_form.json +0 -0
- {covalence-0.1.0.dev4 → covalence-0.1.0.dev5}/crates/lib/dasl/tests/fixtures/simple.json +0 -0
- {covalence-0.1.0.dev4 → covalence-0.1.0.dev5}/crates/lib/dasl/tests/fixtures/tags.json +0 -0
- {covalence-0.1.0.dev4 → covalence-0.1.0.dev5}/crates/lib/dasl/tests/fixtures/utf8.json +0 -0
- {covalence-0.1.0.dev4 → covalence-0.1.0.dev5}/crates/lib/dasl/tests/fixtures.rs +0 -0
- {covalence-0.1.0.dev4 → covalence-0.1.0.dev5}/crates/lib/dasl/tests/json.rs +0 -0
- {covalence-0.1.0.dev4 → covalence-0.1.0.dev5}/crates/lib/dasl/tests/vectors/README.md +0 -0
- {covalence-0.1.0.dev4 → covalence-0.1.0.dev5}/crates/lib/dasl/tests/vectors/cid-vectors.jsonl +0 -0
- {covalence-0.1.0.dev4 → covalence-0.1.0.dev5}/crates/lib/dasl/tests/vectors/dasl-vectors.jsonl +0 -0
- {covalence-0.1.0.dev4 → covalence-0.1.0.dev5}/crates/lib/dasl/tests/vectors.rs +0 -0
- {covalence-0.1.0.dev4 → covalence-0.1.0.dev5}/crates/lib/data/Cargo.toml +0 -0
- {covalence-0.1.0.dev4 → covalence-0.1.0.dev5}/crates/lib/data/LICENSE-APACHE +0 -0
- {covalence-0.1.0.dev4 → covalence-0.1.0.dev5}/crates/lib/data/LICENSE-CC0 +0 -0
- {covalence-0.1.0.dev4 → covalence-0.1.0.dev5}/crates/lib/data/LICENSE-MIT +0 -0
- {covalence-0.1.0.dev4 → covalence-0.1.0.dev5}/crates/lib/data/src/lib.rs +0 -0
- {covalence-0.1.0.dev4 → covalence-0.1.0.dev5}/crates/lib/error/Cargo.toml +0 -0
- {covalence-0.1.0.dev4 → covalence-0.1.0.dev5}/crates/lib/error/LICENSE-APACHE +0 -0
- {covalence-0.1.0.dev4 → covalence-0.1.0.dev5}/crates/lib/error/LICENSE-CC0 +0 -0
- {covalence-0.1.0.dev4 → covalence-0.1.0.dev5}/crates/lib/error/LICENSE-MIT +0 -0
- {covalence-0.1.0.dev4 → covalence-0.1.0.dev5}/crates/lib/error/src/lib.rs +0 -0
- {covalence-0.1.0.dev4 → covalence-0.1.0.dev5}/crates/lib/python/Cargo.toml +0 -0
- {covalence-0.1.0.dev4 → covalence-0.1.0.dev5}/crates/lib/python/LICENSE-APACHE +0 -0
- {covalence-0.1.0.dev4 → covalence-0.1.0.dev5}/crates/lib/python/LICENSE-CC0 +0 -0
- {covalence-0.1.0.dev4 → covalence-0.1.0.dev5}/crates/lib/python/LICENSE-MIT +0 -0
- {covalence-0.1.0.dev4 → covalence-0.1.0.dev5}/crates/lib/python/src/lib.rs +0 -0
- {covalence-0.1.0.dev4 → covalence-0.1.0.dev5}/crates/lib/rand/Cargo.toml +0 -0
- {covalence-0.1.0.dev4 → covalence-0.1.0.dev5}/crates/lib/rand/LICENSE-APACHE +0 -0
- {covalence-0.1.0.dev4 → covalence-0.1.0.dev5}/crates/lib/rand/LICENSE-CC0 +0 -0
- {covalence-0.1.0.dev4 → covalence-0.1.0.dev5}/crates/lib/rand/LICENSE-MIT +0 -0
- {covalence-0.1.0.dev4 → covalence-0.1.0.dev5}/crates/lib/rand/src/lib.rs +0 -0
- {covalence-0.1.0.dev4 → covalence-0.1.0.dev5}/crates/lib/serde/Cargo.toml +0 -0
- {covalence-0.1.0.dev4 → covalence-0.1.0.dev5}/crates/lib/serde/LICENSE-APACHE +0 -0
- {covalence-0.1.0.dev4 → covalence-0.1.0.dev5}/crates/lib/serde/LICENSE-CC0 +0 -0
- {covalence-0.1.0.dev4 → covalence-0.1.0.dev5}/crates/lib/serde/LICENSE-MIT +0 -0
- {covalence-0.1.0.dev4 → covalence-0.1.0.dev5}/crates/lib/serde/src/lib.rs +0 -0
- {covalence-0.1.0.dev4 → covalence-0.1.0.dev5}/crates/lib/sqlite/Cargo.toml +0 -0
- {covalence-0.1.0.dev4 → covalence-0.1.0.dev5}/crates/lib/sqlite/LICENSE-APACHE +0 -0
- {covalence-0.1.0.dev4 → covalence-0.1.0.dev5}/crates/lib/sqlite/LICENSE-CC0 +0 -0
- {covalence-0.1.0.dev4 → covalence-0.1.0.dev5}/crates/lib/sqlite/LICENSE-MIT +0 -0
- {covalence-0.1.0.dev4 → covalence-0.1.0.dev5}/crates/lib/sqlite/src/lib.rs +0 -0
- {covalence-0.1.0.dev4 → covalence-0.1.0.dev5}/crates/python/Cargo.toml +0 -0
- {covalence-0.1.0.dev4 → covalence-0.1.0.dev5}/crates/python/LICENSE-APACHE +0 -0
- {covalence-0.1.0.dev4 → covalence-0.1.0.dev5}/crates/python/LICENSE-CC0 +0 -0
- {covalence-0.1.0.dev4 → covalence-0.1.0.dev5}/crates/python/LICENSE-MIT +0 -0
- {covalence-0.1.0.dev4 → covalence-0.1.0.dev5}/crates/python/about.md.hbs +0 -0
- {covalence-0.1.0.dev4 → covalence-0.1.0.dev5}/crates/python/about.toml +0 -0
- {covalence-0.1.0.dev4 → covalence-0.1.0.dev5}/crates/python/src/errors.rs +0 -0
- {covalence-0.1.0.dev4 → covalence-0.1.0.dev5}/crates/python/src/streams/download.rs +0 -0
- {covalence-0.1.0.dev4 → covalence-0.1.0.dev5}/crates/python/src/streams/mod.rs +0 -0
- {covalence-0.1.0.dev4 → covalence-0.1.0.dev5}/crates/python/src/streams/upload/tests.rs +0 -0
- {covalence-0.1.0.dev4 → covalence-0.1.0.dev5}/crates/python/src/streams/upload.rs +0 -0
- {covalence-0.1.0.dev4 → covalence-0.1.0.dev5}/crates/python/tests/test_bounded_io.py +0 -0
- {covalence-0.1.0.dev4 → covalence-0.1.0.dev5}/crates/python/tests/test_docs.py +0 -0
- {covalence-0.1.0.dev4 → covalence-0.1.0.dev5}/crates/python/tests/test_package.py +0 -0
- {covalence-0.1.0.dev4 → covalence-0.1.0.dev5}/crates/python/tests/test_stubs.py +0 -0
- {covalence-0.1.0.dev4 → covalence-0.1.0.dev5}/crates/python/tests/typed_usage.py +0 -0
- {covalence-0.1.0.dev4 → covalence-0.1.0.dev5}/pyproject.toml +0 -0
- {covalence-0.1.0.dev4 → covalence-0.1.0.dev5}/python/covalence/__init__.py +0 -0
- {covalence-0.1.0.dev4 → covalence-0.1.0.dev5}/python/covalence/py.typed +0 -0
|
@@ -779,7 +779,7 @@ checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b"
|
|
|
779
779
|
|
|
780
780
|
[[package]]
|
|
781
781
|
name = "covalence-blob-trait"
|
|
782
|
-
version = "0.1.0-dev.
|
|
782
|
+
version = "0.1.0-dev.5"
|
|
783
783
|
dependencies = [
|
|
784
784
|
"covalence-lib-cid",
|
|
785
785
|
"covalence-lib-data",
|
|
@@ -788,7 +788,7 @@ dependencies = [
|
|
|
788
788
|
|
|
789
789
|
[[package]]
|
|
790
790
|
name = "covalence-cas-kv"
|
|
791
|
-
version = "0.1.0-dev.
|
|
791
|
+
version = "0.1.0-dev.5"
|
|
792
792
|
dependencies = [
|
|
793
793
|
"blake3",
|
|
794
794
|
"covalence-cas-trait",
|
|
@@ -810,7 +810,7 @@ dependencies = [
|
|
|
810
810
|
|
|
811
811
|
[[package]]
|
|
812
812
|
name = "covalence-cas-memory"
|
|
813
|
-
version = "0.1.0-dev.
|
|
813
|
+
version = "0.1.0-dev.5"
|
|
814
814
|
dependencies = [
|
|
815
815
|
"covalence-cas-trait",
|
|
816
816
|
"covalence-lib-cid",
|
|
@@ -821,7 +821,7 @@ dependencies = [
|
|
|
821
821
|
|
|
822
822
|
[[package]]
|
|
823
823
|
name = "covalence-cas-sqlite"
|
|
824
|
-
version = "0.1.0-dev.
|
|
824
|
+
version = "0.1.0-dev.5"
|
|
825
825
|
dependencies = [
|
|
826
826
|
"covalence-cas-trait",
|
|
827
827
|
"covalence-lib-cid",
|
|
@@ -836,7 +836,7 @@ dependencies = [
|
|
|
836
836
|
|
|
837
837
|
[[package]]
|
|
838
838
|
name = "covalence-cas-trait"
|
|
839
|
-
version = "0.1.0-dev.
|
|
839
|
+
version = "0.1.0-dev.5"
|
|
840
840
|
dependencies = [
|
|
841
841
|
"covalence-lib-cid",
|
|
842
842
|
"covalence-lib-dasl",
|
|
@@ -847,7 +847,7 @@ dependencies = [
|
|
|
847
847
|
|
|
848
848
|
[[package]]
|
|
849
849
|
name = "covalence-kv-memory"
|
|
850
|
-
version = "0.1.0-dev.
|
|
850
|
+
version = "0.1.0-dev.5"
|
|
851
851
|
dependencies = [
|
|
852
852
|
"covalence-kv-trait",
|
|
853
853
|
"covalence-lib-dasl",
|
|
@@ -857,7 +857,7 @@ dependencies = [
|
|
|
857
857
|
|
|
858
858
|
[[package]]
|
|
859
859
|
name = "covalence-kv-s3"
|
|
860
|
-
version = "0.1.0-dev.
|
|
860
|
+
version = "0.1.0-dev.5"
|
|
861
861
|
dependencies = [
|
|
862
862
|
"aws-config",
|
|
863
863
|
"aws-credential-types",
|
|
@@ -881,7 +881,7 @@ dependencies = [
|
|
|
881
881
|
|
|
882
882
|
[[package]]
|
|
883
883
|
name = "covalence-kv-trait"
|
|
884
|
-
version = "0.1.0-dev.
|
|
884
|
+
version = "0.1.0-dev.5"
|
|
885
885
|
dependencies = [
|
|
886
886
|
"covalence-lib-dasl",
|
|
887
887
|
"covalence-lib-data",
|
|
@@ -890,7 +890,7 @@ dependencies = [
|
|
|
890
890
|
|
|
891
891
|
[[package]]
|
|
892
892
|
name = "covalence-lib-cid"
|
|
893
|
-
version = "0.1.0-dev.
|
|
893
|
+
version = "0.1.0-dev.5"
|
|
894
894
|
dependencies = [
|
|
895
895
|
"blake3",
|
|
896
896
|
"covalence-lib-data",
|
|
@@ -905,14 +905,14 @@ dependencies = [
|
|
|
905
905
|
|
|
906
906
|
[[package]]
|
|
907
907
|
name = "covalence-lib-cli"
|
|
908
|
-
version = "0.1.0-dev.
|
|
908
|
+
version = "0.1.0-dev.5"
|
|
909
909
|
dependencies = [
|
|
910
910
|
"clap",
|
|
911
911
|
]
|
|
912
912
|
|
|
913
913
|
[[package]]
|
|
914
914
|
name = "covalence-lib-dasl"
|
|
915
|
-
version = "0.1.0-dev.
|
|
915
|
+
version = "0.1.0-dev.5"
|
|
916
916
|
dependencies = [
|
|
917
917
|
"blake3",
|
|
918
918
|
"covalence-lib-cid",
|
|
@@ -927,7 +927,7 @@ dependencies = [
|
|
|
927
927
|
|
|
928
928
|
[[package]]
|
|
929
929
|
name = "covalence-lib-data"
|
|
930
|
-
version = "0.1.0-dev.
|
|
930
|
+
version = "0.1.0-dev.5"
|
|
931
931
|
dependencies = [
|
|
932
932
|
"arrayvec",
|
|
933
933
|
"bytes",
|
|
@@ -939,7 +939,7 @@ dependencies = [
|
|
|
939
939
|
|
|
940
940
|
[[package]]
|
|
941
941
|
name = "covalence-lib-error"
|
|
942
|
-
version = "0.1.0-dev.
|
|
942
|
+
version = "0.1.0-dev.5"
|
|
943
943
|
dependencies = [
|
|
944
944
|
"miette",
|
|
945
945
|
"snafu",
|
|
@@ -947,14 +947,14 @@ dependencies = [
|
|
|
947
947
|
|
|
948
948
|
[[package]]
|
|
949
949
|
name = "covalence-lib-python"
|
|
950
|
-
version = "0.1.0-dev.
|
|
950
|
+
version = "0.1.0-dev.5"
|
|
951
951
|
dependencies = [
|
|
952
952
|
"pyo3",
|
|
953
953
|
]
|
|
954
954
|
|
|
955
955
|
[[package]]
|
|
956
956
|
name = "covalence-lib-rand"
|
|
957
|
-
version = "0.1.0-dev.
|
|
957
|
+
version = "0.1.0-dev.5"
|
|
958
958
|
dependencies = [
|
|
959
959
|
"getrandom 0.3.4",
|
|
960
960
|
"rand",
|
|
@@ -962,21 +962,21 @@ dependencies = [
|
|
|
962
962
|
|
|
963
963
|
[[package]]
|
|
964
964
|
name = "covalence-lib-serde"
|
|
965
|
-
version = "0.1.0-dev.
|
|
965
|
+
version = "0.1.0-dev.5"
|
|
966
966
|
dependencies = [
|
|
967
967
|
"serde",
|
|
968
968
|
]
|
|
969
969
|
|
|
970
970
|
[[package]]
|
|
971
971
|
name = "covalence-lib-sqlite"
|
|
972
|
-
version = "0.1.0-dev.
|
|
972
|
+
version = "0.1.0-dev.5"
|
|
973
973
|
dependencies = [
|
|
974
974
|
"rusqlite",
|
|
975
975
|
]
|
|
976
976
|
|
|
977
977
|
[[package]]
|
|
978
978
|
name = "covalence-python"
|
|
979
|
-
version = "0.1.0-dev.
|
|
979
|
+
version = "0.1.0-dev.5"
|
|
980
980
|
dependencies = [
|
|
981
981
|
"blake3",
|
|
982
982
|
"covalence-blob-trait",
|
|
@@ -997,7 +997,7 @@ dependencies = [
|
|
|
997
997
|
|
|
998
998
|
[[package]]
|
|
999
999
|
name = "covalence-tools-glu"
|
|
1000
|
-
version = "0.1.0-dev.
|
|
1000
|
+
version = "0.1.0-dev.5"
|
|
1001
1001
|
dependencies = [
|
|
1002
1002
|
"covalence-lib-cli",
|
|
1003
1003
|
"covalence-lib-error",
|
|
@@ -3,7 +3,7 @@ members = ["crates/lib/error", "crates/lib/serde", "crates/lib/data", "crates/li
|
|
|
3
3
|
resolver = "3"
|
|
4
4
|
|
|
5
5
|
[workspace.package]
|
|
6
|
-
version = "0.1.0-dev.
|
|
6
|
+
version = "0.1.0-dev.5"
|
|
7
7
|
edition = "2024"
|
|
8
8
|
rust-version = "1.98"
|
|
9
9
|
license = "CC0-1.0 OR MIT OR Apache-2.0"
|
|
@@ -17,22 +17,22 @@ miette = "7.6.0"
|
|
|
17
17
|
sha2 = "0.11.0"
|
|
18
18
|
snafu = "0.9.2"
|
|
19
19
|
# Published crates, at the workspace version so the set moves together.
|
|
20
|
-
covalence-blob-trait = { path = "crates/blob/trait", version = "=0.1.0-dev.
|
|
21
|
-
covalence-cas-kv = { path = "crates/cas/kv", version = "=0.1.0-dev.
|
|
22
|
-
covalence-cas-memory = { path = "crates/cas/memory", version = "=0.1.0-dev.
|
|
23
|
-
covalence-cas-sqlite = { path = "crates/cas/sqlite", version = "=0.1.0-dev.
|
|
24
|
-
covalence-cas-trait = { path = "crates/cas/trait", version = "=0.1.0-dev.
|
|
25
|
-
covalence-kv-memory = { path = "crates/kv/memory", version = "=0.1.0-dev.
|
|
26
|
-
covalence-kv-s3 = { path = "crates/kv/s3", version = "=0.1.0-dev.
|
|
27
|
-
covalence-kv-trait = { path = "crates/kv/trait", version = "=0.1.0-dev.
|
|
28
|
-
covalence-lib-cid = { path = "crates/lib/cid", version = "=0.1.0-dev.
|
|
29
|
-
covalence-lib-cli = { path = "crates/lib/cli", version = "=0.1.0-dev.
|
|
30
|
-
covalence-lib-dasl = { path = "crates/lib/dasl", version = "=0.1.0-dev.
|
|
31
|
-
covalence-lib-data = { path = "crates/lib/data", version = "=0.1.0-dev.
|
|
32
|
-
covalence-lib-error = { path = "crates/lib/error", version = "=0.1.0-dev.
|
|
33
|
-
covalence-lib-rand = { path = "crates/lib/rand", version = "=0.1.0-dev.
|
|
34
|
-
covalence-lib-serde = { path = "crates/lib/serde", version = "=0.1.0-dev.
|
|
35
|
-
covalence-lib-sqlite = { path = "crates/lib/sqlite", version = "=0.1.0-dev.
|
|
20
|
+
covalence-blob-trait = { path = "crates/blob/trait", version = "=0.1.0-dev.5" }
|
|
21
|
+
covalence-cas-kv = { path = "crates/cas/kv", version = "=0.1.0-dev.5" }
|
|
22
|
+
covalence-cas-memory = { path = "crates/cas/memory", version = "=0.1.0-dev.5" }
|
|
23
|
+
covalence-cas-sqlite = { path = "crates/cas/sqlite", version = "=0.1.0-dev.5" }
|
|
24
|
+
covalence-cas-trait = { path = "crates/cas/trait", version = "=0.1.0-dev.5" }
|
|
25
|
+
covalence-kv-memory = { path = "crates/kv/memory", version = "=0.1.0-dev.5" }
|
|
26
|
+
covalence-kv-s3 = { path = "crates/kv/s3", version = "=0.1.0-dev.5" }
|
|
27
|
+
covalence-kv-trait = { path = "crates/kv/trait", version = "=0.1.0-dev.5" }
|
|
28
|
+
covalence-lib-cid = { path = "crates/lib/cid", version = "=0.1.0-dev.5" }
|
|
29
|
+
covalence-lib-cli = { path = "crates/lib/cli", version = "=0.1.0-dev.5" }
|
|
30
|
+
covalence-lib-dasl = { path = "crates/lib/dasl", version = "=0.1.0-dev.5" }
|
|
31
|
+
covalence-lib-data = { path = "crates/lib/data", version = "=0.1.0-dev.5" }
|
|
32
|
+
covalence-lib-error = { path = "crates/lib/error", version = "=0.1.0-dev.5" }
|
|
33
|
+
covalence-lib-rand = { path = "crates/lib/rand", version = "=0.1.0-dev.5" }
|
|
34
|
+
covalence-lib-serde = { path = "crates/lib/serde", version = "=0.1.0-dev.5" }
|
|
35
|
+
covalence-lib-sqlite = { path = "crates/lib/sqlite", version = "=0.1.0-dev.5" }
|
|
36
36
|
|
|
37
37
|
[workspace.lints.rust]
|
|
38
38
|
unsafe_code = "forbid"
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: covalence
|
|
3
|
-
Version: 0.1.0.
|
|
3
|
+
Version: 0.1.0.dev5
|
|
4
4
|
License-File: LICENSE-APACHE
|
|
5
5
|
License-File: LICENSE-CC0
|
|
6
6
|
License-File: LICENSE-MIT
|
|
@@ -35,7 +35,7 @@ True
|
|
|
35
35
|
## Install
|
|
36
36
|
|
|
37
37
|
```sh
|
|
38
|
-
pip install covalence==0.1.0.
|
|
38
|
+
pip install covalence==0.1.0.dev5
|
|
39
39
|
```
|
|
40
40
|
|
|
41
41
|
- Pre-releases need an exact pin like this one, or `pip install --pre`.
|
|
@@ -120,8 +120,23 @@ pip install covalence==0.1.0.dev4
|
|
|
120
120
|
- S3 keys must be UTF-8 with non-empty `/`-separated segments other than `.`
|
|
121
121
|
and `..`; other keys raise `StoreError`.
|
|
122
122
|
- On S3, values over 8 MiB go up as multipart uploads, and values come down
|
|
123
|
-
in ranged GETs of 8 MiB, 16 parts or ranges at a time.
|
|
124
|
-
|
|
123
|
+
in ranged GETs of 8 MiB, 16 parts or ranges at a time. Upload parts grow
|
|
124
|
+
when needed to stay within 10,000 parts. Covalence checks
|
|
125
|
+
[S3's limits](https://docs.aws.amazon.com/AmazonS3/latest/userguide/qfacts.html)
|
|
126
|
+
before upload: at most 10,000 parts of 5 GiB each, subject to the platform's
|
|
127
|
+
allocation limit.
|
|
128
|
+
- `Kv.s3(..., upload_buffer_bytes=n)` caps aggregate upload payload
|
|
129
|
+
reservations across calls sharing that store. An upload reserves the smaller
|
|
130
|
+
of its length and its part size times its worker count before allocating
|
|
131
|
+
parts, reading its upload source or making requests. Insufficient capacity
|
|
132
|
+
raises `StoreError` immediately; zero admits only empty uploads. The default
|
|
133
|
+
is no budget. Reservations last through retries and cleanup.
|
|
134
|
+
- This budget excludes caller-owned sources, read buffers, allocator overhead,
|
|
135
|
+
HTTP/TLS buffers and request metadata; it is not an RSS limit. Owned byte
|
|
136
|
+
values still reserve their planned payload even though their parts share the
|
|
137
|
+
original allocation. File helpers may hash their source before upload
|
|
138
|
+
admission. A larger legal object can need a larger budget because its parts
|
|
139
|
+
grow; rejection does not consume its upload source or create an upload.
|
|
125
140
|
|
|
126
141
|
## Content-addressed stores
|
|
127
142
|
|
|
@@ -196,8 +211,10 @@ pip install covalence==0.1.0.dev4
|
|
|
196
211
|
- These limits govern source length and output bytes, not the memory a backend
|
|
197
212
|
uses before reading or writing. Python callback transfers are at most 1 MiB.
|
|
198
213
|
- `put_file`, `put_fileobj`, `put_from`, `get_file` and `get_to` stream on `Cas.kv` over `Kv.s3`,
|
|
199
|
-
holding up to 16
|
|
214
|
+
holding up to 16 part payloads at a time, or as many as
|
|
200
215
|
`Kv.s3(..., concurrency=n)` says; blocks over 78 GiB go up in larger parts.
|
|
216
|
+
Upload parts start at 8 MiB; read ranges stay at 8 MiB. Set
|
|
217
|
+
`upload_buffer_bytes=` to enforce a shared upload payload budget.
|
|
201
218
|
Other stores, including `Cas.kv(Kv.memory())`, hold the block in memory.
|
|
202
219
|
- A create-only write can reuse a block without checking its stored bytes.
|
|
203
220
|
Returning a CID does not freshly verify an existing block: use a checked
|
|
@@ -305,16 +322,16 @@ stub; this package writes its stub by hand and checks it against `__doc__`:
|
|
|
305
322
|
later) for manylinux_2_28 x86_64 and aarch64 and macOS arm64, and an sdist,
|
|
306
323
|
and runs `tests/` against each wheel. Pull requests and pushes keep them as
|
|
307
324
|
workflow artifacts. A release tag must match the workspace version, such as
|
|
308
|
-
`0.1.0-dev.
|
|
325
|
+
`0.1.0-dev.5`; the files carry its PEP 440 form, `0.1.0.dev5`:
|
|
309
326
|
|
|
310
327
|
- `v<version>` attaches them to that GitHub release. In the public
|
|
311
328
|
`hiloopai/covalence` repository it also publishes them to PyPI, and
|
|
312
329
|
`.github/workflows/crates.yml` publishes the crates to crates.io.
|
|
313
330
|
- `python-v<version>` only attaches them to the GitHub release.
|
|
314
331
|
|
|
315
|
-
To publish `0.1.0-dev.
|
|
332
|
+
To publish `0.1.0-dev.5`:
|
|
316
333
|
|
|
317
334
|
```sh
|
|
318
|
-
git tag v0.1.0-dev.
|
|
335
|
+
git tag v0.1.0-dev.5 && git push origin v0.1.0-dev.5
|
|
319
336
|
```
|
|
320
337
|
|
|
@@ -21,7 +21,7 @@ True
|
|
|
21
21
|
## Install
|
|
22
22
|
|
|
23
23
|
```sh
|
|
24
|
-
pip install covalence==0.1.0.
|
|
24
|
+
pip install covalence==0.1.0.dev5
|
|
25
25
|
```
|
|
26
26
|
|
|
27
27
|
- Pre-releases need an exact pin like this one, or `pip install --pre`.
|
|
@@ -106,8 +106,23 @@ pip install covalence==0.1.0.dev4
|
|
|
106
106
|
- S3 keys must be UTF-8 with non-empty `/`-separated segments other than `.`
|
|
107
107
|
and `..`; other keys raise `StoreError`.
|
|
108
108
|
- On S3, values over 8 MiB go up as multipart uploads, and values come down
|
|
109
|
-
in ranged GETs of 8 MiB, 16 parts or ranges at a time.
|
|
110
|
-
|
|
109
|
+
in ranged GETs of 8 MiB, 16 parts or ranges at a time. Upload parts grow
|
|
110
|
+
when needed to stay within 10,000 parts. Covalence checks
|
|
111
|
+
[S3's limits](https://docs.aws.amazon.com/AmazonS3/latest/userguide/qfacts.html)
|
|
112
|
+
before upload: at most 10,000 parts of 5 GiB each, subject to the platform's
|
|
113
|
+
allocation limit.
|
|
114
|
+
- `Kv.s3(..., upload_buffer_bytes=n)` caps aggregate upload payload
|
|
115
|
+
reservations across calls sharing that store. An upload reserves the smaller
|
|
116
|
+
of its length and its part size times its worker count before allocating
|
|
117
|
+
parts, reading its upload source or making requests. Insufficient capacity
|
|
118
|
+
raises `StoreError` immediately; zero admits only empty uploads. The default
|
|
119
|
+
is no budget. Reservations last through retries and cleanup.
|
|
120
|
+
- This budget excludes caller-owned sources, read buffers, allocator overhead,
|
|
121
|
+
HTTP/TLS buffers and request metadata; it is not an RSS limit. Owned byte
|
|
122
|
+
values still reserve their planned payload even though their parts share the
|
|
123
|
+
original allocation. File helpers may hash their source before upload
|
|
124
|
+
admission. A larger legal object can need a larger budget because its parts
|
|
125
|
+
grow; rejection does not consume its upload source or create an upload.
|
|
111
126
|
|
|
112
127
|
## Content-addressed stores
|
|
113
128
|
|
|
@@ -182,8 +197,10 @@ pip install covalence==0.1.0.dev4
|
|
|
182
197
|
- These limits govern source length and output bytes, not the memory a backend
|
|
183
198
|
uses before reading or writing. Python callback transfers are at most 1 MiB.
|
|
184
199
|
- `put_file`, `put_fileobj`, `put_from`, `get_file` and `get_to` stream on `Cas.kv` over `Kv.s3`,
|
|
185
|
-
holding up to 16
|
|
200
|
+
holding up to 16 part payloads at a time, or as many as
|
|
186
201
|
`Kv.s3(..., concurrency=n)` says; blocks over 78 GiB go up in larger parts.
|
|
202
|
+
Upload parts start at 8 MiB; read ranges stay at 8 MiB. Set
|
|
203
|
+
`upload_buffer_bytes=` to enforce a shared upload payload budget.
|
|
187
204
|
Other stores, including `Cas.kv(Kv.memory())`, hold the block in memory.
|
|
188
205
|
- A create-only write can reuse a block without checking its stored bytes.
|
|
189
206
|
Returning a CID does not freshly verify an existing block: use a checked
|
|
@@ -291,15 +308,15 @@ stub; this package writes its stub by hand and checks it against `__doc__`:
|
|
|
291
308
|
later) for manylinux_2_28 x86_64 and aarch64 and macOS arm64, and an sdist,
|
|
292
309
|
and runs `tests/` against each wheel. Pull requests and pushes keep them as
|
|
293
310
|
workflow artifacts. A release tag must match the workspace version, such as
|
|
294
|
-
`0.1.0-dev.
|
|
311
|
+
`0.1.0-dev.5`; the files carry its PEP 440 form, `0.1.0.dev5`:
|
|
295
312
|
|
|
296
313
|
- `v<version>` attaches them to that GitHub release. In the public
|
|
297
314
|
`hiloopai/covalence` repository it also publishes them to PyPI, and
|
|
298
315
|
`.github/workflows/crates.yml` publishes the crates to crates.io.
|
|
299
316
|
- `python-v<version>` only attaches them to the GitHub release.
|
|
300
317
|
|
|
301
|
-
To publish `0.1.0-dev.
|
|
318
|
+
To publish `0.1.0-dev.5`:
|
|
302
319
|
|
|
303
320
|
```sh
|
|
304
|
-
git tag v0.1.0-dev.
|
|
321
|
+
git tag v0.1.0-dev.5 && git push origin v0.1.0-dev.5
|
|
305
322
|
```
|
|
@@ -30,17 +30,18 @@
|
|
|
30
30
|
mod read;
|
|
31
31
|
mod write;
|
|
32
32
|
|
|
33
|
+
pub use write::UploadPlan;
|
|
33
34
|
use write::Write;
|
|
34
35
|
|
|
35
36
|
use std::{
|
|
36
|
-
collections::BTreeMap,
|
|
37
|
+
collections::{BTreeMap, TryReserveError},
|
|
37
38
|
fmt,
|
|
38
39
|
io::{self, Read},
|
|
39
40
|
panic::{self, AssertUnwindSafe},
|
|
40
41
|
str::FromStr,
|
|
41
42
|
sync::{
|
|
42
43
|
Mutex, PoisonError,
|
|
43
|
-
atomic::{AtomicBool, Ordering},
|
|
44
|
+
atomic::{AtomicBool, AtomicU64, Ordering},
|
|
44
45
|
mpsc,
|
|
45
46
|
},
|
|
46
47
|
thread,
|
|
@@ -68,6 +69,34 @@ use rusty_s3::{Bucket, BucketError, UrlStyle};
|
|
|
68
69
|
#[snafu(crate_root(covalence_lib_error::snafu))]
|
|
69
70
|
#[non_exhaustive]
|
|
70
71
|
pub enum Error {
|
|
72
|
+
/// An upload exceeds an S3 limit or the platform's allocation limit.
|
|
73
|
+
#[snafu(display("invalid upload {field} {value}: expected {minimum}..={maximum}"))]
|
|
74
|
+
UploadLimit {
|
|
75
|
+
/// The quantity checked.
|
|
76
|
+
field: &'static str,
|
|
77
|
+
/// The requested quantity.
|
|
78
|
+
value: u64,
|
|
79
|
+
/// Its smallest supported value.
|
|
80
|
+
minimum: u64,
|
|
81
|
+
/// Its largest supported value.
|
|
82
|
+
maximum: u64,
|
|
83
|
+
},
|
|
84
|
+
/// The store's upload payload budget cannot admit another upload.
|
|
85
|
+
#[snafu(display("upload needs {required} buffer bytes; {available} available"))]
|
|
86
|
+
UploadBuffer {
|
|
87
|
+
/// The payload reservation required for this upload.
|
|
88
|
+
required: u64,
|
|
89
|
+
/// The unreserved bytes when admission failed.
|
|
90
|
+
available: u64,
|
|
91
|
+
},
|
|
92
|
+
/// Allocating an upload part failed after admission.
|
|
93
|
+
#[snafu(display("could not allocate {bytes} upload bytes: {source}"))]
|
|
94
|
+
UploadAllocation {
|
|
95
|
+
/// The part's size in bytes.
|
|
96
|
+
bytes: u64,
|
|
97
|
+
/// Why the allocator refused it.
|
|
98
|
+
source: TryReserveError,
|
|
99
|
+
},
|
|
71
100
|
/// The endpoint is not a URL.
|
|
72
101
|
#[snafu(display("invalid endpoint {endpoint}: {source}"))]
|
|
73
102
|
Endpoint {
|
|
@@ -254,10 +283,18 @@ const EXPIRY: Duration = Duration::from_secs(60);
|
|
|
254
283
|
const TIMEOUT: Duration = Duration::from_secs(30);
|
|
255
284
|
/// The slowest transfer, in bytes per second, a request attempt waits for.
|
|
256
285
|
const MIN_RATE: u64 = 1 << 20;
|
|
257
|
-
/// The
|
|
258
|
-
const
|
|
286
|
+
/// The body size above which request timeouts stop increasing.
|
|
287
|
+
const MAX_TIMEOUT_BYTES: u64 = 5 << 40;
|
|
259
288
|
/// The most parts a multipart upload may have.
|
|
260
289
|
const MAX_PARTS: u64 = 10_000;
|
|
290
|
+
/// S3's largest multipart part, in bytes.
|
|
291
|
+
const MAX_PART: u64 = 5 << 30;
|
|
292
|
+
/// S3's smallest non-final multipart part, in bytes.
|
|
293
|
+
const MIN_PART: u64 = 5 << 20;
|
|
294
|
+
/// S3's largest object: 10,000 parts of 5 GiB.
|
|
295
|
+
const MAX_OBJECT: u64 = MAX_PARTS * MAX_PART;
|
|
296
|
+
/// The largest body sent with one PUT, in bytes (5 GB).
|
|
297
|
+
const MAX_SINGLE_PUT: u64 = 5_000_000_000;
|
|
261
298
|
/// The default part size, 8 MiB: values larger are uploaded in parts of at
|
|
262
299
|
/// least this size, and values are read in ranges of it.
|
|
263
300
|
pub const PART: u64 = 8 << 20;
|
|
@@ -283,7 +320,7 @@ pub struct AwsOptions {
|
|
|
283
320
|
|
|
284
321
|
/// The limit on a request attempt moving `bytes` of body.
|
|
285
322
|
fn timeout(bytes: u64) -> Duration {
|
|
286
|
-
TIMEOUT + Duration::from_secs(bytes.min(
|
|
323
|
+
TIMEOUT + Duration::from_secs(bytes.min(MAX_TIMEOUT_BYTES) / MIN_RATE)
|
|
287
324
|
}
|
|
288
325
|
|
|
289
326
|
/// Run `future` to completion on `runtime`, on a thread of its own so that
|
|
@@ -400,9 +437,12 @@ impl Cached {
|
|
|
400
437
|
/// # Writes
|
|
401
438
|
///
|
|
402
439
|
/// - A value of up to the part size ([`PART`] unless set with
|
|
403
|
-
/// [`S3Kv::part_size`]) is written with one PUT.
|
|
440
|
+
/// [`S3Kv::part_size`]) and 5 GB is written with one PUT.
|
|
404
441
|
/// - A larger value is written with a multipart upload, in parts of the part
|
|
405
442
|
/// size, or larger parts if needed to keep to S3's limit of 10,000 parts.
|
|
443
|
+
/// - [`S3Kv::upload_plan`] checks S3 and allocation limits before any
|
|
444
|
+
/// upload request or source read. [`S3Kv::upload_buffer_bytes`] can bound
|
|
445
|
+
/// the aggregate upload payload held by calls sharing this store.
|
|
406
446
|
/// - Each PUT or part sends the SHA-256 of its body as
|
|
407
447
|
/// `x-amz-checksum-sha256`, which S3 checks against the bytes it receives.
|
|
408
448
|
/// - A failed multipart upload is aborted once its parts in flight finish.
|
|
@@ -458,6 +498,43 @@ pub struct S3Kv {
|
|
|
458
498
|
agent: ureq::Agent,
|
|
459
499
|
part: u64,
|
|
460
500
|
concurrency: usize,
|
|
501
|
+
upload_budget: Option<UploadBudget>,
|
|
502
|
+
}
|
|
503
|
+
|
|
504
|
+
/// Aggregate payload reservations for calls sharing one store.
|
|
505
|
+
struct UploadBudget {
|
|
506
|
+
limit: u64,
|
|
507
|
+
used: AtomicU64,
|
|
508
|
+
}
|
|
509
|
+
|
|
510
|
+
impl UploadBudget {
|
|
511
|
+
fn reserve(&self, required: u64) -> Result<UploadReservation<'_>, Error> {
|
|
512
|
+
self.used
|
|
513
|
+
.fetch_update(Ordering::Relaxed, Ordering::Relaxed, |used| {
|
|
514
|
+
used.checked_add(required)
|
|
515
|
+
.filter(|next| *next <= self.limit)
|
|
516
|
+
})
|
|
517
|
+
.map_err(|used| Error::UploadBuffer {
|
|
518
|
+
required,
|
|
519
|
+
available: self.limit - used,
|
|
520
|
+
})?;
|
|
521
|
+
Ok(UploadReservation {
|
|
522
|
+
budget: self,
|
|
523
|
+
bytes: required,
|
|
524
|
+
})
|
|
525
|
+
}
|
|
526
|
+
}
|
|
527
|
+
|
|
528
|
+
/// Held until all upload requests, including cleanup, have finished.
|
|
529
|
+
struct UploadReservation<'a> {
|
|
530
|
+
budget: &'a UploadBudget,
|
|
531
|
+
bytes: u64,
|
|
532
|
+
}
|
|
533
|
+
|
|
534
|
+
impl Drop for UploadReservation<'_> {
|
|
535
|
+
fn drop(&mut self) {
|
|
536
|
+
self.budget.used.fetch_sub(self.bytes, Ordering::Relaxed);
|
|
537
|
+
}
|
|
461
538
|
}
|
|
462
539
|
|
|
463
540
|
impl fmt::Debug for S3Kv {
|
|
@@ -498,6 +575,7 @@ impl S3Kv {
|
|
|
498
575
|
agent: agent(CONCURRENCY),
|
|
499
576
|
part: PART,
|
|
500
577
|
concurrency: CONCURRENCY,
|
|
578
|
+
upload_budget: None,
|
|
501
579
|
})
|
|
502
580
|
}
|
|
503
581
|
|
|
@@ -517,6 +595,28 @@ impl S3Kv {
|
|
|
517
595
|
self
|
|
518
596
|
}
|
|
519
597
|
|
|
598
|
+
/// Limit the aggregate upload payload reservation to `bytes`.
|
|
599
|
+
///
|
|
600
|
+
/// Calls sharing this store (including through `Arc`) share the budget.
|
|
601
|
+
/// An upload reserves [`UploadPlan::buffer_bytes`] before reading its
|
|
602
|
+
/// source, allocating parts or sending requests, and keeps the reservation
|
|
603
|
+
/// until all its requests and cleanup finish. Insufficient capacity fails
|
|
604
|
+
/// immediately with [`Error::UploadBuffer`]; it does not wait. Zero admits
|
|
605
|
+
/// only empty uploads. No budget is set by default.
|
|
606
|
+
///
|
|
607
|
+
/// This bounds upload part payloads, including parts held for retries.
|
|
608
|
+
/// Caller-owned sources, read buffers, allocator overhead, HTTP/TLS buffers
|
|
609
|
+
/// and request metadata are additional memory; this is not an RSS limit.
|
|
610
|
+
/// Already owned values still reserve their planned upload payload, even
|
|
611
|
+
/// though their parts share the caller's allocation.
|
|
612
|
+
pub fn upload_buffer_bytes(mut self, bytes: u64) -> Self {
|
|
613
|
+
self.upload_budget = Some(UploadBudget {
|
|
614
|
+
limit: bytes,
|
|
615
|
+
used: AtomicU64::new(0),
|
|
616
|
+
});
|
|
617
|
+
self
|
|
618
|
+
}
|
|
619
|
+
|
|
520
620
|
/// A store for `location`, configured as the AWS SDKs are.
|
|
521
621
|
///
|
|
522
622
|
/// - Profile: `options.profile`, else `AWS_PROFILE`, else `default`.
|
|
@@ -12,10 +12,43 @@ use rusty_s3::{S3Action, actions::CreateMultipartUpload};
|
|
|
12
12
|
use sha2::{Digest, Sha256};
|
|
13
13
|
|
|
14
14
|
use crate::{
|
|
15
|
-
EXPIRY, Error, IoSnafu, MAX_ERROR,
|
|
16
|
-
timeout,
|
|
15
|
+
EXPIRY, Error, IoSnafu, MAX_ERROR, MAX_OBJECT, MAX_PART, MAX_PARTS, MAX_SINGLE_PUT, MIN_PART,
|
|
16
|
+
MissingFieldSnafu, S3Kv, UploadAllocationSnafu, body, status_error, timeout,
|
|
17
17
|
};
|
|
18
18
|
|
|
19
|
+
/// The checked geometry and upload payload reservation for a value.
|
|
20
|
+
///
|
|
21
|
+
/// Produced by [`S3Kv::upload_plan`], which is also used to admit writes.
|
|
22
|
+
/// It does not allocate payloads, reserve budget or send requests.
|
|
23
|
+
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
|
|
24
|
+
pub struct UploadPlan {
|
|
25
|
+
/// The largest part payload, or zero for an empty value.
|
|
26
|
+
pub part_bytes: u64,
|
|
27
|
+
/// The number of parts, including one for a single PUT.
|
|
28
|
+
pub parts: u16,
|
|
29
|
+
/// The maximum payload held by this upload's concurrent workers and reader.
|
|
30
|
+
///
|
|
31
|
+
/// Excludes source, allocator, HTTP/TLS and metadata overhead; see
|
|
32
|
+
/// [`S3Kv::upload_buffer_bytes`].
|
|
33
|
+
pub buffer_bytes: u64,
|
|
34
|
+
/// Whether the value uses multipart upload, rather than one PUT.
|
|
35
|
+
pub multipart: bool,
|
|
36
|
+
}
|
|
37
|
+
|
|
38
|
+
/// Reject unsupported quantities before any allocation or upload request.
|
|
39
|
+
fn within(field: &'static str, value: u64, minimum: u64, maximum: u64) -> Result<(), Error> {
|
|
40
|
+
if (minimum..=maximum).contains(&value) {
|
|
41
|
+
Ok(())
|
|
42
|
+
} else {
|
|
43
|
+
Err(Error::UploadLimit {
|
|
44
|
+
field,
|
|
45
|
+
value,
|
|
46
|
+
minimum,
|
|
47
|
+
maximum,
|
|
48
|
+
})
|
|
49
|
+
}
|
|
50
|
+
}
|
|
51
|
+
|
|
19
52
|
/// The header carrying the base64 SHA-256 of a PUT body, signed with the URL.
|
|
20
53
|
const CHECKSUM: &str = "x-amz-checksum-sha256";
|
|
21
54
|
|
|
@@ -53,6 +86,46 @@ fn escape(text: &str) -> String {
|
|
|
53
86
|
}
|
|
54
87
|
|
|
55
88
|
impl S3Kv {
|
|
89
|
+
/// Plan a write of `len` bytes without reading, allocating or sending.
|
|
90
|
+
///
|
|
91
|
+
/// Checks S3's 10,000-part limit, 5 GiB maximum part, 5 MiB minimum
|
|
92
|
+
/// non-final part, 5 GB single-PUT limit and this platform's allocation
|
|
93
|
+
/// limit. Parts grow beyond the configured size when needed. An invalid
|
|
94
|
+
/// length or part size returns [`Error::UploadLimit`].
|
|
95
|
+
///
|
|
96
|
+
/// The reservation is the smaller of the value's length and the part
|
|
97
|
+
/// size times the number of workers. It covers the reader's next part
|
|
98
|
+
/// as well as in-flight parts: a worker slot is available before the
|
|
99
|
+
/// next part is read. This does not reserve capacity or check current
|
|
100
|
+
/// availability; the write does that atomically before doing any I/O.
|
|
101
|
+
pub fn upload_plan(&self, len: u64) -> Result<UploadPlan, Error> {
|
|
102
|
+
within("object length", len, 0, MAX_OBJECT)?;
|
|
103
|
+
// A small configured part remains useful for ranged reads and tiny
|
|
104
|
+
// single PUTs, but an oversized one is never a supported upload setting.
|
|
105
|
+
if self.part > MAX_PART {
|
|
106
|
+
within("part size", self.part, MIN_PART, MAX_PART)?;
|
|
107
|
+
}
|
|
108
|
+
let part = self.part_for(len);
|
|
109
|
+
let multipart = len > part || len > MAX_SINGLE_PUT;
|
|
110
|
+
if multipart {
|
|
111
|
+
within("part size", part, MIN_PART, MAX_PART)?;
|
|
112
|
+
}
|
|
113
|
+
let parts = if multipart { len.div_ceil(part) } else { 1 };
|
|
114
|
+
within("part count", parts, 1, MAX_PARTS)?;
|
|
115
|
+
let part_bytes = len.min(part);
|
|
116
|
+
// Vec allocations must fit isize, a stronger condition than usize.
|
|
117
|
+
within("part allocation", part_bytes, 0, isize::MAX as u64)?;
|
|
118
|
+
let workers = self.concurrency.min(parts as usize) as u64;
|
|
119
|
+
// Validated S3 geometry bounds this product by MAX_OBJECT.
|
|
120
|
+
let buffer_bytes = len.min(workers * part_bytes);
|
|
121
|
+
Ok(UploadPlan {
|
|
122
|
+
part_bytes,
|
|
123
|
+
parts: parts as u16,
|
|
124
|
+
buffer_bytes,
|
|
125
|
+
multipart,
|
|
126
|
+
})
|
|
127
|
+
}
|
|
128
|
+
|
|
56
129
|
/// Store `value` under `key`, as `write` says; whether it stored.
|
|
57
130
|
pub(crate) fn write(&self, key: &[u8], value: Bytes, write: Write) -> Result<bool, Error> {
|
|
58
131
|
let mut offset = 0;
|
|
@@ -73,7 +146,10 @@ impl S3Kv {
|
|
|
73
146
|
write: Write,
|
|
74
147
|
) -> Result<bool, Error> {
|
|
75
148
|
self.upload(key, len, write, |len| {
|
|
76
|
-
let mut part =
|
|
149
|
+
let mut part = Vec::new();
|
|
150
|
+
part.try_reserve_exact(len)
|
|
151
|
+
.context(UploadAllocationSnafu { bytes: len as u64 })?;
|
|
152
|
+
part.resize(len, 0);
|
|
77
153
|
reader.read_exact(&mut part).context(IoSnafu)?;
|
|
78
154
|
Ok(part.into())
|
|
79
155
|
})
|
|
@@ -91,14 +167,22 @@ impl S3Kv {
|
|
|
91
167
|
write: Write,
|
|
92
168
|
mut next: impl FnMut(usize) -> Result<Bytes, Error>,
|
|
93
169
|
) -> Result<bool, Error> {
|
|
170
|
+
let plan = self.upload_plan(len)?;
|
|
171
|
+
// This guard outlives in_order's scoped worker joins and the abort
|
|
172
|
+
// request, on success, error and unwinding alike.
|
|
173
|
+
let _reservation = self
|
|
174
|
+
.upload_budget
|
|
175
|
+
.as_ref()
|
|
176
|
+
.map(|budget| budget.reserve(plan.buffer_bytes))
|
|
177
|
+
.transpose()?;
|
|
94
178
|
let object = self.object(key)?;
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
return self.put_object(&object, next(len as usize)?, write);
|
|
179
|
+
if !plan.multipart {
|
|
180
|
+
return self.put_object(&object, next(plan.part_bytes as usize)?, write);
|
|
98
181
|
}
|
|
99
182
|
let upload_id = self.create(&object)?;
|
|
100
|
-
//
|
|
101
|
-
let
|
|
183
|
+
// The planner checked every allocation and the part count above.
|
|
184
|
+
let part = plan.part_bytes;
|
|
185
|
+
let count = u64::from(plan.parts);
|
|
102
186
|
let mut parts = Vec::with_capacity(count as usize);
|
|
103
187
|
let jobs = (0..count).map(|index| {
|
|
104
188
|
let size = part.min(len - index * part);
|