covalence 0.1.0.dev3__tar.gz → 0.1.0.dev5__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {covalence-0.1.0.dev3 → covalence-0.1.0.dev5}/Cargo.lock +19 -19
- {covalence-0.1.0.dev3 → covalence-0.1.0.dev5}/Cargo.toml +17 -17
- {covalence-0.1.0.dev3 → covalence-0.1.0.dev5}/PKG-INFO +74 -11
- {covalence-0.1.0.dev3 → covalence-0.1.0.dev5}/README.md +73 -10
- {covalence-0.1.0.dev3 → covalence-0.1.0.dev5}/crates/kv/s3/src/lib.rs +106 -6
- {covalence-0.1.0.dev3 → covalence-0.1.0.dev5}/crates/kv/s3/src/write.rs +92 -8
- {covalence-0.1.0.dev3 → covalence-0.1.0.dev5}/crates/kv/s3/tests/kv.rs +391 -29
- {covalence-0.1.0.dev3 → covalence-0.1.0.dev5}/crates/python/README.md +73 -10
- {covalence-0.1.0.dev3 → covalence-0.1.0.dev5}/crates/python/src/lib.rs +117 -4
- covalence-0.1.0.dev5/crates/python/src/streams/download.rs +295 -0
- covalence-0.1.0.dev5/crates/python/src/streams/mod.rs +20 -0
- covalence-0.1.0.dev5/crates/python/src/streams/upload/tests.rs +241 -0
- covalence-0.1.0.dev5/crates/python/src/streams/upload.rs +215 -0
- {covalence-0.1.0.dev3 → covalence-0.1.0.dev5}/crates/python/tests/test_api.py +38 -0
- covalence-0.1.0.dev5/crates/python/tests/test_bounded_io.py +935 -0
- {covalence-0.1.0.dev3 → covalence-0.1.0.dev5}/python/covalence/THIRD-PARTY-LICENSES.md +17 -17
- {covalence-0.1.0.dev3 → covalence-0.1.0.dev5}/python/covalence/_native.pyi +77 -5
- {covalence-0.1.0.dev3 → covalence-0.1.0.dev5}/LICENSE-APACHE +0 -0
- {covalence-0.1.0.dev3 → covalence-0.1.0.dev5}/LICENSE-CC0 +0 -0
- {covalence-0.1.0.dev3 → covalence-0.1.0.dev5}/LICENSE-MIT +0 -0
- {covalence-0.1.0.dev3 → covalence-0.1.0.dev5}/crates/blob/trait/Cargo.toml +0 -0
- {covalence-0.1.0.dev3 → covalence-0.1.0.dev5}/crates/blob/trait/LICENSE-APACHE +0 -0
- {covalence-0.1.0.dev3 → covalence-0.1.0.dev5}/crates/blob/trait/LICENSE-CC0 +0 -0
- {covalence-0.1.0.dev3 → covalence-0.1.0.dev5}/crates/blob/trait/LICENSE-MIT +0 -0
- {covalence-0.1.0.dev3 → covalence-0.1.0.dev5}/crates/blob/trait/README.md +0 -0
- {covalence-0.1.0.dev3 → covalence-0.1.0.dev5}/crates/blob/trait/src/lib.rs +0 -0
- {covalence-0.1.0.dev3 → covalence-0.1.0.dev5}/crates/blob/trait/tests/blob.rs +0 -0
- {covalence-0.1.0.dev3 → covalence-0.1.0.dev5}/crates/blob/trait/tests/common/mod.rs +0 -0
- {covalence-0.1.0.dev3 → covalence-0.1.0.dev5}/crates/blob/trait/tests/fact.rs +0 -0
- {covalence-0.1.0.dev3 → covalence-0.1.0.dev5}/crates/cas/kv/Cargo.toml +0 -0
- {covalence-0.1.0.dev3 → covalence-0.1.0.dev5}/crates/cas/kv/LICENSE-APACHE +0 -0
- {covalence-0.1.0.dev3 → covalence-0.1.0.dev5}/crates/cas/kv/LICENSE-CC0 +0 -0
- {covalence-0.1.0.dev3 → covalence-0.1.0.dev5}/crates/cas/kv/LICENSE-MIT +0 -0
- {covalence-0.1.0.dev3 → covalence-0.1.0.dev5}/crates/cas/kv/src/lib.rs +0 -0
- {covalence-0.1.0.dev3 → covalence-0.1.0.dev5}/crates/cas/kv/tests/kv.rs +0 -0
- {covalence-0.1.0.dev3 → covalence-0.1.0.dev5}/crates/cas/memory/Cargo.toml +0 -0
- {covalence-0.1.0.dev3 → covalence-0.1.0.dev5}/crates/cas/memory/LICENSE-APACHE +0 -0
- {covalence-0.1.0.dev3 → covalence-0.1.0.dev5}/crates/cas/memory/LICENSE-CC0 +0 -0
- {covalence-0.1.0.dev3 → covalence-0.1.0.dev5}/crates/cas/memory/LICENSE-MIT +0 -0
- {covalence-0.1.0.dev3 → covalence-0.1.0.dev5}/crates/cas/memory/src/lib.rs +0 -0
- {covalence-0.1.0.dev3 → covalence-0.1.0.dev5}/crates/cas/memory/tests/memory.rs +0 -0
- {covalence-0.1.0.dev3 → covalence-0.1.0.dev5}/crates/cas/sqlite/Cargo.toml +0 -0
- {covalence-0.1.0.dev3 → covalence-0.1.0.dev5}/crates/cas/sqlite/LICENSE-APACHE +0 -0
- {covalence-0.1.0.dev3 → covalence-0.1.0.dev5}/crates/cas/sqlite/LICENSE-CC0 +0 -0
- {covalence-0.1.0.dev3 → covalence-0.1.0.dev5}/crates/cas/sqlite/LICENSE-MIT +0 -0
- {covalence-0.1.0.dev3 → covalence-0.1.0.dev5}/crates/cas/sqlite/src/lib.rs +0 -0
- {covalence-0.1.0.dev3 → covalence-0.1.0.dev5}/crates/cas/sqlite/tests/sqlite.rs +0 -0
- {covalence-0.1.0.dev3 → covalence-0.1.0.dev5}/crates/cas/trait/Cargo.toml +0 -0
- {covalence-0.1.0.dev3 → covalence-0.1.0.dev5}/crates/cas/trait/LICENSE-APACHE +0 -0
- {covalence-0.1.0.dev3 → covalence-0.1.0.dev5}/crates/cas/trait/LICENSE-CC0 +0 -0
- {covalence-0.1.0.dev3 → covalence-0.1.0.dev5}/crates/cas/trait/LICENSE-MIT +0 -0
- {covalence-0.1.0.dev3 → covalence-0.1.0.dev5}/crates/cas/trait/src/lib.rs +0 -0
- {covalence-0.1.0.dev3 → covalence-0.1.0.dev5}/crates/cas/trait/tests/trait.rs +0 -0
- {covalence-0.1.0.dev3 → covalence-0.1.0.dev5}/crates/kv/memory/Cargo.toml +0 -0
- {covalence-0.1.0.dev3 → covalence-0.1.0.dev5}/crates/kv/memory/LICENSE-APACHE +0 -0
- {covalence-0.1.0.dev3 → covalence-0.1.0.dev5}/crates/kv/memory/LICENSE-CC0 +0 -0
- {covalence-0.1.0.dev3 → covalence-0.1.0.dev5}/crates/kv/memory/LICENSE-MIT +0 -0
- {covalence-0.1.0.dev3 → covalence-0.1.0.dev5}/crates/kv/memory/src/lib.rs +0 -0
- {covalence-0.1.0.dev3 → covalence-0.1.0.dev5}/crates/kv/memory/tests/memory.rs +0 -0
- {covalence-0.1.0.dev3 → covalence-0.1.0.dev5}/crates/kv/s3/Cargo.toml +0 -0
- {covalence-0.1.0.dev3 → covalence-0.1.0.dev5}/crates/kv/s3/LICENSE-APACHE +0 -0
- {covalence-0.1.0.dev3 → covalence-0.1.0.dev5}/crates/kv/s3/LICENSE-CC0 +0 -0
- {covalence-0.1.0.dev3 → covalence-0.1.0.dev5}/crates/kv/s3/LICENSE-MIT +0 -0
- {covalence-0.1.0.dev3 → covalence-0.1.0.dev5}/crates/kv/s3/src/read.rs +0 -0
- {covalence-0.1.0.dev3 → covalence-0.1.0.dev5}/crates/kv/s3/tests/aws.rs +0 -0
- {covalence-0.1.0.dev3 → covalence-0.1.0.dev5}/crates/kv/trait/Cargo.toml +0 -0
- {covalence-0.1.0.dev3 → covalence-0.1.0.dev5}/crates/kv/trait/LICENSE-APACHE +0 -0
- {covalence-0.1.0.dev3 → covalence-0.1.0.dev5}/crates/kv/trait/LICENSE-CC0 +0 -0
- {covalence-0.1.0.dev3 → covalence-0.1.0.dev5}/crates/kv/trait/LICENSE-MIT +0 -0
- {covalence-0.1.0.dev3 → covalence-0.1.0.dev5}/crates/kv/trait/src/lib.rs +0 -0
- {covalence-0.1.0.dev3 → covalence-0.1.0.dev5}/crates/kv/trait/tests/atomic.rs +0 -0
- {covalence-0.1.0.dev3 → covalence-0.1.0.dev5}/crates/lib/cid/Cargo.toml +0 -0
- {covalence-0.1.0.dev3 → covalence-0.1.0.dev5}/crates/lib/cid/LICENSE-APACHE +0 -0
- {covalence-0.1.0.dev3 → covalence-0.1.0.dev5}/crates/lib/cid/LICENSE-CC0 +0 -0
- {covalence-0.1.0.dev3 → covalence-0.1.0.dev5}/crates/lib/cid/LICENSE-MIT +0 -0
- {covalence-0.1.0.dev3 → covalence-0.1.0.dev5}/crates/lib/cid/README.md +0 -0
- {covalence-0.1.0.dev3 → covalence-0.1.0.dev5}/crates/lib/cid/src/dasl.rs +0 -0
- {covalence-0.1.0.dev3 → covalence-0.1.0.dev5}/crates/lib/cid/src/fact.rs +0 -0
- {covalence-0.1.0.dev3 → covalence-0.1.0.dev5}/crates/lib/cid/src/git.rs +0 -0
- {covalence-0.1.0.dev3 → covalence-0.1.0.dev5}/crates/lib/cid/src/lib.rs +0 -0
- {covalence-0.1.0.dev3 → covalence-0.1.0.dev5}/crates/lib/cid/src/multicodec.rs +0 -0
- {covalence-0.1.0.dev3 → covalence-0.1.0.dev5}/crates/lib/cid/src/o256.rs +0 -0
- {covalence-0.1.0.dev3 → covalence-0.1.0.dev5}/crates/lib/cid/src/storage.rs +0 -0
- {covalence-0.1.0.dev3 → covalence-0.1.0.dev5}/crates/lib/cid/tests/cid.rs +0 -0
- {covalence-0.1.0.dev3 → covalence-0.1.0.dev5}/crates/lib/cid/tests/fact.rs +0 -0
- {covalence-0.1.0.dev3 → covalence-0.1.0.dev5}/crates/lib/cid/tests/git.rs +0 -0
- {covalence-0.1.0.dev3 → covalence-0.1.0.dev5}/crates/lib/cid/tests/o256.rs +0 -0
- {covalence-0.1.0.dev3 → covalence-0.1.0.dev5}/crates/lib/cid/tests/shattered/1.bin +0 -0
- {covalence-0.1.0.dev3 → covalence-0.1.0.dev5}/crates/lib/cid/tests/shattered/2.bin +0 -0
- {covalence-0.1.0.dev3 → covalence-0.1.0.dev5}/crates/lib/cid/tests/shattered/README.md +0 -0
- {covalence-0.1.0.dev3 → covalence-0.1.0.dev5}/crates/lib/dasl/Cargo.toml +0 -0
- {covalence-0.1.0.dev3 → covalence-0.1.0.dev5}/crates/lib/dasl/LICENSE-APACHE +0 -0
- {covalence-0.1.0.dev3 → covalence-0.1.0.dev5}/crates/lib/dasl/LICENSE-CC0 +0 -0
- {covalence-0.1.0.dev3 → covalence-0.1.0.dev5}/crates/lib/dasl/LICENSE-MIT +0 -0
- {covalence-0.1.0.dev3 → covalence-0.1.0.dev5}/crates/lib/dasl/README.md +0 -0
- {covalence-0.1.0.dev3 → covalence-0.1.0.dev5}/crates/lib/dasl/src/drisl.rs +0 -0
- {covalence-0.1.0.dev3 → covalence-0.1.0.dev5}/crates/lib/dasl/src/event.rs +0 -0
- {covalence-0.1.0.dev3 → covalence-0.1.0.dev5}/crates/lib/dasl/src/json.rs +0 -0
- {covalence-0.1.0.dev3 → covalence-0.1.0.dev5}/crates/lib/dasl/src/lib.rs +0 -0
- {covalence-0.1.0.dev3 → covalence-0.1.0.dev5}/crates/lib/dasl/tests/codec.rs +0 -0
- {covalence-0.1.0.dev3 → covalence-0.1.0.dev5}/crates/lib/dasl/tests/events.rs +0 -0
- {covalence-0.1.0.dev3 → covalence-0.1.0.dev5}/crates/lib/dasl/tests/fact.rs +0 -0
- {covalence-0.1.0.dev3 → covalence-0.1.0.dev5}/crates/lib/dasl/tests/fixtures/LICENSE +0 -0
- {covalence-0.1.0.dev3 → covalence-0.1.0.dev5}/crates/lib/dasl/tests/fixtures/README.md +0 -0
- {covalence-0.1.0.dev3 → covalence-0.1.0.dev5}/crates/lib/dasl/tests/fixtures/cid.json +0 -0
- {covalence-0.1.0.dev3 → covalence-0.1.0.dev5}/crates/lib/dasl/tests/fixtures/concat.json +0 -0
- {covalence-0.1.0.dev3 → covalence-0.1.0.dev5}/crates/lib/dasl/tests/fixtures/floats.json +0 -0
- {covalence-0.1.0.dev3 → covalence-0.1.0.dev5}/crates/lib/dasl/tests/fixtures/indefinite.json +0 -0
- {covalence-0.1.0.dev3 → covalence-0.1.0.dev5}/crates/lib/dasl/tests/fixtures/integer_range.json +0 -0
- {covalence-0.1.0.dev3 → covalence-0.1.0.dev5}/crates/lib/dasl/tests/fixtures/map_keys.json +0 -0
- {covalence-0.1.0.dev3 → covalence-0.1.0.dev5}/crates/lib/dasl/tests/fixtures/numeric_reduction.json +0 -0
- {covalence-0.1.0.dev3 → covalence-0.1.0.dev5}/crates/lib/dasl/tests/fixtures/short_form.json +0 -0
- {covalence-0.1.0.dev3 → covalence-0.1.0.dev5}/crates/lib/dasl/tests/fixtures/simple.json +0 -0
- {covalence-0.1.0.dev3 → covalence-0.1.0.dev5}/crates/lib/dasl/tests/fixtures/tags.json +0 -0
- {covalence-0.1.0.dev3 → covalence-0.1.0.dev5}/crates/lib/dasl/tests/fixtures/utf8.json +0 -0
- {covalence-0.1.0.dev3 → covalence-0.1.0.dev5}/crates/lib/dasl/tests/fixtures.rs +0 -0
- {covalence-0.1.0.dev3 → covalence-0.1.0.dev5}/crates/lib/dasl/tests/json.rs +0 -0
- {covalence-0.1.0.dev3 → covalence-0.1.0.dev5}/crates/lib/dasl/tests/vectors/README.md +0 -0
- {covalence-0.1.0.dev3 → covalence-0.1.0.dev5}/crates/lib/dasl/tests/vectors/cid-vectors.jsonl +0 -0
- {covalence-0.1.0.dev3 → covalence-0.1.0.dev5}/crates/lib/dasl/tests/vectors/dasl-vectors.jsonl +0 -0
- {covalence-0.1.0.dev3 → covalence-0.1.0.dev5}/crates/lib/dasl/tests/vectors.rs +0 -0
- {covalence-0.1.0.dev3 → covalence-0.1.0.dev5}/crates/lib/data/Cargo.toml +0 -0
- {covalence-0.1.0.dev3 → covalence-0.1.0.dev5}/crates/lib/data/LICENSE-APACHE +0 -0
- {covalence-0.1.0.dev3 → covalence-0.1.0.dev5}/crates/lib/data/LICENSE-CC0 +0 -0
- {covalence-0.1.0.dev3 → covalence-0.1.0.dev5}/crates/lib/data/LICENSE-MIT +0 -0
- {covalence-0.1.0.dev3 → covalence-0.1.0.dev5}/crates/lib/data/src/lib.rs +0 -0
- {covalence-0.1.0.dev3 → covalence-0.1.0.dev5}/crates/lib/error/Cargo.toml +0 -0
- {covalence-0.1.0.dev3 → covalence-0.1.0.dev5}/crates/lib/error/LICENSE-APACHE +0 -0
- {covalence-0.1.0.dev3 → covalence-0.1.0.dev5}/crates/lib/error/LICENSE-CC0 +0 -0
- {covalence-0.1.0.dev3 → covalence-0.1.0.dev5}/crates/lib/error/LICENSE-MIT +0 -0
- {covalence-0.1.0.dev3 → covalence-0.1.0.dev5}/crates/lib/error/src/lib.rs +0 -0
- {covalence-0.1.0.dev3 → covalence-0.1.0.dev5}/crates/lib/python/Cargo.toml +0 -0
- {covalence-0.1.0.dev3 → covalence-0.1.0.dev5}/crates/lib/python/LICENSE-APACHE +0 -0
- {covalence-0.1.0.dev3 → covalence-0.1.0.dev5}/crates/lib/python/LICENSE-CC0 +0 -0
- {covalence-0.1.0.dev3 → covalence-0.1.0.dev5}/crates/lib/python/LICENSE-MIT +0 -0
- {covalence-0.1.0.dev3 → covalence-0.1.0.dev5}/crates/lib/python/src/lib.rs +0 -0
- {covalence-0.1.0.dev3 → covalence-0.1.0.dev5}/crates/lib/rand/Cargo.toml +0 -0
- {covalence-0.1.0.dev3 → covalence-0.1.0.dev5}/crates/lib/rand/LICENSE-APACHE +0 -0
- {covalence-0.1.0.dev3 → covalence-0.1.0.dev5}/crates/lib/rand/LICENSE-CC0 +0 -0
- {covalence-0.1.0.dev3 → covalence-0.1.0.dev5}/crates/lib/rand/LICENSE-MIT +0 -0
- {covalence-0.1.0.dev3 → covalence-0.1.0.dev5}/crates/lib/rand/src/lib.rs +0 -0
- {covalence-0.1.0.dev3 → covalence-0.1.0.dev5}/crates/lib/serde/Cargo.toml +0 -0
- {covalence-0.1.0.dev3 → covalence-0.1.0.dev5}/crates/lib/serde/LICENSE-APACHE +0 -0
- {covalence-0.1.0.dev3 → covalence-0.1.0.dev5}/crates/lib/serde/LICENSE-CC0 +0 -0
- {covalence-0.1.0.dev3 → covalence-0.1.0.dev5}/crates/lib/serde/LICENSE-MIT +0 -0
- {covalence-0.1.0.dev3 → covalence-0.1.0.dev5}/crates/lib/serde/src/lib.rs +0 -0
- {covalence-0.1.0.dev3 → covalence-0.1.0.dev5}/crates/lib/sqlite/Cargo.toml +0 -0
- {covalence-0.1.0.dev3 → covalence-0.1.0.dev5}/crates/lib/sqlite/LICENSE-APACHE +0 -0
- {covalence-0.1.0.dev3 → covalence-0.1.0.dev5}/crates/lib/sqlite/LICENSE-CC0 +0 -0
- {covalence-0.1.0.dev3 → covalence-0.1.0.dev5}/crates/lib/sqlite/LICENSE-MIT +0 -0
- {covalence-0.1.0.dev3 → covalence-0.1.0.dev5}/crates/lib/sqlite/src/lib.rs +0 -0
- {covalence-0.1.0.dev3 → covalence-0.1.0.dev5}/crates/python/Cargo.toml +0 -0
- {covalence-0.1.0.dev3 → covalence-0.1.0.dev5}/crates/python/LICENSE-APACHE +0 -0
- {covalence-0.1.0.dev3 → covalence-0.1.0.dev5}/crates/python/LICENSE-CC0 +0 -0
- {covalence-0.1.0.dev3 → covalence-0.1.0.dev5}/crates/python/LICENSE-MIT +0 -0
- {covalence-0.1.0.dev3 → covalence-0.1.0.dev5}/crates/python/about.md.hbs +0 -0
- {covalence-0.1.0.dev3 → covalence-0.1.0.dev5}/crates/python/about.toml +0 -0
- {covalence-0.1.0.dev3 → covalence-0.1.0.dev5}/crates/python/src/errors.rs +0 -0
- {covalence-0.1.0.dev3 → covalence-0.1.0.dev5}/crates/python/tests/test_docs.py +0 -0
- {covalence-0.1.0.dev3 → covalence-0.1.0.dev5}/crates/python/tests/test_package.py +0 -0
- {covalence-0.1.0.dev3 → covalence-0.1.0.dev5}/crates/python/tests/test_stubs.py +0 -0
- {covalence-0.1.0.dev3 → covalence-0.1.0.dev5}/crates/python/tests/typed_usage.py +0 -0
- {covalence-0.1.0.dev3 → covalence-0.1.0.dev5}/pyproject.toml +0 -0
- {covalence-0.1.0.dev3 → covalence-0.1.0.dev5}/python/covalence/__init__.py +0 -0
- {covalence-0.1.0.dev3 → covalence-0.1.0.dev5}/python/covalence/py.typed +0 -0
|
@@ -779,7 +779,7 @@ checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b"
|
|
|
779
779
|
|
|
780
780
|
[[package]]
|
|
781
781
|
name = "covalence-blob-trait"
|
|
782
|
-
version = "0.1.0-dev.
|
|
782
|
+
version = "0.1.0-dev.5"
|
|
783
783
|
dependencies = [
|
|
784
784
|
"covalence-lib-cid",
|
|
785
785
|
"covalence-lib-data",
|
|
@@ -788,7 +788,7 @@ dependencies = [
|
|
|
788
788
|
|
|
789
789
|
[[package]]
|
|
790
790
|
name = "covalence-cas-kv"
|
|
791
|
-
version = "0.1.0-dev.
|
|
791
|
+
version = "0.1.0-dev.5"
|
|
792
792
|
dependencies = [
|
|
793
793
|
"blake3",
|
|
794
794
|
"covalence-cas-trait",
|
|
@@ -810,7 +810,7 @@ dependencies = [
|
|
|
810
810
|
|
|
811
811
|
[[package]]
|
|
812
812
|
name = "covalence-cas-memory"
|
|
813
|
-
version = "0.1.0-dev.
|
|
813
|
+
version = "0.1.0-dev.5"
|
|
814
814
|
dependencies = [
|
|
815
815
|
"covalence-cas-trait",
|
|
816
816
|
"covalence-lib-cid",
|
|
@@ -821,7 +821,7 @@ dependencies = [
|
|
|
821
821
|
|
|
822
822
|
[[package]]
|
|
823
823
|
name = "covalence-cas-sqlite"
|
|
824
|
-
version = "0.1.0-dev.
|
|
824
|
+
version = "0.1.0-dev.5"
|
|
825
825
|
dependencies = [
|
|
826
826
|
"covalence-cas-trait",
|
|
827
827
|
"covalence-lib-cid",
|
|
@@ -836,7 +836,7 @@ dependencies = [
|
|
|
836
836
|
|
|
837
837
|
[[package]]
|
|
838
838
|
name = "covalence-cas-trait"
|
|
839
|
-
version = "0.1.0-dev.
|
|
839
|
+
version = "0.1.0-dev.5"
|
|
840
840
|
dependencies = [
|
|
841
841
|
"covalence-lib-cid",
|
|
842
842
|
"covalence-lib-dasl",
|
|
@@ -847,7 +847,7 @@ dependencies = [
|
|
|
847
847
|
|
|
848
848
|
[[package]]
|
|
849
849
|
name = "covalence-kv-memory"
|
|
850
|
-
version = "0.1.0-dev.
|
|
850
|
+
version = "0.1.0-dev.5"
|
|
851
851
|
dependencies = [
|
|
852
852
|
"covalence-kv-trait",
|
|
853
853
|
"covalence-lib-dasl",
|
|
@@ -857,7 +857,7 @@ dependencies = [
|
|
|
857
857
|
|
|
858
858
|
[[package]]
|
|
859
859
|
name = "covalence-kv-s3"
|
|
860
|
-
version = "0.1.0-dev.
|
|
860
|
+
version = "0.1.0-dev.5"
|
|
861
861
|
dependencies = [
|
|
862
862
|
"aws-config",
|
|
863
863
|
"aws-credential-types",
|
|
@@ -881,7 +881,7 @@ dependencies = [
|
|
|
881
881
|
|
|
882
882
|
[[package]]
|
|
883
883
|
name = "covalence-kv-trait"
|
|
884
|
-
version = "0.1.0-dev.
|
|
884
|
+
version = "0.1.0-dev.5"
|
|
885
885
|
dependencies = [
|
|
886
886
|
"covalence-lib-dasl",
|
|
887
887
|
"covalence-lib-data",
|
|
@@ -890,7 +890,7 @@ dependencies = [
|
|
|
890
890
|
|
|
891
891
|
[[package]]
|
|
892
892
|
name = "covalence-lib-cid"
|
|
893
|
-
version = "0.1.0-dev.
|
|
893
|
+
version = "0.1.0-dev.5"
|
|
894
894
|
dependencies = [
|
|
895
895
|
"blake3",
|
|
896
896
|
"covalence-lib-data",
|
|
@@ -905,14 +905,14 @@ dependencies = [
|
|
|
905
905
|
|
|
906
906
|
[[package]]
|
|
907
907
|
name = "covalence-lib-cli"
|
|
908
|
-
version = "0.1.0-dev.
|
|
908
|
+
version = "0.1.0-dev.5"
|
|
909
909
|
dependencies = [
|
|
910
910
|
"clap",
|
|
911
911
|
]
|
|
912
912
|
|
|
913
913
|
[[package]]
|
|
914
914
|
name = "covalence-lib-dasl"
|
|
915
|
-
version = "0.1.0-dev.
|
|
915
|
+
version = "0.1.0-dev.5"
|
|
916
916
|
dependencies = [
|
|
917
917
|
"blake3",
|
|
918
918
|
"covalence-lib-cid",
|
|
@@ -927,7 +927,7 @@ dependencies = [
|
|
|
927
927
|
|
|
928
928
|
[[package]]
|
|
929
929
|
name = "covalence-lib-data"
|
|
930
|
-
version = "0.1.0-dev.
|
|
930
|
+
version = "0.1.0-dev.5"
|
|
931
931
|
dependencies = [
|
|
932
932
|
"arrayvec",
|
|
933
933
|
"bytes",
|
|
@@ -939,7 +939,7 @@ dependencies = [
|
|
|
939
939
|
|
|
940
940
|
[[package]]
|
|
941
941
|
name = "covalence-lib-error"
|
|
942
|
-
version = "0.1.0-dev.
|
|
942
|
+
version = "0.1.0-dev.5"
|
|
943
943
|
dependencies = [
|
|
944
944
|
"miette",
|
|
945
945
|
"snafu",
|
|
@@ -947,14 +947,14 @@ dependencies = [
|
|
|
947
947
|
|
|
948
948
|
[[package]]
|
|
949
949
|
name = "covalence-lib-python"
|
|
950
|
-
version = "0.1.0-dev.
|
|
950
|
+
version = "0.1.0-dev.5"
|
|
951
951
|
dependencies = [
|
|
952
952
|
"pyo3",
|
|
953
953
|
]
|
|
954
954
|
|
|
955
955
|
[[package]]
|
|
956
956
|
name = "covalence-lib-rand"
|
|
957
|
-
version = "0.1.0-dev.
|
|
957
|
+
version = "0.1.0-dev.5"
|
|
958
958
|
dependencies = [
|
|
959
959
|
"getrandom 0.3.4",
|
|
960
960
|
"rand",
|
|
@@ -962,21 +962,21 @@ dependencies = [
|
|
|
962
962
|
|
|
963
963
|
[[package]]
|
|
964
964
|
name = "covalence-lib-serde"
|
|
965
|
-
version = "0.1.0-dev.
|
|
965
|
+
version = "0.1.0-dev.5"
|
|
966
966
|
dependencies = [
|
|
967
967
|
"serde",
|
|
968
968
|
]
|
|
969
969
|
|
|
970
970
|
[[package]]
|
|
971
971
|
name = "covalence-lib-sqlite"
|
|
972
|
-
version = "0.1.0-dev.
|
|
972
|
+
version = "0.1.0-dev.5"
|
|
973
973
|
dependencies = [
|
|
974
974
|
"rusqlite",
|
|
975
975
|
]
|
|
976
976
|
|
|
977
977
|
[[package]]
|
|
978
978
|
name = "covalence-python"
|
|
979
|
-
version = "0.1.0-dev.
|
|
979
|
+
version = "0.1.0-dev.5"
|
|
980
980
|
dependencies = [
|
|
981
981
|
"blake3",
|
|
982
982
|
"covalence-blob-trait",
|
|
@@ -997,7 +997,7 @@ dependencies = [
|
|
|
997
997
|
|
|
998
998
|
[[package]]
|
|
999
999
|
name = "covalence-tools-glu"
|
|
1000
|
-
version = "0.1.0-dev.
|
|
1000
|
+
version = "0.1.0-dev.5"
|
|
1001
1001
|
dependencies = [
|
|
1002
1002
|
"covalence-lib-cli",
|
|
1003
1003
|
"covalence-lib-error",
|
|
@@ -3,7 +3,7 @@ members = ["crates/lib/error", "crates/lib/serde", "crates/lib/data", "crates/li
|
|
|
3
3
|
resolver = "3"
|
|
4
4
|
|
|
5
5
|
[workspace.package]
|
|
6
|
-
version = "0.1.0-dev.
|
|
6
|
+
version = "0.1.0-dev.5"
|
|
7
7
|
edition = "2024"
|
|
8
8
|
rust-version = "1.98"
|
|
9
9
|
license = "CC0-1.0 OR MIT OR Apache-2.0"
|
|
@@ -17,22 +17,22 @@ miette = "7.6.0"
|
|
|
17
17
|
sha2 = "0.11.0"
|
|
18
18
|
snafu = "0.9.2"
|
|
19
19
|
# Published crates, at the workspace version so the set moves together.
|
|
20
|
-
covalence-blob-trait = { path = "crates/blob/trait", version = "=0.1.0-dev.
|
|
21
|
-
covalence-cas-kv = { path = "crates/cas/kv", version = "=0.1.0-dev.
|
|
22
|
-
covalence-cas-memory = { path = "crates/cas/memory", version = "=0.1.0-dev.
|
|
23
|
-
covalence-cas-sqlite = { path = "crates/cas/sqlite", version = "=0.1.0-dev.
|
|
24
|
-
covalence-cas-trait = { path = "crates/cas/trait", version = "=0.1.0-dev.
|
|
25
|
-
covalence-kv-memory = { path = "crates/kv/memory", version = "=0.1.0-dev.
|
|
26
|
-
covalence-kv-s3 = { path = "crates/kv/s3", version = "=0.1.0-dev.
|
|
27
|
-
covalence-kv-trait = { path = "crates/kv/trait", version = "=0.1.0-dev.
|
|
28
|
-
covalence-lib-cid = { path = "crates/lib/cid", version = "=0.1.0-dev.
|
|
29
|
-
covalence-lib-cli = { path = "crates/lib/cli", version = "=0.1.0-dev.
|
|
30
|
-
covalence-lib-dasl = { path = "crates/lib/dasl", version = "=0.1.0-dev.
|
|
31
|
-
covalence-lib-data = { path = "crates/lib/data", version = "=0.1.0-dev.
|
|
32
|
-
covalence-lib-error = { path = "crates/lib/error", version = "=0.1.0-dev.
|
|
33
|
-
covalence-lib-rand = { path = "crates/lib/rand", version = "=0.1.0-dev.
|
|
34
|
-
covalence-lib-serde = { path = "crates/lib/serde", version = "=0.1.0-dev.
|
|
35
|
-
covalence-lib-sqlite = { path = "crates/lib/sqlite", version = "=0.1.0-dev.
|
|
20
|
+
covalence-blob-trait = { path = "crates/blob/trait", version = "=0.1.0-dev.5" }
|
|
21
|
+
covalence-cas-kv = { path = "crates/cas/kv", version = "=0.1.0-dev.5" }
|
|
22
|
+
covalence-cas-memory = { path = "crates/cas/memory", version = "=0.1.0-dev.5" }
|
|
23
|
+
covalence-cas-sqlite = { path = "crates/cas/sqlite", version = "=0.1.0-dev.5" }
|
|
24
|
+
covalence-cas-trait = { path = "crates/cas/trait", version = "=0.1.0-dev.5" }
|
|
25
|
+
covalence-kv-memory = { path = "crates/kv/memory", version = "=0.1.0-dev.5" }
|
|
26
|
+
covalence-kv-s3 = { path = "crates/kv/s3", version = "=0.1.0-dev.5" }
|
|
27
|
+
covalence-kv-trait = { path = "crates/kv/trait", version = "=0.1.0-dev.5" }
|
|
28
|
+
covalence-lib-cid = { path = "crates/lib/cid", version = "=0.1.0-dev.5" }
|
|
29
|
+
covalence-lib-cli = { path = "crates/lib/cli", version = "=0.1.0-dev.5" }
|
|
30
|
+
covalence-lib-dasl = { path = "crates/lib/dasl", version = "=0.1.0-dev.5" }
|
|
31
|
+
covalence-lib-data = { path = "crates/lib/data", version = "=0.1.0-dev.5" }
|
|
32
|
+
covalence-lib-error = { path = "crates/lib/error", version = "=0.1.0-dev.5" }
|
|
33
|
+
covalence-lib-rand = { path = "crates/lib/rand", version = "=0.1.0-dev.5" }
|
|
34
|
+
covalence-lib-serde = { path = "crates/lib/serde", version = "=0.1.0-dev.5" }
|
|
35
|
+
covalence-lib-sqlite = { path = "crates/lib/sqlite", version = "=0.1.0-dev.5" }
|
|
36
36
|
|
|
37
37
|
[workspace.lints.rust]
|
|
38
38
|
unsafe_code = "forbid"
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: covalence
|
|
3
|
-
Version: 0.1.0.
|
|
3
|
+
Version: 0.1.0.dev5
|
|
4
4
|
License-File: LICENSE-APACHE
|
|
5
5
|
License-File: LICENSE-CC0
|
|
6
6
|
License-File: LICENSE-MIT
|
|
@@ -35,7 +35,7 @@ True
|
|
|
35
35
|
## Install
|
|
36
36
|
|
|
37
37
|
```sh
|
|
38
|
-
pip install covalence==0.1.0.
|
|
38
|
+
pip install covalence==0.1.0.dev5
|
|
39
39
|
```
|
|
40
40
|
|
|
41
41
|
- Pre-releases need an exact pin like this one, or `pip install --pre`.
|
|
@@ -72,6 +72,11 @@ pip install covalence==0.1.0.dev3
|
|
|
72
72
|
- `Cid.of` hashes with `"blake3"` and `"raw"` by default.
|
|
73
73
|
- `Cid.of_file(path)` is the raw BLAKE3 `Cid` of a file, as `Cas.put_file`
|
|
74
74
|
returns it, without storing it or reading the file into memory.
|
|
75
|
+
- `Cid.of_fileobj(file, max_bytes=..., expected_length=...)` hashes an
|
|
76
|
+
already-open, seekable binary file object once from offset zero, without
|
|
77
|
+
storing it or holding it in memory. It uses the same bounded reader as
|
|
78
|
+
`Cas.put_fileobj`, and leaves the caller's source open at EOF on success.
|
|
79
|
+
Keep that source unchanged and exclusively owned until it returns.
|
|
75
80
|
|
|
76
81
|
## Values
|
|
77
82
|
|
|
@@ -115,8 +120,23 @@ pip install covalence==0.1.0.dev3
|
|
|
115
120
|
- S3 keys must be UTF-8 with non-empty `/`-separated segments other than `.`
|
|
116
121
|
and `..`; other keys raise `StoreError`.
|
|
117
122
|
- On S3, values over 8 MiB go up as multipart uploads, and values come down
|
|
118
|
-
in ranged GETs of 8 MiB, 16 parts or ranges at a time.
|
|
119
|
-
|
|
123
|
+
in ranged GETs of 8 MiB, 16 parts or ranges at a time. Upload parts grow
|
|
124
|
+
when needed to stay within 10,000 parts. Covalence checks
|
|
125
|
+
[S3's limits](https://docs.aws.amazon.com/AmazonS3/latest/userguide/qfacts.html)
|
|
126
|
+
before upload: at most 10,000 parts of 5 GiB each, subject to the platform's
|
|
127
|
+
allocation limit.
|
|
128
|
+
- `Kv.s3(..., upload_buffer_bytes=n)` caps aggregate upload payload
|
|
129
|
+
reservations across calls sharing that store. An upload reserves the smaller
|
|
130
|
+
of its length and its part size times its worker count before allocating
|
|
131
|
+
parts, reading its upload source or making requests. Insufficient capacity
|
|
132
|
+
raises `StoreError` immediately; zero admits only empty uploads. The default
|
|
133
|
+
is no budget. Reservations last through retries and cleanup.
|
|
134
|
+
- This budget excludes caller-owned sources, read buffers, allocator overhead,
|
|
135
|
+
HTTP/TLS buffers and request metadata; it is not an RSS limit. Owned byte
|
|
136
|
+
values still reserve their planned payload even though their parts share the
|
|
137
|
+
original allocation. File helpers may hash their source before upload
|
|
138
|
+
admission. A larger legal object can need a larger budget because its parts
|
|
139
|
+
grow; rejection does not consume its upload source or create an upload.
|
|
120
140
|
|
|
121
141
|
## Content-addressed stores
|
|
122
142
|
|
|
@@ -174,10 +194,31 @@ pip install covalence==0.1.0.dev3
|
|
|
174
194
|
case `StoreError` reports that failure with the source as its `__cause__`.
|
|
175
195
|
- `Cas.get_file(cid, path)` writes the block to a file and returns its
|
|
176
196
|
length, or `None` if missing.
|
|
177
|
-
- `
|
|
178
|
-
|
|
179
|
-
|
|
180
|
-
|
|
197
|
+
- `Cas.put_fileobj(file, max_bytes=..., expected_length=...)` stores a
|
|
198
|
+
caller-owned, seekable binary file object. It reads from offset zero twice,
|
|
199
|
+
checks the complete source on both passes even when deduplicating, and
|
|
200
|
+
leaves it open at EOF on success. The caller must keep the source unchanged
|
|
201
|
+
and exclusively owned until it returns; the method creates no snapshot.
|
|
202
|
+
`max_bytes` caps the accepted source length; `expected_length`, if given,
|
|
203
|
+
must match it exactly. A length violation raises `ValueError` before a new
|
|
204
|
+
block is published. A pass may read one extra byte to check the end.
|
|
205
|
+
- `Cas.get_to(cid, writer, max_bytes=..., expected_length=...)` writes to a
|
|
206
|
+
caller-owned binary file object and returns the length, or `None` if missing.
|
|
207
|
+
It checks all bytes against the CID, caps the bytes the writer accepts, and
|
|
208
|
+
checks the optional exact length. It handles short writes and never seeks,
|
|
209
|
+
flushes or closes the writer. A failure may leave partial or unchecked bytes
|
|
210
|
+
there: write to private staging and publish only after a successful return.
|
|
211
|
+
- These limits govern source length and output bytes, not the memory a backend
|
|
212
|
+
uses before reading or writing. Python callback transfers are at most 1 MiB.
|
|
213
|
+
- `put_file`, `put_fileobj`, `put_from`, `get_file` and `get_to` stream on `Cas.kv` over `Kv.s3`,
|
|
214
|
+
holding up to 16 part payloads at a time, or as many as
|
|
215
|
+
`Kv.s3(..., concurrency=n)` says; blocks over 78 GiB go up in larger parts.
|
|
216
|
+
Upload parts start at 8 MiB; read ranges stay at 8 MiB. Set
|
|
217
|
+
`upload_buffer_bytes=` to enforce a shared upload payload budget.
|
|
218
|
+
Other stores, including `Cas.kv(Kv.memory())`, hold the block in memory.
|
|
219
|
+
- A create-only write can reuse a block without checking its stored bytes.
|
|
220
|
+
Returning a CID does not freshly verify an existing block: use a checked
|
|
221
|
+
read, such as `get_to`, when that guarantee is needed.
|
|
181
222
|
- `put_file` reads the file twice, to hash it and to store it, and stores the
|
|
182
223
|
bytes it hashed. If they changed in between it raises `IntegrityError`,
|
|
183
224
|
and if the file got shorter `OSError`, storing nothing either way. Bytes
|
|
@@ -188,6 +229,28 @@ pip install covalence==0.1.0.dev3
|
|
|
188
229
|
was. The new file has the permissions of the file it replaces, or those
|
|
189
230
|
the umask gives a new file.
|
|
190
231
|
|
|
232
|
+
Use an already-open file without exposing its path:
|
|
233
|
+
|
|
234
|
+
```pycon
|
|
235
|
+
>>> import io
|
|
236
|
+
>>> source = io.BytesIO(b"hello")
|
|
237
|
+
>>> cid = cas.put_fileobj(source, max_bytes=5, expected_length=5)
|
|
238
|
+
>>> sink = io.BytesIO()
|
|
239
|
+
>>> cas.get_to(cid, sink, max_bytes=5, expected_length=5)
|
|
240
|
+
5
|
|
241
|
+
>>> sink.getvalue()
|
|
242
|
+
b'hello'
|
|
243
|
+
>>> Cid.of_fileobj(sink, max_bytes=5, expected_length=5) == cid
|
|
244
|
+
True
|
|
245
|
+
|
|
246
|
+
```
|
|
247
|
+
|
|
248
|
+
An integer file descriptor can be wrapped with `os.fdopen(os.dup(fd), "wb",
|
|
249
|
+
buffering=0)` for downloads, or `"rb"` for uploads. Close that wrapper when
|
|
250
|
+
finished; the original descriptor remains open. Duplicates share a file
|
|
251
|
+
position, so the caller must also keep the original descriptor exclusively
|
|
252
|
+
owned during the operation.
|
|
253
|
+
|
|
191
254
|
## Blobs and facts
|
|
192
255
|
|
|
193
256
|
- `Blob` is bytes read whole or by range:
|
|
@@ -259,16 +322,16 @@ stub; this package writes its stub by hand and checks it against `__doc__`:
|
|
|
259
322
|
later) for manylinux_2_28 x86_64 and aarch64 and macOS arm64, and an sdist,
|
|
260
323
|
and runs `tests/` against each wheel. Pull requests and pushes keep them as
|
|
261
324
|
workflow artifacts. A release tag must match the workspace version, such as
|
|
262
|
-
`0.1.0-dev.
|
|
325
|
+
`0.1.0-dev.5`; the files carry its PEP 440 form, `0.1.0.dev5`:
|
|
263
326
|
|
|
264
327
|
- `v<version>` attaches them to that GitHub release. In the public
|
|
265
328
|
`hiloopai/covalence` repository it also publishes them to PyPI, and
|
|
266
329
|
`.github/workflows/crates.yml` publishes the crates to crates.io.
|
|
267
330
|
- `python-v<version>` only attaches them to the GitHub release.
|
|
268
331
|
|
|
269
|
-
To publish `0.1.0-dev.
|
|
332
|
+
To publish `0.1.0-dev.5`:
|
|
270
333
|
|
|
271
334
|
```sh
|
|
272
|
-
git tag v0.1.0-dev.
|
|
335
|
+
git tag v0.1.0-dev.5 && git push origin v0.1.0-dev.5
|
|
273
336
|
```
|
|
274
337
|
|
|
@@ -21,7 +21,7 @@ True
|
|
|
21
21
|
## Install
|
|
22
22
|
|
|
23
23
|
```sh
|
|
24
|
-
pip install covalence==0.1.0.
|
|
24
|
+
pip install covalence==0.1.0.dev5
|
|
25
25
|
```
|
|
26
26
|
|
|
27
27
|
- Pre-releases need an exact pin like this one, or `pip install --pre`.
|
|
@@ -58,6 +58,11 @@ pip install covalence==0.1.0.dev3
|
|
|
58
58
|
- `Cid.of` hashes with `"blake3"` and `"raw"` by default.
|
|
59
59
|
- `Cid.of_file(path)` is the raw BLAKE3 `Cid` of a file, as `Cas.put_file`
|
|
60
60
|
returns it, without storing it or reading the file into memory.
|
|
61
|
+
- `Cid.of_fileobj(file, max_bytes=..., expected_length=...)` hashes an
|
|
62
|
+
already-open, seekable binary file object once from offset zero, without
|
|
63
|
+
storing it or holding it in memory. It uses the same bounded reader as
|
|
64
|
+
`Cas.put_fileobj`, and leaves the caller's source open at EOF on success.
|
|
65
|
+
Keep that source unchanged and exclusively owned until it returns.
|
|
61
66
|
|
|
62
67
|
## Values
|
|
63
68
|
|
|
@@ -101,8 +106,23 @@ pip install covalence==0.1.0.dev3
|
|
|
101
106
|
- S3 keys must be UTF-8 with non-empty `/`-separated segments other than `.`
|
|
102
107
|
and `..`; other keys raise `StoreError`.
|
|
103
108
|
- On S3, values over 8 MiB go up as multipart uploads, and values come down
|
|
104
|
-
in ranged GETs of 8 MiB, 16 parts or ranges at a time.
|
|
105
|
-
|
|
109
|
+
in ranged GETs of 8 MiB, 16 parts or ranges at a time. Upload parts grow
|
|
110
|
+
when needed to stay within 10,000 parts. Covalence checks
|
|
111
|
+
[S3's limits](https://docs.aws.amazon.com/AmazonS3/latest/userguide/qfacts.html)
|
|
112
|
+
before upload: at most 10,000 parts of 5 GiB each, subject to the platform's
|
|
113
|
+
allocation limit.
|
|
114
|
+
- `Kv.s3(..., upload_buffer_bytes=n)` caps aggregate upload payload
|
|
115
|
+
reservations across calls sharing that store. An upload reserves the smaller
|
|
116
|
+
of its length and its part size times its worker count before allocating
|
|
117
|
+
parts, reading its upload source or making requests. Insufficient capacity
|
|
118
|
+
raises `StoreError` immediately; zero admits only empty uploads. The default
|
|
119
|
+
is no budget. Reservations last through retries and cleanup.
|
|
120
|
+
- This budget excludes caller-owned sources, read buffers, allocator overhead,
|
|
121
|
+
HTTP/TLS buffers and request metadata; it is not an RSS limit. Owned byte
|
|
122
|
+
values still reserve their planned payload even though their parts share the
|
|
123
|
+
original allocation. File helpers may hash their source before upload
|
|
124
|
+
admission. A larger legal object can need a larger budget because its parts
|
|
125
|
+
grow; rejection does not consume its upload source or create an upload.
|
|
106
126
|
|
|
107
127
|
## Content-addressed stores
|
|
108
128
|
|
|
@@ -160,10 +180,31 @@ pip install covalence==0.1.0.dev3
|
|
|
160
180
|
case `StoreError` reports that failure with the source as its `__cause__`.
|
|
161
181
|
- `Cas.get_file(cid, path)` writes the block to a file and returns its
|
|
162
182
|
length, or `None` if missing.
|
|
163
|
-
- `
|
|
164
|
-
|
|
165
|
-
|
|
166
|
-
|
|
183
|
+
- `Cas.put_fileobj(file, max_bytes=..., expected_length=...)` stores a
|
|
184
|
+
caller-owned, seekable binary file object. It reads from offset zero twice,
|
|
185
|
+
checks the complete source on both passes even when deduplicating, and
|
|
186
|
+
leaves it open at EOF on success. The caller must keep the source unchanged
|
|
187
|
+
and exclusively owned until it returns; the method creates no snapshot.
|
|
188
|
+
`max_bytes` caps the accepted source length; `expected_length`, if given,
|
|
189
|
+
must match it exactly. A length violation raises `ValueError` before a new
|
|
190
|
+
block is published. A pass may read one extra byte to check the end.
|
|
191
|
+
- `Cas.get_to(cid, writer, max_bytes=..., expected_length=...)` writes to a
|
|
192
|
+
caller-owned binary file object and returns the length, or `None` if missing.
|
|
193
|
+
It checks all bytes against the CID, caps the bytes the writer accepts, and
|
|
194
|
+
checks the optional exact length. It handles short writes and never seeks,
|
|
195
|
+
flushes or closes the writer. A failure may leave partial or unchecked bytes
|
|
196
|
+
there: write to private staging and publish only after a successful return.
|
|
197
|
+
- These limits govern source length and output bytes, not the memory a backend
|
|
198
|
+
uses before reading or writing. Python callback transfers are at most 1 MiB.
|
|
199
|
+
- `put_file`, `put_fileobj`, `put_from`, `get_file` and `get_to` stream on `Cas.kv` over `Kv.s3`,
|
|
200
|
+
holding up to 16 part payloads at a time, or as many as
|
|
201
|
+
`Kv.s3(..., concurrency=n)` says; blocks over 78 GiB go up in larger parts.
|
|
202
|
+
Upload parts start at 8 MiB; read ranges stay at 8 MiB. Set
|
|
203
|
+
`upload_buffer_bytes=` to enforce a shared upload payload budget.
|
|
204
|
+
Other stores, including `Cas.kv(Kv.memory())`, hold the block in memory.
|
|
205
|
+
- A create-only write can reuse a block without checking its stored bytes.
|
|
206
|
+
Returning a CID does not freshly verify an existing block: use a checked
|
|
207
|
+
read, such as `get_to`, when that guarantee is needed.
|
|
167
208
|
- `put_file` reads the file twice, to hash it and to store it, and stores the
|
|
168
209
|
bytes it hashed. If they changed in between it raises `IntegrityError`,
|
|
169
210
|
and if the file got shorter `OSError`, storing nothing either way. Bytes
|
|
@@ -174,6 +215,28 @@ pip install covalence==0.1.0.dev3
|
|
|
174
215
|
was. The new file has the permissions of the file it replaces, or those
|
|
175
216
|
the umask gives a new file.
|
|
176
217
|
|
|
218
|
+
Use an already-open file without exposing its path:
|
|
219
|
+
|
|
220
|
+
```pycon
|
|
221
|
+
>>> import io
|
|
222
|
+
>>> source = io.BytesIO(b"hello")
|
|
223
|
+
>>> cid = cas.put_fileobj(source, max_bytes=5, expected_length=5)
|
|
224
|
+
>>> sink = io.BytesIO()
|
|
225
|
+
>>> cas.get_to(cid, sink, max_bytes=5, expected_length=5)
|
|
226
|
+
5
|
|
227
|
+
>>> sink.getvalue()
|
|
228
|
+
b'hello'
|
|
229
|
+
>>> Cid.of_fileobj(sink, max_bytes=5, expected_length=5) == cid
|
|
230
|
+
True
|
|
231
|
+
|
|
232
|
+
```
|
|
233
|
+
|
|
234
|
+
An integer file descriptor can be wrapped with `os.fdopen(os.dup(fd), "wb",
|
|
235
|
+
buffering=0)` for downloads, or `"rb"` for uploads. Close that wrapper when
|
|
236
|
+
finished; the original descriptor remains open. Duplicates share a file
|
|
237
|
+
position, so the caller must also keep the original descriptor exclusively
|
|
238
|
+
owned during the operation.
|
|
239
|
+
|
|
177
240
|
## Blobs and facts
|
|
178
241
|
|
|
179
242
|
- `Blob` is bytes read whole or by range:
|
|
@@ -245,15 +308,15 @@ stub; this package writes its stub by hand and checks it against `__doc__`:
|
|
|
245
308
|
later) for manylinux_2_28 x86_64 and aarch64 and macOS arm64, and an sdist,
|
|
246
309
|
and runs `tests/` against each wheel. Pull requests and pushes keep them as
|
|
247
310
|
workflow artifacts. A release tag must match the workspace version, such as
|
|
248
|
-
`0.1.0-dev.
|
|
311
|
+
`0.1.0-dev.5`; the files carry its PEP 440 form, `0.1.0.dev5`:
|
|
249
312
|
|
|
250
313
|
- `v<version>` attaches them to that GitHub release. In the public
|
|
251
314
|
`hiloopai/covalence` repository it also publishes them to PyPI, and
|
|
252
315
|
`.github/workflows/crates.yml` publishes the crates to crates.io.
|
|
253
316
|
- `python-v<version>` only attaches them to the GitHub release.
|
|
254
317
|
|
|
255
|
-
To publish `0.1.0-dev.
|
|
318
|
+
To publish `0.1.0-dev.5`:
|
|
256
319
|
|
|
257
320
|
```sh
|
|
258
|
-
git tag v0.1.0-dev.
|
|
321
|
+
git tag v0.1.0-dev.5 && git push origin v0.1.0-dev.5
|
|
259
322
|
```
|
|
@@ -30,17 +30,18 @@
|
|
|
30
30
|
mod read;
|
|
31
31
|
mod write;
|
|
32
32
|
|
|
33
|
+
pub use write::UploadPlan;
|
|
33
34
|
use write::Write;
|
|
34
35
|
|
|
35
36
|
use std::{
|
|
36
|
-
collections::BTreeMap,
|
|
37
|
+
collections::{BTreeMap, TryReserveError},
|
|
37
38
|
fmt,
|
|
38
39
|
io::{self, Read},
|
|
39
40
|
panic::{self, AssertUnwindSafe},
|
|
40
41
|
str::FromStr,
|
|
41
42
|
sync::{
|
|
42
43
|
Mutex, PoisonError,
|
|
43
|
-
atomic::{AtomicBool, Ordering},
|
|
44
|
+
atomic::{AtomicBool, AtomicU64, Ordering},
|
|
44
45
|
mpsc,
|
|
45
46
|
},
|
|
46
47
|
thread,
|
|
@@ -68,6 +69,34 @@ use rusty_s3::{Bucket, BucketError, UrlStyle};
|
|
|
68
69
|
#[snafu(crate_root(covalence_lib_error::snafu))]
|
|
69
70
|
#[non_exhaustive]
|
|
70
71
|
pub enum Error {
|
|
72
|
+
/// An upload exceeds an S3 limit or the platform's allocation limit.
|
|
73
|
+
#[snafu(display("invalid upload {field} {value}: expected {minimum}..={maximum}"))]
|
|
74
|
+
UploadLimit {
|
|
75
|
+
/// The quantity checked.
|
|
76
|
+
field: &'static str,
|
|
77
|
+
/// The requested quantity.
|
|
78
|
+
value: u64,
|
|
79
|
+
/// Its smallest supported value.
|
|
80
|
+
minimum: u64,
|
|
81
|
+
/// Its largest supported value.
|
|
82
|
+
maximum: u64,
|
|
83
|
+
},
|
|
84
|
+
/// The store's upload payload budget cannot admit another upload.
|
|
85
|
+
#[snafu(display("upload needs {required} buffer bytes; {available} available"))]
|
|
86
|
+
UploadBuffer {
|
|
87
|
+
/// The payload reservation required for this upload.
|
|
88
|
+
required: u64,
|
|
89
|
+
/// The unreserved bytes when admission failed.
|
|
90
|
+
available: u64,
|
|
91
|
+
},
|
|
92
|
+
/// Allocating an upload part failed after admission.
|
|
93
|
+
#[snafu(display("could not allocate {bytes} upload bytes: {source}"))]
|
|
94
|
+
UploadAllocation {
|
|
95
|
+
/// The part's size in bytes.
|
|
96
|
+
bytes: u64,
|
|
97
|
+
/// Why the allocator refused it.
|
|
98
|
+
source: TryReserveError,
|
|
99
|
+
},
|
|
71
100
|
/// The endpoint is not a URL.
|
|
72
101
|
#[snafu(display("invalid endpoint {endpoint}: {source}"))]
|
|
73
102
|
Endpoint {
|
|
@@ -254,10 +283,18 @@ const EXPIRY: Duration = Duration::from_secs(60);
|
|
|
254
283
|
const TIMEOUT: Duration = Duration::from_secs(30);
|
|
255
284
|
/// The slowest transfer, in bytes per second, a request attempt waits for.
|
|
256
285
|
const MIN_RATE: u64 = 1 << 20;
|
|
257
|
-
/// The
|
|
258
|
-
const
|
|
286
|
+
/// The body size above which request timeouts stop increasing.
|
|
287
|
+
const MAX_TIMEOUT_BYTES: u64 = 5 << 40;
|
|
259
288
|
/// The most parts a multipart upload may have.
|
|
260
289
|
const MAX_PARTS: u64 = 10_000;
|
|
290
|
+
/// S3's largest multipart part, in bytes.
|
|
291
|
+
const MAX_PART: u64 = 5 << 30;
|
|
292
|
+
/// S3's smallest non-final multipart part, in bytes.
|
|
293
|
+
const MIN_PART: u64 = 5 << 20;
|
|
294
|
+
/// S3's largest object: 10,000 parts of 5 GiB.
|
|
295
|
+
const MAX_OBJECT: u64 = MAX_PARTS * MAX_PART;
|
|
296
|
+
/// The largest body sent with one PUT, in bytes (5 GB).
|
|
297
|
+
const MAX_SINGLE_PUT: u64 = 5_000_000_000;
|
|
261
298
|
/// The default part size, 8 MiB: values larger are uploaded in parts of at
|
|
262
299
|
/// least this size, and values are read in ranges of it.
|
|
263
300
|
pub const PART: u64 = 8 << 20;
|
|
@@ -283,7 +320,7 @@ pub struct AwsOptions {
|
|
|
283
320
|
|
|
284
321
|
/// The limit on a request attempt moving `bytes` of body.
|
|
285
322
|
fn timeout(bytes: u64) -> Duration {
|
|
286
|
-
TIMEOUT + Duration::from_secs(bytes.min(
|
|
323
|
+
TIMEOUT + Duration::from_secs(bytes.min(MAX_TIMEOUT_BYTES) / MIN_RATE)
|
|
287
324
|
}
|
|
288
325
|
|
|
289
326
|
/// Run `future` to completion on `runtime`, on a thread of its own so that
|
|
@@ -400,9 +437,12 @@ impl Cached {
|
|
|
400
437
|
/// # Writes
|
|
401
438
|
///
|
|
402
439
|
/// - A value of up to the part size ([`PART`] unless set with
|
|
403
|
-
/// [`S3Kv::part_size`]) is written with one PUT.
|
|
440
|
+
/// [`S3Kv::part_size`]) and 5 GB is written with one PUT.
|
|
404
441
|
/// - A larger value is written with a multipart upload, in parts of the part
|
|
405
442
|
/// size, or larger parts if needed to keep to S3's limit of 10,000 parts.
|
|
443
|
+
/// - [`S3Kv::upload_plan`] checks S3 and allocation limits before any
|
|
444
|
+
/// upload request or source read. [`S3Kv::upload_buffer_bytes`] can bound
|
|
445
|
+
/// the aggregate upload payload held by calls sharing this store.
|
|
406
446
|
/// - Each PUT or part sends the SHA-256 of its body as
|
|
407
447
|
/// `x-amz-checksum-sha256`, which S3 checks against the bytes it receives.
|
|
408
448
|
/// - A failed multipart upload is aborted once its parts in flight finish.
|
|
@@ -458,6 +498,43 @@ pub struct S3Kv {
|
|
|
458
498
|
agent: ureq::Agent,
|
|
459
499
|
part: u64,
|
|
460
500
|
concurrency: usize,
|
|
501
|
+
upload_budget: Option<UploadBudget>,
|
|
502
|
+
}
|
|
503
|
+
|
|
504
|
+
/// Aggregate payload reservations for calls sharing one store.
|
|
505
|
+
struct UploadBudget {
|
|
506
|
+
limit: u64,
|
|
507
|
+
used: AtomicU64,
|
|
508
|
+
}
|
|
509
|
+
|
|
510
|
+
impl UploadBudget {
|
|
511
|
+
fn reserve(&self, required: u64) -> Result<UploadReservation<'_>, Error> {
|
|
512
|
+
self.used
|
|
513
|
+
.fetch_update(Ordering::Relaxed, Ordering::Relaxed, |used| {
|
|
514
|
+
used.checked_add(required)
|
|
515
|
+
.filter(|next| *next <= self.limit)
|
|
516
|
+
})
|
|
517
|
+
.map_err(|used| Error::UploadBuffer {
|
|
518
|
+
required,
|
|
519
|
+
available: self.limit - used,
|
|
520
|
+
})?;
|
|
521
|
+
Ok(UploadReservation {
|
|
522
|
+
budget: self,
|
|
523
|
+
bytes: required,
|
|
524
|
+
})
|
|
525
|
+
}
|
|
526
|
+
}
|
|
527
|
+
|
|
528
|
+
/// Held until all upload requests, including cleanup, have finished.
|
|
529
|
+
struct UploadReservation<'a> {
|
|
530
|
+
budget: &'a UploadBudget,
|
|
531
|
+
bytes: u64,
|
|
532
|
+
}
|
|
533
|
+
|
|
534
|
+
impl Drop for UploadReservation<'_> {
|
|
535
|
+
fn drop(&mut self) {
|
|
536
|
+
self.budget.used.fetch_sub(self.bytes, Ordering::Relaxed);
|
|
537
|
+
}
|
|
461
538
|
}
|
|
462
539
|
|
|
463
540
|
impl fmt::Debug for S3Kv {
|
|
@@ -498,6 +575,7 @@ impl S3Kv {
|
|
|
498
575
|
agent: agent(CONCURRENCY),
|
|
499
576
|
part: PART,
|
|
500
577
|
concurrency: CONCURRENCY,
|
|
578
|
+
upload_budget: None,
|
|
501
579
|
})
|
|
502
580
|
}
|
|
503
581
|
|
|
@@ -517,6 +595,28 @@ impl S3Kv {
|
|
|
517
595
|
self
|
|
518
596
|
}
|
|
519
597
|
|
|
598
|
+
/// Limit the aggregate upload payload reservation to `bytes`.
|
|
599
|
+
///
|
|
600
|
+
/// Calls sharing this store (including through `Arc`) share the budget.
|
|
601
|
+
/// An upload reserves [`UploadPlan::buffer_bytes`] before reading its
|
|
602
|
+
/// source, allocating parts or sending requests, and keeps the reservation
|
|
603
|
+
/// until all its requests and cleanup finish. Insufficient capacity fails
|
|
604
|
+
/// immediately with [`Error::UploadBuffer`]; it does not wait. Zero admits
|
|
605
|
+
/// only empty uploads. No budget is set by default.
|
|
606
|
+
///
|
|
607
|
+
/// This bounds upload part payloads, including parts held for retries.
|
|
608
|
+
/// Caller-owned sources, read buffers, allocator overhead, HTTP/TLS buffers
|
|
609
|
+
/// and request metadata are additional memory; this is not an RSS limit.
|
|
610
|
+
/// Already owned values still reserve their planned upload payload, even
|
|
611
|
+
/// though their parts share the caller's allocation.
|
|
612
|
+
pub fn upload_buffer_bytes(mut self, bytes: u64) -> Self {
|
|
613
|
+
self.upload_budget = Some(UploadBudget {
|
|
614
|
+
limit: bytes,
|
|
615
|
+
used: AtomicU64::new(0),
|
|
616
|
+
});
|
|
617
|
+
self
|
|
618
|
+
}
|
|
619
|
+
|
|
520
620
|
/// A store for `location`, configured as the AWS SDKs are.
|
|
521
621
|
///
|
|
522
622
|
/// - Profile: `options.profile`, else `AWS_PROFILE`, else `default`.
|