covalence 0.1.0.dev2__tar.gz → 0.1.0.dev3__tar.gz

{covalence-0.1.0.dev2 → covalence-0.1.0.dev3}/Cargo.lock

@@ -779,7 +779,7 @@ checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b"
 
 [[package]]
 name = "covalence-blob-trait"
-version = "0.1.0-dev.2"
+version = "0.1.0-dev.3"
 dependencies = [
  "covalence-lib-cid",
  "covalence-lib-data",
@@ -788,7 +788,7 @@ dependencies = [
 
 [[package]]
 name = "covalence-cas-kv"
-version = "0.1.0-dev.2"
+version = "0.1.0-dev.3"
 dependencies = [
  "blake3",
  "covalence-cas-trait",
@@ -810,7 +810,7 @@ dependencies = [
 
 [[package]]
 name = "covalence-cas-memory"
-version = "0.1.0-dev.2"
+version = "0.1.0-dev.3"
 dependencies = [
  "covalence-cas-trait",
  "covalence-lib-cid",
@@ -821,7 +821,7 @@ dependencies = [
 
 [[package]]
 name = "covalence-cas-sqlite"
-version = "0.1.0-dev.2"
+version = "0.1.0-dev.3"
 dependencies = [
  "covalence-cas-trait",
  "covalence-lib-cid",
@@ -836,7 +836,7 @@ dependencies = [
 
 [[package]]
 name = "covalence-cas-trait"
-version = "0.1.0-dev.2"
+version = "0.1.0-dev.3"
 dependencies = [
  "covalence-lib-cid",
  "covalence-lib-dasl",
@@ -847,7 +847,7 @@ dependencies = [
 
 [[package]]
 name = "covalence-kv-memory"
-version = "0.1.0-dev.2"
+version = "0.1.0-dev.3"
 dependencies = [
  "covalence-kv-trait",
  "covalence-lib-dasl",
@@ -857,7 +857,7 @@ dependencies = [
 
 [[package]]
 name = "covalence-kv-s3"
-version = "0.1.0-dev.2"
+version = "0.1.0-dev.3"
 dependencies = [
  "aws-config",
  "aws-credential-types",
@@ -881,7 +881,7 @@ dependencies = [
 
 [[package]]
 name = "covalence-kv-trait"
-version = "0.1.0-dev.2"
+version = "0.1.0-dev.3"
 dependencies = [
  "covalence-lib-dasl",
  "covalence-lib-data",
@@ -890,7 +890,7 @@ dependencies = [
 
 [[package]]
 name = "covalence-lib-cid"
-version = "0.1.0-dev.2"
+version = "0.1.0-dev.3"
 dependencies = [
  "blake3",
  "covalence-lib-data",
@@ -905,14 +905,14 @@ dependencies = [
 
 [[package]]
 name = "covalence-lib-cli"
-version = "0.1.0-dev.2"
+version = "0.1.0-dev.3"
 dependencies = [
  "clap",
 ]
 
 [[package]]
 name = "covalence-lib-dasl"
-version = "0.1.0-dev.2"
+version = "0.1.0-dev.3"
 dependencies = [
  "blake3",
  "covalence-lib-cid",
@@ -927,7 +927,7 @@ dependencies = [
 
 [[package]]
 name = "covalence-lib-data"
-version = "0.1.0-dev.2"
+version = "0.1.0-dev.3"
 dependencies = [
  "arrayvec",
  "bytes",
@@ -939,7 +939,7 @@ dependencies = [
 
 [[package]]
 name = "covalence-lib-error"
-version = "0.1.0-dev.2"
+version = "0.1.0-dev.3"
 dependencies = [
  "miette",
  "snafu",
@@ -947,14 +947,14 @@ dependencies = [
 
 [[package]]
 name = "covalence-lib-python"
-version = "0.1.0-dev.2"
+version = "0.1.0-dev.3"
 dependencies = [
  "pyo3",
 ]
 
 [[package]]
 name = "covalence-lib-rand"
-version = "0.1.0-dev.2"
+version = "0.1.0-dev.3"
 dependencies = [
  "getrandom 0.3.4",
  "rand",
@@ -962,21 +962,21 @@ dependencies = [
 
 [[package]]
 name = "covalence-lib-serde"
-version = "0.1.0-dev.2"
+version = "0.1.0-dev.3"
 dependencies = [
  "serde",
 ]
 
 [[package]]
 name = "covalence-lib-sqlite"
-version = "0.1.0-dev.2"
+version = "0.1.0-dev.3"
 dependencies = [
  "rusqlite",
 ]
 
 [[package]]
 name = "covalence-python"
-version = "0.1.0-dev.2"
+version = "0.1.0-dev.3"
 dependencies = [
  "blake3",
  "covalence-blob-trait",
@@ -997,7 +997,7 @@ dependencies = [
 
 [[package]]
 name = "covalence-tools-glu"
-version = "0.1.0-dev.2"
+version = "0.1.0-dev.3"
 dependencies = [
  "covalence-lib-cli",
  "covalence-lib-error",

{covalence-0.1.0.dev2 → covalence-0.1.0.dev3}/Cargo.toml

@@ -3,7 +3,7 @@ members = ["crates/lib/error", "crates/lib/serde", "crates/lib/data", "crates/li
 resolver = "3"
 
 [workspace.package]
-version = "0.1.0-dev.2"
+version = "0.1.0-dev.3"
 edition = "2024"
 rust-version = "1.98"
 license = "CC0-1.0 OR MIT OR Apache-2.0"
@@ -17,22 +17,22 @@ miette = "7.6.0"
 sha2 = "0.11.0"
 snafu = "0.9.2"
 # Published crates, at the workspace version so the set moves together.
-covalence-blob-trait = { path = "crates/blob/trait", version = "=0.1.0-dev.2" }
-covalence-cas-kv = { path = "crates/cas/kv", version = "=0.1.0-dev.2" }
-covalence-cas-memory = { path = "crates/cas/memory", version = "=0.1.0-dev.2" }
-covalence-cas-sqlite = { path = "crates/cas/sqlite", version = "=0.1.0-dev.2" }
-covalence-cas-trait = { path = "crates/cas/trait", version = "=0.1.0-dev.2" }
-covalence-kv-memory = { path = "crates/kv/memory", version = "=0.1.0-dev.2" }
-covalence-kv-s3 = { path = "crates/kv/s3", version = "=0.1.0-dev.2" }
-covalence-kv-trait = { path = "crates/kv/trait", version = "=0.1.0-dev.2" }
-covalence-lib-cid = { path = "crates/lib/cid", version = "=0.1.0-dev.2" }
-covalence-lib-cli = { path = "crates/lib/cli", version = "=0.1.0-dev.2" }
-covalence-lib-dasl = { path = "crates/lib/dasl", version = "=0.1.0-dev.2" }
-covalence-lib-data = { path = "crates/lib/data", version = "=0.1.0-dev.2" }
-covalence-lib-error = { path = "crates/lib/error", version = "=0.1.0-dev.2" }
-covalence-lib-rand = { path = "crates/lib/rand", version = "=0.1.0-dev.2" }
-covalence-lib-serde = { path = "crates/lib/serde", version = "=0.1.0-dev.2" }
-covalence-lib-sqlite = { path = "crates/lib/sqlite", version = "=0.1.0-dev.2" }
+covalence-blob-trait = { path = "crates/blob/trait", version = "=0.1.0-dev.3" }
+covalence-cas-kv = { path = "crates/cas/kv", version = "=0.1.0-dev.3" }
+covalence-cas-memory = { path = "crates/cas/memory", version = "=0.1.0-dev.3" }
+covalence-cas-sqlite = { path = "crates/cas/sqlite", version = "=0.1.0-dev.3" }
+covalence-cas-trait = { path = "crates/cas/trait", version = "=0.1.0-dev.3" }
+covalence-kv-memory = { path = "crates/kv/memory", version = "=0.1.0-dev.3" }
+covalence-kv-s3 = { path = "crates/kv/s3", version = "=0.1.0-dev.3" }
+covalence-kv-trait = { path = "crates/kv/trait", version = "=0.1.0-dev.3" }
+covalence-lib-cid = { path = "crates/lib/cid", version = "=0.1.0-dev.3" }
+covalence-lib-cli = { path = "crates/lib/cli", version = "=0.1.0-dev.3" }
+covalence-lib-dasl = { path = "crates/lib/dasl", version = "=0.1.0-dev.3" }
+covalence-lib-data = { path = "crates/lib/data", version = "=0.1.0-dev.3" }
+covalence-lib-error = { path = "crates/lib/error", version = "=0.1.0-dev.3" }
+covalence-lib-rand = { path = "crates/lib/rand", version = "=0.1.0-dev.3" }
+covalence-lib-serde = { path = "crates/lib/serde", version = "=0.1.0-dev.3" }
+covalence-lib-sqlite = { path = "crates/lib/sqlite", version = "=0.1.0-dev.3" }
 
 [workspace.lints.rust]
 unsafe_code = "forbid"

{covalence-0.1.0.dev2 → covalence-0.1.0.dev3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: covalence
-Version: 0.1.0.dev2
+Version: 0.1.0.dev3
 License-File: LICENSE-APACHE
 License-File: LICENSE-CC0
 License-File: LICENSE-MIT
@@ -35,7 +35,7 @@ True
 ## Install
 
 ```sh
-pip install covalence==0.1.0.dev2
+pip install covalence==0.1.0.dev3
 ```
 
 - Pre-releases need an exact pin like this one, or `pip install --pre`.
@@ -138,6 +138,9 @@ pip install covalence==0.1.0.dev2
     `prefix/blake3/<digest>/bytes`.
 - `put` returns the raw BLAKE3 `Cid`, after checking that the store kept the
   data under it.
+- `Cas.kv` never replaces a block its `Kv` already holds: on `Kv.s3` each
+  write is conditional (`If-None-Match: *`), so concurrent writers of one
+  block race safely and the first one's bytes stay.
 - `put_value(value)` stores the DRISL encoding of a `Value` (or what
   `Value(...)` takes) and returns its DRISL CID, which is `value.cid()`.
 - `get` returns the bytes and `get_fact` a `BlobFact`, both checked against
@@ -156,16 +159,25 @@ pip install covalence==0.1.0.dev2
   - Ranges are clipped at the end.
   - Stores without native ranges fetch the whole value and slice it.
 
-### Files
+### Files and streams
 
 - `Cas.put_file(path)` stores a file and returns its raw BLAKE3 `Cid`, as
   `put` does.
+- `Cas.put_from(cid, chunks, length)` stores a block from an iterable of
+  bytes, such as a request body, and returns its raw BLAKE3 `Cid`. Before a
+  new block is stored, its bytes must match `length` and the BLAKE3 `cid`,
+  and `chunks` must end there. A store may return an existing block without
+  consuming all the input; unread bytes and source errors are not checked.
+  While consuming input, a hash mismatch (`IntegrityError`), too few or too
+  many bytes (`ValueError`), or a source exception publish no new block.
+  Source exceptions propagate unchanged unless cleanup also fails, in which
+  case `StoreError` reports that failure with the source as its `__cause__`.
 - `Cas.get_file(cid, path)` writes the block to a file and returns its
   length, or `None` if missing.
-- `put_file` and `get_file` stream on `Cas.kv` over `Kv.s3`, holding up to
-  16 parts of 8 MiB (128 MiB) at a time; files over 78 GiB go up in larger
-  parts. Other stores, including `Cas.kv(Kv.memory())`, hold the block in
-  memory.
+- `put_file`, `put_from` and `get_file` stream on `Cas.kv` over `Kv.s3`,
+  holding up to 16 parts of 8 MiB (128 MiB) at a time, or as many as
+  `Kv.s3(..., concurrency=n)` says; blocks over 78 GiB go up in larger parts. Other stores, including `Cas.kv(Kv.memory())`, hold
+  the block in memory.
 - `put_file` reads the file twice, to hash it and to store it, and stores the
   bytes it hashed. If they changed in between it raises `IntegrityError`,
   and if the file got shorter `OSError`, storing nothing either way. Bytes
@@ -254,9 +266,9 @@ workflow artifacts. A release tag must match the workspace version, such as
   `.github/workflows/crates.yml` publishes the crates to crates.io.
 - `python-v<version>` only attaches them to the GitHub release.
 
-To publish `0.1.0-dev.2`:
+To publish `0.1.0-dev.3`:
 
 ```sh
-git tag v0.1.0-dev.2 && git push origin v0.1.0-dev.2
+git tag v0.1.0-dev.3 && git push origin v0.1.0-dev.3
 ```
 

{covalence-0.1.0.dev2 → covalence-0.1.0.dev3}/README.md

@@ -21,7 +21,7 @@ True
 ## Install
 
 ```sh
-pip install covalence==0.1.0.dev2
+pip install covalence==0.1.0.dev3
 ```
 
 - Pre-releases need an exact pin like this one, or `pip install --pre`.
@@ -124,6 +124,9 @@ pip install covalence==0.1.0.dev2
     `prefix/blake3/<digest>/bytes`.
 - `put` returns the raw BLAKE3 `Cid`, after checking that the store kept the
   data under it.
+- `Cas.kv` never replaces a block its `Kv` already holds: on `Kv.s3` each
+  write is conditional (`If-None-Match: *`), so concurrent writers of one
+  block race safely and the first one's bytes stay.
 - `put_value(value)` stores the DRISL encoding of a `Value` (or what
   `Value(...)` takes) and returns its DRISL CID, which is `value.cid()`.
 - `get` returns the bytes and `get_fact` a `BlobFact`, both checked against
@@ -142,16 +145,25 @@ pip install covalence==0.1.0.dev2
   - Ranges are clipped at the end.
   - Stores without native ranges fetch the whole value and slice it.
 
-### Files
+### Files and streams
 
 - `Cas.put_file(path)` stores a file and returns its raw BLAKE3 `Cid`, as
   `put` does.
+- `Cas.put_from(cid, chunks, length)` stores a block from an iterable of
+  bytes, such as a request body, and returns its raw BLAKE3 `Cid`. Before a
+  new block is stored, its bytes must match `length` and the BLAKE3 `cid`,
+  and `chunks` must end there. A store may return an existing block without
+  consuming all the input; unread bytes and source errors are not checked.
+  While consuming input, a hash mismatch (`IntegrityError`), too few or too
+  many bytes (`ValueError`), or a source exception publish no new block.
+  Source exceptions propagate unchanged unless cleanup also fails, in which
+  case `StoreError` reports that failure with the source as its `__cause__`.
 - `Cas.get_file(cid, path)` writes the block to a file and returns its
   length, or `None` if missing.
-- `put_file` and `get_file` stream on `Cas.kv` over `Kv.s3`, holding up to
-  16 parts of 8 MiB (128 MiB) at a time; files over 78 GiB go up in larger
-  parts. Other stores, including `Cas.kv(Kv.memory())`, hold the block in
-  memory.
+- `put_file`, `put_from` and `get_file` stream on `Cas.kv` over `Kv.s3`,
+  holding up to 16 parts of 8 MiB (128 MiB) at a time, or as many as
+  `Kv.s3(..., concurrency=n)` says; blocks over 78 GiB go up in larger parts. Other stores, including `Cas.kv(Kv.memory())`, hold
+  the block in memory.
 - `put_file` reads the file twice, to hash it and to store it, and stores the
   bytes it hashed. If they changed in between it raises `IntegrityError`,
   and if the file got shorter `OSError`, storing nothing either way. Bytes
@@ -240,8 +252,8 @@ workflow artifacts. A release tag must match the workspace version, such as
   `.github/workflows/crates.yml` publishes the crates to crates.io.
 - `python-v<version>` only attaches them to the GitHub release.
 
-To publish `0.1.0-dev.2`:
+To publish `0.1.0-dev.3`:
 
 ```sh
-git tag v0.1.0-dev.2 && git push origin v0.1.0-dev.2
+git tag v0.1.0-dev.3 && git push origin v0.1.0-dev.3
 ```

{covalence-0.1.0.dev2 → covalence-0.1.0.dev3}/crates/cas/kv/src/lib.rs

@@ -46,13 +46,16 @@ use data_encoding::BASE32_NOPAD;
 ///   `get_fact` and `aget_fact` check the bytes against the CID and fail with
 ///   [`Error::Check`] if they do not match. `get_unchecked`, sizes and ranges
 ///   are not checked.
-/// - `has` asks the key-value store whether the key exists (a HEAD request on
-///   S3), without reading the bytes.
-/// - `put` writes without asking first, so it needs only write access. (S3
-///   answers a HEAD for a missing key with 403 to a client that may not list
-///   the bucket.)
+/// - `has` asks the key-value store whether the key exists, without reading
+///   the bytes.
+/// - `put`, `aput` and `put_from` never replace a block: they store through
+///   the key-value store's [`Kv::put_new`], [`Kv::aput_new`] and
+///   [`Kv::put_new_from`], so concurrent writers of one block race safely.
+///   Writes fail if the backend does not implement atomic create-only writes.
 /// - `put_from` and `get_to` stream through the key-value store's own
-///   `put_from` and `get_to`, hashing the bytes as they pass.
+///   `put_new_from` and `get_to`, hashing the bytes as they pass.
+///   If the backend skips an existing block, `put_from` returns without
+///   consuming or checking any remaining input.
 /// - If the bytes `put_from` reads do not hash to the CID, the read that would
 ///   hand over the last byte fails instead. The key-value store's `put_from`
 ///   then fails as [`Kv::put_from`] says, storing nothing, and `put_from`
@@ -60,6 +63,18 @@ use data_encoding::BASE32_NOPAD;
 ///   store reports a further failure, such as failing to clean up.
 /// - [`Cas::info`] is `{"name": "kv", "prefix": <prefix>, "kv": <the
 ///   key-value store's info>}`; [`Cas::stats`] is the key-value store's.
+///
+/// # S3
+///
+/// Writes use conditional requests to preserve existing blocks. They do not
+/// make a preliminary HEAD request, so they need only write access.
+///
+/// `has` uses a HEAD request. S3 answers a HEAD for a missing key with 403 to
+/// a client that may not list the bucket.
+///
+/// S3 may consume the source before discovering that a block already exists.
+/// Once the backend skips insertion, `put_from` leaves any remaining input
+/// unread.
 pub struct KvCas {
     kv: Arc<dyn Kv>,
     prefix: String,
@@ -277,7 +292,7 @@ impl Put<Blake3> for KvCas {
     fn put(&self, bytes: Bytes) -> Result<Blake3, Error> {
         let cid = Blake3::of(&bytes);
         self.kv
-            .put(self.key(&cid).as_bytes(), bytes)
+            .put_new(self.key(&cid).as_bytes(), bytes)
             .map_err(backend)?;
         Ok(cid)
     }
@@ -294,7 +309,10 @@ impl AsyncPut for KvCas {
         Box::pin(async move {
             let cid = Blake3::of(&bytes);
             let key = self.key(&cid);
-            self.kv.aput(key.as_bytes(), bytes).await.map_err(backend)?;
+            self.kv
+                .aput_new(key.as_bytes(), bytes)
+                .await
+                .map_err(backend)?;
             Ok(cid.into())
         })
     }
@@ -334,8 +352,8 @@ impl Cas for KvCas {
             mismatch: false,
         };
         let key = self.key(&cid);
-        match self.kv.put_from(key.as_bytes(), &mut checked, len) {
-            Ok(()) => Ok(cid.into()),
+        match self.kv.put_new_from(key.as_bytes(), &mut checked, len) {
+            Ok(_) => Ok(cid.into()),
             // The store failed as asked; a store error beyond that, such as a
             // failed cleanup, is the one to report.
             Err(covalence_kv_trait::Error::Io { .. }) if checked.mismatch => {

{covalence-0.1.0.dev2 → covalence-0.1.0.dev3}/crates/cas/kv/tests/kv.rs

@@ -25,7 +25,7 @@ use tokio::sync::oneshot;
 const HELLO: &str = "blake3/5khrmpntq2bjexseshc6ldklwnig56gbj23yvbxjbdcwestheahq/bytes";
 
 /// Store "hello" and check reads, then `corrupt` it and check that checked
-/// reads notice, and that `put` writes the block again.
+/// reads notice, and that `put` leaves the block it finds: it never replaces.
 fn check(store: &KvCas, corrupt: impl FnOnce()) {
     let cid: Cid = store.put("hello".into()).unwrap();
     assert_eq!(cid, Blake3::of(b"hello").into());
@@ -59,7 +59,12 @@ fn check(store: &KvCas, corrupt: impl FnOnce()) {
     ));
     assert!(matches!(store.get_fact(&cid), Err(Error::Check { .. })));
     let _: Cid = store.put("hello".into()).unwrap();
-    assert_eq!(store.get(&cid).unwrap().unwrap(), "hello");
+    assert_eq!(store.get_unchecked(&cid).unwrap().unwrap(), "wrong");
+    let _ = block_on(store.aput("hello".into())).unwrap();
+    assert_eq!(store.get_unchecked(&cid).unwrap().unwrap(), "wrong");
+    let put_from = store.put_from(Blake3::of(b"hello"), &mut &b"hello"[..], 5);
+    assert_eq!(put_from.unwrap(), cid);
+    assert_eq!(store.get_unchecked(&cid).unwrap().unwrap(), "wrong");
 }
 
 #[test]
@@ -72,7 +77,7 @@ fn memory() {
 }
 
 /// A key-value store that cannot read values, only whether they exist.
-struct Blind(MemoryKv);
+struct Blind(MemoryKv, usize);
 
 impl Kv for Blind {
     fn get(&self, _: &[u8]) -> Result<Option<Bytes>, covalence_kv_trait::Error> {
@@ -90,13 +95,61 @@ impl Kv for Blind {
     fn exists(&self, key: &[u8]) -> Result<bool, covalence_kv_trait::Error> {
         self.0.exists(key)
     }
+    fn put_new(&self, key: &[u8], value: Bytes) -> Result<bool, covalence_kv_trait::Error> {
+        self.0.put_new(key, value)
+    }
+    fn aput_new<'a>(&'a self, key: &'a [u8], value: Bytes) -> KvFuture<'a, bool> {
+        self.0.aput_new(key, value)
+    }
+    fn put_new_from(
+        &self,
+        key: &[u8],
+        reader: &mut dyn std::io::Read,
+        len: u64,
+    ) -> Result<bool, covalence_kv_trait::Error> {
+        if self.0.exists(key)? {
+            reader
+                .read_exact(&mut vec![0; self.1])
+                .map_err(|source| covalence_kv_trait::Error::Io { source })?;
+            return Ok(false);
+        }
+        self.0.put_new_from(key, reader, len)
+    }
+}
+
+/// An existing block may succeed with invalid or failing input left unread.
+#[test]
+fn existing_blocks_leave_unused_stream_bytes_unchecked() {
+    for consumed in [0, 2] {
+        // A backend may skip immediately or after reading part of the source.
+        let store = KvCas::new(Blind(MemoryKv::default(), consumed).into_dyn());
+        let cid: Cid = store.put("hello".into()).unwrap();
+        let claim = Blake3::of(b"hello");
+        for data in [&b"wrong"[..], &b"bad"[..]] {
+            let mut source = std::io::Cursor::new(data);
+            assert_eq!(store.put_from(claim, &mut source, 5).unwrap(), cid);
+            assert_eq!(source.position(), consumed as u64);
+        }
+        // An error beyond the consumed prefix must not be observed either.
+        let mut source = std::io::Read::chain(std::io::Cursor::new(b"he"), FailingReader);
+        assert_eq!(store.put_from(claim, &mut source, 5).unwrap(), cid);
+        assert_eq!(source.get_ref().0.position(), consumed as u64);
+    }
+}
+
+struct FailingReader;
+
+impl std::io::Read for FailingReader {
+    fn read(&mut self, _: &mut [u8]) -> std::io::Result<usize> {
+        Err(std::io::Error::other("source failed"))
+    }
 }
 
 /// `has` and `missing` ask whether keys exist rather than reading values,
 /// through `Arc<dyn Cas>` too.
 #[test]
 fn existence_without_reads() {
-    let store = KvCas::new(Blind(MemoryKv::default()).into_dyn()).into_dyn();
+    let store = KvCas::new(Blind(MemoryKv::default(), 0).into_dyn()).into_dyn();
     let cid = store.put("hello".into()).unwrap();
     assert_eq!(store.put("hello".into()).unwrap(), cid);
     let missing: Cid = Blake3::of(b"missing").into();
@@ -352,12 +405,12 @@ impl Kv for Leaky {
     fn put(&self, _: &[u8], _: Bytes) -> Result<(), covalence_kv_trait::Error> {
         Ok(())
     }
-    fn put_from(
+    fn put_new_from(
         &self,
         _: &[u8],
         reader: &mut dyn std::io::Read,
         _: u64,
-    ) -> Result<(), covalence_kv_trait::Error> {
+    ) -> Result<bool, covalence_kv_trait::Error> {
         let _ = reader.read_to_end(&mut Vec::new());
         Err(covalence_kv_trait::Error::Backend {
             source: "abort failed".into(),

{covalence-0.1.0.dev2 → covalence-0.1.0.dev3}/crates/cas/trait/src/lib.rs

@@ -350,10 +350,14 @@ pub trait Cas: Get + Put + AsyncGet + AsyncPut {
     }
 
     /// Store the next `len` bytes of `reader`, whose raw BLAKE3 CID is `cid`,
-    /// and return the key [`Put::put_fact`] would. Reads no further; stores
-    /// nothing and fails with [`Error::Check`] if the bytes have another CID,
-    /// or with [`Error::Io`] if `reader` fails or ends early. Reads them into
-    /// memory by default; stores that can stream override it.
+    /// and return the key [`Put::put_fact`] would.
+    ///
+    /// - A store may return an existing block without consuming all of
+    ///   `reader`; unread bytes are not checked.
+    /// - Otherwise, reads exactly `len` bytes and no further. Stores nothing
+    ///   and fails with [`Error::Check`] if the bytes have another CID, or
+    ///   with [`Error::Io`] if `reader` fails or ends early.
+    /// - Reads into memory by default; stores that can stream override it.
     fn put_from(&self, cid: Blake3, reader: &mut dyn Read, len: u64) -> Result<Cid, Error> {
         let mut bytes = Vec::new();
         let read = reader

{covalence-0.1.0.dev2 → covalence-0.1.0.dev3}/crates/kv/memory/src/lib.rs

@@ -17,13 +17,17 @@
 //! ```
 #![deny(missing_docs)]
 
-use std::{collections::BTreeMap, sync::RwLock};
+use std::{
+    collections::BTreeMap,
+    io::{self, Read},
+    sync::RwLock,
+};
 
 use covalence_kv_trait::{Error, Kv, KvFuture, ready};
 use covalence_lib_dasl::Value;
 use covalence_lib_data::Bytes;
 
-/// An in-memory store. It accepts every key.
+/// An in-memory store. It accepts every key, and [`Kv::put_new`] is atomic.
 ///
 /// [`Kv::info`] is `{"name": "memory"}`, and [`Kv::stats`] counts `keys` and
 /// the `bytes` of their values.
@@ -48,6 +52,35 @@ impl Kv for MemoryKv {
         ready(self.put(key, value))
     }
 
+    fn put_new(&self, key: &[u8], value: Bytes) -> Result<bool, Error> {
+        let mut map = self.0.write().unwrap();
+        if map.contains_key(key) {
+            return Ok(false);
+        }
+        map.insert(key.into(), value);
+        Ok(true)
+    }
+
+    /// Reads the value into memory first, so the store is locked only to
+    /// insert it.
+    fn put_new_from(&self, key: &[u8], reader: &mut dyn Read, len: u64) -> Result<bool, Error> {
+        let mut value = Vec::new();
+        let read = reader
+            .take(len)
+            .read_to_end(&mut value)
+            .map_err(|source| Error::Io { source })?;
+        if (read as u64) < len {
+            return Err(Error::Io {
+                source: io::ErrorKind::UnexpectedEof.into(),
+            });
+        }
+        self.put_new(key, value.into())
+    }
+
+    fn aput_new<'a>(&'a self, key: &'a [u8], value: Bytes) -> KvFuture<'a, bool> {
+        ready(self.put_new(key, value))
+    }
+
     fn exists(&self, key: &[u8]) -> Result<bool, Error> {
         Ok(self.0.read().unwrap().contains_key(key))
     }

{covalence-0.1.0.dev2 → covalence-0.1.0.dev3}/crates/kv/memory/tests/memory.rs

@@ -27,6 +27,36 @@ fn put_and_get() -> Result<(), Error> {
     Ok(())
 }
 
+/// A new value goes only under a free key, also through `Arc<dyn Kv>`, and
+/// racing writers of one key store once.
+#[test]
+fn new_values_never_replace() -> Result<(), Error> {
+    let store = store();
+    assert!(store.put_new(b"a", "one".into())?);
+    assert!(!store.put_new(b"a", "two".into())?);
+    assert!(!block_on(store.aput_new(b"a", "two".into()))?);
+    assert!(!store.put_new_from(b"a", &mut &b"two"[..], 3)?);
+    assert_eq!(store.get(b"a")?.unwrap(), "one");
+    assert!(store.put_new_from(b"b", &mut &b"two!"[..], 3)?);
+    assert_eq!(store.get(b"b")?.unwrap(), "two");
+    let error = store.put_new_from(b"c", &mut &b"ab"[..], 3).unwrap_err();
+    assert!(matches!(error, Error::Io { .. }), "{error}");
+    assert!(!store.exists(b"c")?);
+    let stored = std::thread::scope(|scope| {
+        let store = &store;
+        let racers: Vec<_> = (0..8u8)
+            .map(|i| scope.spawn(move || store.put_new(b"race", vec![i].into()).unwrap()))
+            .collect();
+        racers
+            .into_iter()
+            .map(|racer| racer.join().unwrap())
+            .filter(|&stored| stored)
+            .count()
+    });
+    assert_eq!(stored, 1);
+    Ok(())
+}
+
 #[test]
 fn async_matches_sync() -> Result<(), Error> {
     let store = store();