commitkv 1.0.1__tar.gz → 1.1.0__tar.gz

{commitkv-1.0.1 → commitkv-1.1.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: commitkv
-Version: 1.0.1
+Version: 1.1.0
 License-File: LICENSE
 Summary: Compressed key-value store for git-managed caches: append-only segments that never exceed a size limit
 License-Expression: Apache-2.0
@@ -80,6 +80,14 @@ store.refresh(); // pick up what other stores wrote to the directory
   writing (the first `put`, and the first after a failed one); records that other stores write
   stay invisible until one of those happens. Call `refresh` after
   replacing segments under an open store, for example by checking out another branch.
+- Stores that write to a directory at the same time each append to a segment of their own, and
+  the next run continues at most one of them. Open one store per directory in a process and share
+  it; a store per use leaves a small segment behind on every run, and a small segment
+  compresses worse because its values have few earlier ones to refer to.
+- `keys` lists the keys in the order their current values lie in the segments, oldest segment
+  first. One store's writes are listed in the order it made them; the writes of several stores
+  are grouped by segment, so their order does not tell which was made first. Read every value
+  in that order: a scattered order decodes the surrounding values again for most of them.
 - Among records of a key that stores wrote without seeing each other's, which one wins is
   unspecified, but every reader picks the same one.
 - `max_segment_bytes` / `maxSegmentBytes` must be between 1,024 and 100,000,000; the upper

{commitkv-1.0.1 → commitkv-1.1.0}/README.md

@@ -69,6 +69,14 @@ store.refresh(); // pick up what other stores wrote to the directory
   writing (the first `put`, and the first after a failed one); records that other stores write
   stay invisible until one of those happens. Call `refresh` after
   replacing segments under an open store, for example by checking out another branch.
+- Stores that write to a directory at the same time each append to a segment of their own, and
+  the next run continues at most one of them. Open one store per directory in a process and share
+  it; a store per use leaves a small segment behind on every run, and a small segment
+  compresses worse because its values have few earlier ones to refer to.
+- `keys` lists the keys in the order their current values lie in the segments, oldest segment
+  first. One store's writes are listed in the order it made them; the writes of several stores
+  are grouped by segment, so their order does not tell which was made first. Read every value
+  in that order: a scattered order decodes the surrounding values again for most of them.
 - Among records of a key that stores wrote without seeing each other's, which one wins is
   unspecified, but every reader picks the same one.
 - `max_segment_bytes` / `maxSegmentBytes` must be between 1,024 and 100,000,000; the upper

{commitkv-1.0.1 → commitkv-1.1.0}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "commitkv"
-version = "1.0.1"
+version = "1.1.0"
 description = "Compressed key-value store for git-managed caches: append-only segments that never exceed a size limit"
 readme = "README.md"
 requires-python = ">=3.10"

{commitkv-1.0.1 → commitkv-1.1.0}/python/commitkv/_native.pyi

@@ -25,6 +25,9 @@ class Store:
 
     def __contains__(self, key: str | bytes) -> bool: ...
     def __len__(self) -> int: ...
-    def keys(self) -> list[bytes]: ...
+    def keys(self) -> list[bytes]:
+        """Lists the keys in the order their current values lie in the segments, oldest segment
+        first, which is the order one store wrote them; reading every value is fastest in this
+        order."""
     def refresh(self) -> None:
         """Picks up records that other stores wrote to the directory."""

{commitkv-1.0.1 → commitkv-1.1.0}/rust/crates/commitkv/src/store.rs

@@ -214,9 +214,14 @@ impl Store {
         self.len() == 0
     }
 
+    /// The keys in the order their current records lie in the segments, oldest segment first,
+    /// so reading every value in this order decodes each group once. This is the order one
+    /// store wrote them; stores that write at the same time fill segments of their own.
     pub fn keys(&self) -> Vec<Vec<u8>> {
         let state = self.read_state();
-        state.index.keys().map(|key| key.to_vec()).collect()
+        let mut entries: Vec<_> = state.index.iter().collect();
+        entries.sort_unstable_by_key(|(_, location)| state.position(location));
+        entries.into_iter().map(|(key, _)| key.to_vec()).collect()
     }
 
     /// Picks up records that other stores wrote to the directory since the last call, and
@@ -611,6 +616,12 @@ impl State {
         index
     }
 
+    /// Orders records by age: segment names order by creation time and offsets order within a
+    /// segment.
+    fn position(&self, location: &Location) -> (&str, u64) {
+        (&self.segments[location.segment].name, location.end)
+    }
+
     /// Adds a record that follows the scanned part of its segment; `false` when the record
     /// cannot be placed, which leaves the state unchanged.
     fn apply(&mut self, segment_index: usize, record: &Record) -> bool {
@@ -648,12 +659,11 @@ impl State {
         segment.scanned = location.end;
         segment.group_raw = group_raw_end;
         segment.raw_total = raw_total;
-        // The newest record of a key wins: segment names order by creation time and offsets
-        // order within a segment.
-        let newer = self.index.get(record.key).is_none_or(|current| {
-            (&self.segments[current.segment].name, current.end)
-                < (&self.segments[segment_index].name, location.end)
-        });
+        // The newest record of a key wins.
+        let newer = self
+            .index
+            .get(record.key)
+            .is_none_or(|current| self.position(current) < self.position(&location));
         if newer {
             self.index.insert(record.key.into(), location);
         }

{commitkv-1.0.1 → commitkv-1.1.0}/rust/crates/commitkv/tests/store.rs

@@ -55,6 +55,21 @@ fn values_survive_reopening_and_the_latest_value_of_a_key_wins() {
     assert!(keys.binary_search(&b"large".to_vec()).is_ok());
 }
 
+#[test]
+fn keys_are_listed_in_the_order_a_store_wrote_their_current_values() {
+    let dir = tempfile::tempdir().unwrap();
+    {
+        let store = Store::open(dir.path(), small_segments()).unwrap();
+        for i in 0..400 {
+            store.put(&key(i), &noise(i as u64, 1000)).unwrap();
+        }
+        store.put(&key(3), b"replaced").unwrap();
+    }
+    assert!(segment_paths(dir.path()).len() > 1);
+    let expected: Vec<Vec<u8>> = (0..400).filter(|&i| i != 3).chain([3]).map(key).collect();
+    assert_eq!(open(dir.path()).keys(), expected);
+}
+
 #[test]
 fn no_segment_exceeds_the_limit() {
     let dir = tempfile::tempdir().unwrap();