kvgit 0.4.0__tar.gz → 0.4.1__tar.gz

{kvgit-0.4.0 → kvgit-0.4.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: kvgit
-Version: 0.4.0
+Version: 0.4.1
 Summary: Versioned key-value store with git-like commit, branch, and merge semantics.
 Author: ashenfad
 License: MIT

{kvgit-0.4.0 → kvgit-0.4.1}/kvgit/hamt.py

@@ -108,9 +108,10 @@ class Hamt:
         self.prefix = prefix
         self.bucket_max = bucket_max
         self.pending = pending if pending is not None else {}
-        # Stored nodes already read for an update in progress, keyed like
-        # ``pending``. Filled only on the private view ``updated`` works
-        # through, so a shared ``Hamt`` never has two updates writing it.
+        # Stored nodes already read, keyed like ``pending``: by an update
+        # in progress, on the private view ``updated`` works through, and
+        # by ``get_many``. A node never changes once stored, so nothing
+        # here goes stale.
         self._fetched: dict[str, bytes] = {}
 
     # ---- internal helpers ----
@@ -179,6 +180,23 @@ class Hamt:
             node_hash = node["children"][chunk]
             depth += 1
 
+    def get_many(self, keys: Iterable[str]) -> dict[str, bytes]:
+        """The values of the keys that are present, as ``{key: value}``.
+
+        Reads the paths to all of them level by level, one batched read
+        per level, rather than walking the tree once per key. The nodes
+        read stay cached on this view, so later lookups through it reuse
+        them.
+        """
+        keys = list(dict.fromkeys(keys))
+        self._prefetch_paths(keys, [])
+        found: dict[str, bytes] = {}
+        for key in keys:
+            value = self.get(key)
+            if value is not None:
+                found[key] = value
+        return found
+
     def __contains__(self, key: str) -> bool:
         return self.get(key) is not None
 

{kvgit-0.4.0 → kvgit-0.4.1}/kvgit/repo.py

@@ -305,14 +305,6 @@ class Repo:
             raise UnknownCommitError(f"Commit '{target}' does not exist")
         return target
 
-    def _pointers(self, commit: str) -> dict[str, str]:
-        """A commit's keyset as key -> blob pointer."""
-        root_raw = self._store.get(COMMIT_ROOT % commit)
-        if root_raw is None:
-            raise UnknownCommitError(f"Commit '{commit}' does not exist")
-        entries = Keyset(self._store, root=loads(root_raw)).materialize()
-        return {key: entry.blob for key, entry in entries.items()}
-
 
 class Snapshot(Mapping[str, Any]):
     """The state of the repository at one commit, read-only.
@@ -324,23 +316,59 @@ class Snapshot(Mapping[str, Any]):
     def __init__(self, repo: Repo, commit: str) -> None:
         self._repo = repo
         self.commit = commit
+        self._keyset: Keyset | None = None
+        # The whole keyset as key -> blob pointer, once something iterates
+        # it; until then, reads look keys up in the tree one batch at a
+        # time and remember what they found (None: absent).
         self._pointers: dict[str, str] | None = None
+        self._looked_up: dict[str, str | None] = {}
 
     def __repr__(self) -> str:
         return f"Snapshot(commit={self.commit[:8]}...)"
 
+    def _tree(self) -> Keyset:
+        if self._keyset is None:
+            root_raw = self._repo.store.get(COMMIT_ROOT % self.commit)
+            if root_raw is None:
+                raise UnknownCommitError(f"Commit '{self.commit}' does not exist")
+            self._keyset = Keyset(self._repo.store, root=loads(root_raw))
+        return self._keyset
+
     def _index(self) -> dict[str, str]:
+        """The whole keyset, read once, for iteration and length."""
         if self._pointers is None:
-            self._pointers = self._repo._pointers(self.commit)
+            entries = self._tree().materialize()
+            self._pointers = {key: entry.blob for key, entry in entries.items()}
         return self._pointers
 
+    def _pointers_of(self, keys: tuple[str, ...]) -> dict[str, str]:
+        """The blob pointers of those keys that exist.
+
+        Once the whole keyset has been read, from that; before, by looking
+        just these keys up in the tree — its depth in batched reads, not
+        the whole keyset — so reading a few keys at a commit stays cheap
+        however many the commit holds.
+        """
+        if self._pointers is not None:
+            return {k: self._pointers[k] for k in keys if k in self._pointers}
+        unknown = [k for k in dict.fromkeys(keys) if k not in self._looked_up]
+        if unknown:
+            found = self._tree().get_many(unknown)
+            for key in unknown:
+                entry = found.get(key)
+                self._looked_up[key] = None if entry is None else entry.blob
+        return {
+            key: pointer
+            for key in keys
+            if (pointer := self._looked_up.get(key)) is not None
+        }
+
     def _raw_many(self, keys: tuple[str, ...]) -> dict[str, bytes]:
-        index = self._index()
         wanted: dict[str, list[str]] = {}
-        for key in keys:
-            pointer = index.get(key)
-            if pointer is not None:
-                wanted.setdefault(pointer, []).append(key)
+        for key, pointer in self._pointers_of(keys).items():
+            wanted.setdefault(pointer, []).append(key)
+        if not wanted:
+            return {}
         found = self._repo.store.get_many(wanted.keys())
         return {key: raw for pointer, raw in found.items() for key in wanted[pointer]}
 
@@ -362,7 +390,7 @@ class Snapshot(Mapping[str, Any]):
         return len(self._index())
 
     def __contains__(self, key: object) -> bool:
-        return isinstance(key, str) and key in self._index()
+        return isinstance(key, str) and bool(self._pointers_of((key,)))
 
     @property
     def raw(self) -> RawSnapshot:

{kvgit-0.4.0 → kvgit-0.4.1}/kvgit/versioned/keyset.py

@@ -155,6 +155,11 @@ class Keyset:
             return None
         return decode_entry(raw)
 
+    def get_many(self, keys: Iterable[str]) -> dict[str, KeysetEntry]:
+        """The entries of the keys that are present, reading their paths
+        in one batched read per tree level; see ``Hamt.get_many``."""
+        return {k: decode_entry(v) for k, v in self._hamt.get_many(keys).items()}
+
     def get_blob(self, key: str) -> str | None:
         """Shortcut: just the blob pointer, no meta."""
         entry = self.get(key)

{kvgit-0.4.0 → kvgit-0.4.1}/kvgit.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: kvgit
-Version: 0.4.0
+Version: 0.4.1
 Summary: Versioned key-value store with git-like commit, branch, and merge semantics.
 Author: ashenfad
 License: MIT

{kvgit-0.4.0 → kvgit-0.4.1}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "kvgit"
-version = "0.4.0"
+version = "0.4.1"
 description = "Versioned key-value store with git-like commit, branch, and merge semantics."
 authors = [{ name = "ashenfad" }]
 requires-python = ">=3.10"

{kvgit-0.4.0 → kvgit-0.4.1}/tests/test_hamt.py

@@ -954,3 +954,21 @@ def test_an_update_leaves_the_shared_view_untouched():
     h = Hamt(_store(), bucket_max=2).persist({f"k{i}": b"v" for i in range(50)})
     h.updated({"x": b"1"}, ["k1"])
     assert h._fetched == {}
+
+
+@pytest.mark.parametrize("seed", range(20))
+def test_get_many_matches_get(seed):
+    rng = random.Random(seed)
+    items = {f"k{i}": b"%d" % i for i in range(rng.randrange(0, 300))}
+    h = Hamt(_store(), bucket_max=rng.choice([1, 2, 8])).persist(items)
+    keys = [f"k{rng.randrange(0, 400)}" for _ in range(rng.randrange(0, 80))]
+    assert h.get_many(keys) == {k: h.get(k) for k in keys if h.get(k) is not None}
+
+
+def test_get_many_reads_by_level():
+    store = _CountingStore()
+    h = Hamt(store, bucket_max=8).persist({f"k{i}": b"v" for i in range(5000)})
+    store.gets = store.batches = 0
+    found = h.get_many([f"k{i}" for i in range(0, 5000, 10)])
+    assert len(found) == 500
+    assert store.gets == 0 and store.batches <= 8

{kvgit-0.4.0 → kvgit-0.4.1}/tests/test_repo.py

@@ -1,6 +1,7 @@
 """Tests for Repo: branches, tags, the commit graph, snapshots and gc."""
 
 import os
+import random
 import tempfile
 from collections.abc import Mapping
 
@@ -415,6 +416,87 @@ class TestSnapshots:
             repo.snapshot(tag="v1")
 
 
+class TestSnapshotPointReads:
+    """Reading a few keys at a commit costs the tree's depth in batched
+    reads, not the whole keyset; iterating reads the keyset once."""
+
+    class Counting(Memory):
+        def __init__(self):
+            super().__init__()
+            self.calls = 0
+            self.nodes = 0
+
+        def get(self, key):
+            self.calls += 1
+            self.nodes += key.startswith("kvgit:keyset:")
+            return super().get(key)
+
+        def get_many(self, *args):
+            keys = self._normalize_keys(args)
+            self.calls += 1
+            self.nodes += sum(k.startswith("kvgit:keyset:") for k in keys)
+            return super().get_many(*args)
+
+    def _big(self):
+        store = self.Counting()
+        repo = Repo(store)
+        wt = repo.worktree("main", create=True)
+        for i in range(5000):
+            wt[f"k{i}"] = i
+        wt.commit()
+        total = sum(1 for k in store.keys("kvgit:keyset:"))
+        store.calls = store.nodes = 0
+        return store, repo, wt, total
+
+    def test_one_key_reads_a_path_not_the_keyset(self):
+        store, repo, _, total = self._big()
+        snap = repo.snapshot(branch="main")
+        assert snap["k4321"] == 4321
+        assert store.calls <= 8
+        assert store.nodes < 10 < total
+
+    def test_many_keys_read_by_level_in_batches(self):
+        store, repo, _, total = self._big()
+        snap = repo.snapshot(branch="main")
+        keys = [f"k{i}" for i in range(0, 5000, 100)] + ["absent"]
+        found = snap.get_many(*keys)
+        assert found == {k: int(k[1:]) for k in keys if k != "absent"}
+        assert store.calls <= 8
+        assert store.nodes < total
+
+    def test_a_key_read_twice_is_looked_up_once(self):
+        store, repo, _, _ = self._big()
+        snap = repo.snapshot(branch="main")
+        snap["k1"]
+        before = store.calls
+        assert snap["k1"] == 1 and "k1" in snap and snap.raw["k1"]
+        assert store.calls - before == 2  # the blob, twice; no tree reads
+        assert "nope" not in snap
+        with pytest.raises(KeyError):
+            snap["nope"]
+
+    def test_point_reads_agree_with_the_whole_keyset(self):
+        wt = worktree()
+        rng = random.Random(7)
+        for i in range(300):
+            wt[f"k{i}"] = i
+        wt.commit()
+        for i in rng.sample(range(300), 60):
+            del wt[f"k{i}"]
+        wt.commit()
+        repo = wt.repo
+        full = dict(repo.snapshot(commit=wt.head))
+        probe = [f"k{i}" for i in range(0, 320, 7)]
+        lazy = repo.snapshot(commit=wt.head)
+        assert {k: lazy.get(k) for k in probe} == {k: full.get(k) for k in probe}
+        assert [k in lazy for k in probe] == [k in full for k in probe]
+        assert lazy.get_many(*probe) == {k: full[k] for k in probe if k in full}
+        assert dict(lazy) == full  # iterating after point reads
+        assert {k: lazy.get(k) for k in probe} == {k: full.get(k) for k in probe}
+        raw = repo.snapshot(commit=wt.head).raw
+        assert set(raw.get_many(*probe)) == {k for k in probe if k in full}
+
+
 class TestGc:
     def test_gc_reclaims_a_deleted_branch(self):
         repo, _, dev, *_ = repo_with_history()