kvgit 0.4.1__tar.gz → 0.4.3__tar.gz

{kvgit-0.4.1 → kvgit-0.4.3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: kvgit
-Version: 0.4.1
+Version: 0.4.3
 Summary: Versioned key-value store with git-like commit, branch, and merge semantics.
 Author: ashenfad
 License: MIT
@@ -144,7 +144,9 @@ with Repo(Disk("/tmp/kvgit-demo")) as on_disk:
 | `git gc [--prune=now]` | `repo.gc([min_age=0])` |
 
 Where it differs: there is no index; several worktrees can hold one
-branch, and their commits race and merge rather than being refused; a
+branch, and their commits race and merge rather than being refused
+(`commit(on_moved="raise")` refuses instead, raising
+`BranchMovedError`, for data whose keys aren't independent); a
 conflicted merge raises `MergeConflict` and changes nothing (unless a
 merge function such as `text_merge` writes markers into the value);
 commit hashes are not git's, and there are no remotes.

{kvgit-0.4.1 → kvgit-0.4.3}/README.md

@@ -90,7 +90,9 @@ with Repo(Disk("/tmp/kvgit-demo")) as on_disk:
 | `git gc [--prune=now]` | `repo.gc([min_age=0])` |
 
 Where it differs: there is no index; several worktrees can hold one
-branch, and their commits race and merge rather than being refused; a
+branch, and their commits race and merge rather than being refused
+(`commit(on_moved="raise")` refuses instead, raising
+`BranchMovedError`, for data whose keys aren't independent); a
 conflicted merge raises `MergeConflict` and changes nothing (unless a
 merge function such as `text_merge` writes markers into the value);
 commit hashes are not git's, and there are no remotes.

{kvgit-0.4.1 → kvgit-0.4.3}/kvgit/__init__.py

@@ -4,6 +4,7 @@ from ._open import open
 from .content_types import MergeFn, counter, last_writer_wins, text_merge
 from .errors import (
     BranchExistsError,
+    BranchMovedError,
     ConcurrencyError,
     CorruptHeadError,
     GcBusy,
@@ -40,6 +41,7 @@ from .worktree import Status, Worktree
 __all__ = [
     "ROOT_COMMIT",
     "BranchExistsError",
+    "BranchMovedError",
     "Branches",
     "CantMark",
     "Commit",

{kvgit-0.4.1 → kvgit-0.4.3}/kvgit/_codec.py

@@ -61,6 +61,27 @@ class _ChunkReader:
         return None
 
 
+class _CountingReader:
+    """A chunk reader that tallies the bytes of the chunks it returns."""
+
+    def __init__(self, reader: _ChunkReader) -> None:
+        self._reader = reader
+        self.nbytes = 0
+
+    def get(self, ref: str) -> bytes:
+        raw = self._reader.get(ref)
+        self.nbytes += len(raw)
+        return raw
+
+    def get_many(self, refs):
+        found = self._reader.get_many(refs)
+        self.nbytes += sum(len(v) for v in found.values())
+        return found
+
+    def prefetch(self, refs) -> None:
+        self._reader.prefetch(refs)
+
+
 def _is_chunk_aware(fn) -> bool:
     """Whether an encoder/decoder takes a required sink/reader argument.
 
@@ -148,6 +169,14 @@ class Codec:
             return self.decoder(raw, self._reader)
         return self.decoder(raw)
 
+    def decode_sized(self, raw: bytes) -> tuple[Any, int]:
+        """A value, and the bytes it was decoded from: its blob plus any
+        chunks a chunked decoder read, which the value usually keeps."""
+        if self.decoder_chunked:
+            reader = _CountingReader(self._reader)  # type: ignore[arg-type]
+            return self.decoder(raw, reader), len(raw) + reader.nbytes
+        return self.decoder(raw), len(raw)
+
     def encode_merged(self, value: Any) -> bytes:
         """Encode a value a merge function produced.
 

{kvgit-0.4.1 → kvgit-0.4.3}/kvgit/_open.py

@@ -3,6 +3,7 @@
 from typing import Literal
 
 from ._codec import CodecSpec
+from .cache import DEFAULT_CACHE_BYTES, DEFAULT_VALUE_CACHE_BYTES
 from .kv.base import KVStore
 from .kv.memory import Memory
 from .repo import Repo
@@ -39,6 +40,8 @@ def open(
     db_name: str = "kvgit",
     branch: str = "main",
     codec: CodecSpec = "pickle",
+    cache_bytes: int = DEFAULT_CACHE_BYTES,
+    value_cache_bytes: int = DEFAULT_VALUE_CACHE_BYTES,
 ) -> Worktree:
     """Open (or create) a store, and a worktree on ``branch`` of it.
 
@@ -57,6 +60,16 @@ def open(
         db_name: The IndexedDB database name, for ``kind="indexeddb"``.
         branch: The branch to check out (default ``"main"``).
         codec: How values are stored; see :class:`Repo`.
+        cache_bytes: Memory for what the store never changes; see
+            :class:`Repo`.
+        value_cache_bytes: Memory for the values the worktree reads;
+            see :class:`Repo`.
     """
     backend = _make_backend(kind, path=path, db_name=db_name)
-    return Repo(backend, codec=codec).worktree(branch, create=True)
+    repo = Repo(
+        backend,
+        codec=codec,
+        cache_bytes=cache_bytes,
+        value_cache_bytes=value_cache_bytes,
+    )
+    return repo.worktree(branch, create=True)

kvgit-0.4.3/kvgit/cache.py

@@ -0,0 +1,369 @@
+"""A repository's memory of what its store can never change.
+
+Tree nodes and a commit's records — its parents, their generations,
+its time and its info — are written once, under keys named by content
+or by commit, and never rewritten. A read of one can be answered from
+memory for as long as the store still holds it, which saves the round
+trip to a networked backend that most reads of them are.
+
+What the cache must not outlive is a deletion. Only a sweep deletes
+these keys, and every sweep rewrites the GC lease record, which each
+commit reads before it builds. So the cache remembers the record it
+last saw and empties itself when a read of the lease finds another:
+the sweep of any process on any host is noticed before this process
+builds on what it may have removed. A sweep run through this process's
+own store evicts what it removes as it removes it.
+
+A commit's root is never cached. It is the store's answer to whether a
+commit exists — what ``snapshot(commit=)`` and a tag's ``dangling``
+ask — so it is always read, and whatever is reached through a root the
+store still holds is safe to answer from here. Absence is never cached
+either: a key missing now may be written by the next commit.
+
+A worktree keeps a second, separate memory: the decoded values it has
+read at its head (:class:`ValueCache`). That one holds blobs, so it is
+per worktree, emptied whenever the head moves, and bounded in bytes.
+"""
+
+from __future__ import annotations
+
+import threading
+from collections import OrderedDict
+from collections.abc import Iterable, Mapping
+from typing import Any
+
+from .kv.base import KVStore
+from .versioned.keyset import Keyset
+from .versioned.kv import (
+    COMMIT_TIME,
+    GC_LEASE_KEY,
+    INFO_KEY,
+    PARENT_COMMIT,
+    PARENT_GENS,
+)
+
+DEFAULT_CACHE_BYTES = 32 * 1024 * 1024
+"""The default budget: tens of thousands of tree nodes."""
+
+DEFAULT_VALUE_CACHE_BYTES = 32 * 1024 * 1024
+"""The default budget for each worktree's decoded values."""
+
+_CACHEABLE_PREFIXES = (
+    Keyset.DEFAULT_PREFIX,
+    PARENT_COMMIT.replace("%s", ""),
+    PARENT_GENS.replace("%s", ""),
+    COMMIT_TIME.replace("%s", ""),
+    INFO_KEY.replace("%s", ""),
+)
+
+# What an entry costs beyond its bytes: the key, the dict slot, the
+# object headers. An estimate, so the budget bounds memory rather than
+# just payload.
+_ENTRY_OVERHEAD = 96
+
+_UNSEEN: Any = object()
+
+
+def cacheable(key: str) -> bool:
+    """Whether ``key`` is one the store never rewrites."""
+    return key.startswith(_CACHEABLE_PREFIXES)
+
+
+class ContentCache:
+    """Least-recently-used memory of write-once keys, bounded in bytes.
+
+    Shared by every worktree and snapshot of one :class:`~kvgit.Repo`,
+    and safe to use from several threads.
+    """
+
+    def __init__(self, max_bytes: int = DEFAULT_CACHE_BYTES) -> None:
+        if max_bytes <= 0:
+            raise ValueError(f"max_bytes must be positive, got {max_bytes}")
+        self.max_bytes = max_bytes
+        self._entries: OrderedDict[str, bytes] = OrderedDict()
+        self._bytes = 0
+        self._lock = threading.Lock()
+        self._lease: Any = _UNSEEN
+        self.hits = 0
+        """Reads answered from memory."""
+        self.misses = 0
+        """Reads of cacheable keys that went to the store."""
+        self.clears = 0
+        """How often a sweep emptied the cache."""
+
+    def __repr__(self) -> str:
+        return (
+            f"ContentCache({len(self._entries)} entries, "
+            f"{self._bytes}/{self.max_bytes} bytes)"
+        )
+
+    def __len__(self) -> int:
+        return len(self._entries)
+
+    @property
+    def size_bytes(self) -> int:
+        """What the entries held now cost, by the budget's estimate."""
+        return self._bytes
+
+    def clear(self) -> None:
+        with self._lock:
+            self._clear()
+
+    def _clear(self) -> None:
+        self._entries.clear()
+        self._bytes = 0
+
+    def lookup(self, keys: Iterable[str]) -> dict[str, bytes]:
+        """The cached values of those ``keys`` held here."""
+        found: dict[str, bytes] = {}
+        with self._lock:
+            for key in keys:
+                value = self._entries.get(key)
+                if value is not None:
+                    self._entries.move_to_end(key)
+                    found[key] = value
+        return found
+
+    def count(self, hits: int, misses: int) -> None:
+        with self._lock:
+            self.hits += hits
+            self.misses += misses
+
+    def remember(self, items: Mapping[str, bytes]) -> None:
+        """Hold these values, evicting the least recently used to fit."""
+        with self._lock:
+            for key, value in items.items():
+                if not isinstance(value, bytes | bytearray | memoryview):
+                    continue
+                cost = len(value) + len(key) + _ENTRY_OVERHEAD
+                if cost > self.max_bytes:
+                    continue
+                old = self._entries.pop(key, None)
+                if old is not None:
+                    self._bytes -= len(old) + len(key) + _ENTRY_OVERHEAD
+                self._entries[key] = bytes(value)
+                self._bytes += cost
+            while self._bytes > self.max_bytes and self._entries:
+                key, value = self._entries.popitem(last=False)
+                self._bytes -= len(value) + len(key) + _ENTRY_OVERHEAD
+
+    def forget(self, keys: Iterable[str]) -> None:
+        with self._lock:
+            for key in keys:
+                value = self._entries.pop(key, None)
+                if value is not None:
+                    self._bytes -= len(value) + len(key) + _ENTRY_OVERHEAD
+
+    def saw_lease(self, record: bytes | None) -> None:
+        """Note the GC lease record a read found; a different one than
+        last time means a sweep has run, and what it removed may be
+        held here."""
+        with self._lock:
+            if self._lease is not _UNSEEN and record != self._lease:
+                self._clear()
+                self.clears += 1
+            self._lease = record
+
+
+class CachedStore(KVStore):
+    """A backend seen through a :class:`ContentCache`.
+
+    Reads of write-once keys are answered from the cache where it can
+    and remembered where it cannot; everything else passes through
+    untouched. Successful writes of write-once keys are remembered too,
+    so a commit's next read of its own tree is a hit, and removals
+    evict. Anything else the backend offers (``close``, ``drop``, ...)
+    is reached through this object as it would be on the backend.
+    """
+
+    def __init__(self, backend: KVStore, cache: ContentCache) -> None:
+        self.backend = backend
+        self.cache = cache
+
+    def __repr__(self) -> str:
+        return f"CachedStore({self.backend!r})"
+
+    def __getattr__(self, name: str) -> Any:
+        if name.startswith("__") or name in ("backend", "cache"):
+            raise AttributeError(name)
+        return getattr(self.backend, name)
+
+    # -- reads -----------------------------------------------------------
+
+    def get(self, key: str) -> bytes | None:
+        if cacheable(key):
+            held = self.cache.lookup((key,))
+            if key in held:
+                self.cache.count(1, 0)
+                return held[key]
+            value = self.backend.get(key)
+            self.cache.count(0, 1)
+            if value is not None:
+                self.cache.remember({key: value})
+            return value
+        value = self.backend.get(key)
+        if key == GC_LEASE_KEY:
+            self.cache.saw_lease(value)
+        return value
+
+    def get_many(self, *args) -> Mapping[str, bytes]:
+        keys = list(dict.fromkeys(self._normalize_keys(args)))
+        wanted = [k for k in keys if cacheable(k)]
+        if GC_LEASE_KEY in keys:
+            # A read of the lease may reveal a sweep, and nothing the
+            # cache held before it may be answered beside it: the whole
+            # call goes to the store, and the record is noted before
+            # anything it brought back is remembered.
+            held: dict[str, bytes] = {}
+            found = dict(self.backend.get_many(keys))
+            self.cache.saw_lease(found.get(GC_LEASE_KEY))
+        else:
+            held = self.cache.lookup(wanted) if wanted else {}
+            rest = [k for k in keys if k not in held]
+            found = dict(self.backend.get_many(rest)) if rest else {}
+        fetched = {k: v for k, v in found.items() if cacheable(k)}
+        if wanted:
+            self.cache.count(len(held), len(wanted) - len(held))
+        if fetched:
+            self.cache.remember(fetched)
+        found.update(held)
+        return found
+
+    def items(self) -> Iterable[tuple[str, bytes]]:
+        return self.backend.items()
+
+    def keys(self, prefix: str = "") -> Iterable[str]:
+        return self.backend.keys(prefix)
+
+    def __contains__(self, key: str) -> bool:
+        if cacheable(key) and self.cache.lookup((key,)):
+            return True
+        return key in self.backend
+
+    # -- writes ----------------------------------------------------------
+
+    def set(self, key: str, value: bytes) -> None:
+        self.backend.set(key, value)
+        self._wrote({key: value}, ())
+
+    def set_many(
+        self, items: Mapping[str, bytes] | None = None, /, **kwargs: bytes
+    ) -> None:
+        items = dict(self._normalize_items(items, kwargs))
+        self.backend.set_many(items)
+        self._wrote(items, ())
+
+    def remove(self, key: str) -> None:
+        self.backend.remove(key)
+        self._wrote({}, (key,))
+
+    def remove_many(self, *args) -> None:
+        keys = list(self._normalize_keys(args))
+        self.backend.remove_many(keys)
+        self._wrote({}, keys)
+
+    def cas_many(
+        self,
+        expected: Mapping[str, bytes | None],
+        writes: Mapping[str, bytes],
+        removes: Iterable[str] = (),
+    ) -> bool:
+        removes = list(removes)
+        if not self.backend.cas_many(expected, writes, removes):
+            return False
+        self._wrote(writes, removes)
+        return True
+
+    def clear(self) -> None:
+        self.backend.clear()
+        self.cache.clear()
+
+    def _wrote(self, writes: Mapping[str, bytes], removes: Iterable[str]) -> None:
+        gone = [k for k in removes if cacheable(k)]
+        if gone:
+            self.cache.forget(gone)
+        landed = {k: v for k, v in writes.items() if cacheable(k)}
+        if landed:
+            self.cache.remember(landed)
+        if GC_LEASE_KEY in writes:
+            # This process's own sweep taking or releasing the lease.
+            self.cache.saw_lease(writes[GC_LEASE_KEY])
+
+
+class ValueCache:
+    """A worktree's decoded values of committed keys, least recently
+    used first out, bounded in bytes.
+
+    It holds only what the worktree's head already stores, so anything
+    it lets go is read again from the store; pending writes are kept by
+    the worktree itself and never pass through here. A value's cost is
+    the bytes it was decoded from (its blob, plus any chunks a chunked
+    codec read), and a value costing more than a quarter of the budget
+    is not held at all, so one large read cannot push out everything
+    else and the largest values, the ones a caller may want only a few
+    bytes of, are not kept alive by the cache. A budget of ``0`` holds
+    nothing.
+
+    Safe to use from several threads.
+    """
+
+    def __init__(self, max_bytes: int = DEFAULT_VALUE_CACHE_BYTES) -> None:
+        if max_bytes < 0:
+            raise ValueError(f"max_bytes must not be negative, got {max_bytes}")
+        self.max_bytes = max_bytes
+        self._entries: OrderedDict[str, tuple[Any, int]] = OrderedDict()
+        self._bytes = 0
+        self._lock = threading.Lock()
+
+    def __repr__(self) -> str:
+        return (
+            f"ValueCache({len(self._entries)} entries, "
+            f"{self._bytes}/{self.max_bytes} bytes)"
+        )
+
+    def __len__(self) -> int:
+        return len(self._entries)
+
+    def __contains__(self, key: object) -> bool:
+        return key in self._entries
+
+    @property
+    def size_bytes(self) -> int:
+        """What the values held now cost, by the budget's estimate."""
+        return self._bytes
+
+    @property
+    def max_value_bytes(self) -> int:
+        """The largest cost of a value that is held."""
+        return self.max_bytes // 4
+
+    def get(self, key: str, default: Any = None) -> Any:
+        """The value held for ``key``, marked as just used; ``default``
+        if none is."""
+        with self._lock:
+            entry = self._entries.get(key)
+            if entry is None:
+                return default
+            self._entries.move_to_end(key)
+            return entry[0]
+
+    def remember(self, key: str, value: Any, nbytes: int) -> None:
+        """Hold ``value``, decoded from ``nbytes`` bytes, evicting the
+        least recently used to fit."""
+        cost = nbytes + len(key) + _ENTRY_OVERHEAD
+        if cost > self.max_value_bytes:
+            return
+        with self._lock:
+            old = self._entries.pop(key, None)
+            if old is not None:
+                self._bytes -= old[1]
+            self._entries[key] = (value, cost)
+            self._bytes += cost
+            while self._bytes > self.max_bytes:
+                _, (_, evicted) = self._entries.popitem(last=False)
+                self._bytes -= evicted
+
+    def clear(self) -> None:
+        with self._lock:
+            self._entries.clear()
+            self._bytes = 0

{kvgit-0.4.1 → kvgit-0.4.3}/kvgit/errors.py

@@ -18,6 +18,28 @@ class ConcurrencyError(KvgitError):
     """
 
 
+class BranchMovedError(ConcurrencyError):
+    """Raised by a commit made with ``on_moved="raise"`` that found its
+    branch moved: the branch's head is no longer the commit the handle
+    is based on. Nothing was published and pending changes are kept;
+    refresh and redo the work, or give up.
+
+    Attributes:
+        branch: The branch.
+        base: The commit the handle is based on, which the branch no
+            longer names.
+        head: The branch's head as the commit found it.
+    """
+
+    def __init__(self, branch: str, base: str, head: str) -> None:
+        self.branch = branch
+        self.base = base
+        self.head = head
+        super().__init__(
+            f"Branch '{branch}' moved from {base} to {head}; nothing was committed"
+        )
+
+
 class MergeConflict(KvgitError):
     """Raised when a three-way merge encounters unresolvable conflicts.
 

{kvgit-0.4.1 → kvgit-0.4.3}/kvgit/refs.py

@@ -35,10 +35,15 @@ class Branches(Mapping[str, str]):
         return f"Branches({list(self)!r})"
 
     def __getitem__(self, name: str) -> str:
+        return self._resolve(name)
+
+    def _resolve(self, name: str, roots: dict[str, str] | None = None) -> str:
+        """The branch's head, recording its root in ``roots`` when given
+        (see :meth:`Repo._resolve_ref`)."""
         _kv._reject_reserved_branch(name)
-        store = self._repo.store
+        store = self._repo._store
         commit = _kv._resolve_head(
-            store, name, recover_from_corrupt_head=self._repo._recover
+            store, name, recover_from_corrupt_head=self._repo._recover, roots=roots
         )
         if commit is not None:
             return commit
@@ -47,13 +52,13 @@ class Branches(Mapping[str, str]):
         raise UnknownBranchError(f"Branch '{name}' does not exist")
 
     def __contains__(self, name: object) -> bool:
-        return isinstance(name, str) and VersionedKV.exists(self._repo.store, name)
+        return isinstance(name, str) and VersionedKV.exists(self._repo._store, name)
 
     def __iter__(self) -> Iterator[str]:
-        return iter(VersionedKV.branches(self._repo.store))
+        return iter(VersionedKV.branches(self._repo._store))
 
     def __len__(self) -> int:
-        return len(VersionedKV.branches(self._repo.store))
+        return len(VersionedKV.branches(self._repo._store))
 
     def create(self, name: str, at: str | None = None) -> str:
         """Create branch ``name`` at commit ``at`` (default: the empty root
@@ -64,7 +69,7 @@ class Branches(Mapping[str, str]):
             UnknownCommitError: if ``at`` is not in the store.
         """
         target = at or ROOT_COMMIT
-        _kv.create_branch(self._repo.store, name, target)
+        _kv.create_branch(self._repo._store, name, target)
         return target
 
     def delete(self, name: str) -> None:
@@ -72,10 +77,21 @@ class Branches(Mapping[str, str]):
         ``gc``; worktrees still holding it read from their heads and raise
         :class:`UnknownBranchError` on commit.
 
+        The removal is conditional on the head this call read: if another
+        writer moves the branch, or deletes and recreates it, in between,
+        nothing is removed and the call raises rather than retrying, so a
+        newer head is never deleted on the strength of an older one. A
+        branch made again at the same commit is indistinguishable from
+        the one read, and is deleted, as with git's
+        ``update-ref -d <ref> <old-oid>``.
+
         Raises:
             UnknownBranchError: if there is no such branch.
+            ConcurrencyError: if the branch changed while being deleted.
+            ValueError: if ``name`` is empty or in the reserved tag
+                namespace.
         """
-        _kv.delete_branch(self._repo.store, name)
+        _kv.delete_branch(self._repo._store, name)
 
 
 class Tags(Mapping[str, str]):
@@ -95,7 +111,7 @@ class Tags(Mapping[str, str]):
         return f"Tags({list(self)!r})"
 
     def __getitem__(self, name: str) -> str:
-        commit = _kv._resolve_tag(self._repo.store, name)
+        commit = _kv._resolve_tag(self._repo._store, name)
         if commit is None:
             raise UnknownTagError(f"Tag '{name}' does not exist")
         return commit
@@ -103,14 +119,14 @@ class Tags(Mapping[str, str]):
     def __contains__(self, name: object) -> bool:
         return (
             isinstance(name, str)
-            and _kv._resolve_tag(self._repo.store, name) is not None
+            and _kv._resolve_tag(self._repo._store, name) is not None
         )
 
     def __iter__(self) -> Iterator[str]:
-        return iter(_kv.tags(self._repo.store))
+        return iter(_kv.tags(self._repo._store))
 
     def __len__(self) -> int:
-        return len(_kv.tags(self._repo.store))
+        return len(_kv.tags(self._repo._store))
 
     def items(self) -> ItemsView[str, str]:
         """Every tag and its commit: a live view, like the mapping, that
@@ -125,16 +141,22 @@ class Tags(Mapping[str, str]):
             TagExistsError: if the name is taken.
             UnknownCommitError: if ``commit`` is not in the store.
         """
-        _kv.create_tag(self._repo.store, name, commit, info)
+        _kv.create_tag(self._repo._store, name, commit, info)
 
     def delete(self, name: str) -> None:
         """Delete a tag. A commit it alone kept alive becomes collectable
         at the next ``gc``.
 
+        The removal is conditional on the head and info record this call
+        read: a tag deleted and made again by another writer in between,
+        even at the same commit, keeps its new commit and info, and the
+        call raises rather than retrying.
+
         Raises:
             UnknownTagError: if there is no such tag.
+            ConcurrencyError: if the tag changed while being deleted.
         """
-        _kv.delete_tag(self._repo.store, name)
+        _kv.delete_tag(self._repo._store, name)
 
     def info(self, name: str) -> TagInfo:
         """A tag's commit, creation time, info, and whether its commit is
@@ -143,7 +165,7 @@ class Tags(Mapping[str, str]):
         Raises:
             UnknownTagError: if there is no such tag.
         """
-        found = _kv.tag_info(self._repo.store, name)
+        found = _kv.tag_info(self._repo._store, name)
         if found is None:
             raise UnknownTagError(f"Tag '{name}' does not exist")
         return found