kvgit 0.3.1__tar.gz → 0.3.2__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {kvgit-0.3.1 → kvgit-0.3.2}/PKG-INFO +1 -1
- {kvgit-0.3.1 → kvgit-0.3.2}/kvgit/__init__.py +2 -1
- {kvgit-0.3.1 → kvgit-0.3.2}/kvgit/kv/disk.py +7 -0
- kvgit-0.3.2/kvgit/store.py +152 -0
- {kvgit-0.3.1 → kvgit-0.3.2}/kvgit/versioned/kv.py +166 -138
- {kvgit-0.3.1 → kvgit-0.3.2}/kvgit.egg-info/PKG-INFO +1 -1
- {kvgit-0.3.1 → kvgit-0.3.2}/kvgit.egg-info/SOURCES.txt +1 -0
- {kvgit-0.3.1 → kvgit-0.3.2}/pyproject.toml +1 -1
- kvgit-0.3.2/tests/test_delete_branches.py +209 -0
- kvgit-0.3.1/kvgit/store.py +0 -74
- {kvgit-0.3.1 → kvgit-0.3.2}/LICENSE +0 -0
- {kvgit-0.3.1 → kvgit-0.3.2}/README.md +0 -0
- {kvgit-0.3.1 → kvgit-0.3.2}/kvgit/codecs/__init__.py +0 -0
- {kvgit-0.3.1 → kvgit-0.3.2}/kvgit/codecs/_hash.py +0 -0
- {kvgit-0.3.1 → kvgit-0.3.2}/kvgit/codecs/base.py +0 -0
- {kvgit-0.3.1 → kvgit-0.3.2}/kvgit/codecs/numpy.py +0 -0
- {kvgit-0.3.1 → kvgit-0.3.2}/kvgit/codecs/pandas.py +0 -0
- {kvgit-0.3.1 → kvgit-0.3.2}/kvgit/codecs/pickler.py +0 -0
- {kvgit-0.3.1 → kvgit-0.3.2}/kvgit/content_types.py +0 -0
- {kvgit-0.3.1 → kvgit-0.3.2}/kvgit/encoding.py +0 -0
- {kvgit-0.3.1 → kvgit-0.3.2}/kvgit/errors.py +0 -0
- {kvgit-0.3.1 → kvgit-0.3.2}/kvgit/hamt.py +0 -0
- {kvgit-0.3.1 → kvgit-0.3.2}/kvgit/kv/__init__.py +0 -0
- {kvgit-0.3.1 → kvgit-0.3.2}/kvgit/kv/base.py +0 -0
- {kvgit-0.3.1 → kvgit-0.3.2}/kvgit/kv/composite.py +0 -0
- {kvgit-0.3.1 → kvgit-0.3.2}/kvgit/kv/indexeddb.py +0 -0
- {kvgit-0.3.1 → kvgit-0.3.2}/kvgit/kv/memory.py +0 -0
- {kvgit-0.3.1 → kvgit-0.3.2}/kvgit/namespaced.py +0 -0
- {kvgit-0.3.1 → kvgit-0.3.2}/kvgit/py.typed +0 -0
- {kvgit-0.3.1 → kvgit-0.3.2}/kvgit/staged.py +0 -0
- {kvgit-0.3.1 → kvgit-0.3.2}/kvgit/versioned/__init__.py +0 -0
- {kvgit-0.3.1 → kvgit-0.3.2}/kvgit/versioned/base.py +0 -0
- {kvgit-0.3.1 → kvgit-0.3.2}/kvgit/versioned/helpers.py +0 -0
- {kvgit-0.3.1 → kvgit-0.3.2}/kvgit/versioned/keyset.py +0 -0
- {kvgit-0.3.1 → kvgit-0.3.2}/kvgit/versioned/merge.py +0 -0
- {kvgit-0.3.1 → kvgit-0.3.2}/kvgit/versioned/protocol.py +0 -0
- {kvgit-0.3.1 → kvgit-0.3.2}/kvgit.egg-info/dependency_links.txt +0 -0
- {kvgit-0.3.1 → kvgit-0.3.2}/kvgit.egg-info/requires.txt +0 -0
- {kvgit-0.3.1 → kvgit-0.3.2}/kvgit.egg-info/top_level.txt +0 -0
- {kvgit-0.3.1 → kvgit-0.3.2}/setup.cfg +0 -0
- {kvgit-0.3.1 → kvgit-0.3.2}/tests/test_content_types.py +0 -0
- {kvgit-0.3.1 → kvgit-0.3.2}/tests/test_hamt.py +0 -0
- {kvgit-0.3.1 → kvgit-0.3.2}/tests/test_namespaced.py +0 -0
- {kvgit-0.3.1 → kvgit-0.3.2}/tests/test_staged.py +0 -0
- {kvgit-0.3.1 → kvgit-0.3.2}/tests/test_store_factory.py +0 -0
|
@@ -5,7 +5,7 @@ from .errors import ConcurrencyError, MergeConflict
|
|
|
5
5
|
from .namespaced import Namespaced
|
|
6
6
|
from .versioned.protocol import MergeResult, Versioned
|
|
7
7
|
from .staged import Staged
|
|
8
|
-
from .store import store
|
|
8
|
+
from .store import delete_branches, store
|
|
9
9
|
from .versioned.kv import VersionedKV
|
|
10
10
|
|
|
11
11
|
__all__ = [
|
|
@@ -18,6 +18,7 @@ __all__ = [
|
|
|
18
18
|
"Versioned",
|
|
19
19
|
"VersionedKV",
|
|
20
20
|
"counter",
|
|
21
|
+
"delete_branches",
|
|
21
22
|
"last_writer_wins",
|
|
22
23
|
"store",
|
|
23
24
|
]
|
|
@@ -87,3 +87,10 @@ class Disk(KVStore):
|
|
|
87
87
|
|
|
88
88
|
def clear(self) -> None:
|
|
89
89
|
self.store.clear()
|
|
90
|
+
|
|
91
|
+
def close(self) -> None:
|
|
92
|
+
# diskcache holds SQLite connections (one per thread) open until
|
|
93
|
+
# closed. Admin teardown (kvgit.delete_branches) constructs a
|
|
94
|
+
# backend just to sweep and must release the handle so the next
|
|
95
|
+
# opener on the same directory isn't blocked.
|
|
96
|
+
self.store.close()
|
|
@@ -0,0 +1,152 @@
|
|
|
1
|
+
"""Store factory function."""
|
|
2
|
+
|
|
3
|
+
import pickle
|
|
4
|
+
from collections.abc import Iterable
|
|
5
|
+
from typing import Any, Callable, Literal
|
|
6
|
+
|
|
7
|
+
from .kv.base import KVStore
|
|
8
|
+
from .kv.memory import Memory
|
|
9
|
+
from .staged import Staged
|
|
10
|
+
from .versioned.kv import BRANCH_HEAD, BRANCH_HEAD_PREV, VersionedKV, clean_orphans
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
def _make_backend(
|
|
14
|
+
kind: Literal["memory", "disk", "indexeddb"],
|
|
15
|
+
*,
|
|
16
|
+
path: str | None,
|
|
17
|
+
db_name: str,
|
|
18
|
+
) -> KVStore:
|
|
19
|
+
"""Construct a raw ``KVStore`` backend by kind.
|
|
20
|
+
|
|
21
|
+
Shared by :func:`store` and :func:`delete_branches` so both open the
|
|
22
|
+
exact same backends from the same parameters.
|
|
23
|
+
"""
|
|
24
|
+
if kind == "memory":
|
|
25
|
+
return Memory()
|
|
26
|
+
elif kind == "disk":
|
|
27
|
+
if path is None:
|
|
28
|
+
raise ValueError("path is required when kind='disk'")
|
|
29
|
+
from .kv.disk import Disk
|
|
30
|
+
|
|
31
|
+
return Disk(path)
|
|
32
|
+
elif kind == "indexeddb":
|
|
33
|
+
from .kv.indexeddb import IndexedDB
|
|
34
|
+
|
|
35
|
+
return IndexedDB(db_name=db_name)
|
|
36
|
+
else:
|
|
37
|
+
raise ValueError(f"Unknown kind: {kind!r}")
|
|
38
|
+
|
|
39
|
+
|
|
40
|
+
def store(
|
|
41
|
+
kind: Literal["memory", "disk", "indexeddb"] = "memory",
|
|
42
|
+
*,
|
|
43
|
+
path: str | None = None,
|
|
44
|
+
db_name: str = "kvgit",
|
|
45
|
+
branch: str = "main",
|
|
46
|
+
encoder: Callable[..., bytes] = pickle.dumps,
|
|
47
|
+
decoder: Callable[..., Any] = pickle.loads,
|
|
48
|
+
codecs: str | None = None,
|
|
49
|
+
) -> Staged:
|
|
50
|
+
"""Create a Staged store with sensible defaults.
|
|
51
|
+
|
|
52
|
+
Args:
|
|
53
|
+
kind: ``"memory"`` (default), ``"disk"``, or ``"indexeddb"``.
|
|
54
|
+
path: Required when ``kind="disk"``. Directory path for the
|
|
55
|
+
disk backend.
|
|
56
|
+
db_name: IndexedDB database name (default ``"kvgit"``).
|
|
57
|
+
Only used when ``kind="indexeddb"``.
|
|
58
|
+
branch: Branch name (default ``"main"``).
|
|
59
|
+
encoder: Value encoder (default ``pickle.dumps``).
|
|
60
|
+
decoder: Value decoder (default ``pickle.loads``).
|
|
61
|
+
codecs: Optional named codec preset. Currently supported:
|
|
62
|
+
``"scientific"`` — numpy/pandas chunked codecs (requires
|
|
63
|
+
numpy; install with ``pip install kvgit[scientific]``).
|
|
64
|
+
Mutually exclusive with explicit ``encoder`` / ``decoder``.
|
|
65
|
+
|
|
66
|
+
Returns:
|
|
67
|
+
A ``Staged`` store instance.
|
|
68
|
+
|
|
69
|
+
Raises:
|
|
70
|
+
ValueError: if ``codecs`` is given alongside non-default
|
|
71
|
+
``encoder`` / ``decoder``, or if ``codecs`` names an
|
|
72
|
+
unknown preset.
|
|
73
|
+
ImportError: if a codec preset's optional dependency is not
|
|
74
|
+
installed.
|
|
75
|
+
"""
|
|
76
|
+
if codecs is not None:
|
|
77
|
+
if encoder is not pickle.dumps or decoder is not pickle.loads:
|
|
78
|
+
raise ValueError(
|
|
79
|
+
"codecs= is mutually exclusive with explicit encoder/decoder; "
|
|
80
|
+
"pass one or the other"
|
|
81
|
+
)
|
|
82
|
+
from .codecs import _resolve_named
|
|
83
|
+
|
|
84
|
+
encoder, decoder = _resolve_named(codecs)
|
|
85
|
+
|
|
86
|
+
backend = _make_backend(kind, path=path, db_name=db_name)
|
|
87
|
+
|
|
88
|
+
return Staged(VersionedKV(backend, branch=branch), encoder=encoder, decoder=decoder)
|
|
89
|
+
|
|
90
|
+
|
|
91
|
+
def delete_branches(
|
|
92
|
+
names: str | Iterable[str],
|
|
93
|
+
*,
|
|
94
|
+
kind: Literal["memory", "disk", "indexeddb"] = "disk",
|
|
95
|
+
path: str | None = None,
|
|
96
|
+
db_name: str = "kvgit",
|
|
97
|
+
min_age: float = 3600,
|
|
98
|
+
) -> None:
|
|
99
|
+
"""Delete branches with no branch anchor, then sweep orphans.
|
|
100
|
+
|
|
101
|
+
:meth:`VersionedKV.delete_branch` refuses to delete the branch its
|
|
102
|
+
handle is anchored on — and there is no way to open a handle
|
|
103
|
+
*without* a branch. When the branch you want gone is the store's
|
|
104
|
+
only branch, there is nothing safe to anchor on. This admin entry
|
|
105
|
+
point sidesteps the whole problem: it opens the backend directly
|
|
106
|
+
(no ``VersionedKV``, so no current branch) and edits the branch
|
|
107
|
+
keys itself.
|
|
108
|
+
|
|
109
|
+
For each name, ``__branch_head__<name>`` and its
|
|
110
|
+
``__branch_head_prev__<name>`` recovery backup are removed — the
|
|
111
|
+
backup too, so a later same-named branch can't "recover" the
|
|
112
|
+
deleted state through ``_resolve_head``'s fallback. A name with no
|
|
113
|
+
head is simply skipped: this is teardown, and idempotency beats a
|
|
114
|
+
``ValueError`` here. Deleting every branch is legal — the store
|
|
115
|
+
just mints a fresh empty ``main`` on next open.
|
|
116
|
+
|
|
117
|
+
One :func:`clean_orphans` sweep runs after all removals, so commits
|
|
118
|
+
that only the deleted branches referenced are reclaimed. All heads
|
|
119
|
+
(and prev-heads) are gone before the sweep, so it can't mistake a
|
|
120
|
+
doomed branch for a live one. ``min_age`` defaults to
|
|
121
|
+
``clean_orphans``'s one-hour guard (matching ``delete_branch``), so
|
|
122
|
+
an in-flight commit from a concurrent writer on another branch is
|
|
123
|
+
not swept out from under it; an admin who knows the store is quiet
|
|
124
|
+
can pass ``0`` to reclaim young commits immediately.
|
|
125
|
+
|
|
126
|
+
Args:
|
|
127
|
+
names: Branch names to delete — one name or an iterable of
|
|
128
|
+
them. Missing names are no-ops. (A bare string is treated
|
|
129
|
+
as ONE name, not iterated per character.)
|
|
130
|
+
kind: ``"disk"`` (default), ``"memory"``, or ``"indexeddb"``.
|
|
131
|
+
path: Required when ``kind="disk"``. Store directory.
|
|
132
|
+
db_name: IndexedDB database name (``kind="indexeddb"`` only).
|
|
133
|
+
min_age: Passed to :func:`clean_orphans` — commits younger than
|
|
134
|
+
this many seconds survive the sweep.
|
|
135
|
+
"""
|
|
136
|
+
# A bare string is a plausible one-branch call, and iterating it
|
|
137
|
+
# would "delete" each character as a branch name — a silent no-op
|
|
138
|
+
# at best. Treat it as a single name, like nontainer's
|
|
139
|
+
# delete_workspace does one layer up.
|
|
140
|
+
doomed = {names} if isinstance(names, str) else set(names)
|
|
141
|
+
if not doomed:
|
|
142
|
+
return # nothing asked: skip the backend open and the sweep
|
|
143
|
+
backend = _make_backend(kind, path=path, db_name=db_name)
|
|
144
|
+
try:
|
|
145
|
+
for name in doomed:
|
|
146
|
+
backend.remove(BRANCH_HEAD % name)
|
|
147
|
+
backend.remove(BRANCH_HEAD_PREV % name)
|
|
148
|
+
clean_orphans(backend, min_age=min_age)
|
|
149
|
+
finally:
|
|
150
|
+
close = getattr(backend, "close", None)
|
|
151
|
+
if callable(close):
|
|
152
|
+
close()
|
|
@@ -47,6 +47,7 @@ from ..hamt import EMPTY_HASH
|
|
|
47
47
|
from ..kv.base import KVStore
|
|
48
48
|
from ..kv.memory import Memory
|
|
49
49
|
from .base import VersionedBase
|
|
50
|
+
from .helpers import walk_history
|
|
50
51
|
from .keyset import Keyset, KeysetEntry, MetaEntry
|
|
51
52
|
from .merge import MergeResolution
|
|
52
53
|
|
|
@@ -265,6 +266,167 @@ def _scan_for_best_commit(store: KVStore, branch: str) -> str | None:
|
|
|
265
266
|
return max(tips, key=lambda h: all_commits.get(h, 0))
|
|
266
267
|
|
|
267
268
|
|
|
269
|
+
def clean_orphans(store: KVStore, min_age: float = 3600) -> int:
|
|
270
|
+
"""Remove orphaned commits unreachable from any branch HEAD.
|
|
271
|
+
|
|
272
|
+
Traces all reachable commits from live branch HEADs, then deletes
|
|
273
|
+
commit metadata, blobs, and HAMT nodes that are not reachable from
|
|
274
|
+
any reachable commit.
|
|
275
|
+
|
|
276
|
+
Handle-independent by design: it marks from ALL live branch HEADs
|
|
277
|
+
and touches nothing but ``store``, so it works with or without a
|
|
278
|
+
``VersionedKV`` anchored on it. :meth:`VersionedKV.clean_orphans`
|
|
279
|
+
and the anchor-free admin path (:func:`kvgit.delete_branches`)
|
|
280
|
+
share this one implementation.
|
|
281
|
+
|
|
282
|
+
The ``min_age`` guard (default 1 hour) prevents recently created
|
|
283
|
+
commits from being falsely swept during concurrent writes.
|
|
284
|
+
|
|
285
|
+
Returns:
|
|
286
|
+
Number of orphaned commits removed.
|
|
287
|
+
"""
|
|
288
|
+
gc_logger = logging.getLogger("kvgit.orphans")
|
|
289
|
+
cutoff_time = time.time() - min_age
|
|
290
|
+
|
|
291
|
+
def _parent_loader(commit_hash: str) -> tuple[str, ...]:
|
|
292
|
+
parent_bytes = store.get(PARENT_COMMIT % commit_hash)
|
|
293
|
+
if parent_bytes is None:
|
|
294
|
+
return ()
|
|
295
|
+
raw = loads(parent_bytes)
|
|
296
|
+
if raw is None:
|
|
297
|
+
return ()
|
|
298
|
+
if isinstance(raw, str):
|
|
299
|
+
return (raw,)
|
|
300
|
+
return tuple(raw)
|
|
301
|
+
|
|
302
|
+
# Mark phase: walk every branch's history, collecting reachable
|
|
303
|
+
# commits, blob keys, HAMT node hashes, and chunk references.
|
|
304
|
+
reachable_commits: set[str] = set()
|
|
305
|
+
reachable_blobs: set[str] = set()
|
|
306
|
+
reachable_nodes: set[str] = set()
|
|
307
|
+
reachable_chunks: set[str] = set()
|
|
308
|
+
|
|
309
|
+
def _walk_commit_for_marks(commit_hash: str) -> None:
|
|
310
|
+
"""Walk one commit's keyset, accumulating reachable refs."""
|
|
311
|
+
root = _load_root(store, commit_hash)
|
|
312
|
+
if root is None:
|
|
313
|
+
return
|
|
314
|
+
# Single batched walk per commit collects HAMT node hashes
|
|
315
|
+
# and the entries (each carrying blob + optional chunks).
|
|
316
|
+
# ``skip_nodes`` lets us skip subtrees already seen via
|
|
317
|
+
# structural sharing — the blobs under those subtrees are
|
|
318
|
+
# already accounted for.
|
|
319
|
+
entries, new_nodes = Keyset(store, root=root).walk(skip_nodes=reachable_nodes)
|
|
320
|
+
for entry in entries.values():
|
|
321
|
+
reachable_blobs.add(entry.blob)
|
|
322
|
+
if entry.meta.chunks:
|
|
323
|
+
reachable_chunks.update(entry.meta.chunks)
|
|
324
|
+
reachable_nodes.update(new_nodes)
|
|
325
|
+
|
|
326
|
+
branch_prefix = BRANCH_HEAD.replace("%s", "")
|
|
327
|
+
for key in store.keys():
|
|
328
|
+
if not (isinstance(key, str) and key.startswith(branch_prefix)):
|
|
329
|
+
continue
|
|
330
|
+
branch_name = key[len(branch_prefix) :]
|
|
331
|
+
branch_head = _resolve_head(store, branch_name)
|
|
332
|
+
if branch_head is None:
|
|
333
|
+
continue
|
|
334
|
+
for commit in walk_history(branch_head, _parent_loader, all_parents=True):
|
|
335
|
+
if commit in reachable_commits:
|
|
336
|
+
continue
|
|
337
|
+
reachable_commits.add(commit)
|
|
338
|
+
_walk_commit_for_marks(commit)
|
|
339
|
+
|
|
340
|
+
# Sweep phase: find orphaned commits via __commit_root__ scan.
|
|
341
|
+
# Also identify "young orphans" — commits inside the min_age
|
|
342
|
+
# window that aren't branch-reachable. Their chunks must be
|
|
343
|
+
# protected from sweeping (they may be in-flight from another
|
|
344
|
+
# writer), even though we won't delete the commits themselves
|
|
345
|
+
# until they age past the cutoff.
|
|
346
|
+
orphans: list[str] = []
|
|
347
|
+
young_orphan_commits: list[str] = []
|
|
348
|
+
root_prefix = COMMIT_ROOT.replace("%s", "")
|
|
349
|
+
|
|
350
|
+
for key in store.keys():
|
|
351
|
+
if not (isinstance(key, str) and key.startswith(root_prefix)):
|
|
352
|
+
continue
|
|
353
|
+
commit_hash = key[len(root_prefix) :]
|
|
354
|
+
if not commit_hash or commit_hash in reachable_commits:
|
|
355
|
+
continue
|
|
356
|
+
time_bytes = store.get(COMMIT_TIME % commit_hash)
|
|
357
|
+
if time_bytes is None:
|
|
358
|
+
# No timestamp recorded — be conservative, leave it alone.
|
|
359
|
+
continue
|
|
360
|
+
try:
|
|
361
|
+
ts_val = safe_loads(time_bytes)
|
|
362
|
+
if not isinstance(ts_val, (int, float)):
|
|
363
|
+
continue
|
|
364
|
+
if float(ts_val) < cutoff_time:
|
|
365
|
+
orphans.append(commit_hash)
|
|
366
|
+
else:
|
|
367
|
+
young_orphan_commits.append(commit_hash)
|
|
368
|
+
except (TypeError, ValueError):
|
|
369
|
+
continue
|
|
370
|
+
|
|
371
|
+
# Protect chunks referenced by young orphan commits — those
|
|
372
|
+
# may belong to in-flight writers whose CAS has not landed yet.
|
|
373
|
+
for young in young_orphan_commits:
|
|
374
|
+
_walk_commit_for_marks(young)
|
|
375
|
+
|
|
376
|
+
# Collect everything to delete in one batch so the sweep is atomic
|
|
377
|
+
# at the store level (defends against partial sweeps under crash).
|
|
378
|
+
all_removals: list[str] = []
|
|
379
|
+
|
|
380
|
+
for orphan_hash in orphans:
|
|
381
|
+
orphan_root = _load_root(store, orphan_hash)
|
|
382
|
+
if orphan_root is not None and orphan_root != EMPTY_HASH:
|
|
383
|
+
try:
|
|
384
|
+
# Batched walk for the orphan's blob references.
|
|
385
|
+
orphan_entries = Keyset(store, root=orphan_root).materialize()
|
|
386
|
+
for entry in orphan_entries.values():
|
|
387
|
+
if entry.blob not in reachable_blobs:
|
|
388
|
+
all_removals.append(entry.blob)
|
|
389
|
+
# Chunks in the orphan are only swept later via
|
|
390
|
+
# the chunk-namespace scan below; here we only
|
|
391
|
+
# need to ensure we've picked up the blob refs.
|
|
392
|
+
except Exception:
|
|
393
|
+
pass
|
|
394
|
+
all_removals.extend(
|
|
395
|
+
[
|
|
396
|
+
COMMIT_ROOT % orphan_hash,
|
|
397
|
+
PARENT_COMMIT % orphan_hash,
|
|
398
|
+
COMMIT_TIME % orphan_hash,
|
|
399
|
+
INFO_KEY % orphan_hash,
|
|
400
|
+
]
|
|
401
|
+
)
|
|
402
|
+
|
|
403
|
+
# Orphan HAMT nodes: any keyset node not reachable from a live commit
|
|
404
|
+
keyset_prefix = Keyset.DEFAULT_PREFIX
|
|
405
|
+
for key in store.keys():
|
|
406
|
+
if not (isinstance(key, str) and key.startswith(keyset_prefix)):
|
|
407
|
+
continue
|
|
408
|
+
node_hash = key[len(keyset_prefix) :]
|
|
409
|
+
if node_hash and node_hash not in reachable_nodes:
|
|
410
|
+
all_removals.append(key)
|
|
411
|
+
|
|
412
|
+
# Orphan chunks: any chunk not reachable from a live commit
|
|
413
|
+
# (or a young orphan, see above) is fair game.
|
|
414
|
+
for key in store.keys():
|
|
415
|
+
if not (isinstance(key, str) and key.startswith(CHUNK_PREFIX)):
|
|
416
|
+
continue
|
|
417
|
+
chunk_hash = key[len(CHUNK_PREFIX) :]
|
|
418
|
+
if chunk_hash and chunk_hash not in reachable_chunks:
|
|
419
|
+
all_removals.append(key)
|
|
420
|
+
|
|
421
|
+
if all_removals:
|
|
422
|
+
store.remove_many(*all_removals)
|
|
423
|
+
|
|
424
|
+
if orphans:
|
|
425
|
+
gc_logger.debug("Cleaned %d orphaned commit(s)", len(orphans))
|
|
426
|
+
|
|
427
|
+
return len(orphans)
|
|
428
|
+
|
|
429
|
+
|
|
268
430
|
class VersionedKV(VersionedBase):
|
|
269
431
|
"""A commit log over a KV store.
|
|
270
432
|
|
|
@@ -755,148 +917,14 @@ class VersionedKV(VersionedBase):
|
|
|
755
917
|
def clean_orphans(self, min_age: float = 3600) -> int:
|
|
756
918
|
"""Remove orphaned commits unreachable from any branch HEAD.
|
|
757
919
|
|
|
758
|
-
|
|
759
|
-
|
|
760
|
-
|
|
761
|
-
|
|
762
|
-
The ``min_age`` guard (default 1 hour) prevents recently
|
|
763
|
-
created commits from being falsely swept during concurrent
|
|
764
|
-
writes.
|
|
920
|
+
Thin instance wrapper over :func:`clean_orphans`, which does the
|
|
921
|
+
mark-and-sweep against ``self.store``. Kept as a method so
|
|
922
|
+
existing callers (and ``delete_branch``) read naturally.
|
|
765
923
|
|
|
766
924
|
Returns:
|
|
767
925
|
Number of orphaned commits removed.
|
|
768
926
|
"""
|
|
769
|
-
|
|
770
|
-
cutoff_time = time.time() - min_age
|
|
771
|
-
|
|
772
|
-
# Mark phase: walk every branch's history, collecting reachable
|
|
773
|
-
# commits, blob keys, HAMT node hashes, and chunk references.
|
|
774
|
-
reachable_commits: set[str] = set()
|
|
775
|
-
reachable_blobs: set[str] = set()
|
|
776
|
-
reachable_nodes: set[str] = set()
|
|
777
|
-
reachable_chunks: set[str] = set()
|
|
778
|
-
|
|
779
|
-
def _walk_commit_for_marks(commit_hash: str) -> None:
|
|
780
|
-
"""Walk one commit's keyset, accumulating reachable refs."""
|
|
781
|
-
root = _load_root(self.store, commit_hash)
|
|
782
|
-
if root is None:
|
|
783
|
-
return
|
|
784
|
-
# Single batched walk per commit collects HAMT node hashes
|
|
785
|
-
# and the entries (each carrying blob + optional chunks).
|
|
786
|
-
# ``skip_nodes`` lets us skip subtrees already seen via
|
|
787
|
-
# structural sharing — the blobs under those subtrees are
|
|
788
|
-
# already accounted for.
|
|
789
|
-
entries, new_nodes = Keyset(self.store, root=root).walk(
|
|
790
|
-
skip_nodes=reachable_nodes
|
|
791
|
-
)
|
|
792
|
-
for entry in entries.values():
|
|
793
|
-
reachable_blobs.add(entry.blob)
|
|
794
|
-
if entry.meta.chunks:
|
|
795
|
-
reachable_chunks.update(entry.meta.chunks)
|
|
796
|
-
reachable_nodes.update(new_nodes)
|
|
797
|
-
|
|
798
|
-
branch_prefix = BRANCH_HEAD.replace("%s", "")
|
|
799
|
-
for key in self.store.keys():
|
|
800
|
-
if not (isinstance(key, str) and key.startswith(branch_prefix)):
|
|
801
|
-
continue
|
|
802
|
-
branch_name = key[len(branch_prefix) :]
|
|
803
|
-
branch_head = _resolve_head(self.store, branch_name)
|
|
804
|
-
if branch_head is None:
|
|
805
|
-
continue
|
|
806
|
-
for commit in self.history(commit_hash=branch_head, all_parents=True):
|
|
807
|
-
if commit in reachable_commits:
|
|
808
|
-
continue
|
|
809
|
-
reachable_commits.add(commit)
|
|
810
|
-
_walk_commit_for_marks(commit)
|
|
811
|
-
|
|
812
|
-
# Sweep phase: find orphaned commits via __commit_root__ scan.
|
|
813
|
-
# Also identify "young orphans" — commits inside the min_age
|
|
814
|
-
# window that aren't branch-reachable. Their chunks must be
|
|
815
|
-
# protected from sweeping (they may be in-flight from another
|
|
816
|
-
# writer), even though we won't delete the commits themselves
|
|
817
|
-
# until they age past the cutoff.
|
|
818
|
-
orphans: list[str] = []
|
|
819
|
-
young_orphan_commits: list[str] = []
|
|
820
|
-
root_prefix = COMMIT_ROOT.replace("%s", "")
|
|
821
|
-
|
|
822
|
-
for key in self.store.keys():
|
|
823
|
-
if not (isinstance(key, str) and key.startswith(root_prefix)):
|
|
824
|
-
continue
|
|
825
|
-
commit_hash = key[len(root_prefix) :]
|
|
826
|
-
if not commit_hash or commit_hash in reachable_commits:
|
|
827
|
-
continue
|
|
828
|
-
time_bytes = self.store.get(COMMIT_TIME % commit_hash)
|
|
829
|
-
if time_bytes is None:
|
|
830
|
-
# No timestamp recorded — be conservative, leave it alone.
|
|
831
|
-
continue
|
|
832
|
-
try:
|
|
833
|
-
ts_val = safe_loads(time_bytes)
|
|
834
|
-
if not isinstance(ts_val, (int, float)):
|
|
835
|
-
continue
|
|
836
|
-
if float(ts_val) < cutoff_time:
|
|
837
|
-
orphans.append(commit_hash)
|
|
838
|
-
else:
|
|
839
|
-
young_orphan_commits.append(commit_hash)
|
|
840
|
-
except (TypeError, ValueError):
|
|
841
|
-
continue
|
|
842
|
-
|
|
843
|
-
# Protect chunks referenced by young orphan commits — those
|
|
844
|
-
# may belong to in-flight writers whose CAS has not landed yet.
|
|
845
|
-
for young in young_orphan_commits:
|
|
846
|
-
_walk_commit_for_marks(young)
|
|
847
|
-
|
|
848
|
-
# Collect everything to delete in one batch so the sweep is atomic
|
|
849
|
-
# at the store level (defends against partial sweeps under crash).
|
|
850
|
-
all_removals: list[str] = []
|
|
851
|
-
|
|
852
|
-
for orphan_hash in orphans:
|
|
853
|
-
orphan_root = _load_root(self.store, orphan_hash)
|
|
854
|
-
if orphan_root is not None and orphan_root != EMPTY_HASH:
|
|
855
|
-
try:
|
|
856
|
-
# Batched walk for the orphan's blob references.
|
|
857
|
-
orphan_entries = Keyset(self.store, root=orphan_root).materialize()
|
|
858
|
-
for entry in orphan_entries.values():
|
|
859
|
-
if entry.blob not in reachable_blobs:
|
|
860
|
-
all_removals.append(entry.blob)
|
|
861
|
-
# Chunks in the orphan are only swept later via
|
|
862
|
-
# the chunk-namespace scan below; here we only
|
|
863
|
-
# need to ensure we've picked up the blob refs.
|
|
864
|
-
except Exception:
|
|
865
|
-
pass
|
|
866
|
-
all_removals.extend(
|
|
867
|
-
[
|
|
868
|
-
COMMIT_ROOT % orphan_hash,
|
|
869
|
-
PARENT_COMMIT % orphan_hash,
|
|
870
|
-
COMMIT_TIME % orphan_hash,
|
|
871
|
-
INFO_KEY % orphan_hash,
|
|
872
|
-
]
|
|
873
|
-
)
|
|
874
|
-
|
|
875
|
-
# Orphan HAMT nodes: any keyset node not reachable from a live commit
|
|
876
|
-
keyset_prefix = Keyset.DEFAULT_PREFIX
|
|
877
|
-
for key in self.store.keys():
|
|
878
|
-
if not (isinstance(key, str) and key.startswith(keyset_prefix)):
|
|
879
|
-
continue
|
|
880
|
-
node_hash = key[len(keyset_prefix) :]
|
|
881
|
-
if node_hash and node_hash not in reachable_nodes:
|
|
882
|
-
all_removals.append(key)
|
|
883
|
-
|
|
884
|
-
# Orphan chunks: any chunk not reachable from a live commit
|
|
885
|
-
# (or a young orphan, see above) is fair game.
|
|
886
|
-
for key in self.store.keys():
|
|
887
|
-
if not (isinstance(key, str) and key.startswith(CHUNK_PREFIX)):
|
|
888
|
-
continue
|
|
889
|
-
chunk_hash = key[len(CHUNK_PREFIX) :]
|
|
890
|
-
if chunk_hash and chunk_hash not in reachable_chunks:
|
|
891
|
-
all_removals.append(key)
|
|
892
|
-
|
|
893
|
-
if all_removals:
|
|
894
|
-
self.store.remove_many(*all_removals)
|
|
895
|
-
|
|
896
|
-
if orphans:
|
|
897
|
-
gc_logger.debug("Cleaned %d orphaned commit(s)", len(orphans))
|
|
898
|
-
|
|
899
|
-
return len(orphans)
|
|
927
|
+
return clean_orphans(self.store, min_age)
|
|
900
928
|
|
|
901
929
|
# -- Internal --
|
|
902
930
|
|
|
@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
|
|
|
4
4
|
|
|
5
5
|
[project]
|
|
6
6
|
name = "kvgit"
|
|
7
|
-
version = "0.3.
|
|
7
|
+
version = "0.3.2"
|
|
8
8
|
description = "Versioned key-value store with git-like commit, branch, and merge semantics."
|
|
9
9
|
authors = [{ name = "ashenfad" }]
|
|
10
10
|
requires-python = ">=3.10"
|
|
@@ -0,0 +1,209 @@
|
|
|
1
|
+
"""Tests for the anchor-free admin deletion API (``kvgit.delete_branches``)
|
|
2
|
+
and the module-level ``clean_orphans`` it shares with ``VersionedKV``.
|
|
3
|
+
"""
|
|
4
|
+
|
|
5
|
+
import os
|
|
6
|
+
import tempfile
|
|
7
|
+
|
|
8
|
+
import kvgit
|
|
9
|
+
from kvgit import VersionedKV, store
|
|
10
|
+
from kvgit.kv.memory import Memory
|
|
11
|
+
from kvgit.versioned.kv import (
|
|
12
|
+
BRANCH_HEAD,
|
|
13
|
+
BRANCH_HEAD_PREV,
|
|
14
|
+
_resolve_head,
|
|
15
|
+
clean_orphans,
|
|
16
|
+
)
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
def _blob_keys(backend) -> set[str]:
|
|
20
|
+
"""Versioned blob keys (``<commit>:<user_key>``) currently in the store."""
|
|
21
|
+
return {
|
|
22
|
+
k
|
|
23
|
+
for k in backend.keys()
|
|
24
|
+
if isinstance(k, str)
|
|
25
|
+
and ":" in k
|
|
26
|
+
and not k.startswith("__")
|
|
27
|
+
and "kvgit:" not in k
|
|
28
|
+
}
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
class TestDeleteBranches:
|
|
32
|
+
def test_delete_sole_branch(self):
|
|
33
|
+
"""Deleting the store's ONLY branch is legal — the exact case the
|
|
34
|
+
instance method can't reach (no anchor left to open on)."""
|
|
35
|
+
with tempfile.TemporaryDirectory() as d:
|
|
36
|
+
p = os.path.join(d, "store")
|
|
37
|
+
s = store(kind="disk", path=p) # only branch: main
|
|
38
|
+
s["x"] = "1"
|
|
39
|
+
s.commit()
|
|
40
|
+
s.versioned.store.close()
|
|
41
|
+
|
|
42
|
+
kvgit.delete_branches(["main"], kind="disk", path=p)
|
|
43
|
+
|
|
44
|
+
# Reopening mints a fresh empty main.
|
|
45
|
+
s2 = store(kind="disk", path=p)
|
|
46
|
+
assert s2.list_branches() == ["main"]
|
|
47
|
+
assert s2.get("x") is None
|
|
48
|
+
|
|
49
|
+
def test_delete_batch(self):
|
|
50
|
+
with tempfile.TemporaryDirectory() as d:
|
|
51
|
+
p = os.path.join(d, "store")
|
|
52
|
+
s = store(kind="disk", path=p)
|
|
53
|
+
s["base"] = "ok"
|
|
54
|
+
s.commit()
|
|
55
|
+
s.create_branch("dev")["d"] = "1"
|
|
56
|
+
s.create_branch("feature")["f"] = "1"
|
|
57
|
+
assert set(s.list_branches()) == {"main", "dev", "feature"}
|
|
58
|
+
s.versioned.store.close()
|
|
59
|
+
|
|
60
|
+
kvgit.delete_branches(["dev", "feature"], kind="disk", path=p)
|
|
61
|
+
|
|
62
|
+
s2 = store(kind="disk", path=p)
|
|
63
|
+
assert s2.list_branches() == ["main"]
|
|
64
|
+
assert s2.get("base") == "ok"
|
|
65
|
+
|
|
66
|
+
def test_nonexistent_names_noop(self):
|
|
67
|
+
"""Teardown is idempotent: unknown names don't raise (unlike the
|
|
68
|
+
instance method's ValueError)."""
|
|
69
|
+
with tempfile.TemporaryDirectory() as d:
|
|
70
|
+
p = os.path.join(d, "store")
|
|
71
|
+
store(kind="disk", path=p)["x"] = "1" # materialize main
|
|
72
|
+
kvgit.delete_branches(["never", "existed"], kind="disk", path=p)
|
|
73
|
+
# main untouched
|
|
74
|
+
assert store(kind="disk", path=p).list_branches() == ["main"]
|
|
75
|
+
|
|
76
|
+
def test_delete_all_branches_legal(self):
|
|
77
|
+
with tempfile.TemporaryDirectory() as d:
|
|
78
|
+
p = os.path.join(d, "store")
|
|
79
|
+
s = store(kind="disk", path=p)
|
|
80
|
+
s.commit()
|
|
81
|
+
s.create_branch("dev")
|
|
82
|
+
s.versioned.store.close()
|
|
83
|
+
|
|
84
|
+
kvgit.delete_branches(["main", "dev"], kind="disk", path=p)
|
|
85
|
+
|
|
86
|
+
s2 = store(kind="disk", path=p)
|
|
87
|
+
assert s2.list_branches() == ["main"] # fresh empty main
|
|
88
|
+
|
|
89
|
+
def test_prev_head_cleanup_no_resurrection(self):
|
|
90
|
+
"""The prev-HEAD recovery backup must go with the head, or a
|
|
91
|
+
same-named branch created later would 'recover' deleted state
|
|
92
|
+
through _resolve_head's fallback."""
|
|
93
|
+
with tempfile.TemporaryDirectory() as d:
|
|
94
|
+
p = os.path.join(d, "store")
|
|
95
|
+
s = store(kind="disk", path=p)
|
|
96
|
+
s.commit() # main genesis
|
|
97
|
+
dev = s.create_branch("dev")
|
|
98
|
+
dev["secret"] = "deleted data"
|
|
99
|
+
dev.commit()
|
|
100
|
+
dev["more"] = "moves prev-HEAD off genesis"
|
|
101
|
+
dev.commit()
|
|
102
|
+
s.versioned.store.close()
|
|
103
|
+
|
|
104
|
+
kvgit.delete_branches(["dev"], kind="disk", path=p)
|
|
105
|
+
|
|
106
|
+
s2 = store(kind="disk", path=p, branch="dev") # recreate the name
|
|
107
|
+
assert s2.get("secret") is None
|
|
108
|
+
assert s2.get("more") is None
|
|
109
|
+
assert s2.versioned.store.get(BRANCH_HEAD_PREV % "dev") is None
|
|
110
|
+
|
|
111
|
+
def test_memory_kind_smoke(self):
|
|
112
|
+
"""kind='memory' constructs a fresh empty Memory each call, so
|
|
113
|
+
this is a no-op — but it must not raise."""
|
|
114
|
+
kvgit.delete_branches(["anything"], kind="memory")
|
|
115
|
+
|
|
116
|
+
def test_bare_string_is_one_name(self):
|
|
117
|
+
"""A bare string is treated as ONE branch name — iterated, it
|
|
118
|
+
would 'delete' each character as a branch and silently no-op
|
|
119
|
+
the real request."""
|
|
120
|
+
with tempfile.TemporaryDirectory() as d:
|
|
121
|
+
p = os.path.join(d, "store")
|
|
122
|
+
s = store(kind="disk", path=p)
|
|
123
|
+
s["base"] = "ok"
|
|
124
|
+
s.commit()
|
|
125
|
+
s.create_branch("dev")["d"] = "1"
|
|
126
|
+
s.versioned.store.close()
|
|
127
|
+
|
|
128
|
+
kvgit.delete_branches("dev", kind="disk", path=p)
|
|
129
|
+
|
|
130
|
+
s2 = store(kind="disk", path=p)
|
|
131
|
+
assert s2.list_branches() == ["main"]
|
|
132
|
+
assert s2.get("base") == "ok"
|
|
133
|
+
|
|
134
|
+
def test_empty_names_early_return(self):
|
|
135
|
+
"""An empty iterable returns before the backend opens — no
|
|
136
|
+
store directory is created as a side effect."""
|
|
137
|
+
with tempfile.TemporaryDirectory() as d:
|
|
138
|
+
p = os.path.join(d, "store")
|
|
139
|
+
kvgit.delete_branches([], kind="disk", path=p)
|
|
140
|
+
assert not os.path.exists(p)
|
|
141
|
+
|
|
142
|
+
def test_min_age_zero_reclaims_immediately(self):
|
|
143
|
+
"""min_age=0 lets an admin who knows the store is quiet reclaim
|
|
144
|
+
a just-committed branch's blobs in the same call, instead of
|
|
145
|
+
waiting out the one-hour concurrent-writer guard."""
|
|
146
|
+
with tempfile.TemporaryDirectory() as d:
|
|
147
|
+
p = os.path.join(d, "store")
|
|
148
|
+
s = store(kind="disk", path=p)
|
|
149
|
+
s["base"] = "ok"
|
|
150
|
+
s.commit()
|
|
151
|
+
dev = s.create_branch("dev")
|
|
152
|
+
dev["secret"] = "unique-blob-value"
|
|
153
|
+
dev.commit()
|
|
154
|
+
backend = s.versioned.store
|
|
155
|
+
assert {k for k in _blob_keys(backend) if k.endswith(":secret")}
|
|
156
|
+
backend.close()
|
|
157
|
+
|
|
158
|
+
kvgit.delete_branches("dev", kind="disk", path=p, min_age=0)
|
|
159
|
+
|
|
160
|
+
s2 = store(kind="disk", path=p)
|
|
161
|
+
after = {k for k in _blob_keys(s2.versioned.store) if k.endswith(":secret")}
|
|
162
|
+
assert not after # reclaimed without waiting out the guard
|
|
163
|
+
|
|
164
|
+
def test_reopen_after_delete_not_locked(self):
|
|
165
|
+
"""The disk handle is released (finally-close), so the next
|
|
166
|
+
opener on the same directory is not blocked."""
|
|
167
|
+
with tempfile.TemporaryDirectory() as d:
|
|
168
|
+
p = os.path.join(d, "store")
|
|
169
|
+
store(kind="disk", path=p).commit()
|
|
170
|
+
kvgit.delete_branches(["main"], kind="disk", path=p)
|
|
171
|
+
# Would wedge if delete_branches leaked the diskcache handle.
|
|
172
|
+
store(kind="disk", path=p).commit()
|
|
173
|
+
|
|
174
|
+
|
|
175
|
+
class TestSharedOrphanSweep:
|
|
176
|
+
def test_orphan_gc_reclaims_deleted_branch_blobs(self):
|
|
177
|
+
"""The deleted branch's unique blobs are gone after the shared
|
|
178
|
+
sweep. Uses min_age=0 to bypass the age guard on fresh commits."""
|
|
179
|
+
backend = Memory()
|
|
180
|
+
v = VersionedKV(backend) # main
|
|
181
|
+
dev = v.create_branch("dev")
|
|
182
|
+
dev.commit({"secret": b"unique-blob-value"})
|
|
183
|
+
|
|
184
|
+
# A blob key referencing the dev-only value exists.
|
|
185
|
+
before = {k for k in _blob_keys(backend) if k.endswith(":secret")}
|
|
186
|
+
assert before
|
|
187
|
+
|
|
188
|
+
# Mimic delete_branches' removals, then the shared sweep at min_age=0.
|
|
189
|
+
backend.remove(BRANCH_HEAD % "dev")
|
|
190
|
+
backend.remove(BRANCH_HEAD_PREV % "dev")
|
|
191
|
+
removed = clean_orphans(backend, min_age=0)
|
|
192
|
+
|
|
193
|
+
assert removed >= 1
|
|
194
|
+
after = {k for k in _blob_keys(backend) if k.endswith(":secret")}
|
|
195
|
+
assert not after # reclaimed
|
|
196
|
+
|
|
197
|
+
def test_module_clean_orphans_matches_instance(self):
|
|
198
|
+
"""The instance method delegates to the module function; both
|
|
199
|
+
return the orphan count for the same store."""
|
|
200
|
+
backend = Memory()
|
|
201
|
+
v = VersionedKV(backend)
|
|
202
|
+
dev = v.create_branch("dev")
|
|
203
|
+
dev.commit({"k": b"v"})
|
|
204
|
+
backend.remove(BRANCH_HEAD % "dev")
|
|
205
|
+
backend.remove(BRANCH_HEAD_PREV % "dev")
|
|
206
|
+
|
|
207
|
+
# No live branch references dev's commit anymore.
|
|
208
|
+
assert _resolve_head(backend, "dev") is None
|
|
209
|
+
assert clean_orphans(backend, min_age=0) >= 1
|
kvgit-0.3.1/kvgit/store.py
DELETED
|
@@ -1,74 +0,0 @@
|
|
|
1
|
-
"""Store factory function."""
|
|
2
|
-
|
|
3
|
-
import pickle
|
|
4
|
-
from typing import Any, Callable, Literal
|
|
5
|
-
|
|
6
|
-
from .kv.base import KVStore
|
|
7
|
-
from .kv.memory import Memory
|
|
8
|
-
from .staged import Staged
|
|
9
|
-
from .versioned.kv import VersionedKV
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
def store(
|
|
13
|
-
kind: Literal["memory", "disk", "indexeddb"] = "memory",
|
|
14
|
-
*,
|
|
15
|
-
path: str | None = None,
|
|
16
|
-
db_name: str = "kvgit",
|
|
17
|
-
branch: str = "main",
|
|
18
|
-
encoder: Callable[..., bytes] = pickle.dumps,
|
|
19
|
-
decoder: Callable[..., Any] = pickle.loads,
|
|
20
|
-
codecs: str | None = None,
|
|
21
|
-
) -> Staged:
|
|
22
|
-
"""Create a Staged store with sensible defaults.
|
|
23
|
-
|
|
24
|
-
Args:
|
|
25
|
-
kind: ``"memory"`` (default), ``"disk"``, or ``"indexeddb"``.
|
|
26
|
-
path: Required when ``kind="disk"``. Directory path for the
|
|
27
|
-
disk backend.
|
|
28
|
-
db_name: IndexedDB database name (default ``"kvgit"``).
|
|
29
|
-
Only used when ``kind="indexeddb"``.
|
|
30
|
-
branch: Branch name (default ``"main"``).
|
|
31
|
-
encoder: Value encoder (default ``pickle.dumps``).
|
|
32
|
-
decoder: Value decoder (default ``pickle.loads``).
|
|
33
|
-
codecs: Optional named codec preset. Currently supported:
|
|
34
|
-
``"scientific"`` — numpy/pandas chunked codecs (requires
|
|
35
|
-
numpy; install with ``pip install kvgit[scientific]``).
|
|
36
|
-
Mutually exclusive with explicit ``encoder`` / ``decoder``.
|
|
37
|
-
|
|
38
|
-
Returns:
|
|
39
|
-
A ``Staged`` store instance.
|
|
40
|
-
|
|
41
|
-
Raises:
|
|
42
|
-
ValueError: if ``codecs`` is given alongside non-default
|
|
43
|
-
``encoder`` / ``decoder``, or if ``codecs`` names an
|
|
44
|
-
unknown preset.
|
|
45
|
-
ImportError: if a codec preset's optional dependency is not
|
|
46
|
-
installed.
|
|
47
|
-
"""
|
|
48
|
-
if codecs is not None:
|
|
49
|
-
if encoder is not pickle.dumps or decoder is not pickle.loads:
|
|
50
|
-
raise ValueError(
|
|
51
|
-
"codecs= is mutually exclusive with explicit encoder/decoder; "
|
|
52
|
-
"pass one or the other"
|
|
53
|
-
)
|
|
54
|
-
from .codecs import _resolve_named
|
|
55
|
-
|
|
56
|
-
encoder, decoder = _resolve_named(codecs)
|
|
57
|
-
|
|
58
|
-
backend: KVStore
|
|
59
|
-
if kind == "memory":
|
|
60
|
-
backend = Memory()
|
|
61
|
-
elif kind == "disk":
|
|
62
|
-
if path is None:
|
|
63
|
-
raise ValueError("path is required when kind='disk'")
|
|
64
|
-
from .kv.disk import Disk
|
|
65
|
-
|
|
66
|
-
backend = Disk(path)
|
|
67
|
-
elif kind == "indexeddb":
|
|
68
|
-
from .kv.indexeddb import IndexedDB
|
|
69
|
-
|
|
70
|
-
backend = IndexedDB(db_name=db_name)
|
|
71
|
-
else:
|
|
72
|
-
raise ValueError(f"Unknown kind: {kind!r}")
|
|
73
|
-
|
|
74
|
-
return Staged(VersionedKV(backend, branch=branch), encoder=encoder, decoder=decoder)
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|