kvgit 0.3.1__tar.gz → 0.3.3__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {kvgit-0.3.1 → kvgit-0.3.3}/PKG-INFO +2 -2
- {kvgit-0.3.1 → kvgit-0.3.3}/kvgit/__init__.py +3 -2
- {kvgit-0.3.1 → kvgit-0.3.3}/kvgit/codecs/numpy.py +2 -2
- {kvgit-0.3.1 → kvgit-0.3.3}/kvgit/codecs/pickler.py +2 -1
- {kvgit-0.3.1 → kvgit-0.3.3}/kvgit/content_types.py +2 -1
- {kvgit-0.3.1 → kvgit-0.3.3}/kvgit/encoding.py +2 -1
- {kvgit-0.3.1 → kvgit-0.3.3}/kvgit/kv/disk.py +7 -0
- {kvgit-0.3.1 → kvgit-0.3.3}/kvgit/kv/indexeddb.py +4 -3
- {kvgit-0.3.1 → kvgit-0.3.3}/kvgit/namespaced.py +2 -2
- {kvgit-0.3.1 → kvgit-0.3.3}/kvgit/staged.py +2 -2
- kvgit-0.3.3/kvgit/store.py +157 -0
- {kvgit-0.3.1 → kvgit-0.3.3}/kvgit/versioned/base.py +1 -1
- {kvgit-0.3.1 → kvgit-0.3.3}/kvgit/versioned/helpers.py +1 -1
- {kvgit-0.3.1 → kvgit-0.3.3}/kvgit/versioned/kv.py +425 -153
- {kvgit-0.3.1 → kvgit-0.3.3}/kvgit/versioned/merge.py +3 -3
- {kvgit-0.3.1 → kvgit-0.3.3}/kvgit/versioned/protocol.py +2 -2
- {kvgit-0.3.1 → kvgit-0.3.3}/kvgit.egg-info/PKG-INFO +2 -2
- {kvgit-0.3.1 → kvgit-0.3.3}/kvgit.egg-info/SOURCES.txt +1 -0
- {kvgit-0.3.1 → kvgit-0.3.3}/kvgit.egg-info/requires.txt +1 -1
- kvgit-0.3.3/pyproject.toml +152 -0
- kvgit-0.3.3/tests/test_delete_branches.py +209 -0
- {kvgit-0.3.1 → kvgit-0.3.3}/tests/test_hamt.py +4 -5
- kvgit-0.3.1/kvgit/store.py +0 -74
- kvgit-0.3.1/pyproject.toml +0 -69
- {kvgit-0.3.1 → kvgit-0.3.3}/LICENSE +0 -0
- {kvgit-0.3.1 → kvgit-0.3.3}/README.md +0 -0
- {kvgit-0.3.1 → kvgit-0.3.3}/kvgit/codecs/__init__.py +0 -0
- {kvgit-0.3.1 → kvgit-0.3.3}/kvgit/codecs/_hash.py +0 -0
- {kvgit-0.3.1 → kvgit-0.3.3}/kvgit/codecs/base.py +0 -0
- {kvgit-0.3.1 → kvgit-0.3.3}/kvgit/codecs/pandas.py +0 -0
- {kvgit-0.3.1 → kvgit-0.3.3}/kvgit/errors.py +0 -0
- {kvgit-0.3.1 → kvgit-0.3.3}/kvgit/hamt.py +0 -0
- {kvgit-0.3.1 → kvgit-0.3.3}/kvgit/kv/__init__.py +0 -0
- {kvgit-0.3.1 → kvgit-0.3.3}/kvgit/kv/base.py +0 -0
- {kvgit-0.3.1 → kvgit-0.3.3}/kvgit/kv/composite.py +0 -0
- {kvgit-0.3.1 → kvgit-0.3.3}/kvgit/kv/memory.py +0 -0
- {kvgit-0.3.1 → kvgit-0.3.3}/kvgit/py.typed +0 -0
- {kvgit-0.3.1 → kvgit-0.3.3}/kvgit/versioned/__init__.py +0 -0
- {kvgit-0.3.1 → kvgit-0.3.3}/kvgit/versioned/keyset.py +0 -0
- {kvgit-0.3.1 → kvgit-0.3.3}/kvgit.egg-info/dependency_links.txt +0 -0
- {kvgit-0.3.1 → kvgit-0.3.3}/kvgit.egg-info/top_level.txt +0 -0
- {kvgit-0.3.1 → kvgit-0.3.3}/setup.cfg +0 -0
- {kvgit-0.3.1 → kvgit-0.3.3}/tests/test_content_types.py +0 -0
- {kvgit-0.3.1 → kvgit-0.3.3}/tests/test_namespaced.py +0 -0
- {kvgit-0.3.1 → kvgit-0.3.3}/tests/test_staged.py +0 -0
- {kvgit-0.3.1 → kvgit-0.3.3}/tests/test_store_factory.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: kvgit
|
|
3
|
-
Version: 0.3.
|
|
3
|
+
Version: 0.3.3
|
|
4
4
|
Summary: Versioned key-value store with git-like commit, branch, and merge semantics.
|
|
5
5
|
Author: ashenfad
|
|
6
6
|
License: MIT
|
|
@@ -42,7 +42,7 @@ Requires-Dist: pandas>=2.0; extra == "all"
|
|
|
42
42
|
Provides-Extra: dev
|
|
43
43
|
Requires-Dist: pytest; extra == "dev"
|
|
44
44
|
Requires-Dist: diskcache; extra == "dev"
|
|
45
|
-
Requires-Dist: ruff; extra == "dev"
|
|
45
|
+
Requires-Dist: ruff<0.17,>=0.16; extra == "dev"
|
|
46
46
|
Requires-Dist: numpy>=1.24; extra == "dev"
|
|
47
47
|
Requires-Dist: pandas>=2.0; extra == "dev"
|
|
48
48
|
Dynamic: license-file
|
|
@@ -3,10 +3,10 @@
|
|
|
3
3
|
from .content_types import MergeFn, counter, last_writer_wins
|
|
4
4
|
from .errors import ConcurrencyError, MergeConflict
|
|
5
5
|
from .namespaced import Namespaced
|
|
6
|
-
from .versioned.protocol import MergeResult, Versioned
|
|
7
6
|
from .staged import Staged
|
|
8
|
-
from .store import store
|
|
7
|
+
from .store import delete_branches, store
|
|
9
8
|
from .versioned.kv import VersionedKV
|
|
9
|
+
from .versioned.protocol import MergeResult, Versioned
|
|
10
10
|
|
|
11
11
|
__all__ = [
|
|
12
12
|
"ConcurrencyError",
|
|
@@ -18,6 +18,7 @@ __all__ = [
|
|
|
18
18
|
"Versioned",
|
|
19
19
|
"VersionedKV",
|
|
20
20
|
"counter",
|
|
21
|
+
"delete_branches",
|
|
21
22
|
"last_writer_wins",
|
|
22
23
|
"store",
|
|
23
24
|
]
|
|
@@ -33,7 +33,7 @@ storing each unique buffer once instead of N times.)
|
|
|
33
33
|
|
|
34
34
|
from __future__ import annotations
|
|
35
35
|
|
|
36
|
-
from typing import
|
|
36
|
+
from typing import TYPE_CHECKING, Any
|
|
37
37
|
|
|
38
38
|
if TYPE_CHECKING:
|
|
39
39
|
import numpy as np
|
|
@@ -66,7 +66,7 @@ def _descr_to_dtype(descr):
|
|
|
66
66
|
return npf.descr_to_dtype(descr)
|
|
67
67
|
|
|
68
68
|
|
|
69
|
-
def _root_and_offset(arr:
|
|
69
|
+
def _root_and_offset(arr: np.ndarray) -> tuple[np.ndarray, int]:
|
|
70
70
|
"""Walk ``.base`` to the root buffer; return (root, byte_offset).
|
|
71
71
|
|
|
72
72
|
Returns ``(arr, 0)`` if the .base chain leads somewhere we can't
|
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
"""Merge functions for typed values."""
|
|
2
2
|
|
|
3
|
-
from
|
|
3
|
+
from collections.abc import Callable
|
|
4
|
+
from typing import Any
|
|
4
5
|
|
|
5
6
|
MergeFn = Callable[[Any | None, Any, Any], Any]
|
|
6
7
|
"""Merge function: (old_value | None, our_value, their_value) -> merged_value.
|
|
@@ -87,3 +87,10 @@ class Disk(KVStore):
|
|
|
87
87
|
|
|
88
88
|
def clear(self) -> None:
|
|
89
89
|
self.store.clear()
|
|
90
|
+
|
|
91
|
+
def close(self) -> None:
|
|
92
|
+
# diskcache holds SQLite connections (one per thread) open until
|
|
93
|
+
# closed. Admin teardown (kvgit.delete_branches) constructs a
|
|
94
|
+
# backend just to sweep and must release the handle so the next
|
|
95
|
+
# opener on the same directory isn't blocked.
|
|
96
|
+
self.store.close()
|
|
@@ -47,8 +47,8 @@ the handler is set, causing a deadlock.
|
|
|
47
47
|
|
|
48
48
|
from collections.abc import Iterable, Mapping
|
|
49
49
|
|
|
50
|
-
from pyodide.ffi import create_proxy, run_sync, to_js # type: ignore[import-not-found]
|
|
51
50
|
from js import Promise, indexedDB, undefined # type: ignore[import-not-found]
|
|
51
|
+
from pyodide.ffi import create_proxy, run_sync, to_js # type: ignore[import-not-found]
|
|
52
52
|
|
|
53
53
|
from .base import KVStore
|
|
54
54
|
|
|
@@ -155,8 +155,9 @@ def _to_bytes(js_value) -> bytes | None:
|
|
|
155
155
|
return None
|
|
156
156
|
try:
|
|
157
157
|
return js_value.to_py().tobytes()
|
|
158
|
-
except Exception:
|
|
159
|
-
#
|
|
158
|
+
except Exception: # noqa: BLE001 — a JS->Python conversion can fail
|
|
159
|
+
# with anything the FFI decides to raise; there is no narrower
|
|
160
|
+
# type to name. Corrupted or unexpected JS value — treat as missing.
|
|
160
161
|
return None
|
|
161
162
|
|
|
162
163
|
|
|
@@ -2,8 +2,8 @@
|
|
|
2
2
|
|
|
3
3
|
from __future__ import annotations
|
|
4
4
|
|
|
5
|
-
from collections.abc import Iterator, MutableMapping
|
|
6
|
-
from typing import Any
|
|
5
|
+
from collections.abc import Iterable, Iterator, MutableMapping
|
|
6
|
+
from typing import Any
|
|
7
7
|
|
|
8
8
|
|
|
9
9
|
class Namespaced(MutableMapping[str, Any]):
|
|
@@ -2,8 +2,8 @@
|
|
|
2
2
|
|
|
3
3
|
import inspect
|
|
4
4
|
import pickle
|
|
5
|
-
from collections.abc import Iterable, Iterator, MutableMapping
|
|
6
|
-
from typing import Any
|
|
5
|
+
from collections.abc import Callable, Iterable, Iterator, MutableMapping
|
|
6
|
+
from typing import Any
|
|
7
7
|
|
|
8
8
|
from .codecs._hash import hash_bytes
|
|
9
9
|
from .content_types import MergeFn
|
|
@@ -0,0 +1,157 @@
|
|
|
1
|
+
"""Store factory function."""
|
|
2
|
+
|
|
3
|
+
import pickle
|
|
4
|
+
from collections.abc import Callable, Iterable
|
|
5
|
+
from typing import Any, Literal
|
|
6
|
+
|
|
7
|
+
from .kv.base import KVStore
|
|
8
|
+
from .kv.memory import Memory
|
|
9
|
+
from .staged import Staged
|
|
10
|
+
from .versioned.kv import BRANCH_HEAD, BRANCH_HEAD_PREV, VersionedKV, clean_orphans
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
def _make_backend(
|
|
14
|
+
kind: Literal["memory", "disk", "indexeddb"],
|
|
15
|
+
*,
|
|
16
|
+
path: str | None,
|
|
17
|
+
db_name: str,
|
|
18
|
+
) -> KVStore:
|
|
19
|
+
"""Construct a raw ``KVStore`` backend by kind.
|
|
20
|
+
|
|
21
|
+
Shared by :func:`store` and :func:`delete_branches` so both open the
|
|
22
|
+
exact same backends from the same parameters.
|
|
23
|
+
"""
|
|
24
|
+
if kind == "memory":
|
|
25
|
+
return Memory()
|
|
26
|
+
elif kind == "disk":
|
|
27
|
+
if path is None:
|
|
28
|
+
raise ValueError("path is required when kind='disk'")
|
|
29
|
+
from .kv.disk import Disk
|
|
30
|
+
|
|
31
|
+
return Disk(path)
|
|
32
|
+
elif kind == "indexeddb":
|
|
33
|
+
from .kv.indexeddb import IndexedDB
|
|
34
|
+
|
|
35
|
+
return IndexedDB(db_name=db_name)
|
|
36
|
+
else:
|
|
37
|
+
raise ValueError(f"Unknown kind: {kind!r}")
|
|
38
|
+
|
|
39
|
+
|
|
40
|
+
def store(
|
|
41
|
+
kind: Literal["memory", "disk", "indexeddb"] = "memory",
|
|
42
|
+
*,
|
|
43
|
+
path: str | None = None,
|
|
44
|
+
db_name: str = "kvgit",
|
|
45
|
+
branch: str = "main",
|
|
46
|
+
encoder: Callable[..., bytes] = pickle.dumps,
|
|
47
|
+
decoder: Callable[..., Any] = pickle.loads,
|
|
48
|
+
codecs: str | None = None,
|
|
49
|
+
) -> Staged:
|
|
50
|
+
"""Create a Staged store with sensible defaults.
|
|
51
|
+
|
|
52
|
+
Args:
|
|
53
|
+
kind: ``"memory"`` (default), ``"disk"``, or ``"indexeddb"``.
|
|
54
|
+
path: Required when ``kind="disk"``. Directory path for the
|
|
55
|
+
disk backend.
|
|
56
|
+
db_name: IndexedDB database name (default ``"kvgit"``).
|
|
57
|
+
Only used when ``kind="indexeddb"``.
|
|
58
|
+
branch: Branch name (default ``"main"``).
|
|
59
|
+
encoder: Value encoder (default ``pickle.dumps``).
|
|
60
|
+
decoder: Value decoder (default ``pickle.loads``).
|
|
61
|
+
codecs: Optional named codec preset. Currently supported:
|
|
62
|
+
``"scientific"`` — numpy/pandas chunked codecs (requires
|
|
63
|
+
numpy; install with ``pip install kvgit[scientific]``).
|
|
64
|
+
Mutually exclusive with explicit ``encoder`` / ``decoder``.
|
|
65
|
+
|
|
66
|
+
Returns:
|
|
67
|
+
A ``Staged`` store instance.
|
|
68
|
+
|
|
69
|
+
Raises:
|
|
70
|
+
ValueError: if ``codecs`` is given alongside non-default
|
|
71
|
+
``encoder`` / ``decoder``, or if ``codecs`` names an
|
|
72
|
+
unknown preset.
|
|
73
|
+
ImportError: if a codec preset's optional dependency is not
|
|
74
|
+
installed.
|
|
75
|
+
"""
|
|
76
|
+
if codecs is not None:
|
|
77
|
+
if encoder is not pickle.dumps or decoder is not pickle.loads:
|
|
78
|
+
raise ValueError(
|
|
79
|
+
"codecs= is mutually exclusive with explicit encoder/decoder; "
|
|
80
|
+
"pass one or the other"
|
|
81
|
+
)
|
|
82
|
+
from .codecs import _resolve_named
|
|
83
|
+
|
|
84
|
+
encoder, decoder = _resolve_named(codecs)
|
|
85
|
+
|
|
86
|
+
backend = _make_backend(kind, path=path, db_name=db_name)
|
|
87
|
+
|
|
88
|
+
return Staged(VersionedKV(backend, branch=branch), encoder=encoder, decoder=decoder)
|
|
89
|
+
|
|
90
|
+
|
|
91
|
+
def delete_branches(
|
|
92
|
+
names: str | Iterable[str],
|
|
93
|
+
*,
|
|
94
|
+
kind: Literal["memory", "disk", "indexeddb"] = "disk",
|
|
95
|
+
path: str | None = None,
|
|
96
|
+
db_name: str = "kvgit",
|
|
97
|
+
min_age: float = 3600,
|
|
98
|
+
) -> None:
|
|
99
|
+
"""Delete branches with no branch anchor, then sweep orphans.
|
|
100
|
+
|
|
101
|
+
:meth:`VersionedKV.delete_branch` refuses to delete the branch its
|
|
102
|
+
handle is anchored on — and there is no way to open a handle
|
|
103
|
+
*without* a branch. When the branch you want gone is the store's
|
|
104
|
+
only branch, there is nothing safe to anchor on. This admin entry
|
|
105
|
+
point sidesteps the whole problem: it opens the backend directly
|
|
106
|
+
(no ``VersionedKV``, so no current branch) and edits the branch
|
|
107
|
+
keys itself.
|
|
108
|
+
|
|
109
|
+
For each name, ``__branch_head__<name>`` and its
|
|
110
|
+
``__branch_head_prev__<name>`` recovery backup are removed — the
|
|
111
|
+
backup too, so a later same-named branch can't "recover" the
|
|
112
|
+
deleted state through ``_resolve_head``'s fallback. A name with no
|
|
113
|
+
head is simply skipped: this is teardown, and idempotency beats a
|
|
114
|
+
``ValueError`` here. Deleting every branch is legal — the store
|
|
115
|
+
just mints a fresh empty ``main`` on next open.
|
|
116
|
+
|
|
117
|
+
One :func:`clean_orphans` sweep runs after all removals, so commits
|
|
118
|
+
that only the deleted branches referenced are reclaimed. All heads
|
|
119
|
+
(and prev-heads) are gone before the sweep, so it can't mistake a
|
|
120
|
+
doomed branch for a live one. ``min_age`` defaults to
|
|
121
|
+
``clean_orphans``'s one-hour guard (matching ``delete_branch``), so
|
|
122
|
+
an in-flight commit from a concurrent writer on another branch is
|
|
123
|
+
not swept out from under it; an admin who knows the store is quiet
|
|
124
|
+
can pass ``0`` to reclaim young commits immediately.
|
|
125
|
+
|
|
126
|
+
That sweep does not reclaim chunks — the content-addressed bytes a
|
|
127
|
+
chunked codec writes — because a chunk key is not the deleted
|
|
128
|
+
branch's to give away. On a store using chunked codecs, follow up
|
|
129
|
+
with ``deep_clean`` on a quiescent store to get that space back.
|
|
130
|
+
|
|
131
|
+
Args:
|
|
132
|
+
names: Branch names to delete — one name or an iterable of
|
|
133
|
+
them. Missing names are no-ops. (A bare string is treated
|
|
134
|
+
as ONE name, not iterated per character.)
|
|
135
|
+
kind: ``"disk"`` (default), ``"memory"``, or ``"indexeddb"``.
|
|
136
|
+
path: Required when ``kind="disk"``. Store directory.
|
|
137
|
+
db_name: IndexedDB database name (``kind="indexeddb"`` only).
|
|
138
|
+
min_age: Passed to :func:`clean_orphans` — commits younger than
|
|
139
|
+
this many seconds survive the sweep.
|
|
140
|
+
"""
|
|
141
|
+
# A bare string is a plausible one-branch call, and iterating it
|
|
142
|
+
# would "delete" each character as a branch name — a silent no-op
|
|
143
|
+
# at best. Treat it as a single name, like nontainer's
|
|
144
|
+
# delete_workspace does one layer up.
|
|
145
|
+
doomed = {names} if isinstance(names, str) else set(names)
|
|
146
|
+
if not doomed:
|
|
147
|
+
return # nothing asked: skip the backend open and the sweep
|
|
148
|
+
backend = _make_backend(kind, path=path, db_name=db_name)
|
|
149
|
+
try:
|
|
150
|
+
for name in doomed:
|
|
151
|
+
backend.remove(BRANCH_HEAD % name)
|
|
152
|
+
backend.remove(BRANCH_HEAD_PREV % name)
|
|
153
|
+
clean_orphans(backend, min_age=min_age)
|
|
154
|
+
finally:
|
|
155
|
+
close = getattr(backend, "close", None)
|
|
156
|
+
if callable(close):
|
|
157
|
+
close()
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
"""Shared commit/merge orchestration for versioned stores."""
|
|
2
2
|
|
|
3
3
|
from abc import ABC, abstractmethod
|
|
4
|
-
from
|
|
4
|
+
from collections.abc import Iterable
|
|
5
5
|
|
|
6
6
|
from ..errors import ConcurrencyError, MergeConflict
|
|
7
7
|
from .helpers import diff_keysets, walk_history
|