kvgit 0.3.0__tar.gz → 0.3.1__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {kvgit-0.3.0 → kvgit-0.3.1}/PKG-INFO +3 -7
- {kvgit-0.3.0 → kvgit-0.3.1}/README.md +2 -2
- {kvgit-0.3.0 → kvgit-0.3.1}/kvgit/__init__.py +0 -7
- kvgit-0.3.1/kvgit/kv/composite.py +191 -0
- {kvgit-0.3.0 → kvgit-0.3.1}/kvgit/staged.py +1 -2
- {kvgit-0.3.0 → kvgit-0.3.1}/kvgit/store.py +17 -30
- {kvgit-0.3.0 → kvgit-0.3.1}/kvgit/versioned/__init__.py +0 -7
- {kvgit-0.3.0 → kvgit-0.3.1}/kvgit/versioned/kv.py +5 -0
- {kvgit-0.3.0 → kvgit-0.3.1}/kvgit/versioned/merge.py +2 -3
- {kvgit-0.3.0 → kvgit-0.3.1}/kvgit/versioned/protocol.py +1 -2
- {kvgit-0.3.0 → kvgit-0.3.1}/kvgit.egg-info/PKG-INFO +3 -7
- {kvgit-0.3.0 → kvgit-0.3.1}/kvgit.egg-info/SOURCES.txt +0 -1
- {kvgit-0.3.0 → kvgit-0.3.1}/kvgit.egg-info/requires.txt +0 -5
- {kvgit-0.3.0 → kvgit-0.3.1}/pyproject.toml +3 -5
- kvgit-0.3.0/kvgit/kv/composite.py +0 -137
- kvgit-0.3.0/kvgit/versioned/gp.py +0 -360
- {kvgit-0.3.0 → kvgit-0.3.1}/LICENSE +0 -0
- {kvgit-0.3.0 → kvgit-0.3.1}/kvgit/codecs/__init__.py +0 -0
- {kvgit-0.3.0 → kvgit-0.3.1}/kvgit/codecs/_hash.py +0 -0
- {kvgit-0.3.0 → kvgit-0.3.1}/kvgit/codecs/base.py +0 -0
- {kvgit-0.3.0 → kvgit-0.3.1}/kvgit/codecs/numpy.py +0 -0
- {kvgit-0.3.0 → kvgit-0.3.1}/kvgit/codecs/pandas.py +0 -0
- {kvgit-0.3.0 → kvgit-0.3.1}/kvgit/codecs/pickler.py +0 -0
- {kvgit-0.3.0 → kvgit-0.3.1}/kvgit/content_types.py +0 -0
- {kvgit-0.3.0 → kvgit-0.3.1}/kvgit/encoding.py +0 -0
- {kvgit-0.3.0 → kvgit-0.3.1}/kvgit/errors.py +0 -0
- {kvgit-0.3.0 → kvgit-0.3.1}/kvgit/hamt.py +0 -0
- {kvgit-0.3.0 → kvgit-0.3.1}/kvgit/kv/__init__.py +0 -0
- {kvgit-0.3.0 → kvgit-0.3.1}/kvgit/kv/base.py +0 -0
- {kvgit-0.3.0 → kvgit-0.3.1}/kvgit/kv/disk.py +0 -0
- {kvgit-0.3.0 → kvgit-0.3.1}/kvgit/kv/indexeddb.py +0 -0
- {kvgit-0.3.0 → kvgit-0.3.1}/kvgit/kv/memory.py +0 -0
- {kvgit-0.3.0 → kvgit-0.3.1}/kvgit/namespaced.py +0 -0
- {kvgit-0.3.0 → kvgit-0.3.1}/kvgit/py.typed +0 -0
- {kvgit-0.3.0 → kvgit-0.3.1}/kvgit/versioned/base.py +0 -0
- {kvgit-0.3.0 → kvgit-0.3.1}/kvgit/versioned/helpers.py +0 -0
- {kvgit-0.3.0 → kvgit-0.3.1}/kvgit/versioned/keyset.py +0 -0
- {kvgit-0.3.0 → kvgit-0.3.1}/kvgit.egg-info/dependency_links.txt +0 -0
- {kvgit-0.3.0 → kvgit-0.3.1}/kvgit.egg-info/top_level.txt +0 -0
- {kvgit-0.3.0 → kvgit-0.3.1}/setup.cfg +0 -0
- {kvgit-0.3.0 → kvgit-0.3.1}/tests/test_content_types.py +0 -0
- {kvgit-0.3.0 → kvgit-0.3.1}/tests/test_hamt.py +0 -0
- {kvgit-0.3.0 → kvgit-0.3.1}/tests/test_namespaced.py +0 -0
- {kvgit-0.3.0 → kvgit-0.3.1}/tests/test_staged.py +0 -0
- {kvgit-0.3.0 → kvgit-0.3.1}/tests/test_store_factory.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: kvgit
|
|
3
|
-
Version: 0.3.
|
|
3
|
+
Version: 0.3.1
|
|
4
4
|
Summary: Versioned key-value store with git-like commit, branch, and merge semantics.
|
|
5
5
|
Author: ashenfad
|
|
6
6
|
License: MIT
|
|
@@ -27,8 +27,6 @@ Description-Content-Type: text/markdown
|
|
|
27
27
|
License-File: LICENSE
|
|
28
28
|
Provides-Extra: disk
|
|
29
29
|
Requires-Dist: diskcache; extra == "disk"
|
|
30
|
-
Provides-Extra: git
|
|
31
|
-
Requires-Dist: gitpython>=3.1; extra == "git"
|
|
32
30
|
Provides-Extra: numpy
|
|
33
31
|
Requires-Dist: numpy>=1.24; extra == "numpy"
|
|
34
32
|
Provides-Extra: pandas
|
|
@@ -39,13 +37,11 @@ Requires-Dist: numpy>=1.24; extra == "scientific"
|
|
|
39
37
|
Requires-Dist: pandas>=2.0; extra == "scientific"
|
|
40
38
|
Provides-Extra: all
|
|
41
39
|
Requires-Dist: diskcache; extra == "all"
|
|
42
|
-
Requires-Dist: gitpython>=3.1; extra == "all"
|
|
43
40
|
Requires-Dist: numpy>=1.24; extra == "all"
|
|
44
41
|
Requires-Dist: pandas>=2.0; extra == "all"
|
|
45
42
|
Provides-Extra: dev
|
|
46
43
|
Requires-Dist: pytest; extra == "dev"
|
|
47
44
|
Requires-Dist: diskcache; extra == "dev"
|
|
48
|
-
Requires-Dist: gitpython>=3.1; extra == "dev"
|
|
49
45
|
Requires-Dist: ruff; extra == "dev"
|
|
50
46
|
Requires-Dist: numpy>=1.24; extra == "dev"
|
|
51
47
|
Requires-Dist: pandas>=2.0; extra == "dev"
|
|
@@ -61,7 +57,7 @@ Git-style versioning for your data. Commits, branches, and merges -- backed by a
|
|
|
61
57
|
| **Commits** | Immutable, content-addressable snapshots with rollback |
|
|
62
58
|
| **Branches** | Cheap forks with CAS-based optimistic concurrency |
|
|
63
59
|
| **Three-way merge** | Auto-merges non-overlapping changes; pluggable merge fns for conflicts |
|
|
64
|
-
| **Pluggable backends** | In-memory, disk (diskcache),
|
|
60
|
+
| **Pluggable backends** | In-memory, disk (diskcache), IndexedDB (Pyodide/browser), or bring your own `KVStore` |
|
|
65
61
|
| **Chunked codecs** | Optional content-addressed dedup for large numpy arrays and pandas DataFrames -- equal buffers stored once across keys, commits, and branches |
|
|
66
62
|
|
|
67
63
|
## Install
|
|
@@ -69,7 +65,6 @@ Git-style versioning for your data. Commits, branches, and merges -- backed by a
|
|
|
69
65
|
```bash
|
|
70
66
|
pip install kvgit # in-memory only
|
|
71
67
|
pip install kvgit[disk] # adds disk backend via diskcache
|
|
72
|
-
pip install kvgit[git] # adds git backend via GitPython (requires git on PATH)
|
|
73
68
|
pip install kvgit[scientific] # adds chunked codecs for numpy / pandas
|
|
74
69
|
# IndexedDB backend is available automatically in Pyodide (browser) environments
|
|
75
70
|
```
|
|
@@ -131,3 +126,4 @@ See [`docs/`](docs/) for detailed documentation:
|
|
|
131
126
|
|
|
132
127
|
- [Quick Start](docs/quick-start.md) -- common patterns with runnable examples
|
|
133
128
|
- [API Reference](docs/api.md) -- full reference for all classes, methods, and types
|
|
129
|
+
- [Browser persistence (Pyodide)](docs/pyodide.md) -- choosing between the IndexedDB and OPFS-mounted-disk backends, plus the syncfs flush requirement and recommended host-side patterns
|
|
@@ -8,7 +8,7 @@ Git-style versioning for your data. Commits, branches, and merges -- backed by a
|
|
|
8
8
|
| **Commits** | Immutable, content-addressable snapshots with rollback |
|
|
9
9
|
| **Branches** | Cheap forks with CAS-based optimistic concurrency |
|
|
10
10
|
| **Three-way merge** | Auto-merges non-overlapping changes; pluggable merge fns for conflicts |
|
|
11
|
-
| **Pluggable backends** | In-memory, disk (diskcache),
|
|
11
|
+
| **Pluggable backends** | In-memory, disk (diskcache), IndexedDB (Pyodide/browser), or bring your own `KVStore` |
|
|
12
12
|
| **Chunked codecs** | Optional content-addressed dedup for large numpy arrays and pandas DataFrames -- equal buffers stored once across keys, commits, and branches |
|
|
13
13
|
|
|
14
14
|
## Install
|
|
@@ -16,7 +16,6 @@ Git-style versioning for your data. Commits, branches, and merges -- backed by a
|
|
|
16
16
|
```bash
|
|
17
17
|
pip install kvgit # in-memory only
|
|
18
18
|
pip install kvgit[disk] # adds disk backend via diskcache
|
|
19
|
-
pip install kvgit[git] # adds git backend via GitPython (requires git on PATH)
|
|
20
19
|
pip install kvgit[scientific] # adds chunked codecs for numpy / pandas
|
|
21
20
|
# IndexedDB backend is available automatically in Pyodide (browser) environments
|
|
22
21
|
```
|
|
@@ -78,3 +77,4 @@ See [`docs/`](docs/) for detailed documentation:
|
|
|
78
77
|
|
|
79
78
|
- [Quick Start](docs/quick-start.md) -- common patterns with runnable examples
|
|
80
79
|
- [API Reference](docs/api.md) -- full reference for all classes, methods, and types
|
|
80
|
+
- [Browser persistence (Pyodide)](docs/pyodide.md) -- choosing between the IndexedDB and OPFS-mounted-disk backends, plus the syncfs flush requirement and recommended host-side patterns
|
|
@@ -0,0 +1,191 @@
|
|
|
1
|
+
"""N-tier composite cache over multiple KV stores."""
|
|
2
|
+
|
|
3
|
+
import logging
|
|
4
|
+
from collections.abc import Iterable, Mapping
|
|
5
|
+
|
|
6
|
+
from .base import KVStore
|
|
7
|
+
|
|
8
|
+
logger = logging.getLogger("kvgit.kv.composite")
|
|
9
|
+
|
|
10
|
+
# Exceptions we treat as programming bugs (a misconfigured tier, a
|
|
11
|
+
# protocol mismatch) rather than operational tier unavailability.
|
|
12
|
+
# These are re-raised so they surface instead of being silently masked
|
|
13
|
+
# by the cache-fallback machinery. Everything else under Exception
|
|
14
|
+
# (OSError, ConnectionError, Pyodide JsException, ...) is logged and
|
|
15
|
+
# treated as "tier unavailable, try next".
|
|
16
|
+
_BUG_EXCEPTIONS = (TypeError, AttributeError, AssertionError)
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
def _is_bug(exc: BaseException) -> bool:
|
|
20
|
+
return isinstance(exc, _BUG_EXCEPTIONS)
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
class Composite(KVStore):
|
|
24
|
+
"""N-tier cache composing any number of KV stores.
|
|
25
|
+
|
|
26
|
+
On get: check L1, L2, ..., Ln in order. On hit at tier i,
|
|
27
|
+
populate L1..L(i-1) and return.
|
|
28
|
+
|
|
29
|
+
On set: write to all tiers (most durable first).
|
|
30
|
+
|
|
31
|
+
On cas: delegate to Ln (authoritative), update caches on success.
|
|
32
|
+
|
|
33
|
+
Tier failures (``OSError``, network errors, etc.) are logged at
|
|
34
|
+
WARNING and the next tier is tried; programming-error exceptions
|
|
35
|
+
(``TypeError``, ``AttributeError``, ``AssertionError``) propagate
|
|
36
|
+
so they aren't silently swallowed.
|
|
37
|
+
|
|
38
|
+
Args:
|
|
39
|
+
stores: List of KV stores ordered fastest -> most durable.
|
|
40
|
+
"""
|
|
41
|
+
|
|
42
|
+
def __init__(self, stores: list[KVStore]) -> None:
|
|
43
|
+
if not stores:
|
|
44
|
+
raise ValueError("Composite requires at least one store")
|
|
45
|
+
self._stores = stores
|
|
46
|
+
|
|
47
|
+
def _populate_caches(self, upto: int, items: Mapping[str, bytes]) -> None:
|
|
48
|
+
"""Best-effort write to faster tiers after a slow-tier hit."""
|
|
49
|
+
for j in range(upto):
|
|
50
|
+
try:
|
|
51
|
+
self._stores[j].set_many(items)
|
|
52
|
+
except Exception as e:
|
|
53
|
+
if _is_bug(e):
|
|
54
|
+
raise
|
|
55
|
+
logger.warning("Composite cache populate failed at tier %d: %s", j, e)
|
|
56
|
+
|
|
57
|
+
def get(self, key: str) -> bytes | None:
|
|
58
|
+
for i, store in enumerate(self._stores):
|
|
59
|
+
try:
|
|
60
|
+
value = store.get(key)
|
|
61
|
+
except Exception as e:
|
|
62
|
+
if _is_bug(e):
|
|
63
|
+
raise
|
|
64
|
+
logger.warning("Composite get failed at tier %d for %r: %s", i, key, e)
|
|
65
|
+
continue
|
|
66
|
+
if value is not None:
|
|
67
|
+
if i > 0:
|
|
68
|
+
self._populate_caches(i, {key: value})
|
|
69
|
+
return value
|
|
70
|
+
return None
|
|
71
|
+
|
|
72
|
+
def get_many(self, *args) -> Mapping[str, bytes]:
|
|
73
|
+
result: dict[str, bytes] = {}
|
|
74
|
+
remaining = set(self._normalize_keys(args))
|
|
75
|
+
for i, store in enumerate(self._stores):
|
|
76
|
+
if not remaining:
|
|
77
|
+
break
|
|
78
|
+
try:
|
|
79
|
+
# Delegate to the tier's bulk get — backends with high
|
|
80
|
+
# per-call latency (Disk, IndexedDB) collapse N round-trips
|
|
81
|
+
# into one. The protocol guarantees only existing keys
|
|
82
|
+
# appear in the result.
|
|
83
|
+
tier_values = store.get_many(remaining)
|
|
84
|
+
except Exception as e:
|
|
85
|
+
if _is_bug(e):
|
|
86
|
+
raise
|
|
87
|
+
logger.warning("Composite get_many failed at tier %d: %s", i, e)
|
|
88
|
+
continue
|
|
89
|
+
if tier_values and i > 0:
|
|
90
|
+
self._populate_caches(i, tier_values)
|
|
91
|
+
result.update(tier_values)
|
|
92
|
+
remaining -= tier_values.keys()
|
|
93
|
+
return result
|
|
94
|
+
|
|
95
|
+
def __contains__(self, key: str) -> bool:
|
|
96
|
+
for i, store in enumerate(self._stores):
|
|
97
|
+
try:
|
|
98
|
+
if key in store:
|
|
99
|
+
return True
|
|
100
|
+
except Exception as e:
|
|
101
|
+
if _is_bug(e):
|
|
102
|
+
raise
|
|
103
|
+
logger.warning(
|
|
104
|
+
"Composite contains failed at tier %d for %r: %s", i, key, e
|
|
105
|
+
)
|
|
106
|
+
continue
|
|
107
|
+
return False
|
|
108
|
+
|
|
109
|
+
def keys(self) -> Iterable[str]:
|
|
110
|
+
return self._stores[-1].keys()
|
|
111
|
+
|
|
112
|
+
def items(self) -> Iterable[tuple[str, bytes]]:
|
|
113
|
+
return self._stores[-1].items()
|
|
114
|
+
|
|
115
|
+
def set(self, key: str, value: bytes) -> None:
|
|
116
|
+
# Authoritative tier first; failures here propagate (durability
|
|
117
|
+
# is the contract of set()). Cache-tier failures are logged.
|
|
118
|
+
self._stores[-1].set(key, value)
|
|
119
|
+
for i, store in enumerate(self._stores[:-1]):
|
|
120
|
+
try:
|
|
121
|
+
store.set(key, value)
|
|
122
|
+
except Exception as e:
|
|
123
|
+
if _is_bug(e):
|
|
124
|
+
raise
|
|
125
|
+
logger.warning("Composite set failed at tier %d for %r: %s", i, key, e)
|
|
126
|
+
|
|
127
|
+
def set_many(
|
|
128
|
+
self,
|
|
129
|
+
items: Mapping[str, bytes] | None = None,
|
|
130
|
+
/,
|
|
131
|
+
**kwargs: bytes,
|
|
132
|
+
) -> None:
|
|
133
|
+
items = self._normalize_items(items, kwargs)
|
|
134
|
+
self._stores[-1].set_many(items)
|
|
135
|
+
for i, store in enumerate(self._stores[:-1]):
|
|
136
|
+
try:
|
|
137
|
+
store.set_many(items)
|
|
138
|
+
except Exception as e:
|
|
139
|
+
if _is_bug(e):
|
|
140
|
+
raise
|
|
141
|
+
logger.warning("Composite set_many failed at tier %d: %s", i, e)
|
|
142
|
+
|
|
143
|
+
def remove(self, key: str) -> None:
|
|
144
|
+
self._stores[-1].remove(key)
|
|
145
|
+
for i, store in enumerate(self._stores[:-1]):
|
|
146
|
+
try:
|
|
147
|
+
store.remove(key)
|
|
148
|
+
except Exception as e:
|
|
149
|
+
if _is_bug(e):
|
|
150
|
+
raise
|
|
151
|
+
logger.warning(
|
|
152
|
+
"Composite remove failed at tier %d for %r: %s", i, key, e
|
|
153
|
+
)
|
|
154
|
+
|
|
155
|
+
def remove_many(self, *args) -> None:
|
|
156
|
+
keys = list(self._normalize_keys(args))
|
|
157
|
+
self._stores[-1].remove_many(keys)
|
|
158
|
+
for i, store in enumerate(self._stores[:-1]):
|
|
159
|
+
try:
|
|
160
|
+
store.remove_many(keys)
|
|
161
|
+
except Exception as e:
|
|
162
|
+
if _is_bug(e):
|
|
163
|
+
raise
|
|
164
|
+
logger.warning("Composite remove_many failed at tier %d: %s", i, e)
|
|
165
|
+
|
|
166
|
+
def clear(self) -> None:
|
|
167
|
+
self._stores[-1].clear()
|
|
168
|
+
for i, store in enumerate(self._stores[:-1]):
|
|
169
|
+
try:
|
|
170
|
+
store.clear()
|
|
171
|
+
except Exception as e:
|
|
172
|
+
if _is_bug(e):
|
|
173
|
+
raise
|
|
174
|
+
logger.warning("Composite clear failed at tier %d: %s", i, e)
|
|
175
|
+
|
|
176
|
+
def cas(self, key: str, value: bytes, expected: bytes | None) -> bool:
|
|
177
|
+
success = self._stores[-1].cas(key, value, expected)
|
|
178
|
+
if success:
|
|
179
|
+
for i, store in enumerate(self._stores[:-1]):
|
|
180
|
+
try:
|
|
181
|
+
store.set(key, value)
|
|
182
|
+
except Exception as e:
|
|
183
|
+
if _is_bug(e):
|
|
184
|
+
raise
|
|
185
|
+
logger.warning(
|
|
186
|
+
"Composite cas cache-update failed at tier %d for %r: %s",
|
|
187
|
+
i,
|
|
188
|
+
key,
|
|
189
|
+
e,
|
|
190
|
+
)
|
|
191
|
+
return success
|
|
@@ -126,8 +126,7 @@ class Staged(MutableMapping[str, Any]):
|
|
|
126
126
|
"chunked encoder/decoder requires a VersionedKV-backed Staged "
|
|
127
127
|
f"(got {type(versioned).__name__}). "
|
|
128
128
|
"Use kvgit.store(kind='memory'/'disk'/'indexeddb') or build a "
|
|
129
|
-
"VersionedKV directly.
|
|
130
|
-
"support chunked codecs."
|
|
129
|
+
"VersionedKV directly."
|
|
131
130
|
)
|
|
132
131
|
self._chunk_reader = (
|
|
133
132
|
_ChunkReader(versioned.store) if self._decoder_chunked else None
|
|
@@ -7,11 +7,10 @@ from .kv.base import KVStore
|
|
|
7
7
|
from .kv.memory import Memory
|
|
8
8
|
from .staged import Staged
|
|
9
9
|
from .versioned.kv import VersionedKV
|
|
10
|
-
from .versioned.protocol import Versioned
|
|
11
10
|
|
|
12
11
|
|
|
13
12
|
def store(
|
|
14
|
-
kind: Literal["memory", "disk", "
|
|
13
|
+
kind: Literal["memory", "disk", "indexeddb"] = "memory",
|
|
15
14
|
*,
|
|
16
15
|
path: str | None = None,
|
|
17
16
|
db_name: str = "kvgit",
|
|
@@ -23,9 +22,9 @@ def store(
|
|
|
23
22
|
"""Create a Staged store with sensible defaults.
|
|
24
23
|
|
|
25
24
|
Args:
|
|
26
|
-
kind: ``"memory"`` (default), ``"disk"``,
|
|
27
|
-
path: Required when ``kind="disk"
|
|
28
|
-
|
|
25
|
+
kind: ``"memory"`` (default), ``"disk"``, or ``"indexeddb"``.
|
|
26
|
+
path: Required when ``kind="disk"``. Directory path for the
|
|
27
|
+
disk backend.
|
|
29
28
|
db_name: IndexedDB database name (default ``"kvgit"``).
|
|
30
29
|
Only used when ``kind="indexeddb"``.
|
|
31
30
|
branch: Branch name (default ``"main"``).
|
|
@@ -56,32 +55,20 @@ def store(
|
|
|
56
55
|
|
|
57
56
|
encoder, decoder = _resolve_named(codecs)
|
|
58
57
|
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
58
|
+
backend: KVStore
|
|
59
|
+
if kind == "memory":
|
|
60
|
+
backend = Memory()
|
|
61
|
+
elif kind == "disk":
|
|
62
62
|
if path is None:
|
|
63
|
-
raise ValueError("path is required when kind='
|
|
64
|
-
from .
|
|
65
|
-
|
|
66
|
-
versioned = VersionedGP(path, branch=branch)
|
|
67
|
-
else:
|
|
68
|
-
# Build KV backend
|
|
69
|
-
backend: KVStore
|
|
70
|
-
if kind == "memory":
|
|
71
|
-
backend = Memory()
|
|
72
|
-
elif kind == "disk":
|
|
73
|
-
if path is None:
|
|
74
|
-
raise ValueError("path is required when kind='disk'")
|
|
75
|
-
from .kv.disk import Disk
|
|
63
|
+
raise ValueError("path is required when kind='disk'")
|
|
64
|
+
from .kv.disk import Disk
|
|
76
65
|
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
66
|
+
backend = Disk(path)
|
|
67
|
+
elif kind == "indexeddb":
|
|
68
|
+
from .kv.indexeddb import IndexedDB
|
|
80
69
|
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
versioned = VersionedKV(backend, branch=branch)
|
|
70
|
+
backend = IndexedDB(db_name=db_name)
|
|
71
|
+
else:
|
|
72
|
+
raise ValueError(f"Unknown kind: {kind!r}")
|
|
86
73
|
|
|
87
|
-
return Staged(
|
|
74
|
+
return Staged(VersionedKV(backend, branch=branch), encoder=encoder, decoder=decoder)
|
|
@@ -681,6 +681,11 @@ class VersionedKV(VersionedBase):
|
|
|
681
681
|
if self.store.get(branch_key) is None:
|
|
682
682
|
raise ValueError(f"Branch '{name}' does not exist")
|
|
683
683
|
self.store.remove(branch_key)
|
|
684
|
+
# The prev-HEAD recovery backup goes too: left behind, a
|
|
685
|
+
# same-named branch created later would "recover" the deleted
|
|
686
|
+
# state through _resolve_head's fallback. Removed before
|
|
687
|
+
# clean_orphans so commits only it referenced are collectable.
|
|
688
|
+
self.store.remove(BRANCH_HEAD_PREV % name)
|
|
684
689
|
self.clean_orphans()
|
|
685
690
|
|
|
686
691
|
def switch_branch(self, name: str) -> None:
|
|
@@ -37,9 +37,8 @@ def resolve_merge(
|
|
|
37
37
|
) -> MergeResolution:
|
|
38
38
|
"""Resolve a three-way merge between two diverged keysets.
|
|
39
39
|
|
|
40
|
-
Implements the pure merge logic
|
|
41
|
-
|
|
42
|
-
caller handles persistence.
|
|
40
|
+
Implements the pure merge logic used by VersionedKV. Does NOT
|
|
41
|
+
create commits or advance HEAD -- the caller handles persistence.
|
|
43
42
|
|
|
44
43
|
Args:
|
|
45
44
|
lca_keyset: Keyset of the lowest common ancestor commit.
|
|
@@ -38,8 +38,7 @@ class MergeResult:
|
|
|
38
38
|
class Versioned(Protocol):
|
|
39
39
|
"""Protocol for versioned key-value stores.
|
|
40
40
|
|
|
41
|
-
Defines the common interface implemented by ``VersionedKV
|
|
42
|
-
(KVStore-backed) and ``VersionedGP`` (GitPython-backed).
|
|
41
|
+
Defines the common interface implemented by ``VersionedKV``.
|
|
43
42
|
"""
|
|
44
43
|
|
|
45
44
|
last_merge_result: MergeResult | None
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: kvgit
|
|
3
|
-
Version: 0.3.
|
|
3
|
+
Version: 0.3.1
|
|
4
4
|
Summary: Versioned key-value store with git-like commit, branch, and merge semantics.
|
|
5
5
|
Author: ashenfad
|
|
6
6
|
License: MIT
|
|
@@ -27,8 +27,6 @@ Description-Content-Type: text/markdown
|
|
|
27
27
|
License-File: LICENSE
|
|
28
28
|
Provides-Extra: disk
|
|
29
29
|
Requires-Dist: diskcache; extra == "disk"
|
|
30
|
-
Provides-Extra: git
|
|
31
|
-
Requires-Dist: gitpython>=3.1; extra == "git"
|
|
32
30
|
Provides-Extra: numpy
|
|
33
31
|
Requires-Dist: numpy>=1.24; extra == "numpy"
|
|
34
32
|
Provides-Extra: pandas
|
|
@@ -39,13 +37,11 @@ Requires-Dist: numpy>=1.24; extra == "scientific"
|
|
|
39
37
|
Requires-Dist: pandas>=2.0; extra == "scientific"
|
|
40
38
|
Provides-Extra: all
|
|
41
39
|
Requires-Dist: diskcache; extra == "all"
|
|
42
|
-
Requires-Dist: gitpython>=3.1; extra == "all"
|
|
43
40
|
Requires-Dist: numpy>=1.24; extra == "all"
|
|
44
41
|
Requires-Dist: pandas>=2.0; extra == "all"
|
|
45
42
|
Provides-Extra: dev
|
|
46
43
|
Requires-Dist: pytest; extra == "dev"
|
|
47
44
|
Requires-Dist: diskcache; extra == "dev"
|
|
48
|
-
Requires-Dist: gitpython>=3.1; extra == "dev"
|
|
49
45
|
Requires-Dist: ruff; extra == "dev"
|
|
50
46
|
Requires-Dist: numpy>=1.24; extra == "dev"
|
|
51
47
|
Requires-Dist: pandas>=2.0; extra == "dev"
|
|
@@ -61,7 +57,7 @@ Git-style versioning for your data. Commits, branches, and merges -- backed by a
|
|
|
61
57
|
| **Commits** | Immutable, content-addressable snapshots with rollback |
|
|
62
58
|
| **Branches** | Cheap forks with CAS-based optimistic concurrency |
|
|
63
59
|
| **Three-way merge** | Auto-merges non-overlapping changes; pluggable merge fns for conflicts |
|
|
64
|
-
| **Pluggable backends** | In-memory, disk (diskcache),
|
|
60
|
+
| **Pluggable backends** | In-memory, disk (diskcache), IndexedDB (Pyodide/browser), or bring your own `KVStore` |
|
|
65
61
|
| **Chunked codecs** | Optional content-addressed dedup for large numpy arrays and pandas DataFrames -- equal buffers stored once across keys, commits, and branches |
|
|
66
62
|
|
|
67
63
|
## Install
|
|
@@ -69,7 +65,6 @@ Git-style versioning for your data. Commits, branches, and merges -- backed by a
|
|
|
69
65
|
```bash
|
|
70
66
|
pip install kvgit # in-memory only
|
|
71
67
|
pip install kvgit[disk] # adds disk backend via diskcache
|
|
72
|
-
pip install kvgit[git] # adds git backend via GitPython (requires git on PATH)
|
|
73
68
|
pip install kvgit[scientific] # adds chunked codecs for numpy / pandas
|
|
74
69
|
# IndexedDB backend is available automatically in Pyodide (browser) environments
|
|
75
70
|
```
|
|
@@ -131,3 +126,4 @@ See [`docs/`](docs/) for detailed documentation:
|
|
|
131
126
|
|
|
132
127
|
- [Quick Start](docs/quick-start.md) -- common patterns with runnable examples
|
|
133
128
|
- [API Reference](docs/api.md) -- full reference for all classes, methods, and types
|
|
129
|
+
- [Browser persistence (Pyodide)](docs/pyodide.md) -- choosing between the IndexedDB and OPFS-mounted-disk backends, plus the syncfs flush requirement and recommended host-side patterns
|
|
@@ -1,14 +1,12 @@
|
|
|
1
1
|
|
|
2
2
|
[all]
|
|
3
3
|
diskcache
|
|
4
|
-
gitpython>=3.1
|
|
5
4
|
numpy>=1.24
|
|
6
5
|
pandas>=2.0
|
|
7
6
|
|
|
8
7
|
[dev]
|
|
9
8
|
pytest
|
|
10
9
|
diskcache
|
|
11
|
-
gitpython>=3.1
|
|
12
10
|
ruff
|
|
13
11
|
numpy>=1.24
|
|
14
12
|
pandas>=2.0
|
|
@@ -16,9 +14,6 @@ pandas>=2.0
|
|
|
16
14
|
[disk]
|
|
17
15
|
diskcache
|
|
18
16
|
|
|
19
|
-
[git]
|
|
20
|
-
gitpython>=3.1
|
|
21
|
-
|
|
22
17
|
[numpy]
|
|
23
18
|
numpy>=1.24
|
|
24
19
|
|
|
@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
|
|
|
4
4
|
|
|
5
5
|
[project]
|
|
6
6
|
name = "kvgit"
|
|
7
|
-
version = "0.3.
|
|
7
|
+
version = "0.3.1"
|
|
8
8
|
description = "Versioned key-value store with git-like commit, branch, and merge semantics."
|
|
9
9
|
authors = [{ name = "ashenfad" }]
|
|
10
10
|
requires-python = ">=3.10"
|
|
@@ -36,15 +36,13 @@ dependencies = []
|
|
|
36
36
|
|
|
37
37
|
[project.optional-dependencies]
|
|
38
38
|
disk = ["diskcache"]
|
|
39
|
-
git = ["gitpython>=3.1"]
|
|
40
39
|
numpy = ["numpy>=1.24"]
|
|
41
40
|
pandas = ["numpy>=1.24", "pandas>=2.0"]
|
|
42
41
|
scientific = ["numpy>=1.24", "pandas>=2.0"]
|
|
43
|
-
all = ["diskcache", "
|
|
42
|
+
all = ["diskcache", "numpy>=1.24", "pandas>=2.0"]
|
|
44
43
|
dev = [
|
|
45
44
|
"pytest",
|
|
46
45
|
"diskcache",
|
|
47
|
-
"gitpython>=3.1",
|
|
48
46
|
"ruff",
|
|
49
47
|
"numpy>=1.24",
|
|
50
48
|
"pandas>=2.0",
|
|
@@ -57,7 +55,7 @@ include = ["kvgit*"]
|
|
|
57
55
|
warn_unused_configs = true
|
|
58
56
|
|
|
59
57
|
[[tool.mypy.overrides]]
|
|
60
|
-
module = ["diskcache"
|
|
58
|
+
module = ["diskcache"]
|
|
61
59
|
ignore_missing_imports = true
|
|
62
60
|
|
|
63
61
|
[tool.pytest.ini_options]
|
|
@@ -1,137 +0,0 @@
|
|
|
1
|
-
"""N-tier composite cache over multiple KV stores."""
|
|
2
|
-
|
|
3
|
-
from collections.abc import Iterable, Mapping
|
|
4
|
-
|
|
5
|
-
from .base import KVStore
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
class Composite(KVStore):
|
|
9
|
-
"""N-tier cache composing any number of KV stores.
|
|
10
|
-
|
|
11
|
-
On get: check L1, L2, ..., Ln in order. On hit at tier i,
|
|
12
|
-
populate L1..L(i-1) and return.
|
|
13
|
-
|
|
14
|
-
On set: write to all tiers (most durable first).
|
|
15
|
-
|
|
16
|
-
On cas: delegate to Ln (authoritative), update caches on success.
|
|
17
|
-
|
|
18
|
-
Args:
|
|
19
|
-
stores: List of KV stores ordered fastest -> most durable.
|
|
20
|
-
"""
|
|
21
|
-
|
|
22
|
-
def __init__(self, stores: list[KVStore]) -> None:
|
|
23
|
-
if not stores:
|
|
24
|
-
raise ValueError("Composite requires at least one store")
|
|
25
|
-
self._stores = stores
|
|
26
|
-
|
|
27
|
-
def get(self, key: str) -> bytes | None:
|
|
28
|
-
for i, store in enumerate(self._stores):
|
|
29
|
-
try:
|
|
30
|
-
value = store.get(key)
|
|
31
|
-
if value is not None:
|
|
32
|
-
if i > 0:
|
|
33
|
-
for j in range(i):
|
|
34
|
-
try:
|
|
35
|
-
self._stores[j].set(key, value)
|
|
36
|
-
except Exception:
|
|
37
|
-
pass # best-effort cache population
|
|
38
|
-
return value
|
|
39
|
-
except Exception:
|
|
40
|
-
continue # tier unavailable, try next
|
|
41
|
-
return None
|
|
42
|
-
|
|
43
|
-
def get_many(self, *args) -> Mapping[str, bytes]:
|
|
44
|
-
result: dict[str, bytes] = {}
|
|
45
|
-
remaining = set(self._normalize_keys(args))
|
|
46
|
-
for i, store in enumerate(self._stores):
|
|
47
|
-
if not remaining:
|
|
48
|
-
break
|
|
49
|
-
try:
|
|
50
|
-
tier_values: dict[str, bytes] = {}
|
|
51
|
-
for key in remaining:
|
|
52
|
-
value = store.get(key)
|
|
53
|
-
if value is not None:
|
|
54
|
-
tier_values[key] = value
|
|
55
|
-
if tier_values and i > 0:
|
|
56
|
-
for j in range(i):
|
|
57
|
-
try:
|
|
58
|
-
self._stores[j].set_many(tier_values)
|
|
59
|
-
except Exception:
|
|
60
|
-
pass
|
|
61
|
-
result.update(tier_values)
|
|
62
|
-
remaining -= tier_values.keys()
|
|
63
|
-
except Exception:
|
|
64
|
-
continue
|
|
65
|
-
return result
|
|
66
|
-
|
|
67
|
-
def __contains__(self, key: str) -> bool:
|
|
68
|
-
for store in self._stores:
|
|
69
|
-
try:
|
|
70
|
-
if key in store:
|
|
71
|
-
return True
|
|
72
|
-
except Exception:
|
|
73
|
-
continue
|
|
74
|
-
return False
|
|
75
|
-
|
|
76
|
-
def keys(self) -> Iterable[str]:
|
|
77
|
-
return self._stores[-1].keys()
|
|
78
|
-
|
|
79
|
-
def items(self) -> Iterable[tuple[str, bytes]]:
|
|
80
|
-
return self._stores[-1].items()
|
|
81
|
-
|
|
82
|
-
def set(self, key: str, value: bytes) -> None:
|
|
83
|
-
self._stores[-1].set(key, value)
|
|
84
|
-
for store in self._stores[:-1]:
|
|
85
|
-
try:
|
|
86
|
-
store.set(key, value)
|
|
87
|
-
except Exception:
|
|
88
|
-
pass # best-effort cache population
|
|
89
|
-
|
|
90
|
-
def set_many(
|
|
91
|
-
self,
|
|
92
|
-
items: Mapping[str, bytes] | None = None,
|
|
93
|
-
/,
|
|
94
|
-
**kwargs: bytes,
|
|
95
|
-
) -> None:
|
|
96
|
-
items = self._normalize_items(items, kwargs)
|
|
97
|
-
self._stores[-1].set_many(items)
|
|
98
|
-
for store in self._stores[:-1]:
|
|
99
|
-
try:
|
|
100
|
-
store.set_many(items)
|
|
101
|
-
except Exception:
|
|
102
|
-
pass # best-effort cache population
|
|
103
|
-
|
|
104
|
-
def remove(self, key: str) -> None:
|
|
105
|
-
self._stores[-1].remove(key)
|
|
106
|
-
for store in self._stores[:-1]:
|
|
107
|
-
try:
|
|
108
|
-
store.remove(key)
|
|
109
|
-
except Exception:
|
|
110
|
-
pass # best-effort cache population
|
|
111
|
-
|
|
112
|
-
def remove_many(self, *args) -> None:
|
|
113
|
-
keys = list(self._normalize_keys(args))
|
|
114
|
-
self._stores[-1].remove_many(keys)
|
|
115
|
-
for store in self._stores[:-1]:
|
|
116
|
-
try:
|
|
117
|
-
store.remove_many(keys)
|
|
118
|
-
except Exception:
|
|
119
|
-
pass # best-effort cache population
|
|
120
|
-
|
|
121
|
-
def clear(self) -> None:
|
|
122
|
-
self._stores[-1].clear()
|
|
123
|
-
for store in self._stores[:-1]:
|
|
124
|
-
try:
|
|
125
|
-
store.clear()
|
|
126
|
-
except Exception:
|
|
127
|
-
pass # best-effort cache population
|
|
128
|
-
|
|
129
|
-
def cas(self, key: str, value: bytes, expected: bytes | None) -> bool:
|
|
130
|
-
success = self._stores[-1].cas(key, value, expected)
|
|
131
|
-
if success:
|
|
132
|
-
for store in self._stores[:-1]:
|
|
133
|
-
try:
|
|
134
|
-
store.set(key, value)
|
|
135
|
-
except Exception:
|
|
136
|
-
pass
|
|
137
|
-
return success
|
|
@@ -1,360 +0,0 @@
|
|
|
1
|
-
"""GitPython-backed versioned store.
|
|
2
|
-
|
|
3
|
-
Importing this module requires GitPython. Both ``kvgit.__init__`` and
|
|
4
|
-
``kvgit.versioned.__init__`` already wrap their imports of this module
|
|
5
|
-
in ``try: ... except ImportError: pass`` so the public ``kvgit`` API
|
|
6
|
-
stays usable on installs that don't have GitPython. Anyone who imports
|
|
7
|
-
this module directly is signaling intent to use the git backend and
|
|
8
|
-
should get a clear ImportError.
|
|
9
|
-
"""
|
|
10
|
-
|
|
11
|
-
import binascii
|
|
12
|
-
import io
|
|
13
|
-
import json
|
|
14
|
-
import os
|
|
15
|
-
import urllib.parse
|
|
16
|
-
|
|
17
|
-
import git # noqa: F401 (re-exported names below need this)
|
|
18
|
-
from git import Blob, Commit, Repo, Tree
|
|
19
|
-
from git.exc import BadObject, GitCommandError
|
|
20
|
-
from gitdb import IStream
|
|
21
|
-
|
|
22
|
-
from .base import VersionedBase
|
|
23
|
-
from .merge import MergeResolution
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
def _quote(key: str) -> str:
|
|
27
|
-
return urllib.parse.quote_plus(key)
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
def _unquote(quoted: str) -> str:
|
|
31
|
-
return urllib.parse.unquote_plus(quoted)
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
class VersionedGP(VersionedBase):
|
|
35
|
-
"""A commit log backed by a Git repository."""
|
|
36
|
-
|
|
37
|
-
def __init__(
|
|
38
|
-
self,
|
|
39
|
-
repo_path: str,
|
|
40
|
-
*,
|
|
41
|
-
commit_hash: str | None = None,
|
|
42
|
-
branch: str = "main",
|
|
43
|
-
) -> None:
|
|
44
|
-
self.repo_path = repo_path
|
|
45
|
-
|
|
46
|
-
if not os.path.exists(repo_path):
|
|
47
|
-
self.repo = Repo.init(repo_path, bare=True)
|
|
48
|
-
else:
|
|
49
|
-
self.repo = Repo(repo_path)
|
|
50
|
-
|
|
51
|
-
if commit_hash is None:
|
|
52
|
-
# Try to read the branch HEAD
|
|
53
|
-
try:
|
|
54
|
-
commit_hash = self.repo.heads[branch].commit.hexsha
|
|
55
|
-
except (IndexError, AttributeError):
|
|
56
|
-
# Branch doesn't exist, create an initial empty commit
|
|
57
|
-
commit_hash = self._create_empty_commit(branch)
|
|
58
|
-
|
|
59
|
-
if not isinstance(commit_hash, str):
|
|
60
|
-
raise TypeError(
|
|
61
|
-
f"commit_hash must be str, got {type(commit_hash).__name__}"
|
|
62
|
-
)
|
|
63
|
-
|
|
64
|
-
super().__init__(branch=branch, commit_hash=commit_hash)
|
|
65
|
-
|
|
66
|
-
# Load commit keyset
|
|
67
|
-
self._load_commit(commit_hash, update_base=True)
|
|
68
|
-
|
|
69
|
-
def _create_empty_commit(self, branch: str) -> str:
|
|
70
|
-
# Create an empty tree
|
|
71
|
-
|
|
72
|
-
tree_data = bytearray()
|
|
73
|
-
istream = self.repo.odb.store(
|
|
74
|
-
IStream(Tree.type, len(tree_data), io.BytesIO(tree_data))
|
|
75
|
-
)
|
|
76
|
-
tree_sha = istream.binsha
|
|
77
|
-
tree_obj = Tree(self.repo, tree_sha)
|
|
78
|
-
|
|
79
|
-
commit = Commit.create_from_tree(
|
|
80
|
-
self.repo, tree_obj, "Initial commit", parent_commits=[]
|
|
81
|
-
)
|
|
82
|
-
self.repo.create_head(branch, commit)
|
|
83
|
-
return commit.hexsha
|
|
84
|
-
|
|
85
|
-
@property
|
|
86
|
-
def latest_head(self) -> str | None:
|
|
87
|
-
try:
|
|
88
|
-
return self.repo.heads[self._branch].commit.hexsha
|
|
89
|
-
except (IndexError, AttributeError):
|
|
90
|
-
return None
|
|
91
|
-
|
|
92
|
-
# -- Read operations --
|
|
93
|
-
|
|
94
|
-
def get(self, key: str) -> bytes | None:
|
|
95
|
-
"""Get a value from the current commit."""
|
|
96
|
-
hexsha = self._commit_keys.get(key)
|
|
97
|
-
if hexsha is None:
|
|
98
|
-
return None
|
|
99
|
-
return self._read_blob(hexsha)
|
|
100
|
-
|
|
101
|
-
def get_many(self, *keys: str) -> dict[str, bytes]:
|
|
102
|
-
"""Get multiple values from the current commit."""
|
|
103
|
-
result = {}
|
|
104
|
-
for key in keys:
|
|
105
|
-
val = self.get(key)
|
|
106
|
-
if val is not None:
|
|
107
|
-
result[key] = val
|
|
108
|
-
return result
|
|
109
|
-
|
|
110
|
-
# -- Abstract method implementations --
|
|
111
|
-
|
|
112
|
-
def _snapshot_state(self) -> tuple:
|
|
113
|
-
return (self._current_commit, dict(self._commit_keys))
|
|
114
|
-
|
|
115
|
-
def _restore_state(self, saved: tuple) -> None:
|
|
116
|
-
self._current_commit, self._commit_keys = saved
|
|
117
|
-
|
|
118
|
-
def _read_blob(self, content_id: str) -> bytes | None:
|
|
119
|
-
"""Read a blob by its git hex SHA."""
|
|
120
|
-
try:
|
|
121
|
-
blob = Blob(self.repo, binascii.unhexlify(content_id))
|
|
122
|
-
return blob.data_stream.read()
|
|
123
|
-
except (BadObject, ValueError):
|
|
124
|
-
return None
|
|
125
|
-
|
|
126
|
-
def _write_blob(self, data: bytes) -> str:
|
|
127
|
-
istream = self.repo.odb.store(IStream(Blob.type, len(data), io.BytesIO(data)))
|
|
128
|
-
return istream.hexsha
|
|
129
|
-
|
|
130
|
-
def _create_tree(self, keyset: dict[str, str]) -> str:
|
|
131
|
-
# Git tree objects are a sequence of entries sorted by name.
|
|
132
|
-
# Each entry is: "<mode> <name>\0<20-byte binary SHA>"
|
|
133
|
-
# We use mode 100644 (regular file) for all entries.
|
|
134
|
-
tree_data = bytearray()
|
|
135
|
-
entries = []
|
|
136
|
-
for key, hexsha in keyset.items():
|
|
137
|
-
entries.append((0o100644, _quote(key), hexsha))
|
|
138
|
-
|
|
139
|
-
for mode, name, hexsha in sorted(entries, key=lambda e: e[1]):
|
|
140
|
-
tree_data.extend(f"{mode:o} {name}".encode("utf-8"))
|
|
141
|
-
tree_data.append(0)
|
|
142
|
-
tree_data.extend(binascii.unhexlify(hexsha))
|
|
143
|
-
|
|
144
|
-
istream = self.repo.odb.store(
|
|
145
|
-
IStream(Tree.type, len(tree_data), io.BytesIO(tree_data))
|
|
146
|
-
)
|
|
147
|
-
return istream.hexsha
|
|
148
|
-
|
|
149
|
-
def _create_git_commit(
|
|
150
|
-
self,
|
|
151
|
-
updates: dict[str, bytes] | None = None,
|
|
152
|
-
removals: set[str] | None = None,
|
|
153
|
-
*,
|
|
154
|
-
info: dict | None = None,
|
|
155
|
-
parents: list[str] | None = None,
|
|
156
|
-
) -> str:
|
|
157
|
-
"""Create a git commit object.
|
|
158
|
-
|
|
159
|
-
Internal method that supports explicit parents (for merge commits).
|
|
160
|
-
"""
|
|
161
|
-
updates = updates or {}
|
|
162
|
-
removals = removals or set()
|
|
163
|
-
|
|
164
|
-
new_commit_keys = dict(self._commit_keys)
|
|
165
|
-
for key in removals:
|
|
166
|
-
new_commit_keys.pop(key, None)
|
|
167
|
-
|
|
168
|
-
for key, value in updates.items():
|
|
169
|
-
new_commit_keys[key] = self._write_blob(value)
|
|
170
|
-
|
|
171
|
-
tree_hexsha = self._create_tree(new_commit_keys)
|
|
172
|
-
tree_obj = Tree(self.repo, binascii.unhexlify(tree_hexsha))
|
|
173
|
-
|
|
174
|
-
info_str = json.dumps(info, separators=(",", ":")) if info else ""
|
|
175
|
-
msg = f"kvgit commit\n\n{info_str}" if info_str else "kvgit commit"
|
|
176
|
-
|
|
177
|
-
if parents is None:
|
|
178
|
-
parent_objs = [Commit(self.repo, binascii.unhexlify(self._current_commit))]
|
|
179
|
-
else:
|
|
180
|
-
parent_objs = [Commit(self.repo, binascii.unhexlify(p)) for p in parents]
|
|
181
|
-
|
|
182
|
-
commit = Commit.create_from_tree(
|
|
183
|
-
self.repo, tree_obj, msg, parent_commits=parent_objs
|
|
184
|
-
)
|
|
185
|
-
|
|
186
|
-
self._commit_keys = new_commit_keys
|
|
187
|
-
self._current_commit = commit.hexsha
|
|
188
|
-
return commit.hexsha
|
|
189
|
-
|
|
190
|
-
def _create_commit(
|
|
191
|
-
self,
|
|
192
|
-
updates: dict[str, bytes] | None = None,
|
|
193
|
-
removals: set[str] | None = None,
|
|
194
|
-
*,
|
|
195
|
-
info: dict | None = None,
|
|
196
|
-
chunks: dict[str, bytes] | None = None,
|
|
197
|
-
chunk_refs: dict[str, list[str]] | None = None,
|
|
198
|
-
) -> str:
|
|
199
|
-
"""Create a single-parent commit (parent = current commit).
|
|
200
|
-
|
|
201
|
-
``chunks`` and ``chunk_refs`` are part of the ``Versioned``
|
|
202
|
-
contract for the chunked-codec layer but the git backend has
|
|
203
|
-
no chunk namespace and no GC story; they're ignored. Pairing
|
|
204
|
-
a chunked codec with this backend is rejected at the
|
|
205
|
-
``Staged`` construction site, so reaching this path with
|
|
206
|
-
non-None ``chunks`` would indicate a caller bypassing
|
|
207
|
-
``Staged`` and writing to ``VersionedGP`` directly — which is
|
|
208
|
-
documented as unsupported.
|
|
209
|
-
"""
|
|
210
|
-
return self._create_git_commit(updates, removals, info=info)
|
|
211
|
-
|
|
212
|
-
def _create_merge_commit(
|
|
213
|
-
self,
|
|
214
|
-
resolution: MergeResolution,
|
|
215
|
-
parents: tuple[str, ...],
|
|
216
|
-
info: dict | None,
|
|
217
|
-
) -> str:
|
|
218
|
-
"""Create a multi-parent merge commit from a resolved merge."""
|
|
219
|
-
merged_keyset = resolution.merged_keyset
|
|
220
|
-
merged_values = resolution.merged_values
|
|
221
|
-
|
|
222
|
-
removals = (
|
|
223
|
-
set(self._commit_keys.keys())
|
|
224
|
-
- set(merged_keyset.keys())
|
|
225
|
-
- set(merged_values.keys())
|
|
226
|
-
)
|
|
227
|
-
self._commit_keys = merged_keyset
|
|
228
|
-
return self._create_git_commit(
|
|
229
|
-
merged_values, removals, info=info, parents=list(parents)
|
|
230
|
-
)
|
|
231
|
-
|
|
232
|
-
def _cas_head(self, expected: str, new_head: str) -> bool:
|
|
233
|
-
"""Atomically advance branch HEAD via git update-ref."""
|
|
234
|
-
try:
|
|
235
|
-
self.repo.git.update_ref(f"refs/heads/{self._branch}", new_head, expected)
|
|
236
|
-
return True
|
|
237
|
-
except GitCommandError:
|
|
238
|
-
return False
|
|
239
|
-
|
|
240
|
-
def _load_keyset(self, commit_hash: str) -> dict[str, str]:
|
|
241
|
-
c = Commit(self.repo, binascii.unhexlify(commit_hash))
|
|
242
|
-
return {_unquote(blob.name): blob.hexsha for blob in c.tree.blobs}
|
|
243
|
-
|
|
244
|
-
def _load_parents(self, commit_hash: str) -> tuple[str, ...]:
|
|
245
|
-
c = Commit(self.repo, binascii.unhexlify(commit_hash))
|
|
246
|
-
return tuple(p.hexsha for p in c.parents)
|
|
247
|
-
|
|
248
|
-
def _find_lca(self, commit_a: str, commit_b: str) -> str | None:
|
|
249
|
-
try:
|
|
250
|
-
bases = self.repo.merge_base(commit_a, commit_b)
|
|
251
|
-
except GitCommandError:
|
|
252
|
-
return None
|
|
253
|
-
if bases:
|
|
254
|
-
return bases[0].hexsha
|
|
255
|
-
return None
|
|
256
|
-
|
|
257
|
-
# -- Navigation --
|
|
258
|
-
|
|
259
|
-
def refresh(self) -> None:
|
|
260
|
-
"""Reload state from HEAD."""
|
|
261
|
-
head = self.latest_head
|
|
262
|
-
if head is None:
|
|
263
|
-
raise ValueError(f"No HEAD commit found for branch {self._branch}")
|
|
264
|
-
self._load_commit(head, update_base=True)
|
|
265
|
-
|
|
266
|
-
def checkout(
|
|
267
|
-
self, commit_hash: str, *, branch: str | None = None
|
|
268
|
-
) -> "VersionedGP | None":
|
|
269
|
-
"""Return a new VersionedGP at a specific commit, or None if missing."""
|
|
270
|
-
try:
|
|
271
|
-
Commit(self.repo, binascii.unhexlify(commit_hash))
|
|
272
|
-
return VersionedGP(
|
|
273
|
-
self.repo_path, commit_hash=commit_hash, branch=branch or self._branch
|
|
274
|
-
)
|
|
275
|
-
except (BadObject, ValueError):
|
|
276
|
-
return None
|
|
277
|
-
|
|
278
|
-
def create_branch(self, name: str, *, at: str | None = None) -> "VersionedGP":
|
|
279
|
-
"""Fork a commit onto a new branch.
|
|
280
|
-
|
|
281
|
-
Returns a new ``VersionedGP`` on the new branch.
|
|
282
|
-
"""
|
|
283
|
-
target = at or self._current_commit
|
|
284
|
-
if at is not None:
|
|
285
|
-
try:
|
|
286
|
-
c = Commit(self.repo, binascii.unhexlify(at))
|
|
287
|
-
c.tree # force load to verify commit exists
|
|
288
|
-
except (BadObject, ValueError):
|
|
289
|
-
raise ValueError(f"Commit '{at}' does not exist")
|
|
290
|
-
|
|
291
|
-
if name in self.repo.heads:
|
|
292
|
-
raise ValueError(f"Branch '{name}' already exists")
|
|
293
|
-
|
|
294
|
-
self.repo.create_head(name, Commit(self.repo, binascii.unhexlify(target)))
|
|
295
|
-
return VersionedGP(self.repo_path, commit_hash=target, branch=name)
|
|
296
|
-
|
|
297
|
-
def delete_branch(self, name: str) -> None:
|
|
298
|
-
"""Delete a branch by name."""
|
|
299
|
-
if name == self._branch:
|
|
300
|
-
raise ValueError("Cannot delete the current branch")
|
|
301
|
-
if name not in self.repo.heads:
|
|
302
|
-
raise ValueError(f"Branch '{name}' does not exist")
|
|
303
|
-
|
|
304
|
-
git.Head.delete(self.repo, self.repo.heads[name], force=True)
|
|
305
|
-
|
|
306
|
-
def switch_branch(self, name: str) -> None:
|
|
307
|
-
"""Switch this instance to a different branch in-place."""
|
|
308
|
-
if name not in self.repo.heads:
|
|
309
|
-
raise ValueError(f"Branch '{name}' does not exist")
|
|
310
|
-
self._branch = name
|
|
311
|
-
self._load_commit(self.repo.heads[name].commit.hexsha, update_base=True)
|
|
312
|
-
|
|
313
|
-
def peek(self, key: str, *, branch: str) -> bytes | None:
|
|
314
|
-
"""Read a key from another branch's HEAD without switching."""
|
|
315
|
-
if branch not in self.repo.heads:
|
|
316
|
-
return None
|
|
317
|
-
commit_hash = self.repo.heads[branch].commit.hexsha
|
|
318
|
-
keyset = self._load_keyset(commit_hash)
|
|
319
|
-
hexsha = keyset.get(key)
|
|
320
|
-
if not hexsha:
|
|
321
|
-
return None
|
|
322
|
-
return self._read_blob(hexsha)
|
|
323
|
-
|
|
324
|
-
def reset_to(self, commit_hash: str) -> bool:
|
|
325
|
-
"""Reset HEAD to a specific commit."""
|
|
326
|
-
try:
|
|
327
|
-
c = Commit(self.repo, binascii.unhexlify(commit_hash))
|
|
328
|
-
c.tree # force load to verify commit exists
|
|
329
|
-
except (BadObject, ValueError):
|
|
330
|
-
return False
|
|
331
|
-
self.repo.git.update_ref(f"refs/heads/{self._branch}", commit_hash)
|
|
332
|
-
self._load_commit(commit_hash, update_base=True)
|
|
333
|
-
return True
|
|
334
|
-
|
|
335
|
-
def list_branches(self) -> list[str]:
|
|
336
|
-
"""List all branch names."""
|
|
337
|
-
return [h.name for h in self.repo.heads]
|
|
338
|
-
|
|
339
|
-
def commit_info(self, commit_hash: str | None = None) -> dict | None:
|
|
340
|
-
"""Retrieve the info dict for a commit, or None if none was stored."""
|
|
341
|
-
target = commit_hash or self._current_commit
|
|
342
|
-
try:
|
|
343
|
-
c = Commit(self.repo, binascii.unhexlify(target))
|
|
344
|
-
msg = c.message
|
|
345
|
-
except (BadObject, ValueError):
|
|
346
|
-
return None
|
|
347
|
-
if "\n\n" not in msg:
|
|
348
|
-
return None
|
|
349
|
-
info_part = msg.split("\n\n", 1)[1].strip()
|
|
350
|
-
if not info_part:
|
|
351
|
-
return None
|
|
352
|
-
return json.loads(info_part)
|
|
353
|
-
|
|
354
|
-
# -- Internal --
|
|
355
|
-
|
|
356
|
-
def _load_commit(self, commit_hash: str, *, update_base: bool) -> None:
|
|
357
|
-
self._current_commit = commit_hash
|
|
358
|
-
if update_base:
|
|
359
|
-
self._base_commit = commit_hash
|
|
360
|
-
self._commit_keys = self._load_keyset(commit_hash)
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|