kvgit 0.3.7__tar.gz → 0.3.8__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {kvgit-0.3.7 → kvgit-0.3.8}/PKG-INFO +58 -7
- {kvgit-0.3.7 → kvgit-0.3.8}/README.md +57 -6
- {kvgit-0.3.7 → kvgit-0.3.8}/kvgit/__init__.py +16 -4
- kvgit-0.3.8/kvgit/content_types.py +69 -0
- {kvgit-0.3.7 → kvgit-0.3.8}/kvgit/errors.py +10 -0
- {kvgit-0.3.7 → kvgit-0.3.8}/kvgit/kv/composite.py +74 -13
- {kvgit-0.3.7 → kvgit-0.3.8}/kvgit/merges.py +25 -1
- {kvgit-0.3.7 → kvgit-0.3.8}/kvgit/staged.py +87 -20
- {kvgit-0.3.7 → kvgit-0.3.8}/kvgit/store.py +3 -3
- {kvgit-0.3.7 → kvgit-0.3.8}/kvgit/versioned/__init__.py +11 -1
- {kvgit-0.3.7 → kvgit-0.3.8}/kvgit/versioned/base.py +49 -14
- {kvgit-0.3.7 → kvgit-0.3.8}/kvgit/versioned/kv.py +348 -47
- kvgit-0.3.8/kvgit/versioned/merge.py +222 -0
- {kvgit-0.3.7 → kvgit-0.3.8}/kvgit/versioned/protocol.py +52 -8
- {kvgit-0.3.7 → kvgit-0.3.8}/kvgit.egg-info/PKG-INFO +58 -7
- {kvgit-0.3.7 → kvgit-0.3.8}/kvgit.egg-info/SOURCES.txt +1 -0
- {kvgit-0.3.7 → kvgit-0.3.8}/pyproject.toml +1 -1
- kvgit-0.3.8/tests/test_content_types.py +191 -0
- kvgit-0.3.8/tests/test_merge_policy.py +796 -0
- kvgit-0.3.7/kvgit/content_types.py +0 -25
- kvgit-0.3.7/kvgit/versioned/merge.py +0 -131
- kvgit-0.3.7/tests/test_content_types.py +0 -87
- {kvgit-0.3.7 → kvgit-0.3.8}/LICENSE +0 -0
- {kvgit-0.3.7 → kvgit-0.3.8}/kvgit/codecs/__init__.py +0 -0
- {kvgit-0.3.7 → kvgit-0.3.8}/kvgit/codecs/_hash.py +0 -0
- {kvgit-0.3.7 → kvgit-0.3.8}/kvgit/codecs/base.py +0 -0
- {kvgit-0.3.7 → kvgit-0.3.8}/kvgit/codecs/numpy.py +0 -0
- {kvgit-0.3.7 → kvgit-0.3.8}/kvgit/codecs/pandas.py +0 -0
- {kvgit-0.3.7 → kvgit-0.3.8}/kvgit/codecs/pickler.py +0 -0
- {kvgit-0.3.7 → kvgit-0.3.8}/kvgit/encoding.py +0 -0
- {kvgit-0.3.7 → kvgit-0.3.8}/kvgit/hamt.py +0 -0
- {kvgit-0.3.7 → kvgit-0.3.8}/kvgit/kv/__init__.py +0 -0
- {kvgit-0.3.7 → kvgit-0.3.8}/kvgit/kv/base.py +0 -0
- {kvgit-0.3.7 → kvgit-0.3.8}/kvgit/kv/disk.py +0 -0
- {kvgit-0.3.7 → kvgit-0.3.8}/kvgit/kv/indexeddb.py +0 -0
- {kvgit-0.3.7 → kvgit-0.3.8}/kvgit/kv/memory.py +0 -0
- {kvgit-0.3.7 → kvgit-0.3.8}/kvgit/namespaced.py +0 -0
- {kvgit-0.3.7 → kvgit-0.3.8}/kvgit/py.typed +0 -0
- {kvgit-0.3.7 → kvgit-0.3.8}/kvgit/versioned/helpers.py +0 -0
- {kvgit-0.3.7 → kvgit-0.3.8}/kvgit/versioned/keyset.py +0 -0
- {kvgit-0.3.7 → kvgit-0.3.8}/kvgit.egg-info/dependency_links.txt +0 -0
- {kvgit-0.3.7 → kvgit-0.3.8}/kvgit.egg-info/requires.txt +0 -0
- {kvgit-0.3.7 → kvgit-0.3.8}/kvgit.egg-info/top_level.txt +0 -0
- {kvgit-0.3.7 → kvgit-0.3.8}/setup.cfg +0 -0
- {kvgit-0.3.7 → kvgit-0.3.8}/tests/test_delete_branches.py +0 -0
- {kvgit-0.3.7 → kvgit-0.3.8}/tests/test_hamt.py +0 -0
- {kvgit-0.3.7 → kvgit-0.3.8}/tests/test_merges.py +0 -0
- {kvgit-0.3.7 → kvgit-0.3.8}/tests/test_namespaced.py +0 -0
- {kvgit-0.3.7 → kvgit-0.3.8}/tests/test_staged.py +0 -0
- {kvgit-0.3.7 → kvgit-0.3.8}/tests/test_store_factory.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: kvgit
|
|
3
|
-
Version: 0.3.
|
|
3
|
+
Version: 0.3.8
|
|
4
4
|
Summary: Versioned key-value store with git-like commit, branch, and merge semantics.
|
|
5
5
|
Author: ashenfad
|
|
6
6
|
License: MIT
|
|
@@ -109,17 +109,68 @@ print(main["score"]) # 500 (fast-forward: main hadn't diverged)
|
|
|
109
109
|
```
|
|
110
110
|
|
|
111
111
|
Overlapping changes need a merge function per key, or a `default_merge`
|
|
112
|
-
fallback. `
|
|
113
|
-
`<<<<<<<` markers (
|
|
114
|
-
cannot mark -- binary, non-UTF-8, oversized -- raises
|
|
115
|
-
as an ordinary conflict:
|
|
112
|
+
fallback. `text_merge()` resolves line-oriented text with git-style
|
|
113
|
+
`<<<<<<<` markers (its `ours_label` / `theirs_label` arguments name
|
|
114
|
+
them); anything it cannot mark -- binary, non-UTF-8, oversized -- raises
|
|
115
|
+
`CantMark`, filed as an ordinary conflict:
|
|
116
116
|
|
|
117
117
|
```python
|
|
118
|
-
from kvgit
|
|
118
|
+
from kvgit import text_merge
|
|
119
119
|
|
|
120
|
-
|
|
120
|
+
main["notes"] = "alpha\nbeta\n"
|
|
121
|
+
main.commit()
|
|
122
|
+
|
|
123
|
+
edits = main.create_branch("edits")
|
|
124
|
+
edits["notes"] = "alpha\nBETA\n"
|
|
125
|
+
edits.commit()
|
|
126
|
+
|
|
127
|
+
main["notes"] = "ALPHA\nbeta\n"
|
|
128
|
+
main.commit()
|
|
129
|
+
|
|
130
|
+
main.merge(edits.current_commit, default_merge=text_merge())
|
|
131
|
+
print(main["notes"]) # "ALPHA\nBETA\n" -- both edits kept
|
|
132
|
+
```
|
|
133
|
+
|
|
134
|
+
`kvgit.merges.text` is the same merge one level down, over raw bytes,
|
|
135
|
+
for `VersionedKV`. A `Staged` decodes each side before calling a merge
|
|
136
|
+
function, so register `text_merge()` there unless the key's values are
|
|
137
|
+
already `bytes`.
|
|
138
|
+
|
|
139
|
+
Registrations resolve most-specific-first: a key takes its exact-key
|
|
140
|
+
registration, else the longest registered prefix it starts with, else
|
|
141
|
+
`default_merge`. Prefixes cover keys whose names are not known when the
|
|
142
|
+
policy is set:
|
|
143
|
+
|
|
144
|
+
```python
|
|
145
|
+
from kvgit import MergeChoice, text_merge
|
|
146
|
+
|
|
147
|
+
main.set_merge_prefix("runs/", MergeChoice.OURS) # this branch owns runs/
|
|
148
|
+
main.set_merge_fn("runs/index", text_merge()) # except this one key
|
|
121
149
|
```
|
|
122
150
|
|
|
151
|
+
A registration holds either a merge function or a `MergeChoice`, and the
|
|
152
|
+
two reach differently:
|
|
153
|
+
|
|
154
|
+
* A **merge function** is consulted only where both sides changed a key.
|
|
155
|
+
A change only one side made is applied as it always was.
|
|
156
|
+
* A **`MergeChoice`** is a standing policy: it hands that side every key
|
|
157
|
+
either side changed under it. So `OURS` also drops a key the other
|
|
158
|
+
side added and keeps one the other side removed -- what a branch that
|
|
159
|
+
owns a whole namespace needs. Nothing is read or decoded for those
|
|
160
|
+
keys.
|
|
161
|
+
|
|
162
|
+
`kvgit.merges.ours` and `kvgit.merges.theirs` are the merge-function
|
|
163
|
+
form, for keys where one side wins only when both sides collide. They
|
|
164
|
+
keep that side's stored value as it stands rather than rewriting it, so
|
|
165
|
+
the merge writes no new blob -- and if the chosen side removed the key,
|
|
166
|
+
the merge removes it.
|
|
167
|
+
|
|
168
|
+
`commit()` and `merge()` take `merge_fns=` / `merge_prefixes=` /
|
|
169
|
+
`default_merge=` for one call, layered over what is registered.
|
|
170
|
+
|
|
171
|
+
Keys both sides changed to the same bytes merge cleanly with no merge
|
|
172
|
+
function, even though each side wrote its own copy.
|
|
173
|
+
|
|
123
174
|
A `post_check(key, merged_bytes)` predicate runs over every
|
|
124
175
|
merge-produced value; returning `False` files that key as conflicted.
|
|
125
176
|
`on_conflict="abandon"` leaves the branch untouched instead of raising.
|
|
@@ -60,17 +60,68 @@ print(main["score"]) # 500 (fast-forward: main hadn't diverged)
|
|
|
60
60
|
```
|
|
61
61
|
|
|
62
62
|
Overlapping changes need a merge function per key, or a `default_merge`
|
|
63
|
-
fallback. `
|
|
64
|
-
`<<<<<<<` markers (
|
|
65
|
-
cannot mark -- binary, non-UTF-8, oversized -- raises
|
|
66
|
-
as an ordinary conflict:
|
|
63
|
+
fallback. `text_merge()` resolves line-oriented text with git-style
|
|
64
|
+
`<<<<<<<` markers (its `ours_label` / `theirs_label` arguments name
|
|
65
|
+
them); anything it cannot mark -- binary, non-UTF-8, oversized -- raises
|
|
66
|
+
`CantMark`, filed as an ordinary conflict:
|
|
67
67
|
|
|
68
68
|
```python
|
|
69
|
-
from kvgit
|
|
69
|
+
from kvgit import text_merge
|
|
70
70
|
|
|
71
|
-
|
|
71
|
+
main["notes"] = "alpha\nbeta\n"
|
|
72
|
+
main.commit()
|
|
73
|
+
|
|
74
|
+
edits = main.create_branch("edits")
|
|
75
|
+
edits["notes"] = "alpha\nBETA\n"
|
|
76
|
+
edits.commit()
|
|
77
|
+
|
|
78
|
+
main["notes"] = "ALPHA\nbeta\n"
|
|
79
|
+
main.commit()
|
|
80
|
+
|
|
81
|
+
main.merge(edits.current_commit, default_merge=text_merge())
|
|
82
|
+
print(main["notes"]) # "ALPHA\nBETA\n" -- both edits kept
|
|
83
|
+
```
|
|
84
|
+
|
|
85
|
+
`kvgit.merges.text` is the same merge one level down, over raw bytes,
|
|
86
|
+
for `VersionedKV`. A `Staged` decodes each side before calling a merge
|
|
87
|
+
function, so register `text_merge()` there unless the key's values are
|
|
88
|
+
already `bytes`.
|
|
89
|
+
|
|
90
|
+
Registrations resolve most-specific-first: a key takes its exact-key
|
|
91
|
+
registration, else the longest registered prefix it starts with, else
|
|
92
|
+
`default_merge`. Prefixes cover keys whose names are not known when the
|
|
93
|
+
policy is set:
|
|
94
|
+
|
|
95
|
+
```python
|
|
96
|
+
from kvgit import MergeChoice, text_merge
|
|
97
|
+
|
|
98
|
+
main.set_merge_prefix("runs/", MergeChoice.OURS) # this branch owns runs/
|
|
99
|
+
main.set_merge_fn("runs/index", text_merge()) # except this one key
|
|
72
100
|
```
|
|
73
101
|
|
|
102
|
+
A registration holds either a merge function or a `MergeChoice`, and the
|
|
103
|
+
two reach differently:
|
|
104
|
+
|
|
105
|
+
* A **merge function** is consulted only where both sides changed a key.
|
|
106
|
+
A change only one side made is applied as it always was.
|
|
107
|
+
* A **`MergeChoice`** is a standing policy: it hands that side every key
|
|
108
|
+
either side changed under it. So `OURS` also drops a key the other
|
|
109
|
+
side added and keeps one the other side removed -- what a branch that
|
|
110
|
+
owns a whole namespace needs. Nothing is read or decoded for those
|
|
111
|
+
keys.
|
|
112
|
+
|
|
113
|
+
`kvgit.merges.ours` and `kvgit.merges.theirs` are the merge-function
|
|
114
|
+
form, for keys where one side wins only when both sides collide. They
|
|
115
|
+
keep that side's stored value as it stands rather than rewriting it, so
|
|
116
|
+
the merge writes no new blob -- and if the chosen side removed the key,
|
|
117
|
+
the merge removes it.
|
|
118
|
+
|
|
119
|
+
`commit()` and `merge()` take `merge_fns=` / `merge_prefixes=` /
|
|
120
|
+
`default_merge=` for one call, layered over what is registered.
|
|
121
|
+
|
|
122
|
+
Keys both sides changed to the same bytes merge cleanly with no merge
|
|
123
|
+
function, even though each side wrote its own copy.
|
|
124
|
+
|
|
74
125
|
A `post_check(key, merged_bytes)` predicate runs over every
|
|
75
126
|
merge-produced value; returning `False` files that key as conflicted.
|
|
76
127
|
`on_conflict="abandon"` leaves the branch untouched instead of raising.
|
|
@@ -1,19 +1,28 @@
|
|
|
1
1
|
"""kvgit: Versioned key-value store."""
|
|
2
2
|
|
|
3
|
-
from .content_types import MergeFn, counter, last_writer_wins
|
|
4
|
-
from .errors import ConcurrencyError, MergeConflict
|
|
5
|
-
from .merges import CantMark, TextMergeFn, make_text_merge, text
|
|
3
|
+
from .content_types import MergeFn, counter, last_writer_wins, text_merge
|
|
4
|
+
from .errors import ConcurrencyError, GcBusy, MergeConflict
|
|
5
|
+
from .merges import CantMark, TextMergeFn, make_text_merge, ours, text, theirs
|
|
6
6
|
from .namespaced import Namespaced
|
|
7
7
|
from .staged import Staged
|
|
8
8
|
from .store import delete_branches, delete_tags, store
|
|
9
9
|
from .versioned.kv import VersionedKV
|
|
10
|
-
from .versioned.protocol import
|
|
10
|
+
from .versioned.protocol import (
|
|
11
|
+
MergeChoice,
|
|
12
|
+
MergePolicy,
|
|
13
|
+
MergeResult,
|
|
14
|
+
TagInfo,
|
|
15
|
+
Versioned,
|
|
16
|
+
)
|
|
11
17
|
|
|
12
18
|
__all__ = [
|
|
13
19
|
"CantMark",
|
|
14
20
|
"ConcurrencyError",
|
|
21
|
+
"GcBusy",
|
|
22
|
+
"MergeChoice",
|
|
15
23
|
"MergeConflict",
|
|
16
24
|
"MergeFn",
|
|
25
|
+
"MergePolicy",
|
|
17
26
|
"MergeResult",
|
|
18
27
|
"Namespaced",
|
|
19
28
|
"Staged",
|
|
@@ -26,6 +35,9 @@ __all__ = [
|
|
|
26
35
|
"delete_tags",
|
|
27
36
|
"last_writer_wins",
|
|
28
37
|
"make_text_merge",
|
|
38
|
+
"ours",
|
|
29
39
|
"store",
|
|
30
40
|
"text",
|
|
41
|
+
"text_merge",
|
|
42
|
+
"theirs",
|
|
31
43
|
]
|
|
@@ -0,0 +1,69 @@
|
|
|
1
|
+
"""Merge functions for typed values."""
|
|
2
|
+
|
|
3
|
+
from collections.abc import Callable
|
|
4
|
+
from typing import Any
|
|
5
|
+
|
|
6
|
+
from .merges import CantMark, make_text_merge
|
|
7
|
+
|
|
8
|
+
MergeFn = Callable[[Any | None, Any, Any], Any]
|
|
9
|
+
"""Merge function: (old_value | None, our_value, their_value) -> merged_value.
|
|
10
|
+
|
|
11
|
+
Any argument can be None (key absent or removed on that side).
|
|
12
|
+
|
|
13
|
+
Returning a :class:`~kvgit.versioned.protocol.MergeChoice` instead of a
|
|
14
|
+
value keeps that side's committed value as it stands: no value is
|
|
15
|
+
encoded and the merge writes no new blob for the key.
|
|
16
|
+
"""
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
def counter() -> MergeFn:
|
|
20
|
+
"""Counter merge: ours + theirs - old."""
|
|
21
|
+
|
|
22
|
+
def merge(old: int | None, ours: int, theirs: int) -> int:
|
|
23
|
+
base = old if old is not None else 0
|
|
24
|
+
return ours + theirs - base
|
|
25
|
+
|
|
26
|
+
return merge
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
def last_writer_wins() -> MergeFn:
|
|
30
|
+
"""Last-writer-wins: always returns theirs."""
|
|
31
|
+
return lambda old, ours, theirs: theirs
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
def text_merge(*, ours_label: str = "ours", theirs_label: str = "theirs") -> MergeFn:
|
|
35
|
+
"""Marker merge for ``str`` (or ``bytes``) values, for use with ``Staged``.
|
|
36
|
+
|
|
37
|
+
The value-level counterpart to :func:`kvgit.merges.text`. ``Staged``
|
|
38
|
+
decodes both sides before calling a merge function, so a key holding
|
|
39
|
+
``str`` reaches a bytes-level function as ``str`` and fails there;
|
|
40
|
+
this one encodes ``str`` sides as UTF-8, marker-merges, and decodes
|
|
41
|
+
the result back to ``str`` when any side was ``str``. A key whose
|
|
42
|
+
values are ``bytes`` merges as bytes and comes back as ``bytes``.
|
|
43
|
+
|
|
44
|
+
Disjoint line changes merge cleanly and overlapping ones come back
|
|
45
|
+
with git-style ``<<<<<<<`` markers (labelled by ``ours_label`` /
|
|
46
|
+
``theirs_label``) rather than raising. A value that is neither
|
|
47
|
+
``str`` nor ``bytes``, and anything the marker merge cannot handle —
|
|
48
|
+
undecodable bytes, NUL bytes, inputs over the size cap — raises
|
|
49
|
+
:class:`~kvgit.merges.CantMark`, which the merge machinery files as
|
|
50
|
+
an ordinary conflict.
|
|
51
|
+
"""
|
|
52
|
+
merge_bytes = make_text_merge(ours_label=ours_label, theirs_label=theirs_label)
|
|
53
|
+
|
|
54
|
+
def encode(value: Any) -> bytes | None:
|
|
55
|
+
if value is None or isinstance(value, bytes):
|
|
56
|
+
return value
|
|
57
|
+
if isinstance(value, str):
|
|
58
|
+
return value.encode("utf-8")
|
|
59
|
+
raise CantMark(f"not text: {type(value).__name__}")
|
|
60
|
+
|
|
61
|
+
def merge(old: Any, ours: Any, theirs: Any) -> str | bytes:
|
|
62
|
+
# One side holding str is enough to make the merged value str:
|
|
63
|
+
# the sides are the same key's value at different commits, so a
|
|
64
|
+
# bytes side is the same text in another spelling.
|
|
65
|
+
as_str = any(isinstance(v, str) for v in (old, ours, theirs))
|
|
66
|
+
merged = merge_bytes(encode(old), encode(ours), encode(theirs))
|
|
67
|
+
return merged.decode("utf-8") if as_str else merged
|
|
68
|
+
|
|
69
|
+
return merge
|
|
@@ -26,3 +26,13 @@ class MergeConflict(Exception):
|
|
|
26
26
|
self.merge_errors = merge_errors or {}
|
|
27
27
|
keys_str = ", ".join(sorted(conflicting_keys))
|
|
28
28
|
super().__init__(f"Merge conflict on keys: {keys_str}")
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
class GcBusy(Exception):
|
|
32
|
+
"""Raised when a deep clean cannot take the store's GC lease.
|
|
33
|
+
|
|
34
|
+
Another process or thread holds an unexpired lease, so a namespace
|
|
35
|
+
scan started now would run beside a sweep that is already deleting.
|
|
36
|
+
Retry later; the lease carries an expiry, so a holder that dies
|
|
37
|
+
without releasing it blocks nothing past that point.
|
|
38
|
+
"""
|
|
@@ -20,15 +20,45 @@ def _is_bug(exc: BaseException) -> bool:
|
|
|
20
20
|
return isinstance(exc, _BUG_EXCEPTIONS)
|
|
21
21
|
|
|
22
22
|
|
|
23
|
+
MUTABLE_PREFIX = "__"
|
|
24
|
+
"""Key prefix marking a value that may change under a fixed key.
|
|
25
|
+
|
|
26
|
+
Every mutable key kvgit writes starts with it — branch heads, their
|
|
27
|
+
prev-HEAD backups, the storage version stamp, the GC lease — and so does
|
|
28
|
+
commit metadata, which is immutable but small enough that reading it
|
|
29
|
+
from the authoritative tier costs little. Everything else is derived
|
|
30
|
+
from its own content (``kvgit:keyset:``, ``kvgit:chunk:``,
|
|
31
|
+
``<commit>:<key>``): the same key always holds the same bytes, which is
|
|
32
|
+
what makes it cacheable.
|
|
33
|
+
"""
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
def _is_mutable(key: str) -> bool:
|
|
37
|
+
return isinstance(key, str) and key.startswith(MUTABLE_PREFIX)
|
|
38
|
+
|
|
39
|
+
|
|
23
40
|
class Composite(KVStore):
|
|
24
41
|
"""N-tier cache composing any number of KV stores.
|
|
25
42
|
|
|
26
|
-
|
|
27
|
-
|
|
43
|
+
**Cache tiers serve only immutable, content-derived keys.** A key
|
|
44
|
+
starting with ``__`` names a value that changes under a fixed key —
|
|
45
|
+
a branch head, its recovery backup, the storage version stamp, the
|
|
46
|
+
GC lease — so ``get``, ``get_many`` and ``__contains__`` read those
|
|
47
|
+
from Ln alone and never populate a cache with them. Cached, they
|
|
48
|
+
would let a process keep serving a branch head that another process
|
|
49
|
+
has already moved: the handle would take a ``ConcurrencyError`` on
|
|
50
|
+
commit, call ``refresh()``, and read the same stale head back out of
|
|
51
|
+
L1. Everything else is keyed by its own content
|
|
52
|
+
(``kvgit:keyset:``, ``kvgit:chunk:``, ``<commit>:<key>``), so a hit
|
|
53
|
+
at any tier is the right answer forever.
|
|
54
|
+
|
|
55
|
+
On get: for a ``__`` key, read Ln. Otherwise check L1, L2, ..., Ln
|
|
56
|
+
in order; on hit at tier i, populate L1..L(i-1) and return.
|
|
28
57
|
|
|
29
58
|
On set: write to all tiers (most durable first).
|
|
30
59
|
|
|
31
|
-
On cas: delegate to Ln (authoritative), update caches on success
|
|
60
|
+
On cas: delegate to Ln (authoritative), update caches on success —
|
|
61
|
+
except for ``__`` keys, which no cache tier serves.
|
|
32
62
|
|
|
33
63
|
Tier failures (``OSError``, network errors, etc.) are logged at
|
|
34
64
|
WARNING and the next tier is tried; programming-error exceptions
|
|
@@ -54,8 +84,20 @@ class Composite(KVStore):
|
|
|
54
84
|
raise
|
|
55
85
|
logger.warning("Composite cache populate failed at tier %d: %s", j, e)
|
|
56
86
|
|
|
87
|
+
def _first_read_tier(self, key: str) -> int:
|
|
88
|
+
"""Index of the highest tier allowed to answer for ``key``.
|
|
89
|
+
|
|
90
|
+
A ``__``-prefixed key names a value that can change under a
|
|
91
|
+
fixed key, so only the authoritative tier may answer it;
|
|
92
|
+
everything else is content-derived and may be served by any
|
|
93
|
+
tier.
|
|
94
|
+
"""
|
|
95
|
+
return len(self._stores) - 1 if _is_mutable(key) else 0
|
|
96
|
+
|
|
57
97
|
def get(self, key: str) -> bytes | None:
|
|
58
|
-
|
|
98
|
+
start = self._first_read_tier(key)
|
|
99
|
+
for i in range(start, len(self._stores)):
|
|
100
|
+
store = self._stores[i]
|
|
59
101
|
try:
|
|
60
102
|
value = store.get(key)
|
|
61
103
|
except Exception as e:
|
|
@@ -64,17 +106,21 @@ class Composite(KVStore):
|
|
|
64
106
|
logger.warning("Composite get failed at tier %d for %r: %s", i, key, e)
|
|
65
107
|
continue
|
|
66
108
|
if value is not None:
|
|
67
|
-
|
|
109
|
+
# Nothing to populate when the read started at the
|
|
110
|
+
# authoritative tier, which is where a mutable key's
|
|
111
|
+
# read starts and ends.
|
|
112
|
+
if i > start:
|
|
68
113
|
self._populate_caches(i, {key: value})
|
|
69
114
|
return value
|
|
70
115
|
return None
|
|
71
116
|
|
|
72
|
-
def
|
|
117
|
+
def _get_many_from(self, remaining: set[str], start: int) -> dict[str, bytes]:
|
|
118
|
+
"""Bulk read ``remaining``, consulting tiers ``start``..Ln."""
|
|
73
119
|
result: dict[str, bytes] = {}
|
|
74
|
-
|
|
75
|
-
for i, store in enumerate(self._stores):
|
|
120
|
+
for i in range(start, len(self._stores)):
|
|
76
121
|
if not remaining:
|
|
77
122
|
break
|
|
123
|
+
store = self._stores[i]
|
|
78
124
|
try:
|
|
79
125
|
# Delegate to the tier's bulk get — backends with high
|
|
80
126
|
# per-call latency (Disk, IndexedDB) collapse N round-trips
|
|
@@ -86,16 +132,28 @@ class Composite(KVStore):
|
|
|
86
132
|
raise
|
|
87
133
|
logger.warning("Composite get_many failed at tier %d: %s", i, e)
|
|
88
134
|
continue
|
|
89
|
-
if tier_values and i >
|
|
135
|
+
if tier_values and i > start:
|
|
90
136
|
self._populate_caches(i, tier_values)
|
|
91
137
|
result.update(tier_values)
|
|
92
|
-
remaining
|
|
138
|
+
remaining = remaining - tier_values.keys()
|
|
139
|
+
return result
|
|
140
|
+
|
|
141
|
+
def get_many(self, *args) -> Mapping[str, bytes]:
|
|
142
|
+
keys = set(self._normalize_keys(args))
|
|
143
|
+
mutable = {key for key in keys if _is_mutable(key)}
|
|
144
|
+
if not mutable:
|
|
145
|
+
return self._get_many_from(keys, 0)
|
|
146
|
+
# Two passes rather than one: mutable keys must come from the
|
|
147
|
+
# authoritative tier and must not be written into a cache, so
|
|
148
|
+
# they cannot share a batch with content-derived keys.
|
|
149
|
+
result = self._get_many_from(mutable, len(self._stores) - 1)
|
|
150
|
+
result.update(self._get_many_from(keys - mutable, 0))
|
|
93
151
|
return result
|
|
94
152
|
|
|
95
153
|
def __contains__(self, key: str) -> bool:
|
|
96
|
-
for i
|
|
154
|
+
for i in range(self._first_read_tier(key), len(self._stores)):
|
|
97
155
|
try:
|
|
98
|
-
if key in
|
|
156
|
+
if key in self._stores[i]:
|
|
99
157
|
return True
|
|
100
158
|
except Exception as e:
|
|
101
159
|
if _is_bug(e):
|
|
@@ -175,7 +233,10 @@ class Composite(KVStore):
|
|
|
175
233
|
|
|
176
234
|
def cas(self, key: str, value: bytes, expected: bytes | None) -> bool:
|
|
177
235
|
success = self._stores[-1].cas(key, value, expected)
|
|
178
|
-
|
|
236
|
+
# A ``__``-prefixed key is read from the authoritative tier
|
|
237
|
+
# only, so writing it into a cache stores bytes nothing will
|
|
238
|
+
# ever read, and a CAS is almost always against such a key.
|
|
239
|
+
if success and not _is_mutable(key):
|
|
179
240
|
for i, store in enumerate(self._stores[:-1]):
|
|
180
241
|
try:
|
|
181
242
|
store.set(key, value)
|
|
@@ -9,6 +9,9 @@ Anything that cannot be marked — undecodable bytes, NUL bytes, inputs
|
|
|
9
9
|
over the size cap — raises :class:`CantMark`, which the merge machinery
|
|
10
10
|
files as an ordinary conflict. Callers that need to tell "no encoding"
|
|
11
11
|
from "too big" catch it themselves; callers that don't let it flow.
|
|
12
|
+
|
|
13
|
+
Also here: :func:`ours` and :func:`theirs`, the pick-a-side merges, for
|
|
14
|
+
keys where one branch simply owns the value.
|
|
12
15
|
"""
|
|
13
16
|
|
|
14
17
|
from __future__ import annotations
|
|
@@ -16,7 +19,7 @@ from __future__ import annotations
|
|
|
16
19
|
import difflib
|
|
17
20
|
from collections.abc import Callable
|
|
18
21
|
|
|
19
|
-
from .versioned.protocol import BytesMergeFn
|
|
22
|
+
from .versioned.protocol import BytesMergeFn, MergeChoice
|
|
20
23
|
|
|
21
24
|
#: Inputs whose combined size exceeds this are refused with
|
|
22
25
|
#: :class:`CantMark` rather than merged. Line-diffing is quadratic-ish
|
|
@@ -245,3 +248,24 @@ def text(old: bytes | None, ours: bytes | None, theirs: bytes | None) -> bytes:
|
|
|
245
248
|
|
|
246
249
|
TextMergeFn = Callable[[bytes | None, bytes | None, bytes | None], bytes]
|
|
247
250
|
"""Type of what :func:`make_text_merge` builds (== ``BytesMergeFn``)."""
|
|
251
|
+
|
|
252
|
+
|
|
253
|
+
def ours(old: bytes | None, our: bytes | None, their: bytes | None) -> MergeChoice:
|
|
254
|
+
"""Take our side: the merged value is our committed value.
|
|
255
|
+
|
|
256
|
+
Keeps our existing blob pointer rather than rewriting the value, so
|
|
257
|
+
the merge writes nothing for this key. If we removed the key, the
|
|
258
|
+
merge removes it. Usable as a per-key, per-prefix, or default merge.
|
|
259
|
+
"""
|
|
260
|
+
return MergeChoice.OURS
|
|
261
|
+
|
|
262
|
+
|
|
263
|
+
def theirs(old: bytes | None, our: bytes | None, their: bytes | None) -> MergeChoice:
|
|
264
|
+
"""Take their side: the merged value is their committed value.
|
|
265
|
+
|
|
266
|
+
Keeps their existing blob pointer rather than rewriting the value,
|
|
267
|
+
so the merge writes nothing for this key. If they removed the key,
|
|
268
|
+
the merge removes it. Usable as a per-key, per-prefix, or default
|
|
269
|
+
merge.
|
|
270
|
+
"""
|
|
271
|
+
return MergeChoice.THEIRS
|
|
@@ -10,6 +10,7 @@ from .content_types import MergeFn
|
|
|
10
10
|
from .versioned.kv import CHUNK_PREFIX, VersionedKV
|
|
11
11
|
from .versioned.protocol import (
|
|
12
12
|
BytesMergeFn,
|
|
13
|
+
MergeChoice,
|
|
13
14
|
MergeResult,
|
|
14
15
|
PostCheck,
|
|
15
16
|
TagInfo,
|
|
@@ -140,8 +141,9 @@ class Staged(MutableMapping[str, Any]):
|
|
|
140
141
|
self._updates: dict[str, Any] = {}
|
|
141
142
|
self._removals: set[str] = set()
|
|
142
143
|
self._cache: dict[str, Any] = {}
|
|
143
|
-
self._merge_fns: dict[str, MergeFn] = {}
|
|
144
|
-
self.
|
|
144
|
+
self._merge_fns: dict[str, MergeFn | MergeChoice] = {}
|
|
145
|
+
self._merge_prefixes: dict[str, MergeFn | MergeChoice] = {}
|
|
146
|
+
self._default_merge: MergeFn | MergeChoice | None = None
|
|
145
147
|
|
|
146
148
|
def _decode(self, raw: bytes) -> Any:
|
|
147
149
|
if self._decoder_chunked:
|
|
@@ -226,11 +228,28 @@ class Staged(MutableMapping[str, Any]):
|
|
|
226
228
|
|
|
227
229
|
# -- Merge function registry --
|
|
228
230
|
|
|
229
|
-
def set_merge_fn(self, key: str, fn: MergeFn) -> None:
|
|
230
|
-
"""Register a merge function
|
|
231
|
+
def set_merge_fn(self, key: str, fn: MergeFn | MergeChoice) -> None:
|
|
232
|
+
"""Register a merge function, or a ``MergeChoice``, for one key."""
|
|
231
233
|
self._merge_fns[key] = fn
|
|
232
234
|
|
|
233
|
-
def
|
|
235
|
+
def set_merge_prefix(self, prefix: str, fn: MergeFn | MergeChoice) -> None:
|
|
236
|
+
"""Register a merge function, or a ``MergeChoice``, under a prefix.
|
|
237
|
+
|
|
238
|
+
Prefixes cover keys whose names are not known when the policy is
|
|
239
|
+
set (``"runs/"`` for ``runs/<id>``). A key takes the most
|
|
240
|
+
specific registration: its exact key, else the longest
|
|
241
|
+
registered prefix it starts with, else the default.
|
|
242
|
+
|
|
243
|
+
What the registration holds decides how far it reaches. A merge
|
|
244
|
+
function is consulted only where both sides changed a key. A
|
|
245
|
+
``MergeChoice`` is a standing policy over the whole prefix: it
|
|
246
|
+
gives that side every key either side changed under it, so
|
|
247
|
+
``OURS`` also drops a key the other side added and keeps one the
|
|
248
|
+
other side removed.
|
|
249
|
+
"""
|
|
250
|
+
self._merge_prefixes[prefix] = fn
|
|
251
|
+
|
|
252
|
+
def set_default_merge(self, fn: MergeFn | MergeChoice) -> None:
|
|
234
253
|
"""Register a default merge function."""
|
|
235
254
|
self._default_merge = fn
|
|
236
255
|
|
|
@@ -245,19 +264,38 @@ class Staged(MutableMapping[str, Any]):
|
|
|
245
264
|
outputs is not supported in v1; merge outputs are stored as
|
|
246
265
|
opaque blobs. Subsequent commits that overwrite the merged key
|
|
247
266
|
do go through the chunked path normally.
|
|
267
|
+
|
|
268
|
+
A ``MergeChoice`` comes back untouched. It names a side to keep
|
|
269
|
+
rather than a value, so encoding it would write a blob holding
|
|
270
|
+
the sentinel itself instead of keeping the side's own value.
|
|
248
271
|
"""
|
|
249
272
|
decode = self._decode
|
|
250
273
|
|
|
251
274
|
def wrapped(
|
|
252
275
|
old: bytes | None, ours: bytes | None, theirs: bytes | None
|
|
253
|
-
) -> bytes:
|
|
276
|
+
) -> bytes | MergeChoice:
|
|
254
277
|
old_val = decode(old) if old is not None else None
|
|
255
278
|
ours_val = decode(ours) if ours is not None else None
|
|
256
279
|
theirs_val = decode(theirs) if theirs is not None else None
|
|
257
|
-
|
|
280
|
+
result = fn(old_val, ours_val, theirs_val)
|
|
281
|
+
if isinstance(result, MergeChoice):
|
|
282
|
+
return result
|
|
283
|
+
return pickle.dumps(result)
|
|
258
284
|
|
|
259
285
|
return wrapped
|
|
260
286
|
|
|
287
|
+
def _wrap_rule(self, rule: MergeFn | MergeChoice) -> BytesMergeFn | MergeChoice:
|
|
288
|
+
"""Prepare one merge registration for the bytes-level merge.
|
|
289
|
+
|
|
290
|
+
A ``MergeChoice`` names a side rather than computing a value, so
|
|
291
|
+
it goes down untouched — there is nothing to decode for the keys
|
|
292
|
+
it governs, and wrapping it would hide that from the resolver.
|
|
293
|
+
A merge function gets the decode/encode wrapper.
|
|
294
|
+
"""
|
|
295
|
+
if isinstance(rule, MergeChoice):
|
|
296
|
+
return rule
|
|
297
|
+
return self._wrap_merge_fn(rule)
|
|
298
|
+
|
|
261
299
|
# -- Commit / reset --
|
|
262
300
|
|
|
263
301
|
def commit(
|
|
@@ -265,8 +303,9 @@ class Staged(MutableMapping[str, Any]):
|
|
|
265
303
|
*,
|
|
266
304
|
keys: set[str] | None = None,
|
|
267
305
|
on_conflict: str = "raise",
|
|
268
|
-
merge_fns: dict[str, MergeFn] | None = None,
|
|
269
|
-
|
|
306
|
+
merge_fns: dict[str, MergeFn | MergeChoice] | None = None,
|
|
307
|
+
merge_prefixes: dict[str, MergeFn | MergeChoice] | None = None,
|
|
308
|
+
default_merge: MergeFn | MergeChoice | None = None,
|
|
270
309
|
info: dict | None = None,
|
|
271
310
|
) -> MergeResult:
|
|
272
311
|
"""Flush staged changes to the underlying Versioned store.
|
|
@@ -325,23 +364,34 @@ class Staged(MutableMapping[str, Any]):
|
|
|
325
364
|
effective_fns = dict(self._merge_fns)
|
|
326
365
|
if merge_fns:
|
|
327
366
|
effective_fns.update(merge_fns)
|
|
367
|
+
effective_prefixes = dict(self._merge_prefixes)
|
|
368
|
+
if merge_prefixes:
|
|
369
|
+
effective_prefixes.update(merge_prefixes)
|
|
328
370
|
effective_default = default_merge or self._default_merge
|
|
329
371
|
|
|
330
|
-
bytes_merge_fns: dict[str, BytesMergeFn] | None = None
|
|
372
|
+
bytes_merge_fns: dict[str, BytesMergeFn | MergeChoice] | None = None
|
|
331
373
|
if effective_fns:
|
|
332
374
|
bytes_merge_fns = {
|
|
333
|
-
key: self.
|
|
375
|
+
key: self._wrap_rule(rule) for key, rule in effective_fns.items()
|
|
376
|
+
}
|
|
377
|
+
|
|
378
|
+
bytes_prefixes: dict[str, BytesMergeFn | MergeChoice] | None = None
|
|
379
|
+
if effective_prefixes:
|
|
380
|
+
bytes_prefixes = {
|
|
381
|
+
prefix: self._wrap_rule(rule)
|
|
382
|
+
for prefix, rule in effective_prefixes.items()
|
|
334
383
|
}
|
|
335
384
|
|
|
336
|
-
bytes_default: BytesMergeFn | None = None
|
|
337
|
-
if effective_default:
|
|
338
|
-
bytes_default = self.
|
|
385
|
+
bytes_default: BytesMergeFn | MergeChoice | None = None
|
|
386
|
+
if effective_default is not None:
|
|
387
|
+
bytes_default = self._wrap_rule(effective_default)
|
|
339
388
|
|
|
340
389
|
result = self._versioned.commit(
|
|
341
390
|
encoded_updates,
|
|
342
391
|
removals,
|
|
343
392
|
on_conflict=on_conflict,
|
|
344
393
|
merge_fns=bytes_merge_fns,
|
|
394
|
+
merge_prefixes=bytes_prefixes,
|
|
345
395
|
default_merge=bytes_default,
|
|
346
396
|
info=info,
|
|
347
397
|
chunks=chunks,
|
|
@@ -366,8 +416,9 @@ class Staged(MutableMapping[str, Any]):
|
|
|
366
416
|
their_head: str,
|
|
367
417
|
*,
|
|
368
418
|
on_conflict: str = "raise",
|
|
369
|
-
merge_fns: dict[str, MergeFn] | None = None,
|
|
370
|
-
|
|
419
|
+
merge_fns: dict[str, MergeFn | MergeChoice] | None = None,
|
|
420
|
+
merge_prefixes: dict[str, MergeFn | MergeChoice] | None = None,
|
|
421
|
+
default_merge: MergeFn | MergeChoice | None = None,
|
|
371
422
|
post_check: PostCheck | None = None,
|
|
372
423
|
info: dict | None = None,
|
|
373
424
|
) -> MergeResult:
|
|
@@ -378,8 +429,10 @@ class Staged(MutableMapping[str, Any]):
|
|
|
378
429
|
when the staging buffer holds uncommitted changes — commit or
|
|
379
430
|
reset first, so the merge reads committed heads on both sides.
|
|
380
431
|
|
|
381
|
-
``merge_fns`` / ``default_merge`` take
|
|
382
|
-
|
|
432
|
+
``merge_fns`` / ``merge_prefixes`` / ``default_merge`` take
|
|
433
|
+
merge functions over decoded values (wrapped like
|
|
434
|
+
:meth:`commit`) or a ``MergeChoice``; ``post_check`` takes
|
|
435
|
+
merged bytes.
|
|
383
436
|
"""
|
|
384
437
|
if self._updates or self._removals:
|
|
385
438
|
raise ValueError("cannot merge with staged changes; commit or reset first")
|
|
@@ -388,19 +441,33 @@ class Staged(MutableMapping[str, Any]):
|
|
|
388
441
|
effective_fns = dict(self._merge_fns)
|
|
389
442
|
if merge_fns:
|
|
390
443
|
effective_fns.update(merge_fns)
|
|
444
|
+
effective_prefixes = dict(self._merge_prefixes)
|
|
445
|
+
if merge_prefixes:
|
|
446
|
+
effective_prefixes.update(merge_prefixes)
|
|
391
447
|
effective_default = default_merge or self._default_merge
|
|
392
448
|
bytes_merge_fns = (
|
|
393
|
-
{key: self.
|
|
449
|
+
{key: self._wrap_rule(rule) for key, rule in effective_fns.items()}
|
|
394
450
|
if effective_fns
|
|
395
451
|
else None
|
|
396
452
|
)
|
|
453
|
+
bytes_prefixes = (
|
|
454
|
+
{
|
|
455
|
+
prefix: self._wrap_rule(rule)
|
|
456
|
+
for prefix, rule in effective_prefixes.items()
|
|
457
|
+
}
|
|
458
|
+
if effective_prefixes
|
|
459
|
+
else None
|
|
460
|
+
)
|
|
397
461
|
bytes_default = (
|
|
398
|
-
self.
|
|
462
|
+
self._wrap_rule(effective_default)
|
|
463
|
+
if effective_default is not None
|
|
464
|
+
else None
|
|
399
465
|
)
|
|
400
466
|
result = self._versioned.merge_heads(
|
|
401
467
|
their_head,
|
|
402
468
|
on_conflict=on_conflict,
|
|
403
469
|
merge_fns=bytes_merge_fns,
|
|
470
|
+
merge_prefixes=bytes_prefixes,
|
|
404
471
|
default_merge=bytes_default,
|
|
405
472
|
post_check=post_check,
|
|
406
473
|
info=info,
|