kvgit 0.3.7__tar.gz → 0.3.9__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {kvgit-0.3.7 → kvgit-0.3.9}/PKG-INFO +58 -7
- {kvgit-0.3.7 → kvgit-0.3.9}/README.md +57 -6
- kvgit-0.3.9/kvgit/__init__.py +53 -0
- kvgit-0.3.9/kvgit/content_types.py +78 -0
- {kvgit-0.3.7 → kvgit-0.3.9}/kvgit/errors.py +19 -0
- {kvgit-0.3.7 → kvgit-0.3.9}/kvgit/kv/composite.py +74 -13
- {kvgit-0.3.7 → kvgit-0.3.9}/kvgit/merges.py +101 -25
- {kvgit-0.3.7 → kvgit-0.3.9}/kvgit/staged.py +95 -20
- {kvgit-0.3.7 → kvgit-0.3.9}/kvgit/store.py +10 -3
- {kvgit-0.3.7 → kvgit-0.3.9}/kvgit/versioned/__init__.py +11 -1
- {kvgit-0.3.7 → kvgit-0.3.9}/kvgit/versioned/base.py +96 -35
- {kvgit-0.3.7 → kvgit-0.3.9}/kvgit/versioned/kv.py +436 -79
- kvgit-0.3.9/kvgit/versioned/merge.py +222 -0
- {kvgit-0.3.7 → kvgit-0.3.9}/kvgit/versioned/protocol.py +58 -8
- {kvgit-0.3.7 → kvgit-0.3.9}/kvgit.egg-info/PKG-INFO +58 -7
- {kvgit-0.3.7 → kvgit-0.3.9}/kvgit.egg-info/SOURCES.txt +1 -0
- {kvgit-0.3.7 → kvgit-0.3.9}/pyproject.toml +1 -1
- kvgit-0.3.9/tests/test_content_types.py +213 -0
- kvgit-0.3.9/tests/test_merge_policy.py +796 -0
- {kvgit-0.3.7 → kvgit-0.3.9}/tests/test_merges.py +121 -1
- {kvgit-0.3.7 → kvgit-0.3.9}/tests/test_staged.py +34 -0
- {kvgit-0.3.7 → kvgit-0.3.9}/tests/test_store_factory.py +18 -0
- kvgit-0.3.7/kvgit/__init__.py +0 -31
- kvgit-0.3.7/kvgit/content_types.py +0 -25
- kvgit-0.3.7/kvgit/versioned/merge.py +0 -131
- kvgit-0.3.7/tests/test_content_types.py +0 -87
- {kvgit-0.3.7 → kvgit-0.3.9}/LICENSE +0 -0
- {kvgit-0.3.7 → kvgit-0.3.9}/kvgit/codecs/__init__.py +0 -0
- {kvgit-0.3.7 → kvgit-0.3.9}/kvgit/codecs/_hash.py +0 -0
- {kvgit-0.3.7 → kvgit-0.3.9}/kvgit/codecs/base.py +0 -0
- {kvgit-0.3.7 → kvgit-0.3.9}/kvgit/codecs/numpy.py +0 -0
- {kvgit-0.3.7 → kvgit-0.3.9}/kvgit/codecs/pandas.py +0 -0
- {kvgit-0.3.7 → kvgit-0.3.9}/kvgit/codecs/pickler.py +0 -0
- {kvgit-0.3.7 → kvgit-0.3.9}/kvgit/encoding.py +0 -0
- {kvgit-0.3.7 → kvgit-0.3.9}/kvgit/hamt.py +0 -0
- {kvgit-0.3.7 → kvgit-0.3.9}/kvgit/kv/__init__.py +0 -0
- {kvgit-0.3.7 → kvgit-0.3.9}/kvgit/kv/base.py +0 -0
- {kvgit-0.3.7 → kvgit-0.3.9}/kvgit/kv/disk.py +0 -0
- {kvgit-0.3.7 → kvgit-0.3.9}/kvgit/kv/indexeddb.py +0 -0
- {kvgit-0.3.7 → kvgit-0.3.9}/kvgit/kv/memory.py +0 -0
- {kvgit-0.3.7 → kvgit-0.3.9}/kvgit/namespaced.py +0 -0
- {kvgit-0.3.7 → kvgit-0.3.9}/kvgit/py.typed +0 -0
- {kvgit-0.3.7 → kvgit-0.3.9}/kvgit/versioned/helpers.py +0 -0
- {kvgit-0.3.7 → kvgit-0.3.9}/kvgit/versioned/keyset.py +0 -0
- {kvgit-0.3.7 → kvgit-0.3.9}/kvgit.egg-info/dependency_links.txt +0 -0
- {kvgit-0.3.7 → kvgit-0.3.9}/kvgit.egg-info/requires.txt +0 -0
- {kvgit-0.3.7 → kvgit-0.3.9}/kvgit.egg-info/top_level.txt +0 -0
- {kvgit-0.3.7 → kvgit-0.3.9}/setup.cfg +0 -0
- {kvgit-0.3.7 → kvgit-0.3.9}/tests/test_delete_branches.py +0 -0
- {kvgit-0.3.7 → kvgit-0.3.9}/tests/test_hamt.py +0 -0
- {kvgit-0.3.7 → kvgit-0.3.9}/tests/test_namespaced.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: kvgit
|
|
3
|
-
Version: 0.3.
|
|
3
|
+
Version: 0.3.9
|
|
4
4
|
Summary: Versioned key-value store with git-like commit, branch, and merge semantics.
|
|
5
5
|
Author: ashenfad
|
|
6
6
|
License: MIT
|
|
@@ -109,17 +109,68 @@ print(main["score"]) # 500 (fast-forward: main hadn't diverged)
|
|
|
109
109
|
```
|
|
110
110
|
|
|
111
111
|
Overlapping changes need a merge function per key, or a `default_merge`
|
|
112
|
-
fallback. `
|
|
113
|
-
`<<<<<<<` markers (
|
|
114
|
-
cannot mark -- binary, non-UTF-8, oversized -- raises
|
|
115
|
-
as an ordinary conflict:
|
|
112
|
+
fallback. `text_merge()` resolves line-oriented text with git-style
|
|
113
|
+
`<<<<<<<` markers (its `ours_label` / `theirs_label` arguments name
|
|
114
|
+
them); anything it cannot mark -- binary, non-UTF-8, oversized -- raises
|
|
115
|
+
`CantMark`, filed as an ordinary conflict:
|
|
116
116
|
|
|
117
117
|
```python
|
|
118
|
-
from kvgit
|
|
118
|
+
from kvgit import text_merge
|
|
119
119
|
|
|
120
|
-
|
|
120
|
+
main["notes"] = "alpha\nbeta\n"
|
|
121
|
+
main.commit()
|
|
122
|
+
|
|
123
|
+
edits = main.create_branch("edits")
|
|
124
|
+
edits["notes"] = "alpha\nBETA\n"
|
|
125
|
+
edits.commit()
|
|
126
|
+
|
|
127
|
+
main["notes"] = "ALPHA\nbeta\n"
|
|
128
|
+
main.commit()
|
|
129
|
+
|
|
130
|
+
main.merge(edits.current_commit, default_merge=text_merge())
|
|
131
|
+
print(main["notes"]) # "ALPHA\nBETA\n" -- both edits kept
|
|
132
|
+
```
|
|
133
|
+
|
|
134
|
+
`kvgit.merges.text` is the same merge one level down, over raw bytes,
|
|
135
|
+
for `VersionedKV`. A `Staged` decodes each side before calling a merge
|
|
136
|
+
function, so register `text_merge()` there unless the key's values are
|
|
137
|
+
already `bytes`.
|
|
138
|
+
|
|
139
|
+
Registrations resolve most-specific-first: a key takes its exact-key
|
|
140
|
+
registration, else the longest registered prefix it starts with, else
|
|
141
|
+
`default_merge`. Prefixes cover keys whose names are not known when the
|
|
142
|
+
policy is set:
|
|
143
|
+
|
|
144
|
+
```python
|
|
145
|
+
from kvgit import MergeChoice, text_merge
|
|
146
|
+
|
|
147
|
+
main.set_merge_prefix("runs/", MergeChoice.OURS) # this branch owns runs/
|
|
148
|
+
main.set_merge_fn("runs/index", text_merge()) # except this one key
|
|
121
149
|
```
|
|
122
150
|
|
|
151
|
+
A registration holds either a merge function or a `MergeChoice`, and the
|
|
152
|
+
two reach differently:
|
|
153
|
+
|
|
154
|
+
* A **merge function** is consulted only where both sides changed a key.
|
|
155
|
+
A change only one side made is applied as it always was.
|
|
156
|
+
* A **`MergeChoice`** is a standing policy: it hands that side every key
|
|
157
|
+
either side changed under it. So `OURS` also drops a key the other
|
|
158
|
+
side added and keeps one the other side removed -- what a branch that
|
|
159
|
+
owns a whole namespace needs. Nothing is read or decoded for those
|
|
160
|
+
keys.
|
|
161
|
+
|
|
162
|
+
`kvgit.merges.ours` and `kvgit.merges.theirs` are the merge-function
|
|
163
|
+
form, for keys where one side wins only when both sides collide. They
|
|
164
|
+
keep that side's stored value as it stands rather than rewriting it, so
|
|
165
|
+
the merge writes no new blob -- and if the chosen side removed the key,
|
|
166
|
+
the merge removes it.
|
|
167
|
+
|
|
168
|
+
`commit()` and `merge()` take `merge_fns=` / `merge_prefixes=` /
|
|
169
|
+
`default_merge=` for one call, layered over what is registered.
|
|
170
|
+
|
|
171
|
+
Keys both sides changed to the same bytes merge cleanly with no merge
|
|
172
|
+
function, even though each side wrote its own copy.
|
|
173
|
+
|
|
123
174
|
A `post_check(key, merged_bytes)` predicate runs over every
|
|
124
175
|
merge-produced value; returning `False` files that key as conflicted.
|
|
125
176
|
`on_conflict="abandon"` leaves the branch untouched instead of raising.
|
|
@@ -60,17 +60,68 @@ print(main["score"]) # 500 (fast-forward: main hadn't diverged)
|
|
|
60
60
|
```
|
|
61
61
|
|
|
62
62
|
Overlapping changes need a merge function per key, or a `default_merge`
|
|
63
|
-
fallback. `
|
|
64
|
-
`<<<<<<<` markers (
|
|
65
|
-
cannot mark -- binary, non-UTF-8, oversized -- raises
|
|
66
|
-
as an ordinary conflict:
|
|
63
|
+
fallback. `text_merge()` resolves line-oriented text with git-style
|
|
64
|
+
`<<<<<<<` markers (its `ours_label` / `theirs_label` arguments name
|
|
65
|
+
them); anything it cannot mark -- binary, non-UTF-8, oversized -- raises
|
|
66
|
+
`CantMark`, filed as an ordinary conflict:
|
|
67
67
|
|
|
68
68
|
```python
|
|
69
|
-
from kvgit
|
|
69
|
+
from kvgit import text_merge
|
|
70
70
|
|
|
71
|
-
|
|
71
|
+
main["notes"] = "alpha\nbeta\n"
|
|
72
|
+
main.commit()
|
|
73
|
+
|
|
74
|
+
edits = main.create_branch("edits")
|
|
75
|
+
edits["notes"] = "alpha\nBETA\n"
|
|
76
|
+
edits.commit()
|
|
77
|
+
|
|
78
|
+
main["notes"] = "ALPHA\nbeta\n"
|
|
79
|
+
main.commit()
|
|
80
|
+
|
|
81
|
+
main.merge(edits.current_commit, default_merge=text_merge())
|
|
82
|
+
print(main["notes"]) # "ALPHA\nBETA\n" -- both edits kept
|
|
83
|
+
```
|
|
84
|
+
|
|
85
|
+
`kvgit.merges.text` is the same merge one level down, over raw bytes,
|
|
86
|
+
for `VersionedKV`. A `Staged` decodes each side before calling a merge
|
|
87
|
+
function, so register `text_merge()` there unless the key's values are
|
|
88
|
+
already `bytes`.
|
|
89
|
+
|
|
90
|
+
Registrations resolve most-specific-first: a key takes its exact-key
|
|
91
|
+
registration, else the longest registered prefix it starts with, else
|
|
92
|
+
`default_merge`. Prefixes cover keys whose names are not known when the
|
|
93
|
+
policy is set:
|
|
94
|
+
|
|
95
|
+
```python
|
|
96
|
+
from kvgit import MergeChoice, text_merge
|
|
97
|
+
|
|
98
|
+
main.set_merge_prefix("runs/", MergeChoice.OURS) # this branch owns runs/
|
|
99
|
+
main.set_merge_fn("runs/index", text_merge()) # except this one key
|
|
72
100
|
```
|
|
73
101
|
|
|
102
|
+
A registration holds either a merge function or a `MergeChoice`, and the
|
|
103
|
+
two reach differently:
|
|
104
|
+
|
|
105
|
+
* A **merge function** is consulted only where both sides changed a key.
|
|
106
|
+
A change only one side made is applied as it always was.
|
|
107
|
+
* A **`MergeChoice`** is a standing policy: it hands that side every key
|
|
108
|
+
either side changed under it. So `OURS` also drops a key the other
|
|
109
|
+
side added and keeps one the other side removed -- what a branch that
|
|
110
|
+
owns a whole namespace needs. Nothing is read or decoded for those
|
|
111
|
+
keys.
|
|
112
|
+
|
|
113
|
+
`kvgit.merges.ours` and `kvgit.merges.theirs` are the merge-function
|
|
114
|
+
form, for keys where one side wins only when both sides collide. They
|
|
115
|
+
keep that side's stored value as it stands rather than rewriting it, so
|
|
116
|
+
the merge writes no new blob -- and if the chosen side removed the key,
|
|
117
|
+
the merge removes it.
|
|
118
|
+
|
|
119
|
+
`commit()` and `merge()` take `merge_fns=` / `merge_prefixes=` /
|
|
120
|
+
`default_merge=` for one call, layered over what is registered.
|
|
121
|
+
|
|
122
|
+
Keys both sides changed to the same bytes merge cleanly with no merge
|
|
123
|
+
function, even though each side wrote its own copy.
|
|
124
|
+
|
|
74
125
|
A `post_check(key, merged_bytes)` predicate runs over every
|
|
75
126
|
merge-produced value; returning `False` files that key as conflicted.
|
|
76
127
|
`on_conflict="abandon"` leaves the branch untouched instead of raising.
|
|
@@ -0,0 +1,53 @@
|
|
|
1
|
+
"""kvgit: Versioned key-value store."""
|
|
2
|
+
|
|
3
|
+
from .content_types import MergeFn, counter, last_writer_wins, text_merge
|
|
4
|
+
from .errors import ConcurrencyError, GcBusy, MergeConflict, UnknownBranchError
|
|
5
|
+
from .merges import (
|
|
6
|
+
CantMark,
|
|
7
|
+
TextMergeFn,
|
|
8
|
+
make_text_merge,
|
|
9
|
+
ours,
|
|
10
|
+
text,
|
|
11
|
+
text_merge_result,
|
|
12
|
+
theirs,
|
|
13
|
+
)
|
|
14
|
+
from .namespaced import Namespaced
|
|
15
|
+
from .staged import Staged
|
|
16
|
+
from .store import delete_branches, delete_tags, store
|
|
17
|
+
from .versioned.kv import VersionedKV
|
|
18
|
+
from .versioned.protocol import (
|
|
19
|
+
MergeChoice,
|
|
20
|
+
MergePolicy,
|
|
21
|
+
MergeResult,
|
|
22
|
+
TagInfo,
|
|
23
|
+
Versioned,
|
|
24
|
+
)
|
|
25
|
+
|
|
26
|
+
__all__ = [
|
|
27
|
+
"CantMark",
|
|
28
|
+
"ConcurrencyError",
|
|
29
|
+
"GcBusy",
|
|
30
|
+
"MergeChoice",
|
|
31
|
+
"MergeConflict",
|
|
32
|
+
"MergeFn",
|
|
33
|
+
"MergePolicy",
|
|
34
|
+
"MergeResult",
|
|
35
|
+
"Namespaced",
|
|
36
|
+
"Staged",
|
|
37
|
+
"TagInfo",
|
|
38
|
+
"TextMergeFn",
|
|
39
|
+
"UnknownBranchError",
|
|
40
|
+
"Versioned",
|
|
41
|
+
"VersionedKV",
|
|
42
|
+
"counter",
|
|
43
|
+
"delete_branches",
|
|
44
|
+
"delete_tags",
|
|
45
|
+
"last_writer_wins",
|
|
46
|
+
"make_text_merge",
|
|
47
|
+
"ours",
|
|
48
|
+
"store",
|
|
49
|
+
"text",
|
|
50
|
+
"text_merge",
|
|
51
|
+
"text_merge_result",
|
|
52
|
+
"theirs",
|
|
53
|
+
]
|
|
@@ -0,0 +1,78 @@
|
|
|
1
|
+
"""Merge functions for typed values."""
|
|
2
|
+
|
|
3
|
+
from collections.abc import Callable
|
|
4
|
+
from typing import Any
|
|
5
|
+
|
|
6
|
+
from .merges import CantMark, make_text_merge
|
|
7
|
+
|
|
8
|
+
MergeFn = Callable[[Any | None, Any, Any], Any]
|
|
9
|
+
"""Merge function: (old_value | None, our_value, their_value) -> merged_value.
|
|
10
|
+
|
|
11
|
+
Any argument can be None (key absent or removed on that side).
|
|
12
|
+
|
|
13
|
+
Returning a :class:`~kvgit.versioned.protocol.MergeChoice` instead of a
|
|
14
|
+
value keeps that side's committed value as it stands: no value is
|
|
15
|
+
encoded and the merge writes no new blob for the key.
|
|
16
|
+
"""
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
def counter() -> MergeFn:
|
|
20
|
+
"""Counter merge: ours + theirs - old."""
|
|
21
|
+
|
|
22
|
+
def merge(old: int | None, ours: int, theirs: int) -> int:
|
|
23
|
+
base = old if old is not None else 0
|
|
24
|
+
return ours + theirs - base
|
|
25
|
+
|
|
26
|
+
return merge
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
def last_writer_wins() -> MergeFn:
|
|
30
|
+
"""Last-writer-wins: always returns theirs."""
|
|
31
|
+
return lambda old, ours, theirs: theirs
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
def text_merge(
|
|
35
|
+
*,
|
|
36
|
+
ours_label: str = "ours",
|
|
37
|
+
theirs_label: str = "theirs",
|
|
38
|
+
strict: bool = False,
|
|
39
|
+
) -> MergeFn:
|
|
40
|
+
"""Marker merge for ``str`` (or ``bytes``) values, for use with ``Staged``.
|
|
41
|
+
|
|
42
|
+
The value-level counterpart to :func:`kvgit.merges.text`. ``Staged``
|
|
43
|
+
decodes both sides before calling a merge function, so a key holding
|
|
44
|
+
``str`` reaches a bytes-level function as ``str`` and fails there;
|
|
45
|
+
this one encodes ``str`` sides as UTF-8, marker-merges, and decodes
|
|
46
|
+
the result back to ``str`` when any side was ``str``. A key whose
|
|
47
|
+
values are ``bytes`` merges as bytes and comes back as ``bytes``.
|
|
48
|
+
|
|
49
|
+
Disjoint line changes merge cleanly and overlapping ones come back
|
|
50
|
+
with git-style ``<<<<<<<`` markers (labelled by ``ours_label`` /
|
|
51
|
+
``theirs_label``) rather than raising. With ``strict=True`` a
|
|
52
|
+
conflict raises instead of marking, so the merge aborts rather than
|
|
53
|
+
landing hunks. A value that is neither ``str`` nor ``bytes``, and
|
|
54
|
+
anything the marker merge cannot handle — undecodable bytes, NUL
|
|
55
|
+
bytes, inputs over the size cap — raises
|
|
56
|
+
:class:`~kvgit.merges.CantMark`, which the merge machinery files as
|
|
57
|
+
an ordinary conflict.
|
|
58
|
+
"""
|
|
59
|
+
merge_bytes = make_text_merge(
|
|
60
|
+
ours_label=ours_label, theirs_label=theirs_label, strict=strict
|
|
61
|
+
)
|
|
62
|
+
|
|
63
|
+
def encode(value: Any) -> bytes | None:
|
|
64
|
+
if value is None or isinstance(value, bytes):
|
|
65
|
+
return value
|
|
66
|
+
if isinstance(value, str):
|
|
67
|
+
return value.encode("utf-8")
|
|
68
|
+
raise CantMark(f"not text: {type(value).__name__}")
|
|
69
|
+
|
|
70
|
+
def merge(old: Any, ours: Any, theirs: Any) -> str | bytes:
|
|
71
|
+
# One side holding str is enough to make the merged value str:
|
|
72
|
+
# the sides are the same key's value at different commits, so a
|
|
73
|
+
# bytes side is the same text in another spelling.
|
|
74
|
+
as_str = any(isinstance(v, str) for v in (old, ours, theirs))
|
|
75
|
+
merged = merge_bytes(encode(old), encode(ours), encode(theirs))
|
|
76
|
+
return merged.decode("utf-8") if as_str else merged
|
|
77
|
+
|
|
78
|
+
return merge
|
|
@@ -26,3 +26,22 @@ class MergeConflict(Exception):
|
|
|
26
26
|
self.merge_errors = merge_errors or {}
|
|
27
27
|
keys_str = ", ".join(sorted(conflicting_keys))
|
|
28
28
|
super().__init__(f"Merge conflict on keys: {keys_str}")
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
class UnknownBranchError(ValueError):
|
|
32
|
+
"""Raised when opening or switching to a branch that does not exist.
|
|
33
|
+
|
|
34
|
+
A subclass of ``ValueError``, so existing ``except ValueError``
|
|
35
|
+
handlers keep working. Open with ``create=True`` (the default) to
|
|
36
|
+
mint the branch instead.
|
|
37
|
+
"""
|
|
38
|
+
|
|
39
|
+
|
|
40
|
+
class GcBusy(Exception):
|
|
41
|
+
"""Raised when a deep clean cannot take the store's GC lease.
|
|
42
|
+
|
|
43
|
+
Another process or thread holds an unexpired lease, so a namespace
|
|
44
|
+
scan started now would run beside a sweep that is already deleting.
|
|
45
|
+
Retry later; the lease carries an expiry, so a holder that dies
|
|
46
|
+
without releasing it blocks nothing past that point.
|
|
47
|
+
"""
|
|
@@ -20,15 +20,45 @@ def _is_bug(exc: BaseException) -> bool:
|
|
|
20
20
|
return isinstance(exc, _BUG_EXCEPTIONS)
|
|
21
21
|
|
|
22
22
|
|
|
23
|
+
MUTABLE_PREFIX = "__"
|
|
24
|
+
"""Key prefix marking a value that may change under a fixed key.
|
|
25
|
+
|
|
26
|
+
Every mutable key kvgit writes starts with it — branch heads, their
|
|
27
|
+
prev-HEAD backups, the storage version stamp, the GC lease — and so does
|
|
28
|
+
commit metadata, which is immutable but small enough that reading it
|
|
29
|
+
from the authoritative tier costs little. Everything else is derived
|
|
30
|
+
from its own content (``kvgit:keyset:``, ``kvgit:chunk:``,
|
|
31
|
+
``<commit>:<key>``): the same key always holds the same bytes, which is
|
|
32
|
+
what makes it cacheable.
|
|
33
|
+
"""
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
def _is_mutable(key: str) -> bool:
|
|
37
|
+
return isinstance(key, str) and key.startswith(MUTABLE_PREFIX)
|
|
38
|
+
|
|
39
|
+
|
|
23
40
|
class Composite(KVStore):
|
|
24
41
|
"""N-tier cache composing any number of KV stores.
|
|
25
42
|
|
|
26
|
-
|
|
27
|
-
|
|
43
|
+
**Cache tiers serve only immutable, content-derived keys.** A key
|
|
44
|
+
starting with ``__`` names a value that changes under a fixed key —
|
|
45
|
+
a branch head, its recovery backup, the storage version stamp, the
|
|
46
|
+
GC lease — so ``get``, ``get_many`` and ``__contains__`` read those
|
|
47
|
+
from Ln alone and never populate a cache with them. Cached, they
|
|
48
|
+
would let a process keep serving a branch head that another process
|
|
49
|
+
has already moved: the handle would take a ``ConcurrencyError`` on
|
|
50
|
+
commit, call ``refresh()``, and read the same stale head back out of
|
|
51
|
+
L1. Everything else is keyed by its own content
|
|
52
|
+
(``kvgit:keyset:``, ``kvgit:chunk:``, ``<commit>:<key>``), so a hit
|
|
53
|
+
at any tier is the right answer forever.
|
|
54
|
+
|
|
55
|
+
On get: for a ``__`` key, read Ln. Otherwise check L1, L2, ..., Ln
|
|
56
|
+
in order; on hit at tier i, populate L1..L(i-1) and return.
|
|
28
57
|
|
|
29
58
|
On set: write to all tiers (most durable first).
|
|
30
59
|
|
|
31
|
-
On cas: delegate to Ln (authoritative), update caches on success
|
|
60
|
+
On cas: delegate to Ln (authoritative), update caches on success —
|
|
61
|
+
except for ``__`` keys, which no cache tier serves.
|
|
32
62
|
|
|
33
63
|
Tier failures (``OSError``, network errors, etc.) are logged at
|
|
34
64
|
WARNING and the next tier is tried; programming-error exceptions
|
|
@@ -54,8 +84,20 @@ class Composite(KVStore):
|
|
|
54
84
|
raise
|
|
55
85
|
logger.warning("Composite cache populate failed at tier %d: %s", j, e)
|
|
56
86
|
|
|
87
|
+
def _first_read_tier(self, key: str) -> int:
|
|
88
|
+
"""Index of the highest tier allowed to answer for ``key``.
|
|
89
|
+
|
|
90
|
+
A ``__``-prefixed key names a value that can change under a
|
|
91
|
+
fixed key, so only the authoritative tier may answer it;
|
|
92
|
+
everything else is content-derived and may be served by any
|
|
93
|
+
tier.
|
|
94
|
+
"""
|
|
95
|
+
return len(self._stores) - 1 if _is_mutable(key) else 0
|
|
96
|
+
|
|
57
97
|
def get(self, key: str) -> bytes | None:
|
|
58
|
-
|
|
98
|
+
start = self._first_read_tier(key)
|
|
99
|
+
for i in range(start, len(self._stores)):
|
|
100
|
+
store = self._stores[i]
|
|
59
101
|
try:
|
|
60
102
|
value = store.get(key)
|
|
61
103
|
except Exception as e:
|
|
@@ -64,17 +106,21 @@ class Composite(KVStore):
|
|
|
64
106
|
logger.warning("Composite get failed at tier %d for %r: %s", i, key, e)
|
|
65
107
|
continue
|
|
66
108
|
if value is not None:
|
|
67
|
-
|
|
109
|
+
# Nothing to populate when the read started at the
|
|
110
|
+
# authoritative tier, which is where a mutable key's
|
|
111
|
+
# read starts and ends.
|
|
112
|
+
if i > start:
|
|
68
113
|
self._populate_caches(i, {key: value})
|
|
69
114
|
return value
|
|
70
115
|
return None
|
|
71
116
|
|
|
72
|
-
def
|
|
117
|
+
def _get_many_from(self, remaining: set[str], start: int) -> dict[str, bytes]:
|
|
118
|
+
"""Bulk read ``remaining``, consulting tiers ``start``..Ln."""
|
|
73
119
|
result: dict[str, bytes] = {}
|
|
74
|
-
|
|
75
|
-
for i, store in enumerate(self._stores):
|
|
120
|
+
for i in range(start, len(self._stores)):
|
|
76
121
|
if not remaining:
|
|
77
122
|
break
|
|
123
|
+
store = self._stores[i]
|
|
78
124
|
try:
|
|
79
125
|
# Delegate to the tier's bulk get — backends with high
|
|
80
126
|
# per-call latency (Disk, IndexedDB) collapse N round-trips
|
|
@@ -86,16 +132,28 @@ class Composite(KVStore):
|
|
|
86
132
|
raise
|
|
87
133
|
logger.warning("Composite get_many failed at tier %d: %s", i, e)
|
|
88
134
|
continue
|
|
89
|
-
if tier_values and i >
|
|
135
|
+
if tier_values and i > start:
|
|
90
136
|
self._populate_caches(i, tier_values)
|
|
91
137
|
result.update(tier_values)
|
|
92
|
-
remaining
|
|
138
|
+
remaining = remaining - tier_values.keys()
|
|
139
|
+
return result
|
|
140
|
+
|
|
141
|
+
def get_many(self, *args) -> Mapping[str, bytes]:
|
|
142
|
+
keys = set(self._normalize_keys(args))
|
|
143
|
+
mutable = {key for key in keys if _is_mutable(key)}
|
|
144
|
+
if not mutable:
|
|
145
|
+
return self._get_many_from(keys, 0)
|
|
146
|
+
# Two passes rather than one: mutable keys must come from the
|
|
147
|
+
# authoritative tier and must not be written into a cache, so
|
|
148
|
+
# they cannot share a batch with content-derived keys.
|
|
149
|
+
result = self._get_many_from(mutable, len(self._stores) - 1)
|
|
150
|
+
result.update(self._get_many_from(keys - mutable, 0))
|
|
93
151
|
return result
|
|
94
152
|
|
|
95
153
|
def __contains__(self, key: str) -> bool:
|
|
96
|
-
for i
|
|
154
|
+
for i in range(self._first_read_tier(key), len(self._stores)):
|
|
97
155
|
try:
|
|
98
|
-
if key in
|
|
156
|
+
if key in self._stores[i]:
|
|
99
157
|
return True
|
|
100
158
|
except Exception as e:
|
|
101
159
|
if _is_bug(e):
|
|
@@ -175,7 +233,10 @@ class Composite(KVStore):
|
|
|
175
233
|
|
|
176
234
|
def cas(self, key: str, value: bytes, expected: bytes | None) -> bool:
|
|
177
235
|
success = self._stores[-1].cas(key, value, expected)
|
|
178
|
-
|
|
236
|
+
# A ``__``-prefixed key is read from the authoritative tier
|
|
237
|
+
# only, so writing it into a cache stores bytes nothing will
|
|
238
|
+
# ever read, and a CAS is almost always against such a key.
|
|
239
|
+
if success and not _is_mutable(key):
|
|
179
240
|
for i, store in enumerate(self._stores[:-1]):
|
|
180
241
|
try:
|
|
181
242
|
store.set(key, value)
|
|
@@ -9,6 +9,9 @@ Anything that cannot be marked — undecodable bytes, NUL bytes, inputs
|
|
|
9
9
|
over the size cap — raises :class:`CantMark`, which the merge machinery
|
|
10
10
|
files as an ordinary conflict. Callers that need to tell "no encoding"
|
|
11
11
|
from "too big" catch it themselves; callers that don't let it flow.
|
|
12
|
+
|
|
13
|
+
Also here: :func:`ours` and :func:`theirs`, the pick-a-side merges, for
|
|
14
|
+
keys where one branch simply owns the value.
|
|
12
15
|
"""
|
|
13
16
|
|
|
14
17
|
from __future__ import annotations
|
|
@@ -16,7 +19,7 @@ from __future__ import annotations
|
|
|
16
19
|
import difflib
|
|
17
20
|
from collections.abc import Callable
|
|
18
21
|
|
|
19
|
-
from .versioned.protocol import BytesMergeFn
|
|
22
|
+
from .versioned.protocol import BytesMergeFn, MergeChoice
|
|
20
23
|
|
|
21
24
|
#: Inputs whose combined size exceeds this are refused with
|
|
22
25
|
#: :class:`CantMark` rather than merged. Line-diffing is quadratic-ish
|
|
@@ -24,13 +27,24 @@ from .versioned.protocol import BytesMergeFn
|
|
|
24
27
|
#: is a whole-file conflict by inspection anyway.
|
|
25
28
|
MAX_MARK_BYTES = 1024 * 1024
|
|
26
29
|
|
|
30
|
+
#: Line boundaries recognized by ``str.splitlines`` beyond ``"\n"``. A
|
|
31
|
+
#: line ending in any of these is already terminated — gluing the next
|
|
32
|
+
#: line after it re-splits to the same lines — so :func:`_terminate`
|
|
33
|
+
#: leaves it alone, and labels may not contain any of them.
|
|
34
|
+
_LINE_BOUNDARIES = frozenset(
|
|
35
|
+
["\n", "\r", "\v", "\f", "\x1c", "\x1d", "\x1e", "\x85", "\u2028", "\u2029"]
|
|
36
|
+
)
|
|
37
|
+
|
|
27
38
|
|
|
28
39
|
class CantMark(Exception):
|
|
29
|
-
"""A value cannot be marker-merged: not text, or
|
|
40
|
+
"""A value cannot be marker-merged: not text, too big, or a conflict
|
|
41
|
+
under ``strict=True``.
|
|
30
42
|
|
|
31
43
|
Raised instead of returning bytes when the inputs are undecodable as
|
|
32
|
-
UTF-8, contain NUL bytes, or exceed ``MAX_MARK_BYTES`` combined
|
|
33
|
-
|
|
44
|
+
UTF-8, contain NUL bytes, or exceed ``MAX_MARK_BYTES`` combined — or,
|
|
45
|
+
when built with ``strict=True``, when the merge would have written
|
|
46
|
+
conflict markers. The merge machinery reports it as a conflict for
|
|
47
|
+
that key.
|
|
34
48
|
"""
|
|
35
49
|
|
|
36
50
|
|
|
@@ -66,15 +80,17 @@ def _changes(base: list[str], side: list[str]) -> list[tuple[int, int, list[str]
|
|
|
66
80
|
|
|
67
81
|
|
|
68
82
|
def _terminate(lines: list[str]) -> list[str]:
|
|
69
|
-
"""Ensure every non-final line ends with a
|
|
83
|
+
"""Ensure every non-final line ends with a line break.
|
|
70
84
|
|
|
71
|
-
A line without a trailing
|
|
85
|
+
A line without a trailing break glued onto following output would
|
|
72
86
|
corrupt both; clean regions stay byte-exact (a final line keeps its
|
|
73
|
-
missing
|
|
87
|
+
missing break, and lines ending in a non-LF ``splitlines`` boundary
|
|
88
|
+
such as lone ``\\r`` are already terminated, so no ``\\n`` is added),
|
|
89
|
+
markers pay the newline tax.
|
|
74
90
|
"""
|
|
75
91
|
fixed: list[str] = []
|
|
76
92
|
for i, ln in enumerate(lines):
|
|
77
|
-
if
|
|
93
|
+
if i + 1 < len(lines) and ln[-1:] not in _LINE_BOUNDARIES:
|
|
78
94
|
ln += "\n"
|
|
79
95
|
fixed.append(ln)
|
|
80
96
|
return fixed
|
|
@@ -207,33 +223,72 @@ def _merge_lines(
|
|
|
207
223
|
return _terminate(out), conflicted
|
|
208
224
|
|
|
209
225
|
|
|
226
|
+
def _check_labels(ours_label: str, theirs_label: str) -> None:
|
|
227
|
+
for name, label in (("ours_label", ours_label), ("theirs_label", theirs_label)):
|
|
228
|
+
if any(boundary in label for boundary in _LINE_BOUNDARIES):
|
|
229
|
+
raise ValueError(f"{name} may not contain a newline: {label!r}")
|
|
230
|
+
|
|
231
|
+
|
|
232
|
+
def text_merge_result(
|
|
233
|
+
old: bytes | None,
|
|
234
|
+
ours: bytes | None,
|
|
235
|
+
theirs: bytes | None,
|
|
236
|
+
*,
|
|
237
|
+
ours_label: str = "ours",
|
|
238
|
+
theirs_label: str = "theirs",
|
|
239
|
+
) -> tuple[bytes, bool]:
|
|
240
|
+
"""Marker-merge returning ``(merged_bytes, conflicted)``.
|
|
241
|
+
|
|
242
|
+
Same merge as :func:`make_text_merge`, but the flag tells whether the
|
|
243
|
+
merge introduced conflict hunks — reliably, where counting
|
|
244
|
+
``<<<<<<<`` lines cannot, because the inputs may legitimately contain
|
|
245
|
+
marker-like lines that merge through untouched. Anything unmarkable
|
|
246
|
+
still raises :class:`CantMark`.
|
|
247
|
+
"""
|
|
248
|
+
_check_labels(ours_label, theirs_label)
|
|
249
|
+
total = sum(len(d) for d in (old, ours, theirs) if d is not None)
|
|
250
|
+
if total > MAX_MARK_BYTES:
|
|
251
|
+
raise CantMark(f"inputs total {total} bytes over cap {MAX_MARK_BYTES}")
|
|
252
|
+
merged, conflicted = _merge_lines(
|
|
253
|
+
_decode(old),
|
|
254
|
+
_decode(ours),
|
|
255
|
+
_decode(theirs),
|
|
256
|
+
ours_label,
|
|
257
|
+
theirs_label,
|
|
258
|
+
ours_deleted=ours is None and old is not None,
|
|
259
|
+
theirs_deleted=theirs is None and old is not None,
|
|
260
|
+
)
|
|
261
|
+
return "".join(merged).encode("utf-8"), conflicted
|
|
262
|
+
|
|
263
|
+
|
|
210
264
|
def make_text_merge(
|
|
211
|
-
*,
|
|
265
|
+
*,
|
|
266
|
+
ours_label: str = "ours",
|
|
267
|
+
theirs_label: str = "theirs",
|
|
268
|
+
strict: bool = False,
|
|
212
269
|
) -> BytesMergeFn:
|
|
213
270
|
"""Build a marker-merge fn with custom conflict labels.
|
|
214
271
|
|
|
215
272
|
Labels ride git's positions (``<<<<<<< <ours>`` /
|
|
216
273
|
``>>>>>>> <theirs>``); pass branch names so conflicts read
|
|
217
|
-
attributably. Labels may not contain newlines.
|
|
274
|
+
attributably. Labels may not contain newlines (any ``str.splitlines``
|
|
275
|
+
boundary: LF, lone CR, VT/FF, FS/GS/RS, NEL, U+2028/2029).
|
|
276
|
+
|
|
277
|
+
With ``strict=True`` the fn raises :class:`CantMark` instead of
|
|
278
|
+
writing markers when sides conflict — for branches where a true
|
|
279
|
+
conflict must abort the commit rather than land hunks. The merge
|
|
280
|
+
machinery files it as an ordinary conflict, same as unmarkable
|
|
281
|
+
input.
|
|
218
282
|
"""
|
|
219
|
-
|
|
220
|
-
if "\n" in label:
|
|
221
|
-
raise ValueError(f"{name} may not contain a newline: {label!r}")
|
|
283
|
+
_check_labels(ours_label, theirs_label)
|
|
222
284
|
|
|
223
285
|
def merge(old: bytes | None, ours: bytes | None, theirs: bytes | None) -> bytes:
|
|
224
|
-
|
|
225
|
-
|
|
226
|
-
raise CantMark(f"inputs total {total} bytes over cap {MAX_MARK_BYTES}")
|
|
227
|
-
merged, _ = _merge_lines(
|
|
228
|
-
_decode(old),
|
|
229
|
-
_decode(ours),
|
|
230
|
-
_decode(theirs),
|
|
231
|
-
ours_label,
|
|
232
|
-
theirs_label,
|
|
233
|
-
ours_deleted=ours is None and old is not None,
|
|
234
|
-
theirs_deleted=theirs is None and old is not None,
|
|
286
|
+
merged, conflicted = text_merge_result(
|
|
287
|
+
old, ours, theirs, ours_label=ours_label, theirs_label=theirs_label
|
|
235
288
|
)
|
|
236
|
-
|
|
289
|
+
if conflicted and strict:
|
|
290
|
+
raise CantMark("strict text merge: conflicting changes cannot be marked")
|
|
291
|
+
return merged
|
|
237
292
|
|
|
238
293
|
return merge
|
|
239
294
|
|
|
@@ -245,3 +300,24 @@ def text(old: bytes | None, ours: bytes | None, theirs: bytes | None) -> bytes:
|
|
|
245
300
|
|
|
246
301
|
TextMergeFn = Callable[[bytes | None, bytes | None, bytes | None], bytes]
|
|
247
302
|
"""Type of what :func:`make_text_merge` builds (== ``BytesMergeFn``)."""
|
|
303
|
+
|
|
304
|
+
|
|
305
|
+
def ours(old: bytes | None, our: bytes | None, their: bytes | None) -> MergeChoice:
|
|
306
|
+
"""Take our side: the merged value is our committed value.
|
|
307
|
+
|
|
308
|
+
Keeps our existing blob pointer rather than rewriting the value, so
|
|
309
|
+
the merge writes nothing for this key. If we removed the key, the
|
|
310
|
+
merge removes it. Usable as a per-key, per-prefix, or default merge.
|
|
311
|
+
"""
|
|
312
|
+
return MergeChoice.OURS
|
|
313
|
+
|
|
314
|
+
|
|
315
|
+
def theirs(old: bytes | None, our: bytes | None, their: bytes | None) -> MergeChoice:
|
|
316
|
+
"""Take their side: the merged value is their committed value.
|
|
317
|
+
|
|
318
|
+
Keeps their existing blob pointer rather than rewriting the value,
|
|
319
|
+
so the merge writes nothing for this key. If they removed the key,
|
|
320
|
+
the merge removes it. Usable as a per-key, per-prefix, or default
|
|
321
|
+
merge.
|
|
322
|
+
"""
|
|
323
|
+
return MergeChoice.THEIRS
|