recordstore 0.11.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,25 @@
1
+ from .recordstore import (
2
+ RecordStore,
3
+ MemoryBytesStore,
4
+ BeeBytesStore,
5
+ MemoryPointer,
6
+ FilePointer,
7
+ SwarmFeedPointer,
8
+ MergeConflict,
9
+ ABSENT,
10
+ DELETE,
11
+ canonical_bytes,
12
+ )
13
+
14
+ __all__ = [
15
+ "RecordStore",
16
+ "MemoryBytesStore",
17
+ "BeeBytesStore",
18
+ "MemoryPointer",
19
+ "FilePointer",
20
+ "SwarmFeedPointer",
21
+ "MergeConflict",
22
+ "ABSENT",
23
+ "DELETE",
24
+ "canonical_bytes",
25
+ ]
@@ -0,0 +1,1201 @@
1
+ """recordstore: a versioned record store over a content-addressed bytes store.
2
+
3
+ This is the thin database kernel between Swarm (immutable chunks + a mutable
4
+ feed pointer) and an application that wants to think in records and versions.
5
+ It knows nothing about graphs, edges, or ontologies.
6
+
7
+ Model
8
+ -----
9
+ - A *record* is any JSON-compatible value, stored under a string key.
10
+ - All records live in a persistent (copy-on-write) compacted radix trie whose
11
+ nodes are each stored as one content-addressed blob; the trie's root
12
+ reference identifies one immutable, self-consistent snapshot of the entire
13
+ dataset.
14
+ - Mutations are staged in memory and flushed by `commit()`, which produces a
15
+ single new root reference. Readers pin a root and see a frozen snapshot.
16
+ - Encodings are canonical (sorted keys, fixed separators), so equal content
17
+ yields byte-equal blobs and therefore an equal root: same dataset =>
18
+ same root reference, regardless of insertion order or history.
19
+
20
+ Layering
21
+ --------
22
+ BytesStore : put(bytes) -> ref, get(ref) -> bytes (Memory / Bee HTTP)
23
+ Trie : canonical persistent radix trie over the bytes store
24
+ RecordStore : staging, commit, snapshots, prefix iteration
25
+ Pointer : mutable "latest root" (Memory / File; Swarm feed = follow-up)
26
+
27
+ Nothing above this layer should ever see a stored blob or a trie node.
28
+ """
29
+
30
+ from __future__ import annotations
31
+
32
+ import json
33
+ import hashlib
34
+ import os
35
+ import time
36
+ from concurrent.futures import ThreadPoolExecutor
37
+ from typing import Dict, Iterable, Iterator, List, Optional, Protocol, Tuple
38
+
39
+ Ref = str # hex-encoded reference to a stored blob
40
+
41
+ _SCHEMA_VERSION = 1
42
+ _TOMBSTONE = object()
43
+
44
+ # Merge sentinels (see RecordStore.merge).
45
+ ABSENT = object() # a resolver sees this for a side where the key is absent
46
+ DELETE = object() # a resolver returns this to drop a key from the merge
47
+
48
+
49
+ class MergeConflict(Exception):
50
+ """Raised by `RecordStore.merge` when both sides changed the same key to
51
+ different values and no `resolver` settled it. `.conflicts` is the list of
52
+ conflicting keys."""
53
+
54
+ def __init__(self, conflicts):
55
+ self.conflicts = list(conflicts)
56
+ shown = ", ".join(sorted(self.conflicts)[:5])
57
+ if len(self.conflicts) > 5:
58
+ shown += ", ..."
59
+ super().__init__(
60
+ f"unresolved merge conflict on {len(self.conflicts)} key(s): {shown}")
61
+
62
+
63
+ # ---------------------------------------------------------------------------
64
+ # Canonical encoding
65
+ # ---------------------------------------------------------------------------
66
+
67
+ def canonical_bytes(obj) -> bytes:
68
+ """Deterministic byte encoding: equal values => equal bytes.
69
+
70
+ Content addressing makes this a correctness requirement, not a style
71
+ choice. Rejects NaN/Infinity (not canonical in JSON).
72
+ """
73
+ return json.dumps(
74
+ obj, sort_keys=True, separators=(",", ":"),
75
+ ensure_ascii=False, allow_nan=False,
76
+ ).encode("utf-8")
77
+
78
+
79
+ def _common_prefix(a: bytes, b: bytes) -> bytes:
80
+ """Longest shared byte prefix of `a` and `b`. Leaner than
81
+ `os.path.commonprefix` (no list/min/max wrapping) — this is on the trie's
82
+ hot insert path."""
83
+ n = min(len(a), len(b))
84
+ i = 0
85
+ while i < n and a[i] == b[i]:
86
+ i += 1
87
+ return a[:i]
88
+
89
+
90
+ def _encode_value(value) -> bytes:
91
+ return canonical_bytes({"rsv": _SCHEMA_VERSION, "val": value})
92
+
93
+
94
+ def _decode_value(data: bytes):
95
+ obj = json.loads(data.decode("utf-8"))
96
+ if obj.get("rsv") != _SCHEMA_VERSION:
97
+ raise ValueError(f"unsupported record schema version: {obj.get('rsv')!r}")
98
+ return obj["val"]
99
+
100
+
101
+ # ---------------------------------------------------------------------------
102
+ # Bytes store backends
103
+ # ---------------------------------------------------------------------------
104
+
105
+ class BytesStore(Protocol):
106
+ # A backend may also implement `get_many(refs) -> {ref: bytes}` and
107
+ # `put_many(datas) -> [ref]` for batched/parallel I/O; recordstore uses them
108
+ # when present and falls back to serial `get`/`put` otherwise.
109
+ def put(self, data: bytes) -> Ref: ...
110
+ def get(self, ref: Ref) -> bytes: ...
111
+
112
+
113
+ class MemoryBytesStore:
114
+ """In-memory content-addressed store; the test double for Swarm."""
115
+
116
+ def __init__(self):
117
+ self.blobs: Dict[Ref, bytes] = {}
118
+
119
+ def put(self, data: bytes) -> Ref:
120
+ ref = hashlib.sha256(data).hexdigest()
121
+ self.blobs[ref] = data
122
+ return ref
123
+
124
+ def get(self, ref: Ref) -> bytes:
125
+ try:
126
+ return self.blobs[ref]
127
+ except KeyError:
128
+ raise KeyError(f"reference not found: {ref}") from None
129
+
130
+ def get_many(self, refs: Iterable[Ref]) -> Dict[Ref, bytes]:
131
+ return {ref: self.get(ref) for ref in refs}
132
+
133
+ def put_many(self, datas: Iterable[bytes]) -> List[Ref]:
134
+ return [self.put(d) for d in datas]
135
+
136
+ def __len__(self):
137
+ return len(self.blobs)
138
+
139
+
140
+ def _auto_batch(api_url: str) -> str:
141
+ """Resolve 'auto' to a validated usable batch id via swarmfs's
142
+ StampManager (an optional dependency, imported lazily like requests).
143
+
144
+ Selection only, never purchase: a library must not spend the node
145
+ wallet's xBZZ on its own. To buy programmatically use swarmfs
146
+ (``StampManager.plan``/``buy``) and pass the resulting id here.
147
+ """
148
+ try:
149
+ from swarmfs._client import SwarmClient
150
+ from swarmfs.stamps import StampManager
151
+ except ImportError:
152
+ raise ImportError(
153
+ "postage_batch_id='auto' needs swarmfs for stamp selection — "
154
+ "install it (pip install -e path/to/swarmfs) or pass an "
155
+ "explicit batch id (see GET /stamps on your node)"
156
+ ) from None
157
+ import asyncio
158
+
159
+ async def resolve() -> str:
160
+ client = SwarmClient(api_url)
161
+ try:
162
+ return await StampManager(client).resolve("auto")
163
+ finally:
164
+ await client.close()
165
+
166
+ return asyncio.run(resolve())
167
+
168
+
169
+ class BeeBytesStore:
170
+ """BytesStore over a Bee node's `/bytes` endpoint.
171
+
172
+ Named for the endpoint it actually uses: `/bytes` is Bee's blob-level
173
+ API, not the raw `/chunks/{address}` single-chunk primitive. Values of
174
+ any length are handled transparently — Bee's splitter turns the payload
175
+ into a chunk tree server-side and returns one reference. Requires a
176
+ usable postage batch id for writes; ``"auto"`` picks one via swarmfs
177
+ (validated, longest TTL — see ``_auto_batch``; selection only, buying
178
+ is deliberately left to the caller).
179
+ """
180
+
181
+ def __init__(self, api_url: str, postage_batch_id: str = "auto",
182
+ deferred_upload: bool = True, max_concurrent_reads: int = 16):
183
+ import requests # lazy: only needed for the real backend
184
+ self.api_url = api_url.rstrip("/")
185
+ if postage_batch_id in (None, "auto"):
186
+ postage_batch_id = _auto_batch(self.api_url)
187
+ self.batch = postage_batch_id
188
+ self.deferred = deferred_upload
189
+ self.max_concurrent_reads = max(1, max_concurrent_reads)
190
+ # A persistent session with a connection pool: keep-alive avoids a fresh
191
+ # TCP (and TLS) handshake on every blob op — the dominant per-op cost on
192
+ # a high-latency link — and gives the read pool reusable connections.
193
+ self._session = requests.Session()
194
+ adapter = requests.adapters.HTTPAdapter(
195
+ pool_connections=1, pool_maxsize=self.max_concurrent_reads
196
+ )
197
+ self._session.mount("http://", adapter)
198
+ self._session.mount("https://", adapter)
199
+
200
+ def put(self, data: bytes) -> Ref:
201
+ r = self._session.post(
202
+ f"{self.api_url}/bytes",
203
+ data=data,
204
+ headers={
205
+ "Content-Type": "application/octet-stream",
206
+ "Swarm-Postage-Batch-Id": self.batch,
207
+ "Swarm-Deferred-Upload": "true" if self.deferred else "false",
208
+ },
209
+ timeout=120,
210
+ )
211
+ r.raise_for_status()
212
+ return r.json()["reference"]
213
+
214
+ def get_many(self, refs: Iterable[Ref]) -> Dict[Ref, bytes]:
215
+ """Fetch many references concurrently — the fast path for hydrating a
216
+ store over a network backend, where each read is otherwise one serial
217
+ HTTP round trip (painful on a high-latency link). Reads are safe to
218
+ parallelise freely: everything here is immutable and content-addressed,
219
+ so there is nothing to lock."""
220
+ refs = list(refs)
221
+ if not refs:
222
+ return {}
223
+ workers = min(self.max_concurrent_reads, len(refs))
224
+ with ThreadPoolExecutor(max_workers=workers) as pool:
225
+ return dict(zip(refs, pool.map(self.get, refs)))
226
+
227
+ def put_many(self, datas: Iterable[bytes]) -> List[Ref]:
228
+ """Upload independent blobs concurrently, preserving order. Used for a
229
+ commit's value blobs, which have no dependencies on one another."""
230
+ datas = list(datas)
231
+ if not datas:
232
+ return []
233
+ workers = min(self.max_concurrent_reads, len(datas))
234
+ with ThreadPoolExecutor(max_workers=workers) as pool:
235
+ return list(pool.map(self.put, datas))
236
+
237
+ def get(self, ref: Ref) -> bytes:
238
+ r = self._session.get(f"{self.api_url}/bytes/{ref}", timeout=120)
239
+ if r.status_code == 404:
240
+ raise KeyError(f"reference not found: {ref}")
241
+ r.raise_for_status()
242
+ return r.content
243
+
244
+
245
+ # ---------------------------------------------------------------------------
246
+ # Persistent compacted radix trie (canonical)
247
+ #
248
+ # Node wire format (canonical JSON):
249
+ # {"tn": 1, "p": "<hex prefix>", "v": "<value ref>"|null, "c": {"<hex byte>": "<ref>", ...}}
250
+ #
251
+ # Canonical-form invariants (make the structure a pure function of content):
252
+ # - a node with no value and no children does not exist (empty map => root None)
253
+ # - a node with no value and exactly one child is merged into that child
254
+ # ---------------------------------------------------------------------------
255
+
256
+ class _Node:
257
+ __slots__ = ("prefix", "value_ref", "children")
258
+
259
+ def __init__(self, prefix: bytes, value_ref: Optional[Ref],
260
+ children: Dict[int, Ref]):
261
+ self.prefix = prefix
262
+ self.value_ref = value_ref
263
+ self.children = children # first-byte -> child node ref
264
+
265
+
266
+ class _Trie:
267
+ def __init__(self, bytes_store: BytesStore):
268
+ self._blobs = bytes_store
269
+ self._cache: Dict[Ref, _Node] = {} # nodes are immutable => safe
270
+ # Commit-scoped write buffer. While buffering, `_store` defers to
271
+ # placeholder refs instead of uploading; `_flush` then writes only the
272
+ # nodes surviving in the final root, bottom-up and one level per batch.
273
+ self._buffering = False
274
+ self._pending: Dict[Ref, _Node] = {}
275
+ self._pn = 0
276
+
277
+ # -- node io -----------------------------------------------------------
278
+
279
+ @staticmethod
280
+ def _decode(data: bytes) -> _Node:
281
+ obj = json.loads(data.decode("utf-8"))
282
+ if obj.get("tn") != 1:
283
+ raise ValueError("not a trie node or unsupported version")
284
+ return _Node(
285
+ bytes.fromhex(obj["p"]),
286
+ obj["v"],
287
+ {int(k, 16): v for k, v in obj["c"].items()},
288
+ )
289
+
290
+ def _load(self, ref: Ref) -> _Node:
291
+ node = self._cache.get(ref)
292
+ if node is None:
293
+ node = self._decode(self._blobs.get(ref))
294
+ self._cache[ref] = node
295
+ return node
296
+
297
+ def _load_many(self, refs: List[Ref]) -> Dict[Ref, _Node]:
298
+ """Load several nodes, fetching the uncached ones in one batch so a
299
+ network store can parallelise the round trips (falls back to serial
300
+ `get` if the store has no `get_many`)."""
301
+ missing = list({r for r in refs if r not in self._cache})
302
+ if missing:
303
+ get_many = getattr(self._blobs, "get_many", None)
304
+ blobs = (get_many(missing) if get_many
305
+ else {r: self._blobs.get(r) for r in missing})
306
+ for r in missing:
307
+ self._cache[r] = self._decode(blobs[r])
308
+ return {r: self._cache[r] for r in refs}
309
+
310
+ @staticmethod
311
+ def _serialize(prefix: bytes, value_ref: Optional[Ref],
312
+ children: Dict[int, Ref]) -> bytes:
313
+ return canonical_bytes({
314
+ "tn": 1,
315
+ "p": prefix.hex(),
316
+ "v": value_ref,
317
+ "c": {format(b, "02x"): r for b, r in sorted(children.items())},
318
+ })
319
+
320
+ def _store(self, node: _Node) -> Ref:
321
+ if self._buffering:
322
+ # Defer: hand back a placeholder. The real (server-assigned) ref is
323
+ # resolved bottom-up in `_flush`, once this node's children are real.
324
+ pid = f"pending:{self._pn}"
325
+ self._pn += 1
326
+ self._pending[pid] = node
327
+ self._cache[pid] = node # so `_load` serves it during the build
328
+ return pid
329
+ ref = self._blobs.put(
330
+ self._serialize(node.prefix, node.value_ref, node.children))
331
+ self._cache[ref] = node
332
+ return ref
333
+
334
+ def _flush(self, root: Optional[Ref]) -> Optional[Ref]:
335
+ """Write the buffered nodes reachable from `root`, bottom-up with one
336
+ concurrent batch per level, and return the real root ref. Nodes not
337
+ reachable from the final root (orphaned intermediates left by
338
+ one-key-at-a-time insertion) are simply never written."""
339
+ if root is None or root not in self._pending:
340
+ return root # empty result, or the root subtree was unchanged
341
+ reachable = set()
342
+ stack = [root]
343
+ while stack:
344
+ pid = stack.pop()
345
+ if pid in reachable:
346
+ continue
347
+ reachable.add(pid)
348
+ for cref in self._pending[pid].children.values():
349
+ if cref in self._pending:
350
+ stack.append(cref)
351
+ put_many = getattr(self._blobs, "put_many", None)
352
+ resolved: Dict[Ref, Ref] = {}
353
+ remaining = set(reachable)
354
+ while root not in resolved:
355
+ ready = [pid for pid in remaining
356
+ if all(c not in self._pending or c in resolved
357
+ for c in self._pending[pid].children.values())]
358
+ if not ready: # impossible for an acyclic trie; guard against a hang
359
+ raise RuntimeError("trie flush stalled: no writable nodes")
360
+ batch = [] # (pid, node-with-real-children, bytes)
361
+ for pid in ready:
362
+ node = self._pending[pid]
363
+ children = {b: resolved.get(c, c) for b, c in node.children.items()}
364
+ real = _Node(node.prefix, node.value_ref, children)
365
+ batch.append((pid, real, self._serialize(
366
+ real.prefix, real.value_ref, real.children)))
367
+ datas = [b[2] for b in batch]
368
+ refs = put_many(datas) if put_many else [self._blobs.put(d) for d in datas]
369
+ for (pid, real, _), ref in zip(batch, refs):
370
+ resolved[pid] = ref
371
+ self._cache[ref] = real # cache with resolved children for reads
372
+ remaining.discard(pid)
373
+ return resolved[root]
374
+
375
+ def _reset_buffer(self) -> None:
376
+ for pid in self._pending:
377
+ self._cache.pop(pid, None)
378
+ self._pending.clear()
379
+ self._buffering = False
380
+ self._pn = 0
381
+
382
+ # -- operations (functional: take a root ref, return a new root ref) ----
383
+
384
+ def get(self, root: Optional[Ref], key: bytes) -> Optional[Ref]:
385
+ while root is not None:
386
+ node = self._load(root)
387
+ if not key.startswith(node.prefix):
388
+ return None
389
+ key = key[len(node.prefix):]
390
+ if key == b"":
391
+ return node.value_ref
392
+ root = node.children.get(key[0])
393
+ key = key[1:]
394
+ return None
395
+
396
+ # -- diff (structural, prunes shared subtrees) --------------------------
397
+
398
+ def _node_or_none(self, ref: Optional[Ref]) -> Optional[_Node]:
399
+ return self._load(ref) if ref is not None else None
400
+
401
+ def _node_items(self, node: _Node, acc: bytes):
402
+ """(key, value_ref) for every value in the subtree rooted at `node`
403
+ (which may be synthetic, i.e. not itself stored)."""
404
+ stack = [(node, acc)]
405
+ while stack:
406
+ n, a = stack.pop()
407
+ full = a + n.prefix
408
+ if n.value_ref is not None:
409
+ yield (full, n.value_ref)
410
+ for byte, cref in n.children.items():
411
+ stack.append((self._load(cref), full + bytes([byte])))
412
+
413
+ def _diff(self, a_root: Optional[Ref], b_root: Optional[Ref]):
414
+ """Yield (key, a_value_ref|None, b_value_ref|None) for every key where
415
+ `a_root` and `b_root` differ. Subtrees with equal refs are pruned, so
416
+ the cost is proportional to the difference, not the dataset."""
417
+ if a_root == b_root:
418
+ return
419
+ yield from self._diff_nodes(
420
+ self._node_or_none(a_root), self._node_or_none(b_root), b"")
421
+
422
+ def _diff_nodes(self, a: Optional[_Node], b: Optional[_Node], acc: bytes):
423
+ if a is None:
424
+ if b is not None:
425
+ for k, v in self._node_items(b, acc):
426
+ yield (k, None, v)
427
+ return
428
+ if b is None:
429
+ for k, v in self._node_items(a, acc):
430
+ yield (k, v, None)
431
+ return
432
+
433
+ pa, pb = a.prefix, b.prefix
434
+ if pa == pb:
435
+ ka = acc + pa
436
+ if a.value_ref != b.value_ref:
437
+ yield (ka, a.value_ref, b.value_ref)
438
+ for byte in set(a.children) | set(b.children):
439
+ ca, cb = a.children.get(byte), b.children.get(byte)
440
+ if ca == cb:
441
+ continue # shared subtree
442
+ yield from self._diff_nodes(
443
+ self._node_or_none(ca), self._node_or_none(cb),
444
+ ka + bytes([byte]))
445
+ return
446
+
447
+ common = _common_prefix(pa, pb)
448
+ if len(common) < len(pa) and len(common) < len(pb):
449
+ # prefixes diverge => the two subtrees cover disjoint keys
450
+ for k, v in self._node_items(a, acc):
451
+ yield (k, v, None)
452
+ for k, v in self._node_items(b, acc):
453
+ yield (k, None, v)
454
+ elif len(common) == len(pa):
455
+ # a's key is a proper prefix of b's: b lives under one of a's branches
456
+ ka = acc + pa
457
+ bb = pb[len(pa)]
458
+ b_split = _Node(pb[len(pa) + 1:], b.value_ref, b.children)
459
+ if a.value_ref is not None:
460
+ yield (ka, a.value_ref, None) # no key at ka on b's side
461
+ for byte, cref in a.children.items():
462
+ if byte == bb:
463
+ yield from self._diff_nodes(
464
+ self._load(cref), b_split, ka + bytes([byte]))
465
+ else:
466
+ for k, v in self._node_items(self._load(cref), ka + bytes([byte])):
467
+ yield (k, v, None)
468
+ if bb not in a.children:
469
+ for k, v in self._node_items(b_split, ka + bytes([bb])):
470
+ yield (k, None, v)
471
+ else:
472
+ # symmetric: b's key is a proper prefix of a's
473
+ kb = acc + pb
474
+ ab = pa[len(pb)]
475
+ a_split = _Node(pa[len(pb) + 1:], a.value_ref, a.children)
476
+ if b.value_ref is not None:
477
+ yield (kb, None, b.value_ref)
478
+ for byte, cref in b.children.items():
479
+ if byte == ab:
480
+ yield from self._diff_nodes(
481
+ a_split, self._load(cref), kb + bytes([byte]))
482
+ else:
483
+ for k, v in self._node_items(self._load(cref), kb + bytes([byte])):
484
+ yield (k, None, v)
485
+ if ab not in b.children:
486
+ for k, v in self._node_items(a_split, kb + bytes([ab])):
487
+ yield (k, v, None)
488
+
489
+ def insert(self, root: Optional[Ref], key: bytes, value_ref: Ref) -> Ref:
490
+ if root is None:
491
+ return self._store(_Node(key, value_ref, {}))
492
+ node = self._load(root)
493
+ common = _common_prefix(node.prefix, key)
494
+
495
+ if len(common) < len(node.prefix):
496
+ # split: demote the existing node under the diverging byte
497
+ demoted = _Node(node.prefix[len(common) + 1:], node.value_ref,
498
+ dict(node.children))
499
+ children = {node.prefix[len(common)]: self._store(demoted)}
500
+ rest = key[len(common):]
501
+ if rest == b"":
502
+ return self._store(_Node(common, value_ref, children))
503
+ leaf = self._store(_Node(rest[1:], value_ref, {}))
504
+ children[rest[0]] = leaf
505
+ return self._store(_Node(common, None, children))
506
+
507
+ rest = key[len(node.prefix):]
508
+ if rest == b"":
509
+ return self._store(_Node(node.prefix, value_ref, dict(node.children)))
510
+ children = dict(node.children)
511
+ child_ref = children.get(rest[0])
512
+ if child_ref is None:
513
+ children[rest[0]] = self._store(_Node(rest[1:], value_ref, {}))
514
+ else:
515
+ children[rest[0]] = self.insert(child_ref, rest[1:], value_ref)
516
+ return self._store(_Node(node.prefix, node.value_ref, children))
517
+
518
+ def delete(self, root: Optional[Ref], key: bytes) -> Optional[Ref]:
519
+ if root is None:
520
+ raise KeyError(key)
521
+ node = self._load(root)
522
+ if not key.startswith(node.prefix):
523
+ raise KeyError(key)
524
+ rest = key[len(node.prefix):]
525
+
526
+ if rest == b"":
527
+ if node.value_ref is None:
528
+ raise KeyError(key)
529
+ return self._canonicalize(node.prefix, None, dict(node.children))
530
+
531
+ child_ref = node.children.get(rest[0])
532
+ if child_ref is None:
533
+ raise KeyError(key)
534
+ new_child = self.delete(child_ref, rest[1:])
535
+ children = dict(node.children)
536
+ if new_child is None:
537
+ del children[rest[0]]
538
+ else:
539
+ children[rest[0]] = new_child
540
+ return self._canonicalize(node.prefix, node.value_ref, children)
541
+
542
+ def _canonicalize(self, prefix: bytes, value_ref: Optional[Ref],
543
+ children: Dict[int, Ref]) -> Optional[Ref]:
544
+ """Restore canonical-form invariants after a removal."""
545
+ if value_ref is None and not children:
546
+ return None
547
+ if value_ref is None and len(children) == 1:
548
+ (byte, child_ref), = children.items()
549
+ child = self._load(child_ref)
550
+ merged = _Node(prefix + bytes([byte]) + child.prefix,
551
+ child.value_ref, dict(child.children))
552
+ return self._store(merged)
553
+ return self._store(_Node(prefix, value_ref, children))
554
+
555
+ def items(self, root: Optional[Ref],
556
+ prefix: bytes = b"") -> Iterator[Tuple[bytes, Ref]]:
557
+ """All (key, value_ref) with key under `prefix`, in sorted key order."""
558
+ if root is None:
559
+ return
560
+ # Sorted pre-order DFS: a node's own key precedes its descendants',
561
+ # children visited in byte order, so keys come out sorted with no final
562
+ # sort and no result-set-sized buffer. Each node's children are
563
+ # prefetched in one batch so a network store still parallelises sibling
564
+ # loads (children pop from the stack as cache hits).
565
+ self._load_many([root]) # route the root through the batch path too
566
+ stack = [(root, b"")]
567
+ while stack:
568
+ ref, acc = stack.pop()
569
+ node = self._load(ref)
570
+ full = acc + node.prefix
571
+ # prune subtrees that cannot contain the prefix
572
+ probe = min(len(full), len(prefix))
573
+ if full[:probe] != prefix[:probe]:
574
+ continue
575
+ if node.value_ref is not None and full.startswith(prefix):
576
+ yield (full, node.value_ref)
577
+ child_bytes = sorted(node.children)
578
+ if child_bytes:
579
+ self._load_many([node.children[b] for b in child_bytes])
580
+ for byte in reversed(child_bytes): # reverse: smallest pops first
581
+ stack.append((node.children[byte], full + bytes([byte])))
582
+
583
+
584
+ # ---------------------------------------------------------------------------
585
+ # Pointers ("latest root")
586
+ # ---------------------------------------------------------------------------
587
+
588
+ class Pointer(Protocol):
589
+ def get(self) -> Optional[Ref]: ...
590
+ def set(self, root: Ref) -> None: ...
591
+
592
+
593
+ class MemoryPointer:
594
+ def __init__(self, root: Optional[Ref] = None):
595
+ self._root = root
596
+
597
+ def get(self) -> Optional[Ref]:
598
+ return self._root
599
+
600
+ def set(self, root: Ref) -> None:
601
+ self._root = root
602
+
603
+ def compare_and_set(self, expected: Optional[Ref], new: Optional[Ref]) -> bool:
604
+ """Atomic in-process compare-and-set — lets reconciling commits over a
605
+ shared in-process pointer converge without a lost-update race."""
606
+ if self._root == expected:
607
+ self._root = new
608
+ return True
609
+ return False
610
+
611
+
612
+ class FilePointer:
613
+ """Local-file pointer, useful during development."""
614
+
615
+ def __init__(self, path: str):
616
+ self.path = path
617
+
618
+ def get(self) -> Optional[Ref]:
619
+ try:
620
+ with open(self.path) as f:
621
+ content = f.read().strip()
622
+ return content or None
623
+ except FileNotFoundError:
624
+ return None
625
+
626
+ def set(self, root: Ref) -> None:
627
+ tmp = self.path + ".tmp"
628
+ with open(tmp, "w") as f:
629
+ f.write(root)
630
+ os.replace(tmp, self.path) # atomic on POSIX
631
+
632
+
633
+ class SwarmFeedPointer:
634
+ """Mutable "latest root" backed by a Swarm feed.
635
+
636
+ A Swarm feed is an owner-signed, mutable pointer: each update is a
637
+ single-owner chunk (SOC), BMT-hashed and secp256k1-signed with the feed
638
+ owner's key, posted to Bee's ``/soc/{owner}/{id}`` endpoint; readers
639
+ resolve "the latest" by sequence-index lookup at
640
+ ``GET /feeds/{owner}/{topic}``. This maps a feed onto the `Pointer`
641
+ protocol: ``set(root)`` publishes a new signed update, ``get()`` resolves
642
+ the latest root.
643
+
644
+ Requires the ``swarm-bee`` package (``pip install "recordstore[feeds]"``),
645
+ which performs the SOC/secp256k1 signing correctly — independently verified
646
+ against a live Bee 2.8.1 node (2026-07). It is imported lazily, so the
647
+ recordstore core stays stdlib-only.
648
+
649
+ Reliability. Swarm feed *lookups* are unreliable per call on a light node,
650
+ especially over a high-latency link: a lookup can 404 ("lookup failed";
651
+ ~10/12 calls in one hotspot measurement) or return a *stale-early* index
652
+ instead of the latest (ethersphere/bee#5251). The SOC *writes* are fine and
653
+ the chunks are individually retrievable; it is the lookup — which fetches
654
+ candidate index chunks from the network — that flakes. So this class never
655
+ trusts a single lookup:
656
+
657
+ - **Read-your-writes cache.** After ``set(ref)``, ``ref`` is served from a
658
+ local cache for ``feed_ttl`` seconds with no network round-trip, so a
659
+ writer never waits on a flaky lookup to see its own commit.
660
+ - **Monotonic index floor.** The next write index is
661
+ ``max(network_next, local_floor)``. Without the floor, back-to-back
662
+ commits would reuse an index while the first SOC is still propagating,
663
+ and the second update would be silently dropped.
664
+ - **Reliable index discovery.** The tip index is found by probing the feed's
665
+ SOC chunks directly (exponential + binary search) rather than trusting the
666
+ flaky /feeds lookup — the SOC chunks are individually retrievable even when
667
+ the lookup 404s. This is what makes a *cold* read (no cached index to hint
668
+ from) reliable. The warm path still tries the cheaper ``after``-hinted
669
+ lookup first and only falls back to probing when it flakes.
670
+ - **Retry-until-stable reads.** ``get()`` retries with exponential backoff on
671
+ transient chunk-fetch errors and never adopts a result whose index
672
+ regresses below what it has already seen (the stale-early guard).
673
+
674
+ This policy follows swarmfs's ``bzzf://`` feed layer, the reference
675
+ implementation for this Swarm characteristic. Once a feed has been resolved
676
+ at least once, ``get()`` also passes Bee's ``after`` index hint
677
+ (``GET /feeds/...?after=N``) so the lookup resumes just below the last
678
+ confirmed index instead of probing from scratch — much cheaper and less
679
+ flaky as the feed grows. swarm-bee's typed API does not expose ``after``
680
+ (see bee-py#2), so it is sent through the client transport directly, and
681
+ falls back to the plain lookup when that transport is unavailable or when
682
+ there is no confirmed index yet to resume from. This is a Swarm/light-node
683
+ characteristic, not a swarm-bee defect — any client hits it identically.
684
+
685
+ Construction. Pass a ``signer`` (32-byte secp256k1 private key, hex) to read
686
+ *and* write; the owner address is derived from it. For a read-only pointer,
687
+ pass ``owner`` (20-byte address, hex) instead. Writing also needs
688
+ ``postage_batch_id``. ``topic`` is a namespace string, hashed to the 32-byte
689
+ feed topic.
690
+ """
691
+
692
+ def __init__(
693
+ self,
694
+ api_url: str,
695
+ topic: str,
696
+ *,
697
+ signer: Optional[str] = None,
698
+ owner: Optional[str] = None,
699
+ postage_batch_id: Optional[str] = None,
700
+ feed_ttl: float = 15.0,
701
+ max_lookup_retries: int = 15,
702
+ retry_backoff: float = 0.5,
703
+ retry_backoff_cap: float = 5.0,
704
+ ):
705
+ try:
706
+ from bee import Bee
707
+ from bee.feeds import make_feed_identifier
708
+ from bee.swarm.keys import PrivateKey
709
+ from bee.swarm.typed_bytes import BatchId, EthAddress, Reference, Topic
710
+ from bee.swarm.errors import BeeResponseError
711
+ except ImportError as e: # pragma: no cover - only without the extra
712
+ raise ImportError(
713
+ "SwarmFeedPointer requires the 'swarm-bee' package; install it "
714
+ 'with: pip install "recordstore[feeds]"'
715
+ ) from e
716
+
717
+ self._Reference = Reference
718
+ self._make_feed_identifier = make_feed_identifier
719
+ self._BeeResponseError = BeeResponseError
720
+ self._bee = Bee(api_url)
721
+ self._topic = Topic.from_string(topic)
722
+
723
+ self._signer = PrivateKey.from_hex(signer) if signer else None
724
+ if self._signer is not None:
725
+ self._owner = self._signer.public_key().address()
726
+ elif owner is not None:
727
+ self._owner = EthAddress.from_hex(owner)
728
+ else:
729
+ raise ValueError(
730
+ "SwarmFeedPointer needs a signer (to read and write) or an "
731
+ "owner address (read-only)"
732
+ )
733
+ self._batch = BatchId.from_hex(postage_batch_id) if postage_batch_id else None
734
+
735
+ # Bee honours GET /feeds/...?after=N (resume a lookup from a known
736
+ # index); swarm-bee's typed API can't pass it, so hint via the client
737
+ # transport when present (bee-py#2), falling back cleanly otherwise.
738
+ self._can_hint = hasattr(getattr(self._bee.feeds, "_inner", None), "send")
739
+
740
+ self._ttl = feed_ttl
741
+ self._max_retries = max(1, max_lookup_retries)
742
+ self._backoff = retry_backoff
743
+ self._backoff_cap = retry_backoff_cap
744
+
745
+ # read-your-writes cache + monotonic index floor
746
+ self._cached_ref: Optional[Ref] = None
747
+ self._next_index = 0
748
+ self._cache_expiry = 0.0
749
+
750
+ def set(self, root: Ref) -> None:
751
+ if self._signer is None or self._batch is None:
752
+ raise RuntimeError(
753
+ "SwarmFeedPointer.set requires both a signer and a "
754
+ "postage_batch_id"
755
+ )
756
+ # A persistent writer's floor is authoritative (single-writer model);
757
+ # only a cold instance has to discover where the feed currently ends,
758
+ # and it does so by probing SOC chunks — reliable even when the /feeds
759
+ # lookup flakes on a high-latency link.
760
+ if self._next_index > 0:
761
+ index = self._next_index
762
+ else:
763
+ probed = self._probe_latest_index()
764
+ index = probed + 1 if probed is not None else 0
765
+ self._bee.feeds.update_feed_with_reference(
766
+ batch_id=self._batch,
767
+ signer=self._signer,
768
+ topic=self._topic,
769
+ reference=self._Reference.from_hex(root),
770
+ index=index,
771
+ )
772
+ self._cached_ref = root
773
+ self._next_index = index + 1
774
+ self._cache_expiry = time.monotonic() + self._ttl
775
+
776
+ def get(self) -> Optional[Ref]:
777
+ if self._cached_ref is not None and time.monotonic() < self._cache_expiry:
778
+ return self._cached_ref # read-your-writes / fresh cache
779
+
780
+ delay = self._backoff
781
+ for attempt in range(self._max_retries):
782
+ try:
783
+ latest_index = self._resolve_latest_index()
784
+ if latest_index is None:
785
+ return self._cached_ref # feed is empty (definitive)
786
+ index_next = latest_index + 1
787
+ if index_next > self._next_index or self._cached_ref is None:
788
+ # A newer update (or we've never resolved): read the
789
+ # reference from the feed's single-owner chunk — NOT from a
790
+ # plain feed GET, which Bee dereferences to the pointed-to
791
+ # content rather than returning the reference.
792
+ identifier = self._make_feed_identifier(self._topic, latest_index)
793
+ soc = self._bee.file.download_soc(self._owner, identifier)
794
+ self._cached_ref = self._soc_reference(soc)
795
+ self._next_index = index_next
796
+ self._cache_expiry = time.monotonic() + self._ttl
797
+ return self._cached_ref
798
+ if index_next == self._next_index:
799
+ # confirmed unchanged; serve cache and refresh the TTL.
800
+ self._cache_expiry = time.monotonic() + self._ttl
801
+ return self._cached_ref
802
+ # index_next < floor: stale-early lookup; retry for a fresher one.
803
+ except self._BeeResponseError as e:
804
+ if getattr(e, "status", None) not in (404, 500):
805
+ raise
806
+ # transient flake or empty feed; fall through to backoff/retry.
807
+ if attempt < self._max_retries - 1:
808
+ time.sleep(delay)
809
+ delay = min(delay * 2, self._backoff_cap)
810
+ return self._cached_ref # last-known ref, or None if never resolved
811
+
812
+ def _get_fresh(self) -> Optional[Ref]:
813
+ """Resolve the latest root from the network, bypassing the
814
+ read-your-writes/TTL cache — used by `compare_and_set` so the check
815
+ reflects other writers, not our own cached value."""
816
+ self._cache_expiry = 0.0
817
+ return self.get()
818
+
819
+ def compare_and_set(self, expected: Optional[Ref], new: Ref) -> bool:
820
+ """Best-effort compare-and-set for reconciling commits. Returns True
821
+ only if the feed still resolved to `expected` and `new` was written and
822
+ read back as the latest.
823
+
824
+ Caveat: a Swarm feed has no atomic index claim — Bee accepts (and
825
+ overwrites with) a second update at an already-used index. So this
826
+ cannot be a true CAS. It reads the current head *fresh* (so it reliably
827
+ detects a feed that already advanced — the common case) and verifies its
828
+ own write read-back, which resolves most races; but two writers hitting
829
+ the exact same index simultaneously can still both believe they won.
830
+ This narrows the window rather than closing it (see the limitations in
831
+ the user guide)."""
832
+ if self._get_fresh() != expected:
833
+ return False
834
+ self.set(new)
835
+ return self._get_fresh() == new
836
+
837
+ def _resolve_latest_index(self) -> Optional[int]:
838
+ """Latest feed index, or ``None`` for an empty feed.
839
+
840
+ Warm path: resume the (flaky) /feeds lookup near the tip via Bee's
841
+ ``after`` hint — one round trip when it works. Cold path, or when the
842
+ hinted lookup flakes: probe the feed's SOC chunks directly, which are
843
+ individually retrievable even when the /feeds lookup does not resolve.
844
+ Raises ``BeeResponseError`` only on transient chunk-fetch errors, which
845
+ the retry loop in ``get`` absorbs."""
846
+ hint = self._next_index - 2 # one below our last-confirmed index
847
+ if self._can_hint and hint >= 1:
848
+ try:
849
+ resp = self._bee.feeds._inner.send(
850
+ "GET",
851
+ f"feeds/{self._owner.to_hex()}/{self._topic.to_hex()}",
852
+ params={"after": str(hint)},
853
+ headers=[("Swarm-Only-Root-Chunk", "true")],
854
+ )
855
+ idx_hex = resp.headers.get("swarm-feed-index")
856
+ if idx_hex is not None:
857
+ return int(idx_hex, 16)
858
+ except self._BeeResponseError as e:
859
+ if getattr(e, "status", None) not in (404, 500):
860
+ raise
861
+ # hinted lookup flaked; fall through to the reliable probe.
862
+ return self._probe_latest_index()
863
+
864
+ def _probe_latest_index(self) -> Optional[int]:
865
+ """Highest existing feed index (``None`` if the feed is empty), found by
866
+ probing single-owner-chunk addresses. Sequential feeds have no gaps, so
867
+ an exponential + binary search over SOC existence pins the tip in
868
+ O(log n) reliable chunk fetches — no /feeds lookup involved."""
869
+ if not self._soc_exists(0):
870
+ return None
871
+ lo, hi = 0, 1
872
+ while self._soc_exists(hi):
873
+ lo, hi = hi, hi * 2
874
+ while hi - lo > 1:
875
+ mid = (lo + hi) // 2
876
+ if self._soc_exists(mid):
877
+ lo = mid
878
+ else:
879
+ hi = mid
880
+ return lo
881
+
882
+ def _soc_exists(self, index: int) -> bool:
883
+ identifier = self._make_feed_identifier(self._topic, index)
884
+ try:
885
+ self._bee.file.download_soc(self._owner, identifier)
886
+ return True
887
+ except self._BeeResponseError as e:
888
+ if getattr(e, "status", None) == 404:
889
+ return False
890
+ raise # transient (e.g. 500): let the caller retry
891
+
892
+ @staticmethod
893
+ def _soc_reference(soc) -> Ref:
894
+ # SOC payload is timestamp(8 BE) || reference; strip the timestamp.
895
+ payload = soc.payload
896
+ payload = payload.as_bytes() if hasattr(payload, "as_bytes") else bytes(payload)
897
+ return payload[8:].hex()
898
+
899
+
900
+ # ---------------------------------------------------------------------------
901
+ # RecordStore
902
+ # ---------------------------------------------------------------------------
903
+
904
+ class RecordStore:
905
+ """Staged, versioned key->record store over a BytesStore.
906
+
907
+ Reads are read-your-writes (staged changes shadow the committed trie).
908
+ `commit()` flushes staged changes and returns the new root reference;
909
+ `RecordStore.at(root, bytes_store)` opens a read-only snapshot of any root.
910
+ Returned records are deep copies: mutating them never mutates the store.
911
+ """
912
+
913
+ def __init__(self, bytes_store: BytesStore, root: Optional[Ref] = None,
914
+ pointer: Optional[Pointer] = None, _readonly: bool = False):
915
+ self._blobs = bytes_store
916
+ self._trie = _Trie(bytes_store)
917
+ self._root = pointer.get() if (pointer and root is None) else root
918
+ self._pointer = pointer
919
+ self._staged: Dict[str, object] = {}
920
+ self._readonly = _readonly
921
+
922
+ # -- snapshots -----------------------------------------------------------
923
+
924
+ @classmethod
925
+ def at(cls, root: Optional[Ref], bytes_store: BytesStore) -> "RecordStore":
926
+ return cls(bytes_store, root=root, _readonly=True)
927
+
928
+ @property
929
+ def root(self) -> Optional[Ref]:
930
+ """Root of the last committed state (staged changes not included)."""
931
+ return self._root
932
+
933
+ # -- record operations -----------------------------------------------------
934
+
935
+ @staticmethod
936
+ def _check_key(key: str) -> bytes:
937
+ if not isinstance(key, str) or key == "":
938
+ raise ValueError("key must be a non-empty string")
939
+ return key.encode("utf-8")
940
+
941
+ def get(self, key: str):
942
+ kb = self._check_key(key)
943
+ if key in self._staged:
944
+ staged = self._staged[key]
945
+ if staged is _TOMBSTONE:
946
+ raise KeyError(key)
947
+ return json.loads(canonical_bytes(staged)) # deep copy
948
+ vref = self._trie.get(self._root, kb)
949
+ if vref is None:
950
+ raise KeyError(key)
951
+ return _decode_value(self._blobs.get(vref))
952
+
953
+ def contains(self, key: str) -> bool:
954
+ try:
955
+ self.get(key)
956
+ return True
957
+ except KeyError:
958
+ return False
959
+
960
+ def put(self, key: str, value) -> None:
961
+ if self._readonly:
962
+ raise TypeError("read-only snapshot")
963
+ self._check_key(key)
964
+ # One canonical encode both validates (rejects non-JSON values and
965
+ # NaN/Infinity) and, via the round trip, detaches from the caller's
966
+ # object — no need to encode twice.
967
+ self._staged[key] = json.loads(canonical_bytes(value))
968
+
969
+ def delete(self, key: str) -> None:
970
+ if self._readonly:
971
+ raise TypeError("read-only snapshot")
972
+ kb = self._check_key(key)
973
+ if key not in self._staged and self._trie.get(self._root, kb) is None:
974
+ raise KeyError(key)
975
+ self._staged[key] = _TOMBSTONE
976
+
977
+ def _merged(self, prefix: str):
978
+ """Lazily yield `(key, vref, staged)` in sorted key order, merging the
979
+ committed trie stream with the staged overlay. For a committed record
980
+ `vref` is set and `staged` is None; for a staged put `vref` is None and
981
+ `staged` is the raw staged value; tombstones are dropped. Both inputs
982
+ are already sorted, so this is a streaming merge — nothing proportional
983
+ to the result set is buffered (only the small staged overlay)."""
984
+ pb = prefix.encode("utf-8")
985
+ committed = self._trie.items(self._root, pb) # lazy, sorted
986
+ staged = sorted(k for k in self._staged if k.startswith(prefix))
987
+ si, ns = 0, len(staged)
988
+ for kb, vref in committed:
989
+ ck = kb.decode("utf-8")
990
+ while si < ns and staged[si] < ck:
991
+ sk = staged[si]; si += 1
992
+ if self._staged[sk] is not _TOMBSTONE:
993
+ yield sk, None, self._staged[sk]
994
+ if si < ns and staged[si] == ck: # staged entry shadows the trie
995
+ if self._staged[ck] is not _TOMBSTONE:
996
+ yield ck, None, self._staged[ck]
997
+ si += 1
998
+ else:
999
+ yield ck, vref, None
1000
+ while si < ns:
1001
+ sk = staged[si]; si += 1
1002
+ if self._staged[sk] is not _TOMBSTONE:
1003
+ yield sk, None, self._staged[sk]
1004
+
1005
+ def keys(self, prefix: str = "") -> Iterator[str]:
1006
+ """Sorted keys under `prefix`, staged overlay included, yielded lazily
1007
+ (no result-set-sized buffer)."""
1008
+ for key, _vref, _staged in self._merged(prefix):
1009
+ yield key
1010
+
1011
+ def items(self, prefix: str = ""):
1012
+ """Sorted `(key, value)` pairs under `prefix`, staged overlay included.
1013
+
1014
+ Streams in windows: value blobs are fetched a window at a time, so over
1015
+ a network store that implements `get_many` the reads parallelise within
1016
+ each window (the fast path for hydrating a store) while memory stays
1017
+ bounded to one window rather than the whole result set. Values are
1018
+ deep-copied, exactly like `get`."""
1019
+ window = max(1, getattr(self._blobs, "max_concurrent_reads", 256))
1020
+ buf: list = []
1021
+ refs: List[Ref] = []
1022
+ for key, vref, staged in self._merged(prefix):
1023
+ buf.append((key, vref, staged))
1024
+ if vref is not None:
1025
+ refs.append(vref)
1026
+ if len(refs) >= window:
1027
+ yield from self._flush_items(buf, refs)
1028
+ buf, refs = [], []
1029
+ if buf:
1030
+ yield from self._flush_items(buf, refs)
1031
+
1032
+ def _flush_items(self, buf, refs: List[Ref]):
1033
+ blobs = self._fetch_blobs(refs) if refs else {}
1034
+ for key, vref, staged in buf:
1035
+ if vref is None:
1036
+ yield key, json.loads(canonical_bytes(staged)) # deep copy
1037
+ else:
1038
+ yield key, _decode_value(blobs[vref])
1039
+
1040
+ def _fetch_blobs(self, refs: List[Ref]) -> Dict[Ref, bytes]:
1041
+ get_many = getattr(self._blobs, "get_many", None)
1042
+ if get_many is not None:
1043
+ return get_many(refs)
1044
+ return {r: self._blobs.get(r) for r in refs}
1045
+
1046
+ # -- commit ---------------------------------------------------------------
1047
+
1048
+ def _build_root(self, base: Optional[Ref]) -> Optional[Ref]:
1049
+ """Apply the staged changes on top of `base` and return the new root.
1050
+ Value blobs go up front (concurrently if supported); trie nodes are
1051
+ buffered and flushed bottom-up, one batch per level."""
1052
+ writes = [(k, self._staged[k]) for k in sorted(self._staged)
1053
+ if self._staged[k] is not _TOMBSTONE]
1054
+ put_many = getattr(self._blobs, "put_many", None)
1055
+ datas = [_encode_value(v) for _, v in writes]
1056
+ refs = put_many(datas) if put_many is not None else [self._blobs.put(d) for d in datas]
1057
+ vref = {k: r for (k, _), r in zip(writes, refs)}
1058
+
1059
+ self._trie._buffering = True
1060
+ try:
1061
+ root = base
1062
+ for key in sorted(self._staged): # deterministic write order
1063
+ staged = self._staged[key]
1064
+ kb = key.encode("utf-8")
1065
+ if staged is _TOMBSTONE:
1066
+ try:
1067
+ root = self._trie.delete(root, kb)
1068
+ except KeyError:
1069
+ pass # deleted a key that never existed in the trie
1070
+ else:
1071
+ root = self._trie.insert(root, kb, vref[key])
1072
+ return self._trie._flush(root)
1073
+ finally:
1074
+ self._trie._reset_buffer()
1075
+
1076
+ def commit(self, *, reconcile: bool = False, resolver=None,
1077
+ retries: int = 5) -> Optional[Ref]:
1078
+ """Flush staged changes; return the new root and update the pointer.
1079
+
1080
+ The root/pointer changes only after every blob write has succeeded, so
1081
+ a reader following the pointer sees all of a commit or none of it.
1082
+
1083
+ With `reconcile=True` and a pointer attached, the commit converges with
1084
+ concurrent writers instead of overwriting them: if the pointer has moved
1085
+ past the root this commit built on, the two versions are three-way
1086
+ merged (see `merge`; `resolver` settles conflicts) and the merge is
1087
+ retried up to `retries` times until the pointer lands. A pointer that
1088
+ exposes `compare_and_set` gets race-free updates; otherwise the
1089
+ read-then-set is best-effort (there is no lower-level CAS)."""
1090
+ if self._readonly:
1091
+ raise TypeError("read-only snapshot")
1092
+ base = self._root
1093
+ new = self._build_root(base)
1094
+ if self._pointer is not None:
1095
+ if reconcile:
1096
+ new = self._reconcile(base, new, resolver, retries)
1097
+ else:
1098
+ self._pointer.set(new)
1099
+ self._staged.clear()
1100
+ self._root = new
1101
+ return new
1102
+
1103
+ def _reconcile(self, base, new, resolver, retries):
1104
+ pointer = self._pointer
1105
+ cas = getattr(pointer, "compare_and_set", None)
1106
+ expected = base
1107
+ for _ in range(max(1, retries)):
1108
+ current = pointer.get()
1109
+ if current == expected:
1110
+ if cas is None:
1111
+ pointer.set(new) # best-effort (no CAS at this layer)
1112
+ return new
1113
+ if cas(expected, new):
1114
+ return new
1115
+ continue # lost the race; re-read and retry
1116
+ # pointer advanced under us: fold their version into ours
1117
+ new = self.merge(self._blobs, expected, new, current, resolver)
1118
+ expected = current
1119
+ raise RuntimeError(
1120
+ f"commit could not reconcile the pointer after {retries} tries")
1121
+
1122
+ # -- merge ----------------------------------------------------------------
1123
+
1124
+ @classmethod
1125
+ def merge(cls, bytes_store: BytesStore, base: Optional[Ref],
1126
+ ours: Optional[Ref], theirs: Optional[Ref], resolver=None) -> Optional[Ref]:
1127
+ """Three-way merge of two roots that diverged from a common `base`.
1128
+
1129
+ Returns the merged root. Because roots are canonical, this leans on
1130
+ reference equality: if a subtree is unchanged on a side its root ref
1131
+ still equals `base`'s, so whole branches merge for free.
1132
+
1133
+ Per key: a change made on only one side is taken; a change made on both
1134
+ sides to the *same* value is taken once; a change made on both sides to
1135
+ *different* values is a conflict. Conflicts are settled by
1136
+ ``resolver(key, base, ours, theirs)`` — each argument is the decoded
1137
+ value or the ``ABSENT`` sentinel; return the value to keep, or the
1138
+ ``DELETE`` sentinel to drop the key. Without a resolver, conflicts raise
1139
+ `MergeConflict`. The merge is commutative iff the resolver is symmetric
1140
+ in its ours/theirs arguments (the built-in conflict = raise is).
1141
+
1142
+ Only the changed keys are touched: both the read (a structural diff that
1143
+ prunes subtrees equal on both sides) and the write (the merged diff
1144
+ applied to `base`, bulk-flushed) are proportional to the divergence, not
1145
+ the dataset. Unchanged subtrees are shared with `base`.
1146
+ """
1147
+ if ours == theirs:
1148
+ return ours # identical (incl. both None)
1149
+ if ours == base:
1150
+ return theirs # only they changed
1151
+ if theirs == base:
1152
+ return ours # only we changed
1153
+
1154
+ trie = _Trie(bytes_store)
1155
+ # (base_ref, side_ref) per key that changed from base on each side.
1156
+ our_diff = {k: (bv, sv) for k, bv, sv in trie._diff(base, ours)}
1157
+ their_diff = {k: (bv, sv) for k, bv, sv in trie._diff(base, theirs)}
1158
+
1159
+ changes: Dict[bytes, object] = {} # key -> value_ref | _TOMBSTONE
1160
+ conflicts: List[str] = []
1161
+ for k in set(our_diff) | set(their_diff):
1162
+ if k in our_diff and k in their_diff:
1163
+ bv, ov = our_diff[k]
1164
+ tv = their_diff[k][1]
1165
+ if ov == tv:
1166
+ merged = ov # both changed it the same way
1167
+ elif resolver is None:
1168
+ conflicts.append(k.decode("utf-8"))
1169
+ continue
1170
+ else:
1171
+ decode = (lambda r: _decode_value(bytes_store.get(r))
1172
+ if r is not None else ABSENT)
1173
+ res = resolver(k.decode("utf-8"), decode(bv), decode(ov), decode(tv))
1174
+ merged = None if res is DELETE else bytes_store.put(_encode_value(res))
1175
+ elif k in our_diff:
1176
+ bv, merged = our_diff[k] # changed by us only
1177
+ else:
1178
+ bv, merged = their_diff[k] # changed by them only
1179
+ if merged != bv: # (a resolver could land back on base)
1180
+ changes[k] = merged if merged is not None else _TOMBSTONE
1181
+
1182
+ if conflicts:
1183
+ raise MergeConflict(conflicts)
1184
+
1185
+ # Apply only the diff to base (O(diff) node writes, bulk-flushed).
1186
+ root = base
1187
+ trie._buffering = True
1188
+ try:
1189
+ for k in sorted(changes):
1190
+ c = changes[k]
1191
+ if c is _TOMBSTONE:
1192
+ try:
1193
+ root = trie.delete(root, k)
1194
+ except KeyError:
1195
+ pass
1196
+ else:
1197
+ root = trie.insert(root, k, c)
1198
+ root = trie._flush(root)
1199
+ finally:
1200
+ trie._reset_buffer()
1201
+ return root
@@ -0,0 +1,12 @@
1
+ Metadata-Version: 2.4
2
+ Name: recordstore
3
+ Version: 0.11.0
4
+ Summary: Versioned key-record store over a content-addressed bytes store (memory or Ethereum Swarm/Bee backends)
5
+ License: BSD-3-Clause
6
+ Requires-Python: >=3.9
7
+ License-File: LICENSE
8
+ Provides-Extra: bee
9
+ Requires-Dist: requests; extra == "bee"
10
+ Provides-Extra: feeds
11
+ Requires-Dist: swarm-bee; extra == "feeds"
12
+ Dynamic: license-file
@@ -0,0 +1,7 @@
1
+ recordstore/__init__.py,sha256=YcL41Fm1Jv6ghs6bwgI4jo8qmkKP0SIITfaVnw_MrHk,424
2
+ recordstore/recordstore.py,sha256=B3L42Gx7YxOqXW86ZwbuMadxvqhXDLJ-xmdj2syKKsc,52211
3
+ recordstore-0.11.0.dist-info/licenses/LICENSE,sha256=fN8KKdDmAODXBl8NeF83bIuVLXzyuH8SuwXSLYpZgr4,1500
4
+ recordstore-0.11.0.dist-info/METADATA,sha256=0F7tbgyrUoGpWMm9ndgqwKfIc-qKwolxwPoNOW5OjjE,383
5
+ recordstore-0.11.0.dist-info/WHEEL,sha256=K260EYznzXsJYBQGqmI8VTxEdiZYNvDZwW9cBh9-_MA,91
6
+ recordstore-0.11.0.dist-info/top_level.txt,sha256=rWW_kG4BjHRqO-y9BwZ-VjbKpbA9hezCBNtSKbWGyv0,12
7
+ recordstore-0.11.0.dist-info/RECORD,,
@@ -0,0 +1,5 @@
1
+ Wheel-Version: 1.0
2
+ Generator: setuptools (83.0.0)
3
+ Root-Is-Purelib: true
4
+ Tag: py3-none-any
5
+
@@ -0,0 +1,28 @@
1
+ BSD 3-Clause License
2
+
3
+ Copyright (c) 2026, Peter Foldiak
4
+
5
+ Redistribution and use in source and binary forms, with or without
6
+ modification, are permitted provided that the following conditions are met:
7
+
8
+ 1. Redistributions of source code must retain the above copyright notice, this
9
+ list of conditions and the following disclaimer.
10
+
11
+ 2. Redistributions in binary form must reproduce the above copyright notice,
12
+ this list of conditions and the following disclaimer in the documentation
13
+ and/or other materials provided with the distribution.
14
+
15
+ 3. Neither the name of the copyright holder nor the names of its
16
+ contributors may be used to endorse or promote products derived from
17
+ this software without specific prior written permission.
18
+
19
+ THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
20
+ AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
21
+ IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
22
+ DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
23
+ FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
24
+ DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
25
+ SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
26
+ CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
27
+ OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
28
+ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
@@ -0,0 +1 @@
1
+ recordstore