moat-kv 0.70.14__py3-none-any.whl → 0.70.18__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- moat/kv/__init__.py +4 -1
- moat/kv/_main.py +12 -4
- moat/kv/actor/deletor.py +4 -1
- moat/kv/auth/__init__.py +16 -10
- moat/kv/auth/_test.py +12 -4
- moat/kv/auth/password.py +1 -3
- moat/kv/client.py +44 -15
- moat/kv/code.py +3 -1
- moat/kv/command/acl.py +17 -4
- moat/kv/command/auth.py +14 -3
- moat/kv/command/code.py +41 -10
- moat/kv/command/codec.py +33 -14
- moat/kv/command/data.py +51 -16
- moat/kv/command/dump/__init__.py +10 -1
- moat/kv/command/error.py +12 -4
- moat/kv/command/internal.py +25 -7
- moat/kv/command/job.py +31 -9
- moat/kv/command/server.py +13 -3
- moat/kv/command/type.py +24 -8
- moat/kv/data.py +3 -1
- moat/kv/errors.py +23 -5
- moat/kv/mock/__init__.py +5 -2
- moat/kv/mock/mqtt.py +9 -3
- moat/kv/mock/serf.py +6 -2
- moat/kv/mock/tracer.py +3 -1
- moat/kv/model.py +21 -7
- moat/kv/obj/__init__.py +26 -5
- moat/kv/obj/command.py +10 -6
- moat/kv/runner.py +41 -12
- moat/kv/server.py +113 -32
- moat/kv/types.py +6 -2
- {moat_kv-0.70.14.dist-info → moat_kv-0.70.18.dist-info}/METADATA +5 -5
- moat_kv-0.70.18.dist-info/RECORD +49 -0
- moat_kv-0.70.14.dist-info/RECORD +0 -49
- {moat_kv-0.70.14.dist-info → moat_kv-0.70.18.dist-info}/LICENSE +0 -0
- {moat_kv-0.70.14.dist-info → moat_kv-0.70.18.dist-info}/LICENSE.APACHE2 +0 -0
- {moat_kv-0.70.14.dist-info → moat_kv-0.70.18.dist-info}/LICENSE.MIT +0 -0
- {moat_kv-0.70.14.dist-info → moat_kv-0.70.18.dist-info}/WHEEL +0 -0
- {moat_kv-0.70.14.dist-info → moat_kv-0.70.18.dist-info}/top_level.txt +0 -0
moat/kv/model.py
CHANGED
@@ -39,7 +39,9 @@ class Node:
|
|
39
39
|
tick: int = None
|
40
40
|
_present: RangeSet = None # I have these as valid data. Superset of ``._deleted``.
|
41
41
|
_deleted: RangeSet = None # I have these as no-longer-valid data
|
42
|
-
_reported: RangeSet =
|
42
|
+
_reported: RangeSet = (
|
43
|
+
None # somebody else reported these missing data for this node
|
44
|
+
)
|
43
45
|
_superseded: RangeSet = None # I know these once existed, but no more.
|
44
46
|
entries: dict = None
|
45
47
|
tock: int = 0 # tock when node was last observed
|
@@ -536,7 +538,9 @@ class NodeEvent:
|
|
536
538
|
class UpdateEvent:
|
537
539
|
"""Represents an event which updates something."""
|
538
540
|
|
539
|
-
def __init__(
|
541
|
+
def __init__(
|
542
|
+
self, event: NodeEvent, entry: "Entry", new_value, old_value=NotGiven, tock=None
|
543
|
+
):
|
540
544
|
self.event = event
|
541
545
|
self.entry = entry
|
542
546
|
self.new_value = new_value
|
@@ -875,8 +879,12 @@ class Entry:
|
|
875
879
|
|
876
880
|
if evt.event == self.chain:
|
877
881
|
if self._data != evt.new_value:
|
878
|
-
logger.error(
|
879
|
-
|
882
|
+
logger.error(
|
883
|
+
"Diff %r: has\n%r\nbut should have\n%r\n",
|
884
|
+
evt.event,
|
885
|
+
self._data,
|
886
|
+
evt.new_value,
|
887
|
+
)
|
880
888
|
return
|
881
889
|
|
882
890
|
if hasattr(evt, "new_value"):
|
@@ -896,7 +904,9 @@ class Entry:
|
|
896
904
|
if not loading:
|
897
905
|
logger.warning("*** inconsistency ***")
|
898
906
|
logger.warning("Node: %s", self.path)
|
899
|
-
logger.warning(
|
907
|
+
logger.warning(
|
908
|
+
"Current: %s :%s: %r", self.chain, self.tock, self._data
|
909
|
+
)
|
900
910
|
logger.warning("New: %s :%s: %r", evt.event, evt.tock, evt_val)
|
901
911
|
if evt.tock < self.tock:
|
902
912
|
if not loading:
|
@@ -924,7 +934,9 @@ class Entry:
|
|
924
934
|
n.seen(t, self)
|
925
935
|
await self.updated(evt)
|
926
936
|
|
927
|
-
async def walk(
|
937
|
+
async def walk(
|
938
|
+
self, proc, acl=None, max_depth=-1, min_depth=0, _depth=0, full=False
|
939
|
+
):
|
928
940
|
"""
|
929
941
|
Call coroutine ``proc`` on this node and all its children).
|
930
942
|
|
@@ -948,7 +960,9 @@ class Entry:
|
|
948
960
|
if k is None and not full:
|
949
961
|
continue
|
950
962
|
a = acl.step(k) if acl is not None else None
|
951
|
-
await v.walk(
|
963
|
+
await v.walk(
|
964
|
+
proc, acl=a, max_depth=max_depth, min_depth=min_depth, _depth=_depth
|
965
|
+
)
|
952
966
|
|
953
967
|
def serialize(self, chop_path=0, nchain=2, conv=None):
|
954
968
|
"""Serialize this entry for msgpack.
|
moat/kv/obj/__init__.py
CHANGED
@@ -16,7 +16,7 @@ try:
|
|
16
16
|
except ImportError:
|
17
17
|
from async_generator import asynccontextmanager
|
18
18
|
|
19
|
-
from moat.util import NoLock, NotGiven, Path, PathLongener, combine_dict
|
19
|
+
from moat.util import NoLock, NotGiven, Path, PathLongener, combine_dict, yload
|
20
20
|
|
21
21
|
__all__ = ["ClientEntry", "AttrClientEntry", "ClientRoot"]
|
22
22
|
|
@@ -261,7 +261,9 @@ class ClientEntry:
|
|
261
261
|
self.chain = r.chain
|
262
262
|
return r
|
263
263
|
|
264
|
-
async def delete(
|
264
|
+
async def delete(
|
265
|
+
self, _locked=False, nchain=0, chain=True, wait=False, recursive=False
|
266
|
+
):
|
265
267
|
"""Delete this node's value.
|
266
268
|
|
267
269
|
This is a coroutine.
|
@@ -417,7 +419,9 @@ class MirrorRoot(ClientEntry):
|
|
417
419
|
self._seen = dict()
|
418
420
|
|
419
421
|
@classmethod
|
420
|
-
async def as_handler(
|
422
|
+
async def as_handler(
|
423
|
+
cls, client, cfg=None, key="prefix", subpath=(), name=None, **kw
|
424
|
+
):
|
421
425
|
"""Return an (or "the") instance of this class.
|
422
426
|
|
423
427
|
The handler is created if it doesn't exist.
|
@@ -430,6 +434,19 @@ class MirrorRoot(ClientEntry):
|
|
430
434
|
d.append(cfg)
|
431
435
|
|
432
436
|
defcfg = client._cfg.get(cls.CFG)
|
437
|
+
if not defcfg:
|
438
|
+
# seems we didn't load the class' default config yet.
|
439
|
+
import inspect
|
440
|
+
from pathlib import Path as _Path
|
441
|
+
|
442
|
+
md = inspect.getmodule(cls)
|
443
|
+
try:
|
444
|
+
f = (_Path(md.__file__).parent / "_config.yaml").open("r")
|
445
|
+
except EnvironmentError:
|
446
|
+
pass
|
447
|
+
else:
|
448
|
+
with f:
|
449
|
+
defcfg = yload(f, attr=True).get("kv",{}).get(cls.CFG)
|
433
450
|
if cfg:
|
434
451
|
if defcfg:
|
435
452
|
cfg = combine_dict(cfg, defcfg)
|
@@ -444,7 +461,9 @@ class MirrorRoot(ClientEntry):
|
|
444
461
|
name = str(Path("_moat.kv", client.name, cls.CFG, *subpath))
|
445
462
|
|
446
463
|
def make():
|
447
|
-
return client.mirror(
|
464
|
+
return client.mirror(
|
465
|
+
cfg[key] + subpath, root_type=cls, need_wait=True, cfg=cfg, **kw
|
466
|
+
)
|
448
467
|
|
449
468
|
return await client.unique_helper(name, factory=make)
|
450
469
|
|
@@ -532,7 +551,9 @@ class MirrorRoot(ClientEntry):
|
|
532
551
|
pass
|
533
552
|
|
534
553
|
# update entry
|
535
|
-
entry.chain =
|
554
|
+
entry.chain = (
|
555
|
+
None if val is NotGiven else r.get("chain", None)
|
556
|
+
)
|
536
557
|
await entry.set_value(val)
|
537
558
|
|
538
559
|
if val is NotGiven and not entry:
|
moat/kv/obj/command.py
CHANGED
@@ -54,7 +54,9 @@ class _InvSub:
|
|
54
54
|
def id_arg(self, proc):
|
55
55
|
if self.id_name is None:
|
56
56
|
return proc
|
57
|
-
return click.argument(
|
57
|
+
return click.argument(
|
58
|
+
self.id_name, type=self.id_typ, callback=self.id_cb, nargs=1
|
59
|
+
)(proc)
|
58
60
|
|
59
61
|
def apply_aux(self, proc):
|
60
62
|
for t in self.aux:
|
@@ -110,9 +112,7 @@ def std_command(cli, *a, **kw):
|
|
110
112
|
\b
|
111
113
|
Use '… {tname} -' to list all entries.
|
112
114
|
Use '… {tname} NAME' to show details of a single entry.
|
113
|
-
""".format(
|
114
|
-
tname=tname, tlname=tinv.long_name
|
115
|
-
),
|
115
|
+
""".format(tname=tname, tlname=tinv.long_name),
|
116
116
|
)
|
117
117
|
@click.argument("name", type=str, nargs=1)
|
118
118
|
@click.pass_context
|
@@ -123,7 +123,9 @@ def std_command(cli, *a, **kw):
|
|
123
123
|
|
124
124
|
if name == "-":
|
125
125
|
if ctx.invoked_subcommand is not None:
|
126
|
-
raise click.BadParameter(
|
126
|
+
raise click.BadParameter(
|
127
|
+
"The name '-' triggers a list and precludes subcommands."
|
128
|
+
)
|
127
129
|
cnt = 0
|
128
130
|
for n in this(obj).all_children if tinv.list_recursive else this(obj):
|
129
131
|
cnt += 1
|
@@ -150,7 +152,9 @@ def std_command(cli, *a, **kw):
|
|
150
152
|
else:
|
151
153
|
vv = "-"
|
152
154
|
elif isinstance(vv, dict):
|
153
|
-
vv = " ".join(
|
155
|
+
vv = " ".join(
|
156
|
+
"%s=%s" % (x, y) for x, y in sorted(vv.items())
|
157
|
+
)
|
154
158
|
print("%s %s %s" % (k, kk, vv), file=obj.stdout)
|
155
159
|
else:
|
156
160
|
print("%s %s" % (k, v), file=obj.stdout)
|
moat/kv/runner.py
CHANGED
@@ -24,7 +24,14 @@ from moat.util import (
|
|
24
24
|
spawn,
|
25
25
|
)
|
26
26
|
|
27
|
-
from .actor import
|
27
|
+
from .actor import (
|
28
|
+
ActorState,
|
29
|
+
BrokenState,
|
30
|
+
ClientActor,
|
31
|
+
CompleteState,
|
32
|
+
DetachedState,
|
33
|
+
PartialState,
|
34
|
+
)
|
28
35
|
from .exceptions import ServerError
|
29
36
|
from .obj import AttrClientEntry, ClientRoot, MirrorRoot
|
30
37
|
|
@@ -287,7 +294,9 @@ class CallAdmin:
|
|
287
294
|
async with slf.client.watch(path, **kw) as watcher:
|
288
295
|
yield watcher
|
289
296
|
else:
|
290
|
-
raise RuntimeError(
|
297
|
+
raise RuntimeError(
|
298
|
+
f"What should I do with a path marked {path.mark !r}?"
|
299
|
+
)
|
291
300
|
|
292
301
|
with anyio.CancelScope() as sc:
|
293
302
|
slf.scope = sc
|
@@ -309,7 +318,9 @@ class CallAdmin:
|
|
309
318
|
elif msg.get("state", "") == "uptodate":
|
310
319
|
slf.admin._n_watch_seen += 1
|
311
320
|
if slf.admin._n_watch_seen == slf.admin._n_watch:
|
312
|
-
await slf.runner.send_event(
|
321
|
+
await slf.runner.send_event(
|
322
|
+
ReadyMsg(slf.admin._n_watch_seen)
|
323
|
+
)
|
313
324
|
|
314
325
|
def cancel(slf):
|
315
326
|
if slf.scope is None:
|
@@ -544,7 +555,9 @@ class RunnerEntry(AttrClientEntry):
|
|
544
555
|
if state.node is not None:
|
545
556
|
raise RuntimeError(f"already running on {state.node}")
|
546
557
|
code = self.root.code.follow(self.code, create=False)
|
547
|
-
data = combine_dict(
|
558
|
+
data = combine_dict(
|
559
|
+
self.data or {}, {} if code.value is NotGiven else code.value.get("default", {}), deep=True
|
560
|
+
)
|
548
561
|
|
549
562
|
if code.is_async:
|
550
563
|
data["_info"] = self._q = create_queue(QLEN)
|
@@ -561,7 +574,9 @@ class RunnerEntry(AttrClientEntry):
|
|
561
574
|
|
562
575
|
await state.save(wait=True)
|
563
576
|
if state.node != state.root.name:
|
564
|
-
raise RuntimeError(
|
577
|
+
raise RuntimeError(
|
578
|
+
"Rudely taken away from us.", state.node, state.root.name
|
579
|
+
)
|
565
580
|
|
566
581
|
data["_self"] = calls = CallAdmin(self, state, data)
|
567
582
|
res = await calls._run(code, data)
|
@@ -778,7 +793,10 @@ class StateEntry(AttrClientEntry):
|
|
778
793
|
await self.delete()
|
779
794
|
return
|
780
795
|
|
781
|
-
if self.node is None
|
796
|
+
if self.node is None:
|
797
|
+
return
|
798
|
+
if self.node != self.root.name:
|
799
|
+
self.root.runner.get_node(self.node)
|
782
800
|
return
|
783
801
|
|
784
802
|
self.stopped = time.time()
|
@@ -834,7 +852,10 @@ class StateEntry(AttrClientEntry):
|
|
834
852
|
run.scope.cancel()
|
835
853
|
elif n is not None:
|
836
854
|
logger.warning(
|
837
|
-
"Runner %s at %r: running but node is %s",
|
855
|
+
"Runner %s at %r: running but node is %s",
|
856
|
+
self.root.name,
|
857
|
+
self.subpath,
|
858
|
+
n,
|
838
859
|
)
|
839
860
|
|
840
861
|
await run.root.trigger_rescan()
|
@@ -940,7 +961,9 @@ class _BaseRunnerRoot(ClientRoot):
|
|
940
961
|
cfg_ = client._cfg["runner"]
|
941
962
|
else:
|
942
963
|
cfg_ = combine_dict(cfg, client._cfg["runner"])
|
943
|
-
return await super().as_handler(
|
964
|
+
return await super().as_handler(
|
965
|
+
client, subpath=subpath, _subpath=subpath, cfg=cfg_, **kw
|
966
|
+
)
|
944
967
|
|
945
968
|
async def run_starting(self):
|
946
969
|
from .code import CodeRoot
|
@@ -1073,7 +1096,9 @@ class AnyRunnerRoot(_BaseRunnerRoot):
|
|
1073
1096
|
def __init__(self, *a, **kw):
|
1074
1097
|
super().__init__(*a, **kw)
|
1075
1098
|
self.group = (
|
1076
|
-
P(self.client.config.server["root"]) + P(self._cfg["name"])
|
1099
|
+
P(self.client.config.server["root"]) + P(self._cfg["name"])
|
1100
|
+
| "any"
|
1101
|
+
| self._path[-1]
|
1077
1102
|
)
|
1078
1103
|
|
1079
1104
|
def get_node(self, name):
|
@@ -1151,7 +1176,7 @@ class AnyRunnerRoot(_BaseRunnerRoot):
|
|
1151
1176
|
self._stale_times.append(cur)
|
1152
1177
|
if self._stale_times[0] > cur - self.max_age:
|
1153
1178
|
return
|
1154
|
-
if len(self._stale_times)
|
1179
|
+
if len(self._stale_times) <= 2*self._cfg["actor"]["n_hosts"]+1:
|
1155
1180
|
return
|
1156
1181
|
cut = self._stale_times.pop(0)
|
1157
1182
|
|
@@ -1253,7 +1278,9 @@ class SingleRunnerRoot(_BaseRunnerRoot):
|
|
1253
1278
|
async with anyio.create_task_group() as tg:
|
1254
1279
|
age_q = create_queue(1)
|
1255
1280
|
|
1256
|
-
async with ClientActor(
|
1281
|
+
async with ClientActor(
|
1282
|
+
self.client, self.name, topic=self.group, cfg=self._cfg
|
1283
|
+
) as act:
|
1257
1284
|
self._act = act
|
1258
1285
|
tg.start_soon(self._age_notifier, age_q)
|
1259
1286
|
await self.spawn(self._run_now)
|
@@ -1303,7 +1330,9 @@ class AllRunnerRoot(SingleRunnerRoot):
|
|
1303
1330
|
def __init__(self, *a, **kw):
|
1304
1331
|
super().__init__(*a, **kw)
|
1305
1332
|
self.group = (
|
1306
|
-
P(self.client.config.server["root"]) + P(self._cfg["name"])
|
1333
|
+
P(self.client.config.server["root"]) + P(self._cfg["name"])
|
1334
|
+
| "all"
|
1335
|
+
| self._path[-1]
|
1307
1336
|
)
|
1308
1337
|
|
1309
1338
|
async def _state_runner(self):
|
moat/kv/server.py
CHANGED
@@ -151,7 +151,9 @@ class StreamCommand:
|
|
151
151
|
self.client.in_stream[self.seq] = self
|
152
152
|
self.qlen = self.client.qlen
|
153
153
|
if self.qlen:
|
154
|
-
self.qr = DelayedRead(
|
154
|
+
self.qr = DelayedRead(
|
155
|
+
self.qlen, get_seq=self._get_seq, send_ack=self._send_ack
|
156
|
+
)
|
155
157
|
self.dw = DelayedWrite(self.qlen)
|
156
158
|
else:
|
157
159
|
self.qr = create_queue(1)
|
@@ -225,7 +227,9 @@ class StreamCommand:
|
|
225
227
|
await self.send(**res)
|
226
228
|
except Exception as exc:
|
227
229
|
if not isinstance(exc, CancelledError):
|
228
|
-
self.client.logger.exception(
|
230
|
+
self.client.logger.exception(
|
231
|
+
"ERS%d %r", self.client._client_nr, self.msg
|
232
|
+
)
|
229
233
|
await self.send(error=repr(exc))
|
230
234
|
finally:
|
231
235
|
with anyio.move_on_after(2, shield=True):
|
@@ -475,7 +479,11 @@ class SCmd_get_tree(StreamCommand):
|
|
475
479
|
if root is None:
|
476
480
|
root = client.root
|
477
481
|
entry, acl = root.follow_acl(
|
478
|
-
msg.path,
|
482
|
+
msg.path,
|
483
|
+
create=False,
|
484
|
+
nulls_ok=client.nulls_ok,
|
485
|
+
acl=client.acl,
|
486
|
+
acl_key="e",
|
479
487
|
)
|
480
488
|
else:
|
481
489
|
entry, _ = root.follow_acl(msg.path, create=False, nulls_ok=client.nulls_ok)
|
@@ -560,7 +568,9 @@ class SCmd_watch(StreamCommand):
|
|
560
568
|
return
|
561
569
|
if entry.tock < tock:
|
562
570
|
res = entry.serialize(
|
563
|
-
chop_path=client._chop_path,
|
571
|
+
chop_path=client._chop_path,
|
572
|
+
nchain=nchain,
|
573
|
+
conv=conv,
|
564
574
|
)
|
565
575
|
shorter(res)
|
566
576
|
if not acl.allows("r"):
|
@@ -697,7 +707,9 @@ class ServerClient:
|
|
697
707
|
with anyio.CancelScope() as s:
|
698
708
|
self.tasks[seq] = s
|
699
709
|
if "chain" in msg:
|
700
|
-
msg.chain = NodeEvent.deserialize(
|
710
|
+
msg.chain = NodeEvent.deserialize(
|
711
|
+
msg.chain, cache=self.server.node_cache
|
712
|
+
)
|
701
713
|
|
702
714
|
fn = None
|
703
715
|
if msg.get("state", "") != "start":
|
@@ -912,7 +924,9 @@ class ServerClient:
|
|
912
924
|
if msg.get("nchain", 0):
|
913
925
|
entry["chain"] = None
|
914
926
|
else:
|
915
|
-
entry = entry.serialize(
|
927
|
+
entry = entry.serialize(
|
928
|
+
chop_path=-1, nchain=msg.get("nchain", 0), conv=self.conv
|
929
|
+
)
|
916
930
|
return entry
|
917
931
|
|
918
932
|
async def cmd_set_value(self, msg, **kw):
|
@@ -941,7 +955,9 @@ class ServerClient:
|
|
941
955
|
entry, acl = root.follow_acl(msg.path, acl=acl, acl_key="W", nulls_ok=_nulls_ok)
|
942
956
|
if root is self.root and "match" in self.metaroot:
|
943
957
|
try:
|
944
|
-
self.metaroot["match"].check_value(
|
958
|
+
self.metaroot["match"].check_value(
|
959
|
+
None if value is NotGiven else value, entry
|
960
|
+
)
|
945
961
|
except ClientError:
|
946
962
|
raise
|
947
963
|
except Exception as exc:
|
@@ -952,7 +968,11 @@ class ServerClient:
|
|
952
968
|
send_prev = True
|
953
969
|
nchain = msg.get("nchain", 1)
|
954
970
|
|
955
|
-
if
|
971
|
+
if (
|
972
|
+
msg.get("idem", False)
|
973
|
+
and type(entry.data) is type(value)
|
974
|
+
and entry.data == value
|
975
|
+
):
|
956
976
|
res = attrdict(tock=entry.tock, changed=False)
|
957
977
|
if nchain > 0:
|
958
978
|
res.chain = entry.chain.serialize(nchain=nchain)
|
@@ -960,7 +980,9 @@ class ServerClient:
|
|
960
980
|
|
961
981
|
if "prev" in msg:
|
962
982
|
if entry.data != msg.prev:
|
963
|
-
raise ClientError(
|
983
|
+
raise ClientError(
|
984
|
+
f"Data is {entry.data !r} not {msg.prev !r} at {msg.path}"
|
985
|
+
)
|
964
986
|
send_prev = False
|
965
987
|
if "chain" in msg:
|
966
988
|
if msg.chain is None:
|
@@ -987,7 +1009,9 @@ class ServerClient:
|
|
987
1009
|
async with self.server.next_event() as event:
|
988
1010
|
await entry.set_data(
|
989
1011
|
event,
|
990
|
-
NotGiven
|
1012
|
+
NotGiven
|
1013
|
+
if value is NotGiven
|
1014
|
+
else self.conv.dec_value(value, entry=entry),
|
991
1015
|
server=self.server,
|
992
1016
|
tock=self.server.tock,
|
993
1017
|
)
|
@@ -1004,7 +1028,11 @@ class ServerClient:
|
|
1004
1028
|
You usually do this via a stream command.
|
1005
1029
|
"""
|
1006
1030
|
msg = UpdateEvent.deserialize(
|
1007
|
-
self.root,
|
1031
|
+
self.root,
|
1032
|
+
msg,
|
1033
|
+
nulls_ok=self.nulls_ok,
|
1034
|
+
conv=self.conv,
|
1035
|
+
cache=self.server._nodes,
|
1008
1036
|
)
|
1009
1037
|
res = await msg.entry.apply(msg, server=self, root=self.root)
|
1010
1038
|
if res is None:
|
@@ -1066,10 +1094,15 @@ class ServerClient:
|
|
1066
1094
|
res = 0
|
1067
1095
|
if entry.data is not None and acl.allows("d"):
|
1068
1096
|
async with self.server.next_event() as event:
|
1069
|
-
evt = await entry.set_data(
|
1097
|
+
evt = await entry.set_data(
|
1098
|
+
event, NotGiven, server=self, tock=self.server.tock
|
1099
|
+
)
|
1070
1100
|
if nchain:
|
1071
1101
|
r = evt.serialize(
|
1072
|
-
chop_path=self._chop_path,
|
1102
|
+
chop_path=self._chop_path,
|
1103
|
+
nchain=nchain,
|
1104
|
+
with_old=True,
|
1105
|
+
conv=self.conv,
|
1073
1106
|
)
|
1074
1107
|
r["seq"] = seq
|
1075
1108
|
r.pop("new_value", None) # always None
|
@@ -1119,7 +1152,9 @@ class ServerClient:
|
|
1119
1152
|
if msg.typ is None:
|
1120
1153
|
val.pop("current", None)
|
1121
1154
|
elif msg.typ not in a or not len(a[msg.typ]["user"].keys()):
|
1122
|
-
raise RuntimeError(
|
1155
|
+
raise RuntimeError(
|
1156
|
+
"You didn't configure this method yet:" + repr((msg.typ, vars(a)))
|
1157
|
+
)
|
1123
1158
|
else:
|
1124
1159
|
val["current"] = msg.typ
|
1125
1160
|
msg.value = val
|
@@ -1203,7 +1238,9 @@ class ServerClient:
|
|
1203
1238
|
await evt.wait()
|
1204
1239
|
except Exception as exc:
|
1205
1240
|
msg = {"error": str(exc)}
|
1206
|
-
if isinstance(
|
1241
|
+
if isinstance(
|
1242
|
+
exc, ClientError
|
1243
|
+
): # pylint doesn't seem to see this, so …:
|
1207
1244
|
msg["etype"] = exc.etype # pylint: disable=no-member ### YES IT HAS
|
1208
1245
|
else:
|
1209
1246
|
self.logger.exception(
|
@@ -1241,7 +1278,12 @@ class _RecoverControl:
|
|
1241
1278
|
_id = 0
|
1242
1279
|
|
1243
1280
|
def __init__(
|
1244
|
-
self,
|
1281
|
+
self,
|
1282
|
+
server,
|
1283
|
+
scope,
|
1284
|
+
prio,
|
1285
|
+
local_history,
|
1286
|
+
sources, # pylint:disable=redefined-outer-name
|
1245
1287
|
):
|
1246
1288
|
self.server = server
|
1247
1289
|
self.scope = scope
|
@@ -1431,13 +1473,17 @@ class Server:
|
|
1431
1473
|
yield n
|
1432
1474
|
except BaseException as exc:
|
1433
1475
|
if n is not None:
|
1434
|
-
self.logger.warning(
|
1476
|
+
self.logger.warning(
|
1477
|
+
"Deletion %s %d due to %r", self.node, n.tick, exc
|
1478
|
+
)
|
1435
1479
|
self.node.report_deleted(
|
1436
|
-
RangeSet((nt,)),
|
1480
|
+
RangeSet((nt,)),
|
1481
|
+
self, # pylint: disable=used-before-assignment
|
1437
1482
|
)
|
1438
1483
|
with anyio.move_on_after(2, shield=True):
|
1439
1484
|
await self._send_event(
|
1440
|
-
"info",
|
1485
|
+
"info",
|
1486
|
+
dict(node="", tick=0, deleted={self.node.name: (nt,)}),
|
1441
1487
|
)
|
1442
1488
|
raise
|
1443
1489
|
finally:
|
@@ -1693,7 +1739,9 @@ class Server:
|
|
1693
1739
|
"""
|
1694
1740
|
Process an update message: deserialize it and apply the result.
|
1695
1741
|
"""
|
1696
|
-
msg = UpdateEvent.deserialize(
|
1742
|
+
msg = UpdateEvent.deserialize(
|
1743
|
+
self.root, msg, cache=self.node_cache, nulls_ok=True
|
1744
|
+
)
|
1697
1745
|
await msg.entry.apply(msg, server=self, root=self.paranoid_root)
|
1698
1746
|
|
1699
1747
|
async def user_info(self, msg):
|
@@ -1967,7 +2015,9 @@ class Server:
|
|
1967
2015
|
try:
|
1968
2016
|
# First try to read the host name from the meta-root's
|
1969
2017
|
# "hostmap" entry, if any.
|
1970
|
-
hme = self.root.follow(
|
2018
|
+
hme = self.root.follow(
|
2019
|
+
Path(None, "hostmap", host), create=False, nulls_ok=True
|
2020
|
+
)
|
1971
2021
|
if hme.data is NotGiven:
|
1972
2022
|
raise KeyError(host)
|
1973
2023
|
except KeyError:
|
@@ -2064,7 +2114,11 @@ class Server:
|
|
2064
2114
|
|
2065
2115
|
pl = PathLongener(())
|
2066
2116
|
res = await client._request(
|
2067
|
-
"get_tree",
|
2117
|
+
"get_tree",
|
2118
|
+
iter=True,
|
2119
|
+
from_server=self.node.name,
|
2120
|
+
nchain=-1,
|
2121
|
+
path=(),
|
2068
2122
|
)
|
2069
2123
|
async for r in res:
|
2070
2124
|
pl(r)
|
@@ -2118,7 +2172,9 @@ class Server:
|
|
2118
2172
|
if len(self.fetch_missing):
|
2119
2173
|
self.fetch_running = False
|
2120
2174
|
for nm in self.fetch_missing:
|
2121
|
-
self.logger.error(
|
2175
|
+
self.logger.error(
|
2176
|
+
"Sync: missing: %s %s", nm.name, nm.local_missing
|
2177
|
+
)
|
2122
2178
|
await self.spawn(self.do_send_missing)
|
2123
2179
|
if self.force_startup or not len(self.fetch_missing):
|
2124
2180
|
if self.node.tick is None:
|
@@ -2298,7 +2354,9 @@ class Server:
|
|
2298
2354
|
if prio is None:
|
2299
2355
|
await anyio.sleep(clock * (1 + self._actor.random / 3))
|
2300
2356
|
else:
|
2301
|
-
await anyio.sleep(
|
2357
|
+
await anyio.sleep(
|
2358
|
+
clock * (1 - (1 / (1 << prio)) / 2 - self._actor.random / 5)
|
2359
|
+
)
|
2302
2360
|
|
2303
2361
|
self.logger.debug("SendMissingGo %s %s", prio, self.sending_missing)
|
2304
2362
|
while self.sending_missing:
|
@@ -2309,7 +2367,10 @@ class Server:
|
|
2309
2367
|
deleted = {}
|
2310
2368
|
for n in nodes:
|
2311
2369
|
self.logger.debug(
|
2312
|
-
"SendMissingGo %s %r %r",
|
2370
|
+
"SendMissingGo %s %r %r",
|
2371
|
+
n.name,
|
2372
|
+
n.remote_missing,
|
2373
|
+
n.local_superseded,
|
2313
2374
|
)
|
2314
2375
|
k = n.remote_missing & n.local_superseded
|
2315
2376
|
for r in n.remote_missing & n.local_present:
|
@@ -2370,12 +2431,18 @@ class Server:
|
|
2370
2431
|
await self.tock_seen(m.tock)
|
2371
2432
|
else:
|
2372
2433
|
m.tock = self.tock
|
2373
|
-
m = UpdateEvent.deserialize(
|
2434
|
+
m = UpdateEvent.deserialize(
|
2435
|
+
self.root, m, cache=self.node_cache, nulls_ok=True
|
2436
|
+
)
|
2374
2437
|
await self.tock_seen(m.tock)
|
2375
|
-
await m.entry.apply(
|
2438
|
+
await m.entry.apply(
|
2439
|
+
m, server=self, root=self.paranoid_root, loading=True
|
2440
|
+
)
|
2376
2441
|
elif "info" in m:
|
2377
2442
|
await self._process_info(m["info"])
|
2378
|
-
elif
|
2443
|
+
elif (
|
2444
|
+
"nodes" in m or "known" in m or "deleted" in m or "tock" in m
|
2445
|
+
): # XXX LEGACY
|
2379
2446
|
await self._process_info(m)
|
2380
2447
|
else:
|
2381
2448
|
self.logger.warning("Unknown message in stream: %s", repr(m))
|
@@ -2501,7 +2568,11 @@ class Server:
|
|
2501
2568
|
self._savers.append(state)
|
2502
2569
|
try:
|
2503
2570
|
await self.save_stream(
|
2504
|
-
path=path,
|
2571
|
+
path=path,
|
2572
|
+
stream=stream,
|
2573
|
+
done=done,
|
2574
|
+
done_val=s,
|
2575
|
+
save_state=save_state,
|
2505
2576
|
)
|
2506
2577
|
except EnvironmentError as err:
|
2507
2578
|
if done is None:
|
@@ -2511,7 +2582,9 @@ class Server:
|
|
2511
2582
|
with anyio.CancelScope(shield=True):
|
2512
2583
|
sd.set()
|
2513
2584
|
|
2514
|
-
async def run_saver(
|
2585
|
+
async def run_saver(
|
2586
|
+
self, path: str = None, stream=None, save_state=False, wait: bool = True
|
2587
|
+
):
|
2515
2588
|
"""
|
2516
2589
|
Start a task that continually saves to disk.
|
2517
2590
|
|
@@ -2532,7 +2605,13 @@ class Server:
|
|
2532
2605
|
res = None
|
2533
2606
|
if path is not None:
|
2534
2607
|
await self.spawn(
|
2535
|
-
partial(
|
2608
|
+
partial(
|
2609
|
+
self._saver,
|
2610
|
+
path=path,
|
2611
|
+
stream=stream,
|
2612
|
+
save_state=save_state,
|
2613
|
+
done=done,
|
2614
|
+
)
|
2536
2615
|
)
|
2537
2616
|
if wait:
|
2538
2617
|
res = await done.get()
|
@@ -2653,7 +2732,9 @@ class Server:
|
|
2653
2732
|
assert self.node.tick is None
|
2654
2733
|
self.node.tick = 0
|
2655
2734
|
async with self.next_event() as event:
|
2656
|
-
await self.root.set_data(
|
2735
|
+
await self.root.set_data(
|
2736
|
+
event, self._init, tock=self.tock, server=self
|
2737
|
+
)
|
2657
2738
|
|
2658
2739
|
await self.spawn(self._sigterm)
|
2659
2740
|
|
moat/kv/types.py
CHANGED
@@ -306,7 +306,9 @@ class CodecEntry(Entry):
|
|
306
306
|
) from exc
|
307
307
|
else:
|
308
308
|
if r != w:
|
309
|
-
raise ValueError(
|
309
|
+
raise ValueError(
|
310
|
+
f"Decoding at {self.path}: {v!r} got {r!r}, not {w!r}"
|
311
|
+
)
|
310
312
|
|
311
313
|
if value is not None and value.encode is not None:
|
312
314
|
if not value["out"]:
|
@@ -321,7 +323,9 @@ class CodecEntry(Entry):
|
|
321
323
|
) from exc
|
322
324
|
else:
|
323
325
|
if r != w:
|
324
|
-
raise ValueError(
|
326
|
+
raise ValueError(
|
327
|
+
f"Encoding at {self.path}: {v!r} got {r!r}, not {w!r}"
|
328
|
+
)
|
325
329
|
|
326
330
|
await super().set(value)
|
327
331
|
self._enc = enc
|