moat-kv 0.70.23__py3-none-any.whl → 0.70.24__py3-none-any.whl

examples/pathify.py

@@ -0,0 +1,45 @@
+#!/usr/bin/python3
+
+# Batch-convert data. In this case I had some entries which were stored as
+# a list, but using Path made much more sense (esp when you need to
+# view/edit the yaml export).
+
+import anyio
+from moat.kv.client import open_client
+from moat.util import P, yload, Path
+import asyncclick as click
+
+
+def conv(m, s: str) -> bool:
+    try:
+        d = m.value[s]
+    except KeyError:
+        return 0
+    if isinstance(d, Path):
+        return 0
+    if not isinstance(d, Sequence):
+        return 0
+    d = Path.build(d)
+    m.value[s] = d
+    return 1
+
+
+@click.command()
+@click.argument("path", type=P)
+@click.argument("keys", type=str, nargs=-1)
+async def main(path, keys):
+    if not keys:
+        keys = "src dest dst state".split()
+    with open("/etc/moat.kv.cfg") as cff:
+        cfg = yload(cff)
+    async with open_client(**cfg) as client:
+        async for m in client.get_tree(path, nchain=2):
+            n = 0
+            for k in keys:
+                n += conv(m, k)
+            if n:
+                await client.set(ORIG + m.path, value=m.value, chain=m.chain)
+
+
+if __name__ == "__main__":
+    main()

moat/kv/actor/__init__.py

@@ -0,0 +1,98 @@
+"""
+This module implements a :class:`asyncactor.Actor` which works on top of
+a MoaT-KV client.
+"""
+from __future__ import annotations
+
+from asyncactor import Actor
+from asyncactor.abc import MonitorStream, Transport
+
+__all__ = [
+    "ClientActor",
+    "ActorState",
+    "BrokenState",
+    "DetachedState",
+    "PartialState",
+    "CompleteState",
+]
+
+
+class ClientActor(Actor):
+    def __init__(self, client, *a, topic, **kw):
+        super().__init__(ClientTransport(client, topic), *a, **kw)
+
+
+class ClientTransport(Transport):
+    """
+    This class exports the client's direct messaging interface to the
+    actor.
+    """
+
+    def __init__(self, client, topic):
+        self.client = client
+        self.topic = topic
+
+    def monitor(self):
+        return ClientMonitor(self)
+
+    async def send(self, payload):
+        await self.client.msg_send(self.topic, payload)
+
+
+class ClientMonitor(MonitorStream):
+    _mon1 = None
+    _mon2 = None
+    _it = None
+
+    async def __aenter__(self):
+        self._mon1 = self.transport.client.msg_monitor(self.transport.topic)
+        self._mon2 = await self._mon1.__aenter__()
+        return self
+
+    async def __aexit__(self, *tb):
+        return await self._mon1.__aexit__(*tb)
+
+    def __aiter__(self):
+        self._it = self._mon2.__aiter__()
+        return self
+
+    async def __anext__(self):
+        msg = await self._it.__anext__()
+        return msg.data
+
+
+# The following events are used by Runner etc. to notify running jobs
+# about the current connectivity state.
+#
+class ActorState:
+    """base class for states"""
+
+    def __init__(self, msg=None):
+        self.msg = msg
+
+    def __repr__(self):
+        return "<%s:%r>" % (self.__class__.__name__, self.msg)
+
+
+class BrokenState(ActorState):
+    """I have no idea what's happening, probably nothing good"""
+
+    pass
+
+
+class DetachedState(ActorState):
+    """I am detached, my actor group is not visible"""
+
+    pass
+
+
+class PartialState(ActorState):
+    """Some but not all members of my actor group are visible"""
+
+    pass
+
+
+class CompleteState(ActorState):
+    """All members of my actor group are visible"""
+
+    pass

moat/kv/actor/deletor.py

@@ -0,0 +1,139 @@
+"""
+This module implements additional code for the server-side DeleteActor,
+which is used to clean up the list of deleted nodes.
+"""
+
+from __future__ import annotations
+
+import weakref
+from collections import deque
+
+import anyio
+from asyncactor import Actor, PingEvent, TagEvent
+from asyncactor.backend import get_transport
+
+TAGS = 4
+
+
+class DeleteActor:
+    _enabled = None
+
+    def __init__(self, server):
+        self._server = weakref.ref(server)
+        self.deleted = deque()
+        self.tags = []
+        self.actor = None
+
+        self.max_seen = 0
+        self.n_tags = 0
+        self.n_pings = 0
+        self.n_nodes = 0
+
+    @property
+    def server(self):
+        return self._server()
+
+    async def tock_me(self):
+        """
+        Add the current tock to our buffer.
+
+        This is updated whenever a new leader is selected.
+        """
+        self.tags.append(self.server.tock)
+        self.tags = self.tags[-TAGS:]
+        await self.actor.set_value((self.tags[0], self.tags[-1]))
+
+    def add_deleted(self, nodes: NodeSet):  # noqa: F821
+        """
+        These nodes are deleted. Remember them for some time.
+        """
+        if self.n_nodes == 0:
+            return
+        self.deleted.append((self.server.tock, nodes))
+
+    def purge_to(self, tock):
+        """
+        Sufficient time has passed since this tock was seen, while all
+        Delete actor nodes were active. Finally flush the entries that have
+        been deleted before it.
+        """
+        while self.deleted and self.deleted[0][0] < tock:
+            d = self.deleted.popleft()
+            self.server.purge_deleted(d[1])
+
+    async def enable(self, n):
+        """
+        Enable this actor, as a group of N.
+        """
+        if self.actor is None:
+            self._enabled = True
+        else:
+            await self.actor.enable(n)
+        self.n_tags = 0
+        self.n_pings = 0
+        self.n_nodes = n
+
+    async def disable(self, n: int = 0):
+        """
+        Disable this actor. It will still listen, and require N Delete
+        actor members in order to flush its deletion entries.
+
+        Completely disable deletion flushing by passing n=0.
+        """
+        if self.actor is None:
+            self._enabled = False
+        else:
+            await self.actor.disable()
+        self.n_tags = 0
+        self.n_pings = 0
+        self.n_nodes = n
+
+    async def run(self, evt: anyio.abc.Event = None):
+        """
+        The task that monitors the Delete actor.
+        """
+        try:
+            T = get_transport("moat_kv")
+            async with Actor(
+                T(self.server.backend, *self.server.cfg.server.root, "del"),
+                name=self.server.node.name,
+                cfg=self.server.cfg.server.delete,
+                enabled=False,
+            ) as actor:
+                self.actor = actor
+                if self._enabled is not None:
+                    if self._enabled:
+                        await actor.enable()
+                    else:
+                        await actor.disable()
+                if evt is not None:
+                    evt.set()
+                async for evt in actor:
+                    if isinstance(evt, PingEvent):
+                        val = evt.value
+                        if val is None:
+                            self.n_pings = self.n_tags = 0
+                            continue
+                        if len(evt.msg.history) < self.n_nodes:
+                            self.n_pings = self.n_tags = 0
+                            continue
+                        self.n_pings += 1
+                        if self.n_pings > self.n_nodes:
+                            mx, self.max_seen = (
+                                self.max_seen,
+                                max(self.max_seen, val[1]),
+                            )
+                            if val[0] > mx > 0:
+                                await self.server.resync_deleted(evt.msg.history)
+                                continue
+                            self.purge_to(val[0])
+                            self.max_seen = max(self.max_seen, val[1])
+
+                    elif isinstance(evt, TagEvent):
+                        if actor.history_size == self.n_nodes:
+                            self.n_tags += 1
+                            if self.n_tags > 2:
+                                self.purge_to(self.tags[0])
+                            await self.tock_me()
+        finally:
+            self.actor = None