moat-kv 0.70.23__py3-none-any.whl → 0.71.0__py3-none-any.whl

docs/source/v3.rst

@@ -0,0 +1,168 @@
+=================
+MoaT-KV Version 3
+=================
+
++++++++++
+Rationale
++++++++++
+
+The V2 servers worked, for the most part, but they had a couple of problems.
+
+* sometimes there are strange deadlocks
+
+* msgpack isn't that widely supported compared to CBOR
+
+* the initial client/server negotiation isn't versioned
+
+* the sync protocol is somewhat overengineered
+
+* speed of updates is limited by the client>server>MQTT>server>client
+  chain; two possibly-high-load servers in between cause too much delay
+
+* Messages tended to be more verbose than necessary
+
+++++++++++++++++++
+V3 design overview
+++++++++++++++++++
+
+Server start
+++++++++++++
+
+* connect to MQTT, listen to all MoaT messages,
+  update internal state as messages come in
+
+* ask for a server link on the Join/Actor topic
+
+* load the local backup if present
+
+* fetch full tree from designated server is replied
+
+* merge data
+
+* Join actor topic
+
+
+Client connection setup A
++++++++++++++++++++++++++
+
+* Client connects to MQTT and sends Query message
+* Designated server replies with connection data
+* Client connects to server
+
+* Server greets client
+* Client authenticates
+* Server sends MQTT connection information
+* Client connects to MQTT and sends Birth message
+* Server sees the client's message and sends ACK to the client
+
+Client connection setup B
++++++++++++++++++++++++++
+
+This method is slower; it can be used when the configured server doesn't work.
+
+* Client connects to MQTT and sends Query message
+* Designated server sends connection data
+
+Reading initial data
+++++++++++++++++++++
+
+* Client subscribes to MQTT topic
+* Client requests initial data from server
+* Server sends data, advises client that no data exist, or tells the client
+  that MQTT uses retaining (if so, which codec to use)
+
+Updates
++++++++
+
+The most notable pain point of the old design is the speed of updates.
+Thus in V3 all updates will be MQTT messages.
+
+This section does not apply if the MQTT server retains the data.
+
+MoaT update messages are CBOR maps. Keys are small integers for message brevity and
+decode speed.
+
+* 0: tock
+
+  The system-wide update counter. May be missing.
+
+* 1: value
+
+  Contents: Whatever is CBOR-encodeable. A missing value means that the
+  object shall be deleted.
+
+* 2: tick.
+
+  Update tracker. This is a ``((name, seq, counter), …)`` list of tuples.
+
+  * name
+    The server's name
+
+  * seq
+    Client connection. Zero is reserved for server-generated content.
+
+  * counter
+    A per-client update counter, managed by the client. Must start at 1 and
+    increment in steps of 1, in order to detect skipped updates.
+
+
+The server sees the update and sends an ACK message to the client, assuming
+that there was no conflict.
+
+
+Skipped updates
++++++++++++++++
+
+Servers listen to all messages. If there's a gap in a client's sequence
+numbers, the server will ask it to repeat the message.
+
+
+Update conflict resolution
+++++++++++++++++++++++++++
+
+If a client sends a message which the server determines
+
+
+MQTT topics
++++++++++++
+
+All are under a common configured prefix, the default is "moat/main".
+
+svc/act
+-------
+
+The Actor topic for server identification.
+
+The transmitted value contains the server's name, host and port.
+
+
+svc/query
+---------
+
+Connect requests from clients.
+
+svc/server
+----------
+
+Reply queue for messages to ``svc/query``. Contains the server value as
+above.
+
+
+d/*
+---
+
+Update messages.
+
+Topic translation
++++++++++++++++++
+
+Topics are encoded like MoaT paths, except for these differences:
+
+* The path separator is ``/``  instead of ``.``
+* Slashes are escaped as ``:_``.
+* Spaces in paths are never escaped: that would collide with the previous rule
+* Dots are not escaped, obviously.
+* `None` is encoded as "$NULL" when it's a top-level element.
+* The sequence ``:.`` is used to shield both wildcards and strings with a
+  leading ``$`` character. It translates back to an empty string, not a
+  dot, and may be treated as an illegal sequence otherwise.

examples/code/transform.scale.yml

@@ -0,0 +1,21 @@
+code: |
+  await _self.watch(src, fetch=False)
+  async for msg in _info:
+      if isinstance(msg, _cls.ChangeMsg):
+          try:
+              val = msg.value * factor + offset
+          except AttributeError:
+              continue
+          await _client.set(dst, value=val, idem=False)
+info: Apply factor+offset
+is_async: true
+vars:
+- src
+- dst
+- factor
+- offset
+
+# Whenever the value at 'src', changes, this code multiplies it by
+# 'factor', adds 'offset', and writes it to 'dst'.
+# 
+# This takes ~0.025 seconds, end-to-end, on a Raspberry Pi 3.

examples/code/transform.switch.yml

@@ -0,0 +1,82 @@
+code: |
+  await _self.watch(src, fetch=False)
+  inv = high < low
+  if inv:
+      high,low = low,high
+  res = await _client.get(flip)
+  if "value" in res:
+      is_high = res.value != inv
+  else:
+      await _client.set(flip, value=inv)
+      is_high = False
+
+  async for msg in _info:
+      if isinstance(msg, _cls.ChangeMsg):
+          try:
+              val = msg.value
+          except AttributeError:
+              continue
+          if is_high and val < low:
+              is_high = False
+              await _client.set(flip, value=inv)
+          elif not is_high and val > high:
+              is_high = True
+              await _client.set(flip, value=not inv)
+          else:
+              if is_high:
+                  val += high-low
+              await _client.set(dst, value=val)
+info: Switch input between triggers
+is_async: true
+vars:
+- src
+- dst
+- flip
+- high
+- low
+
+# Consider this simple hook-up of a photoresistor:
+#
+#         +5V --- PHOTO --+-- R1 --+-- R2 --- GND
+#                         |        |
+#                       sensor    port
+#
+# The port is a pull-down output: thus, this setup switches resistance
+# between R1 and R1+R2, which allows us to measure the wide range of
+# resistance which a typical photoresistor has, without requiring more
+# fancy circuitry.
+# ‹low› and ‹high› need to be calculated so that
+# 
+# * when the photoresistor's resistance is such that the voltage at the
+#   sensor is at ‹high›, switching on the port will set the voltage to
+#   ‹low›, so you don't get a break in the brightness curve
+# * the rate of change is roughly equal, so you don't get a kink in that
+#   curve.
+# 
+# These conditions are satisfied when R(photo) = √(R1*(R1+R2)).
+# So if R1=100Ω and R2=10kΩ, R(photo) at the switching point would be 1005Ω,
+# thus you can calculate V(high) as 4.55 Volt and V(low) as 0.45 Volt.
+# 
+# The actual choice of resistors is up to you and depends on the
+# photoresistor's behavior in "interesting" lighting conditions. In general
+# you'd want a high-resolution capture of the values at both ends of the
+# range. Thus the above resistor values are sensible if the photoresistor
+# is at 5kΩ when the light gets bright enough to read the newspaper (you
+# want to capture as many nuances of the dark phase as possible), and at
+# 50Ω on a bright summer day just as a cloud has obscured the sun.
+# Or something like that.
+# 
+# 'src' is the path which the sensor value is written to. Presumably it's
+# polled periodically.
+#
+# 'dst' is the entry which receives the adjusted value.
+#
+# 'flip' is the entry which corresponds to the port. It receives a `bool`
+# value (`False` on startup).
+#
+# 'low' and 'high' are the threshold values. If 'low' is greater than
+# 'high', the port is inverted (i.e. it's set to `True` on startup).
+#
+# When a ‹src› update causes the port to be inverted, writing to ‹dst› is
+# skipped: if the new value is close to the threshold, the delay doesn't
+# matter much, and if it's not the value would be too inaccurate.

examples/code/transform.timeslot.yml

@@ -0,0 +1,63 @@
+code: |
+  last_val = None
+  this_val = None
+  timer = None
+  await _client.set(dst, value=0, idem=True)
+  await _self.watch(src, fetch=True)
+
+  async for msg in _info:
+      if isinstance(msg, _cls.ChangeMsg):
+          try:
+              val = msg.value
+          except AttributeError:
+              continue
+          if last_val is None:
+              last_val = val
+              this_val = val
+              continue
+          this_val = val
+          delta = this_val - last_val
+          if delta > 0 and timer is None:
+              # fire an immediate update at first change
+              await _client.set(dst, value=delta*factor, idem=(delta == 0))
+              timer = await _self.timer(seconds)
+          elif delta < 0:
+              # wraparound or whatever
+              last_val = this_val
+              if timer is not None:
+                  await timer.cancel()
+                  timer = None
+      elif isinstance(msg, _cls.TimerMsg):
+          delta = this_val - last_val
+          if delta >= 0:
+              await _client.set(dst, value=delta*factor, idem=(delta == 0))
+          if delta > 0:
+              await timer.run(seconds)
+          else:
+              timer = None
+          last_val = this_val
+info: generate timeslots for counter deltas
+is_async: true
+vars:
+- src
+- dst
+- seconds
+- factor
+
+# This code converts a randomly-updating counter into one that carries a
+# defined meaning.
+#
+# Consider a rain meter. The counter triggers whenever the meter's counter
+# triggers possibly aggregated so that you don't get more than one update
+# every ten seconds even if it's raining buckets.
+#
+# However, in your display you want the rate of rain over, say, the last
+# minute, so you can show some approximation of "how much rain is there
+# right now".
+#
+# This code sends an initial update immediately so that receiving code sees
+# some value > 0 ASAP, then another update every ‹seconds›.
+#
+# Updates are scaled by ‹factor› so you can translate the counter's output
+# to something understandable like "how many mm of water would there be on
+# the ground if this amount of rain continued for an hour".

examples/pathify.py

@@ -0,0 +1,45 @@
+#!/usr/bin/python3
+
+# Batch-convert data. In this case I had some entries which were stored as
+# a list, but using Path made much more sense (esp when you need to
+# view/edit the yaml export).
+
+import anyio
+from moat.kv.client import open_client
+from moat.util import P, yload, Path
+import asyncclick as click
+
+
+def conv(m, s: str) -> bool:
+    try:
+        d = m.value[s]
+    except KeyError:
+        return 0
+    if isinstance(d, Path):
+        return 0
+    if not isinstance(d, Sequence):
+        return 0
+    d = Path.build(d)
+    m.value[s] = d
+    return 1
+
+
+@click.command()
+@click.argument("path", type=P)
+@click.argument("keys", type=str, nargs=-1)
+async def main(path, keys):
+    if not keys:
+        keys = "src dest dst state".split()
+    with open("/etc/moat.kv.cfg") as cff:
+        cfg = yload(cff)
+    async with open_client(**cfg) as client:
+        async for m in client.get_tree(path, nchain=2):
+            n = 0
+            for k in keys:
+                n += conv(m, k)
+            if n:
+                await client.set(ORIG + m.path, value=m.value, chain=m.chain)
+
+
+if __name__ == "__main__":
+    main()

moat/kv/_cfg.yaml

@@ -26,7 +26,6 @@ runner: # for moat.kv.runner.RunnerRoot
   state: !P :.moat.kv.state"
 
   name: "run"
-  # Serf event name, suffixed by subpath
 
   start_delay: 1
   # time to wait between job starts. Not optional.
@@ -56,9 +55,6 @@ server:
   # default
   mqtt:
     uri: "mqtt://localhost:1883"
-  serf:
-    host: "localhost"
-    port: 7373
 
   # event message path/topic prefix
   root: !P moat.kv
@@ -79,10 +75,10 @@ server:
   ping:
     cycle: 10
     gap: 2
-    # asyncserf.Actor config timing for server sync
+    # asyncactor config timing for server sync
   # ping also controls minimum server startup time
   delete:
-    # asyncserf.Actor config timing for deletion
+    # asyncactor config timing for deletion
     cycle: 100
     gap: 10
     version: 1

moat/kv/actor/__init__.py

@@ -0,0 +1,98 @@
+"""
+This module implements a :class:`asyncactor.Actor` which works on top of
+a MoaT-KV client.
+"""
+from __future__ import annotations
+
+from asyncactor import Actor
+from asyncactor.abc import MonitorStream, Transport
+
+__all__ = [
+    "ClientActor",
+    "ActorState",
+    "BrokenState",
+    "DetachedState",
+    "PartialState",
+    "CompleteState",
+]
+
+
+class ClientActor(Actor):
+    def __init__(self, client, *a, topic, **kw):
+        super().__init__(ClientTransport(client, topic), *a, **kw)
+
+
+class ClientTransport(Transport):
+    """
+    This class exports the client's direct messaging interface to the
+    actor.
+    """
+
+    def __init__(self, client, topic):
+        self.client = client
+        self.topic = topic
+
+    def monitor(self):
+        return ClientMonitor(self)
+
+    async def send(self, payload):
+        await self.client.msg_send(self.topic, payload)
+
+
+class ClientMonitor(MonitorStream):
+    _mon1 = None
+    _mon2 = None
+    _it = None
+
+    async def __aenter__(self):
+        self._mon1 = self.transport.client.msg_monitor(self.transport.topic)
+        self._mon2 = await self._mon1.__aenter__()
+        return self
+
+    async def __aexit__(self, *tb):
+        return await self._mon1.__aexit__(*tb)
+
+    def __aiter__(self):
+        self._it = self._mon2.__aiter__()
+        return self
+
+    async def __anext__(self):
+        msg = await self._it.__anext__()
+        return msg.data
+
+
+# The following events are used by Runner etc. to notify running jobs
+# about the current connectivity state.
+#
+class ActorState:
+    """base class for states"""
+
+    def __init__(self, msg=None):
+        self.msg = msg
+
+    def __repr__(self):
+        return "<%s:%r>" % (self.__class__.__name__, self.msg)
+
+
+class BrokenState(ActorState):
+    """I have no idea what's happening, probably nothing good"""
+
+    pass
+
+
+class DetachedState(ActorState):
+    """I am detached, my actor group is not visible"""
+
+    pass
+
+
+class PartialState(ActorState):
+    """Some but not all members of my actor group are visible"""
+
+    pass
+
+
+class CompleteState(ActorState):
+    """All members of my actor group are visible"""
+
+    pass

moat/kv/actor/deletor.py

@@ -0,0 +1,139 @@
+"""
+This module implements additional code for the server-side DeleteActor,
+which is used to clean up the list of deleted nodes.
+"""
+
+from __future__ import annotations
+
+import weakref
+from collections import deque
+
+import anyio
+from asyncactor import Actor, PingEvent, TagEvent
+from asyncactor.backend import get_transport
+
+TAGS = 4
+
+
+class DeleteActor:
+    _enabled = None
+
+    def __init__(self, server):
+        self._server = weakref.ref(server)
+        self.deleted = deque()
+        self.tags = []
+        self.actor = None
+
+        self.max_seen = 0
+        self.n_tags = 0
+        self.n_pings = 0
+        self.n_nodes = 0
+
+    @property
+    def server(self):
+        return self._server()
+
+    async def tock_me(self):
+        """
+        Add the current tock to our buffer.
+
+        This is updated whenever a new leader is selected.
+        """
+        self.tags.append(self.server.tock)
+        self.tags = self.tags[-TAGS:]
+        await self.actor.set_value((self.tags[0], self.tags[-1]))
+
+    def add_deleted(self, nodes: NodeSet):  # noqa: F821
+        """
+        These nodes are deleted. Remember them for some time.
+        """
+        if self.n_nodes == 0:
+            return
+        self.deleted.append((self.server.tock, nodes))
+
+    def purge_to(self, tock):
+        """
+        Sufficient time has passed since this tock was seen, while all
+        Delete actor nodes were active. Finally flush the entries that have
+        been deleted before it.
+        """
+        while self.deleted and self.deleted[0][0] < tock:
+            d = self.deleted.popleft()
+            self.server.purge_deleted(d[1])
+
+    async def enable(self, n):
+        """
+        Enable this actor, as a group of N.
+        """
+        if self.actor is None:
+            self._enabled = True
+        else:
+            await self.actor.enable(n)
+        self.n_tags = 0
+        self.n_pings = 0
+        self.n_nodes = n
+
+    async def disable(self, n: int = 0):
+        """
+        Disable this actor. It will still listen, and require N Delete
+        actor members in order to flush its deletion entries.
+
+        Completely disable deletion flushing by passing n=0.
+        """
+        if self.actor is None:
+            self._enabled = False
+        else:
+            await self.actor.disable()
+        self.n_tags = 0
+        self.n_pings = 0
+        self.n_nodes = n
+
+    async def run(self, evt: anyio.abc.Event = None):
+        """
+        The task that monitors the Delete actor.
+        """
+        try:
+            T = get_transport("moat_kv")
+            async with Actor(
+                T(self.server.backend, *self.server.cfg.server.root, "del"),
+                name=self.server.node.name,
+                cfg=self.server.cfg.server.delete,
+                enabled=False,
+            ) as actor:
+                self.actor = actor
+                if self._enabled is not None:
+                    if self._enabled:
+                        await actor.enable()
+                    else:
+                        await actor.disable()
+                if evt is not None:
+                    evt.set()
+                async for evt in actor:
+                    if isinstance(evt, PingEvent):
+                        val = evt.value
+                        if val is None:
+                            self.n_pings = self.n_tags = 0
+                            continue
+                        if len(evt.msg.history) < self.n_nodes:
+                            self.n_pings = self.n_tags = 0
+                            continue
+                        self.n_pings += 1
+                        if self.n_pings > self.n_nodes:
+                            mx, self.max_seen = (
+                                self.max_seen,
+                                max(self.max_seen, val[1]),
+                            )
+                            if val[0] > mx > 0:
+                                await self.server.resync_deleted(evt.msg.history)
+                                continue
+                            self.purge_to(val[0])
+                            self.max_seen = max(self.max_seen, val[1])
+
+                    elif isinstance(evt, TagEvent):
+                        if actor.history_size == self.n_nodes:
+                            self.n_tags += 1
+                            if self.n_tags > 2:
+                                self.purge_to(self.tags[0])
+                            await self.tock_me()
+        finally:
+            self.actor = None