moat-kv 0.70.14__tar.gz → 0.70.18__tar.gz

moat-kv-0.70.18/.gitmodules

@@ -0,0 +1,21 @@
+[submodule "ext/inv"]
+	path = ext/inv
+	url = git://git.smurf.noris.de/moat-kv-inv.git
+[submodule "ext/ow"]
+	path = ext/ow
+	url = git://git.smurf.noris.de/moat-kv-ow.git
+[submodule "ext/ha"]
+	path = ext/ha
+	url = git://git.smurf.noris.de/moat-kv-ha.git
+[submodule "ext/moat-knx"]
+	path = ext/knx
+	url = git://git.smurf.noris.de/moat-kv-knx.git
+[submodule "ext/gpio"]
+	path = ext/gpio
+	url = git://git.smurf.noris.de/moat-kv-gpio.git
+[submodule "moat-kv-akumuli"]
+	path = ext/akumuli
+	url = git://git.smurf.noris.de/moat-kv-akumuli.git
+[submodule "ext/wago"]
+	path = ext/wago
+	url = git://git.smurf.noris.de/moat-kv-wago.git

{moat-kv-0.70.14 → moat-kv-0.70.18}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: moat-kv
-Version: 0.70.14
+Version: 0.70.18
 Summary: A distributed no-master key-value store
 Author-email: Matthias Urlichs <matthias@urlichs.de>
 Project-URL: homepage, https://m-o-a-t.org

{moat-kv-0.70.14 → moat-kv-0.70.18}/docs/source/TODO.rst

@@ -1,6 +1,10 @@
 Open issues
 ===========
 
+* Exchange a version code on startup
+
+* CBOR
+
 * Ping: ignore messages with decreasing tock (per node)
 
 * chroot operation: add and test proper sub-roots, including auth and
@@ -22,14 +26,9 @@ Open issues
 
 * Rather than mangling split messages, use a MsgPack extension type.
 
-* Rate limiting. Currently the server can flood the client with data and no
-  other call can get a word in edgewise.
-
 * AnyRunner: Do proper load balancing; the leader should be able to tell
   some other node to run a job if it's busy.
 
-* Add an EveryRunner (with per-node status of course).
-
 * Keep an error index on the server?  Something more general?
 
 * Restart code that's been changed (without waiting for restart/retry).
@@ -39,8 +38,6 @@ Open issues
 
 * Implement a shared secret to sign server-to-server messages.
 
-* Re-implement Serf (or something like it) in Python, it's large and adds latency
-
 * Runner: switch to monotonic time (except for target time!)
 
 * Error consolidation: if a conflict doesn't get resolved on its own, do it

{moat-kv-0.70.14 → moat-kv-0.70.18}/moat/kv/__init__.py

@@ -3,10 +3,13 @@
 __path__ = __import__("pkgutil").extend_path(__path__, __name__)
 
 try:
+    import warning
     import pkg_resources  # part of setuptools
 
-    _version = pkg_resources.require("moat.kv")[0].version
+    with warnings.filterwarnings("ignore"):
+        _version = pkg_resources.require("moat.kv")[0].version
     del pkg_resources
+    del warnings
 
     _version_tuple = tuple(int(x) for x in _version.split("."))
 

{moat-kv-0.70.14 → moat-kv-0.70.18}/moat/kv/_main.py

@@ -35,15 +35,23 @@ class NullObj:
         raise self._exc
 
     def __getattr__(self, k):
-        if k[0] == "_" and k not in ("_request","_cfg"):
+        if k[0] == "_" and k not in ("_request", "_cfg"):
             return object.__getattribute__(self, k)
         raise self._exc
 
 
-@load_subgroup(sub_pre="moat.kv.command", sub_post="cli", ext_pre="moat.kv", ext_post="_main.cli")
-@click.option("-h", "--host", default=None, help=f"Host to use. Default: {CFG.kv.conn.host}")
+@load_subgroup(
+    sub_pre="moat.kv.command", sub_post="cli", ext_pre="moat.kv", ext_post="_main.cli"
+)
+@click.option(
+    "-h", "--host", default=None, help=f"Host to use. Default: {CFG.kv.conn.host}"
+)
 @click.option(
-    "-p", "--port", type=int, default=None, help=f"Port to use. Default: {CFG.kv.conn.port}"
+    "-p",
+    "--port",
+    type=int,
+    default=None,
+    help=f"Port to use. Default: {CFG.kv.conn.port}",
 )
 @click.option(
     "-a",

{moat-kv-0.70.14 → moat-kv-0.70.18}/moat/kv/actor/deletor.py

@@ -117,7 +117,10 @@ class DeleteActor:
                             continue
                         self.n_pings += 1
                         if self.n_pings > self.n_nodes:
-                            mx, self.max_seen = (self.max_seen, max(self.max_seen, val[1]))
+                            mx, self.max_seen = (
+                                self.max_seen,
+                                max(self.max_seen, val[1]),
+                            )
                             if val[0] > mx > 0:
                                 await self.server.resync_deleted(evt.msg.history)
                                 continue

{moat-kv-0.70.14 → moat-kv-0.70.18}/moat/kv/auth/__init__.py

@@ -279,7 +279,11 @@ class BaseClientAuthMaker(_AuthLoaded):
         """
         # pragma: no cover
         res = await client._request(
-            "auth_get", typ=cls._auth_method, kind=_kind, ident=ident, nchain=0 if _initial else 2
+            "auth_get",
+            typ=cls._auth_method,
+            kind=_kind,
+            ident=ident,
+            nchain=0 if _initial else 2,
         )
         self = cls(_initial=_initial)
         self._chain = res.chain
@@ -338,7 +342,9 @@ class BaseServerAuth(_AuthLoaded):
 
         try:
             data = data["conv"].data["key"]
-            res, _ = root.follow_acl(Path(None, "conv", data), create=False, nulls_ok=True)
+            res, _ = root.follow_acl(
+                Path(None, "conv", data), create=False, nulls_ok=True
+            )
             return res
         except (KeyError, AttributeError):
             return ConvNull
@@ -348,7 +354,9 @@ class BaseServerAuth(_AuthLoaded):
             data = data["acl"].data["key"]
             if data == "*":
                 return NullACL
-            acl, _ = root.follow_acl(Path(None, "acl", data), create=False, nulls_ok=True)
+            acl, _ = root.follow_acl(
+                Path(None, "acl", data), create=False, nulls_ok=True
+            )
             return ACLFinder(acl)
         except (KeyError, AttributeError):
             return NullACL
@@ -362,17 +370,13 @@ class BaseServerAuth(_AuthLoaded):
         """
         return {}
 
-    async def check_read(
-        self, *path, client: ServerClient, data=None
-    ):  # pylint: disable=unused-argument
+    async def check_read(self, *path, client: ServerClient, data=None):  # pylint: disable=unused-argument
         """Check that this user may read the element at this location.
         This method may modify the data.
         """
         return data
 
-    async def check_write(
-        self, *path, client: ServerClient, data=None
-    ):  # pylint: disable=unused-argument
+    async def check_write(self, *path, client: ServerClient, data=None):  # pylint: disable=unused-argument
         """Check that this user may write the element at this location.
         This method may modify the data.
         """
@@ -415,7 +419,9 @@ class BaseServerAuthMaker(_AuthLoaded):
 
     @classmethod
     async def recv(
-        cls, cmd: StreamCommand, data: attrdict  # pylint: disable=unused-argument
+        cls,
+        cmd: StreamCommand,
+        data: attrdict,  # pylint: disable=unused-argument
     ) -> "BaseServerAuthMaker":
         """Create/update a new user by reading the record from the client"""
         dt = data.get("data", None) or {}

{moat-kv-0.70.14 → moat-kv-0.70.18}/moat/kv/auth/_test.py

@@ -62,7 +62,9 @@ class ServerUserMaker(BaseServerAuthMaker):
         await cmd.send(step="SendWant")
         msg = await cmd.recv()
         assert msg.step == "WantName"
-        await cmd.send(step="SendName", name=self.name, chain=self._chain.serialize(nchain=3))
+        await cmd.send(
+            step="SendName", name=self.name, chain=self._chain.serialize(nchain=3)
+        )
         msg = await cmd.recv()
 
     # Annoying methods to read+save the user name from/to KV
@@ -82,13 +84,17 @@ class ClientUserMaker(BaseClientAuthMaker):
     gen_schema = dict(
         type="object",
         additionalProperties=False,
-        properties=dict(name=dict(type="string", minLength=1, pattern="^[a-zA-Z][a-zA-Z0-9_]*$")),
+        properties=dict(
+            name=dict(type="string", minLength=1, pattern="^[a-zA-Z][a-zA-Z0-9_]*$")
+        ),
         required=["name"],
     )
     mod_schema = dict(
         type="object",
         additionalProperties=False,
-        properties=dict(name=dict(type="string", minLength=1, pattern="^[a-zA-Z][a-zA-Z0-9_]*$")),
+        properties=dict(
+            name=dict(type="string", minLength=1, pattern="^[a-zA-Z][a-zA-Z0-9_]*$")
+        ),
         # required=[],
     )
     name = None
@@ -148,7 +154,9 @@ class ClientUser(BaseClientAuth):
     schema = dict(
         type="object",
         additionalProperties=False,
-        properties=dict(name=dict(type="string", minLength=1, pattern="^[a-zA-Z][a-zA-Z0-9_]*$")),
+        properties=dict(
+            name=dict(type="string", minLength=1, pattern="^[a-zA-Z][a-zA-Z0-9_]*$")
+        ),
         required=["name"],
     )
     _name = None

{moat-kv-0.70.14 → moat-kv-0.70.18}/moat/kv/auth/password.py

@@ -168,9 +168,7 @@ class ClientUserMaker(BaseClientAuthMaker):
             pass
         return self
 
-    async def send(
-        self, client: Client, _kind="user", **msg
-    ):  # pylint: disable=unused-argument,arguments-differ
+    async def send(self, client: Client, _kind="user", **msg):  # pylint: disable=unused-argument,arguments-differ
         """Send a record representing this user to the server."""
         if self._pass is not None:
             msg["password"] = await pack_pwd(client, self._pass, self._length)

{moat-kv-0.70.14 → moat-kv-0.70.18}/moat/kv/client.py

@@ -110,14 +110,16 @@ async def client_scope(_name=None, **cfg):
     """
 
     if _name is None:
-        _name = cfg.get("conn",{}).get("name", "conn")
+        _name = cfg.get("conn", {}).get("name", "conn")
         if _name is None:
             global _cid
             _cid += 1
             _name = f"_{_cid}"
             # uniqueness required for testing.
             # TODO replace with a dependency on the test server.
-    return await scope.service(f"moat.kv.client.{_name}", _scoped_client, _name=_name, **cfg)
+    return await scope.service(
+        f"moat.kv.client.{_name}", _scoped_client, _name=_name, **cfg
+    )
 
 
 class StreamedRequest:
@@ -153,7 +155,9 @@ class StreamedRequest:
         self._path_long = lambda x: x
         if client.qlen > 0:
             self.dw = DelayedWrite(client.qlen)
-            self.qr = DelayedRead(client.qlen, get_seq=self._get_seq, send_ack=self._send_ack)
+            self.qr = DelayedRead(
+                client.qlen, get_seq=self._get_seq, send_ack=self._send_ack
+            )
         else:
             self.qr = create_queue(client.config.server.buffer)
 
@@ -480,7 +484,9 @@ class Client:
             res = await self._request(
                 "diffie_hellman", pubkey=num2byte(k.public_key), length=length
             )  # length=k.key_length
-            await anyio.to_thread.run_sync(k.generate_shared_secret, byte2num(res.pubkey))
+            await anyio.to_thread.run_sync(
+                k.generate_shared_secret, byte2num(res.pubkey)
+            )
             self._dh_key = num2byte(k.shared_secret)[0:32]
         return self._dh_key
 
@@ -529,6 +535,7 @@ class Client:
 
             except BaseException as exc:
                 logger.warning("Reader died: %r", exc, exc_info=exc)
+                raise
             finally:
                 with anyio.fail_after(2, shield=True):
                     hdl, self._handlers = self._handlers, None
@@ -540,9 +547,7 @@ class Client:
                         except ClosedResourceError:
                             pass
 
-    async def _request(
-        self, action, iter=None, seq=None, _async=False, **params
-    ):  # pylint: disable=redefined-builtin  # iter
+    async def _request(self, action, iter=None, seq=None, _async=False, **params):  # pylint: disable=redefined-builtin  # iter
         """Send a request. Wait for a reply.
 
         Args:
@@ -676,12 +681,16 @@ class Client:
         if not sa or not sa[0]:
             # no auth required
             if auth:
-                logger.info("Tried to use auth=%s, but not required.", auth._auth_method)
+                logger.info(
+                    "Tried to use auth=%s, but not required.", auth._auth_method
+                )
             return
         if not auth:
             raise ClientAuthRequiredError("You need to log in using:", sa[0])
         if auth._auth_method != sa[0]:
-            raise ClientAuthMethodError(f"You cannot use {auth._auth_method!r} auth", sa)
+            raise ClientAuthMethodError(
+                f"You cannot use {auth._auth_method!r} auth", sa
+            )
         if getattr(auth, "_DEBUG", False):
             auth._length = 16
         await auth.auth(self)
@@ -727,7 +736,9 @@ class Client:
                         self._server_init = msg = await hello.get()
                         self.logger.debug("Hello %s", msg)
                         self.server_name = msg.node
-                        self.client_name = cfg["name"] or socket.gethostname() or self.server_name
+                        self.client_name = (
+                            cfg["name"] or socket.gethostname() or self.server_name
+                        )
                         if "qlen" in msg:
                             self.qlen = min(msg.qlen, 99)  # self.config.server.buffer
                             await self._send(seq=0, qlen=self.qlen)
@@ -735,7 +746,9 @@ class Client:
 
                     from .config import ConfigRoot
 
-                    self._config = await ConfigRoot.as_handler(self, require_client=False)
+                    self._config = await ConfigRoot.as_handler(
+                        self, require_client=False
+                    )
 
                 except TimeoutError:
                     raise
@@ -779,7 +792,16 @@ class Client:
             raise RuntimeError("You need a path, not a string")
         return self._request(action="get_value", path=path, iter=False, nchain=nchain)
 
-    def set(self, path, value=NotGiven, *, chain=NotGiven, prev=NotGiven, nchain=0, idem=None):
+    def set(
+        self,
+        path,
+        value=NotGiven,
+        *,
+        chain=NotGiven,
+        prev=NotGiven,
+        nchain=0,
+        idem=None,
+    ):
         """
         Set or update a value.
 
@@ -856,8 +878,13 @@ class Client:
             raise RuntimeError("You need a path, not a string")
         if empty is None:
             empty = not with_data
-        res = await self._request(action="enum", path=path, with_data=with_data, empty=empty, **kw)
-        return res.result
+        res = await self._request(
+            action="enum", path=path, with_data=with_data, empty=empty, **kw
+        )
+        try:
+            return res.result
+        except AttributeError:
+            raise res.q.value.error from None  # XXX fix this
 
     async def get_tree(self, path, *, long_path=True, **kw):
         """
@@ -936,7 +963,9 @@ class Client:
         """
         if isinstance(path, str):
             raise RuntimeError("You need a path, not a string")
-        return self._stream(action="watch", path=path, iter=True, long_path=long_path, **kw)
+        return self._stream(
+            action="watch", path=path, iter=True, long_path=long_path, **kw
+        )
 
     def mirror(self, path, *, root_type=None, **kw):
         """An async context manager that affords an update-able mirror

{moat-kv-0.70.14 → moat-kv-0.70.18}/moat/kv/code.py

@@ -159,7 +159,9 @@ class CodeRoot(ClientRoot):
         make_proc(code, variables, path, use_async=is_async)
 
         r = await self.client.set(
-            self._path + path, value=dict(code=code, is_async=is_async, vars=variables), nchain=2
+            self._path + path,
+            value=dict(code=code, is_async=is_async, vars=variables),
+            nchain=2,
         )
         await self.wait_chain(r.chain)
 

{moat-kv-0.70.14 → moat-kv-0.70.18}/moat/kv/command/acl.py

@@ -94,10 +94,15 @@ async def set_(obj, acl, name, path):
     acl = set(acl)
 
     if acl - ACL:
-        raise click.UsageError(f"You're trying to set an unknown ACL flag: {acl - ACL !r}")
+        raise click.UsageError(
+            f"You're trying to set an unknown ACL flag: {acl - ACL !r}"
+        )
 
     res = await obj.client._request(
-        action="get_internal", path=("acl", name) + path, iter=False, nchain=3 if obj.meta else 1
+        action="get_internal",
+        path=("acl", name) + path,
+        iter=False,
+        nchain=3 if obj.meta else 1,
     )
     ov = set(res.get("value", ""))
     if ov - ACL:
@@ -105,7 +110,10 @@ async def set_(obj, acl, name, path):
 
     if mode == "-" and not acl:
         res = await obj.client._request(
-            action="delete_internal", path=("acl", name) + path, iter=False, chain=res.chain
+            action="delete_internal",
+            path=("acl", name) + path,
+            iter=False,
+            chain=res.chain,
         )
         v = "-"
 
@@ -125,7 +133,12 @@ async def set_(obj, acl, name, path):
         )
 
     if obj.meta:
-        res = {"old": "".join(ov), "new": "".join(v), "chain": res.chain, "tock": res.tock}
+        res = {
+            "old": "".join(ov),
+            "new": "".join(v),
+            "chain": res.chain,
+            "tock": res.tock,
+        }
         yprint(res, stream=obj.stdout)
     else:
         res = {"old": "".join(ov), "new": "".join(v)}

{moat-kv-0.70.14 → moat-kv-0.70.18}/moat/kv/command/auth.py

@@ -63,7 +63,12 @@ async def enum_typ(obj, kind="user", ident=None, nchain=0):
     async for auth in enum_auth(obj):
         if ident is not None:
             res = await obj.client._request(
-                action="auth_list", typ=auth, kind=kind, ident=ident, iter=False, nchain=nchain
+                action="auth_list",
+                typ=auth,
+                kind=kind,
+                ident=ident,
+                iter=False,
+                nchain=nchain,
             )
             yield res
         else:
@@ -106,7 +111,10 @@ async def user():
 
 @user.command("list")
 @click.option(
-    "-v", "--verbose", is_flag=True, help="Print complete results. Default: just the names"
+    "-v",
+    "--verbose",
+    is_flag=True,
+    help="Print complete results. Default: just the names",
 )
 @click.pass_obj  # pylint: disable=function-redefined
 async def list_user(obj, verbose):
@@ -157,7 +165,10 @@ async def param(obj, new, ident, type, key, args):  # pylint: disable=redefined-
         u._length = 16
     # ou = await u.recv(obj.client, ident, _initial=False)  # unused
     res = await obj.client._request(
-        action="get_internal", path=("auth", auth, "user", ident, type), iter=False, nchain=3
+        action="get_internal",
+        path=("auth", auth, "user", ident, type),
+        iter=False,
+        nchain=3,
     )
 
     kw = res.get("value", NotGiven)

{moat-kv-0.70.14 → moat-kv-0.70.18}/moat/kv/command/code.py

@@ -3,7 +3,16 @@
 import sys
 
 import asyncclick as click
-from moat.util import NotGiven, P, Path, PathLongener, attr_args, process_args, yload, yprint
+from moat.util import (
+    NotGiven,
+    P,
+    Path,
+    PathLongener,
+    attr_args,
+    process_args,
+    yload,
+    yprint,
+)
 
 
 @click.group(invoke_without_command=True)  # pylint: disable=undefined-variable
@@ -23,14 +32,18 @@ async def cli(ctx, path):
 
 
 @cli.command()
-@click.option("-s", "--script", type=click.File(mode="w", lazy=True), help="Save the code here")
+@click.option(
+    "-s", "--script", type=click.File(mode="w", lazy=True), help="Save the code here"
+)
 @click.pass_obj
 async def get(obj, script):
     """Read a code entry"""
     if not len(obj.codepath):
         raise click.UsageError("You need a non-empty path.")
 
-    res = await obj.client._request(action="get_value", path=obj.path, iter=False, nchain=obj.meta)
+    res = await obj.client._request(
+        action="get_value", path=obj.path, iter=False, nchain=obj.meta
+    )
     if "value" not in res:
         if obj.debug:
             print("No entry here.", file=sys.stderr)
@@ -53,10 +66,14 @@ async def get(obj, script):
     help="The code is async / sync (default: async)",
     default=True,
 )
-@click.option("-t", "--thread", is_flag=True, help="The code should run in a worker thread")
+@click.option(
+    "-t", "--thread", is_flag=True, help="The code should run in a worker thread"
+)
 @click.option("-s", "--script", type=click.File(mode="r"), help="File with the code")
 @click.option("-i", "--info", type=str, help="one-liner info about the code")
-@click.option("-d", "--data", type=click.File(mode="r"), help="load the metadata (YAML)")
+@click.option(
+    "-d", "--data", type=click.File(mode="r"), help="load the metadata (YAML)"
+)
 @attr_args
 @click.pass_obj
 async def set_(obj, thread, script, data, vars_, eval_, path_, async_, info):
@@ -124,7 +141,9 @@ async def mod():
 
 
 @mod.command("get")
-@click.option("-s", "--script", type=click.File(mode="w", lazy=True), help="Save the code here")
+@click.option(
+    "-s", "--script", type=click.File(mode="w", lazy=True), help="Save the code here"
+)
 @click.argument("path", nargs=1)
 @click.pass_obj  # pylint: disable=function-redefined
 async def get_mod(obj, path, script):
@@ -153,8 +172,12 @@ async def get_mod(obj, path, script):
 
 
 @mod.command("set")
-@click.option("-s", "--script", type=click.File(mode="r"), help="File with the module's code")
-@click.option("-d", "--data", type=click.File(mode="r"), help="load the metadata (YAML)")
+@click.option(
+    "-s", "--script", type=click.File(mode="r"), help="File with the module's code"
+)
+@click.option(
+    "-d", "--data", type=click.File(mode="r"), help="load the metadata (YAML)"
+)
 @click.argument("path", nargs=1)  # pylint: disable=function-redefined
 @click.pass_obj
 async def set_mod(obj, path, script, data):
@@ -201,10 +224,18 @@ async def set_mod(obj, path, script, data):
     "for values. Default: return as list",
 )
 @click.option(
-    "-m", "--maxdepth", type=int, default=None, help="Limit recursion depth. Default: whole tree"
+    "-m",
+    "--maxdepth",
+    type=int,
+    default=None,
+    help="Limit recursion depth. Default: whole tree",
 )
 @click.option(
-    "-M", "--mindepth", type=int, default=None, help="Starting depth. Default: whole tree"
+    "-M",
+    "--mindepth",
+    type=int,
+    default=None,
+    help="Starting depth. Default: whole tree",
 )
 @click.option("-f", "--full", is_flag=True, help="print complete entries.")
 @click.option("-s", "--short", is_flag=True, help="print shortened entries.")