moat-kv 0.70.22__py3-none-any.whl → 0.70.24__py3-none-any.whl

examples/pathify.py

@@ -0,0 +1,45 @@
+#!/usr/bin/python3
+
+# Batch-convert data. In this case I had some entries which were stored as
+# a list, but using Path made much more sense (esp when you need to
+# view/edit the yaml export).
+
+import anyio
+from moat.kv.client import open_client
+from moat.util import P, yload, Path
+import asyncclick as click
+
+
+def conv(m, s: str) -> bool:
+    try:
+        d = m.value[s]
+    except KeyError:
+        return 0
+    if isinstance(d, Path):
+        return 0
+    if not isinstance(d, Sequence):
+        return 0
+    d = Path.build(d)
+    m.value[s] = d
+    return 1
+
+
+@click.command()
+@click.argument("path", type=P)
+@click.argument("keys", type=str, nargs=-1)
+async def main(path, keys):
+    if not keys:
+        keys = "src dest dst state".split()
+    with open("/etc/moat.kv.cfg") as cff:
+        cfg = yload(cff)
+    async with open_client(**cfg) as client:
+        async for m in client.get_tree(path, nchain=2):
+            n = 0
+            for k in keys:
+                n += conv(m, k)
+            if n:
+                await client.set(ORIG + m.path, value=m.value, chain=m.chain)
+
+
+if __name__ == "__main__":
+    main()

moat/kv/__init__.py

@@ -1,4 +1,5 @@
 # pylint: disable=W0703,C0103
+from __future__ import annotations
 
 __path__ = __import__("pkgutil").extend_path(__path__, __name__)
 

moat/kv/_cfg.yaml

@@ -0,0 +1,97 @@
+conn:
+  # client: controls how to talk to the MoaT-KV server
+  host: localhost
+  port: 27586
+  ssl: false
+  # ssl:
+  #   cert: '/path/to/cert.pem',key='/path/to/cert.key'
+  init_timeout: 5
+  # time to wait for connection plus greeting
+  auth: null
+  # no auth used by default
+  name: null
+  # defaults to a seqnum
+config:
+  prefix: !P :.moat.kv.config
+errors:
+  prefix: !P :.moat.kv.error
+codes:
+  prefix: !P :.moat.kv.code.proc
+modules:
+  prefix: !P :.moat.kv.code.module
+runner: # for moat.kv.runner.RunnerRoot
+  # storage for runnable commands
+  prefix: !P :.moat.kv.run"
+  # storage for runner states
+  state: !P :.moat.kv.state"
+
+  name: "run"
+  # Serf event name, suffixed by subpath
+
+  start_delay: 1
+  # time to wait between job starts. Not optional.
+
+  ping: -15
+  # set an I-am-running message every those-many seconds
+  # positive: set in moat.kv, negative: broadcast to :moat.kv.run tag
+
+  actor:
+    # Actor config, required for Runner
+    cycle: 20
+    nodes: -1
+    splits: 5
+    n_hosts: 3
+    version: 1
+  sub:
+    # tags for various runner modes
+    group: "any"
+    single: "at"
+    all: "all"
+server:
+  # server-side configuration
+  buffer: 10
+  # per-stream buffer
+
+  backend: "mqtt"
+  # default
+  mqtt:
+    uri: "mqtt://localhost:1883"
+  serf:
+    host: "localhost"
+    port: 7373
+
+  # event message path/topic prefix
+  root: !P moat.kv
+
+  paranoia: False
+  # typecheck server-to-server updates?
+  #
+  # which addresses/ports to accept MoaT-KV connections on
+  bind: [{}]
+  bind_default:
+    # default values for all elements of "bind"
+    host: "localhost"
+    port: PORT
+    ssl: False
+  change:
+    length: 5
+    # chain length: use max nr of network sections +1
+  ping:
+    cycle: 10
+    gap: 2
+    # asyncserf.Actor config timing for server sync
+  # ping also controls minimum server startup time
+  delete:
+    # asyncserf.Actor config timing for deletion
+    cycle: 100
+    gap: 10
+    version: 1
+paranoia: false
+# typecheck server>server updates?
+
+# how does a new server reach existing nodes, to download state?
+domain: null
+# domain in which to look up node names, if not in hostmap
+hostmap:   # map MoaT-KV server names to connect destinations
+  test1: ["localhost", 27586]
+  test2: ["does-not-exist.invalid", 27586]

moat/kv/_main.py

@@ -4,11 +4,12 @@ Basic DistKV support
 
 """
 
+from __future__ import annotations
+
 import logging
-from pathlib import Path
 
 import asyncclick as click
-from moat.util import attrdict, combine_dict, load_subgroup, yload
+from moat.util import attrdict, combine_dict, load_subgroup, CFG, ensure_cfg
 
 from moat.kv.auth import gen_auth
 from moat.kv.client import client_scope
@@ -16,7 +17,7 @@ from moat.kv.client import client_scope
 logger = logging.getLogger(__name__)
 
 
-CFG = yload(Path(__file__).parent / "_config.yaml", attr=True)
+ensure_cfg("moat.kv")
 
 
 class NullObj:
@@ -40,12 +41,8 @@ class NullObj:
         raise self._exc
 
 
-@load_subgroup(
-    sub_pre="moat.kv.command", sub_post="cli", ext_pre="moat.kv", ext_post="_main.cli"
-)
-@click.option(
-    "-h", "--host", default=None, help=f"Host to use. Default: {CFG.kv.conn.host}"
-)
+@load_subgroup(sub_pre="moat.kv.command", sub_post="cli", ext_pre="moat.kv", ext_post="_main.cli")
+@click.option("-h", "--host", default=None, help=f"Host to use. Default: {CFG.kv.conn.host}")
 @click.option(
     "-p",
     "--port",

moat/kv/actor/__init__.py

@@ -2,8 +2,9 @@
 This module implements a :class:`asyncactor.Actor` which works on top of
 a MoaT-KV client.
 """
+from __future__ import annotations
 
-from asyncactor import Actor  # noqa
+from asyncactor import Actor
 from asyncactor.abc import MonitorStream, Transport
 
 __all__ = [

moat/kv/actor/deletor.py

@@ -3,6 +3,8 @@ This module implements additional code for the server-side DeleteActor,
 which is used to clean up the list of deleted nodes.
 """
 
+from __future__ import annotations
+
 import weakref
 from collections import deque
 
@@ -41,7 +43,7 @@ class DeleteActor:
         self.tags = self.tags[-TAGS:]
         await self.actor.set_value((self.tags[0], self.tags[-1]))
 
-    def add_deleted(self, nodes: "NodeSet"):  # noqa: F821
+    def add_deleted(self, nodes: NodeSet):  # noqa: F821
         """
         These nodes are deleted. Remember them for some time.
         """

moat/kv/auth/__init__.py

@@ -59,6 +59,8 @@ The server process is:
 
 """
 
+from __future__ import annotations
+
 import io
 from importlib import import_module
 
@@ -87,7 +89,7 @@ add_schema = {
             "additionalProperties": False,
             "properties": {"key": {type: "string", "minLength": 1}},
         },
-    }
+    },
 }
 
 
@@ -111,7 +113,7 @@ def gen_auth(s: str):
 
     m, *p = s.split()
     if len(p) == 0 and m[0] == "=":
-        with io.open(m[1:], "r", encoding="utf-8") as f:
+        with open(m[1:], encoding="utf-8") as f:
             kw = yload(f)
             m = kw.pop("type")
     else:
@@ -147,7 +149,7 @@ async def null_server_login(stream):
     return stream
 
 
-async def null_client_login(stream, user: "BaseClientAuth"):  # pylint: disable=unused-argument
+async def null_client_login(stream, user: BaseClientAuth):  # pylint: disable=unused-argument
     return stream
 
 
@@ -342,9 +344,7 @@ class BaseServerAuth(_AuthLoaded):
 
         try:
             data = data["conv"].data["key"]
-            res, _ = root.follow_acl(
-                Path(None, "conv", data), create=False, nulls_ok=True
-            )
+            res, _ = root.follow_acl(Path(None, "conv", data), create=False, nulls_ok=True)
             return res
         except (KeyError, AttributeError):
             return ConvNull
@@ -354,9 +354,7 @@ class BaseServerAuth(_AuthLoaded):
             data = data["acl"].data["key"]
             if data == "*":
                 return NullACL
-            acl, _ = root.follow_acl(
-                Path(None, "acl", data), create=False, nulls_ok=True
-            )
+            acl, _ = root.follow_acl(Path(None, "acl", data), create=False, nulls_ok=True)
             return ACLFinder(acl)
         except (KeyError, AttributeError):
             return NullACL
@@ -422,7 +420,7 @@ class BaseServerAuthMaker(_AuthLoaded):
         cls,
         cmd: StreamCommand,
         data: attrdict,  # pylint: disable=unused-argument
-    ) -> "BaseServerAuthMaker":
+    ) -> BaseServerAuthMaker:
         """Create/update a new user by reading the record from the client"""
         dt = data.get("data", None) or {}
         jsonschema.validate(instance=dt, schema=cls.schema)

moat/kv/auth/_test.py

@@ -5,6 +5,8 @@ Test auth method.
 Does not limit anything, allows everything.
 """
 
+from __future__ import annotations
+
 import logging
 
 log = logging.getLogger(__name__)
@@ -62,9 +64,7 @@ class ServerUserMaker(BaseServerAuthMaker):
         await cmd.send(step="SendWant")
         msg = await cmd.recv()
         assert msg.step == "WantName"
-        await cmd.send(
-            step="SendName", name=self.name, chain=self._chain.serialize(nchain=3)
-        )
+        await cmd.send(step="SendName", name=self.name, chain=self._chain.serialize(nchain=3))
         msg = await cmd.recv()
 
     # Annoying methods to read+save the user name from/to KV
@@ -84,17 +84,13 @@ class ClientUserMaker(BaseClientAuthMaker):
     gen_schema = dict(
         type="object",
         additionalProperties=False,
-        properties=dict(
-            name=dict(type="string", minLength=1, pattern="^[a-zA-Z][a-zA-Z0-9_]*$")
-        ),
+        properties=dict(name=dict(type="string", minLength=1, pattern="^[a-zA-Z][a-zA-Z0-9_]*$")),
         required=["name"],
     )
     mod_schema = dict(
         type="object",
         additionalProperties=False,
-        properties=dict(
-            name=dict(type="string", minLength=1, pattern="^[a-zA-Z][a-zA-Z0-9_]*$")
-        ),
+        properties=dict(name=dict(type="string", minLength=1, pattern="^[a-zA-Z][a-zA-Z0-9_]*$")),
         # required=[],
     )
     name = None
@@ -154,9 +150,7 @@ class ClientUser(BaseClientAuth):
     schema = dict(
         type="object",
         additionalProperties=False,
-        properties=dict(
-            name=dict(type="string", minLength=1, pattern="^[a-zA-Z][a-zA-Z0-9_]*$")
-        ),
+        properties=dict(name=dict(type="string", minLength=1, pattern="^[a-zA-Z][a-zA-Z0-9_]*$")),
         required=["name"],
     )
     _name = None

moat/kv/auth/password.py

@@ -5,6 +5,8 @@ Password-based auth method.
 Does not limit anything, allows everything.
 """
 
+from __future__ import annotations
+
 import nacl.secret
 
 from ..client import Client, NoData

moat/kv/auth/root.py

@@ -5,6 +5,8 @@ Null auth method.
 Does not limit anything, allows everything.
 """
 
+from __future__ import annotations
+
 from . import (
     BaseClientAuth,
     BaseClientAuthMaker,

moat/kv/backend/__init__.py

@@ -1,3 +1,4 @@
+from __future__ import annotations
 from abc import ABCMeta, abstractmethod
 from contextlib import asynccontextmanager
 

moat/kv/backend/mqtt.py

@@ -1,9 +1,11 @@
+from __future__ import annotations
 import logging
 from contextlib import asynccontextmanager
 
 import anyio
 from moat.mqtt.client import MQTTClient
 from moat.mqtt.codecs import NoopCodec
+from moat.util import NotGiven
 
 from . import Backend
 
@@ -24,9 +26,7 @@ class MqttBackend(Backend):
 
     @asynccontextmanager
     async def connect(self, *a, **kw):
-        codec = kw.pop("codec", None)
-        if codec is None:
-            codec = NoopCodec()
+        codec = kw.pop("codec", NotGiven)
         C = MQTTClient(self._tg, codec=codec)
         try:
             await C.connect(*a, **kw)

moat/kv/backend/serf.py

@@ -1,3 +1,4 @@
+from __future__ import annotations
 from contextlib import asynccontextmanager
 
 import anyio

moat/kv/client.py

@@ -4,17 +4,18 @@ Client code.
 Main entry point: :func:`open_client`.
 """
 
+from __future__ import annotations
+
 import logging
 import os
 import socket
 from contextlib import AsyncExitStack, asynccontextmanager
 from inspect import iscoroutine
-from pathlib import Path
-from typing import Tuple
 
 import anyio
 from asyncscope import Scope, main_scope, scope
 from moat.util import (  # pylint: disable=no-name-in-module
+    CFG,
     DelayedRead,
     DelayedWrite,
     NotGiven,
@@ -25,9 +26,10 @@ from moat.util import (  # pylint: disable=no-name-in-module
     byte2num,
     combine_dict,
     create_queue,
+    ensure_cfg,
     gen_ssl,
+    gen_ident,al_lower,
     num2byte,
-    yload,
 )
 
 from .codec import packer, stream_unpacker
@@ -45,17 +47,6 @@ logger = logging.getLogger(__name__)
 
 ClosedResourceError = anyio.ClosedResourceError
 
-rs = os.environ.get("PYTHONHASHSEED", None)
-if rs is None:
-    import random
-else:  # pragma: no cover
-    try:
-        import trio._core._run as tcr
-    except ImportError:
-        import random
-    else:
-        random = tcr._r
-
 __all__ = ["NoData", "ManyData", "open_client", "client_scope", "StreamedRequest"]
 
 
@@ -117,9 +108,7 @@ async def client_scope(_name=None, **cfg):
             _name = f"_{_cid}"
             # uniqueness required for testing.
             # TODO replace with a dependency on the test server.
-    return await scope.service(
-        f"moat.kv.client.{_name}", _scoped_client, _name=_name, **cfg
-    )
+    return await scope.service(f"moat.kv.client.{_name}", _scoped_client, _name=_name, **cfg)
 
 
 class StreamedRequest:
@@ -155,9 +144,7 @@ class StreamedRequest:
         self._path_long = lambda x: x
         if client.qlen > 0:
             self.dw = DelayedWrite(client.qlen)
-            self.qr = DelayedRead(
-                client.qlen, get_seq=self._get_seq, send_ack=self._send_ack
-            )
+            self.qr = DelayedRead(client.qlen, get_seq=self._get_seq, send_ack=self._send_ack)
         else:
             self.qr = create_queue(client.config.server.buffer)
 
@@ -227,7 +214,7 @@ class StreamedRequest:
 
     async def get(self):
         """Receive a single reply"""
-        pass  # receive reply
+        # receive reply
         if self._reply_stream:
             raise RuntimeError("Unexpected multi stream msg")
         msg = await self.recv()
@@ -411,7 +398,7 @@ class Client:
     qlen: int = 0
 
     def __init__(self, cfg: dict):
-        CFG = yload(Path(__file__).parent / "_config.yaml")
+        ensure_cfg("moat.kv")
         self._cfg = combine_dict(cfg, CFG["kv"], cls=attrdict)
         self.config = ClientConfig(self)
 
@@ -419,7 +406,7 @@ class Client:
         self._handlers = {}
         self._send_lock = anyio.Lock()
         self._helpers = {}
-        self._name = "".join(random.choices("abcdefghjkmnopqrstuvwxyz23456789", k=9))
+        self._name = gen_ident(9, alphabet=al_lower)
         self.logger = logging.getLogger(f"moat.kv.client.{self._name}")
 
     @property
@@ -482,11 +469,11 @@ class Client:
 
             k = await anyio.to_thread.run_sync(gen_key)
             res = await self._request(
-                "diffie_hellman", pubkey=num2byte(k.public_key), length=length
+                "diffie_hellman",
+                pubkey=num2byte(k.public_key),
+                length=length,
             )  # length=k.key_length
-            await anyio.to_thread.run_sync(
-                k.generate_shared_secret, byte2num(res.pubkey)
-            )
+            await anyio.to_thread.run_sync(k.generate_shared_secret, byte2num(res.pubkey))
             self._dh_key = num2byte(k.shared_secret)[0:32]
         return self._dh_key
 
@@ -681,16 +668,12 @@ class Client:
         if not sa or not sa[0]:
             # no auth required
             if auth:
-                logger.info(
-                    "Tried to use auth=%s, but not required.", auth._auth_method
-                )
+                logger.info("Tried to use auth=%s, but not required.", auth._auth_method)
             return
         if not auth:
             raise ClientAuthRequiredError("You need to log in using:", sa[0])
         if auth._auth_method != sa[0]:
-            raise ClientAuthMethodError(
-                f"You cannot use {auth._auth_method!r} auth", sa
-            )
+            raise ClientAuthMethodError(f"You cannot use {auth._auth_method!r} auth", sa)
         if getattr(auth, "_DEBUG", False):
             auth._length = 16
         await auth.auth(self)
@@ -736,9 +719,7 @@ class Client:
                         self._server_init = msg = await hello.get()
                         self.logger.debug("Hello %s", msg)
                         self.server_name = msg.node
-                        self.client_name = (
-                            cfg["name"] or socket.gethostname() or self.server_name
-                        )
+                        self.client_name = cfg["name"] or socket.gethostname() or self.server_name
                         if "qlen" in msg:
                             self.qlen = min(msg.qlen, 99)  # self.config.server.buffer
                             await self._send(seq=0, qlen=self.qlen)
@@ -746,13 +727,11 @@ class Client:
 
                     from .config import ConfigRoot
 
-                    self._config = await ConfigRoot.as_handler(
-                        self, require_client=False
-                    )
+                    self._config = await ConfigRoot.as_handler(self, require_client=False)
 
                 except TimeoutError:
                     raise
-                except socket.error as e:
+                except OSError as e:
                     raise ServerConnectionError(host, port) from e
                 else:
                     yield self
@@ -830,7 +809,12 @@ class Client:
             kw["idem"] = idem
 
         return self._request(
-            action="set_value", path=path, value=value, iter=False, nchain=nchain, **kw
+            action="set_value",
+            path=path,
+            value=value,
+            iter=False,
+            nchain=nchain,
+            **kw,
         )
 
     def delete(self, path, *, chain=NotGiven, prev=NotGiven, nchain=0, recursive=False):
@@ -878,9 +862,7 @@ class Client:
             raise RuntimeError("You need a path, not a string")
         if empty is None:
             empty = not with_data
-        res = await self._request(
-            action="enum", path=path, with_data=with_data, empty=empty, **kw
-        )
+        res = await self._request(action="enum", path=path, with_data=with_data, empty=empty, **kw)
         try:
             return res.result
         except AttributeError:
@@ -910,7 +892,11 @@ class Client:
         if long_path:
             lp = PathLongener()
         async for r in await self._request(
-            action="get_tree", path=path, iter=True, long_path=True, **kw
+            action="get_tree",
+            path=path,
+            iter=True,
+            long_path=True,
+            **kw,
         ):
             if long_path:
                 lp(r)
@@ -963,9 +949,7 @@ class Client:
         """
         if isinstance(path, str):
             raise RuntimeError("You need a path, not a string")
-        return self._stream(
-            action="watch", path=path, iter=True, long_path=long_path, **kw
-        )
+        return self._stream(action="watch", path=path, iter=True, long_path=long_path, **kw)
 
     def mirror(self, path, *, root_type=None, **kw):
         """An async context manager that affords an update-able mirror
@@ -997,7 +981,7 @@ class Client:
         root = root_type(self, path, **kw)
         return root.run()
 
-    def msg_monitor(self, topic: Tuple[str], raw: bool = False):
+    def msg_monitor(self, topic: tuple[str], raw: bool = False):
         """
         Return an async iterator of tunneled messages. This receives
         all messages sent using :meth:`msg_send` with the same topic.
@@ -1023,7 +1007,7 @@ class Client:
         """
         return self._stream(action="msg_monitor", topic=topic, raw=raw)
 
-    def msg_send(self, topic: Tuple[str], data=None, raw: bytes = None):
+    def msg_send(self, topic: tuple[str], data=None, raw: bytes = None):
         """
         Tunnel a user-tagged message. This sends the message
         to all active callers of :meth:`msg_monitor` which use the same topic.

moat/kv/code.py

@@ -5,6 +5,7 @@ so that it can be called easily.
 "Code" consists of either Python modules or single procedures.
 
 """
+from __future__ import annotations
 
 import logging
 import sys
@@ -36,7 +37,7 @@ class ModuleRoot(ClientRoot):
 
     CFG = "modules"
 
-    err: "ErrorRoot" = None  # noqa: F821
+    err: ErrorRoot = None  # noqa: F821
 
     @classmethod
     def child_type(cls, name):
@@ -95,7 +96,10 @@ class ModuleEntry(ClientEntry):
             self._module = None
             logger.warning("Could not compile @%r", self.subpath)
             await self.root.err.record_error(
-                "compile", self.subpath, exc=exc, message="compiler error"
+                "compile",
+                self.subpath,
+                exc=exc,
+                message="compiler error",
             )
         else:
             await self.root.err.record_working("compile", self.subpath)
@@ -207,7 +211,10 @@ class CodeEntry(ClientEntry):
         except Exception as exc:
             logger.warning("Could not compile @%s", self.subpath)
             await self.root.err.record_error(
-                "compile", self.subpath, exc=exc, message="compiler error"
+                "compile",
+                self.subpath,
+                exc=exc,
+                message="compiler error",
             )
             self._code = None
         else:

moat/kv/codec.py

@@ -4,6 +4,7 @@ plus an unpacker factory for streams.
 """
 
 # compatibility
+from __future__ import annotations
 
 from moat.util import *  # noqa: F403