moat-kv 0.70.20__py3-none-any.whl → 0.70.23__py3-none-any.whl

moat/kv/__init__.py

@@ -1,4 +1,5 @@
 # pylint: disable=W0703,C0103
+from __future__ import annotations
 
 __path__ = __import__("pkgutil").extend_path(__path__, __name__)
 

moat/kv/_cfg.yaml

@@ -0,0 +1,97 @@
+conn:
+  # client: controls how to talk to the MoaT-KV server
+  host: localhost
+  port: 27586
+  ssl: false
+  # ssl:
+  #   cert: '/path/to/cert.pem',key='/path/to/cert.key'
+  init_timeout: 5
+  # time to wait for connection plus greeting
+  auth: null
+  # no auth used by default
+  name: null
+  # defaults to a seqnum
+config:
+  prefix: !P :.moat.kv.config
+errors:
+  prefix: !P :.moat.kv.error
+codes:
+  prefix: !P :.moat.kv.code.proc
+modules:
+  prefix: !P :.moat.kv.code.module
+runner: # for moat.kv.runner.RunnerRoot
+  # storage for runnable commands
+  prefix: !P :.moat.kv.run"
+  # storage for runner states
+  state: !P :.moat.kv.state"
+
+  name: "run"
+  # Serf event name, suffixed by subpath
+
+  start_delay: 1
+  # time to wait between job starts. Not optional.
+
+  ping: -15
+  # set an I-am-running message every those-many seconds
+  # positive: set in moat.kv, negative: broadcast to :moat.kv.run tag
+
+  actor:
+    # Actor config, required for Runner
+    cycle: 20
+    nodes: -1
+    splits: 5
+    n_hosts: 3
+    version: 1
+  sub:
+    # tags for various runner modes
+    group: "any"
+    single: "at"
+    all: "all"
+server:
+  # server-side configuration
+  buffer: 10
+  # per-stream buffer
+
+  backend: "mqtt"
+  # default
+  mqtt:
+    uri: "mqtt://localhost:1883"
+  serf:
+    host: "localhost"
+    port: 7373
+
+  # event message path/topic prefix
+  root: !P moat.kv
+
+  paranoia: False
+  # typecheck server-to-server updates?
+  #
+  # which addresses/ports to accept MoaT-KV connections on
+  bind: [{}]
+  bind_default:
+    # default values for all elements of "bind"
+    host: "localhost"
+    port: PORT
+    ssl: False
+  change:
+    length: 5
+    # chain length: use max nr of network sections +1
+  ping:
+    cycle: 10
+    gap: 2
+    # asyncserf.Actor config timing for server sync
+  # ping also controls minimum server startup time
+  delete:
+    # asyncserf.Actor config timing for deletion
+    cycle: 100
+    gap: 10
+    version: 1
+paranoia: false
+# typecheck server>server updates?
+
+# how does a new server reach existing nodes, to download state?
+domain: null
+# domain in which to look up node names, if not in hostmap
+hostmap:   # map MoaT-KV server names to connect destinations
+  test1: ["localhost", 27586]
+  test2: ["does-not-exist.invalid", 27586]

moat/kv/_main.py

@@ -4,11 +4,12 @@ Basic DistKV support
 
 """
 
+from __future__ import annotations
+
 import logging
-from pathlib import Path
 
 import asyncclick as click
-from moat.util import attrdict, combine_dict, load_subgroup, yload
+from moat.util import attrdict, combine_dict, load_subgroup, CFG, ensure_cfg
 
 from moat.kv.auth import gen_auth
 from moat.kv.client import client_scope
@@ -16,7 +17,7 @@ from moat.kv.client import client_scope
 logger = logging.getLogger(__name__)
 
 
-CFG = yload(Path(__file__).parent / "_config.yaml", attr=True)
+ensure_cfg("moat.kv")
 
 
 class NullObj:
@@ -40,12 +41,8 @@ class NullObj:
         raise self._exc
 
 
-@load_subgroup(
-    sub_pre="moat.kv.command", sub_post="cli", ext_pre="moat.kv", ext_post="_main.cli"
-)
-@click.option(
-    "-h", "--host", default=None, help=f"Host to use. Default: {CFG.kv.conn.host}"
-)
+@load_subgroup(sub_pre="moat.kv.command", sub_post="cli", ext_pre="moat.kv", ext_post="_main.cli")
+@click.option("-h", "--host", default=None, help=f"Host to use. Default: {CFG.kv.conn.host}")
 @click.option(
     "-p",
     "--port",

moat/kv/client.py

@@ -4,17 +4,18 @@ Client code.
 Main entry point: :func:`open_client`.
 """
 
+from __future__ import annotations
+
 import logging
 import os
 import socket
 from contextlib import AsyncExitStack, asynccontextmanager
 from inspect import iscoroutine
-from pathlib import Path
-from typing import Tuple
 
 import anyio
 from asyncscope import Scope, main_scope, scope
 from moat.util import (  # pylint: disable=no-name-in-module
+    CFG,
     DelayedRead,
     DelayedWrite,
     NotGiven,
@@ -25,9 +26,10 @@ from moat.util import (  # pylint: disable=no-name-in-module
     byte2num,
     combine_dict,
     create_queue,
+    ensure_cfg,
     gen_ssl,
+    gen_ident,al_lower,
     num2byte,
-    yload,
 )
 
 from .codec import packer, stream_unpacker
@@ -45,17 +47,6 @@ logger = logging.getLogger(__name__)
 
 ClosedResourceError = anyio.ClosedResourceError
 
-rs = os.environ.get("PYTHONHASHSEED", None)
-if rs is None:
-    import random
-else:  # pragma: no cover
-    try:
-        import trio._core._run as tcr
-    except ImportError:
-        import random
-    else:
-        random = tcr._r
-
 __all__ = ["NoData", "ManyData", "open_client", "client_scope", "StreamedRequest"]
 
 
@@ -117,9 +108,7 @@ async def client_scope(_name=None, **cfg):
             _name = f"_{_cid}"
             # uniqueness required for testing.
             # TODO replace with a dependency on the test server.
-    return await scope.service(
-        f"moat.kv.client.{_name}", _scoped_client, _name=_name, **cfg
-    )
+    return await scope.service(f"moat.kv.client.{_name}", _scoped_client, _name=_name, **cfg)
 
 
 class StreamedRequest:
@@ -155,9 +144,7 @@ class StreamedRequest:
         self._path_long = lambda x: x
         if client.qlen > 0:
             self.dw = DelayedWrite(client.qlen)
-            self.qr = DelayedRead(
-                client.qlen, get_seq=self._get_seq, send_ack=self._send_ack
-            )
+            self.qr = DelayedRead(client.qlen, get_seq=self._get_seq, send_ack=self._send_ack)
         else:
             self.qr = create_queue(client.config.server.buffer)
 
@@ -227,7 +214,7 @@ class StreamedRequest:
 
     async def get(self):
         """Receive a single reply"""
-        pass  # receive reply
+        # receive reply
         if self._reply_stream:
             raise RuntimeError("Unexpected multi stream msg")
         msg = await self.recv()
@@ -246,7 +233,7 @@ class StreamedRequest:
     async def __anext__(self):
         try:
             res = await self.qr.get()
-        except (anyio.EndOfStream, anyio.ClosedResourceError):
+        except (anyio.EndOfStream, anyio.ClosedResourceError, EOFError):
             raise StopAsyncIteration
         except CancelledError:
             raise StopAsyncIteration  # just terminate
@@ -274,7 +261,7 @@ class StreamedRequest:
     async def cancel(self):
         try:
             await self.qr.put_error(CancelledError())
-        except (anyio.BrokenResourceError, anyio.ClosedResourceError):
+        except (anyio.BrokenResourceError, anyio.ClosedResourceError, EOFError):
             pass
         else:
             try:
@@ -411,7 +398,7 @@ class Client:
     qlen: int = 0
 
     def __init__(self, cfg: dict):
-        CFG = yload(Path(__file__).parent / "_config.yaml")
+        ensure_cfg("moat.kv")
         self._cfg = combine_dict(cfg, CFG["kv"], cls=attrdict)
         self.config = ClientConfig(self)
 
@@ -419,7 +406,7 @@ class Client:
         self._handlers = {}
         self._send_lock = anyio.Lock()
         self._helpers = {}
-        self._name = "".join(random.choices("abcdefghjkmnopqrstuvwxyz23456789", k=9))
+        self._name = gen_ident(9, alphabet=al_lower)
         self.logger = logging.getLogger(f"moat.kv.client.{self._name}")
 
     @property
@@ -482,11 +469,11 @@ class Client:
 
             k = await anyio.to_thread.run_sync(gen_key)
             res = await self._request(
-                "diffie_hellman", pubkey=num2byte(k.public_key), length=length
+                "diffie_hellman",
+                pubkey=num2byte(k.public_key),
+                length=length,
             )  # length=k.key_length
-            await anyio.to_thread.run_sync(
-                k.generate_shared_secret, byte2num(res.pubkey)
-            )
+            await anyio.to_thread.run_sync(k.generate_shared_secret, byte2num(res.pubkey))
             self._dh_key = num2byte(k.shared_secret)[0:32]
         return self._dh_key
 
@@ -681,16 +668,12 @@ class Client:
         if not sa or not sa[0]:
             # no auth required
             if auth:
-                logger.info(
-                    "Tried to use auth=%s, but not required.", auth._auth_method
-                )
+                logger.info("Tried to use auth=%s, but not required.", auth._auth_method)
             return
         if not auth:
             raise ClientAuthRequiredError("You need to log in using:", sa[0])
         if auth._auth_method != sa[0]:
-            raise ClientAuthMethodError(
-                f"You cannot use {auth._auth_method!r} auth", sa
-            )
+            raise ClientAuthMethodError(f"You cannot use {auth._auth_method!r} auth", sa)
         if getattr(auth, "_DEBUG", False):
             auth._length = 16
         await auth.auth(self)
@@ -736,9 +719,7 @@ class Client:
                         self._server_init = msg = await hello.get()
                         self.logger.debug("Hello %s", msg)
                         self.server_name = msg.node
-                        self.client_name = (
-                            cfg["name"] or socket.gethostname() or self.server_name
-                        )
+                        self.client_name = cfg["name"] or socket.gethostname() or self.server_name
                         if "qlen" in msg:
                             self.qlen = min(msg.qlen, 99)  # self.config.server.buffer
                             await self._send(seq=0, qlen=self.qlen)
@@ -746,13 +727,11 @@ class Client:
 
                     from .config import ConfigRoot
 
-                    self._config = await ConfigRoot.as_handler(
-                        self, require_client=False
-                    )
+                    self._config = await ConfigRoot.as_handler(self, require_client=False)
 
                 except TimeoutError:
                     raise
-                except socket.error as e:
+                except OSError as e:
                     raise ServerConnectionError(host, port) from e
                 else:
                     yield self
@@ -830,7 +809,12 @@ class Client:
             kw["idem"] = idem
 
         return self._request(
-            action="set_value", path=path, value=value, iter=False, nchain=nchain, **kw
+            action="set_value",
+            path=path,
+            value=value,
+            iter=False,
+            nchain=nchain,
+            **kw,
         )
 
     def delete(self, path, *, chain=NotGiven, prev=NotGiven, nchain=0, recursive=False):
@@ -878,9 +862,7 @@ class Client:
             raise RuntimeError("You need a path, not a string")
         if empty is None:
             empty = not with_data
-        res = await self._request(
-            action="enum", path=path, with_data=with_data, empty=empty, **kw
-        )
+        res = await self._request(action="enum", path=path, with_data=with_data, empty=empty, **kw)
         try:
             return res.result
         except AttributeError:
@@ -910,7 +892,11 @@ class Client:
         if long_path:
             lp = PathLongener()
         async for r in await self._request(
-            action="get_tree", path=path, iter=True, long_path=True, **kw
+            action="get_tree",
+            path=path,
+            iter=True,
+            long_path=True,
+            **kw,
         ):
             if long_path:
                 lp(r)
@@ -963,9 +949,7 @@ class Client:
         """
         if isinstance(path, str):
             raise RuntimeError("You need a path, not a string")
-        return self._stream(
-            action="watch", path=path, iter=True, long_path=long_path, **kw
-        )
+        return self._stream(action="watch", path=path, iter=True, long_path=long_path, **kw)
 
     def mirror(self, path, *, root_type=None, **kw):
         """An async context manager that affords an update-able mirror
@@ -997,7 +981,7 @@ class Client:
         root = root_type(self, path, **kw)
         return root.run()
 
-    def msg_monitor(self, topic: Tuple[str], raw: bool = False):
+    def msg_monitor(self, topic: tuple[str], raw: bool = False):
         """
         Return an async iterator of tunneled messages. This receives
         all messages sent using :meth:`msg_send` with the same topic.
@@ -1023,7 +1007,7 @@ class Client:
         """
         return self._stream(action="msg_monitor", topic=topic, raw=raw)
 
-    def msg_send(self, topic: Tuple[str], data=None, raw: bytes = None):
+    def msg_send(self, topic: tuple[str], data=None, raw: bytes = None):
         """
         Tunnel a user-tagged message. This sends the message
         to all active callers of :meth:`msg_monitor` which use the same topic.

moat/kv/code.py

@@ -5,6 +5,7 @@ so that it can be called easily.
 "Code" consists of either Python modules or single procedures.
 
 """
+from __future__ import annotations
 
 import logging
 import sys
@@ -36,7 +37,7 @@ class ModuleRoot(ClientRoot):
 
     CFG = "modules"
 
-    err: "ErrorRoot" = None  # noqa: F821
+    err: ErrorRoot = None  # noqa: F821
 
     @classmethod
     def child_type(cls, name):
@@ -95,7 +96,10 @@ class ModuleEntry(ClientEntry):
             self._module = None
             logger.warning("Could not compile @%r", self.subpath)
             await self.root.err.record_error(
-                "compile", self.subpath, exc=exc, message="compiler error"
+                "compile",
+                self.subpath,
+                exc=exc,
+                message="compiler error",
             )
         else:
             await self.root.err.record_working("compile", self.subpath)
@@ -207,7 +211,10 @@ class CodeEntry(ClientEntry):
         except Exception as exc:
             logger.warning("Could not compile @%s", self.subpath)
             await self.root.err.record_error(
-                "compile", self.subpath, exc=exc, message="compiler error"
+                "compile",
+                self.subpath,
+                exc=exc,
+                message="compiler error",
             )
             self._code = None
         else:

moat/kv/codec.py

@@ -4,6 +4,7 @@ plus an unpacker factory for streams.
 """
 
 # compatibility
+from __future__ import annotations
 
 from moat.util import *  # noqa: F403
 

moat/kv/config.py

@@ -3,6 +3,8 @@ An online-updated config store
 
 """
 
+from __future__ import annotations
+
 try:
     from contextlib import asynccontextmanager
 except ImportError:  # pragma: no cover

moat/kv/data.py

@@ -1,6 +1,8 @@
 """
 Data access
 """
+from __future__ import annotations
+
 import datetime
 import os
 import sys
@@ -35,7 +37,8 @@ def add_dates(d):
                 continue
             if start <= v <= stop:
                 d[f"_{k}"] = datetime.datetime.fromtimestamp(v).isoformat(
-                    sep=" ", timespec="milliseconds"
+                    sep=" ",
+                    timespec="milliseconds",
                 )
 
     _add(d)
@@ -81,9 +84,7 @@ async def data_get(
             kw.setdefault("nchain", obj.meta)
         y = {}
         if internal:
-            res = await obj.client._request(
-                action="get_tree_internal", path=path, iter=True, **kw
-            )
+            res = await obj.client._request(action="get_tree_internal", path=path, iter=True, **kw)
         else:
             res = obj.client.get_tree(path, **kw)
         async for r in res:
@@ -182,16 +183,16 @@ def res_update(res, attr: Path, value=None, **kw):  # pylint: disable=redefined-
     return val._update(attr, value=value, **kw)
 
 
-async def node_attr(obj, path, vars_, eval_, path_, res=None, chain=None):
+async def node_attr(obj, path, res=None, chain=None, **kw):
     """
     Sub-attr setter.
 
     Args:
         obj: command object
         path: address of the node to change
-        vars_, eval_, path_: the results of `attr_args`
         res: old node, if it has been read already
         chain: change chain of node, copied from res if clear
+        **kw: the results of `attr_args`
 
     Returns the result of setting the attribute.
     """
@@ -207,7 +208,7 @@ async def node_attr(obj, path, vars_, eval_, path_, res=None, chain=None):
     except AttributeError:
         chain = None
         val = NotGiven
-    val = process_args(val, vars_, eval_, path_)
+    val = process_args(val, **kw)
     if val is NotGiven:
         res = await obj.client.delete(path, nchain=obj.meta, chain=chain)
     else:

moat/kv/errors.py

@@ -79,6 +79,8 @@ of this record.
 
 """
 
+from __future__ import annotations
+
 import logging
 import traceback
 from collections import defaultdict
@@ -212,7 +214,7 @@ class ErrorEntry(AttrClientEntry):
                 try:
                     m = message.format(exc=exc, **data)
                 except Exception as exc:  # pylint: disable=unused-argument  # OH COME ON
-                    m = message + f" (FORMAT {exc !r})"
+                    m = message + f" (FORMAT {exc!r})"
             else:
                 m = message
             if m:
@@ -304,9 +306,7 @@ class ErrorEntry(AttrClientEntry):
         if value is NotGiven:
             if self.value is NotGiven:
                 return
-            keep = await self.root.get_error_record(
-                self.subsystem, self.path, create=False
-            )
+            keep = await self.root.get_error_record(self.subsystem, self.path, create=False)
             if keep is not None:
                 self._real_entry = keep.real_entry
                 await self.move_to_real()
@@ -449,7 +449,13 @@ class ErrorRoot(ClientRoot):
         raise RuntimeError(f"This cannot happen: {entry.node} {entry.tock}")
 
     async def record_working(  # pylint: disable=dangerous-default-value
-        self, subsystem, path, *, comment=None, data={}, force=False
+        self,
+        subsystem,
+        path,
+        *,
+        comment=None,
+        data={},
+        force=False,
     ):
         """This exception has been fixed.
 
@@ -522,7 +528,11 @@ class ErrorRoot(ClientRoot):
             return  # owch, but can't be helped
 
         r = await rec.real_entry.add_exc(
-            self.name, exc=exc, data=data, comment=comment, message=message
+            self.name,
+            exc=exc,
+            data=data,
+            comment=comment,
+            message=message,
         )
         return r
 
@@ -535,9 +545,7 @@ class ErrorRoot(ClientRoot):
             return
 
         try:
-            del (self._done if entry.resolved else self._active)[entry.subsystem][
-                entry.path
-            ]
+            del (self._done if entry.resolved else self._active)[entry.subsystem][entry.path]
         except KeyError:
             pass
 

moat/kv/exceptions.py

@@ -3,7 +3,7 @@ This module affords all MoaT-KV exceptions.
 """
 
 # pylint: disable=unnecessary-pass
-
+from __future__ import annotations
 
 error_types = {}
 
@@ -39,8 +39,6 @@ class ClientError(MoaTKVError):
 
     etype: str = None
 
-    pass
-
 
 @_typed
 class ClientChainError(ClientError):
@@ -48,8 +46,6 @@ class ClientChainError(ClientError):
 
     etype = "chain"
 
-    pass
-
 
 @_typed
 class ClientConnectionError(ClientError):
@@ -57,8 +53,6 @@ class ClientConnectionError(ClientError):
 
     etype = "conn"
 
-    pass
-
 
 class ServerClosedError(ServerError):
     """The server closed our connection."""