moat-kv 0.70.23__py3-none-any.whl → 0.70.24__py3-none-any.whl

moat/kv/command/codec.py

@@ -0,0 +1,186 @@
+# command line interface
+from __future__ import annotations
+
+import asyncclick as click
+from moat.util import NotGiven, P, Path, PathLongener, yload, yprint
+
+
+@click.group()  # pylint: disable=undefined-variable
+async def cli():
+    """Manage codecs and converters. Usage: … codec …"""
+    pass
+
+
+@cli.command()
+@click.option("-e", "--encode", type=click.File(mode="w", lazy=True), help="Save the encoder here")
+@click.option("-d", "--decode", type=click.File(mode="w", lazy=True), help="Save the decoder here")
+@click.option("-s", "--script", type=click.File(mode="w", lazy=True), help="Save the data here")
+@click.argument("path", nargs=1)
+@click.pass_obj
+async def get(obj, path, script, encode, decode):
+    """Read type information"""
+    path = P(path)
+    if not len(path):
+        raise click.UsageError("You need a non-empty path.")
+    res = await obj.client._request(
+        action="get_internal",
+        path=Path("codec") + path,
+        iter=False,
+        nchain=obj.meta,
+    )
+    if encode and res.get("encode", None) is not None:
+        encode.write(res.pop("encode"))
+    if decode and res.get("decode", None) is not None:
+        decode.write(res.pop("decode"))
+
+    if not obj.meta:
+        res = res.value
+    yprint(res, stream=script or obj.stdout)
+
+
+@cli.command(name="list")
+@click.pass_obj
+@click.argument("path", nargs=1)
+async def list_(obj, path):
+    """List type information entries"""
+    res = await obj.client._request(
+        action="get_tree_internal",
+        path=Path("codec") + P(path),
+        iter=True,
+        nchain=obj.meta,
+    )
+    pl = PathLongener(())
+    async for r in res:
+        pl(r)
+        print(" ".join(str(x) for x in r.path), file=obj.stdout)
+
+
+@cli.command("set")
+@click.option("-e", "--encode", type=click.File(mode="r"), help="File with the encoder")
+@click.option("-d", "--decode", type=click.File(mode="r"), help="File with the decoder")
+@click.option("-D", "--data", type=click.File(mode="r"), help="File with the rest")
+@click.option("-i", "--in", "in_", nargs=2, multiple=True, help="Decoding sample")
+@click.option("-o", "--out", nargs=2, multiple=True, help="Encoding sample")
+@click.argument("path", nargs=1)
+@click.pass_obj
+async def set_(obj, path, encode, decode, data, in_, out):
+    """Save codec information"""
+    path = P(path)
+    if not len(path):
+        raise click.UsageError("You need a non-empty path.")
+
+    if data:
+        msg = yload(data)
+    else:
+        msg = {}
+    chain = NotGiven
+    if "value" in msg:
+        chain = msg.get("chain", NotGiven)
+        msg = msg["value"]
+
+    if "encode" in msg:
+        if encode:
+            raise click.UsageError("Duplicate encode script")
+    else:
+        if not encode:
+            raise click.UsageError("Missing encode script")
+        msg["encode"] = encode.read()
+    if "decode" in msg:
+        if decode:
+            raise click.UsageError("Duplicate decode script")
+    else:
+        if not decode:
+            raise click.UsageError("Missing decode script")
+        msg["decode"] = decode.read()
+    if in_:
+        msg["in"] = [(eval(a), eval(b)) for a, b in in_]  # pylint: disable=eval-used
+    if out:
+        msg["out"] = [(eval(a), eval(b)) for a, b in out]  # pylint: disable=eval-used
+
+    if not msg["in"]:
+        raise click.UsageError("Missing decode tests")
+    if not msg["out"]:
+        raise click.UsageError("Missing encode tests")
+
+    res = await obj.client._request(
+        action="set_internal",
+        value=msg,
+        path=Path("codec") + path,
+        iter=False,
+        nchain=obj.meta,
+        **({} if chain is NotGiven else {"chain": chain}),
+    )
+    if obj.meta:
+        yprint(res, stream=obj.stdout)
+
+
+@cli.command()
+@click.option("-c", "--codec", type=P, help="Codec to link to.")
+@click.option("-d", "--delete", is_flag=True, help="Use to delete this converter.")
+@click.option(
+    "-l",
+    "--list",
+    "list_",
+    is_flag=True,
+    help="Use to list this converter; '-' to list all.",
+)
+@click.argument("name", nargs=1)
+@click.argument("path", type=P, nargs=1)
+@click.pass_obj
+async def convert(obj, path, codec, name, delete, list_):
+    """Match a codec to a path (read, if no codec given)"""
+    path = P(path)
+    if delete and list_:
+        raise click.UsageError("You can't both list and delete a path.")
+    if not len(path) and not list_:
+        raise click.UsageError("You need a non-empty path.")
+    if codec and delete:
+        raise click.UsageError("You can't both set and delete a path.")
+
+    if list_:
+        if name == "-":
+            if len(path):
+                raise click.UsageError("You can't use a path here.")
+            res = await obj.client._request(
+                action="enum_internal",
+                path=Path("conv"),
+                iter=False,
+                nchain=0,
+                empty=True,
+            )
+            for r in res.result:
+                print(r, file=obj.stdout)
+
+        else:
+            res = await obj.client._request(
+                action="get_tree_internal",
+                path=Path("conv", name) + path,
+                iter=True,
+                nchain=obj.meta,
+            )
+            pl = PathLongener(())
+            async for r in res:
+                pl(r)
+                try:
+                    print(f"{r.path} : {Path.build(r.value['codec'])}", file=obj.stdout)
+                except Exception as e:
+                    print(f"{Path(r.path)} {e!r}", file=obj.stdout)
+
+        return
+    if delete:
+        res = await obj.client._request(action="delete_internal", path=Path("conv", name) + path)
+    else:
+        msg = {"codec": codec}
+        res = await obj.client._request(
+            action="set_internal",
+            value=msg,
+            path=Path("conv", name) + path,
+            iter=False,
+            nchain=obj.meta,
+        )
+    if obj.meta:
+        yprint(res, stream=obj.stdout)
+    elif type or delete:
+        print(res.tock, file=obj.stdout)
+    else:
+        print(" ".join(res.type), file=obj.stdout)

moat/kv/command/data.py

@@ -0,0 +1,265 @@
+# command line interface
+from __future__ import annotations
+
+import datetime
+import time
+
+import asyncclick as click
+from moat.util import MsgReader, NotGiven, P, PathLongener, attr_args, yprint
+
+from moat.kv.client import StreamedRequest
+from moat.kv.data import add_dates, data_get, node_attr
+
+
+@click.group(short_help="Manage data.", invoke_without_command=True)  # pylint: disable=undefined-variable
+@click.argument("path", type=P, nargs=1)
+@click.pass_context
+async def cli(ctx, path):
+    """
+    This subcommand accesses the actual user data stored in your MoaT-KV tree.
+    """
+    if ctx.invoked_subcommand is None:
+        await data_get(ctx.obj, path, recursive=False)
+    else:
+        ctx.obj.path = path
+
+
+@cli.command()
+@click.option(
+    "-d",
+    "--as-dict",
+    default=None,
+    help="Structure as dictionary. The argument is the key to use "
+    "for values. Default: return as list",
+)
+@click.option(
+    "-m",
+    "--maxdepth",
+    type=int,
+    default=None,
+    help="Limit recursion depth. Default: whole tree",
+)
+@click.option(
+    "-M",
+    "--mindepth",
+    type=int,
+    default=None,
+    help="Starting depth. Default: whole tree",
+)
+@click.option("-r", "--recursive", is_flag=True, help="Read a complete subtree")
+@click.option("-e", "--empty", is_flag=True, help="Include empty nodes")
+@click.option("-R", "--raw", is_flag=True, help="Print string values without quotes etc.")
+@click.option("-D", "--add-date", is_flag=True, help="Add *_date entries")
+@click.pass_obj
+async def get(obj, **k):
+    """
+    Read a MoaT-KV value.
+
+    If you read a sub-tree recursively, be aware that the whole subtree
+    will be read before anything is printed. Use the "watch --state" subcommand
+    for incremental output.
+    """
+
+    await data_get(obj, obj.path, **k)
+
+
+@cli.command("list")
+@click.option(
+    "-d",
+    "--as-dict",
+    default=None,
+    help="Structure as dictionary. The argument is the key to use "
+    "for values. Default: return as list",
+)
+@click.option(
+    "-m",
+    "--maxdepth",
+    type=int,
+    default=1,
+    help="Limit recursion depth. Default: 1 (single layer).",
+)
+@click.option(
+    "-M",
+    "--mindepth",
+    type=int,
+    default=1,
+    help="Starting depth. Default: 1 (single layer).",
+)
+@click.pass_obj
+async def list_(obj, **k):
+    """
+    List MoaT-KV values.
+
+    This is like "get" but with "--mindepth=1 --maxdepth=1 --recursive --empty"
+
+    If you read a sub-tree recursively, be aware that the whole subtree
+    will be read before anything is printed. Use the "watch --state" subcommand
+    for incremental output.
+    """
+
+    k["recursive"] = True
+    k["raw"] = True
+    k["empty"] = True
+    await data_get(obj, obj.path, **k)
+
+
+@cli.command("set", short_help="Add or update an entry")
+@attr_args
+@click.option("-l", "--last", nargs=2, help="Previous change entry (node serial)")
+@click.option("-n", "--new", is_flag=True, help="This is a new entry.")
+@click.pass_obj
+async def set_(obj, last, new, **kw):
+    """
+    Store a value at some MoaT-KV position.
+
+    If you update a value you can use "--last" to ensure that no other
+    change arrived between reading and writing the entry. (This is not
+    foolproof but reduces the window to a fraction of a second.)
+
+    When adding a new entry use "--new" to ensure that you don't
+    accidentally overwrite something.
+
+    MoaT-KV entries typically are mappings. Use a colon as the path if you
+    want to replace the top level.
+    """
+    if new:
+        if last:
+            raise click.UsageError("'new' and 'last' are mutually exclusive")
+        kw["chain"] = None
+    else:
+        if last:
+            kw["chain"] = {"node": last[0], "tick": int(last[1])}
+
+    res = await node_attr(obj, obj.path, **kw)
+
+    if obj.meta:
+        yprint(res, stream=obj.stdout)
+
+
+class nstr:
+    def __new__(cls, val):
+        if val is NotGiven:
+            return val
+        return str(val)
+
+
+@cli.command(short_help="Delete an entry / subtree")
+@click.option(
+    "-p",
+    "--prev",
+    type=nstr,
+    default=NotGiven,
+    help="Previous value. Deprecated; use 'last'",
+)
+@click.option("-l", "--last", nargs=2, help="Previous change entry (node serial)")
+@click.option("-r", "--recursive", is_flag=True, help="Delete a complete subtree")
+@click.option("--internal", is_flag=True, help="Affect the internal tree. DANGER.")
+@click.option("-e", "--eval", "eval_", is_flag=True, help="The previous value shall be evaluated.")
+@click.pass_obj
+async def delete(obj, prev, last, recursive, eval_, internal):
+    """
+    Delete an entry, or a whole subtree.
+
+    You really should use "--last" (preferred) or "--prev" (if you must) to
+    ensure that no other change arrived (but note that this doesn't work
+    when deleting a subtree).
+
+    Non-recursively deleting an entry with children works and does *not*
+    affect the child entries.
+
+    The root entry cannot be deleted.
+    """
+    args = {}
+    if eval_ and prev is NotGiven:
+        raise click.UsageError("You need to add a value that can be evaluated")
+    if recursive:
+        if prev is not NotGiven or last:
+            raise click.UsageError("You can't use a prev value when deleting recursively.")
+        if internal:
+            raise click.UsageError("'internal' and 'recursive' are mutually exclusive")
+    else:
+        if prev is not NotGiven:
+            if eval_:
+                prev = eval(prev)  # pylint: disable=eval-used
+            args["prev"] = prev
+        if last:
+            args["chain"] = {"node": last[0], "tick": int(last[1])}
+
+    res = await obj.client.delete(path=obj.path, nchain=obj.meta, recursive=recursive, **args)
+    if isinstance(res, StreamedRequest):
+        pl = PathLongener(obj.path)
+        async for r in res:
+            pl(r)
+            if obj.meta:
+                yprint(r, stream=obj.stdout)
+    else:
+        if obj.meta:
+            yprint(res, stream=obj.stdout)
+
+
+@cli.command()
+@click.option("-s", "--state", is_flag=True, help="Also get the current state.")
+@click.option("-o", "--only", is_flag=True, help="Value only, nothing fancy.")
+@click.option("-p", "--path-only", is_flag=True, help="Value only, nothing fancy.")
+@click.option("-D", "--add-date", is_flag=True, help="Add *_date entries")
+@click.option("-i", "--ignore", multiple=True, type=P, help="Skip this (sub)tree")
+@click.pass_obj
+async def monitor(obj, state, only, path_only, add_date, ignore):
+    """Monitor a MoaT-KV subtree"""
+
+    flushing = not state
+    seen = False
+
+    async with obj.client.watch(
+        obj.path,
+        nchain=obj.meta,
+        fetch=state,
+        max_depth=0 if only else -1,
+        long_path=False,
+    ) as res:
+        pl = PathLongener(() if path_only else obj.path)
+        async for r in res:
+            if add_date and "value" in r:
+                add_dates(r.value)
+            if any(p == r.path[: len(p)] for p in ignore):
+                continue
+            pl(r)
+            if r.get("state", "") == "uptodate":
+                if only and not seen:
+                    # value doesn't exist
+                    return
+                flushing = True
+                if only or path_only:
+                    continue
+            else:
+                del r["seq"]
+                if only or path_only:
+                    try:
+                        if path_only:
+                            print(f"{r.path} {r.value}", file=obj.stdout)
+                        else:
+                            print(r.value, file=obj.stdout)
+                        continue
+                    except AttributeError:
+                        # value has been deleted
+                        continue
+            if flushing:
+                r["time"] = time.time()
+                r["_time"] = datetime.datetime.now().isoformat(sep=" ", timespec="milliseconds")
+            yprint(r, stream=obj.stdout)
+            print("---", file=obj.stdout)
+            if flushing:
+                obj.stdout.flush()
+            seen = True
+
+
+@cli.command()
+@click.option("-i", "--infile", type=click.Path(), help="File to read (msgpack).")
+@click.pass_obj
+async def update(obj, infile):
+    """Send a list of updates to a MoaT-KV subtree"""
+    async with MsgReader(path=infile) as reader:
+        async for msg in reader:
+            if not hasattr(msg, "path"):
+                continue
+            await obj.client.set(obj.path + msg.path, value=msg.value)

moat/kv/command/dump/__init__.py

@@ -0,0 +1,143 @@
+# command line interface
+from __future__ import annotations
+
+import datetime
+import sys
+from collections.abc import Mapping
+
+import asyncclick as click
+from moat.mqtt.codecs import MsgPackCodec
+from moat.util import (
+    MsgReader,
+    MsgWriter,
+    P,
+    Path,
+    PathLongener,
+    load_subgroup,
+    yload,
+    yprint,
+)
+
+from moat.kv.codec import unpacker
+
+
+@load_subgroup(short_help="Local data mangling", sub_pre="dump")
+async def cli():
+    """
+    Low-level tools that don't depend on a running MoaT-KV server.
+    """
+    pass
+
+
+@cli.command()
+@click.argument("node", nargs=1)
+@click.argument("file", type=click.Path(), nargs=1)
+async def init(node, file):
+    """Write an initial preload file.
+
+    Usage: moat kv dump init <node> <outfile>
+
+    Writes an initial MoaT-KV file that behaves as if it was generated by <node>.
+
+    Using this command, followed by "moat kv server -l <outfile> <node>", is
+    equivalent to running "moat kv server -i 'Initial data' <node>.
+    """
+    async with MsgWriter(path=file) as f:
+        await f(
+            dict(
+                chain=dict(node=node, tick=1, prev=None),
+                depth=0,
+                path=[],
+                tock=1,
+                value="Initial data",
+            ),
+        )
+
+
+@cli.command("msg")
+@click.argument("path", nargs=1)
+@click.pass_obj
+async def msg_(obj, path):
+    """
+    Monitor the server-to-sever message stream.
+
+    Use ':' for the main server's "update" stream.
+    Use '+NAME' to monitor a different stream instead.
+    Use '+' to monitor all streams.
+    Otherwise use the given name as-is; Mosquitto wildcard rules apply.
+
+    \b
+    Common streams (prefix with '+'):
+    * ping    all servers
+    * update  data changes
+    * del     nodes responsible for cleaning up deleted records
+    """
+    from moat.kv.backend import get_backend
+
+    class _Unpack:
+        def __init__(self):
+            self._part_cache = dict()
+
+    import moat.kv.server
+
+    _Unpack._unpack_multiple = moat.kv.server.Server._unpack_multiple
+    _unpacker = _Unpack()._unpack_multiple
+
+    path = P(path)
+    if len(path) == 0:
+        path = P(obj.cfg.server["root"]) | "update"
+    elif len(path) == 1 and path[0].startswith("+"):  # pylint: disable=no-member  # owch
+        p = path[0][1:]
+        path = P(obj.cfg.server["root"])
+        path |= p or "#"
+    be = obj.cfg.server.backend
+    kw = obj.cfg.server[be]
+
+    async with get_backend(be)(**kw) as conn:
+        async with conn.monitor(*path) as stream:
+            async for msg in stream:
+                v = vars(msg)
+                if isinstance(v.get("payload"), (bytearray, bytes)):
+                    t = msg.topic
+                    v = unpacker(v["payload"])
+                    v = _unpacker(v)
+                    if v is None:
+                        continue
+                    if not isinstance(v, Mapping):
+                        v = {"_data": v}
+                    v["_topic"] = Path.build(t)
+                else:
+                    v["_type"] = type(msg).__name__
+
+                v["_timestamp"] = datetime.datetime.now().isoformat(
+                    sep=" ",
+                    timespec="milliseconds",
+                )
+
+                yprint(v, stream=obj.stdout)
+                print("---", file=obj.stdout)
+
+
+@cli.command("post")
+@click.argument("path", nargs=1)
+@click.pass_obj
+async def post_(obj, path):
+    """
+    Send a msgpack-encoded message (or several) to a specific MQTT topic.
+
+    Messages are read from YAML.
+    Common streams:
+    * ping: sync: all servers (default)
+    * update: data changes
+    * del: sync: nodes responsible for cleaning up deleted records
+    """
+    from moat.kv.backend import get_backend
+
+    path = P(path)
+    be = obj.cfg.server.backend
+    kw = obj.cfg.server[be]
+
+    async with get_backend(be)(codec=MsgPackCodec, **kw) as conn:
+        for d in yload(sys.stdin, multi=True):
+            topic = d.pop("_topic", path)
+            await conn.send(*topic, payload=d)