moat-kv 0.70.20__py3-none-any.whl → 0.70.23__py3-none-any.whl

moat/kv/command/codec.py

@@ -1,190 +0,0 @@
-# command line interface
-
-import asyncclick as click
-from moat.util import NotGiven, P, Path, PathLongener, yload, yprint
-
-
-@click.group()  # pylint: disable=undefined-variable
-async def cli():
-    """Manage codecs and converters. Usage: … codec …"""
-    pass
-
-
-@cli.command()
-@click.option(
-    "-e", "--encode", type=click.File(mode="w", lazy=True), help="Save the encoder here"
-)
-@click.option(
-    "-d", "--decode", type=click.File(mode="w", lazy=True), help="Save the decoder here"
-)
-@click.option(
-    "-s", "--script", type=click.File(mode="w", lazy=True), help="Save the data here"
-)
-@click.argument("path", nargs=1)
-@click.pass_obj
-async def get(obj, path, script, encode, decode):
-    """Read type information"""
-    path = P(path)
-    if not len(path):
-        raise click.UsageError("You need a non-empty path.")
-    res = await obj.client._request(
-        action="get_internal", path=Path("codec") + path, iter=False, nchain=obj.meta
-    )
-    if encode and res.get("encode", None) is not None:
-        encode.write(res.pop("encode"))
-    if decode and res.get("decode", None) is not None:
-        decode.write(res.pop("decode"))
-
-    if not obj.meta:
-        res = res.value
-    yprint(res, stream=script or obj.stdout)
-
-
-@cli.command(name="list")
-@click.pass_obj
-@click.argument("path", nargs=1)
-async def list_(obj, path):
-    """List type information entries"""
-    res = await obj.client._request(
-        action="get_tree_internal",
-        path=Path("codec") + P(path),
-        iter=True,
-        nchain=obj.meta,
-    )
-    pl = PathLongener(())
-    async for r in res:
-        pl(r)
-        print(" ".join(str(x) for x in r.path), file=obj.stdout)
-
-
-@cli.command("set")
-@click.option("-e", "--encode", type=click.File(mode="r"), help="File with the encoder")
-@click.option("-d", "--decode", type=click.File(mode="r"), help="File with the decoder")
-@click.option("-D", "--data", type=click.File(mode="r"), help="File with the rest")
-@click.option("-i", "--in", "in_", nargs=2, multiple=True, help="Decoding sample")
-@click.option("-o", "--out", nargs=2, multiple=True, help="Encoding sample")
-@click.argument("path", nargs=1)
-@click.pass_obj
-async def set_(obj, path, encode, decode, data, in_, out):
-    """Save codec information"""
-    path = P(path)
-    if not len(path):
-        raise click.UsageError("You need a non-empty path.")
-
-    if data:
-        msg = yload(data)
-    else:
-        msg = {}
-    chain = NotGiven
-    if "value" in msg:
-        chain = msg.get("chain", NotGiven)
-        msg = msg["value"]
-
-    if "encode" in msg:
-        if encode:
-            raise click.UsageError("Duplicate encode script")
-    else:
-        if not encode:
-            raise click.UsageError("Missing encode script")
-        msg["encode"] = encode.read()
-    if "decode" in msg:
-        if decode:
-            raise click.UsageError("Duplicate decode script")
-    else:
-        if not decode:
-            raise click.UsageError("Missing decode script")
-        msg["decode"] = decode.read()
-    if in_:
-        msg["in"] = [(eval(a), eval(b)) for a, b in in_]  # pylint: disable=eval-used
-    if out:
-        msg["out"] = [(eval(a), eval(b)) for a, b in out]  # pylint: disable=eval-used
-
-    if not msg["in"]:
-        raise click.UsageError("Missing decode tests")
-    if not msg["out"]:
-        raise click.UsageError("Missing encode tests")
-
-    res = await obj.client._request(
-        action="set_internal",
-        value=msg,
-        path=Path("codec") + path,
-        iter=False,
-        nchain=obj.meta,
-        **({} if chain is NotGiven else {"chain": chain}),
-    )
-    if obj.meta:
-        yprint(res, stream=obj.stdout)
-
-
-@cli.command()
-@click.option("-c", "--codec", type=P, help="Codec to link to.")
-@click.option("-d", "--delete", is_flag=True, help="Use to delete this converter.")
-@click.option(
-    "-l",
-    "--list",
-    "list_",
-    is_flag=True,
-    help="Use to list this converter; '-' to list all.",
-)
-@click.argument("name", nargs=1)
-@click.argument("path", type=P, nargs=1)
-@click.pass_obj
-async def convert(obj, path, codec, name, delete, list_):
-    """Match a codec to a path (read, if no codec given)"""
-    path = P(path)
-    if delete and list_:
-        raise click.UsageError("You can't both list and delete a path.")
-    if not len(path) and not list_:
-        raise click.UsageError("You need a non-empty path.")
-    if codec and delete:
-        raise click.UsageError("You can't both set and delete a path.")
-
-    if list_:
-        if name == "-":
-            if len(path):
-                raise click.UsageError("You can't use a path here.")
-            res = await obj.client._request(
-                action="enum_internal",
-                path=Path("conv"),
-                iter=False,
-                nchain=0,
-                empty=True,
-            )
-            for r in res.result:
-                print(r, file=obj.stdout)
-
-        else:
-            res = await obj.client._request(
-                action="get_tree_internal",
-                path=Path("conv", name) + path,
-                iter=True,
-                nchain=obj.meta,
-            )
-            pl = PathLongener(())
-            async for r in res:
-                pl(r)
-                try:
-                    print(f"{r.path} : {Path.build(r.value['codec'])}", file=obj.stdout)
-                except Exception as e:
-                    print(f"{Path(r.path)} {e !r}", file=obj.stdout)
-
-        return
-    if delete:
-        res = await obj.client._request(
-            action="delete_internal", path=Path("conv", name) + path
-        )
-    else:
-        msg = {"codec": codec}
-        res = await obj.client._request(
-            action="set_internal",
-            value=msg,
-            path=Path("conv", name) + path,
-            iter=False,
-            nchain=obj.meta,
-        )
-    if obj.meta:
-        yprint(res, stream=obj.stdout)
-    elif type or delete:
-        print(res.tock, file=obj.stdout)
-    else:
-        print(" ".join(res.type), file=obj.stdout)

moat/kv/command/data.py

@@ -1,274 +0,0 @@
-# command line interface
-
-import datetime
-import time
-
-import asyncclick as click
-from moat.util import MsgReader, NotGiven, P, PathLongener, attr_args, yprint
-
-from moat.kv.client import StreamedRequest
-from moat.kv.data import add_dates, data_get, node_attr
-
-
-@click.group(short_help="Manage data.", invoke_without_command=True)  # pylint: disable=undefined-variable
-@click.argument("path", type=P, nargs=1)
-@click.pass_context
-async def cli(ctx, path):
-    """
-    This subcommand accesses the actual user data stored in your MoaT-KV tree.
-    """
-    if ctx.invoked_subcommand is None:
-        await data_get(ctx.obj, path, recursive=False)
-    else:
-        ctx.obj.path = path
-
-
-@cli.command()
-@click.option(
-    "-d",
-    "--as-dict",
-    default=None,
-    help="Structure as dictionary. The argument is the key to use "
-    "for values. Default: return as list",
-)
-@click.option(
-    "-m",
-    "--maxdepth",
-    type=int,
-    default=None,
-    help="Limit recursion depth. Default: whole tree",
-)
-@click.option(
-    "-M",
-    "--mindepth",
-    type=int,
-    default=None,
-    help="Starting depth. Default: whole tree",
-)
-@click.option("-r", "--recursive", is_flag=True, help="Read a complete subtree")
-@click.option("-e", "--empty", is_flag=True, help="Include empty nodes")
-@click.option(
-    "-R", "--raw", is_flag=True, help="Print string values without quotes etc."
-)
-@click.option("-D", "--add-date", is_flag=True, help="Add *_date entries")
-@click.pass_obj
-async def get(obj, **k):
-    """
-    Read a MoaT-KV value.
-
-    If you read a sub-tree recursively, be aware that the whole subtree
-    will be read before anything is printed. Use the "watch --state" subcommand
-    for incremental output.
-    """
-
-    await data_get(obj, obj.path, **k)
-
-
-@cli.command("list")
-@click.option(
-    "-d",
-    "--as-dict",
-    default=None,
-    help="Structure as dictionary. The argument is the key to use "
-    "for values. Default: return as list",
-)
-@click.option(
-    "-m",
-    "--maxdepth",
-    type=int,
-    default=1,
-    help="Limit recursion depth. Default: 1 (single layer).",
-)
-@click.option(
-    "-M",
-    "--mindepth",
-    type=int,
-    default=1,
-    help="Starting depth. Default: 1 (single layer).",
-)
-@click.pass_obj
-async def list_(obj, **k):
-    """
-    List MoaT-KV values.
-
-    This is like "get" but with "--mindepth=1 --maxdepth=1 --recursive --empty"
-
-    If you read a sub-tree recursively, be aware that the whole subtree
-    will be read before anything is printed. Use the "watch --state" subcommand
-    for incremental output.
-    """
-
-    k["recursive"] = True
-    k["raw"] = True
-    k["empty"] = True
-    await data_get(obj, obj.path, **k)
-
-
-@cli.command("set", short_help="Add or update an entry")
-@attr_args
-@click.option("-l", "--last", nargs=2, help="Previous change entry (node serial)")
-@click.option("-n", "--new", is_flag=True, help="This is a new entry.")
-@click.pass_obj
-async def set_(obj, vars_, eval_, path_, last, new):
-    """
-    Store a value at some MoaT-KV position.
-
-    If you update a value you can use "--last" to ensure that no other
-    change arrived between reading and writing the entry. (This is not
-    foolproof but reduces the window to a fraction of a second.)
-
-    When adding a new entry use "--new" to ensure that you don't
-    accidentally overwrite something.
-
-    MoaT-KV entries typically are mappings. Use a colon as the path if you
-    want to replace the top level.
-    """
-    args = {}
-    if new:
-        if last:
-            raise click.UsageError("'new' and 'last' are mutually exclusive")
-        args["chain"] = None
-    else:
-        if last:
-            args["chain"] = {"node": last[0], "tick": int(last[1])}
-
-    res = await node_attr(obj, obj.path, vars_, eval_, path_, **args)
-
-    if obj.meta:
-        yprint(res, stream=obj.stdout)
-
-
-class nstr:
-    def __new__(cls, val):
-        if val is NotGiven:
-            return val
-        return str(val)
-
-
-@cli.command(short_help="Delete an entry / subtree")
-@click.option(
-    "-p",
-    "--prev",
-    type=nstr,
-    default=NotGiven,
-    help="Previous value. Deprecated; use 'last'",
-)
-@click.option("-l", "--last", nargs=2, help="Previous change entry (node serial)")
-@click.option("-r", "--recursive", is_flag=True, help="Delete a complete subtree")
-@click.option("--internal", is_flag=True, help="Affect the internal tree. DANGER.")
-@click.option(
-    "-e", "--eval", "eval_", is_flag=True, help="The previous value shall be evaluated."
-)
-@click.pass_obj
-async def delete(obj, prev, last, recursive, eval_, internal):
-    """
-    Delete an entry, or a whole subtree.
-
-    You really should use "--last" (preferred) or "--prev" (if you must) to
-    ensure that no other change arrived (but note that this doesn't work
-    when deleting a subtree).
-
-    Non-recursively deleting an entry with children works and does *not*
-    affect the child entries.
-
-    The root entry cannot be deleted.
-    """
-    args = {}
-    if eval_ and prev is NotGiven:
-        raise click.UsageError("You need to add a value that can be evaluated")
-    if recursive:
-        if prev is not NotGiven or last:
-            raise click.UsageError(
-                "You can't use a prev value when deleting recursively."
-            )
-        if internal:
-            raise click.UsageError("'internal' and 'recursive' are mutually exclusive")
-    else:
-        if prev is not NotGiven:
-            if eval_:
-                prev = eval(prev)  # pylint: disable=eval-used
-            args["prev"] = prev
-        if last:
-            args["chain"] = {"node": last[0], "tick": int(last[1])}
-
-    res = await obj.client.delete(
-        path=obj.path, nchain=obj.meta, recursive=recursive, **args
-    )
-    if isinstance(res, StreamedRequest):
-        pl = PathLongener(obj.path)
-        async for r in res:
-            pl(r)
-            if obj.meta:
-                yprint(r, stream=obj.stdout)
-    else:
-        if obj.meta:
-            yprint(res, stream=obj.stdout)
-
-
-@cli.command()
-@click.option("-s", "--state", is_flag=True, help="Also get the current state.")
-@click.option("-o", "--only", is_flag=True, help="Value only, nothing fancy.")
-@click.option("-p", "--path-only", is_flag=True, help="Value only, nothing fancy.")
-@click.option("-D", "--add-date", is_flag=True, help="Add *_date entries")
-@click.option("-i", "--ignore", multiple=True, type=P, help="Skip this (sub)tree")
-@click.pass_obj
-async def monitor(obj, state, only, path_only, add_date, ignore):
-    """Monitor a MoaT-KV subtree"""
-
-    flushing = not state
-    seen = False
-
-    async with obj.client.watch(
-        obj.path,
-        nchain=obj.meta,
-        fetch=state, max_depth=0 if only else -1, 
-        long_path=False,
-    ) as res:
-        pl = PathLongener(() if path_only else obj.path)
-        async for r in res:
-            if add_date and "value" in r:
-                add_dates(r.value)
-            if any(p == r.path[: len(p)] for p in ignore):
-                continue
-            pl(r)
-            if r.get("state", "") == "uptodate":
-                if only and not seen:
-                    # value doesn't exist
-                    return
-                flushing = True
-                if only or path_only:
-                    continue
-            else:
-                del r["seq"]
-                if only or path_only:
-                    try:
-                        if path_only:
-                            print(f"{r.path} {r.value}", file=obj.stdout)
-                        else:
-                            print(r.value, file=obj.stdout)
-                        continue
-                    except AttributeError:
-                        # value has been deleted
-                        continue
-            if flushing:
-                r["time"] = time.time()
-                r["_time"] = datetime.datetime.now().isoformat(
-                    sep=" ", timespec="milliseconds"
-                )
-            yprint(r, stream=obj.stdout)
-            print("---", file=obj.stdout)
-            if flushing:
-                obj.stdout.flush()
-            seen = True
-
-
-@cli.command()
-@click.option("-i", "--infile", type=click.Path(), help="File to read (msgpack).")
-@click.pass_obj
-async def update(obj, infile):
-    """Send a list of updates to a MoaT-KV subtree"""
-    async with MsgReader(path=infile) as reader:
-        async for msg in reader:
-            if not hasattr(msg, "path"):
-                continue
-            await obj.client.set(obj.path + msg.path, value=msg.value)

moat/kv/command/dump/__init__.py

@@ -1,141 +0,0 @@
-# command line interface
-
-import datetime
-import sys
-from collections.abc import Mapping
-
-import asyncclick as click
-from moat.mqtt.codecs import MsgPackCodec
-from moat.util import (
-    MsgReader,
-    MsgWriter,
-    P,
-    Path,
-    PathLongener,
-    load_subgroup,
-    yload,
-    yprint,
-)
-
-from moat.kv.codec import unpacker
-
-
-@load_subgroup(short_help="Local data mangling", sub_pre="dump")
-async def cli():
-    """
-    Low-level tools that don't depend on a running MoaT-KV server.
-    """
-    pass
-
-
-@cli.command()
-@click.argument("node", nargs=1)
-@click.argument("file", type=click.Path(), nargs=1)
-async def init(node, file):
-    """Write an initial preload file.
-
-    Usage: moat kv dump init <node> <outfile>
-
-    Writes an initial MoaT-KV file that behaves as if it was generated by <node>.
-
-    Using this command, followed by "moat kv server -l <outfile> <node>", is
-    equivalent to running "moat kv server -i 'Initial data' <node>.
-    """
-    async with MsgWriter(path=file) as f:
-        await f(
-            dict(
-                chain=dict(node=node, tick=1, prev=None),
-                depth=0,
-                path=[],
-                tock=1,
-                value="Initial data",
-            )
-        )
-
-
-@cli.command("msg")
-@click.argument("path", nargs=1)
-@click.pass_obj
-async def msg_(obj, path):
-    """
-    Monitor the server-to-sever message stream.
-
-    Use ':' for the main server's "update" stream.
-    Use '+NAME' to monitor a different stream instead.
-    Use '+' to monitor all streams.
-    Otherwise use the given name as-is; Mosquitto wildcard rules apply.
-
-    \b
-    Common streams (prefix with '+'):
-    * ping    all servers
-    * update  data changes
-    * del     nodes responsible for cleaning up deleted records
-    """
-    from moat.kv.backend import get_backend
-
-    class _Unpack:
-        def __init__(self):
-            self._part_cache = dict()
-
-    import moat.kv.server
-
-    _Unpack._unpack_multiple = moat.kv.server.Server._unpack_multiple
-    _unpacker = _Unpack()._unpack_multiple
-
-    path = P(path)
-    if len(path) == 0:
-        path = P(obj.cfg.server["root"]) | "update"
-    elif len(path) == 1 and path[0].startswith("+"):  # pylint: disable=no-member  # owch
-        p = path[0][1:]
-        path = P(obj.cfg.server["root"])
-        path |= p or "#"
-    be = obj.cfg.server.backend
-    kw = obj.cfg.server[be]
-
-    async with get_backend(be)(**kw) as conn:
-        async with conn.monitor(*path) as stream:
-            async for msg in stream:
-                v = vars(msg)
-                if isinstance(v.get("payload"), (bytearray, bytes)):
-                    t = msg.topic
-                    v = unpacker(v["payload"])
-                    v = _unpacker(v)
-                    if v is None:
-                        continue
-                    if not isinstance(v, Mapping):
-                        v = {"_data": v}
-                    v["_topic"] = Path.build(t)
-                else:
-                    v["_type"] = type(msg).__name__
-
-                v["_timestamp"] = datetime.datetime.now().isoformat(
-                    sep=" ", timespec="milliseconds"
-                )
-
-                yprint(v, stream=obj.stdout)
-                print("---", file=obj.stdout)
-
-
-@cli.command("post")
-@click.argument("path", nargs=1)
-@click.pass_obj
-async def post_(obj, path):
-    """
-    Send a msgpack-encoded message (or several) to a specific MQTT topic.
-
-    Messages are read from YAML.
-    Common streams:
-    * ping: sync: all servers (default)
-    * update: data changes
-    * del: sync: nodes responsible for cleaning up deleted records
-    """
-    from moat.kv.backend import get_backend
-
-    path = P(path)
-    be = obj.cfg.server.backend
-    kw = obj.cfg.server[be]
-
-    async with get_backend(be)(codec=MsgPackCodec, **kw) as conn:
-        for d in yload(sys.stdin, multi=True):
-            topic = d.pop("_topic", path)
-            await conn.send(*topic, payload=d)