moat-kv 0.71.0__py3-none-any.whl → 0.71.7__py3-none-any.whl

build/lib/moat/kv/command/internal.py

@@ -1,248 +0,0 @@
-# command line interface
-from __future__ import annotations
-
-from collections.abc import Mapping
-
-import asyncclick as click
-from moat.util import P, PathLongener, yprint
-from range_set import RangeSet
-
-
-@click.group(short_help="Control internal state.")  # pylint: disable=undefined-variable
-async def cli():
-    """
-    This subcommand queries and controls the server's internal state.
-    """
-    pass
-
-
-@cli.command()
-@click.option("-n", "--nodes", is_flag=True, help="Get node status.")
-@click.option("-d", "--deleted", is_flag=True, help="Get deletion status.")
-@click.option("-m", "--missing", is_flag=True, help="Get missing-node status.")
-@click.option(
-    "-r",
-    "--remote-missing",
-    "remote_missing",
-    is_flag=True,
-    help="Get remote-missing-node status.",
-)
-@click.option("-p", "--present", is_flag=True, help="Get known-data status.")
-@click.option("-s", "--superseded", is_flag=True, help="Get superseded-data status.")
-@click.option("-D", "--debug", is_flag=True, help="Get internal verbosity.")
-@click.option("--debugger", is_flag=True, help="Start a remote debugger. DO NOT USE.")
-@click.option("-k", "--known", hidden=True, is_flag=True, help="Get superseded-data status.")
-@click.option("-a", "--all", is_flag=True, help="All available data.")
-@click.pass_obj
-async def state(obj, **flags):
-    """
-    Dump the server's state.
-    """
-    if flags.pop("known", None):
-        flags["superseded"] = True
-    if flags.pop("all", None):
-        flags["superseded"] = True
-        flags["present"] = True
-        flags["nodes"] = True
-        flags["deleted"] = True
-        flags["missing"] = True
-        flags["remote_missing"] = True
-    res = await obj.client._request("get_state", iter=False, **flags)
-    k = res.pop("known", None)
-    if k is not None:
-        res["superseded"] = k
-    yprint(res, stream=obj.stdout)
-
-
-@cli.command()
-@click.option("-d", "--deleted", is_flag=True, help="Mark as deleted. Default: superseded")
-@click.option(
-    "-n",
-    "--node",
-    "source",
-    default="?",
-    help="The node this message is faked as being from.",
-)
-@click.option("-b", "--broadcast", is_flag=True, help="Send to all servers")
-@click.argument("node", nargs=1)
-@click.argument("items", type=int, nargs=-1)
-@click.pass_obj
-async def mark(obj, deleted, source, node, items, broadcast):
-    """
-    Fix internal state. Use no items to fetch the current list from the
-    server's ``missing`` state. Use an empty node name to add the whole
-    list, not just a single node's.
-
-    This is a dangerous command.
-    """
-
-    k = "deleted" if deleted else "superseded"
-    if not items:
-        r = await obj.client._request("get_state", iter=False, missing=True)
-        r = r["missing"]
-        if node != "":
-            r = {node: r[node]}
-    elif node == "":
-        raise click.UsageError("You can't do that with an empty node")
-    else:
-        r = RangeSet()
-        for i in items:
-            r.add(i)
-        r = {node: r.__getstate__()}
-
-    msg = {k: r, "node": source}
-
-    await obj.client._request("fake_info", iter=False, **msg)
-    if broadcast:
-        await obj.client._request("fake_info_send", iter=False, **msg)
-
-    res = await obj.client._request("get_state", iter=False, **{k: True})
-    yprint(res, stream=obj.stdout)
-
-
-@cli.command(short_help="Manage the Deleter list")
-@click.option("-d", "--delete", is_flag=True, help="Remove these nodes")
-@click.argument("nodes", nargs=-1)
-@click.pass_obj
-async def deleter(obj, delete, nodes):
-    """
-    Manage the Deleter list
-
-    This is the set of nodes that must be online for removal of deleted
-    entries from MoaT-KV's data.
-
-    There should be one such node in every possible network partition.
-    Also, all nodes with permanent storage should be on the list.
-
-    Usage:
-    - … deleter          -- list state
-    - … deleter NODE…    -- add this node
-    - … deleter -d NODE… -- remove this node
-    """
-
-    res = await obj.client._request(
-        action="get_internal",
-        path=("actor", "del"),
-        iter=False,
-        nchain=3 if delete or nodes else 2,
-    )
-    val = res.get("value", {})
-    if isinstance(val, Mapping):
-        val = val.get("nodes", [])
-    # else: compatibility, TODO remove
-    val = set(val)
-    if delete:
-        if nodes:
-            val -= set(nodes)
-        else:
-            val = set()
-    elif nodes:
-        val |= set(nodes)
-    else:
-        yprint(res, stream=obj.stdout)
-        return
-
-    val = {"nodes": list(val)}
-    res = await obj.client._request(
-        action="set_internal",
-        path=("actor", "del"),
-        iter=False,
-        chain=res.chain,
-        value=val,
-    )
-    res.value = val
-    yprint(res, stream=obj.stdout)
-
-
-@cli.command()
-@click.argument("path", nargs=1)
-@click.pass_obj
-async def dump(obj, path):
-    """
-    Dump internal state.
-
-    This displays MoaT-KV's internal state.
-    """
-
-    path = P(path)
-    y = {}
-    pl = PathLongener()
-    async for r in await obj.client._request("get_tree_internal", path=path, iter=True, nchain=0):
-        pl(r)
-        path = r["path"]
-        yy = y
-        for p in path:
-            yy = yy.setdefault(p, {})
-        try:
-            yy["_"] = r["value"]
-        except KeyError:
-            pass
-    yprint(y, stream=obj.stdout)
-
-
-@cli.command()
-@click.argument("node", nargs=1)
-@click.argument("tick", type=int, nargs=1)
-@click.pass_obj
-async def get(obj, node, tick):
-    """
-    Fetch data by node+tick.
-
-    This looks up internal data.
-    """
-
-    res = await obj.client._request("get_value", node=node, tick=tick, nchain=99)
-    if not obj.meta:
-        res = res.value
-    yprint(res, stream=obj.stdout)
-
-
-@cli.command()
-@click.option("-n", "--num", type=int, help="Return at most this many IDs")
-@click.option("-c", "--current", is_flag=True, help="Return only IDs with current data")
-@click.option("-C", "--copy", is_flag=True, help="Create an no-op change")
-@click.argument("node", nargs=1)
-@click.pass_obj
-async def enum(obj, node, num, current, copy):
-    """
-    List IDs of live data by a specific node.
-
-    Can be used to determine whether a node still has live data,
-    otherwise it can be deleted.
-
-    If '--current' is set, only the IDs of the entries that have last been
-    updated by that node are shown. '--copy' rewrites these entries.
-
-    Increase verbosity to also show the oject paths.
-    """
-
-    res = await obj.client._request("enum_node", node=node, max=num, current=current)
-    if obj.meta and not copy and obj.debug <= 1:
-        yprint(res, stream=obj.stdout)
-    else:
-        for k in res.result:
-            if copy or obj.debug > 1:
-                res = await obj.client._request("get_value", node=node, tick=k, nchain=3)
-                if obj.debug > 1:
-                    print(k, res.path)
-                else:
-                    print(k)
-                if copy and res.chain.node == node:
-                    res = await obj.client.set(res.path, value=res.value, chain=res.chain)
-            else:
-                print(k)
-
-
-@cli.command()
-@click.argument("node", nargs=1)
-@click.pass_obj
-async def kill(obj, node):
-    """
-    Remove a node from the node list.
-
-    This command only works if this node does not have any current data in
-    the system.
-    """
-    res = await obj.client._request("kill_node", node=node)
-    if obj.meta:
-        yprint(res, stream=obj.stdout)

build/lib/moat/kv/command/job.py

@@ -1,433 +0,0 @@
-# command line interface
-from __future__ import annotations
-
-import sys
-import time
-from functools import partial
-
-import anyio
-import asyncclick as click
-from moat.util import P, Path, attr_args, attrdict, process_args, yprint
-
-from moat.kv.code import CodeRoot
-from moat.kv.data import add_dates, data_get
-from moat.kv.runner import AllRunnerRoot, AnyRunnerRoot, SingleRunnerRoot
-
-
-@click.group()  # pylint: disable=undefined-variable
-@click.option("-n", "--node", help="node to run this code on. Empty: any one node, '-': all nodes")
-@click.option("-g", "--group", help="group to run this code on. Empty: default")
-@click.pass_context
-async def cli(ctx, node, group):
-    """Run code stored in MoaT-KV.
-
-    \b
-    The option '-n' is somewhat special:
-    -n -     Jobs for all hosts
-    -n XXX   Jobs for host XXX
-    (no -n)  Jobs for any host
-
-    The default group is 'default'.
-    """
-    obj = ctx.obj
-    if group is None:
-        group = "default"
-    if group == "-":
-        if node is not None:
-            raise click.UsageError("'-g -' doesn't make sense with '-n'")
-        if ctx.invoked_subcommand != "info":
-            raise click.UsageError("'-g -' only makes sense with the 'info' command")
-        obj.runner_root = SingleRunnerRoot
-        subpath = (None,)
-    elif not node:
-        obj.runner_root = AnyRunnerRoot
-        subpath = (group,)
-    elif node == "-":
-        obj.runner_root = AllRunnerRoot
-        subpath = (group,)
-    else:
-        obj.runner_root = SingleRunnerRoot
-        subpath = (node, group)
-
-    cfg = obj.cfg["kv"]["runner"]
-    obj.subpath = Path(cfg["sub"][obj.runner_root.SUB]) + subpath
-    obj.path = cfg["prefix"] + obj.subpath
-    obj.statepath = cfg["state"] + obj.subpath
-
-
-@cli.group("at", short_help="path of the job to operate on", invoke_without_command=True)
-@click.argument("path", nargs=1, type=P)
-@click.pass_context
-async def at_cli(ctx, path):
-    """
-    Add, list, modify, delete jobs at/under this path.
-    """
-    obj = ctx.obj
-    obj.jobpath = path
-
-    if ctx.invoked_subcommand is None:
-        res = await obj.client.get(obj.path + path, nchain=obj.meta)
-        yprint(
-            res if obj.meta else res.value if "value" in res else None,
-            stream=obj.stdout,
-        )
-
-
-@cli.command("info")
-@click.pass_obj
-async def info_(obj):
-    """
-    List available groups for the node in question.
-
-    \b
-    Options (between 'job' and 'info')
-    (none)    list groups with jobs for any host
-    -n -      list groups with jobs for all hosts
-    -g -      list hosts that have specific jobs
-    -n XXX    list groups with jobs for a specific host
-    """
-    path = obj.path[:-1]
-    async for r in obj.client.get_tree(path=path, min_depth=1, max_depth=1, empty=True):
-        print(r.path[-1], file=obj.stdout)
-
-
-@at_cli.command("--help", hidden=True)
-@click.pass_context
-def help_(ctx):  # pylint:disable=unused-variable  # oh boy
-    print(at_cli.get_help(ctx))
-
-
-@at_cli.command("path")
-@click.pass_obj
-async def path__(obj):
-    """
-    Emit the full path leading to the specified runner object.
-
-    Useful for copying or for state monitoring.
-
-    NEVER directly write to the state object. It's controlled by the
-    runner. You'll confuse it if you do that.
-
-    Updating the control object will cancel any running code.
-    """
-    path = obj.jobpath
-    res = dict(command=obj.path + path, state=obj.statepath + path)
-    yprint(res, stream=obj.stdout)
-
-
-@cli.command("run")
-@click.option(
-    "-n",
-    "--nodes",
-    type=int,
-    default=0,
-    help="Size of the group (not for single-node runners)",
-)
-@click.pass_obj
-async def run(obj, nodes):
-    """
-    Run code that needs to run.
-
-    This does not return.
-    """
-    from moat.util import as_service
-
-    if obj.subpath[-1] == "-":
-        raise click.UsageError("Group '-' can only be used for listing.")
-    if nodes and obj.runner_root is SingleRunnerRoot:
-        raise click.UsageError("A single-site runner doesn't have a size.")
-
-    async with as_service(obj) as evt:
-        c = obj.client
-        cr = await CodeRoot.as_handler(c)
-        await obj.runner_root.as_handler(
-            c,
-            subpath=obj.subpath,
-            code=cr,
-            **({"nodes": nodes} if nodes else {}),
-        )
-        evt.set()
-        await anyio.sleep_forever()
-
-
-async def _state_fix(obj, state, state_only, path, r):
-    try:
-        val = r.value
-    except AttributeError:
-        return
-    if state:
-        rs = await obj.client._request(
-            action="get_value",
-            path=state + r.path,
-            iter=False,
-            nchain=obj.meta,
-        )
-        if state_only:
-            r.value = rs
-        else:
-            if obj.meta:
-                val["state"] = rs
-            elif "value" in rs:
-                val["state"] = rs.value
-        if "value" in rs:
-            add_dates(rs.value)
-    if not state_only:
-        if path:
-            r.path = path + r.path
-        add_dates(val)
-
-    return r
-
-
-@at_cli.command("list")
-@click.option("-s", "--state", is_flag=True, help="Add state data")
-@click.option("-S", "--state-only", is_flag=True, help="Only output state data")
-@click.option("-t", "--table", is_flag=True, help="one-line output")
-@click.option(
-    "-d",
-    "--as-dict",
-    default=None,
-    help="Structure as dictionary. The argument is the key to use "
-    "for values. Default: return as list",
-)
-@click.pass_obj
-async def list_(obj, state, state_only, table, as_dict):
-    """List run entries."""
-    if table and state:
-        raise click.UsageError("'--table' and '--state' are mutually exclusive")
-
-    path = obj.jobpath
-
-    if state or state_only or table:
-        state = obj.statepath + path
-
-    if table:
-        from moat.kv.errors import ErrorRoot
-
-        err = await ErrorRoot.as_handler(obj.client)
-
-        async for r in obj.client.get_tree(obj.path + path):
-            p = path + r.path
-            s = await obj.client.get(state + r.path)
-            if "value" not in s:
-                st = "-never-"
-            elif s.value.started > s.value.stopped:
-                st = s.value.node
-            else:
-                try:
-                    e = await err.get_error_record("run", obj.path + p, create=False)
-                except KeyError:
-                    st = "-stopped-"
-                else:
-                    if e is None or e.resolved:
-                        st = "-stopped-"
-                    else:
-                        st = " | ".join(
-                            "%s %s"
-                            % (
-                                Path.build(e.subpath)
-                                if e._path[-2] == ee._path[-1]
-                                else Path.build(ee.subpath),
-                                getattr(ee, "message", None)
-                                or getattr(ee, "comment", None)
-                                or "-",
-                            )
-                            for ee in e
-                        )
-            print(p, r.value.code, st, file=obj.stdout)
-
-    else:
-        await data_get(
-            obj,
-            obj.path + path,
-            as_dict=as_dict,
-            item_mangle=partial(_state_fix, obj, state, state_only, None if as_dict else path),
-        )
-
-
-@at_cli.command("state")
-@click.option("-r", "--result", is_flag=True, help="Just print the actual result.")
-@click.pass_obj
-async def state_(obj, result):
-    """Get the status of a runner entry."""
-    path = obj.jobpath
-
-    if obj.subpath[-1] == "-":
-        raise click.UsageError("Group '-' can only be used for listing.")
-    if result and obj.meta:
-        raise click.UsageError("You can't use '-v' and '-r' at the same time.")
-    if not len(path):
-        raise click.UsageError("You need a non-empty path.")
-    path = obj.statepath + obj.jobpath
-
-    res = await obj.client.get(path, nchain=obj.meta)
-    if "value" not in res:
-        if obj.debug:
-            print("Not found (yet?)", file=sys.stderr)
-        sys.exit(1)
-
-    add_dates(res.value)
-    if not obj.meta:
-        res = res.value
-    yprint(res, stream=obj.stdout)
-
-
-@at_cli.command()
-@click.option("-s", "--state", is_flag=True, help="Add state data")
-@click.pass_obj
-async def get(obj, state):
-    """Read a runner entry"""
-    path = obj.jobpath
-    if obj.subpath[-1] == "-":
-        raise click.UsageError("Group '-' can only be used for listing.")
-    if not path:
-        raise click.UsageError("You need a non-empty path.")
-
-    res = await obj.client._request(
-        action="get_value",
-        path=obj.path + path,
-        iter=False,
-        nchain=obj.meta,
-    )
-    if "value" not in res:
-        print("Not found.", file=sys.stderr)
-        return
-    res.path = path
-    if state:
-        state = obj.statepath
-    await _state_fix(obj, state, False, path, res)
-    if not obj.meta:
-        res = res.value
-
-    yprint(res, stream=obj.stdout)
-
-
-@at_cli.command()
-@click.option("-f", "--force", is_flag=True, help="Force deletion even if messy")
-@click.pass_obj
-async def delete(obj, force):
-    """Remove a runner entry"""
-    path = obj.jobpath
-
-    if obj.subpath[-1] == "-":
-        raise click.UsageError("Group '-' can only be used for listing.")
-    if not path:
-        raise click.UsageError("You need a non-empty path.")
-
-    res = await obj.client.get(obj.path + path, nchain=3)
-    if "value" not in res:
-        res.info = "Does not exist."
-    else:
-        val = res.value
-        if "target" not in val:
-            val.target = None
-        if val.target is not None:
-            val.target = None
-            res = await obj.client.set(obj.path + path, value=val, nchain=3, chain=res.chain)
-            if not force:
-                res.info = "'target' was set: cleared but not deleted."
-        if force or val.target is None:
-            sres = await obj.client.get(obj.statepath + path, nchain=3)
-            if not force and "value" in sres and sres.value.stopped < sres.value.started:
-                res.info = "Still running, not deleted."
-            else:
-                sres = await obj.client.delete(obj.statepath + path, chain=sres.chain)
-                res = await obj.client.delete(obj.path + path, chain=res.chain)
-                if "value" in res and res.value.stopped < res.value.started:
-                    res.info = "Deleted (unclean!)."
-                else:
-                    res.info = "Deleted."
-
-    if obj.meta:
-        yprint(res, stream=obj.stdout)
-    elif obj.debug:
-        print(res.info)
-
-
-@at_cli.command("set")
-@click.option("-c", "--code", help="Path to the code that should run.")
-@click.option("-C", "--copy", help="Use this entry as a template.")
-@click.option("-t", "--time", "tm", help="time the code should next run at. '-':not")
-@click.option("-r", "--repeat", type=int, help="Seconds the code should re-run after")
-@click.option("-k", "--ok", type=float, help="Code is OK if it ran this many seconds")
-@click.option("-b", "--backoff", type=float, help="Back-off factor. Default: 1.4")
-@click.option("-d", "--delay", type=int, help="Seconds the code should retry after (w/ backoff)")
-@click.option("-i", "--info", help="Short human-readable information")
-@attr_args
-@click.pass_obj
-async def set_(obj, code, tm, info, ok, repeat, delay, backoff, copy, **kw):
-    """Add or modify a runner.
-
-    Code typically requires some input parameters.
-
-    You should use '-v NAME VALUE' for string values, '-p NAME VALUE' for
-    paths, and '-e NAME VALUE' for other data. '-e NAME -' deletes an item.
-    """
-    path = obj.jobpath
-
-    if obj.subpath[-1] == "-":
-        raise click.UsageError("Group '-' can only be used for listing.")
-
-    if code is not None:
-        code = P(code)
-    if copy:
-        copy = P(copy)
-    path = obj.path + P(path)
-
-    res = await obj.client._request(action="get_value", path=copy or path, iter=False, nchain=3)
-    if "value" not in res:
-        if copy:
-            raise click.UsageError("--copy: use the complete path to an existing entry")
-        elif code is None:
-            raise click.UsageError("New entry, need code")
-        res = {}
-        chain = None
-    else:
-        chain = None if copy else res["chain"]
-        res = res["value"]
-        if copy and "code" not in res:
-            raise click.UsageError("'--copy' needs a runner entry")
-
-    vl = attrdict(**res.setdefault("data", {}))
-    vl = process_args(vl, **kw)
-    res["data"] = vl
-
-    if code is not None:
-        res["code"] = code
-    if ok is not None:
-        res["ok_after"] = ok
-    if info is not None:
-        res["info"] = info
-    if backoff is not None:
-        res["backoff"] = backoff
-    if delay is not None:
-        res["delay"] = delay
-    if repeat is not None:
-        res["repeat"] = repeat
-    if tm is not None:
-        if tm == "-":
-            res["target"] = None
-        else:
-            res["target"] = time.time() + float(tm)
-
-    res = await obj.client.set(path, value=res, nchain=3, chain=chain)
-    if obj.meta:
-        yprint(res, stream=obj.stdout)
-
-
-@cli.command(short_help="Show runners' keepalive messages")
-@click.pass_obj
-async def monitor(obj):
-    """
-    Runners periodically send a keepalive message. Show them.
-    """
-
-    # TODO this does not watch changes in MoaT-KV.
-    # It also should watch individual jobs' state changes.
-    if obj.subpath[-1] == "-":
-        raise click.UsageError("Group '-' can only be used for listing.")
-
-    async with obj.client.msg_monitor("run") as cl:
-        async for msg in cl:
-            yprint(msg, stream=obj.stdout)
-            print("---", file=obj.stdout)