moat-kv 0.70.23__py3-none-any.whl → 0.70.24__py3-none-any.whl

moat/kv/command/type.py

@@ -0,0 +1,201 @@
+# command line interface
+from __future__ import annotations
+
+import json
+
+import asyncclick as click
+from moat.util import NotGiven, P, Path, PathLongener, yload, yprint
+
+
+@click.group()  # pylint: disable=undefined-variable
+async def cli():
+    """Manage types and type matches. Usage: … type …"""
+    pass
+
+
+@cli.command()
+@click.option("-s", "--script", type=click.File(mode="w", lazy=True), help="Save the script here")
+@click.option("-S", "--schema", type=click.File(mode="w", lazy=True), help="Save the schema here")
+@click.option("-y", "--yaml", "yaml_", is_flag=True, help="Write schema as YAML. Default: JSON.")
+@click.argument("path", type=P, nargs=1)
+@click.pass_obj
+async def get(obj, path, script, schema, yaml_):
+    """Read type checker information"""
+    if not len(path):
+        raise click.UsageError("You need a non-empty path.")
+    res = await obj.client._request(
+        action="get_internal",
+        path=Path("type") + path,
+        iter=False,
+        nchain=obj.meta,
+    )
+    try:
+        r = res.value
+    except AttributeError:
+        raise click.UsageError(f"No data at {Path('type') + path}") from None
+
+    if not obj.meta:
+        res = res.value
+    if script:
+        script.write(r.pop("code"))
+    if schema:
+        if yaml_:
+            yprint(r.pop("schema"), stream=schema)
+        else:
+            json.dump(r.pop("schema"), schema)
+    yprint(res, stream=obj.stdout)
+
+
+@cli.command("set")
+@click.option("-g", "--good", multiple=True, help="Example for passing values")
+@click.option("-b", "--bad", multiple=True, help="Example for failing values")
+@click.option("-d", "--data", type=click.File(mode="r"), help="Load metadata from this YAML file.")
+@click.option("-s", "--script", type=click.File(mode="r"), help="File with the checking script")
+@click.option("-S", "--schema", type=click.File(mode="r"), help="File with the JSON schema")
+@click.option("-y", "--yaml", "yaml_", is_flag=True, help="load the schema as YAML. Default: JSON")
+@click.argument("path", type=P, nargs=1)
+@click.pass_obj
+async def set_(obj, path, good, bad, script, schema, yaml_, data):
+    """Write type checker information."""
+    if not len(path):
+        raise click.UsageError("You need a non-empty path.")
+
+    if data:
+        msg = yload(data)
+    else:
+        msg = {}
+    chain = NotGiven
+    if "value" in msg:
+        chain = msg.get("chain", NotGiven)
+        msg = msg["value"]
+
+    msg.setdefault("good", [])
+    msg.setdefault("bad", [])
+    for x in good:
+        msg["good"].append(eval(x))  # pylint: disable=eval-used
+    for x in bad:
+        msg["bad"].append(eval(x))  # pylint: disable=eval-used
+
+    if "code" in msg:
+        if script:
+            raise click.UsageError("Duplicate script")
+    elif script:
+        msg["code"] = script.read()
+
+    if "schema" in msg:
+        raise click.UsageError("Missing schema")
+    elif schema:
+        if yaml_:
+            msg["schema"] = yload(schema)
+        else:
+            msg["schema"] = json.load(schema)
+
+    if "schema" not in msg and "code" not in msg:
+        raise click.UsageError("I need a schema, Python code, or both.")
+
+    if len(msg["good"]) < 2:
+        raise click.UsageError("Missing known-good test values (at least two)")
+    if not msg["bad"]:
+        raise click.UsageError("Missing known-bad test values")
+
+    res = await obj.client._request(
+        action="set_internal",
+        value=msg,
+        path=Path("type") + path,
+        iter=False,
+        nchain=obj.meta,
+        **({} if chain is NotGiven else {"chain": chain}),
+    )
+    if obj.meta:
+        yprint(res, stream=obj.stdout)
+
+
+@cli.command()
+@click.option("-R", "--raw", is_flag=True, help="Print just the path.")
+@click.option("-t", "--type", "type_", help="Type path to link to.")
+@click.option("-d", "--delete", help="Use to delete this mapping.")
+@click.argument("path", type=P, nargs=1)
+@click.pass_obj
+async def match(obj, path, type_, delete, raw):  # pylint: disable=redefined-builtin
+    """Match a type to a path (read, if no type given; list if empty path)"""
+    if not len(path):
+        if raw or type_ or delete:
+            raise click.UsageError("No options allowed when dumping the match tree..")
+        y = {}
+        pl = PathLongener()
+        async for r in await obj.client._request(
+            "get_tree_internal",
+            path=Path("match") + path,
+            iter=True,
+            nchain=0,
+        ):
+            pl(r)
+            path = r["path"]
+            yy = y
+            for p in path:
+                yy = yy.setdefault(p, {})
+            try:
+                yy["_"] = r["value"]
+            except KeyError:
+                pass
+        yprint(y, stream=obj.stdout)
+        return
+
+    if type_ and delete:
+        raise click.UsageError("You can't both set and delete a path.")
+    if raw and (type_ or delete):
+        raise click.UsageError("You can only print the raw path when reading a match.")
+
+    if delete:
+        res = await obj.client._request(action="delete_internal", path=Path("type") + path)
+        if obj.meta:
+            yprint(res, stream=obj.stdout)
+        return
+
+    msg = {}
+    if type_:
+        msg["type"] = P(type_)
+        act = "set_internal"
+    elif delete:
+        act = "delete_internal"
+    else:
+        act = "get_internal"
+    res = await obj.client._request(
+        action=act,
+        value=msg,
+        path=Path("match") + path,
+        iter=False,
+        nchain=obj.meta,
+    )
+    if obj.meta:
+        yprint(res, stream=obj.stdout)
+    elif type_ or delete:
+        pass
+    else:
+        print(" ".join(str(x) for x in res.type), file=obj.stdout)
+
+
+@cli.command()
+@click.argument("path", type=P, nargs=1)
+@click.pass_obj
+async def list(obj, path):  # pylint: disable=redefined-builtin
+    """Dump type data"""
+
+    y = {}
+    pl = PathLongener()
+    async for r in await obj.client._request(
+        "get_tree_internal",
+        path=Path("type") + path,
+        iter=True,
+        nchain=0,
+    ):
+        pl(r)
+        path = r["path"]
+        yy = y
+        for p in path:
+            yy = yy.setdefault(p, {})
+        try:
+            yy["_"] = r["value"]
+        except KeyError:
+            pass
+    yprint(y, stream=obj.stdout)

moat/kv/mock/__init__.py

@@ -0,0 +1,101 @@
+# from asyncclick.testing import CliRunner
+from __future__ import annotations
+import io
+import logging
+import shlex
+import socket
+import sys
+from pathlib import Path
+
+import attr
+from asyncscope import main_scope, scope
+from moat.src.test import run  # pylint:disable=import-error,no-name-in-module
+from moat.util import (  # pylint:disable=no-name-in-module
+    CFG,
+    OptCtx,
+    attrdict,
+    combine_dict,
+    ensure_cfg,
+    list_ext,
+    load_ext,
+    wrap_main,
+    yload,
+)
+
+from moat.kv.client import _scoped_client, client_scope
+
+logger = logging.getLogger(__name__)
+try:
+    from contextlib import asynccontextmanager
+except ImportError:
+    from async_generator import asynccontextmanager
+
+ensure_cfg("moat.kv")
+
+
+@attr.s
+class S:
+    tg = attr.ib()
+    client_ctx = attr.ib()
+    s = attr.ib(factory=list)  # servers
+    c = attr.ib(factory=list)  # clients
+    _seq = 1
+
+    async def ready(self, i=None):
+        if i is not None:
+            await self.s[i].is_ready
+            return self.s[i]
+        for s in self.s:
+            if s is not None:
+                await s.is_ready
+        return self.s
+
+    def __iter__(self):
+        return iter(self.s)
+
+    @asynccontextmanager
+    async def client(self, i: int = 0, **kv):
+        """Get a (new) client for the i'th server."""
+        await self.s[i].is_serving
+        self._seq += 1
+        for host, port, *_ in self.s[i].ports:
+            if host != "::" and host[0] == ":":
+                continue
+            try:
+                cfg = combine_dict(
+                    dict(conn=dict(host=host, port=port, ssl=self.client_ctx, **kv)),
+                    CFG["kv"],
+                )
+
+                async def scc(s, **cfg):
+                    scope.requires(s._scope)
+                    return await _scoped_client(scope.name, **cfg)
+
+                async with scope.using_scope():
+                    c = await scope.service(
+                        f"moat.kv.client.{i}.{self._seq}",
+                        scc,
+                        self.s[i],
+                        **cfg,
+                    )
+                    yield c
+                return
+            except socket.gaierror:
+                pass
+        raise RuntimeError("Duh? no connection")
+
+    async def run(self, *args, do_stdout=True):
+        h = p = None
+        for s in self.s:
+            for h, p, *_ in s.ports:
+                if h[0] != ":":
+                    break
+            else:
+                continue
+            break
+        if len(args) == 1:
+            args = args[0]
+            if isinstance(args, str):
+                args = args.split(" ")
+        async with scope.using_scope():
+            return await run("-VV", "kv", "-h", h, "-p", p, *args, do_stdout=do_stdout)

moat/kv/mock/mqtt.py

@@ -0,0 +1,159 @@
+from __future__ import annotations
+import copy
+import logging
+import os
+import time
+from contextlib import AsyncExitStack, asynccontextmanager
+from functools import partial
+
+import anyio
+from unittest import mock
+import trio
+from asyncscope import main_scope, scope
+from moat.mqtt.broker import create_broker
+from moat.util import NotGiven, attrdict, combine_dict
+
+from moat.kv.mock import S
+from moat.kv.server import Server
+
+from . import CFG
+
+logger = logging.getLogger(__name__)
+
+otm = time.time
+
+PORT = 40000 + (os.getpid() + 10) % 10000
+
+broker_cfg = {
+    "listeners": {"default": {"type": "tcp", "bind": f"127.0.0.1:{PORT}"}},
+    "timeout-disconnect-delay": 2,
+    "auth": {"allow-anonymous": True, "password-file": None},
+}
+
+URI = f"mqtt://127.0.0.1:{PORT}/"
+
+
+@asynccontextmanager
+async def stdtest(n=1, run=True, ssl=False, tocks=20, **kw):
+    C_OUT = CFG.get("_stdout", NotGiven)
+    if C_OUT is not NotGiven:
+        del CFG["_stdout"]
+    TESTCFG = copy.deepcopy(CFG["kv"])
+    TESTCFG.server.port = None
+    TESTCFG.root = "test"
+    if C_OUT is not NotGiven:
+        CFG["_stdout"] = C_OUT
+        TESTCFG["_stdout"] = C_OUT
+
+    if ssl:
+        import ssl
+
+        import trustme
+
+        ca = trustme.CA()
+        cert = ca.issue_server_cert("127.0.0.1")
+        server_ctx = ssl.create_default_context(purpose=ssl.Purpose.CLIENT_AUTH)
+        client_ctx = ssl.create_default_context(purpose=ssl.Purpose.SERVER_AUTH)
+        ca.configure_trust(client_ctx)
+        cert.configure_cert(server_ctx)
+    else:
+        server_ctx = client_ctx = False
+
+    clock = trio.lowlevel.current_clock()
+    try:
+        clock.autojump_threshold = 0.02  # networking
+    except Exception:
+        pass  # test doesn't have autojump_clock fixture
+
+    async def mock_get_host_port(st, host):
+        i = int(host[host.rindex("_") + 1 :])
+        s = st.s[i]
+        await s.is_serving
+        for host, port, *_ in s.ports:
+            if host == "::" or host[0] != ":":
+                return host, port
+
+    def tm():
+        try:
+            return trio.current_time()
+        except RuntimeError:
+            return otm()
+
+    async def mock_set_tock(self, old):
+        assert self._tock < tocks, "Test didn't terminate. Limit:" + str(tocks)
+        await old()
+
+    done = False
+    async with main_scope("moat.kv.test.mqtt") as scp:
+        tg = scp._tg
+        st = S(tg, client_ctx)
+        async with AsyncExitStack() as ex:
+            st.ex = ex  # pylint: disable=attribute-defined-outside-init
+            ex.enter_context(mock.patch("time.time", new=tm))
+            ex.enter_context(mock.patch("time.monotonic", new=tm))
+            logging._startTime = tm()
+
+            async def run_broker(cfg):
+                async with create_broker(config=cfg) as srv:
+                    # NB: some services use "async with await …"
+                    scope.register(srv)
+                    await scope.no_more_dependents()
+
+            async def with_broker(s, *a, **k):
+                await scope.service("moat.mqtt.broker", run_broker, broker_cfg)
+                s._scope = scope.get()
+                return await s._scoped_serve(*a, **k)
+
+            args_def = kw.get("args", attrdict())
+            for i in range(n):
+                name = "test_" + str(i)
+                args = kw.get(name, args_def)
+                args = combine_dict(
+                    args,
+                    args_def,
+                    {
+                        "cfg": {
+                            "conn": {"ssl": client_ctx},
+                            "server": {
+                                "bind_default": {
+                                    "host": "127.0.0.1",
+                                    "port": i + PORT + 1,
+                                    "ssl": server_ctx,
+                                },
+                                "backend": "mqtt",
+                                "mqtt": {"uri": URI},
+                            },
+                        },
+                    },
+                    {"cfg": TESTCFG},
+                )
+                args_def.pop("init", None)
+                s = Server(name, **args)
+                ex.enter_context(
+                    mock.patch.object(s, "_set_tock", new=partial(mock_set_tock, s, s._set_tock)),
+                )
+                ex.enter_context(
+                    mock.patch.object(s, "_get_host_port", new=partial(mock_get_host_port, st)),
+                )
+                st.s.append(s)
+
+            evts = []
+            for i in range(n):
+                if kw.get(f"run_{i}", run):
+                    evt = anyio.Event()
+                    await scp.spawn_service(with_broker, st.s[i], ready_evt=evt)
+                    evts.append(evt)
+                else:
+                    setattr(st, f"run_{i}", partial(scp.spawn_service, with_broker, st.s[i]))
+
+            for e in evts:
+                await e.wait()
+            try:
+                done = True
+                yield st
+            finally:
+                with anyio.fail_after(2, shield=True):
+                    logger.info("Runtime: %s", clock.current_time())
+                    tg.cancel_scope.cancel()
+    if not done:
+        yield None

moat/kv/mock/serf.py

@@ -0,0 +1,250 @@
+from __future__ import annotations
+import copy
+import logging
+import time
+from contextlib import AsyncExitStack, asynccontextmanager
+from functools import partial
+
+import anyio
+import attr
+from unittest import mock
+import trio
+from asyncscope import main_scope, scope
+from asyncserf.stream import SerfEvent
+from moat.util import NotGiven, ValueEvent, attrdict, combine_dict, create_queue
+
+from moat.kv.codec import unpacker
+from moat.kv.mock import S as _S
+from moat.kv.server import Server
+
+logger = logging.getLogger(__name__)
+
+otm = time.time
+
+from . import CFG
+
+
+@asynccontextmanager
+async def stdtest(n=1, run=True, ssl=False, tocks=20, **kw):
+    C_OUT = CFG.get("_stdout", NotGiven)
+    if C_OUT is not NotGiven:
+        del CFG["_stdout"]
+    TESTCFG = copy.deepcopy(CFG["kv"])
+    TESTCFG.server.port = None
+    TESTCFG.server.backend = "serf"
+    TESTCFG.root = "test"
+    if C_OUT is not NotGiven:
+        CFG["_stdout"] = C_OUT
+        TESTCFG["_stdout"] = C_OUT
+
+    if ssl:
+        import ssl
+
+        import trustme
+
+        ca = trustme.CA()
+        cert = ca.issue_server_cert("127.0.0.1")
+        server_ctx = ssl.create_default_context(purpose=ssl.Purpose.CLIENT_AUTH)
+        client_ctx = ssl.create_default_context(purpose=ssl.Purpose.SERVER_AUTH)
+        ca.configure_trust(client_ctx)
+        cert.configure_cert(server_ctx)
+    else:
+        server_ctx = client_ctx = False
+
+    clock = trio.lowlevel.current_clock()
+    clock.autojump_threshold = 0.0
+    # clock.rate = 5
+
+    @attr.s
+    class S(_S):
+        splits = attr.ib(factory=set)
+        serfs = attr.ib(factory=set)
+
+        def split(self, s):
+            assert s not in self.splits
+            logger.debug("Split: add %d", s)
+            self.splits.add(s)
+
+        def join(self, s):
+            logger.debug("Split: join %d", s)
+            self.splits.remove(s)
+
+    async def mock_get_host_port(st, host):
+        i = int(host[host.rindex("_") + 1 :])
+        s = st.s[i]
+        await s.is_serving
+        for host, port, *_ in s.ports:
+            if host == "::" or host[0] != ":":
+                return host, port
+
+    def tm():
+        try:
+            return trio.current_time()
+        except RuntimeError:
+            return otm()
+
+    async def mock_set_tock(self, old):
+        assert self._tock < tocks, "Test didn't terminate. Limit:" + str(tocks)
+        await old()
+
+    async with main_scope("moat.kv.test.serf") as scp:
+        tg = scp._tg
+        st = S(tg, client_ctx)
+        async with AsyncExitStack() as ex:
+            st.ex = ex  # pylint: disable=attribute-defined-outside-init
+            ex.enter_context(mock.patch("time.time", new=tm))
+            ex.enter_context(mock.patch("time.monotonic", new=tm))
+            logging._startTime = tm()
+
+            ex.enter_context(
+                mock.patch("asyncserf.serf_client", new=partial(mock_serf_client, st)),
+            )
+
+            for i in range(n):
+                name = "test_" + str(i)
+                args = kw.get(name, kw.get("args", attrdict()))
+                args["cfg"] = combine_dict(
+                    args.get("cfg", {}),
+                    {
+                        "kv": {
+                            "conn": {"ssl": client_ctx},
+                        },
+                        "server": {
+                            "bind_default": {
+                                "host": "127.0.0.1",
+                                "port": i + 50120,
+                                "ssl": server_ctx,
+                            },
+                            "serf": {"i": i},
+                        },
+                    },
+                    TESTCFG,
+                )
+                s = Server(name, **args)
+                ex.enter_context(
+                    mock.patch.object(s, "_set_tock", new=partial(mock_set_tock, s, s._set_tock)),
+                )
+                ex.enter_context(
+                    mock.patch.object(s, "_get_host_port", new=partial(mock_get_host_port, st)),
+                )
+                st.s.append(s)
+
+            async def with_serf(s, *a, **k):
+                s._scope = scope.get()
+                return await s._scoped_serve(*a, **k)
+
+            evts = []
+            for i in range(n):
+                if kw.get("run_" + str(i), run):
+                    evt = anyio.Event()
+                    await scp.spawn_service(with_serf, st.s[i], ready_evt=evt)
+                    evts.append(evt)
+            for e in evts:
+                await e.wait()
+            try:
+                yield st
+            finally:
+                with anyio.fail_after(2, shield=True):
+                    logger.info("Runtime: %s", clock.current_time())
+                    tg.cancel_scope.cancel()
+        logger.info("End")
+        pass  # unwinding ex:AsyncExitStack
+
+
+@asynccontextmanager
+async def mock_serf_client(master, **cfg):
+    async with scope.using_scope():
+        ms = MockServ(master, **cfg)
+        master.serfs.add(ms)
+        ms._scope = scope.get()  # pylint:disable=attribute-defined-outside-init
+        try:
+            yield ms
+        finally:
+            master.serfs.remove(ms)
+        pass  # terminating mock_serf_client nursery
+
+
+class MockServ:
+    def __init__(self, master, **cfg):
+        self.cfg = cfg
+        self._tg = scope._tg
+        self.streams = {}
+        self._master = master
+
+    def __hash__(self):
+        return id(self)
+
+    async def spawn(self, fn, *args, **kw):
+        async def run(evt):
+            with anyio.CancelScope() as sc:
+                await evt.set(sc)
+                await fn(*args, **kw)
+
+        evt = ValueEvent()
+        self._tg.spawn(run, evt)
+        return await evt.get()
+
+    async def event(self, name, payload, coalesce=True):
+        try:
+            logger.debug("SERF:%s: %r", name, unpacker(payload))
+        except Exception:
+            logger.debug("SERF:%s: %r (raw)", name, payload)
+        assert not coalesce, "'coalesce' must be cleared!"
+
+        i_self = self.cfg.get("i", 0)
+        for s in list(self._master.serfs):
+            i_s = s.cfg.get("i", 0)
+            for x in self._master.splits:
+                if (i_s < x) != (i_self < x):
+                    break
+            else:
+                n = tuple(name.split("."))
+                while n:
+                    sl = s.streams.get(n, ())
+                    for sn in sl:
+                        await sn.q.put((name, payload))
+                    n = n[:-1]
+
+    def stream(self, typ):
+        """compat for supporting asyncactor"""
+        if not typ.startswith("user:"):
+            raise RuntimeError("not supported")
+        typ = typ[5:]
+        return self.serf_mon(typ)
+
+    def serf_mon(self, typ):
+        if "," in typ:
+            raise RuntimeError("not supported")
+        s = MockSerfStream(self, "user:" + typ)
+        return s
+
+    async def serf_send(self, typ, payload):
+        """compat for supporting asyncactor"""
+        return await self.event(typ, payload)
+
+
+class MockSerfStream:
+    q = None
+
+    def __init__(self, serf, typ):
+        self.serf = serf
+        assert typ.startswith("user:")
+        self.typ = tuple(typ[5:].split("."))
+
+    async def __aenter__(self):
+        self.q = create_queue(100)
+        self.serf.streams.setdefault(self.typ, []).append(self)
+        return self
+
+    async def __aexit__(self, *tb):
+        self.serf.streams[self.typ].remove(self)
+        del self.q
+
+    def __aiter__(self):
+        return self
+
+    async def __anext__(self):
+        res = await self.q.get()
+        evt = SerfEvent(self)
+        evt.topic, evt.payload = res
+        return evt