moat-kv 0.70.24__py3-none-any.whl → 0.71.0__py3-none-any.whl

scripts/run

@@ -0,0 +1,29 @@
+#!/bin/bash
+set -ex
+
+# This script starts up MoaT-KV.
+
+# Normally, starting MoaT-KV on a host where clients don't connect locally is
+# a bad idea. However, on a system that has more than one network namespace
+# you need to use a specific address. So set LOCAL=yes to force starting
+# anyway.
+
+if test "$LOCAL" != "yes" && test "$(moat util cfg kv.conn.host)" != "127.0.0.1"; then
+    echo "Server is not localhost: not starting."
+    exit 42  # mark as succeeded
+fi
+
+if test "$MODE" = "master" ; then
+    /usr/lib/moat/kv/current "$TEMP"
+fi
+
+if test ! -v NAME ; then
+    echo "NAME is not set. Exiting." >&2
+    exit 1
+fi
+
+if test -v TEMP && test -s "$TEMP" ; then
+    exec moat -c /etc/moat/moat.cfg kv server -a -l "$TEMP" "$NAME"
+else
+    exec moat -c /etc/moat/moat.cfg kv server "$NAME"
+fi

scripts/run-all

@@ -0,0 +1,10 @@
+#!/bin/bash
+set -ex
+
+# This script file starts up a MoaT-KV any-node runner.
+
+if test $# = 1 ; then
+    exec moat -c /etc/moat/moat.cfg kv job -n - -g "$1" run
+else
+    exec moat -c /etc/moat/moat.cfg kv job -n - run
+fi

scripts/run-any

@@ -0,0 +1,10 @@
+#!/bin/bash
+set -ex
+
+# This script file starts up a MoaT-KV any-node runner.
+
+if test $# = 1 ; then
+    exec moat -c /etc/moat/moat.cfg kv job -g "$1" run
+else
+    exec moat -c /etc/moat/moat.cfg kv job run
+fi

scripts/run-single

@@ -0,0 +1,15 @@
+#!/bin/bash
+set -ex
+
+# This script file starts up a MoaT-KV single-node runner.
+
+if test ! -v NAME ; then
+    echo "NAME is not set. Exiting." >&2
+    exit 1
+fi
+
+if test $# = 1 ; then
+    exec moat -c /etc/moat/moat.cfg kv job -n "$NAME" -g "$1" run
+else
+    exec moat -c /etc/moat/moat.cfg kv job -n "$NAME" run
+fi

scripts/success

@@ -0,0 +1,4 @@
+#!/bin/sh
+set -ex
+
+rm -f "$TEMP"

systemd/moat-kv-recover.service

@@ -0,0 +1,21 @@
+[Unit]
+Description=MoaT-KV state recovery
+Conflicts=moat-kv.service
+Conflicts=distkv.service
+
+AssertFileNotEmpty=/etc/moat/moat.cfg
+
+[Install]
+
+[Service]
+Type=oneshot
+
+ExecStart=/usr/lib/moat/kv/recover
+ExecStartPost=!/bin/systemctl restart moat-kv.service
+
+EnvironmentFile=/usr/lib/moat/kv/env
+EnvironmentFile=-/etc/moat/kv.env
+
+User=MoaT-KV
+Group=MoaT
+

systemd/moat-kv-rotate.service

@@ -0,0 +1,20 @@
+[Unit]
+Description=MoaT-KV log rotate
+Requires=moat-kv.service
+After=moat-kv.service
+Conflicts=distkv-rotate.service
+
+ConditionFileNotEmpty=/etc/moat/moat.cfg
+ConditionPathIsDirectory=/var/lib/moat/kv
+
+[Install]
+
+[Service]
+ExecStart=/usr/lib/moat/kv/rotate
+
+EnvironmentFile=/usr/lib/moat/kv/env
+EnvironmentFile=-/etc/moat/kv.env
+
+User=MoaT-KV
+Group=MoaT
+

systemd/moat-kv-rotate.timer

@@ -0,0 +1,10 @@
+[Unit]
+Description=The MoaT-KV server state save timer
+Requires=moat-kv.service
+
+[Timer]
+OnUnitActiveSec=900
+OnActiveSec=100
+
+[Install]
+WantedBy=moat-kv.service

systemd/moat-kv-run-all.service

@@ -0,0 +1,26 @@
+[Unit]
+Description=MoaT-KV default all-nodes runner
+After=moat-kv.service
+Requires=moat-kv.service
+Conflicts=distkv-run-all.service
+
+ConditionFileNotEmpty=/etc/moat/moat.cfg
+
+[Install]
+WantedBy=multi-user.target
+
+[Service]
+Type=notify
+ExecStart=/usr/lib/moat/kv/run-all
+
+User=MoaT-KV
+Group=MoaT
+
+EnvironmentFile=/usr/lib/moat/kv/env
+EnvironmentFile=-/etc/moat/kv.env
+
+TimeoutSec=300
+WatchdogSec=10
+
+Restart=always
+RestartSec=10

systemd/moat-kv-run-all@.service

@@ -0,0 +1,25 @@
+[Unit]
+Description=MoaT-KV all-nodes runner for %i
+After=moat-kv.service
+Requires=moat-kv.service
+
+ConditionFileNotEmpty=/etc/moat/moat.cfg
+
+[Install]
+WantedBy=multi-user.target
+
+[Service]
+Type=notify
+ExecStart=/usr/lib/moat/kv/run-all %i
+
+User=MoaT-KV
+Group=MoaT
+
+EnvironmentFile=/usr/lib/moat/kv/env
+EnvironmentFile=-/etc/moat/kv.env
+
+TimeoutSec=300
+WatchdogSec=10
+
+Restart=always
+RestartSec=10

systemd/moat-kv-run-any.service

@@ -0,0 +1,26 @@
+[Unit]
+Description=MoaT-KV default any-nodes runner
+After=moat-kv.service
+Requires=moat-kv.service
+Conflicts=distkv-run-any.service
+
+ConditionFileNotEmpty=/etc/moat/moat.cfg
+
+[Install]
+WantedBy=multi-user.target
+
+[Service]
+Type=notify
+ExecStart=/usr/lib/moat/kv/run-any
+
+User=MoaT-KV
+Group=MoaT
+
+EnvironmentFile=/usr/lib/moat/kv/env
+EnvironmentFile=-/etc/moat/kv.env
+
+TimeoutSec=300
+WatchdogSec=10
+
+Restart=always
+RestartSec=10

systemd/moat-kv-run-any@.service

@@ -0,0 +1,25 @@
+[Unit]
+Description=MoaT-KV any-nodes runner for %i
+After=moat-kv.service
+Requires=moat-kv.service
+
+ConditionFileNotEmpty=/etc/moat/moat.cfg
+
+[Install]
+WantedBy=multi-user.target
+
+[Service]
+Type=notify
+ExecStart=/usr/lib/moat/kv/run-any %i
+
+User=MoaT-KV
+Group=MoaT
+
+EnvironmentFile=/usr/lib/moat/kv/env
+EnvironmentFile=-/etc/moat/kv.env
+
+TimeoutSec=300
+WatchdogSec=10
+
+Restart=always
+RestartSec=10

systemd/moat-kv-run-single.service

@@ -0,0 +1,26 @@
+[Unit]
+Description=MoaT-KV default single-node runner
+After=moat-kv.service
+Requires=moat-kv.service
+Conflicts=distkv-run-single.service
+
+ConditionFileNotEmpty=/etc/moat/moat.cfg
+
+[Install]
+WantedBy=multi-user.target
+
+[Service]
+Type=notify
+ExecStart=/usr/lib/moat/kv/run-single
+
+User=MoaT-KV
+Group=MoaT
+
+EnvironmentFile=/usr/lib/moat/kv/env
+EnvironmentFile=-/etc/moat/kv.env
+
+TimeoutSec=300
+WatchdogSec=10
+
+Restart=always
+RestartSec=10

systemd/moat-kv-run-single@.service

@@ -0,0 +1,25 @@
+[Unit]
+Description=MoaT-KV single-node runner for %i
+After=moat-kv.service
+Requires=moat-kv.service
+
+ConditionFileNotEmpty=/etc/moat/moat.cfg
+
+[Install]
+WantedBy=multi-user.target
+
+[Service]
+Type=notify
+ExecStart=/usr/lib/moat/kv/run-single %i
+
+User=MoaT-KV
+Group=MoaT
+
+EnvironmentFile=/usr/lib/moat/kv/env
+EnvironmentFile=-/etc/moat/kv.env
+
+TimeoutSec=300
+WatchdogSec=10
+
+Restart=always
+RestartSec=10

systemd/moat-kv.service

@@ -0,0 +1,27 @@
+[Unit]
+Description=MoaT-KV main server
+After=mosquitto.service
+Wants=mosquitto.service
+Conflicts=moat-kv-recover.service
+Conflicts=distkv.service
+OnFailure=moat-kv-recover.service
+
+ConditionFileNotEmpty=/etc/moat/moat.cfg
+
+[Install]
+WantedBy=multi-user.target
+
+[Service]
+Type=notify
+ExecStart=/usr/lib/moat/kv/run
+ExecStartPost=/usr/lib/moat/kv/success
+SuccessExitStatus=42
+
+User=MoaT-KV
+Group=MoaT
+
+EnvironmentFile=/usr/lib/moat/kv/env
+EnvironmentFile=-/etc/moat/kv.env
+
+TimeoutSec=300
+WatchdogSec=10

systemd/postinst

@@ -0,0 +1,7 @@
+#!/bin/sh
+
+set -ex
+/usr/lib/moat/kv/init
+systemd-sysusers --no-pager /usr/lib/sysusers.d/moat-kv.conf
+systemctl enable moat-kv.service
+systemctl enable moat-kv-rotate.timer || true  # may be masked

systemd/sysusers

@@ -0,0 +1,3 @@
+g   MoaT   -
+u   MoaT-KV   -  "MoaT-KV"   /var/lib/moat/kv
+m   MoaT-KV   MoaT

build/lib/moat/kv/backend/serf.py

@@ -1,45 +0,0 @@
-from __future__ import annotations
-from contextlib import asynccontextmanager
-
-import anyio
-import asyncserf
-
-from . import Backend
-
-# Simply setting connect=asyncserf.serf_client interferes with mocking
-# when testing.
-
-
-class SerfBackend(Backend):
-    client = None
-
-    @asynccontextmanager
-    async def connect(self, *a, **k):
-        async with asyncserf.serf_client(*a, **k) as c:
-            self.client = c
-            try:
-                yield self
-            finally:
-                with anyio.CancelScope(shield=True):
-                    await self.aclose()
-                self.client = None
-
-    def monitor(self, *topic):  # pylint: disable=invalid-overridden-method
-        topic = "user:" + ".".join(topic)
-        # self.client.stream is also async, pass thru
-        return self.client.stream(topic)
-
-    def send(self, *topic, payload):  # pylint: disable=invalid-overridden-method
-        """
-        Send this payload to this topic.
-        """
-        # self.client.event is also async, pass thru
-        return self.client.event(".".join(topic), payload=payload, coalesce=False)
-
-
-@asynccontextmanager
-async def connect(*a, **kw):
-    async with anyio.create_task_group() as tg:
-        c = SerfBackend(tg)
-        async with c.connect(*a, **kw):
-            yield c

build/lib/moat/kv/mock/serf.py

@@ -1,250 +0,0 @@
-from __future__ import annotations
-import copy
-import logging
-import time
-from contextlib import AsyncExitStack, asynccontextmanager
-from functools import partial
-
-import anyio
-import attr
-from unittest import mock
-import trio
-from asyncscope import main_scope, scope
-from asyncserf.stream import SerfEvent
-from moat.util import NotGiven, ValueEvent, attrdict, combine_dict, create_queue
-
-from moat.kv.codec import unpacker
-from moat.kv.mock import S as _S
-from moat.kv.server import Server
-
-logger = logging.getLogger(__name__)
-
-otm = time.time
-
-from . import CFG
-
-
-@asynccontextmanager
-async def stdtest(n=1, run=True, ssl=False, tocks=20, **kw):
-    C_OUT = CFG.get("_stdout", NotGiven)
-    if C_OUT is not NotGiven:
-        del CFG["_stdout"]
-    TESTCFG = copy.deepcopy(CFG["kv"])
-    TESTCFG.server.port = None
-    TESTCFG.server.backend = "serf"
-    TESTCFG.root = "test"
-    if C_OUT is not NotGiven:
-        CFG["_stdout"] = C_OUT
-        TESTCFG["_stdout"] = C_OUT
-
-    if ssl:
-        import ssl
-
-        import trustme
-
-        ca = trustme.CA()
-        cert = ca.issue_server_cert("127.0.0.1")
-        server_ctx = ssl.create_default_context(purpose=ssl.Purpose.CLIENT_AUTH)
-        client_ctx = ssl.create_default_context(purpose=ssl.Purpose.SERVER_AUTH)
-        ca.configure_trust(client_ctx)
-        cert.configure_cert(server_ctx)
-    else:
-        server_ctx = client_ctx = False
-
-    clock = trio.lowlevel.current_clock()
-    clock.autojump_threshold = 0.0
-    # clock.rate = 5
-
-    @attr.s
-    class S(_S):
-        splits = attr.ib(factory=set)
-        serfs = attr.ib(factory=set)
-
-        def split(self, s):
-            assert s not in self.splits
-            logger.debug("Split: add %d", s)
-            self.splits.add(s)
-
-        def join(self, s):
-            logger.debug("Split: join %d", s)
-            self.splits.remove(s)
-
-    async def mock_get_host_port(st, host):
-        i = int(host[host.rindex("_") + 1 :])
-        s = st.s[i]
-        await s.is_serving
-        for host, port, *_ in s.ports:
-            if host == "::" or host[0] != ":":
-                return host, port
-
-    def tm():
-        try:
-            return trio.current_time()
-        except RuntimeError:
-            return otm()
-
-    async def mock_set_tock(self, old):
-        assert self._tock < tocks, "Test didn't terminate. Limit:" + str(tocks)
-        await old()
-
-    async with main_scope("moat.kv.test.serf") as scp:
-        tg = scp._tg
-        st = S(tg, client_ctx)
-        async with AsyncExitStack() as ex:
-            st.ex = ex  # pylint: disable=attribute-defined-outside-init
-            ex.enter_context(mock.patch("time.time", new=tm))
-            ex.enter_context(mock.patch("time.monotonic", new=tm))
-            logging._startTime = tm()
-
-            ex.enter_context(
-                mock.patch("asyncserf.serf_client", new=partial(mock_serf_client, st)),
-            )
-
-            for i in range(n):
-                name = "test_" + str(i)
-                args = kw.get(name, kw.get("args", attrdict()))
-                args["cfg"] = combine_dict(
-                    args.get("cfg", {}),
-                    {
-                        "kv": {
-                            "conn": {"ssl": client_ctx},
-                        },
-                        "server": {
-                            "bind_default": {
-                                "host": "127.0.0.1",
-                                "port": i + 50120,
-                                "ssl": server_ctx,
-                            },
-                            "serf": {"i": i},
-                        },
-                    },
-                    TESTCFG,
-                )
-                s = Server(name, **args)
-                ex.enter_context(
-                    mock.patch.object(s, "_set_tock", new=partial(mock_set_tock, s, s._set_tock)),
-                )
-                ex.enter_context(
-                    mock.patch.object(s, "_get_host_port", new=partial(mock_get_host_port, st)),
-                )
-                st.s.append(s)
-
-            async def with_serf(s, *a, **k):
-                s._scope = scope.get()
-                return await s._scoped_serve(*a, **k)
-
-            evts = []
-            for i in range(n):
-                if kw.get("run_" + str(i), run):
-                    evt = anyio.Event()
-                    await scp.spawn_service(with_serf, st.s[i], ready_evt=evt)
-                    evts.append(evt)
-            for e in evts:
-                await e.wait()
-            try:
-                yield st
-            finally:
-                with anyio.fail_after(2, shield=True):
-                    logger.info("Runtime: %s", clock.current_time())
-                    tg.cancel_scope.cancel()
-        logger.info("End")
-        pass  # unwinding ex:AsyncExitStack
-
-
-@asynccontextmanager
-async def mock_serf_client(master, **cfg):
-    async with scope.using_scope():
-        ms = MockServ(master, **cfg)
-        master.serfs.add(ms)
-        ms._scope = scope.get()  # pylint:disable=attribute-defined-outside-init
-        try:
-            yield ms
-        finally:
-            master.serfs.remove(ms)
-        pass  # terminating mock_serf_client nursery
-
-
-class MockServ:
-    def __init__(self, master, **cfg):
-        self.cfg = cfg
-        self._tg = scope._tg
-        self.streams = {}
-        self._master = master
-
-    def __hash__(self):
-        return id(self)
-
-    async def spawn(self, fn, *args, **kw):
-        async def run(evt):
-            with anyio.CancelScope() as sc:
-                await evt.set(sc)
-                await fn(*args, **kw)
-
-        evt = ValueEvent()
-        self._tg.spawn(run, evt)
-        return await evt.get()
-
-    async def event(self, name, payload, coalesce=True):
-        try:
-            logger.debug("SERF:%s: %r", name, unpacker(payload))
-        except Exception:
-            logger.debug("SERF:%s: %r (raw)", name, payload)
-        assert not coalesce, "'coalesce' must be cleared!"
-
-        i_self = self.cfg.get("i", 0)
-        for s in list(self._master.serfs):
-            i_s = s.cfg.get("i", 0)
-            for x in self._master.splits:
-                if (i_s < x) != (i_self < x):
-                    break
-            else:
-                n = tuple(name.split("."))
-                while n:
-                    sl = s.streams.get(n, ())
-                    for sn in sl:
-                        await sn.q.put((name, payload))
-                    n = n[:-1]
-
-    def stream(self, typ):
-        """compat for supporting asyncactor"""
-        if not typ.startswith("user:"):
-            raise RuntimeError("not supported")
-        typ = typ[5:]
-        return self.serf_mon(typ)
-
-    def serf_mon(self, typ):
-        if "," in typ:
-            raise RuntimeError("not supported")
-        s = MockSerfStream(self, "user:" + typ)
-        return s
-
-    async def serf_send(self, typ, payload):
-        """compat for supporting asyncactor"""
-        return await self.event(typ, payload)
-
-
-class MockSerfStream:
-    q = None
-
-    def __init__(self, serf, typ):
-        self.serf = serf
-        assert typ.startswith("user:")
-        self.typ = tuple(typ[5:].split("."))
-
-    async def __aenter__(self):
-        self.q = create_queue(100)
-        self.serf.streams.setdefault(self.typ, []).append(self)
-        return self
-
-    async def __aexit__(self, *tb):
-        self.serf.streams[self.typ].remove(self)
-        del self.q
-
-    def __aiter__(self):
-        return self
-
-    async def __anext__(self):
-        res = await self.q.get()
-        evt = SerfEvent(self)
-        evt.topic, evt.payload = res
-        return evt

debian/moat-kv/usr/lib/python3/dist-packages/moat/kv/backend/serf.py

@@ -1,45 +0,0 @@
-from __future__ import annotations
-from contextlib import asynccontextmanager
-
-import anyio
-import asyncserf
-
-from . import Backend
-
-# Simply setting connect=asyncserf.serf_client interferes with mocking
-# when testing.
-
-
-class SerfBackend(Backend):
-    client = None
-
-    @asynccontextmanager
-    async def connect(self, *a, **k):
-        async with asyncserf.serf_client(*a, **k) as c:
-            self.client = c
-            try:
-                yield self
-            finally:
-                with anyio.CancelScope(shield=True):
-                    await self.aclose()
-                self.client = None
-
-    def monitor(self, *topic):  # pylint: disable=invalid-overridden-method
-        topic = "user:" + ".".join(topic)
-        # self.client.stream is also async, pass thru
-        return self.client.stream(topic)
-
-    def send(self, *topic, payload):  # pylint: disable=invalid-overridden-method
-        """
-        Send this payload to this topic.
-        """
-        # self.client.event is also async, pass thru
-        return self.client.event(".".join(topic), payload=payload, coalesce=False)
-
-
-@asynccontextmanager
-async def connect(*a, **kw):
-    async with anyio.create_task_group() as tg:
-        c = SerfBackend(tg)
-        async with c.connect(*a, **kw):
-            yield c