moat-kv 0.70.24__py3-none-any.whl → 0.71.6__py3-none-any.whl

debian/moat-kv/usr/lib/python3/dist-packages/moat/kv/runner.py

@@ -1,1347 +0,0 @@
-"""
-This module's job is to run code, resp. to keep it running.
-
-
-"""
-from __future__ import annotations
-
-import time
-from collections.abc import Mapping
-from contextlib import AsyncExitStack, asynccontextmanager
-from weakref import ref
-
-import anyio
-import psutil
-from asyncactor import AuthPingEvent, NodeList, PingEvent, TagEvent, UntagEvent
-from moat.util import (
-    NotGiven,
-    P,
-    Path,
-    attrdict,
-    combine_dict,
-    create_queue,
-    digits,
-    logger_for,
-    spawn,
-)
-
-from .actor import (
-    ActorState,
-    BrokenState,
-    ClientActor,
-    CompleteState,
-    DetachedState,
-    PartialState,
-)
-from .exceptions import ServerError
-from .obj import AttrClientEntry, ClientRoot, MirrorRoot
-
-try:
-    ClosedResourceError = anyio.exceptions.ClosedResourceError
-except AttributeError:
-    ClosedResourceError = anyio.ClosedResourceError
-
-import logging
-
-logger = logging.getLogger(__name__)
-
-QLEN = 10
-
-
-class NotSelected(RuntimeError):
-    """
-    This node has not been selected for a very long time. Something is amiss.
-    """
-
-    pass
-
-
-class ErrorRecorded(RuntimeError):
-    pass
-
-
-class RunnerMsg(ActorState):
-    """Superclass for runner-generated messages.
-
-    Not directly instantiated.
-
-    This message and its descendants take one opaque parameter: ``msg``.
-    """
-
-    pass
-
-
-class ChangeMsg(RunnerMsg):
-    """A message telling your code that some entry has been updated.
-
-    Subclass this and use it as `CallAdmin.watch`'s ``cls`` parameter for easier
-    disambiguation.
-
-    The runner sets ``path`` and ``value`` attributes.
-    """
-
-    pass
-
-
-class MQTTmsg(RunnerMsg):
-    """A message transporting some MQTT data.
-
-    `value` is the MsgPack-decoded content. If that doesn't exist the
-    message is not decodeable.
-
-    The runner also sets the ``path`` attribute.
-    """
-
-    pass
-
-
-class ReadyMsg(RunnerMsg):
-    """
-    This message is queued when the last watcher has read all data.
-    """
-
-    pass
-
-
-class TimerMsg(RunnerMsg):
-    """
-    A message telling your code that a timer triggers.
-
-    Subclass this and use it as `CallAdmin.timer`'s ``cls`` parameter for easier
-    disambiguation.
-    """
-
-    pass
-
-
-_CLASSES = attrdict()
-for _c in (
-    DetachedState,
-    PartialState,
-    CompleteState,
-    ActorState,
-    BrokenState,
-    TimerMsg,
-    ReadyMsg,
-    ChangeMsg,
-    MQTTmsg,
-    RunnerMsg,
-    ErrorRecorded,
-):
-    _CLASSES[_c.__name__] = _c
-
-_CLASSES["NotGiven"] = NotGiven  # ellipsis
-
-
-class CallAdmin:
-    """
-    This class collects some standard tasks which async MoaT-KV-embedded
-    code might want to do.
-    """
-
-    _taskgroup = None
-    _stack = None
-    _restart = False
-    _n_watch: int = None
-    _n_watch_seen: int = None
-
-    def __init__(self, runner, state, data):
-        self._runner = runner
-        self._state = state
-        self._data = data
-        self._client = runner.root.client
-        self._err = runner.root.err
-        self._q = runner._q
-        self._path = runner._path
-        self._subpath = runner.subpath
-        self._logger = runner._logger
-
-    async def _run(self, code, data):
-        while True:
-            self._n_watch = 0
-            self._n_watch_seen = 0
-
-            res = await self._run2(code, data)
-            if self._restart:
-                self._restart = False
-            else:
-                return res
-
-    async def _run2(self, code, data):
-        """Called by the runner to actually execute the code."""
-        self._logger.debug("Start %s with %s", self._runner._path, self._runner.code)
-        async with (
-            anyio.create_task_group() as tg,
-            AsyncExitStack() as stack,
-        ):
-            self._stack = stack
-            self._taskgroup = tg
-            self._runner.scope = sc = tg.cancel_scope
-
-            data["_self"] = self
-
-            oka = getattr(self._runner, "ok_after", 0)
-            if oka > 0:
-
-                async def is_ok(oka):
-                    await anyio.sleep(oka)
-                    await self.setup_done()
-
-                tg.start_soon(is_ok, oka)
-            tg.start_soon(self._changed_code, code)
-
-            await self._runner.send_event(ReadyMsg(0))
-            res = code(**data)
-            if code.is_async is not None:
-                res = await res
-
-            sc.cancel()
-            return res
-
-    async def _pinger(self):
-        t = self._runner.ok_after or 10
-        await anyio.sleep(t / 2)
-        while True:
-            await self._runner.state.ping()
-            await anyio.sleep(t * 5)
-
-    async def _changed_code(self, code):
-        """
-        Kill the job if the underlying code has changed
-        """
-        await code.reload_event.wait()
-        self._restart = True
-        self.cancel()
-
-    def cancel(self):
-        """
-        Cancel the running task
-        """
-        self._taskgroup.cancel_scope.cancel()
-
-    async def spawn(self, proc, *a, **kw):
-        """
-        Start a background subtask.
-
-        The task is auto-cancelled when your code ends.
-
-        Returns: an `anyio.abc.CancelScope` which you can use to cancel the
-            subtask.
-        """
-        return await spawn(self._taskgroup, proc, *a, **kw)
-
-    async def setup_done(self, **kw):
-        """
-        Call this when your code has successfully started up.
-        """
-        self._state.backoff = 0
-        await self._state.save()
-        await self._err.record_working("run", self._runner._path, **kw)
-
-    async def error(self, path=None, **kw):
-        """
-        Record that an error has occurred. This function records specific
-        error data, then raises `ErrorRecorded` which the code is not
-        supposed to catch.
-
-        See `moat.kv.errors.ErrorRoot.record_error` for keyword details. The
-        ``path`` argument is auto-filled to point to the current task.
-        """
-        if path is None:
-            path = self._path
-        r = await self._err.record_error("run", path, **kw)
-        await self._err.root.wait_chain(r.chain)
-        raise ErrorRecorded
-
-    async def open_context(self, ctx):
-        return await self._stack.enter_async_context(ctx)
-
-    async def watch(self, path, cls=ChangeMsg, **kw):
-        """
-        Create a watcher. This path is monitored as per `moat.kv.client.Client.watch`;
-        messages are encapsulated in `ChangeMsg` objects.
-        A `ReadyMsg` will be sent when all watchers have transmitted their
-        initial state.
-
-        By default a watcher will only monitor a single entry. Set
-        ``max_depth`` if you also want child entries.
-
-        By default a watcher will not report existing entries. Set
-        ``fetch=True`` if you want them.
-        """
-
-        class Watcher:
-            """Helper class for watching an entry"""
-
-            # pylint: disable=no-self-argument
-
-            def __init__(slf, admin, runner, client, cls, path, kw):
-                kw.setdefault("fetch", True)
-
-                slf.admin = admin
-                slf.runner = runner
-                slf.client = client
-                slf.path = path
-                slf.kw = kw
-                slf.cls = cls
-                slf.scope = None
-
-            async def run(slf):
-                @asynccontextmanager
-                async def _watch(path, kw):
-                    if path.mark == "r":
-                        async with slf.client.msg_monitor(path, **kw) as watcher:
-                            yield watcher
-                    elif not path.mark:
-                        async with slf.client.watch(path, **kw) as watcher:
-                            yield watcher
-                    else:
-                        raise RuntimeError(f"What should I do with a path marked {path.mark!r}?")
-
-                with anyio.CancelScope() as sc:
-                    slf.scope = sc
-                    async with _watch(path, kw) as watcher:
-                        async for msg in watcher:
-                            if "path" in msg:
-                                chg = cls(msg)
-                                try:
-                                    chg.value = (  # pylint:disable=attribute-defined-outside-init
-                                        msg.value
-                                    )
-                                except AttributeError:
-                                    pass
-                                chg.path = (  # pylint:disable=attribute-defined-outside-init
-                                    msg.path
-                                )
-                                await slf.runner.send_event(chg)
-
-                            elif msg.get("state", "") == "uptodate":
-                                slf.admin._n_watch_seen += 1
-                                if slf.admin._n_watch_seen == slf.admin._n_watch:
-                                    await slf.runner.send_event(ReadyMsg(slf.admin._n_watch_seen))
-
-            def cancel(slf):
-                if slf.scope is None:
-                    return False
-                sc, slf.scope = slf.scope, None
-                sc.cancel()
-
-        if isinstance(path, (tuple, list)):
-            path = Path.build(path)
-        elif not isinstance(path, Path):
-            raise RuntimeError(f"You didn't pass in a path: {path!r}")
-
-        kw.setdefault("max_depth", 0)
-        if kw.setdefault("fetch", True):
-            self._n_watch += 1
-
-        w = Watcher(self, self._runner, self._client, cls, path, kw)
-        self._taskgroup.start_soon(w.run)
-        return w
-
-    async def send(self, path, value=NotGiven, raw=None):
-        """
-        Publish an MQTT message.
-
-        Set either ``value`` or ``raw``.
-        """
-        if isinstance(path, (tuple, list)):
-            path = Path.build(path)
-        elif not isinstance(path, Path):
-            raise RuntimeError(f"You didn't pass in a path: {path!r}")
-
-        await self._client.msg_send(topic=path, data=value, raw=raw)
-
-    async def set(self, path, value, chain=NotGiven):
-        """
-        Set a MoaT-KV value.
-        """
-        if path.mark == "r":
-            return await self.send(path, value)
-        elif path.mark:
-            raise RuntimeError(f"What should I do with a path marked {path.mark!r}")
-
-        if isinstance(path, (tuple, list)):
-            path = Path.build(path)
-        elif not isinstance(path, Path):
-            raise RuntimeError(f"You didn't pass in a path: {path!r}")
-
-        return await self._client.set(path, value=value, chain=chain)
-
-    async def get(self, path, value):
-        """
-        Get a MoaT-KV value.
-        """
-        if isinstance(path, (tuple, list)):
-            path = Path.build(path)
-        elif not isinstance(path, Path):
-            raise RuntimeError(f"You didn't pass in a path: {path!r}")
-
-        res = await self._client.get(path, value=value)  # TODO chain
-
-        if "value" in res:
-            return res.value
-        else:
-            return KeyError(path)
-
-    async def monitor(self, path, cls=MQTTmsg, **kw):
-        """
-        Create an MQTT monitor.
-        Messages are encapsulated in `MQTTmsg` objects.
-
-        By default a monitor will only monitor a single entry. You may use
-        MQTT wildcards.
-
-        The message is decoded and stored in the ``value`` attribute unless
-        it's either undecodeable or ``raw`` is set, in which case it's
-        stored in ``.msg``. The topic the message was sent to is in
-        ``topic``.
-        """
-
-        class Monitor:
-            """Helper class for reading an MQTT topic"""
-
-            def __init__(self, admin, runner, client, cls, path, kw):
-                self.admin = admin
-                self.runner = runner
-                self.client = client
-                self.path = path
-                self.kw = kw
-                self.cls = cls
-                self.scope = None
-
-            async def run(self):
-                with anyio.CancelScope() as sc:
-                    self.scope = sc
-                    async with self.client.msg_monitor(path, **kw) as watcher:
-                        async for msg in watcher:
-                            if "topic" in msg:
-                                # pylint:disable=attribute-defined-outside-init
-                                chg = cls(msg.get("raw", None))
-                                try:
-                                    chg.value = msg.data
-                                except AttributeError:
-                                    pass
-                                chg.path = Path.build(msg.topic)
-                                await self.runner.send_event(chg)
-
-            async def cancel(self):
-                if self.scope is None:
-                    return False
-                sc, self.scope = self.scope, None
-                sc.cancel()
-
-        if isinstance(path, (tuple, list)):
-            path = Path.build(path)
-        elif not isinstance(path, Path):
-            raise RuntimeError(f"You didn't pass in a path: {path!r}")
-
-        w = Monitor(self, self._runner, self._client, cls, path, kw)
-        self._taskgroup.start_soon(w.run)
-        return w
-
-    async def timer(self, delay, cls=TimerMsg):
-        class Timer:
-            def __init__(self, runner, cls, tg):
-                self.runner = runner
-                self.cls = cls
-                self.scope = None
-                self._taskgroup = tg
-                self.delay = None
-
-            async def _run(self):
-                with anyio.CancelScope() as sc:
-                    self.scope = sc
-                    await anyio.sleep(self.delay)
-                    self.scope = None
-                    await self.runner.send_event(self.cls(self))
-
-            def cancel(self):
-                if self.scope is None:
-                    return False
-                sc, self.scope = self.scope, None
-                sc.cancel()
-                return True
-
-            async def run(self, delay):
-                self.cancel()
-                self.delay = delay
-                if self.delay > 0:
-                    self._taskgroup.start_soon(t._run)
-
-        t = Timer(self._runner, cls, self._taskgroup)
-        await t.run(delay)
-        return t
-
-
-class RunnerEntry(AttrClientEntry):
-    """
-    An entry representing some hopefully-running code.
-
-    The code will run some time after ``target`` has passed.
-    On success, it will run again ``repeat`` seconds later (if >0).
-    On error, it will run ``delay`` seconds later (if >0), multiplied by 2**backoff.
-
-    Arguments:
-        code (list): pointer to the code that's to be started.
-        data (dict): additional data for the code.
-        delay (float): time before restarting the job on error.
-            Default 100.
-        repeat (float): time before restarting on success.
-            Default: zero: no restart.
-        target (float): time the job should be started at.
-            Default: zero: don't start.
-        ok_after (float): the job is marked OK if it has run this long.
-            Default: zero: the code will do that itself.
-        backoff (float): Exponential back-off factor on errors.
-            Default: 1.1.
-
-    The code runs with these additional keywords::
-
-        _self: the `CallEnv` object, which the task can use to actually do things.
-        _client: the MoaT-KV client connection.
-        _info: a queue which the task can use to receive events. A message of
-            ``None`` signals that the queue was overflowing and no further
-            messages will be delivered. Your task should use that as its
-            mainloop.
-        _P: build a path from a string
-        _Path: build a path from its arguments
-
-    Some possible messages are defined in :mod:`moat.kv.actor`.
-    """
-
-    ATTRS = "code data delay ok_after repeat backoff target".split()
-
-    delay = 100  # timedelta, before restarting
-    repeat = 0
-    target = 0
-    backoff = 1.1
-    ok_after = 0
-
-    code = ()  # what to execute
-    data = None
-    scope = None  # scope to kill off
-    _comment = None  # used for error entries, i.e. mainly Cancel
-    _q = None  # send events to the running task. Async tasks only.
-    _running = False  # .run is active. Careful with applying updates.
-    _task = None
-    retry = None
-
-    def __init__(self, *a, **k):
-        self.data = {}  # local data
-
-        super().__init__(*a, **k)
-
-        self._logger = logger_for(self._path)
-
-    def __repr__(self):
-        return "<%s %r:%r>" % (self.__class__.__name__, self.subpath, self.code)
-
-    @property
-    def state(self):
-        return self.root.state.follow(self.subpath, create=None)
-
-    async def run(self):
-        if self.code is None:
-            return  # nothing to do here
-
-        state = self.state
-        try:
-            self._running = True
-            try:
-                self._logger.debug("Start")
-                if state.node is not None:
-                    raise RuntimeError(f"already running on {state.node}")
-                code = self.root.code.follow(self.code, create=False)
-                data = combine_dict(
-                    self.data or {},
-                    {} if code.value is NotGiven else code.value.get("default", {}),
-                    deep=True,
-                )
-
-                if code.is_async:
-                    data["_info"] = self._q = create_queue(QLEN)
-                data["_client"] = self.root.client
-                data["_cfg"] = self.root.client._cfg
-                data["_cls"] = _CLASSES
-                data["_P"] = P
-                data["_Path"] = Path
-                data["_log"] = self._logger
-                data["_digits"] = digits
-
-                state.started = time.time()
-                state.node = state.root.name
-
-                await state.save(wait=True)
-                if state.node != state.root.name:
-                    raise RuntimeError("Rudely taken away from us.", state.node, state.root.name)
-
-                data["_self"] = calls = CallAdmin(self, state, data)
-                res = await calls._run(code, data)
-
-            except BaseException as exc:
-                self._logger.info("Error: %r", exc)
-                raise
-            else:
-                self._logger.debug("End")
-            finally:
-                self.scope = None
-                self._q = None
-                t = time.time()
-
-        except ErrorRecorded:
-            # record_error() has already been called
-            self._comment = None
-            state.backoff = min(state.backoff + 1, 20)
-
-        except BaseException as exc:
-            c, self._comment = self._comment, None
-            with anyio.move_on_after(5, shield=True):
-                r = await self.root.err.record_error(
-                    "run",
-                    self._path,
-                    exc=exc,
-                    data=self.data,
-                    comment=c,
-                )
-                if r is not None:
-                    await self.root.err.wait_chain(r.chain)
-            if isinstance(exc, Exception):
-                state.backoff = min(state.backoff + 1, 20)
-            else:
-                raise
-
-        else:
-            state.result = res
-            state.backoff = 0
-            await self.root.err.record_working("run", self._path)
-
-        finally:
-            with anyio.fail_after(2, shield=True):
-                if state.node == state.root.name:
-                    state.node = None
-                self._running = False
-                state.stopped = t
-
-                if state.backoff > 0:
-                    self.retry = t + (self.backoff**state.backoff) * self.delay
-                else:
-                    self.retry = None
-
-                try:
-                    await state.save()
-                except ClosedResourceError:
-                    pass
-                except ServerError:
-                    logger.exception("Could not save")
-
-                await self.root.trigger_rescan()
-
-    async def send_event(self, evt):
-        """Send an event to the running process."""
-        if self._q is None:
-            if self._running:
-                self._logger.info("Discarding %r", evt)
-        elif self._q.qsize() < QLEN - 1:
-            self._logger.debug("Event: %r", evt)
-            await self._q.put(evt)
-        elif self._q.qsize() == QLEN - 1:
-            self._logger.warning("Queue full")
-            await self._q.put(None)
-            self._q = None
-            self._logger.info("Discarding %r", evt)
-
-    async def seems_down(self):
-        state = self.state
-        state.node = None
-        if not self._running:
-            await state.save(wait=True)
-
-    async def set_value(self, value):
-        """Process incoming value changes"""
-        c = self.code
-        await super().set_value(value)
-
-        # Check whether running code needs to be killed off
-        if self.scope is not None:
-            if value is NotGiven or c is not self.code:
-                # The code changed.
-                self._comment = "Cancel: Code changed"
-                self.scope.cancel()
-            elif not getattr(self, "target", None):
-                self._comment = "Cancel: target zeroed"
-                self.scope.cancel()
-
-        await self.root.trigger_rescan()
-
-    async def seen_value(self):
-        await super().seen_value()
-        await self.root.trigger_rescan()
-
-    async def run_at(self, t: float):
-        """Next run at this time."""
-        self.target = t
-        if not self._running:
-            await self.save()
-
-    def should_start(self):
-        """Tell whether this job might want to be started.
-
-        Returns:
-          ``False``: No, it's running (or has run and doesn't restart).
-          ``0``: No, it should not start
-          ``>0``: timestamp at which it should start, or should have started
-
-        """
-
-        state = self.state
-        if self.code is None:
-            return False, "no code"
-        if state.node is not None:
-            # not our responsibility!
-            return None, "node set"
-        if state.started and not state.stopped:
-            raise RuntimeError("Running! should not be called")
-
-        if self.target is None:
-            return False, "no target"
-        elif self.target > state.started:
-            return self.target, "target > started"
-        elif state.backoff:
-            return state.stopped + self.delay * (self.backoff**state.backoff), "backoff"
-        elif self.repeat:
-            return state.stopped + self.repeat, "repeat"
-        elif state.started and state.started > state.stopped:
-            return False, "is started"
-        else:
-            return 0, "no target"
-
-    def __hash__(self):
-        return hash(self.subpath)
-
-    def __eq__(self, other):
-        other = getattr(other, "subpath", other)
-        return self.subpath == other
-
-    def __lt__(self, other):
-        other = getattr(other, "subpath", other)
-        return self.subpath < other
-
-    @property
-    def age(self):
-        return time.time() - self.state.started
-
-
-class RunnerNode:
-    """
-    Represents all nodes in this runner group.
-
-    This is used for load balancing and such. TODO.
-    """
-
-    seen = 0
-    load = 999
-
-    def __new__(cls, root, name):
-        try:
-            self = root._nodes[name]
-        except KeyError:
-            self = object.__new__(cls)
-            self.root = root
-            self.name = name
-            root._nodes[name] = self
-        return self
-
-    def __init__(self, *a, **k):
-        pass
-
-
-class StateEntry(AttrClientEntry):
-    """
-    This is the actual state associated with a RunnerEntry.
-    It must only be managed by the node that actually runs the code.
-
-    Arguments:
-      started (float): timestamp when the job was last started
-      stopped (float): timestamp when the job last terminated
-      pinged (float): timestamp when the state was last verified by the runner
-      result (Any): the code's return value
-      node (str): the node running this code
-      backoff (float): on error, the multiplier to apply to the restart timeout
-      computed (float): computed start time
-      reason (str): reason why (not) starting
-    """
-
-    ATTRS = "started stopped pinged computed reason result node backoff".split()
-
-    started = 0  # timestamp
-    stopped = 0  # timestamp
-    pinged = 0  # timestamp
-    node = None  # on which the code is currently running
-    result = NotGiven
-    backoff = 0
-    computed = 0  # timestamp
-    reason = ""
-
-    @property
-    def runner(self):
-        return self.root.runner.follow(self.subpath, create=False)
-
-    async def startup(self):
-        try:
-            self.runner
-        except KeyError:
-            # The code entry doesn't exist any more.
-            await self.delete()
-            return
-
-        if self.node is None:
-            return
-        if self.node != self.root.name:
-            self.root.runner.get_node(self.node)
-            return
-
-        self.stopped = time.time()
-        self.node = None
-        self.backoff = min(20, self.backoff + 1)
-        await self.root.runner.err.record_error(
-            "run",
-            self.runner._path,
-            message="Runner restarted",
-        )
-        await self.save()
-
-    async def ping(self):
-        if self.node != self.root.name:
-            raise RuntimeError("Not our state!")
-        self.pinged = time.time()
-        await self.save()
-
-    async def stale(self):
-        self.stopped = time.time()
-        node, self.node = self.node, None
-        self.backoff = min(20, self.backoff + 2)
-        await self.root.runner.err.record_error(
-            "run",
-            self.runner._path,
-            message="Runner killed: {node} {state}",
-            data={"node": node, "state": "offline" if self.stopped else "stale"},
-        )
-        await self.save()
-
-    async def set_value(self, value):
-        n = self.node
-
-        await super().set_value(value)
-        if not self.root.runner_ready:
-            return
-        try:
-            run = self.runner
-        except KeyError:
-            return
-
-        # side effect: add to the global node list
-        if n is not None:
-            run.root.get_node(n)
-
-        # Check whether running code needs to be killed off
-        if run.scope is None:
-            return
-        if self.node is not None and self.node == n:
-            # Nothing changed.
-            return
-        elif self.node is None or n == self.root.runner.name:
-            # Owch. Our job got taken away from us.
-            run._comment = f"Cancel: Node set to {self.node!r}"
-            run.scope.cancel()
-        elif n is not None:
-            logger.warning(
-                "Runner %s at %r: running but node is %s",
-                self.root.name,
-                self.subpath,
-                n,
-            )
-
-        await run.root.trigger_rescan()
-
-
-class StateRoot(MirrorRoot):
-    """Base class for handling the state of entries.
-
-    This is separate from the RunnerRoot hierarchy because the latter may
-    be changed by anybody while this subtree may only be affected by the
-    actual runner. Otherwise we get interesting race conditions.
-    """
-
-    _runner = None
-
-    @classmethod
-    def child_type(cls, name):
-        return StateEntry
-
-    @property
-    def name(self):
-        return self.client.client_name
-
-    @property
-    def runner(self):
-        return self._runner()
-
-    @property
-    def runner_ready(self):
-        r = self._runner
-        if r is None:
-            return False
-        r = r()
-        if r is None:
-            return False
-        return r.ready
-
-    def set_runner(self, runner):
-        self._runner = ref(runner)
-
-    async def runner_running(self):
-        for n in self.all_children:
-            await n.startup()
-
-    async def kill_stale_nodes(self, names):
-        """States with node names in the "names" set are stale. Kill them."""
-        for s in self.all_children:
-            if s.node in names:
-                await s.stale()
-
-    _last_t = 0
-
-    async def ping(self):
-        t = time.time()
-        if t - self._last_t >= abs(self._cfg["ping"]):
-            self._last_t = t
-            if self._cfg["ping"] > 0:
-                val = self.value_or({}, Mapping)
-                val["alive"] = t
-                await self.update(val)
-            else:
-                await self.client.msg_send(
-                    "run",
-                    {"group": self.runner.group, "time": t, "node": self.name},
-                )
-
-
-class _BaseRunnerRoot(ClientRoot):
-    """common code for AnyRunnerRoot and SingleRunnerRoot"""
-
-    _trigger: anyio.abc.Event = None
-    _run_now_task: anyio.abc.CancelScope = None
-    _tagged: bool = True
-
-    err = None
-    code = None
-    ready = False
-    _run_next = 0
-    node_history = None
-    _start_delay = None
-    state = None
-    _act = None
-
-    CFG = "runner"
-    SUB = None
-
-    def __init__(self, *a, _subpath, err=None, code=None, nodes=0, **kw):
-        super().__init__(*a, **kw)
-        self.err = err
-        self.code = code
-        self._nodes = {}
-        self.n_nodes = nodes
-        self._trigger = anyio.Event()
-        self._x_subpath = _subpath
-
-    @classmethod
-    def child_type(cls, name):
-        return RunnerEntry
-
-    @classmethod
-    async def as_handler(cls, client, subpath, cfg=None, **kw):  # pylint: disable=arguments-differ
-        assert cls.SUB is not None
-        if cfg is None:
-            cfg_ = client._cfg["runner"]
-        else:
-            cfg_ = combine_dict(cfg, client._cfg["runner"])
-        return await super().as_handler(client, subpath=subpath, _subpath=subpath, cfg=cfg_, **kw)
-
-    async def run_starting(self):
-        from .code import CodeRoot
-        from .errors import ErrorRoot
-
-        if self.err is None:
-            self.err = await ErrorRoot.as_handler(self.client)
-        if self.code is None:
-            self.code = await CodeRoot.as_handler(self.client)
-
-        await self._state_runner()
-        self.state.set_runner(self)
-
-        self.node_history = NodeList(0)
-        self._start_delay = self._cfg["start_delay"]
-
-        await super().run_starting()
-
-    async def _state_runner(self):
-        self.state = await StateRoot.as_handler(
-            self.client,
-            cfg=self._cfg,
-            subpath=self._x_subpath,
-            key="state",
-        )
-
-    @property
-    def name(self):
-        """my node name"""
-        return self.client.client_name
-
-    def get_node(self, name):  # pylint: disable=unused-argument
-        """
-        If the runner keeps track of "foreign" nodes, allocate them
-        """
-        return None
-
-    async def running(self):
-        self._tg.start_soon(self._run_actor)
-
-        # the next block needs to be atomic
-        self.ready = True
-
-        await self.state.runner_running()
-        await super().running()
-
-    async def _run_actor(self):
-        """
-        This method is started as a long-lived task of the root as soon as
-        the subtree's data are loaded.
-
-        Its job is to control which tasks are started.
-        """
-        raise RuntimeError("You want to override me.")
-
-    async def trigger_rescan(self):
-        """Tell the _run_actor task to rescan our job list prematurely"""
-        if self._trigger is not None:
-            self._trigger.set()
-
-    async def _run_now(self, evt=None):
-        t_next = self._run_next
-        with anyio.CancelScope() as sc:
-            self._run_now_task = sc
-            if evt is not None:
-                evt.set()
-            t = time.time()
-            while True:
-                if t_next > t:
-                    with anyio.move_on_after(t_next - t):
-                        await self._trigger.wait()
-                        self._trigger = anyio.Event()
-
-                t = time.time()
-                t_next = t + 999
-                for j in self.all_children:
-                    d, r = j.should_start()
-                    if d is None:
-                        continue
-                    if not d or d > t:
-                        j.state.computed = d
-                        j.state.reason = r
-                        if self._tagged:
-                            try:
-                                await j.state.save()
-                            except anyio.ClosedResourceError:  # owch
-                                logger.error("Could not update state %r %r", j, j.state)
-                                return
-                        if d and t_next > d:
-                            t_next = d
-                        continue
-                    self._tg.start_soon(j.run)
-                    await anyio.sleep(self._start_delay)
-
-    async def notify_actor_state(self, msg=None):
-        """
-        Notify all running jobs about a possible change in active status.
-        """
-        ac = len(self.node_history)
-        if ac == 0 or msg is None:
-            ac = BrokenState
-        elif self.name in self.node_history and ac == 1:
-            ac = DetachedState
-        elif self._act is not None and ac >= self.n_nodes:  # TODO configureable
-            ac = CompleteState
-        else:
-            ac = PartialState
-
-        logger.debug("State %r %r", ac, msg)
-
-        ac = ac(msg)
-        for n in self.all_children:
-            await n.send_event(ac)
-
-
-class AnyRunnerRoot(_BaseRunnerRoot):
-    """
-    This class represents the root of a code runner. Its job is to start
-    (and periodically restart, if required) the entry points stored under it.
-
-    :class:`AnyRunnerRoot` tries to ensure that the code in question runs
-    on one single cluster member. In case of a network split, the code will
-    run once in each split areas until the split is healed.
-    """
-
-    SUB = "group"
-
-    _stale_times = None
-    tg = None
-    seen_load = None
-    _tagged: bool = False
-
-    def __init__(self, *a, **kw):
-        super().__init__(*a, **kw)
-        self.group = (
-            P(self.client.config.server["root"]) + P(self._cfg["name"]) | "any" | self._path[-1]
-        )
-
-    def get_node(self, name):
-        return RunnerNode(self, name)
-
-    @property
-    def max_age(self):
-        """Timeout after which we really should have gotten another go"""
-        # allow one go-around without being in the list; happens when the
-        # bus is slow
-        return self._act.cycle_time_max * (self._act.history_size + 2.5)
-
-    async def _run_actor(self):
-        """
-        Monitor the Actor state, run a :meth:`_run_now` subtask whenever we're 'it'.
-        """
-        async with ClientActor(
-            self.client,
-            self.name,
-            topic=self.group,
-            cfg=self._cfg["actor"],
-        ) as act:
-            self._act = act
-
-            age_q = create_queue(10)
-            await self.spawn(self._age_killer, age_q)
-
-            psutil.cpu_percent(interval=None)
-            await act.set_value(0)
-            self.seen_load = None
-
-            async for msg in act:
-                logger.debug("Actor %r", msg)
-                if isinstance(msg, PingEvent):
-                    await act.set_value(100 - psutil.cpu_percent(interval=None))
-
-                    node = self.get_node(msg.node)
-                    node.load = msg.value
-                    node.seen = time.time()
-                    if self.seen_load is not None:
-                        self.seen_load += msg.value
-                    self.node_history += node
-
-                elif isinstance(msg, TagEvent):
-                    self._tagged = True
-                    load = 100 - psutil.cpu_percent(interval=None)
-                    await act.set_value(load)
-                    if self.seen_load is not None:
-                        pass  # TODO
-
-                    self.get_node(self.name).seen = time.time()
-                    self.node_history += self.name
-                    evt = anyio.Event()
-                    await self.spawn(self._run_now, evt)
-                    await age_q.put(None)
-                    await evt.wait()
-
-                    await self.state.ping()
-
-                elif isinstance(msg, UntagEvent):
-                    self._tagged = False
-                    await act.set_value(100 - psutil.cpu_percent(interval=None))
-                    self.seen_load = 0
-
-                    self._run_now_task.cancel()
-                    # TODO if this is a DetagEvent, kill everything?
-
-                await self.notify_actor_state(msg)
-
-            pass  # end of actor task
-
-    async def find_stale_nodes(self, cur):
-        """
-        Find stale nodes (i.e. last seen < cur) and clean them.
-        """
-        if self._stale_times is None:
-            self._stale_times = []
-        self._stale_times.append(cur)
-        if self._stale_times[0] > cur - self.max_age:
-            return
-        if len(self._stale_times) <= 2 * self._cfg["actor"]["n_hosts"] + 1:
-            return
-        cut = self._stale_times.pop(0)
-
-        dropped = []
-        for node in self._nodes.values():
-            if node.seen < cut:
-                dropped.append(node)
-        names = set()
-        for node in dropped:
-            names.add(node.name)
-            del self._nodes[node.name]
-        if names:
-            await self.state.kill_stale_nodes(names)
-
-    async def _age_killer(self, age_q):
-        """
-        Subtask which cleans up stale tasks
-
-        TODO check where to use time.monotonic
-        """
-        t0 = None
-        t1 = time.time()
-        while True:
-            with anyio.move_on_after(self.max_age):
-                await age_q.get()
-                t1 = time.time()
-            t2 = time.time()
-            if t1 + self.max_age < t2:
-                raise NotSelected(self.max_age, t2, t1)
-            t0 = t1
-            t1 = t2
-            if t0 is not None:
-                await self.find_stale_nodes(t0)
-
-
-class SingleRunnerRoot(_BaseRunnerRoot):
-    """
-    This class represents the root of a code runner. Its job is to start
-    (and periodically restart, if required) the entry points stored under it.
-
-    While :class:`AnyRunnerRoot` tries to ensure that the code in question runs
-    on any cluster member, this class runs tasks on a single node.
-    The code is able to check whether any and/or all of the cluster's main
-    nodes are reachable; this way, the code can default to local operation
-    if connectivity is lost.
-
-    Local data (dict):
-
-    Arguments:
-      cores (tuple): list of nodes whose reachability may determine
-        whether the code uses local/emergency/??? mode.
-
-    Config file:
-
-    Arguments:
-      path (tuple): the location this entry is stored at. Defaults to
-        ``('.moat', 'kv', 'process')``.
-      name (str): this runner's name. Defaults to the client's name plus
-        the name stored in the root node, if any.
-      actor (dict): the configuration for the underlying actor. See
-        ``asyncactor`` for details.
-    """
-
-    SUB = "single"
-
-    err = None
-    code = None
-    state = None
-    tg = None
-
-    def __init__(self, *a, **kw):
-        super().__init__(*a, **kw)
-        self.group = (
-            P(self.client.config.server["root"]) + P(self._cfg["name"])
-            | "single"
-            | self._path[-2]
-            | self._path[-1]
-        )
-
-    async def set_value(self, value):
-        await super().set_value(value)
-        try:
-            cores = value["cores"]
-        except (TypeError, KeyError):
-            if self._act is not None:
-                await self._act.disable(0)
-        else:
-            if self.name in cores:
-                await self._act.enable(len(cores))
-            else:
-                await self._act.disable(len(cores))
-
-    @property
-    def max_age(self):
-        """Timeout after which we really should have gotten another ping"""
-        return self._act.cycle_time_max * 1.5
-
-    async def _run_actor(self):
-        async with anyio.create_task_group() as tg:
-            age_q = create_queue(1)
-
-            async with ClientActor(self.client, self.name, topic=self.group, cfg=self._cfg) as act:
-                self._act = act
-                tg.start_soon(self._age_notifier, age_q)
-                await self.spawn(self._run_now)
-                await act.set_value(0)
-
-                async for msg in act:
-                    if isinstance(msg, AuthPingEvent):
-                        # Some node, not necessarily us, is "it".
-                        # We're the SingleNode runner: we manage our jobs
-                        # when triggered by this, no matter whether we're
-                        # "it" or not.
-                        self.node_history += msg.node
-                        await age_q.put(None)
-                        await self.notify_actor_state(msg)
-
-                        await self.state.ping()
-
-                pass  # end of actor task
-
-    async def _age_notifier(self, age_q):
-        """
-        This background job triggers :meth:`notify_actor_state` when too much
-        time has passed without an AuthPing.
-        """
-        while True:
-            try:
-                with anyio.fail_after(self.max_age):
-                    await age_q.get()
-            except TimeoutError:
-                await self.notify_actor_state()
-
-
-class AllRunnerRoot(SingleRunnerRoot):
-    """
-    This class represents the root of a code runner. Its job is to start
-    (and periodically restart, if required) the entry points stored under it.
-
-    This class behaves like `SingleRunner`, except that it runs tasks on all nodes.
-    """
-
-    SUB = "all"
-
-    err = None
-    _act = None
-    code = None
-
-    def __init__(self, *a, **kw):
-        super().__init__(*a, **kw)
-        self.group = (
-            P(self.client.config.server["root"]) + P(self._cfg["name"]) | "all" | self._path[-1]
-        )
-
-    async def _state_runner(self):
-        self.state = await StateRoot.as_handler(
-            self.client,
-            cfg=self._cfg,
-            subpath=self._x_subpath + (self.client.client_name,),
-            key="state",
-        )