fleetmesh-node 0.2.0__py3-none-any.whl

node/compartment/registry.py

@@ -0,0 +1,790 @@
+"""The Registry: the compartments a node has, and the uniform ways to reach them.
+
+Split out of node/compartment.py when it became a package.
+"""
+from __future__ import annotations
+
+import collections
+import json
+import threading
+import time
+from typing import Any, Callable, Iterator, Mapping, Optional, Sequence, TypeVar
+
+from . import lifecycle as lifecycle_mod
+from . import stores as stores_mod
+from ..interfaces import CommandSpec
+from ..metrics import COUNTER, Metric
+from .base import Compartment, CompartmentProtocol
+from .problem import Note, NoteList, Trouble, TroubleList
+from .refusals import Refusal, why_detail
+
+
+T = TypeVar('T', bound=CompartmentProtocol)
+
+
+class Registry:
+    """The compartments a node has, and the uniform ways to reach all of them.
+
+    It also owns where each one is in its lifecycle (node/lifecycle): every
+    verb below moves a compartment's Tracker, a tick that raises degrades it,
+    enough of them trip its breaker, and a quarantined compartment is left
+    alone until its backoff passes. `say`, when given, is the node log's
+    `say`, and every transition goes out on a declared lifecycle channel.
+    """
+
+    def __init__(self, say: Optional[Callable[..., Any]] = None,
+                 clock: Optional[Callable[[], int]] = None) -> None:
+        self._by_name: collections.OrderedDict[str, CompartmentProtocol] = collections.OrderedDict()
+        self._life: dict[str, lifecycle_mod.Tracker] = {}
+        self.say: Optional[Callable[..., Any]] = say
+        self.clock: Callable[[], int] = clock or (lambda: int(time.time()))
+        # The tick and an operator's `resume` arrive on different threads.
+        self._life_lock: threading.Lock = threading.Lock()
+        # And a `restart` must never stop a compartment in the middle of its
+        # tick: held around each compartment's tick and around each restart.
+        # Re-entrant because a restart from inside a tick (a quarantine trial)
+        # already holds it.
+        self._busy: threading.RLock = threading.RLock()
+        # name -> error, for each compartment whose last report() raised.
+        self._report_failed: dict[str, str] = {}
+
+    def add(self, sub: T) -> T:
+        if not sub.name:
+            raise ValueError('a compartment must have a name')
+        if sub.name in self._by_name:
+            raise ValueError('two compartments called %r' % sub.name)
+        self._by_name[sub.name] = sub
+        decl = getattr(sub, 'LIFECYCLE', None)
+        if not isinstance(decl, lifecycle_mod.Lifecycle):
+            decl = lifecycle_mod.DEFAULT
+        problems = decl.problems()
+        if decl.snapshot and not (callable(getattr(sub, 'snapshot', None)) and callable(getattr(sub, 'restore', None))):
+            # The declaration is the promise; the two methods are the code that
+            # keeps it. One without the other would read as continuity the
+            # node does not have.
+            problems.append('Lifecycle(snapshot=%d) needs snapshot() and restore(state)' % decl.snapshot)
+        # The base bound each declared store's container as `self.<key>`. One
+        # that is not there any more was replaced in __init__ - by a plain
+        # list or dict, which is the unbounded collection the declaration
+        # promised was not there.
+        holdings = getattr(sub, 'holdings', None)
+        declared_stores: tuple[object, ...] = tuple(getattr(sub, 'STORES', None) or ())
+        if holdings is None and declared_stores:
+            # Declared but never built: an __init__ that did not call
+            # Compartment.__init__. The compartment would run without one of
+            # its containers and fail at its first write - or at shutdown,
+            # when snapshot() asks for holdings that are not there.
+            problems.append('it declares STORES but has no holdings - its __init__ never called '
+                            'Compartment.__init__, which builds them')
+        for key, container in (holdings.items() if holdings is not None else ()):
+            if getattr(sub, key, None) is not container:
+                problems.append('store %s was replaced after the base built it (self.%s = ...); '
+                                'write into the container instead' % (key, key))
+        if problems:
+            del self._by_name[sub.name]
+            raise ValueError('%s declares what it cannot keep: %s' % (sub.name, '; '.join(problems)))
+        self._life[sub.name] = lifecycle_mod.Tracker(sub.name, decl, at=self.clock())
+        return sub
+
+    # ---- lifecycle ----------------------------------------------------------
+    def _moved(self, t: Optional[lifecycle_mod.Transition]) -> None:
+        """Say a transition, if there was one and there is a log to say it to."""
+        if t is None or self.say is None:
+            return
+        try:
+            self.say(lifecycle_mod.channel_for(t), t)
+        except Exception:
+            pass                    # a log must never fail the lifecycle it records
+
+    def _move(self, name: Optional[str], to: str, at: Optional[int] = None, why: str = '') -> None:
+        tr = self._life.get(name or '')
+        if tr is None:
+            return
+        with self._life_lock:
+            t = tr.move(to, self.clock() if at is None else at, why)
+        self._moved(t)
+
+    def state_of(self, name: str) -> Optional[str]:
+        tr = self._life.get(name)
+        return tr.state if tr is not None else None
+
+    def lifecycle(self) -> dict[str, lifecycle_mod.LifecycleRecord]:
+        """Where every compartment is, for health and `meshnode lifecycle`."""
+        return {n: tr.describe() for n, tr in self._life.items()}
+
+    def resume(self, name: str, why: str = 'resumed by the operator') -> str:
+        """End a compartment's quarantine now. Returns the state it is left in.
+
+        Raises KeyError for a compartment this node does not have. Resuming
+        one that is not quarantined is not an error - it is already ticking -
+        and the state it is in is returned unchanged.
+        """
+        tr = self._life.get(name)
+        if tr is None:
+            raise KeyError('no compartment called %r; compartments: %s' % (name, ', '.join(self._life)))
+        s = self._by_name.get(name)
+        if s is not None and tr.state == lifecycle_mod.QUARANTINED and tr.started is False:
+            # Quarantined because its start() raised: it never began, so
+            # ending the quarantine means starting it, not ticking it.
+            with self._busy:
+                self._restart(s, tr, 'resumed: %s' % why)
+            return tr.state
+        with self._life_lock:
+            t = tr.resume(self.clock(), why)
+        self._moved(t)
+        return tr.state
+
+    def quarantined(self) -> list[str]:
+        return [n for n, tr in self._life.items() if tr.state == lifecycle_mod.QUARANTINED]
+
+    def failed(self) -> list[str]:
+        return [n for n, tr in self._life.items() if tr.state == lifecycle_mod.FAILED]
+
+    def restartable(self, name: str) -> bool:
+        tr = self._life.get(name)
+        return tr is not None and tr.lifecycle.restart
+
+    def restart(self, name: str, why: str = 'by the operator') -> str:
+        """Stop and start one compartment in place. Returns the state it is left in.
+
+        Only for a compartment that declares `Lifecycle(restart=True)`; any
+        other is refused with ValueError, because a stop() and start() nobody
+        designed to run twice could leave threads or sockets behind and call
+        it a recovery. KeyError for a name this node does not have.
+
+        The same instance is stopped and started: every other compartment
+        that reached it in `wire()` still holds it, and a fresh instance would
+        leave them holding the old one. A start() that raises quarantines it
+        with backoff; the trial that follows restarts it again.
+        """
+        tr = self._life.get(name)
+        s = self._by_name.get(name)
+        if tr is None or s is None:
+            raise KeyError('no compartment called %r; compartments: %s' % (name, ', '.join(self._life)))
+        if not tr.lifecycle.restart:
+            raise ValueError('%s does not declare Lifecycle(restart=True): its stop() and start() are not '
+                             'known to be safe to run twice on one instance, so only a node restart '
+                             'restarts it' % name)
+        if tr.state in (lifecycle_mod.DRAINING, lifecycle_mod.RETIRED):
+            raise ValueError('%s is %s; the node is stopping' % (name, tr.state))
+        with self._busy:
+            self._restart(s, tr, why)
+        return tr.state
+
+    def _restart(self, s: CompartmentProtocol, tr: lifecycle_mod.Tracker, why: str,
+                 trial: bool = False, now: Optional[int] = None) -> bool:
+        """stop() then start(). True when start() returned. Holds `_busy`.
+
+        A trial restart (from quarantine) moves nothing on success: the trial
+        tick that follows decides whether it recovered, and says so.
+        """
+        now = self.clock() if now is None else now
+        if tr.started and not trial:
+            self._move(s.name, lifecycle_mod.DRAINING, now, 'restart: %s' % why)
+        if tr.started:
+            try:
+                s.stop()
+            except Exception as e:
+                # Said and gone past: a stop() that raised has still stopped
+                # as far as it is going to, and start() is the recovery.
+                s.refusals.refuse('restart', 'stop raised', detail=why_detail(e))
+            finally:
+                if hasattr(s, 'stop_workers'):
+                    try:
+                        s.stop_workers()
+                    except Exception:
+                        pass
+            tr.started = False
+        try:
+            if hasattr(s, 'start_workers'):
+                s.start_workers()
+            s.start()
+        except Exception as e:
+            s.refusals.refuse('restart', 'start raised', detail=why_detail(e))
+            with self._life_lock:
+                t = tr.start_failed(now, '%s: %s' % (type(e).__name__, e))
+            self._moved(t)
+            return False
+        if trial:
+            tr.started = True
+            tr.restarts += 1
+            return True
+        with self._life_lock:
+            t = tr.restarted(now, why)
+        self._moved(t)
+        return True
+
+    # ---- carrying state across a process restart ----------------------------
+    def snapshot(self) -> dict[str, lifecycle_mod.SnapshotEntry]:
+        """What every started compartment that declares `snapshot` hands on.
+
+        One whose snapshot() raises, or returns something that is not a
+        JSON object, is a refusal against itself and is left out: the next
+        process starts it fresh, and every other compartment's state still
+        goes across.
+        """
+        out: dict[str, lifecycle_mod.SnapshotEntry] = {}
+        for s in self:
+            tr = self._life.get(s.name or '')
+            if tr is None or not tr.lifecycle.snapshot or tr.started is not True:
+                continue
+            try:
+                state = getattr(s, 'snapshot')()
+                if not isinstance(state, Mapping):
+                    raise TypeError('snapshot() returned %s, not a mapping' % type(state).__name__)
+                state = json.loads(json.dumps(dict(state)))
+            except Exception as e:
+                s.refusals.refuse('snapshot', 'raised', detail=why_detail(e))
+                continue
+            out[s.name or ''] = {'version': tr.lifecycle.snapshot, 'state': state}
+        return out
+
+    def restore(self, saved: Mapping[str, lifecycle_mod.SnapshotEntry], at: int = 0) -> dict[str, str]:
+        """Hand each compartment the state the last process left for it.
+        Before start(), so a compartment starts from where it was.
+
+        Returns {name: what happened} for every compartment that declares
+        `snapshot`, and records the same in its lifecycle. Refused - and so
+        started fresh - when there is no entry, the entry's version is not
+        the one the compartment reads now, or restore() raises. Never raises
+        itself: continuity is worth having, not worth refusing to start for.
+        """
+        out: dict[str, str] = {}
+        for s in self:
+            tr = self._life.get(s.name or '')
+            if tr is None or not tr.lifecycle.snapshot:
+                continue
+            e = saved.get(s.name or '')
+            if e is None:
+                tr.restored = ''
+                continue
+            if e['version'] != tr.lifecycle.snapshot:
+                tr.restored = ('refused: the snapshot is v%d and this %s reads v%d; started fresh'
+                               % (e['version'], s.name, tr.lifecycle.snapshot))
+            else:
+                try:
+                    getattr(s, 'restore')(e['state'])
+                    tr.restored = 'restored (v%d, written at %d)' % (e['version'], at)
+                except Exception as exc:
+                    s.refusals.refuse('restore', 'raised', detail=why_detail(exc))
+                    tr.restored = 'refused: restore raised %s: %s; started fresh' % (type(exc).__name__, exc)
+            out[s.name or ''] = tr.restored
+        return out
+
+    def lifecycle_metrics(self) -> tuple[Any, ...]:
+        """The lifecycle's counters, and gauges sampled from these trackers."""
+        return lifecycle_mod.METRICS + (
+            lifecycle_mod.gauge(lambda: len(self.quarantined()), 'compartments_quarantined',
+                                'Compartments whose breaker is open now.'),
+            lifecycle_mod.gauge(
+                lambda: sum(1 for tr in self._life.values() if tr.state == lifecycle_mod.DEGRADED),
+                'compartments_degraded', 'Compartments whose last tick raised.'),
+            lifecycle_mod.gauge(lambda: len(self.failed()), 'compartments_failed',
+                                'Optional compartments whose start() raised and that cannot restart.'),
+        )
+
+    # ---- what the compartments keep -------------------------------------------
+    def _containers(self) -> list[tuple[str, str, stores_mod.Container]]:
+        out: list[tuple[str, str, stores_mod.Container]] = []
+        for s in self:
+            holdings = getattr(s, 'holdings', None)
+            for key, c in (holdings.items() if holdings is not None else ()):
+                out.append((s.name or '', key, c))
+        return out
+
+    def store_metrics(self) -> tuple[Any, ...]:
+        """Every declared store, metered without its compartment writing a
+        metric: size, bound and high-water as gauges, and the writes, the
+        evictions and the refusals as counters. One series per (compartment,
+        store), sampled from the containers when /metrics is read."""
+        def per(read: Callable[[stores_mod.Container], int]) -> Callable[[], dict[tuple[str, str], int]]:
+            return lambda: {(n, k): read(c) for n, k, c in self._containers()}
+        labels = ('compartment', 'store')
+        return (
+            Metric(name='store_entries', description='Entries a compartment store holds now.',
+                   labels=labels, sample=per(len), tags=('stores',)),
+            Metric(name='store_bound', description='The most entries a store may hold, as declared.',
+                   labels=labels, sample=per(lambda c: c.store.limit), tags=('stores',)),
+            Metric(name='store_high_water', description='The most entries a store has held since start.',
+                   labels=labels, sample=per(lambda c: c.high_water), tags=('stores',)),
+            Metric(name='store_writes', description='Entries written to a store (and counters incremented).',
+                   kind=COUNTER, labels=labels, sample=per(lambda c: c.inserts), tags=('stores',)),
+            Metric(name='store_evictions', description='Entries a store dropped at its bound (OLDEST or LRU).',
+                   kind=COUNTER, labels=labels, sample=per(lambda c: c.evictions), tags=('stores',)),
+            Metric(name='store_refused', description='Writes a full REFUSE store turned away.',
+                   kind=COUNTER, labels=labels, sample=per(lambda c: c.refused), tags=('stores',)),
+        )
+
+    def store_trouble(self) -> list[str]:
+        """A REFUSE store is trouble once it is nearly full, and while it has
+        refused anything: the state it holds is the kind that must not be
+        lost, so the write that failed was a decision the compartment had to
+        make without it. A ring at its bound is a log doing its job."""
+        out: list[str] = []
+        for n, k, c in self._containers():
+            st = c.store
+            if st.kind == stores_mod.COUNTERS or st.evict != stores_mod.REFUSE:
+                continue
+            if c.refused or c.nearly_full():
+                out.append('%s: store %s holds %d of %d %s%s' % (
+                    n, k, len(c), st.limit, st.keyed_by or 'entries',
+                    '; %d write(s) refused' % c.refused if c.refused else ' and refuses new ones when full'))
+        return out
+
+    def store_notes(self) -> list[str]:
+        """A TABLE or SET that has evicted is dropping state it was keeping
+        on purpose. Not a fault - the bound is the decision - but worth
+        reading when something it tracked is unexpectedly gone."""
+        out: list[str] = []
+        for n, k, c in self._containers():
+            if c.store.kind in (stores_mod.TABLE, stores_mod.SET) and c.evictions:
+                out.append('%s: store %s has dropped %d %s at its bound of %d (%s first)' % (
+                    n, k, c.evictions, c.store.keyed_by or 'entries', c.store.limit, c.store.evict))
+        return out
+
+    def __iter__(self) -> Iterator[CompartmentProtocol]:
+        return iter(self._by_name.values())
+
+    def __contains__(self, name: object) -> bool:
+        return name in self._by_name
+
+    def get(self, name: str) -> Optional[CompartmentProtocol]:
+        return self._by_name.get(name)
+
+    def names(self) -> list[str]:
+        return list(self._by_name)
+
+    # ---- assembly -----------------------------------------------------------
+    def assemble(self, ctx: Any, manifest: Sequence[type[Any]],
+                 enabled: Optional[Callable[[str], bool]] = None) -> list[str]:
+        """Build every compartment in the manifest from one context, in the
+        manifest's order, and register each as it is built.
+
+        The manifest is the node's declared tick order. It is *validated*
+        against each class's `requires` rather than re-sorted from it, because
+        tick order is a reviewed decision with reasons the dependency graph
+        does not carry (the relay journal ticks first so a dropped
+        subscription is said before anything that depends on it; the
+        operator's decisions tick before the mechanisms they outrank). A
+        manifest that lists a compartment before one it requires, or omits
+        one it requires, is a programming error and is refused at startup
+        with the two names in the message - not discovered three ticks in.
+
+        Optional compartments are built only when `enabled(name)` says so;
+        the default asks whether `enabled_by` is a present config key. A
+        compartment whose `api` major differs from this registry's, or whose
+        `protocol` is set and differs from the digest in `ctx`, is refused
+        the same way: the node would rather not start than start wrong.
+
+        Returns the names built, in order. The context's `node` (when it is
+        a NodeApp) gets each compartment set on it under its name, which is
+        what the rest of the node and the tests reach for today.
+        """
+        is_on: Callable[[str], bool] = enabled or (lambda key: bool(ctx.cfg.get(key)))
+        digest = getattr(ctx, 'digest', None)
+        setattr(ctx, '_registry', self)
+        built: list[str] = []
+        seen: list[str] = []            # every manifest name passed, built or not
+        for cls in manifest:
+            name = getattr(cls, 'name', None)
+            if not name:
+                raise ValueError('%s declares no name' % cls.__name__)
+            for dep in getattr(cls, 'uses', ()):
+                if dep not in seen:
+                    raise ValueError('%s uses %s, which the manifest does not list before it (manifest order: %s)'
+                                     % (name, dep, ' '.join(seen) or '(nothing yet)'))
+            seen.append(name)
+            api = getattr(cls, 'api', Compartment.API)
+            if api.split('/')[0] != Compartment.API.split('/')[0] or \
+                    api.split('/')[1].split('.')[0] != Compartment.API.split('/')[1].split('.')[0]:
+                raise ValueError('%s is written against %s; this node speaks %s' % (name, api, Compartment.API))
+            proto = getattr(cls, 'protocol', '')
+            if proto and digest and proto != digest:
+                raise ValueError('%s was built against schema %s; this node speaks %s' % (name, proto[:12], digest[:12]))
+            if getattr(cls, 'optional', False):
+                key = getattr(cls, 'enabled_by', None) or name
+                if not is_on(key):
+                    continue
+            for dep in getattr(cls, 'requires', ()):
+                if dep not in self._by_name:
+                    raise ValueError('%s requires %s, which is not built before it (manifest order: %s)'
+                                     % (name, dep, ' '.join(built) or '(nothing yet)'))
+            sub = cls.from_context(ctx)
+            self.add(sub)
+            refs = self.store_ref_problems(sub)
+            if refs:
+                del self._by_name[name]
+                self._life.pop(name, None)
+                raise ValueError('%s reads what is not there: %s' % (name, '; '.join(refs)))
+            built.append(name)
+            if ctx.node is not None:
+                setattr(ctx.node, getattr(cls, 'attr', None) or name, sub)
+        return built
+
+    def store_ref_problems(self, sub: Any) -> list[str]:
+        """Every `other.key` a command of `sub` reads, where `other` is built
+        and declares no store `key`.
+
+        The class lint (Compartment._command_data_problems) holds the ref to
+        `requires`/`uses`; only here do both sides exist, so only here can
+        the store itself be checked. The manifest puts every required and
+        used compartment before `sub`, so all that will be built already is.
+        One that is not built is an absent `uses` - optional by definition,
+        and the command must cope without it - and is skipped.
+        """
+        out: list[str] = []
+        refs = getattr(type(sub), 'store_refs', None)
+        for cmd, other, key in (refs() if callable(refs) else ()):
+            target = self._by_name.get(other)
+            if target is None:
+                continue
+            declared = {getattr(s, 'key', None) for s in getattr(target, 'STORES', ())}
+            if key not in declared:
+                out.append('command %s reads %s.%s, and %s declares no store %r (it declares: %s)'
+                           % (cmd, other, key, other, key, ', '.join(sorted(str(k) for k in declared)) or 'none'))
+        return out
+
+    # ---- the uniform verbs --------------------------------------------------
+    def wire(self, ctx: Any) -> None:
+        """Every compartment's back-edges, in manifest order, once all are built."""
+        for s in self:
+            for need in getattr(type(s), 'NEEDS', ()) or ():
+                if getattr(need, 'late', False):
+                    val = need.resolve_late(ctx, s, registry=self)
+                    setattr(s, need.key, val)
+            s.wire(self, ctx)
+            self._move(s.name, lifecycle_mod.WIRED)
+
+    def route(self, router: Any) -> list[tuple[str, str]]:
+        """Register every compartment's declared protocols on the router.
+
+        Returns [(compartment, piuri)] as registered. A compartment that
+        declares a protocol and has no `handle` is a programming error and
+        is refused here, at startup, with its name.
+        """
+        out: list[tuple[str, str]] = []
+        for s in self:
+            piuris = tuple(getattr(s, 'piuris', ()) or ())
+            exact = tuple(getattr(s, 'exact_types', ()) or ())
+            if not piuris and not exact:
+                continue
+            handle = getattr(s, 'handle', None)
+            is_base = getattr(handle, '__func__', handle) is Compartment.handle
+            if not callable(handle) or (is_base and not getattr(s, 'HANDLERS', ())):
+                raise ValueError('%s declares protocols %s but has no handle(inbound)'
+                                 % (s.name, ', '.join(piuris + exact)))
+
+
+            for p in piuris:
+                router.register(p, handle)
+                out.append((s.name or '', p))
+            for t in exact:
+                router.register_exact(t, handle)
+                out.append((s.name or '', t))
+        return out
+
+    def start(self) -> None:
+        """Start every compartment, in manifest order, each isolated.
+
+        One that raises is a refusal against itself, and what happens next
+        depends on whether the node can run without it (audit L1):
+
+          required (not `optional`)  every compartment already started is
+                                     stopped, newest first, and the raise goes
+                                     on and stops the node. A node that cannot
+                                     begin is a configuration the operator
+                                     must hear about at boot - and it must not
+                                     have half-begun, with a farewell sent by
+                                     something that never served.
+          optional                   the rest still start. It is QUARANTINED
+                                     when it declares restart (its trial
+                                     restarts it), FAILED when it does not.
+        """
+        begun: list[CompartmentProtocol] = []
+        for s in self:
+            tr = self._life.get(s.name or '')
+            try:
+                if hasattr(s, 'start_workers'):
+                    s.start_workers()
+                s.start()
+            except Exception as e:
+                s.refusals.refuse('start', 'raised', detail=why_detail(e))
+                err = '%s: %s' % (type(e).__name__, e)
+                if not getattr(s, 'optional', False):
+                    for o in reversed(begun):
+                        self._move(o.name, lifecycle_mod.DRAINING, why='%s could not start' % s.name)
+                        try:
+                            o.stop()
+                        except Exception as e2:
+                            o.refusals.refuse('stop', 'raised', detail=why_detail(e2))
+                        self._move(o.name, lifecycle_mod.RETIRED)
+                    if tr is not None:
+                        tr.started = False
+                        self._move(s.name, lifecycle_mod.FAILED, why='start raised: %s' % err)
+                    raise
+                if tr is not None:
+                    with self._life_lock:
+                        t = tr.start_failed(self.clock(), err)
+                    self._moved(t)
+                continue
+            if tr is not None:
+                tr.started = True
+            begun.append(s)
+            self._move(s.name, lifecycle_mod.RUNNING)
+
+    def tick(self, now: int) -> list[tuple[Any, ...]]:
+        """Every compartment's periodic work. One that raises does not stop the
+        others, and its failure is a refusal against itself rather than a
+        traceback nobody reads.
+
+        A raise also moves the compartment: to DEGRADED, or to QUARANTINED once
+        its breaker trips. A quarantined compartment is skipped until its
+        backoff passes, then given one trial tick - restarted first when it
+        declares restart, because "turn it off and on again" is the recovery a
+        stuck connection or a wedged worker needs. A retired or failed one is
+        skipped.
+        """
+        did: list[tuple[Any, ...]] = []
+        for s in self:
+            tr = self._life.get(s.name or '')
+            if tr is not None and not tr.due(now):
+                continue
+            with self._busy:
+                if tr is not None and tr.state == lifecycle_mod.QUARANTINED and tr.lifecycle.restart:
+                    if not self._restart(s, tr, 'trial after quarantine', trial=True, now=now):
+                        if tr.state == lifecycle_mod.QUARANTINED:
+                            s.refusals.refuse('tick', 'quarantined', detail={'until': tr.until, 'trips': tr.trips})
+                        did.append((s.name, 'restart failed', tr.last_error))
+                        continue
+                try:
+                    did += list(s.run_tick(now) if hasattr(s, 'run_tick') else (s.tick(now) or []))
+                except Exception as e:
+                    s.refusals.refuse('tick', 'raised', detail=why_detail(e))
+                    did.append((s.name, 'tick failed', '%s: %s' % (type(e).__name__, e)))
+                    if tr is not None:
+                        with self._life_lock:
+                            t = tr.failed(now, '%s: %s' % (type(e).__name__, e))
+                        self._moved(t)
+                        if tr.state == lifecycle_mod.QUARANTINED:
+                            s.refusals.refuse('tick', 'quarantined', detail={'until': tr.until, 'trips': tr.trips})
+                    continue
+            if tr is not None:
+                with self._life_lock:
+                    t = tr.succeeded(now)
+                self._moved(t)
+        return did
+
+    def stop(self) -> list[tuple[Optional[str], Any]]:
+        """Stop every compartment, and hand back what each had to say.
+
+        Returns [(name, whatever stop returned)]. One that raises does not stop
+        the others: a node that cannot say goodbye must still stop. One whose
+        start() raised is not stopped - it never began - and returns None.
+        """
+        out: list[tuple[Optional[str], Any]] = []
+        for s in self:
+            tr = self._life.get(s.name or '')
+            if tr is not None and tr.started is False:
+                out.append((s.name, None))
+                self._move(s.name, lifecycle_mod.RETIRED, why='never started')
+                continue
+            self._move(s.name, lifecycle_mod.DRAINING)
+            try:
+                out.append((s.name, s.stop()))
+            except Exception as e:
+                s.refusals.refuse('stop', 'raised', detail=why_detail(e))
+                out.append((s.name, None))
+            finally:
+                if hasattr(s, 'stop_workers'):
+                    try:
+                        s.stop_workers()
+                    except Exception:
+                        pass
+            self._move(s.name, lifecycle_mod.RETIRED)
+        return out
+
+    def report(self) -> dict[str, Mapping[str, Any]]:
+        """Every compartment's report, each isolated as trouble() and notes()
+        already were.
+
+        This is what health(), status() and status.json are made of, and the
+        tick writes status.json. One compartment whose report() raised used to
+        raise here, so the node did its work and then failed the tick, every
+        tick, and `meshnode health` failed with it (audit 2026-10-15, B2). A
+        report that raises is now that compartment's entry, an `error`, and a
+        `report_error` trouble until a later report() succeeds - the others
+        still report.
+        """
+        out: dict[str, Mapping[str, Any]] = {}
+        failed: dict[str, str] = {}
+        for s in self:
+            try:
+                out[s.name or ''] = s.report()
+            except Exception as e:
+                failed[s.name or ''] = '%s: %s' % (type(e).__name__, e)
+                out[s.name or ''] = {'error': failed[s.name or '']}
+        self._report_failed = failed
+        return out
+
+    def trouble(self) -> list[str]:
+        out = TroubleList()
+        for s in self:
+            try:
+                for t in (s.trouble() or []):
+                    if isinstance(t, Trouble):
+                        comp = t.compartment or s.name or ''
+                        msg = '%s: %s' % (comp, t.message)
+                        out.append(Trouble(
+                            msg,
+                            code=t.code,
+                            compartment=comp,
+                            severity=t.severity,
+                            since=t.since,
+                            hint=t.hint,
+                            **t.metadata,
+                        ))
+                    else:
+                        out.append(Trouble(
+                            '%s: %s' % (s.name, t),
+                            code='fault',
+                            compartment=s.name or '',
+                            severity='trouble',
+                        ))
+            except Exception as e:
+                out.append(Trouble(
+                    '%s: could not report trouble (%s)' % (s.name, e),
+                    code='report_error',
+                    compartment=s.name or '',
+                    severity='fatal',
+                ))
+        # A quarantine is trouble whatever the compartment itself reports:
+        # while its breaker is open it is not ticking, and it may not be in a
+        # position to say so.
+        for n in self.quarantined():
+            tr = self._life[n]
+            msg = ('%s: quarantined - not ticking until %s (trip %d; last: %s); '
+                   'the control command `resume` with name=%s ends it now'
+                   % (n, time.strftime('%H:%M:%SZ', time.gmtime(tr.until)), tr.trips,
+                      tr.last_error or 'unknown', n))
+            out.append(Trouble(msg, code='quarantined', compartment=n, severity='fault'))
+        for n in self.failed():
+            tr = self._life[n]
+            msg = ('%s: failed to start (%s) and does not declare restart; it is not running '
+                   'until the node restarts' % (n, tr.last_error or 'unknown'))
+            out.append(Trouble(msg, code='failed_start', compartment=n, severity='fatal'))
+        # As of the last report(): health() takes the report first, and a
+        # second report() here would double the cost of every health read.
+        for n, err in self._report_failed.items():
+            out.append(Trouble('%s: could not report its state (%s)' % (n, err),
+                               code='report_error', compartment=n, severity='fatal'))
+        for st in self.store_trouble():
+            out.append(Trouble(st, code='store_trouble', severity='trouble'))
+        return out
+
+    def trouble_items(self) -> list[Trouble]:
+        """All active troubles as structured Trouble instances."""
+        res = self.trouble()
+        return [t if isinstance(t, Trouble) else Trouble(str(t)) for t in res]
+
+
+    def notes(self) -> NoteList:
+        out = NoteList()
+        for s in self:
+            try:
+                for t in (s.notes() or []):
+                    if isinstance(t, Note):
+                        out.append(Note(
+                            '%s: %s' % (s.name, t),
+                            topic=t.topic,
+                            compartment=s.name or '',
+                            level=t.level,
+                            since=t.since,
+                            **t.metadata,
+                        ))
+                    else:
+                        out.append(Note('%s: %s' % (s.name, t), compartment=s.name or ''))
+            except Exception as e:
+                out.append(Note('%s: could not report notes (%s)' % (s.name, e), compartment=s.name or '', level='notice'))
+        for n, tr in self._life.items():
+            if tr.state == lifecycle_mod.DEGRADED:
+                out.append(Note(
+                    '%s: degraded - the last tick raised (%s); it recovers after one clean tick'
+                    % (n, tr.last_error or 'unknown'),
+                    compartment=n,
+                    topic='degraded',
+                    level='notice',
+                ))
+        for sn in self.store_notes():
+            out.append(Note(sn, topic='store', level='info'))
+        return out
+
+    def note_items(self) -> list[Note]:
+        """All active notes as structured Note instances."""
+        res = self.notes()
+        return [n if isinstance(n, Note) else Note(str(n)) for n in res]
+
+
+    def refusals(self, n: int = 5) -> list[tuple[Optional[str], Refusal]]:
+        """Every no this node said, across every compartment, newest last."""
+        out: list[tuple[Optional[str], Refusal]] = []
+        for s in self:
+            for r in s.refusals.last(n):
+                out.append((s.name, r))
+        return sorted(out, key=lambda p: p[1].at)
+
+    def live_keys(self) -> tuple[str, ...]:
+        keys: list[str] = []
+        for s in self:
+            keys += [k for k in s.live if k not in keys]
+        return tuple(keys)
+
+    def commands(self) -> collections.OrderedDict[str, CommandSpec]:
+        """name -> spec, prefixed by subsystem, for the control socket."""
+        out: collections.OrderedDict[str, CommandSpec] = collections.OrderedDict()
+        for s in self:
+            for cmd, spec in (s.commands or {}).items():
+                named: CommandSpec = {**spec, 'subsystem': s.name or ''}
+                out['%s.%s' % (s.name, cmd)] = named
+        return out
+
+    def declared_commands(self) -> tuple[Any, ...]:
+        """Every Command declared across compartments."""
+        out: list[Any] = []
+        for s in self:
+            out.extend(getattr(s, 'COMMANDS', ()))
+        return tuple(out)
+
+    def dispatch(self, name: str, args: Optional[Mapping[str, Any]]) -> Any:
+        # All three refusals below are the caller naming something that does
+        # not exist - a typo, not a fault in this node. They were plain
+        # KeyErrors, which the control socket's boundary treats as a bug: it
+        # prefixes the type name and appends to its 200-entry ring, and
+        # health reports the last five of that ring. So five `meshctl` typos
+        # put five errors in the node's own health and pushed any real bug
+        # out of the window (audit 2026-10-07, I14). ControlError is the
+        # declared way to answer the caller and leave no trace. The import is
+        # function-local to keep compartment/ off app/ at module scope, which
+        # is how the compartments that already raise it do it.
+        from ..app import socket as control_mod
+        sub, _, cmd = name.partition('.')
+        s = self.get(sub)
+        if s is None:
+            raise control_mod.ControlError("unknown compartment %r in command %r" % (sub, name))
+        if not cmd:
+            raise control_mod.ControlError("missing command in %r" % (name,))
+        if s.commands and cmd not in s.commands:
+            raise control_mod.ControlError(
+                "unknown command %r on compartment %r; available: %s" % (cmd, sub, list(s.commands)))
+        try:
+            return s.command(cmd, args or {})
+        except KeyError as e:
+            # `Compartment.command` raises KeyError(name) for a name it does
+            # not declare - the same caller mistake as the three above, one
+            # level down, reached when a compartment publishes no `commands`
+            # mapping for the check above to screen against. Only that exact
+            # shape is converted: a KeyError raised from inside a command's
+            # own body is a real bug and must keep reaching the error ring.
+            if e.args == (cmd,):
+                raise control_mod.ControlError(
+                    "unknown command %r on compartment %r" % (cmd, sub)) from None
+            raise