fleetmesh-node 0.2.0__py3-none-any.whl

node/planes/peering/budget/core.py

@@ -0,0 +1,537 @@
+"""The Budget subsystem itself: construction and restore, the meters, the
+window arithmetic, what the sync layer reports, and the Compartment contract.
+Separate from the mixins because this is where the state they all share is
+created; the mixins are the concerns that act on it.
+"""
+from __future__ import annotations
+
+import collections
+import threading
+import time
+from typing import Any, Callable, Counter, Mapping, Optional
+
+from node.interfaces import ReceiptRefusedRow
+
+from fleetmesh_core.accounting import accounting, standing
+from fleetmesh_core.wire.interfaces import ReceiptBody
+from fleetmesh_core.wire.wire_types import GoverningRow
+from node.compartment import stores as st
+from node.storage import accounts as accounts_mod, grants as grants_mod
+from node.compartment import Command, Handler, Need, Outbound, WireProtocol
+import dataclasses
+from node.config.parameter import Parameter, ParamBlock
+from node.channels import Channel, NOTE, TROUBLE, bus as eventbus_mod
+from node.metrics import COUNTER, Metric
+from .chain import OutChain
+from .evaluation import Evaluation, Repricing
+from .meter import C, KEEP_FINDINGS, PIURI, SYNC, TOLERANCE, Meter, _observe
+from .receipts import ReceiptsMixin
+from .tick import TickMixin
+from .windows import WindowsMixin
+from .types import BudgetStateReport
+
+
+@dataclasses.dataclass(frozen=True)
+class BudgetConfig(ParamBlock):
+    key = 'budget'
+    name = 'budget'
+    DERIVED: bool = False
+    PARAMS = (
+        Parameter(
+            key='tolerance', name='Divergence Tolerance',
+            description='Ratio by which measured usage may exceed limit before divergence',
+            type=float, default=TOLERANCE, minimum=0.0, maximum=1.0,
+            tags=('budget', 'tolerance'),
+        ),
+        Parameter(
+            key='keep_windows', name='Keep Windows',
+            description='Number of closed windows retained before pruning',
+            type=int, default=24 * 7, minimum=1,
+            tags=('budget', 'history'),
+        ),
+        Parameter(
+            key='keep_findings', name='Keep Findings',
+            description='Maximum adverse findings kept in memory',
+            type=int, default=KEEP_FINDINGS, minimum=10,
+            tags=('budget', 'findings'),
+        ),
+    )
+    tolerance: float = TOLERANCE
+    keep_windows: int = 24 * 7
+    keep_findings: int = KEEP_FINDINGS
+
+
+class Budget(ReceiptsMixin, WindowsMixin, TickMixin, WireProtocol):
+    requires: tuple[str, ...] = ('peering', 'strain')
+    """The accounting, as a compartment the operator can watch.
+
+    Its refusals are the ones that mattered: a receipt this node would not read,
+    one naming a window this grant cannot be about, and one naming a grant that
+    did not govern the window. That last was refused silently for a night on a
+    four-node mesh while every window closed as active silence, and nothing
+    anywhere said the two ends were naming different grants. It goes through the
+    channel every compartment has now.
+
+    Its `stop` is the farewell: a node that vanishes mid-window leaves its issuer
+    with traffic measured and no receipt, which is an S2 nobody earned.
+    """
+
+    name: str = 'budget'
+    piuris: tuple[str, ...] = (PIURI,)
+    config_cls = BudgetConfig
+    CHANNELS = (
+        Channel('window_evaluated', 'A closed window was evaluated against peer limits',
+                level=NOTE, wire=eventbus_mod._wire_evaluated,
+                say='evaluated window {epoch} for peer {peer} severity {severity}',
+                tags=('budget', 'evaluation')),
+        Channel('window_repriced', 'A peer limit was repriced under AIMD congestion control',
+                level=NOTE, wire=eventbus_mod._wire_repriced,
+                say='repriced peer {peer} for window {epoch}: {was} -> {now}',
+                tags=('budget', 'repricing')),
+        Channel('receipt_refused', 'A receipt from a peer was refused',
+                level=TROUBLE, say='receipt refused from peer {peer}: {reason}',
+                tags=('budget', 'receipt')),
+    )
+    METRICS = (
+        Metric(name='budget_windows_evaluated', description='Windows evaluated.',
+               kind=COUNTER, counts='window_evaluated', tags=('budget', 'evaluation')),
+        Metric(name='budget_windows_repriced', description='Windows repriced under AIMD.',
+               kind=COUNTER, counts='window_repriced', tags=('budget', 'repricing')),
+        Metric(name='budget_receipts_refused', description='Receipts refused from peers.',
+               kind=COUNTER, counts='receipt_refused', tags=('budget', 'receipt')),
+    )
+    OUTBOUND = (
+        Outbound('receipt', reply='receipt-ack', doc='deliver window accounting receipt'),
+        Outbound('close', doc='signal cooperative window closure'),
+        Outbound('receipt-ack', doc='acknowledge received receipt and head status'),
+    )
+    HANDLERS = (
+        Handler('receipt', run='handle_receipt', doc='answer inbound budget/1.0 receipt'),
+        Handler('close', run='handle_close', doc='answer inbound budget/1.0 window close'),
+        Handler('receipt-ack', run='handle_receipt_ack', doc='take inbound budget/1.0 receipt-ack'),
+    )
+    STORES = (
+        st.Store(key='evaluations', description='completed window evaluations',
+                 kind=st.RING, shape=Evaluation, bound=KEEP_FINDINGS),
+        st.Store(key='findings', description='adverse window findings',
+                 kind=st.RING, shape=tuple, bound=KEEP_FINDINGS),
+        st.Store(key='clock_drift', description='receipt clock drift events',
+                 kind=st.RING, shape=tuple, bound=KEEP_FINDINGS),
+        st.Store(key='receipt_gaps', description='receipt sequence gaps',
+                 kind=st.RING, shape=tuple, bound=KEEP_FINDINGS),
+        st.Store(key='unpublished', description='receipts unpublishable to relays',
+                 kind=st.RING, shape=tuple, bound=KEEP_FINDINGS),
+    )
+    COMMANDS = (
+        Command(
+            key='show',
+            name='Budget',
+            description='what this node is accounting: peers metered, windows evaluated, findings'
+                        ' held',
+            tags=('budget', 'status'),
+            permission='budget/show',
+            run='state',
+            reply='{peers, windows_evaluated, findings, grants_waiting_for_a_boundary, '
+                  'clock_drift, receipt_chains_open, idle_streaks, rejected, receipts_refused}',
+        ),
+    )
+
+    # Windows kept after they close. A window is history once it has been
+    # evaluated and the complaint deadline on it has passed, but a report may
+    # still cite the receipt that backs it, so the horizon is generous. Without
+    # one the accounting store grows for as long as the node runs. It is also
+    # how far back a receipt may reach; see on_receipt.
+    KEEP_WINDOWS: int = 24 * 7
+
+    evaluations: st.Ring[Evaluation]
+    findings: st.Ring[tuple[Any, ...]]
+    clock_drift: st.Ring[tuple[Any, ...]]
+    receipt_gaps: st.Ring[tuple[Any, ...]]
+    unpublished: st.Ring[tuple[Any, ...]]
+
+    NEEDS = (
+        Need('keystore', 'ks'),
+        Need('transport'),
+        Need('grant_store', 'stores.grants'),
+        Need('peering'),
+        Need('ledger'),
+        Need('accounts', 'stores.accounts'),
+        Need('strain'),
+        Need('events'),
+        Need('reporter', late=True),
+    )
+
+    def start(self) -> None:
+        if self.reporter is not None:
+            if self.reporter.on_evaluated not in self.on_evaluated:
+                self.on_evaluated.append(self.reporter.on_evaluated)
+            if self.reporter.on_repriced not in self.on_repriced:
+                self.on_repriced.append(self.reporter.on_repriced)
+        if self.events is not None:
+            events = self.events
+            self.on_evaluated.append(lambda rec: events.emit(eventbus_mod.TOPIC_WINDOW_EVALUATED, rec.to_dict()))
+            self.on_repriced.append(lambda *a: events.emit(eventbus_mod.TOPIC_WINDOW_REPRICED, *a))
+
+    def __init__(
+        self,
+        keystore: Any,
+        transport: Any,
+        grant_store: grants_mod.GrantStore,
+        peering: Any,
+        clock: Optional[Callable[[], int]] = None,
+        ledger: Optional[standing.Ledger] = None,
+        accounts: Optional[Any] = None,
+        strain: Optional[Any] = None,
+        node: Optional[Any] = None,
+        events: Optional[Any] = None,
+        reporter: Optional[Any] = None,
+    ) -> None:
+        super().__init__(node, clock)
+        self.strain: Optional[Any] = strain            # what this node has survived, or None
+        self.accounts: Optional[Any] = accounts        # AccountStore, or None for a node that forgets
+        self.events: Optional[Any] = events
+        self.reporter: Optional[Any] = reporter
+        self.ks: Any = keystore
+        self.transport: Any = transport
+        self.grants: grants_mod.GrantStore = grant_store
+        self.peering: Any = peering                  # for the WireNode that writes envelopes
+        self.ledger: standing.Ledger = ledger or standing.Ledger()
+        self._lock: threading.RLock = threading.RLock()
+        # (peer, direction) -> (window seconds, store change counter, monotonic at)
+        self._window_memo: dict[tuple[str, str], tuple[int, int, float]] = {}  # not a store: memoized window calculations
+        self.usage_by: dict[str, dict[int, Meter]] = {}  # not a store: restored from accounts, for persist=DURABLE
+        self.usage_of_me: dict[str, dict[int, Meter]] = {}  # not a store: restored from accounts, for persist=DURABLE
+        self.receipts_in: dict[str, dict[int, list[ReceiptBody]]] = {}  # not a store: restored from accounts, for persist=DURABLE
+        self.heads_in: dict[str, dict[int, str]] = {}  # not a store: derived from restored receipts, for persist=DURABLE
+        self.chain_out: dict[str, dict[int, OutChain]] = {}  # not a store: restored from accounts, for persist=DURABLE
+        self.acks: dict[str, Mapping[str, Any]] = {}  # not a store: last ack per live peer
+        self.evaluated: set[tuple[str, int]] = set()  # not a store: restored from accounts, for persist=DURABLE
+        self.zero_until: dict[str, int] = {}  # not a store: time penalty per zeroed peer
+        self.idle: dict[str, int] = {}  # not a store: restored from accounts evaluations, consecutive closed idle windows
+        # Each of these carries the time it happened, and `trouble` reads only
+        # the ones inside TROUBLE_WINDOW. A cumulative counter with no time
+        # bound warns forever about a fault that has stopped: every node here
+        # spent a day reporting 151 clock-drift events that were hours dead
+        # against NTP-synced boxes, and a peering warned for four hours about
+        # one message lost in a restart while completing thirty-four sessions.
+        # The compartment contract already says what to do - short enough that a
+        # fault which has stopped stops being reported - and the six
+        # compartments that report through `Refusals.within` already did it.
+        self.started: int = self.clock()                         # a receipt dated before this cannot be told from a relay replay
+        self.ack_lag: dict[str, tuple[int, int, int]] = {}  # not a store: ack lag per live peer
+        self.ack_head: dict[str, tuple[int, int]] = {}  # not a store: ack head per live peer
+        # Refusals go through the compartment channel. `rejected` remains as a
+        # read-only view of the same counts, because the health block, the CLI
+        # and three suites already read it by that name.
+        self.on_evaluated: list[Callable[[Evaluation], Any]] = []       # not a store: callbacks after every evaluated window
+        self.reopen: set[tuple[str, int]] = set()  # not a store: priced windows with late receipts pending evaluation
+        self.repriced: list[Repricing] = []  # not a store: late receipt repricing records
+        self.on_repriced: list[Callable[[str, int, str, str], Any]] = []    # not a store: callbacks after a correction
+        if self.accounts is not None:
+            self._restore()
+
+    def _restore(self) -> None:
+        """Pick the accounting back up where the last process left it.
+
+        Without this a restart begins a fresh receipt chain at seq 0 while the
+        issuer is still expecting seq N, and reports counters lower than it
+        already reported: a broken chain and a counter going backwards, which
+        is an S2 misreport filed against a node that did nothing wrong.
+        """
+        assert self.accounts is not None
+        m = self.accounts.meters()
+        for side, table in ((accounts_mod.BY, self.usage_by), (accounts_mod.OF_ME, self.usage_of_me)):
+            for peer, by_epoch in m.get(side, {}).items():
+                for epoch, counters in by_epoch.items():
+                    meter = table.setdefault(peer, {}).setdefault(epoch, Meter())
+                    meter.c.update(counters)
+        self.receipts_in = self.accounts.receipts()
+        for peer, by_epoch in self.receipts_in.items():
+            for epoch, chain in by_epoch.items():
+                if chain:
+                    self.heads_in.setdefault(peer, {})[epoch] = accounting.link(chain[-1])
+        self.chain_out = {peer: {epoch: OutChain.from_dict(s) for epoch, s in by_epoch.items()}  # not a store: restored from accounts, for persist=DURABLE
+                          for peer, by_epoch in self.accounts.chains().items()}
+        self.evaluated = self.accounts.evaluated()
+        # The ledger is what standing/1.0 tells a stranger and what weights a
+        # reporter. It was memory only, so a restart set every streak to zero
+        # and a reporter this node had watched for a week weighed nothing.
+        for peer, epoch, severity, tier in self.accounts.evaluations():
+            if severity is None:
+                continue
+            if tier is None:        # recorded before the tier was kept: the grant that governed it
+                g = self.grants.governing(peer, grants_mod.ISSUED, epoch)
+                tier = (g['payload'].get('tier') if g else None) or 'probation'
+            self.ledger.record(peer, epoch, severity, tier)
+        # The idle streak decides when a dormant grant decays. Counted only in
+        # memory it would begin again at zero on every restart, and a node that
+        # is redeployed weekly would never reach the threshold at all - the same
+        # shape as the standing streak above.
+        rows = self.accounts.evaluations()
+        for peer in {r[0] for r in rows}:
+            run = 0
+            for _p, _e, sev, _t in [r for r in rows if r[0] == peer]:
+                run = run + 1 if sev is None else 0
+            self.idle[peer] = run
+
+    def _save_meters(self, peer: Optional[str] = None, epoch: Optional[int] = None) -> None:
+        if self.accounts is None:
+            return
+        # The sync thread adds a peer's first meter the moment its first
+        # session opens; the tick walks these tables. Snapshot under the lock.
+        with self._lock:
+            rows = [(side, pk, e, meter)
+                    for side, table in ((accounts_mod.BY, self.usage_by), (accounts_mod.OF_ME, self.usage_of_me))
+                    for pk, by_epoch in table.items() if peer is None or pk == peer
+                    for e, meter in by_epoch.items() if epoch is None or e == epoch]
+        for side, pk, e, meter in rows:
+            self.accounts.put_meter(pk, e, side, meter.snapshot(), self.clock())
+
+    # ---- window arithmetic ----------------------------------------------------
+    # How long a remembered window size may be trusted without re-reading the
+    # grant. A window is a grant payload field that changes only when a grant
+    # is re-issued; any write through this node's own store connection drops
+    # the memo at once (sqlite's total_changes moves), and the TTL covers a
+    # write from another process on the same file.
+    WINDOW_MEMO_SECONDS: float = 2.0
+
+    def _window_of(self, peer: str, direction: grants_mod.GrantDirection) -> int:
+        """The window size of the grant in force, in seconds.
+
+        Every observed sync message asked this three times, and each ask was a
+        sqlite read plus a json.loads of the whole payload: measured at ~70%
+        of the cost of metering a message. The memo below is keyed on the
+        connection's change counter so a grant written through this store is
+        seen immediately.
+        """
+        key = (peer, direction)
+        changes = self.grants.db.total_changes
+        now_mono = time.monotonic()
+        memo = self._window_memo.get(key)
+        if memo is not None and memo[1] == changes and now_mono - memo[2] < self.WINDOW_MEMO_SECONDS:
+            return memo[0]
+        row = self.grants.get(peer, direction)
+        w = int((row['payload'].get('window') if row else None) or C['window_seconds_default'])
+        self._window_memo[key] = (w, changes, now_mono)
+        return w
+
+    def _peers(self) -> list[str]:
+        """The peers holding a live grant, as a plain list of pubkeys.
+
+        GrantStore.list() shadows the builtin inside the class body, so the
+        checker cannot read GrantStore.peers()' own annotation; this is the one
+        place the node re-states it (the fix belongs in node/grants.py).
+        """
+        # Read through Any: the store's return annotation is unusable to the
+        # checker, so its shape is re-stated on this side and validated by the
+        # call sites that index/sort it.
+        raw: Any = self.grants.peers()
+        peers: list[str] = [str(p) for p in raw]
+        return peers
+
+    def governing(self, peer: str, direction: grants_mod.GrantDirection, epoch: int) -> Optional[GoverningRow]:
+        """The grant that governed one window, which is what the window is
+        priced under and what a receipt for it names."""
+        return self.grants.governing(peer, direction, epoch)
+
+    def _acceptable_grants(self, peer: str, epoch: int) -> set[str]:
+        """Which grant ids a receipt for this window may name: the one that
+        governed it, the one before (an amend published at the boundary may
+        not have reached the subject yet) and the one current now (a seed
+        applied mid-window)."""
+        ids: set[str] = set()
+        for e in (epoch - 1, epoch):
+            g = self.grants.governing(peer, grants_mod.ISSUED, e)
+            if g:
+                ids.add(g['envelope_id'])
+        row = self.grants.get(peer, grants_mod.ISSUED)
+        if row:
+            ids.add(row['envelope_id'])
+        return ids
+
+    def epoch(self, peer: str, direction: grants_mod.GrantDirection, at: Optional[int] = None) -> int:
+        return accounting.epoch(at if at is not None else self.clock(), self._window_of(peer, direction))
+
+    def aggregate(self, at: Optional[int] = None) -> dict[str, int]:
+        """What every peer together is doing to this node, in the open window.
+
+        The issuer frame - `usage_by` is what a peer did to me - because that is
+        the frame a ceiling is about. A grant bounds one peer and nothing bounded
+        the node, so this is the number `capacity` measures pressure against.
+
+        Windows are per peering, so each peer contributes its own open epoch.
+        With the default window that is one hour for all of them; with mixed
+        windows it is still each peer's current promise, which is the honest
+        reading of "what is happening to me now".
+        """
+        now = at if at is not None else self.clock()
+        total: Counter[str] = collections.Counter()
+        with self._lock:
+            peers = list(self.usage_by)
+        for peer in peers:
+            e = self.epoch(peer, grants_mod.ISSUED, now)
+            with self._lock:
+                m = self.usage_by.get(peer, {}).get(e)
+                snap = m.snapshot() if m is not None else None
+            if snap:
+                total.update(snap)
+        return dict(total)
+
+    def _meter(self, table: dict[str, dict[int, Meter]], peer: str, epoch: int) -> Meter:
+        with self._lock:
+            return table.setdefault(peer, {}).setdefault(epoch, Meter())
+
+    # ---- what the sync layer reports ----------------------------------------
+    def observe(self, peer: str, message: Mapping[str, Any], i_initiated: bool, outbound: bool) -> None:
+        """One sync message, charged to the session's initiator.
+
+        If I opened the session it is my usage under the peer's grant,
+        whichever way this message went; if the peer opened it, it is the
+        peer's usage under mine. The wire knows who opened it by the thid.
+
+        An `open` is flushed before this returns, because the counter it moves
+        is the one this node's own guard reads to decide whether it may open
+        the next session. `observe_session` flushes at session END, which
+        bounds the loss to one session in flight - and one session in flight
+        is exactly what a restart loses. The node then reloaded a meter one
+        short, `throttled()` found room that was not there, and the node
+        opened a session the grant did not cover while believing it did. The
+        peer measured the overrun and priced it: S1 on conn_rate at 5 over a
+        grant of 4, which cost a peer 51 clean windows for restarting. See
+        I15 in docs/audit-2026-10-07-4-issues.md, where the arithmetic is
+        reproduced. Nothing else here needs to be durable this early - the
+        volume counters are reconciled by the receipt and their divergence
+        floor already absorbs a lost session by construction - so this is the
+        one message kind that writes through.
+        """
+        if not message.get('type', '').startswith(SYNC + '/'):
+            return
+        mine = self._meter(self.usage_of_me, peer, self.epoch(peer, grants_mod.RECEIVED))
+        theirs = self._meter(self.usage_by, peer, self.epoch(peer, grants_mod.ISSUED))
+        session_meter = mine if i_initiated else theirs
+        deliverer_meter = mine if outbound else theirs
+        _observe(session_meter, deliverer_meter, message, from_initiator=(i_initiated == outbound))
+        if message.get('type', '').rsplit('/', 1)[-1] == 'open' and self.accounts is not None:
+            # Exactly the one row the open moved, not every epoch on both
+            # sides: this is on the sync path, and `_save_meters(peer)` would
+            # rewrite each remembered window of each side per session.
+            side = accounts_mod.OF_ME if i_initiated else accounts_mod.BY
+            at = self.epoch(peer, grants_mod.RECEIVED if i_initiated else grants_mod.ISSUED)
+            self.accounts.put_meter(peer, at, side, session_meter.snapshot(), self.clock())
+
+    def observe_in(self, peer: str, message: Mapping[str, Any], i_initiated: bool = False) -> None:
+        """A sync message the peer sent me."""
+        self.observe(peer, message, i_initiated, outbound=False)
+
+    def observe_out(self, peer: str, message: Mapping[str, Any], i_initiated: bool = True) -> None:
+        """A sync message I sent the peer."""
+        self.observe(peer, message, i_initiated, outbound=True)
+
+    def observe_session(self, peer: str, role: str, result: Mapping[str, Any]) -> None:
+        """After a session: duplicates the peer delivered to me count against
+        them, and both meters for this peer reach the store.
+
+        `usage_of_me` was written down at every receipt - `send_receipt` puts
+        the meter before the message leaves - and `usage_by` was written only
+        by the tick. So a node that stopped between ticks came back having
+        forgotten some of what a peer had done to it while remembering all of
+        what it had done to the peer, and an issuer that under-measures reads
+        an honest claim as too high: divergence, and an S2 misreport against a
+        peer that did nothing but keep working across somebody else's crash.
+        The asymmetry was the bug. A session is the natural boundary, the same
+        one the other side already uses, and it bounds the loss to one session
+        in flight rather than a whole tick interval.
+        """
+        dups = result.get('counters', {}).get('dups_sent', 0)
+        if role == 'initiator' and dups:
+            self._meter(self.usage_by, peer, self.epoch(peer, grants_mod.ISSUED)).add('dups_sent', dups)
+        self._save_meters(peer)
+
+    # ---- the compartment contract -------------------------------------------------
+    @property
+    def rejected(self) -> Counter[str]:
+        """The refusal counts, under the name the health block and the suites
+        already use. One channel, two readers."""
+        return self.refusals.counts
+
+    def state(self) -> BudgetStateReport:
+        pending = sum(1 for p in self._peers() if self.grants.pending(p))
+        return {
+            'peers': len(self._peers()),
+            'windows_evaluated': len(self.evaluated),
+            'findings': len(self.findings),
+            'grants_waiting_for_a_boundary': pending,
+            'clock_drift': len(self.clock_drift),
+            'receipt_gaps': len(self.receipt_gaps),
+            'receipts_behind': {pk: n for pk, (_e, n, _at) in self.ack_lag.items() if n},
+            'receipts_unpublished': len(self.unpublished),
+            'receipt_chains_open': sum(len(v) for v in self.chain_out.values()),
+            'idle_streaks': {p[:16]: n for p, n in self.idle.items() if n},
+            # Declared rather than reached for: the health block read both
+            # of these off this object by name, which is why `rejected` had
+            # to survive as a property after the refusal channel replaced it.
+            'rejected': dict(self.refusals.counts),
+            'receipts_refused': self.receipts_refused(),
+        }
+
+    def receipts_refused(self) -> list[ReceiptRefusedRow]:
+        """The windows this node would not take a receipt for, because the two
+        ends name different grants. Five at most: it is a fault to fix, not a
+        history to read, and the counts keep the history."""
+        return [ReceiptRefusedRow(peer=r.subject and r.subject[:16], window=r.detail.get('window'),
+                                  named=r.detail.get('named'), acceptable=r.detail.get('acceptable') or [],
+                                  at=r.at)
+                for r in self.refusals.within(self.TROUBLE_WINDOW, self.clock(),
+                                              why='names a grant that did not govern that window')][-5:]
+
+    def trouble(self) -> list[str]:
+        out: list[str] = []
+        disagree = self.refusals.within(self.TROUBLE_WINDOW, self.clock(),
+                                        why='names a grant that did not govern that window')
+        if disagree:
+            peers = sorted({(r.subject or '')[:16] for r in disagree})
+            out.append('receipts refused over a grant disagreement with %s - every window '
+                       'those peerings close will be active silence until it is resolved'
+                       % ', '.join(peers))
+        unaddressable = self.refusals.within(self.TROUBLE_WINDOW, self.clock(),
+                                             kind='accounting', why='the peer raised')
+        if unaddressable:
+            peers = sorted({(r.subject or '')[:16] for r in unaddressable})
+            out.append('accounting raised against %s and that window\'s work was skipped for '
+                       'them: most often a descriptor this node will not accept, which is what a '
+                       'peer left behind a protocol digest looks like from here' % ', '.join(peers))
+        # Only what is still happening. The record stays whole in `state`.
+        now = self.clock()
+        recent = lambda rows: [r for r in rows if now - r[-1] <= self.TROUBLE_WINDOW]
+        drifting, unsent_now, gaps_now = recent(self.clock_drift), recent(self.unpublished), recent(self.receipt_gaps)
+        if drifting:
+            out.append('%d receipts have been timed outside the tolerance: check NTP at both ends '
+                       'before reading any divergence' % len(drifting))
+        lagging = {pk: n for pk, (_e, n, at) in self.ack_lag.items() if n and now - at <= self.TROUBLE_WINDOW}
+        if lagging:
+            out.append('%d peer(s) are acking a chain head behind the one this node sent, by up to %d '
+                       'receipt(s): the claims exist and are not arriving'
+                       % (len(lagging), max(lagging.values())))
+        if unsent_now:
+            out.append('%d receipt(s) no relay would take at all' % len(unsent_now))
+        if gaps_now:
+            out.append('%d receipt(s) never arrived and the chain carried on past them: the wire is '
+                       'dropping claims, and a window missing its tail still prices as an under-claim'
+                       % len(gaps_now))
+        return out
+
+    def stop(self) -> list[tuple[Any, ...]]:
+        """Say what this node used before it goes. Never raises."""
+        try:
+            return self.farewell()
+        except Exception as e:
+            self.refusals.refuse('farewell', 'could not be sent', detail={'error': str(e)})
+            return []
+
+    def run(self, stop: threading.Event, interval: float = 5.0) -> None:
+        while not stop.is_set():
+            try:
+                self.tick()
+            except Exception as e:      # the loop must outlive one bad tick
+                self.findings.append(('self', None, None, 'tick failed: %s' % e))
+            stop.wait(interval)

node/planes/peering/budget/evaluation.py

@@ -0,0 +1,80 @@
+"""The record one closed window leaves behind.
+
+`Budget.evaluate` builds one per (peer, epoch) and appends it to
+`budget.evaluations`; the reporter reads it to decide whether a finding
+opens, the event bus wires a six-key slice of it, `status()` shows the tail,
+and tests reach in for severity and detail. Nothing is stored whole: the
+AccountStore keeps (peer, epoch, severity, tier) and the rest is rebuilt on
+the next window.
+
+A window that neither moved nor claimed anything is `not-evaluated`: it
+has no severity, no detail, and no remedy. Those fields are Optional for
+that reason only - an evaluated window has all of them.
+"""
+
+from __future__ import annotations
+
+from dataclasses import asdict, dataclass, field
+from typing import Any, Mapping, Optional, cast
+from fleetmesh_core.wire.interfaces import (  # noqa: E402
+    ReceiptBody,
+    SilenceEvaluation,
+    WindowEvaluationDetail,
+)
+from .types import EvaluationDict, RepricingDict
+
+__all__ = ['Evaluation', 'Repricing', 'evaluation_like']
+
+
+@dataclass(slots=True)
+class Evaluation:
+    peer: str
+    epoch: int
+    state: str                                 # fleetmesh_core.accounting.window_claim: claim / active-silence / ...
+    measured: dict[str, int]                   # what this node counted, by metric
+    claimed: Optional[dict[str, int]]          # what the subject's receipt said, if one came
+    tier_before: str
+    grant: str                                 # envelope id of the grant that governed the window
+    limits: Mapping[str, int | float]          # the ceilings it was priced against (fleetmesh_core.accounting.window's limit set)
+    severity: Optional[str] = None             # None only when state == 'not-evaluated'
+    note: str = ''                             # fleetmesh_core.accounting.window.adjust's one-line reason
+    detail: dict[str, WindowEvaluationDetail | SilenceEvaluation] = field(default_factory=dict)   # by metric, plus 'silence'
+    tier_after: Optional[str] = None
+    limits_after: Optional[Mapping[str, int | float]] = None
+    effective_from: Optional[int] = None
+    receipt_heads: list[str] = field(default_factory=list)
+    receipts: list[ReceiptBody] = field(default_factory=list)   # the bodies as received, which are the evidence
+
+    def to_dict(self) -> EvaluationDict:
+        return cast(EvaluationDict, asdict(self))
+
+
+@dataclass(slots=True)
+class Repricing:
+    """A window this node had priced, read again after its receipt arrived late."""
+    peer: str
+    epoch: int
+    was: str
+    severity: str
+    state: str
+    measured: dict[str, int]
+    claimed: Optional[dict[str, int]]
+    receipts: int
+    remedy_cancelled: bool
+    at: int
+
+    def to_dict(self) -> RepricingDict:
+        return cast(RepricingDict, asdict(self))
+
+
+def evaluation_like(rec: Evaluation | Mapping[str, Any]) -> Evaluation:
+    """The reporter is handed Evaluations by the budget and plain mappings by
+    tests and plugins. Both are one shape."""
+    if isinstance(rec, Evaluation):
+        return rec
+    known = {f for f in Evaluation.__dataclass_fields__}
+    body = {k: v for k, v in rec.items() if k in known}
+    for k, default in (('state', ''), ('measured', {}), ('claimed', None), ('tier_before', ''),
+                       ('grant', ''), ('limits', {})):
+        body.setdefault(k, default)
+    return Evaluation(**body)