fleetmesh-node 0.2.0__py3-none-any.whl

node/planes/operations/scheduler/compartment.py

@@ -0,0 +1,353 @@
+"""The pulls this node is configured to make, and every reason one did not happen.
+
+This was seventy lines in the middle of the node's tick, and it was the worst
+place in the node to be a stranger. Five things can stop a scheduled pull: the
+peer has no peering yet and the proposal raised; the proposal came back
+declined; this node is over its own ceiling and has stopped pulling as the last
+shed rung; the grant this node holds covers no session this window; or the
+session itself failed. Every one of them was appended to the tick's return value
+as a tuple, printed once, and then gone. `meshnode refusals` could not show any
+of them, `health` counted none of them, and an operator watching a peer go quiet
+had to read the journal back far enough to find the line - if the journal still
+had it.
+
+A refusal that is not recorded is indistinguishable from silence. This
+compartment is the five of them given a channel.
+"""
+from __future__ import annotations
+
+from typing import TYPE_CHECKING, Any, Callable, Mapping, Optional
+
+if TYPE_CHECKING:
+    from node.planes.peering.budget.core import Budget
+    from node.planes.peering.capacity.compartment import Capacity
+    from node.planes.peering.strain.compartment import StrainLedger
+
+import dataclasses
+from node.interfaces import ThrottledRow
+from node.planes.store.sync.shape import SyncSessionResult
+
+
+from .types import SchedulerStateReport
+
+from fleetmesh_core.accounting import accounting
+from node.compartment import stores as st
+from node.storage import grants as grants_mod
+from node.channels import Channel, NOTE
+from node.metrics import COUNTER, Metric
+from node.compartment import Command, Compartment, Need
+from node.config.parameter import Parameter, ParamBlock
+
+# A decline names a reason that does not change in ten minutes: a digest, a
+# The cadence when a peer has no grant to derive one from: there is no
+# granted rate to pace against until one is issued.
+DEFAULT_PULL_EVERY = 900
+
+# missing protocol, a closed door. Ask again in an hour, not at the pull cadence.
+DECLINE_RETRY: int = 3600
+
+
+@dataclasses.dataclass(frozen=True)
+class SchedulerConfig(ParamBlock):
+    key = 'scheduler'
+    name = 'scheduler'
+    DERIVED: bool = False
+    PARAMS = (
+        Parameter(key='default_pull_every', name='Default Pull Cadence', description='Seconds between pulls when peer grant has no rate', type=int, default=DEFAULT_PULL_EVERY, tags=('scheduler', 'pull')),
+        Parameter(key='decline_retry', name='Decline Retry Horizon', description='Seconds before re-proposing to declined peer', type=int, default=DECLINE_RETRY, tags=('scheduler', 'pull')),
+    )
+    default_pull_every: int = DEFAULT_PULL_EVERY
+    decline_retry: int = DECLINE_RETRY
+
+
+class Scheduler(Compartment):
+    requires: tuple[str, ...] = ('budget', 'capacity', 'strain')
+
+    name: str = 'scheduler'
+    reads: tuple[str, ...] = ('peers',)
+    live: tuple[str, ...] = ('peers',)
+    config_cls = SchedulerConfig
+    CHANNELS = (
+        Channel('pull_scheduled', 'Pull scheduled for peer and shard', level=NOTE, say='scheduled pull for peer {peer} on shard {shard}', tags=('scheduler', 'pull')),
+        Channel('pull_refused', 'Scheduled pull refused', level=NOTE, say='pull refused for peer {peer} on shard {shard}: {reason}', tags=('scheduler', 'pull')),
+    )
+    METRICS = (
+        Metric(name='scheduler_pulls_scheduled', description='Times pulls were scheduled.', kind=COUNTER, counts='pull_scheduled', tags=('scheduler', 'pull')),
+        Metric(name='scheduler_pulls_refused', description='Times scheduled pulls were refused.', kind=COUNTER, counts='pull_refused', tags=('scheduler', 'pull')),
+    )
+    NEEDS = (
+        Need('grant_store', 'stores.grants'),
+        Need('budget'),
+        Need('capacity'),
+        Need('propose'),
+        Need('pull'),
+        Need('strain'),
+    )
+    STORES = (
+        st.Store(
+            key='_last_pull',
+            description='when a session last ran for (peer, shard)',
+            bound=1024,
+            evict=st.LRU,
+            kind=st.TABLE,
+            keyed_by='pull',
+        ),
+        st.Store(
+            key='_declined',
+            description='when a declined proposal may be tried again for (peer, shard)',
+            bound=1024,
+            evict=st.LRU,
+            kind=st.TABLE,
+            keyed_by='pull',
+        ),
+        st.Store(
+            key='_throttled',
+            description='(epoch, why) last recorded for (peer, shard)',
+            bound=1024,
+            evict=st.LRU,
+            kind=st.TABLE,
+            keyed_by='pull',
+        ),
+        st.Store(
+            key='_shed',
+            description='whether the pull is held under the ceiling for (peer, shard)',
+            bound=1024,
+            evict=st.LRU,
+            kind=st.TABLE,
+            keyed_by='pull',
+        ),
+    )
+    COMMANDS = (
+        Command(
+            key='show',
+            name='Scheduler',
+            description='every scheduled pull, when it last ran, and what is holding it back now',
+            tags=('scheduler', 'status'),
+            permission='scheduler/show',
+            run='state',
+            reply='{pulls, throttled, held, declined_until}',
+        ),
+    )
+
+    def __init__(
+        self,
+        grant_store: grants_mod.GrantStore,
+        budget: Budget,
+        capacity: Optional[Capacity],
+        propose: Callable[..., tuple[str, Any]],
+        pull: Callable[..., SyncSessionResult],
+        strain: Optional[StrainLedger] = None,
+        clock: Optional[Callable[[], int]] = None,
+        node: Optional[Any] = None,
+    ) -> None:
+        super().__init__(node, clock)
+        self.grants: grants_mod.GrantStore = grant_store
+        self.budget: Budget = budget
+        self.capacity: Optional[Capacity] = capacity
+        self.propose: Callable[..., tuple[str, Any]] = propose
+        self.pull: Callable[..., SyncSessionResult] = pull
+        self.strain: Optional[StrainLedger] = strain
+        self._last_pull: st.Table[tuple[str, str], int]     # (peer, shard) -> when a session last ran
+        self._declined: st.Table[tuple[str, str], int]      # (peer, shard) -> when a declined proposal may be tried again
+        self._throttled: st.Table[tuple[str, str], tuple[int, str]]     # (peer, shard) -> (epoch, why) last recorded
+        self._shed: st.Table[tuple[str, str], bool]          # (peer, shard) -> whether the pull is held under the ceiling
+
+    def tick(self, now: int) -> list[tuple[Any, ...]]:
+        did: list[tuple[Any, ...]] = []
+        for p in self.config('peers') or []:
+            key = (p['pubkey'], p['shard'])
+            held = self.grants.get(p['pubkey'], grants_mod.RECEIVED)
+            last = self._last_pull.get(key)
+            if held is not None and held['status'] == 'live':
+                # The budget is a ceiling, not a timetable. `conn_rate` is a
+                # rate limit - so many sessions per window - and a node is
+                # entitled to spend it as fast as it likes up to the limit.
+                # An earlier draft of this spaced the remaining sessions
+                # evenly across the remaining window, which is a politeness
+                # policy nothing asked for and which slowed exactly the node a
+                # tier promotion was meant to speed up.
+                #
+                # So: a spent budget is a hard stop, and the cadence is a
+                # preference whose DEFAULT is derived from the grant -
+                # window / conn_rate. An unconfigured node therefore paces at
+                # the rate it was granted, and a promotion from four sessions
+                # an hour to twelve moves it from 900s to 300s by itself,
+                # where before the two agreed only by the coincidence that
+                # 3600/4 is the configured 900.
+                #
+                # The hard stop is also what closes a defect. `_last_pull` is
+                # in memory, so after a restart `last is None` and no cadence
+                # gate can fire - the node pulled immediately however recently
+                # it had pulled, spending budget the window had already
+                # allocated. Three restarts inside window 497614 is how that
+                # window came to carry more sessions than the grant allowed.
+                # `sessions_left` reads the meter, which is persisted and is
+                # the record the peer prices, so a restart inherits what was
+                # spent instead of forgetting it.
+                left, closes = self.budget.sessions_left(p['pubkey'], now)
+                if left <= 0:
+                    # Spent: pace, do not attempt. Before this the node tried
+                    # anyway and `throttled()` below refused it, recording
+                    # "the grant covers no session this window" for what is
+                    # ordinary pacing. That path stays as the backstop for a
+                    # real overrun on another counter.
+                    continue
+                every = self._cadence(p)
+                if last is not None and now - last < int(every):
+                    continue
+            if held is None or held['status'] != 'live':
+                # No grant, or one a terminate zeroed. `terminate` leaves the
+                # revoked row so a restart knows the peering ended, and `peer`
+                # afterwards is the operator saying it should begin again; a
+                # row that is not live is as much a reason to propose as none,
+                # and pull_every is a cadence for pulling, not for asking. A
+                # decline still holds its own retry below.
+                if now < self._declined.get(key, 0):
+                    continue
+                try:
+                    kw = {'timeout': p['timeout']} if 'timeout' in p else {}
+                    try:
+                        status, detail = self.propose(p['pubkey'], p['shard'], p['filter'], **kw)
+                    except TypeError:
+                        status, detail = self.propose(p['pubkey'], p['shard'], p['filter'])
+                except Exception as e:
+                    # A peer with no descriptor yet raised out of the whole tick,
+                    # so every other peer's receipts waited on it.
+                    self.refusals.refuse('propose', 'the handshake raised', subject=p['pubkey'],
+                                         detail={'shard': p['shard'],
+                                                 'error': '%s: %s' % (type(e).__name__, e)}, at=now)
+                    did.append(('propose failed', p['pubkey'], '%s: %s' % (type(e).__name__, e)))
+                    self._last_pull[key] = now
+                    continue
+                did.append(('proposed', p['pubkey'], status))
+                if status != 'peered':
+                    self._last_pull[key] = now
+                    retry = int(self.strain.reserve(DECLINE_RETRY)) if self.strain is not None else DECLINE_RETRY
+                    self._declined[key] = now + retry
+                    self.refusals.refuse('propose', 'the peer declined', subject=p['pubkey'],
+                                         detail={'shard': p['shard'], 'status': status,
+                                                 'retry_at': self._declined[key]}, at=now)
+                    continue
+                self._declined.pop(key, None)
+            if self.capacity is not None and self.capacity.leafing():
+                # The last rung. A node over its own ceiling stops pulling, which
+                # is the only load it chooses. It keeps answering receipts and
+                # complaints, because accounting is what makes a shed legible
+                # rather than a partition.
+                if self._shed.get(key) is not True:
+                    self._shed[key] = True
+                    self.refusals.refuse('pull', 'this node is over its own ceiling',
+                                         subject=p['pubkey'], detail={'shard': p['shard']}, at=now)
+                    did.append(('pull held', p['pubkey'], 'this node is over its own ceiling'))
+                self._last_pull[key] = now
+                continue
+            self._shed.pop(key, None)
+            # The grant this node holds is read before it acts, not only after.
+            # pull_every is the operator's ceiling; the grant's is the lower one.
+            why = self.budget.throttled(p['pubkey'], now)
+            if why is not None:
+                epoch = self.budget.epoch(p['pubkey'], grants_mod.RECEIVED, now)
+                if self._throttled.get(key) != (epoch, why):
+                    self._throttled[key] = (epoch, why)
+                    # The reason is the category and the numbers are the detail:
+                    # "sync_ranges 48 over the grant's 40" as a reason would put
+                    # a new key in the counter for every window that overran.
+                    self.refusals.refuse('pull', 'the grant covers no session this window',
+                                         subject=p['pubkey'],
+                                         detail={'shard': p['shard'], 'epoch': epoch, 'over': why}, at=now)
+                    did.append(('throttled', p['pubkey'], why))
+                self._last_pull[key] = now
+                continue
+            try:
+                res = self.pull(p['pubkey'], p['shard'], p['filter'])
+                did.append(('pulled', p['pubkey'], res['status'], res['counters'].get('events_in', 0)))
+            except Exception as e:
+                self.refusals.refuse('pull', 'the session raised', subject=p['pubkey'],
+                                     detail={'shard': p['shard'],
+                                             'error': '%s: %s' % (type(e).__name__, e)}, at=now)
+                did.append(('pull failed', p['pubkey'], str(e)))
+            self._last_pull[key] = now
+        return did
+
+    # ---- being watched --------------------------------------------------------
+    def throttled(self) -> list[ThrottledRow]:
+        """The pulls this node is holding back this window, and until when.
+
+        A throttled peer goes quiet in both directions: no sessions, so no
+        usage, so no receipts due either way. The relay line then says "no
+        event yet" and an operator reads a dead node. Four nodes on three
+        machines produced exactly that reading for twenty minutes. Nothing was
+        wrong; the grant's ceiling had been reached. Say so, and say when it
+        turns.
+        """
+        out: list[ThrottledRow] = []
+        for (pk, shard), (epoch, why) in sorted(self._throttled.items()):
+            try:
+                w = self.budget._window_of(pk, grants_mod.RECEIVED)
+                if self.budget.epoch(pk, grants_mod.RECEIVED) != epoch:
+                    continue
+                until = accounting.window_bounds(epoch, w)[1]
+            except Exception:
+                continue
+            out.append({'peer': pk, 'shard': shard, 'why': why, 'until': until})
+        return out
+
+    def _cadence(self, p: Mapping[str, Any]) -> int:
+        """Seconds between pulls of this peer: the operator's number, or the
+        grant's own rate when there is none.
+
+        Here rather than inline because `state()` reports it and the loop
+        obeys it, and the two read a hardcoded 900 and a derived value
+        respectively - so an unconfigured node pulled every window/conn_rate
+        while telling its operator 900. Two places agreeing by hand is how
+        that happened; one method is the fix.
+        """
+        every = p.get('pull_every')
+        if every is not None:
+            return int(every)
+        allowed, w = self.budget.granted_rate(p['pubkey'])
+        return (w // allowed) if allowed > 0 else DEFAULT_PULL_EVERY
+
+    def state(self) -> SchedulerStateReport:
+        return {
+            'pulls': [
+                {
+                    'peer': p['pubkey'][:16],
+                    'shard': p['shard'],
+                    'every': self._cadence(p),
+                    'last': self._last_pull.get((p['pubkey'], p['shard'])),
+                }
+                for p in self.config('peers') or []
+            ],
+            'throttled': self.throttled(),
+            'held': sorted('%s %s' % (pk[:16], shard) for (pk, shard), v in self._shed.items() if v),
+            'declined_until': {
+                '%s %s' % (pk[:16], shard): at
+                for (pk, shard), at in self._declined.items()
+            },
+        }
+
+    def _live_with(self, peer: str) -> bool:
+        held = self.grants.get(peer, grants_mod.RECEIVED)
+        return held is not None and held['status'] == 'live'
+
+    def trouble(self) -> list[str]:
+        out: list[str] = []
+        now = self.clock()
+        failed = self.refusals.within(self.TROUBLE_WINDOW, now, kind='pull', why='the session raised')
+        if failed:
+            out.append('%d scheduled pull(s) failed: %s'
+                       % (len(failed), ', '.join(sorted({(r.subject or '')[:16] for r in failed}))))
+        raised = self.refusals.within(self.TROUBLE_WINDOW, now, kind='propose', why='the handshake raised')
+        # A proposal that got no answer because the peer was not up yet - the
+        # first minute of a mesh deploy, every time - is answered by the
+        # retry. Once the grant is live the failure is history, not trouble.
+        raised = [r for r in raised if not self._live_with(r.subject or '')]
+        if raised:
+            out.append('%d peering(s) could not be proposed: %s'
+                       % (len(raised), ', '.join(sorted({(r.subject or '')[:16] for r in raised}))))
+        held = sorted(k for k, v in self._shed.items() if v)
+        if held:
+            out.append('%d scheduled pull(s) held because this node is over its own ceiling: %s'
+                       % (len(held), ', '.join('%s %s' % (pk[:16], shard) for pk, shard in held)))
+        return out

node/planes/operations/scheduler/types.py

@@ -0,0 +1,21 @@
+"""Types and records for the Scheduler compartment."""
+from __future__ import annotations
+
+from typing import Optional, TypedDict
+from node.interfaces import ThrottledRow
+
+
+class SchedulerPullStatus(TypedDict, total=False):
+    """Status of one scheduled pull."""
+    peer: str
+    shard: str
+    every: int
+    last: Optional[int]
+
+
+class SchedulerStateReport(TypedDict, total=False):
+    """The report Scheduler.state() gives of itself."""
+    pulls: list[SchedulerPullStatus]
+    throttled: list[ThrottledRow]
+    held: list[str]
+    declined_until: dict[str, int]

node/planes/operations/updates/__init__.py

@@ -0,0 +1,17 @@
+"""The `updates` compartment: check-and-report. See docs/updates.md, Tier 2.
+
+    release.py   a kind 9900 release event -> ReleaseInfo, or a refusal reason
+    store.py     what was seen and acknowledged, durable across restarts
+    core.py      Updates, the compartment: check, trouble, commands
+
+What it does: asks the channel relays for release events signed by the key
+the operator pinned, keeps the newest one that speaks this node's protocol,
+and says so where the node says things. What it does not do: fetch an
+artefact, run anything, or treat a track policy as more than a refused
+config line. That is the apply tier, and it is not here.
+"""
+from fleetmesh_core.peering.release import ReleaseInfo, parse_release_event, parse_revocation
+from .store import UpdateStore
+from .core import Updates
+
+__all__ = ['ReleaseInfo', 'parse_release_event', 'parse_revocation', 'UpdateStore', 'Updates']