fleetmesh-node 0.2.0__py3-none-any.whl

node/planes/identity/presence/compartment.py

@@ -0,0 +1,409 @@
+"""What this node says it is alive, and what it makes of a peer saying so.
+
+Three things that are one thing: the descriptor this node publishes, the
+heartbeat that says the descriptor is still true, and the subscription that
+reads a peer's. All three ran loose in the node's tick, and the pair of them
+that can find a fault - a heartbeat whose class contradicts the peer's live
+descriptor, one whose endpoint digest says the descriptor has gone stale - fed
+a finding into the budget and recorded nothing an operator could read back.
+"""
+from __future__ import annotations
+
+import threading
+import time
+from typing import Any, Callable, Optional, Sequence
+
+from fleetmesh_core.crypto import giftwrap
+from fleetmesh_core.identity import relaylist
+from fleetmesh_core.services import lifecycle, telemetry
+from fleetmesh_core.storage import store as ref_store
+from fleetmesh_core.wire.interfaces import NostrEvent
+from node.channels import journal
+from node.compartment import stores as st
+from node.protocols.nostr import client as relay
+import dataclasses
+from node.config.parameter import Parameter, ParamBlock
+from node.channels import Channel, NOTE, INFO
+from node.metrics import COUNTER, Metric
+from node.compartment import Command, Compartment, Every, Need, Worker
+from .types import PresenceStateReport
+
+# One kind 21802 per window. The counters are the transport's and the
+# strain ledger's totals, binned by fleetmesh_core/telemetry so nothing finer than
+# an order of magnitude leaves the node and no peer is named.
+TELEMETRY_WINDOW: int = 3600
+
+
+@dataclasses.dataclass(frozen=True)
+class PresenceConfig(ParamBlock):
+    key = 'presence'
+    name = 'presence'
+    DERIVED: bool = False
+    PARAMS = (
+        Parameter(
+            key='heartbeat', name='Heartbeat Enabled',
+            description='Whether to publish periodic heartbeat events',
+            type=bool, default=True, tags=('presence', 'heartbeat'),
+        ),
+        Parameter(
+            key='telemetry_window', name='Telemetry Window',
+            description='Window size in seconds for telemetry aggregation',
+            type=int, default=3600, minimum=60, tags=('presence', 'telemetry'),
+        ),
+    )
+    heartbeat: bool = True
+    telemetry_window: int = 3600
+
+
+class Presence(Compartment):
+    requires: tuple[str, ...] = ('budget', 'strain')   # strain totals are one of the five telemetry metrics
+
+    name: str = 'presence'
+    # `heartbeat` and `class` are read every tick but neither is live: a node's
+    # class is what the node IS, and a heartbeat switched off under a running
+    # node would take its liveness off the mesh without anything on the mesh
+    # being told. Both change on a restart.
+    reads: tuple[str, ...] = ('heartbeat', 'class', 'telemetry')
+    config_cls = PresenceConfig
+    NEEDS = (
+        Need('keystore', 'ks'),
+        Need('peers'),
+        Need('grant_store', 'stores.grants'),
+        Need('relays'),
+        Need('budget'),
+        Need('stop'),
+        Need('timeout'),
+        Need('strain'),
+        Need('descriptor'),
+        Need('rebuild_descriptor', 'build_descriptor'),
+        Need('transport'),
+    )
+    TELEMETRY_WINDOW: int = TELEMETRY_WINDOW
+    CHANNELS = (
+        Channel('peer_retiring', 'Peer indicated graceful retirement', level=NOTE,
+                say='peer {peer} says it is retiring; the partition that follows is not stress from that direction'),
+        Channel('inbound_event', 'Inbound heartbeat event processed', level=INFO,
+                say='< {event}'),
+    )
+    METRICS = (
+        Metric(name='presence_peers_retiring', description='Times peers reported retirement.',
+               kind=COUNTER, counts='peer_retiring', tags=('presence', 'peers')),
+        Metric(name='presence_events_inbound', description='Times heartbeat events were processed.',
+               kind=COUNTER, counts='inbound_event', tags=('presence', 'heartbeat')),
+    )
+    EVERY = (Every(seconds=TELEMETRY_WINDOW, run='_run_telemetry'),)
+    WORKERS = (Worker(run='_listen', name='presence-listen'),)
+    STORES = (
+        st.Store(
+            key='_seen',
+            description='peer -> the id of the last heartbeat read from it',
+            bound=1024,
+            evict=st.LRU,
+            kind=st.TABLE,
+            keyed_by='peer',
+        ),
+        st.Store(
+            key='_retiring',
+            description='peers that said they were going, and when',
+            bound=1024,
+            evict=st.LRU,
+            kind=st.TABLE,
+            keyed_by='peer',
+        ),
+    )
+    COMMANDS = (
+        Command(
+            key='show',
+            name='Presence',
+            description='the descriptor this node publishes, when it expires, and the peers it '
+                        'has heard from',
+            tags=('presence', 'status'),
+            permission='presence/show',
+            run='state',
+            reply='{descriptor, expires, renews_at, last_heartbeat, inbox_relays, heard_from, '
+                  'retiring}',
+        ),
+    )
+
+    def __init__(
+        self,
+        keystore: Any,
+        peers: Any,
+        grant_store: Any,
+        relays: Sequence[str],
+        budget: Optional[Any] = None,
+        stop: Optional[threading.Event] = None,
+        clock: Optional[Callable[[], int]] = None,
+        timeout: float = 15.0,
+        node: Optional[Any] = None,
+        strain: Optional[Any] = None,
+        descriptor: Optional[Callable[[], Optional[NostrEvent]]] = None,
+        rebuild_descriptor: Optional[Callable[[], NostrEvent]] = None,
+        transport: Optional[Any] = None,
+        publisher: Optional[Callable[[Sequence[str], Any], Any]] = None,
+    ) -> None:
+        super().__init__(node, clock)
+        # How a heartbeat, telemetry or renewed descriptor reaches the relays;
+        # given so a suite can record them without a socket.
+        self.publisher: Callable[[Sequence[str], Any], Any] = publisher or relay.publish_all
+        # The transport whose nostr-rung counters the telemetry event reports.
+        self.transport: Optional[Any] = transport if transport is not None else getattr(node, 'transport', None)
+        # The node's current descriptor, and the one call that rebuilds it
+        # from what the compartments say now. Suites that pass a fake node
+        # get both off it.
+        self.descriptor: Callable[[], Optional[NostrEvent]] = descriptor or (lambda: getattr(node, 'descriptor', None))
+        self.rebuild_descriptor: Callable[[], NostrEvent] = rebuild_descriptor or (lambda: getattr(node, 'build_descriptor')())
+        self.ks: Any = keystore
+        self.peers: Any = peers
+        self.grants: Any = grant_store
+        self.relays: list[str] = list(relays)  # not a store: the configured relays
+        self.budget: Optional[Any] = budget
+        self.strain: Optional[Any] = strain
+        self.timeout: float = timeout
+        self._stop: threading.Event = stop or getattr(self, '_stop_event', None) or threading.Event()
+        self._last_heartbeat: Optional[int] = None
+        self._last_telemetry: Optional[int] = None
+        # The relay list this node last published (kind 10050): the relays it
+        # reads, as a tuple, so a changed set is published once and an
+        # unchanged one never again.
+        self._inbox_said: Optional[tuple[str, ...]] = None
+        # The event itself, for whoever projects the floor (the linked face).
+        self.last_inbox: Optional[NostrEvent] = None
+        self._seen: st.Table[str, str]          # peer -> the id of the last heartbeat read from it
+        # Peers that said they were going, and when. A partition from one of
+        # these is not stress: this node was told. Cleared when the peer is
+        # heard from again without the flag, because that is a node that came
+        # back and is expecting to be counted like any other.
+        self._retiring: st.Table[str, int]
+
+    # ---- what this node says --------------------------------------------------
+    def _renews_at(self) -> tuple[int, int]:
+        """A descriptor is renewed inside the last tenth of its life."""
+        desc = self.descriptor()
+        created_at = desc.get('created_at', self.clock()) if desc else self.clock()
+        exp = None
+        if desc:
+            for t in desc.get('tags', []):
+                if isinstance(t, (list, tuple)) and len(t) > 1 and t[0] == 'expiration':
+                    try:
+                        exp = int(t[1])
+                        break
+                    except (ValueError, TypeError):
+                        pass
+        if exp is None:
+            exp = created_at + 86400
+        return exp, exp - (exp - created_at) // 10
+
+    def publish_inbox(self, now: int) -> Optional[NostrEvent]:
+        """The kind 10050 that states this node's relay floor: the relays it
+        reads are where a stored wrap for it goes (NIP-17). Every node
+        publishes one - the relay rung is the one way a node can always be
+        reached, and this is where that is said, in the place any Nostr
+        client already looks. Published on the first tick and again only
+        when the set changes; None when nothing was due."""
+        if not self.relays:
+            return None
+        now_set = tuple(self.relays)
+        if self._inbox_said == now_set:
+            return None
+        tags = relaylist.build(self.relays)
+        ev: NostrEvent = ref_store.make_signed_event(self.ks.node_sk, now, relaylist.KIND, tags, '')
+        self.publisher(self.relays, ev)
+        self._inbox_said = now_set
+        self.last_inbox = ev
+        return ev
+
+    def tick(self, now: int) -> list[tuple[Any, ...]]:
+        did: list[tuple[Any, ...]] = []
+        inbox = self.publish_inbox(now)
+        if inbox is not None:
+            did.append(('inbox relays published', inbox['id']))
+        exp, renew = self._renews_at()
+        if now >= renew:
+            desc = self.rebuild_descriptor()
+            if self.relays:
+                self.publisher(self.relays, desc)
+            did.append(('descriptor renewed', desc['id']))
+        # anchors must, edges should, leaves never
+        if self.config('heartbeat', True) and lifecycle.heartbeat_due(
+                self.config('class'), self._last_heartbeat, now):
+            cur = self.descriptor()
+            eps = [t for t in cur['tags'] if t[0] == 'endpoint'] if cur is not None else []
+            hb = lifecycle.heartbeat(self.ks.node_pk, self.config('class'), eps, now)
+            signed_hb = ref_store.make_signed_event(self.ks.node_sk, now, hb['kind'], hb['tags'], hb['content'])
+            if self.relays:
+                self.publisher(self.relays, signed_hb)
+            self._last_heartbeat = now
+            did.append(('heartbeat', signed_hb['id']))
+        did += self.run_every(now)
+        return did
+
+    _tick_runs_every = True
+
+    def _run_telemetry(self, now: Optional[int] = None) -> list[tuple[Any, ...]]:
+        now = now if now is not None else self.clock()
+        if not (self.config('telemetry', False) and self.relays):
+            return []
+        tm = self._telemetry(now)
+        signed_tm = ref_store.make_signed_event(self.ks.node_sk, now, tm['kind'], tm['tags'], tm['content'])
+        self.publisher(self.relays, signed_tm)
+        self._last_telemetry = now
+        return [('telemetry', signed_tm['id'])]
+
+    # One kind 21802 per window. The counters are the transport's and the
+    # strain ledger's totals, binned by fleetmesh_core/telemetry so nothing finer than
+    # an order of magnitude leaves the node and no peer is named.
+
+    def _telemetry(self, now: int) -> Any:
+        tr = getattr(self.transport, 'nostr', None) if self.transport is not None else None
+        counters: dict[str, int] = {'peers': len(self.grants.peers())}
+        if tr is not None:
+            counters['wraps_in'] = sum(getattr(tr, 'wraps_in', {}).values())
+            counters['wraps_seen_again'] = sum(getattr(tr, 'wraps_seen_again', {}).values())
+            counters['wraps_dropped'] = len(getattr(tr, 'dropped', ()))
+        if self.strain is not None:
+            counters['strain'] = sum(self.strain.by_direction().values())
+        return telemetry.telemetry(self.ks.node_pk, now, self.TELEMETRY_WINDOW, counters,
+                                   expiration=now + 2 * self.TELEMETRY_WINDOW)
+
+    def announce_retiring(self, now: Optional[int] = None) -> list[tuple[Any, ...]]:
+        """Say this node is going, before it starts giving things away.
+
+        Published directly rather than through the tick's due check: a
+        retirement does not wait for the hour to come round, and the whole
+        value of the flag is that it arrives before the silence it explains.
+        A leaf publishes no heartbeat and so cannot announce one, which is
+        correct - a leaf claims nothing and hands nothing over.
+
+        Ephemeral, like every heartbeat: only a peer listening now hears it.
+        That is the right answer and it is why it is advisory. A peer that was
+        itself down records the partition, and the partition is true.
+        """
+        now_val = now if now is not None else self.clock()
+        if self.config('class') == 'leaf' or not self.relays or not self.config('heartbeat', True):
+            return []
+        desc = self.descriptor()
+        eps = [t for t in desc['tags'] if t[0] == 'endpoint'] if desc is not None else []
+        hb = lifecycle.heartbeat(self.ks.node_pk, self.config('class'), eps, now_val, load=1.0, retiring=True)
+        signed = ref_store.make_signed_event(self.ks.node_sk, now_val, hb['kind'], hb['tags'], hb['content'])
+        self.publisher(self.relays, signed)
+        self._last_heartbeat = now_val
+        return [('retiring announced', signed['id'])]
+
+    def is_retiring(self, peer_pk: str) -> bool:
+        """Did this peer tell us it was going? Read by anything that would
+        otherwise price the silence that follows."""
+        return peer_pk in self._retiring
+
+    # ---- what a peer says -----------------------------------------------------
+    def on_heartbeat(self, hb: Any) -> list[tuple[Any, ...]]:
+        """One kind 21801 from a peer, checked against what that peer published.
+
+        A node published one of these every hour and nothing ever read one, so
+        two rules the reference already implements were dead. A heartbeat whose
+        class contradicts the peer's live descriptor is a misreport about
+        itself and an S2. One whose endpoint digest differs from the
+        descriptor's says the descriptor has gone stale, which is the whole
+        reason that digest is in there: a peer can tell without dialling every
+        address it was given.
+
+        A heartbeat is *ephemeral*, so no relay stores one and none can be
+        asked for after the fact. Liveness is a statement about now, and the
+        only way to hear it is to be listening when it is made - which is why
+        this is fed by a subscription and not by a query. A node that was down
+        misses them, and that is the right answer.
+        """
+        did: list[tuple[Any, ...]] = []
+        # A subscription by author returns whatever a relay chooses to serve
+        # under that name. Believing it unsigned would let a relay file an S2
+        # against any peer this node holds a grant with.
+        if not isinstance(hb, dict) or not giftwrap.verify_event(hb) or hb.get('kind') != lifecycle.HEARTBEAT:
+            self.refusals.refuse('heartbeat', 'not a signed kind 21801',
+                                 subject=(hb or {}).get('pubkey') if isinstance(hb, dict) else None)
+            return did
+        pk = hb.get('pubkey')
+        if not isinstance(pk, str) or pk not in set(self.grants.peers()):
+            self.refusals.refuse('heartbeat', 'from a node this one holds no grant with', subject=pk)
+            return did
+        hb_id = hb.get('id')
+        if not isinstance(hb_id, str) or self._seen.get(pk) == hb_id:
+            return did
+        self._seen[pk] = hb_id
+        desc = self.peers.descriptor(pk)
+        if desc is None:
+            self.refusals.refuse('heartbeat', 'the peer publishes no descriptor to check it against', subject=pk)
+            return did
+        problems, findings = lifecycle.check_heartbeat(hb, desc)
+        for severity, why in findings:
+            if self.budget is not None:
+                self.budget.findings.append((pk, None, severity, 'heartbeat: %s' % why))
+            self.refusals.refuse('heartbeat', 'contradicts the descriptor the peer publishes',
+                                 subject=pk, detail={'severity': severity, 'what': why})
+            did.append(('heartbeat contradicts its descriptor', pk, severity))
+        if lifecycle.is_retiring(hb):
+            # Not a finding and not a fault. The peer is telling us the silence
+            # that follows is a departure, so nothing downstream prices it.
+            if pk not in self._retiring:
+                self._retiring[pk] = self.clock()
+                self.say('peer_retiring', peer=pk[:16])
+                did.append(('peer is retiring', pk))
+        elif self._retiring.pop(pk, None) is not None:
+            did.append(('peer came back', pk))
+        if problems:
+            self.refusals.refuse('heartbeat', 'malformed', subject=pk,
+                                 detail={'problems': problems[:2]})
+            did.append(('heartbeat is malformed', pk, '; '.join(problems[:2])))
+        elif lifecycle.descriptor_is_stale(hb, desc):
+            self.peers.descriptor(pk, refresh=True)
+            did.append(('descriptor was stale, refreshed', pk))
+        return did
+
+    def _listen(self) -> None:
+        """One subscription per relay for the liveness of peers we hold grants with."""
+        while not self._stop.is_set():
+            peers_held = self.grants.peers()
+            if not peers_held or not self.relays:
+                self._stop.wait(30)
+                continue
+            r = relay.Relay(self.relays[0], timeout=5.0)
+            try:
+                sid = r.connect().subscribe([{'kinds': [lifecycle.HEARTBEAT], 'authors': peers_held}])
+                started, watching = time.time(), set(peers_held)
+                for hb in r.events(sid):
+                    if self._stop.is_set():
+                        break
+                    for d in self.on_heartbeat(hb):
+                        self.say('inbound_event', event=' '.join(journal.short(x) for x in d))
+                    # re-subscribe when the peer set has moved on
+                    if time.time() - started > 300 and set(self.grants.peers()) != watching:
+                        break
+            except Exception as e:
+                self.refusals.refuse('subscription', 'could not be held open',
+                                     subject=self.relays[0], detail={'error': str(e)})
+            finally:
+                r.close()
+            self._stop.wait(5)
+
+    # ---- being watched --------------------------------------------------------
+    def state(self) -> PresenceStateReport:
+        desc = self.descriptor()
+        exp, renew = self._renews_at() if desc else (None, None)
+        return {
+            'descriptor': desc['id'] if desc else None,
+            'expires': exp,
+            'renews_at': renew,
+            'last_heartbeat': self._last_heartbeat,
+            'inbox_relays': list(self._inbox_said or ()),
+            'heard_from': sorted(pk[:16] for pk in self._seen),
+            'retiring': sorted(pk[:16] for pk in self._retiring),
+        }
+
+    def trouble(self) -> list[str]:
+        out: list[str] = []
+        wrong = self.refusals.within(self.TROUBLE_WINDOW, self.clock(), kind='heartbeat',
+                                     why='contradicts the descriptor the peer publishes')
+        if wrong:
+            out.append('%d heartbeat(s) contradict the descriptor their peer publishes: %s. '
+                       'Each is a finding against that peer'
+                       % (len(wrong), ', '.join(sorted({(r.subject or '')[:16] for r in wrong}))))
+        return out

node/planes/identity/presence/types.py

@@ -0,0 +1,14 @@
+"""Types and records for the Presence compartment."""
+from __future__ import annotations
+
+from typing import Optional, TypedDict
+
+
+class PresenceStateReport(TypedDict):
+    descriptor: Optional[str]
+    expires: Optional[int]
+    renews_at: Optional[int]
+    last_heartbeat: Optional[int]
+    inbox_relays: list[str]
+    heard_from: list[str]
+    retiring: list[str]

node/planes/operations/__init__.py

@@ -0,0 +1,13 @@
+"""Operations plane: node lifecycle, task scheduling, health, and software updates.
+
+The 2 compartments of the operations plane:
+- scheduler: cadence pacing, peer sync throttling, and task scheduling
+- updates: check-and-report software updates and release notifications
+"""
+from .scheduler import Scheduler
+from .updates import Updates
+
+__all__ = [
+    'Scheduler',
+    'Updates',
+]

node/planes/operations/scheduler/__init__.py

@@ -0,0 +1,9 @@
+"""The pulls this node is configured to make, and every reason one did not happen.
+
+One-module compartment: everything is in compartment.py.
+"""
+from .compartment import Scheduler
+from .types import SchedulerPullStatus, SchedulerStateReport
+
+__all__ = ['Scheduler', 'SchedulerPullStatus', 'SchedulerStateReport']
+