fleetmesh-node 0.2.0__py3-none-any.whl

node/compartment/sidecar.py

@@ -0,0 +1,499 @@
+"""A sidecar, and the bridge compartment that is the node's only way to reach it.
+
+Three words, kept apart because the node had one word for two things (audit
+2026-10-07-3, finding "two things called sidecar"):
+
+  compartment   in-process. Built, wired, started, ticked, restarted and
+                retired by the Registry; its whole lifecycle is the node's.
+
+  sidecar       a separate process the node reaches but does not contain:
+                fips (Rust), libp2p (sixty-three packages, trio, will not
+                install on 3.14), bacalhau, grasp, strfry. Kept out because its
+                codebase, its dependencies, its runtime or its licence do not
+                belong in `node/` - which is standard library plus
+                `cryptography`, so that a second implementer never inherits
+                any of it. The node never starts or supervises one
+                (`supervised` is False, and stays False until code honours it).
+
+  bridge        the ONE compartment that fronts one sidecar. Everything on the
+                node that needs a sidecar goes through its bridge - names it in
+                `requires`/`uses` and calls `bridge.call()` or a method the
+                bridge offers - and nothing else opens the sidecar's socket.
+                So the sidecar's settings (PARAMS: endpoint, timeout), what it
+                says (CHANNELS), what it measures (METRICS), what an operator
+                can ask of it (COMMANDS) and how it fails (LIFECYCLE) are
+                declared once, on the bridge, the way every other compartment
+                declares them.
+
+What a bridge gets from this base, so that every sidecar is reached the same way:
+
+  - `SIDECAR`, a manifest: the protocol it speaks (`family/major`), what it
+    wraps upstream, the command that runs it standalone, and the request that
+    proves it is there.
+  - a link state - unknown, up, down, incompatible - moved only by `call()`,
+    said once each way on three declared channels and counted:
+    `<name>.sidecar.reachable|unreachable|incompatible`.
+  - the version gate (audit S1): every reply carries the protocol the sidecar
+    speaks, and a different family or major is refused - a bridge that read a
+    `/2` reply as `/1` would be wrong silently. On a link that is not up the
+    manifest's `probe` goes first as a handshake, so a sidecar that speaks
+    another major is refused before it is sent anything that carries data.
+  - two commands on the control socket: `<name>.sidecar` (the manifest and the
+    link now) and `<name>.probe` (ask the sidecar now).
+  - `Lifecycle(restart=True)`: a bridge holds a link and nothing more, so a
+    restart forgets the link and the next call finds out afresh. A wedged link
+    that trips the breaker is restarted by its quarantine trial.
+  - trouble when the sidecar is wanted and down or incompatible, naming what
+    this node loses (`CONSEQUENCE`).
+
+Two ways a sidecar runs, both described by the same manifest:
+
+  bridge mode     the sidecar is already running - ours, or one on the market
+                  (a strfry, a bacalhau) - and the bridge compartment in the
+                  node talks to it.
+  standalone      `SIDECAR.standalone` is the command that runs it on its own,
+                  beside the node or on another host; the bridge does not care
+                  who started it, only that its endpoint answers.
+
+The transport is a unix socket carrying one JSON object per line, which is what
+announce, p2p and fips already speak. HTTP (the class-A manifests in
+node/sidecars: grasp, ncontainer, magpie) arrives with the first bridge that
+needs it; `TRANSPORTS` says what is implemented and a manifest naming another
+is refused at import rather than at the first call.
+
+Named `sidecar_bridge` because `node/bridge` is the Universal Nostr Relay
+Bridge, a compartment called `bridge` that fronts relays rather than a sidecar.
+A package rather than a root module for the reason node/lifecycle is one.
+Standard library only.
+"""
+from __future__ import annotations
+
+import dataclasses
+import json
+import socket
+import threading
+import time
+from typing import Any, Callable, Mapping, Optional, TypedDict
+
+from .. import channels as ch_mod
+from .. import metrics as mx_mod
+from . import lifecycle as lifecycle_mod
+from .base import Compartment
+from .command import Command
+
+TRANSPORTS: tuple[str, ...] = ('unix',)
+MAX_LINE: int = 1 << 20
+
+# The link, as the bridge last found it.
+UNKNOWN = 'unknown'            # not asked since start (or restart)
+UP = 'up'                      # the last call was answered, in a protocol this bridge speaks
+DOWN = 'down'                  # the last call was not answered, or was refused
+INCOMPATIBLE = 'incompatible'  # answered, in a protocol family or major this bridge does not speak
+LINKS: tuple[str, ...] = (UNKNOWN, UP, DOWN, INCOMPATIBLE)
+
+
+class SidecarError(Exception):
+    """No sidecar, a sidecar that will not answer, or one that says no."""
+
+
+class SidecarIncompatible(SidecarError):
+    """A sidecar that answered in a protocol this bridge does not speak."""
+
+
+class SidecarRecord(TypedDict):
+    name: str
+    protocol: str
+    upstream: str
+    transport: str
+    standalone: str
+    probe: str
+    supervised: bool
+
+
+class LinkRecord(TypedDict):
+    sidecar: SidecarRecord
+    endpoint: str
+    link: str
+    since: int
+    protocol: str            # what the sidecar last said it speaks; '' when it did not say
+    last_error: str
+    calls: int
+    failures: int
+
+
+class ProbeRecord(TypedDict, total=False):
+    """What `<name>.probe` answers: the link, and the status or the error."""
+    link: str
+    protocol: str
+    status: object
+    error: str
+
+
+@dataclasses.dataclass(frozen=True)
+class Sidecar:
+    """What a bridge fronts. Declared on the bridge as `SIDECAR`.
+
+      name         the sidecar's name; also the bridge compartment's
+      protocol     `family/major` this bridge speaks, e.g. 'fleetmesh-announce/1'
+      upstream     what the sidecar wraps, for a person: 'py-libp2p gossipsub'
+      transport    how it is reached; one of TRANSPORTS
+      standalone   the command that runs it on its own, for the docs and the
+                   `sidecar` command; '' when it has none
+      probe        the request that proves it is there and returns its status
+      supervised   False: the node depends on it and never starts or restarts
+                   it. Refused when True until the code that would honour it
+                   exists - a declared supervisor nothing runs is worse than none.
+    """
+    name: str
+    protocol: str
+    upstream: str
+    transport: str = 'unix'
+    standalone: str = ''
+    probe: str = 'status'
+    supervised: bool = False
+
+    def __post_init__(self) -> None:
+        family, _, major = self.protocol.rpartition('/')
+        if not family or not major.isdigit():
+            raise ValueError('sidecar %s: protocol is family/major, got %r' % (self.name, self.protocol))
+        if self.transport not in TRANSPORTS:
+            raise ValueError('sidecar %s: transport %r is not implemented (have: %s)'
+                             % (self.name, self.transport, ', '.join(TRANSPORTS)))
+        if self.supervised:
+            raise ValueError('sidecar %s: supervised=True is not honoured yet; the node depends on a '
+                             'sidecar and never starts it' % self.name)
+
+    @property
+    def family(self) -> str:
+        return self.protocol.rpartition('/')[0]
+
+    @property
+    def major(self) -> str:
+        return self.protocol.rpartition('/')[2]
+
+    def compatible(self, said: str) -> bool:
+        """Whether a sidecar that says it speaks `said` speaks this bridge's
+        protocol. Same family, same major; a minor (`/1.3`) is additive. An
+        unstated protocol is accepted - the sidecars predate the field - and
+        the link records that it was not stated."""
+        if not said:
+            return True
+        family, _, version = said.rpartition('/')
+        return family == self.family and version.split('.')[0] == self.major
+
+    def describe(self) -> SidecarRecord:
+        return {'name': self.name, 'protocol': self.protocol, 'upstream': self.upstream,
+                'transport': self.transport, 'standalone': self.standalone, 'probe': self.probe,
+                'supervised': self.supervised}
+
+
+def jsonline_call(path: str, cmd: str, timeout: float = 5.0, **args: Any) -> tuple[Any, str]:
+    """One request, one reply, one connection, over a unix socket.
+
+    Returns (result, the protocol the reply said it speaks, or ''). Raises
+    SidecarError for all of no sidecar, a sidecar that will not answer, and a
+    sidecar that says no. The wire was announce's; it is every unix sidecar's.
+    """
+    s = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
+    s.settimeout(timeout)
+    try:
+        try:
+            s.connect(path)
+        except OSError as e:
+            raise SidecarError('no sidecar at %s (%s)' % (path, e.strerror or e))
+        try:
+            s.sendall((json.dumps({'cmd': cmd, 'args': args}) + '\n').encode('utf-8'))
+            buf = b''
+            while not buf.endswith(b'\n'):
+                chunk = s.recv(65536)
+                if not chunk:
+                    break
+                buf += chunk
+                if len(buf) > MAX_LINE:
+                    raise SidecarError('the sidecar answered with more than %d bytes' % MAX_LINE)
+        except socket.timeout:
+            raise SidecarError('the sidecar did not answer within %.0fs' % timeout)
+        except OSError as e:
+            raise SidecarError('the sidecar closed the socket (%s)' % (e.strerror or e))
+    finally:
+        s.close()
+    if not buf:
+        raise SidecarError('the sidecar closed the socket without answering')
+    try:
+        reply = json.loads(buf.decode('utf-8'))
+    except (ValueError, UnicodeDecodeError) as e:
+        raise SidecarError('the sidecar answered something that is not JSON (%s)' % e)
+    if not isinstance(reply, dict):
+        raise SidecarError('a reply is an object')
+    said = str(reply.get('protocol') or '')
+    if not reply.get('ok'):
+        raise SidecarError(str(reply.get('error') or 'refused'))
+    return reply.get('result'), said
+
+
+def probe_unix_socket(path: str, request: Mapping[str, Any], timeout: float = 1.0) -> Optional[Mapping[str, Any]]:
+    """Send one JSON line over a unix socket and return the parsed JSON object, or None on error."""
+    try:
+        s = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
+        s.settimeout(timeout)
+        s.connect(path)
+        s.sendall(json.dumps(dict(request)).encode('utf-8') + b'\n')
+        from ..transport import lineproto
+        resp = lineproto.read_json_line(s, cap=MAX_LINE)
+        s.close()
+        if isinstance(resp, dict):
+            return resp
+    except Exception:
+        pass
+    return None
+
+
+# ---- what every bridge declares, derived from its manifest ---------------------
+def bridge_channels(name: str) -> tuple[ch_mod.Channel, ...]:
+    return (
+        ch_mod.Channel(
+            name='%s.sidecar.reachable' % name,
+            description='The %s sidecar answered, in a protocol this bridge speaks.' % name,
+            level=ch_mod.INFO, wire=lambda endpoint, protocol: {'endpoint': endpoint, 'protocol': protocol},
+            say='sidecar up at {endpoint} ({protocol})', tags=(name, 'sidecar', 'bridge')),
+        ch_mod.Channel(
+            name='%s.sidecar.unreachable' % name,
+            description='The %s sidecar did not answer, or refused; what needs it waits.' % name,
+            level=ch_mod.NOTE, wire=lambda why, endpoint: {'why': why, 'endpoint': endpoint},
+            say='sidecar not reachable at {endpoint} ({why})', tags=(name, 'sidecar', 'bridge')),
+        ch_mod.Channel(
+            name='%s.sidecar.incompatible' % name,
+            description='The %s sidecar answered in a protocol this bridge does not speak.' % name,
+            level=ch_mod.TROUBLE, wire=lambda said, wants: {'said': said, 'wants': wants},
+            say='sidecar speaks {said}; this bridge speaks {wants}', tags=(name, 'sidecar', 'bridge')),
+    )
+
+
+def bridge_metrics(name: str) -> tuple[mx_mod.Metric, ...]:
+    return (
+        mx_mod.Metric(name='%s_sidecar_reachable' % name,
+                      description='Times the %s sidecar was found reachable.' % name,
+                      kind=mx_mod.COUNTER, counts='%s.sidecar.reachable' % name, tags=(name, 'sidecar')),
+        mx_mod.Metric(name='%s_sidecar_unreachable' % name,
+                      description='Times the %s sidecar was found unreachable.' % name,
+                      kind=mx_mod.COUNTER, counts='%s.sidecar.unreachable' % name, tags=(name, 'sidecar')),
+        mx_mod.Metric(name='%s_sidecar_incompatible' % name,
+                      description='Times the %s sidecar answered in a protocol this bridge does not speak.' % name,
+                      kind=mx_mod.COUNTER, counts='%s.sidecar.incompatible' % name, tags=(name, 'sidecar')),
+    )
+
+
+def bridge_commands(sc: Sidecar) -> tuple[Command, ...]:
+    return (
+        Command(
+            key='sidecar', name='%s sidecar' % sc.name.capitalize(),
+            description='the sidecar this bridge fronts - what it wraps, the protocol it must speak, how to '
+                        'run it standalone - and the link to it as last found',
+            tags=(sc.name, 'sidecar', 'bridge'), permission='%s/sidecar' % sc.name,
+            run='bridge_state',
+            reply='{sidecar, endpoint, link, since, protocol, last_error, calls, failures}'),
+        Command(
+            key='probe', name='Probe %s sidecar' % sc.name,
+            description='ask the sidecar now (its `%s` request) rather than wait for the next tick' % sc.probe,
+            tags=(sc.name, 'sidecar', 'bridge'), permission='%s/probe' % sc.name,
+            process=(sc.name,), run='probe',
+            reply='{link, protocol, status | error}'),
+    )
+
+
+def _merged(base: tuple[Any, ...], own: tuple[Any, ...], key: Callable[[Any], str]) -> tuple[Any, ...]:
+    """The bridge's own declarations, then the base's it did not declare itself."""
+    have = {key(x) for x in own}
+    return tuple(own) + tuple(x for x in base if key(x) not in have)
+
+
+class Bridge(Compartment, abstract=True):
+    """The compartment that fronts one sidecar. Subclass, declare `SIDECAR`,
+    implement `endpoint()`, and reach the sidecar only through `call()`.
+
+    Optional hooks: `wanted()` (False turns the link checks and trouble off,
+    e.g. a sidecar the config has not enabled) and `forget()` (what the bridge
+    must forget when its link is reset, at start and on every restart).
+    """
+
+    SIDECAR: Optional[Sidecar] = None
+    # What this node loses while the sidecar is down, for the trouble line.
+    CONSEQUENCE: str = 'what needs it waits'
+    LIFECYCLE: lifecycle_mod.Lifecycle = lifecycle_mod.Lifecycle(restart=True)
+    timeout: float = 5.0
+
+    def __init_subclass__(cls, **kw: Any) -> None:
+        sc = cls.__dict__.get('SIDECAR')
+        if isinstance(sc, Sidecar):
+            if cls.name and cls.name != sc.name:
+                raise TypeError('bridge %s fronts sidecar %s: a bridge is named for its sidecar'
+                                % (cls.name, sc.name))
+            name = sc.name
+            cls.CHANNELS = _merged(bridge_channels(name), tuple(cls.__dict__.get('CHANNELS', ())),
+                                   lambda c: c.name)
+            cls.METRICS = _merged(bridge_metrics(name), tuple(cls.__dict__.get('METRICS', ())),
+                                  lambda m: m.name)
+            cls.COMMANDS = _merged(bridge_commands(sc), tuple(cls.__dict__.get('COMMANDS', ())),
+                                   lambda c: c.key)
+        super().__init_subclass__(**kw)
+
+    def __init__(self, node: Any, clock: Optional[Callable[[], int]] = None) -> None:
+        super().__init__(node, clock)
+        self._link_lock: threading.Lock = threading.Lock()
+        self._reset_link()
+
+    # ---- what a subclass provides ---------------------------------------------
+    @property
+    def sidecar(self) -> Sidecar:
+        if self.SIDECAR is None:
+            raise TypeError('%s declares no SIDECAR' % type(self).__name__)
+        return self.SIDECAR
+
+    def endpoint(self) -> str:
+        raise NotImplementedError('%s must say where its sidecar listens' % type(self).__name__)
+
+    def wanted(self) -> bool:
+        return True
+
+    def forget(self) -> None:
+        """What a reset link must take with it. Nothing, by default."""
+
+    # ---- the link --------------------------------------------------------------
+    def _reset_link(self) -> None:
+        self.link: str = UNKNOWN
+        self.link_since: int = self._now()
+        self.link_protocol: str = ''
+        self.link_error: str = ''
+        self.calls: int = 0
+        self.call_failures: int = 0
+        self._up: Optional[bool] = None       # the reachability the journal was last told
+        self._said: Optional[str] = None      # the last unreachable reason it was told
+
+    def start(self) -> None:
+        # A fresh link: the next call finds out. Nothing is dialled here, so a
+        # sidecar that hangs cannot hold the node's start.
+        with self._link_lock:
+            self._reset_link()
+        self.forget()
+
+    def stop(self) -> Any:
+        with self._link_lock:
+            self._reset_link()
+        return None
+
+    def _now(self) -> int:
+        return int(getattr(self, 'clock', time.time)())
+
+    def call(self, cmd: str, **args: Any) -> Any:
+        """The only way to the sidecar. Moves the link and says so once each way.
+
+        On a link that is not up, the manifest's `probe` request goes first,
+        as a handshake: the version gate must refuse BEFORE a request carrying
+        data reaches a sidecar that would misread it, not after. One extra
+        round trip per (re)connection, none while the link is up.
+
+        Raises SidecarError (SidecarIncompatible for a protocol this bridge
+        does not speak); the caller decides what a missing sidecar costs it.
+        """
+        if self.link != UP and cmd != self.sidecar.probe:
+            self._exchange(self.sidecar.probe, {})
+        return self._exchange(cmd, args)
+
+    def _exchange(self, cmd: str, args: Mapping[str, Any]) -> Any:
+        now = self._now()
+        kw = dict(args)
+        to = float(kw.pop('timeout', self.timeout))
+        try:
+            result, said = jsonline_call(self.endpoint(), cmd, timeout=to, **kw)
+        except SidecarError as e:
+            self.call_failures += 1
+            self._down(str(e), now)
+            raise
+        if not self.sidecar.compatible(said):
+            self.call_failures += 1
+            self._incompatible(said, now)
+            raise SidecarIncompatible('the %s sidecar speaks %s; this bridge speaks %s'
+                                      % (self.sidecar.name, said, self.sidecar.protocol))
+        self.calls += 1
+        self.link_protocol = said
+        self._reachable()
+        return result
+
+    def probe(self) -> ProbeRecord:
+        """Ask the sidecar now. Never raises: the answer says what happened."""
+        try:
+            status = self.call(self.sidecar.probe)
+        except SidecarError as e:
+            return {'link': self.link, 'protocol': self.link_protocol, 'error': str(e)}
+        return {'link': self.link, 'protocol': self.link_protocol, 'status': status}
+
+    def _move_link(self, to: str, now: int, error: str = '') -> bool:
+        with self._link_lock:
+            moved = to != self.link
+            if moved:
+                self.link, self.link_since = to, now
+            self.link_error = error
+        return moved
+
+    def _down(self, why: str, now: int) -> None:
+        self._move_link(DOWN, now, why)
+        if self._up is not False or self._said != why:
+            self.say('%s.sidecar.unreachable' % self.sidecar.name, why, self.endpoint())
+            self._said = why
+        self.refusals.refuse('sidecar', 'not reachable', subject=self.endpoint(),
+                             detail={'error': why}, at=now)
+        self._up = False
+        self.forget()               # it must be told again what it was told before
+
+    def _incompatible(self, said: str, now: int) -> None:
+        if self._move_link(INCOMPATIBLE, now, 'speaks %s' % said):
+            self.say('%s.sidecar.incompatible' % self.sidecar.name, said, self.sidecar.protocol)
+        self.refusals.refuse('sidecar', 'incompatible', subject=self.endpoint(),
+                             detail={'said': said, 'wants': self.sidecar.protocol}, at=now)
+        self._up = False
+        self.forget()
+
+    def _reachable(self) -> None:
+        self._move_link(UP, self._now())
+        if self._up is not True:
+            self.say('%s.sidecar.reachable' % self.sidecar.name, self.endpoint(),
+                     self.link_protocol or 'protocol unstated')
+        self._up = True
+        self._said = None
+
+    @property
+    def available(self) -> bool:
+        """Whether a compartment that depends on this sidecar should try now."""
+        return self.link == UP
+
+    # ---- being watched ---------------------------------------------------------
+    def bridge_state(self) -> LinkRecord:
+        return {'sidecar': self.sidecar.describe(), 'endpoint': self.endpoint(), 'link': self.link,
+                'since': self.link_since, 'protocol': self.link_protocol, 'last_error': self.link_error,
+                'calls': self.calls, 'failures': self.call_failures}
+
+    def sections(self) -> Mapping[str, Any]:
+        return {'bridge': self.bridge_state()}
+
+    def trouble(self) -> list[str]:
+        if not self.wanted():
+            return []
+        if self.link == INCOMPATIBLE:
+            return ['the %s sidecar at %s %s; this bridge speaks %s - upgrade one of them'
+                    % (self.sidecar.name, self.endpoint(), self.link_error, self.sidecar.protocol)]
+        if self.link == DOWN:
+            return ['the %s sidecar is not reachable at %s; %s'
+                    % (self.sidecar.name, self.endpoint(), self.CONSEQUENCE)]
+        return []
+
+
+def bridges(compartments: Any) -> list[Bridge]:
+    """The bridges among a node's compartments: every sidecar it reaches."""
+    return [c for c in compartments if isinstance(c, Bridge)]
+
+
+__all__ = [
+    'Bridge', 'DOWN', 'INCOMPATIBLE', 'LINKS', 'LinkRecord', 'MAX_LINE', 'ProbeRecord', 'Sidecar', 'SidecarError',
+    'SidecarIncompatible', 'SidecarRecord', 'TRANSPORTS', 'UNKNOWN', 'UP', 'bridge_channels',
+    'bridge_commands', 'bridge_metrics', 'bridges', 'jsonline_call',
+]