fleetmesh-node 0.2.0__py3-none-any.whl

node/planes/services/sidecars/compartment.py

@@ -0,0 +1,461 @@
+"""Sidecars: fronting off-the-shelf projects the node does not reimplement.
+
+This compartment does not speak git smart-http, the OCI distribution
+protocol or Magpie's media API itself - ngit-grasp, an ncontainer-compatible
+gateway and magpie already do, and reimplementing any of them (Option B,
+considered and rejected each time) buys nothing a sidecar does not already
+give for less code and less to keep current with upstream. What this
+compartment owns is the bridge:
+
+  * which capabilities this node declares, and the endpoint tags a leaf
+    reads to find the sidecar - all opt-in, all carried on the kind 11801
+    descriptor the same way mint/price already are (see
+    PaidFetch.descriptor_tags).
+  * whether the sidecar actually speaks its protocol. Every sidecar ships
+    its own self-tests (a git `info/refs`, an OCI `/v2/`, a magpie files
+    route): they run when the compartment starts, on the usual cadence after
+    that, and on demand through the `sidecar_test` verb. A capability is
+    declared ONLY while the tests pass - a config flag alone declares
+    nothing - and when the verified set changes the compartment rebuilds the
+    descriptor there and then, so a kind 11801 never advertises a protocol
+    nothing behind this node can answer. A failing sidecar is `trouble()`
+    the way every other mechanism here is.
+  * nothing about payment: PaidFetch.quote_session (GRASP, coarse per
+    clone/fetch session) and PaidFetch.quote (ncontainer and magpie, per blob -
+    a resolved OCI digest or a Magpie file sha256 is priced exactly like any
+    other Blossom fetch) are the payment surface, called from the reverse
+    proxy in front of each sidecar, not from here.
+
+What a sidecar IS lives in a manifest (node/sidecar_manifest.py), not here:
+the node ships three in node/planes/services/sidecars/manifests/ (grasp, ncontainer, magpie);
+an operator adds a fourth with `sidecars.<name>.manifest`; a plugin may
+declare more. This compartment reads the `sidecars` config block, resolves
+each entry to a manifest, runs the manifest's checks and declares what
+passes. Adding a sidecar is a JSON file - not a change to this one.
+
+A node with no `sidecars` block behaves exactly as before: no capability
+tag, no endpoint tag, nothing to test, nothing in `trouble()`.
+"""
+from __future__ import annotations
+
+import urllib.parse
+from typing import Any, Callable, Mapping, MutableMapping, NamedTuple, Optional
+
+from node.transport import netfetch
+from . import manifest as manifest_mod
+from node import compartment as compartment_mod
+from node.compartment import stores as st
+from node.channels import Channel, INFO, NOTE
+from node.metrics import COUNTER, Metric
+from node.compartment import Command, Every, Need
+from node.config.parameter import Parameter
+from fleetmesh_core.wire import validate
+from node.config import SidecarEntry, sidecars_of, SidecarsConfig
+from .manifest import Check, ManifestError, Sidecar  # noqa: F401 - the compartment's own vocabulary
+
+# The origin whose capability identifiers the schema digest governs: only a
+# shipped manifest may declare under it. Derived from the schema, never
+# written down here (node/ names no host; see test_node's unplug scan).
+RESERVED_ORIGIN = validate.FLEETMESH_ORIGIN.rstrip('/')
+
+# The manifests the node ships, in descriptor order. Operator and plugin
+# manifests are appended behind them, in config order.
+BUILTINS: tuple[Sidecar, ...] = manifest_mod.builtins()
+_BUILTIN_CAPS = {s.name: s.capability for s in BUILTINS}
+CAP_GRASP = _BUILTIN_CAPS['grasp']
+CAP_NCONTAINER = _BUILTIN_CAPS['ncontainer']
+CAP_MAGPIE = _BUILTIN_CAPS['magpie']
+SIDECARS = BUILTINS   # the historical name; the live set is Sidecars.registry()
+_BUILTIN_BY_NAME: dict[str, Sidecar] = {s.name: s for s in BUILTINS}
+
+
+from .types import SidecarStatusReport
+
+
+def resolve_manifest(name: str, entry: Mapping[str, Any],
+                     extra: Mapping[str, Sidecar] = {}) -> Sidecar:
+    """The manifest a config entry stands for: inline or `@file` when given,
+    else a built-in or plugin-declared manifest of that name. A manifest's
+    `name` must match the entry key it is configured under - the key is the
+    systemd unit and the Caddy route, the manifest the protocol."""
+    raw = entry.get('manifest')
+    if raw is None:
+        s = _BUILTIN_BY_NAME.get(name) or extra.get(name)
+        if s is None:
+            raise ManifestError('sidecars.%s: not a built-in (%s) and no manifest given' % (
+                name, ', '.join(_BUILTIN_BY_NAME)))
+        return s
+    if isinstance(raw, str):
+        if not raw.startswith('@'):
+            raise ManifestError('sidecars.%s.manifest must be "@/path/to/manifest.json" or an object' % name)
+        s = manifest_mod.load_file(raw[1:], origin=RESERVED_ORIGIN)
+    else:
+        s = manifest_mod.parse(raw, origin=RESERVED_ORIGIN)
+    if s.name != name:
+        raise ManifestError('sidecars.%s: the manifest calls itself %r; the key and the name must agree' % (name, s.name))
+    return s
+
+
+# How often a tick re-runs the checks - matching the cadence other
+# compartments use for anything that costs a network round trip. The first
+# run is at start(), not a cadence later.
+_CHECK_EVERY_SECONDS = 300
+_PROBE_TIMEOUT = 3.0
+
+
+class ProbeResult(NamedTuple):
+    """What one check said. `status` is the HTTP status when there was one."""
+    name: str
+    ok: bool
+    status: Optional[int]
+    why: str
+
+
+Fetcher = Callable[[str, str], tuple[int, Mapping[str, str]]]
+
+
+def _http(url: str, method: str, timeout: float = _PROBE_TIMEOUT) -> tuple[int, Mapping[str, str]]:
+    """One request to the sidecar: (status, headers). Only http/https are
+    ever dialled - the URL is operator config, but urlopen would otherwise
+    honour file:// and ftp:// too - and it goes through netfetch so redirects
+    stay inside the same policy. allow_private=True because sidecars are
+    local by design. Raises on anything that is not an HTTP answer."""
+    parts = urllib.parse.urlsplit(url)
+    if parts.scheme not in ('http', 'https'):
+        raise ValueError('not an http(s) URL')
+    try:
+        _body, headers = netfetch.fetch(url, max_bytes=4096, timeout=timeout, allow_private=True, method=method)
+        return 200, headers
+    except netfetch.FetchHTTPError as e:
+        return e.code, e.headers or {}
+
+
+def _reachable(url: str, timeout: float = _PROBE_TIMEOUT) -> bool:
+    """HEAD the sidecar. Reachable means it answered HTTP at all: a 4xx is a
+    live process that disliked HEAD, not an absent one."""
+    try:
+        status, _ = _http(url, 'HEAD', timeout)
+        return status < 500
+    except Exception:
+        return False
+
+
+def run_check(fetch: Fetcher, endpoint: str, check: Check) -> ProbeResult:
+    """One check against one endpoint, never raising."""
+    url = endpoint.rstrip('/') + check.path
+    try:
+        status, headers = fetch(url, check.method)
+    except Exception as e:
+        return ProbeResult(check.name, False, None, '%s: %s' % (type(e).__name__, str(e)[:80] or 'no answer'))
+    if status not in check.expect:
+        return ProbeResult(check.name, False, status, 'HTTP %d, wanted one of %s' % (status, sorted(check.expect)))
+    if status == 200 and check.content_type:
+        ctype = next((v for k, v in headers.items() if k.lower() == 'content-type'), '')
+        if not str(ctype).lower().startswith(check.content_type):
+            return ProbeResult(check.name, False, status, 'content-type %r is not %s' % (ctype, check.content_type))
+    return ProbeResult(check.name, True, status, '')
+
+
+class Sidecars(compartment_mod.Compartment):
+    """Declares one capability per sidecar whose self-tests pass, and keeps
+    testing the sidecars behind them. See module docstring for what it does
+    not do."""
+
+    name = 'sidecars'
+    reads = ('sidecars',) + tuple(s.name for s in BUILTINS)   # the old top-level keys, one release more
+    live = ('sidecars',) + tuple(s.name for s in BUILTINS)
+    config_cls = SidecarsConfig
+    STORES = (
+        st.Store(key='probes', description='recent sidecar probe results',
+                 kind=st.RING, shape=dict, bound=256),
+    )
+    CHANNELS = (
+        Channel('sidecar_verified', 'Sidecar self-tests passed and capability declared', level=INFO, say='sidecar {name} verified and declared', tags=('sidecars', 'probe')),
+        Channel('sidecar_failed', 'Sidecar self-test failed and capability retracted', level=NOTE, say='sidecar {name} self-test failed: {why}', tags=('sidecars', 'probe')),
+    )
+    METRICS = (
+        Metric(name='sidecars_verified_total', description='Times sidecars verified.', kind=COUNTER, counts='sidecar_verified', tags=('sidecars', 'probe')),
+        Metric(name='sidecars_failed_total', description='Times sidecar probes failed.', kind=COUNTER, counts='sidecar_failed', tags=('sidecars', 'probe')),
+    )
+    NEEDS = (
+        Need('cfg', 'cfg', getter=True),
+        Need('descriptor', 'descriptor'),
+        Need('redescribe', 'redescribe'),
+    )
+    EVERY = (Every(seconds=_CHECK_EVERY_SECONDS, run='_check_endpoints'),)
+    COMMANDS = (
+        Command(
+            key='sidecars',
+            name='Configured Sidecars',
+            description='each configured sidecar, its endpoint, whether it is verified and its last self-test',
+            tags=('sidecars', 'status'),
+            permission='sidecars/sidecars',
+            run='list_sidecars',
+            reply='dict',
+        ),
+        Command(
+            key='sidecar_test',
+            name='Sidecar Self-Test',
+            description='run the self-tests of one sidecar (or all) now and redeclare if the result changed',
+            tags=('sidecars', 'test'),
+            parameters=(
+                Parameter(key='name', name='name', description='test: only this sidecar (default all enabled)', type=str, default=None, required=False),
+            ),
+            permission='sidecars/sidecar_test',
+            run='run_sidecar_test',
+            reply='{name: {verified, checks: [{name, ok, status, why}], at}}',
+        ),
+    )
+
+    def __init__(self, cfg: Callable[[], Mapping[str, Any]], clock: Optional[Callable[[], int]] = None,
+                 checker: Optional[Callable[[str], bool]] = None, node: Any = None,
+                 fetch: Fetcher = _http,
+                 descriptor: Optional[Callable[[], Optional[Mapping[str, Any]]]] = None,
+                 redescribe: Optional[Callable[[], Any]] = None) -> None:
+        super().__init__(node, clock)
+        self._cfg = cfg
+        # `checker(url) -> bool` is the older, coarser injection: when given it
+        # stands in for every check of a sidecar. `fetch` is the real seam.
+        self._checker = checker
+        self._fetch = fetch
+        self._descriptor = descriptor or (lambda: None)
+        self._redescribe = redescribe
+        self._last_checked: dict[str, int] = {}  # not a store: local configured sidecars probe timestamps
+        self._results: dict[str, list[ProbeResult]] = {}  # not a store: local configured sidecars probe results
+        self._verified: dict[str, bool] = {}  # not a store: local configured sidecars verified flags
+        self._declared: dict[str, Sidecar] = {}  # not a store: plugin-declared manifests, by name
+        self._manifest_errors: dict[str, str] = {}  # not a store: manifest errors for configured sidecars
+        self.redeclared = 0
+        self._closed = False
+
+    # -- config -----------------------------------------------------------
+    def _entries(self) -> dict[str, SidecarEntry]:
+        """The `sidecars` block as it is now (old top-level keys folded in);
+        every sidecar reads live."""
+        out = {}
+        for name, e in sidecars_of(self._cfg()).items():
+            e = e or {}
+            out[name] = SidecarEntry(enabled=bool(e.get('enabled', False)), endpoint=e.get('endpoint'),
+                                     manifest=e.get('manifest'))
+        return out
+
+    def block(self, name: str) -> SidecarEntry:
+        return self._entries().get(name, SidecarEntry())
+
+    def enabled(self, name: str) -> bool:
+        return self.block(name).on
+
+    def registry(self) -> list[Sidecar]:
+        """Every sidecar this node knows of, in descriptor order: the
+        built-ins, then plugin-declared ones, then any the operator named
+        only by manifest. A configured entry whose manifest does not resolve
+        is remembered as a manifest problem (trouble()) and skipped - a
+        bad manifest declares nothing, same as a failing check."""
+        self._manifest_errors = {}  # not a store: reset on registry evaluation
+        out: list[Sidecar] = list(BUILTINS)
+        seen = {s.name for s in out}
+        for name, s in self._declared.items():
+            if name not in seen:
+                out.append(s); seen.add(name)
+        for name, e in self._entries().items():
+            if e.manifest is None and name in seen:
+                continue
+            try:
+                s = resolve_manifest(name, {'manifest': e.manifest}, self._declared)
+            except (ManifestError, OSError) as err:
+                self._manifest_errors[name] = str(err)
+                continue
+            if name in seen:            # operator manifest overrides a plugin's, never a built-in
+                if name in _BUILTIN_BY_NAME:
+                    self._manifest_errors[name] = 'sidecars.%s: %s is a built-in; its manifest is not overridable' % (name, name)
+                    continue
+                out = [s if x.name == name else x for x in out]
+            else:
+                out.append(s); seen.add(name)
+        return out
+
+    def manifest(self, name: str) -> Sidecar:
+        for s in self.registry():
+            if s.name == name:
+                return s
+        raise KeyError('no sidecar named %r (have: %s)' % (name, ', '.join(x.name for x in self.registry())))
+
+    def declare(self, sidecar: Sidecar) -> None:
+        """A plugin (or test) adds a manifest the node did not ship. Refused
+        under the reserved origin, like any other third-party manifest."""
+        manifest_mod.parse(sidecar.to_dict(), origin=RESERVED_ORIGIN)
+        self._declared[sidecar.name] = sidecar
+
+    # Readable names for the individual sidecars; enabled(name) is the
+    # generic form and what the table-driven paths use.
+    def grasp_enabled(self) -> bool:
+        return self.enabled('grasp')
+
+    def ncontainer_enabled(self) -> bool:
+        return self.enabled('ncontainer')
+
+    def magpie_enabled(self) -> bool:
+        return self.enabled('magpie')
+
+    def _endpoints(self) -> dict[str, str]:
+        """Enabled entries whose manifest resolved. One with a manifest
+        problem is not an endpoint: it is neither tested nor declared."""
+        known = {s.name for s in self.registry()} - set(self._manifest_errors)
+        return {n: str(e.endpoint) for n, e in self._entries().items() if e.on and n in known}
+
+    def verified(self, name: str) -> bool:
+        """Enabled now AND its last self-test passed."""
+        return self.enabled(name) and self._verified.get(name, False)
+
+    # -- what the descriptor says ------------------------------------------
+    def capability_protocols(self) -> list[str]:
+        """Extra `c` protocol identifiers this node's descriptor should carry,
+        alongside the required ones `descriptor.build` always adds. Only the
+        sidecars whose self-tests pass: enabling one declares nothing until
+        it answers."""
+        return [s.capability for s in self.registry() if self.verified(s.name)]
+
+    def descriptor_tags(self) -> list[list[str]]:
+        """<name>-endpoint, only for a verified sidecar - see
+        schema/kinds.json's note on kind 11801."""
+        eps = self._endpoints()
+        return [[s.tag, eps[s.name]] for s in self.registry() if s.name in eps and self.verified(s.name)]
+
+    def _declared_on_descriptor(self) -> set[str]:
+        desc = self._descriptor()
+        if not desc:
+            return set()
+        caps = {t[1] for t in desc.get('tags', []) if len(t) > 1 and t[0] == 'c'}
+        return {s.name for s in self.registry() if s.capability in caps}
+
+    # -- the self-tests ------------------------------------------------------
+    def test(self, name: str, now: Optional[int] = None) -> list[ProbeResult]:
+        """Run one sidecar's checks now. Records the verdict; does not redeclare."""
+        s = self.manifest(name)
+        at = now if now is not None else self.clock()
+        url = self._endpoints().get(name)
+        if url is None:
+            results = [ProbeResult('enabled', False, None, 'not enabled')]
+        elif self._checker is not None:
+            up = self._checker(url)
+            results = [ProbeResult(c.name, up, 200 if up else None, '' if up else 'no answer') for c in s.checks]
+        else:
+            results = [run_check(self._fetch, url, c) for c in s.checks]
+        self._last_checked[name] = at
+        self._results[name] = results
+        self._verified[name] = url is not None and all(r.ok for r in results)
+        for r in results:
+            if not r.ok:
+                self.refusals.refuse('sidecar', 'self-test %s failed' % r.name, subject=name,
+                                     detail={'endpoint': url or '', 'status': r.status, 'why': r.why}, at=at)
+        return results
+
+    def test_all(self, now: Optional[int] = None, only: Optional[str] = None) -> dict[str, list[ProbeResult]]:
+        names = [only] if only else list(self._endpoints())
+        return {n: self.test(n, now) for n in names}
+
+    def redeclare_if_changed(self) -> bool:
+        """Rebuild the descriptor when what should be declared differs from
+        what the current one says. True when it was rebuilt."""
+        want = {s.name for s in self.registry() if self.verified(s.name)}
+        if want == self._declared_on_descriptor():
+            return False
+        if self._redescribe is None:
+            return False
+        self._redescribe()
+        self.redeclared += 1
+        return True
+
+    # -- lifecycle -----------------------------------------------------------
+    def start(self) -> None:
+        """Test every enabled sidecar before anything is declared, and bring
+        the descriptor into agreement at once rather than a cadence later."""
+        if self._endpoints():
+            self.test_all(self.clock())
+        self._every_last['_check_endpoints'] = self.clock()
+        self.redeclare_if_changed()
+
+    def _check_endpoints(self, now: Optional[int] = None) -> list[tuple[Any, ...]]:
+        now = now if now is not None else self.clock()
+        out: list[tuple[Any, ...]] = []
+        changed = False
+        for name in self._endpoints():
+            was = self._verified.get(name)
+            self.test(name, now)
+            is_ok = self._verified[name]
+            if was is not None and was != is_ok:
+                out.append(('sidecar', name, 'verified' if is_ok else 'failing'))
+            changed = changed or (was != is_ok)
+        if changed and self.redeclare_if_changed():
+            out.append(('sidecar', 'descriptor', 'redeclared'))
+        return out
+
+    _tick_runs_every = True
+
+    def tick(self, now: int) -> list[tuple[Any, ...]]:
+        out: list[tuple[Any, ...]] = []
+        if any(name not in self._last_checked for name in self._endpoints()):
+            self._every_last['_check_endpoints'] = now
+            out += self._check_endpoints(now)
+        else:
+            out += self.run_every(now)
+        changed = False
+        # A sidecar removed from the config stops being verified without a probe.
+        for name in [n for n in self._verified if n not in self._endpoints()]:
+            del self._verified[name]
+            changed = True
+        if changed and self.redeclare_if_changed():
+            out.append(('sidecar', 'descriptor', 'redeclared'))
+        return out
+
+    def _report(self, name: str) -> SidecarStatusReport:
+        return {'verified': self.verified(name), 'last_checked': self._last_checked.get(name),
+                'checks': [r._asdict() for r in self._results.get(name, [])]}
+
+    def state(self) -> Mapping[str, Any]:
+        if self._closed:
+            return {'stopped': True}
+        out: MutableMapping[str, Any] = {}
+        entries = self._entries()
+        for s in self.registry():
+            if s.name in entries:
+                out[s.name] = {'enabled': self.enabled(s.name), 'endpoint': self.block(s.name).endpoint,
+                               'capability': s.capability, 'builtin': s.builtin,
+                               'reachable': self._verified.get(s.name), **self._report(s.name)}
+            else:
+                out[s.name] = {'enabled': False, 'capability': s.capability, 'builtin': s.builtin}
+        for name, why in self._manifest_errors.items():
+            out[name] = {'enabled': entries[name].enabled, 'manifest_error': why}
+        out['redeclared'] = self.redeclared
+        return out
+
+    def trouble(self) -> list[str]:
+        out = []
+        eps = self._endpoints()
+        self.registry()
+        for name, why in self._manifest_errors.items():
+            out.append('%s sidecar has no usable manifest: %s' % (name, why))
+        for name, url in eps.items():
+            if self._verified.get(name) is False:
+                failed = [r for r in self._results.get(name, []) if not r.ok]
+                out.append('%s sidecar is configured at %s but not verified: %s' % (
+                    name, url, '; '.join('%s (%s)' % (r.name, r.why) for r in failed) or 'untested'))
+        return out
+
+    def list_sidecars(self) -> Mapping[str, Any]:
+        return {n: {'endpoint': url, **self._report(n)} for n, url in self._endpoints().items()}
+
+    def run_sidecar_test(self, args: Mapping[str, Any]) -> Mapping[str, Any]:
+        only = args.get('name')
+        if only is not None:
+            self.manifest(only)        # KeyError names the ones there are
+        if only is not None and not self.enabled(only):
+            raise KeyError('%s is not enabled' % only)
+        now = self.clock()
+        results = self.test_all(now, only)
+        redeclared = self.redeclare_if_changed()
+        return {n: {'verified': all(r.ok for r in rs), 'checks': [r._asdict() for r in rs], 'at': now}
+                for n, rs in results.items()} | {'redeclared': redeclared}
+
+    def stop(self) -> Any:
+        self._closed = True