fleetmesh-node 0.2.0__py3-none-any.whl

node/planes/services/sidecars/manifest.py

@@ -0,0 +1,304 @@
+"""The sidecar manifest: one JSON document that says what a fronted service
+is, which capability its presence declares, and the self-tests that have to
+pass before the node says so. `node/sidecars.py` runs them; this module only
+reads and checks the document, so a developer can lint one without a node.
+
+    {
+      "sidecar": 1,
+      "name": "grasp",
+      "capability": "<origin>/grasp/1.0",
+      "endpoint_tag": "grasp-endpoint",
+      "checks": [
+        {"name": "http", "method": "HEAD", "path": "/"},
+        {"name": "git-smart-http", "method": "GET",
+         "path": "/fleetmesh-selftest/none.git/info/refs?service=git-upload-pack",
+         "expect": [200, 404],
+         "content_type": "application/x-git-upload-pack-advertisement"}
+      ],
+      "docs": "https://..."
+    }
+
+Three sources feed the same loader: the manifests shipped in
+`node/sidecar_manifests/` (the built-ins: grasp, ncontainer, magpie), a
+manifest an operator names or inlines in the `sidecars` config block, and
+the ones a plugin class declares. The rules are the same for all three but
+one: only a built-in may declare a capability under the node's own
+origin (the one schema/messages.json's identifiers share), because those identifiers are governed by the schema digest.
+Everyone else uses their own origin, as a dialect does (docs/extending.md).
+
+A manifest with no checks is refused. A listener that answers HTTP is not
+evidence that it speaks the protocol, and a capability declared on that
+evidence is a lie a peer acts on.
+"""
+from __future__ import annotations
+
+import json
+import os
+import re
+from typing import Any, Mapping, NamedTuple, Optional
+
+from fleetmesh_core.identity import origin as origin_mod
+
+VERSION = 1
+METHODS = ('HEAD', 'GET')
+# A GET/HEAD that answered with any of these is a live handler for the route:
+# 200 is the happy path, 401/402 a gate in front of it, 404 the route saying
+# "no such object" - which only a handler that owns the route can say. A 5xx,
+# a refused connection or a timeout is not; nor is a 404 from a server that
+# never heard of the path, which is why a check's path is the protocol's own.
+LIVE = frozenset({200, 401, 402, 404})
+_LOCAL_MANIFESTS = os.path.abspath(os.path.join(os.path.dirname(os.path.abspath(__file__)), 'manifests'))
+BUILTIN_DIR = _LOCAL_MANIFESTS if os.path.isdir(_LOCAL_MANIFESTS) else os.path.abspath(os.path.join(os.path.dirname(os.path.abspath(__file__)), '../../../sidecar_manifests'))
+
+_NAME = re.compile(r'^[a-z][a-z0-9_-]{0,31}$')
+_TAG = re.compile(r'^[a-z][a-z0-9-]{0,63}$')
+_CAPABILITY = re.compile(r'^https://[^/\s]+/[A-Za-z0-9_.-]+/\d+\.\d+$')
+_KEYS = frozenset({'sidecar', 'name', 'capability', 'endpoint_tag', 'checks', 'docs', 'mount', 'pricing', 'run'})
+# How to run it, for whoever deploys it (ops/roles/meshnode reads this;
+# the node never does - it only asks whether the configured endpoint answers
+# the checks). An image published to 127.0.0.1:<port> on the host, the path
+# the container's /data is kept at, env pairs and arguments after the image.
+# Which tag of the image is a host's business (an operator pins a reviewed
+# one in its inventory); the manifest names the image and the default tag.
+_RUN_KEYS = frozenset({'image', 'port', 'data', 'env', 'cmd'})
+_IMAGE = re.compile(r'^[a-z0-9][a-z0-9._/-]*(:[A-Za-z0-9._-]+)?(@sha256:[0-9a-f]{64})?$')
+_ENV_KEY = re.compile(r'^[A-Z][A-Z0-9_]*$')
+_PRICING_KEYS = frozenset({'mode', 'blob_route'})
+# How access is metered, if at all. The proxy in front of a sidecar can only
+# enforce what it can see: one 402 before a whole session (git smart-http is
+# many small requests per clone), or one per content-addressed object whose
+# sha256 is in the URL (then the node's own per-blob route is reused). A
+# protocol whose objects are already fetched through the node's Blossom
+# face (ncontainer) needs nothing here.
+PRICING_MODES = ('none', 'session', 'blob')
+_MOUNT = re.compile(r'^/[A-Za-z0-9_.-]+$')
+_CHECK_KEYS = frozenset({'name', 'method', 'path', 'expect', 'content_type'})
+
+
+class ManifestError(ValueError):
+    """The document is not a sidecar manifest; the message says why."""
+
+
+from .types import (
+    CheckDict,
+    RunDict,
+    PricingDict,
+    SidecarDict,
+)
+
+
+class Check(NamedTuple):
+    """One self-test: a request the sidecar's protocol must answer."""
+    name: str
+    method: str
+    path: str
+    expect: frozenset[int] = LIVE
+    content_type: Optional[str] = None     # required prefix on a 200, when the protocol fixes one
+
+    def to_dict(self) -> CheckDict:
+        d: CheckDict = {'name': self.name, 'method': self.method, 'path': self.path, 'expect': sorted(self.expect)}
+        if self.content_type:
+            d['content_type'] = self.content_type
+        return d
+
+
+class Run(NamedTuple):
+    """How a deployer runs the sidecar: a container image published to
+    127.0.0.1:<port>, its data at `data` (default /var/lib/meshnode-<name>)."""
+    image: str
+    port: int
+    data: str
+    env: tuple[tuple[str, str], ...] = ()
+    cmd: str = ''
+
+    def to_dict(self) -> RunDict:
+        d: RunDict = {'image': self.image, 'port': self.port, 'data': self.data}
+        if self.env:
+            d['env'] = dict(self.env)
+        if self.cmd:
+            d['cmd'] = self.cmd
+        return d
+
+
+class Sidecar(NamedTuple):
+    """One fronted service: the name its config block uses, the capability
+    its presence declares, the descriptor tag that carries its URL and the
+    checks that have to pass before any of that is said."""
+    name: str
+    capability: str
+    tag: str
+    checks: tuple[Check, ...]
+    docs: str = ''
+    builtin: bool = False
+    mount: str = ''                       # public path prefix the proxy fronts it at; '' = '/' + name
+    pricing: str = 'none'                 # one of PRICING_MODES
+    blob_route: Optional[str] = None      # pricing blob: anchored regex, group 1 = sha256
+    run: Optional[Run] = None             # how a deployer runs it; the node does not read it
+
+    @property
+    def mount_path(self) -> str:
+        return self.mount or '/' + self.name
+
+    def to_dict(self) -> SidecarDict:
+        d: SidecarDict = {'sidecar': VERSION, 'name': self.name, 'capability': self.capability,
+                          'endpoint_tag': self.tag, 'checks': [c.to_dict() for c in self.checks]}
+        if self.docs:
+            d['docs'] = self.docs
+        if self.mount:
+            d['mount'] = self.mount
+        if self.pricing != 'none':
+            pr: PricingDict = {'mode': self.pricing}
+            if self.blob_route:
+                pr['blob_route'] = self.blob_route
+            d['pricing'] = pr
+        if self.run:
+            d['run'] = self.run.to_dict()
+        return d
+
+
+origin_of = origin_mod.origin_of  # one spelling of the rule; see node/origin.py
+
+
+def _check(raw: Any, i: int) -> Check:
+    where = 'checks[%d]' % i
+    if not isinstance(raw, Mapping):
+        raise ManifestError('%s must be an object' % where)
+    extra = set(raw) - _CHECK_KEYS
+    if extra:
+        raise ManifestError('%s has unknown keys: %s' % (where, ', '.join(sorted(extra))))
+    name = raw.get('name')
+    if not isinstance(name, str) or not _TAG.match(name):
+        raise ManifestError('%s.name must be a short lowercase identifier' % where)
+    method = raw.get('method', 'GET')
+    if method not in METHODS:
+        raise ManifestError('%s.method must be one of %s' % (where, '/'.join(METHODS)))
+    path = raw.get('path')
+    if not isinstance(path, str) or not path.startswith('/') or any(ch.isspace() for ch in path):
+        raise ManifestError('%s.path must start with / and is appended to the endpoint' % where)
+    expect: frozenset[int] = LIVE
+    if 'expect' in raw:
+        e = raw['expect']
+        if (not isinstance(e, list) or not e
+                or not all(isinstance(s, int) and not isinstance(s, bool) and 100 <= s <= 599 for s in e)):
+            raise ManifestError('%s.expect must be a non-empty list of HTTP statuses' % where)
+        if any(s >= 500 for s in e):
+            raise ManifestError('%s.expect accepts a 5xx: a server error is never proof of a protocol' % where)
+        expect = frozenset(e)
+    ctype = raw.get('content_type')
+    if ctype is not None and (not isinstance(ctype, str) or '/' not in ctype or any(ch.isspace() for ch in ctype)):
+        raise ManifestError('%s.content_type must be a media type such as application/json' % where)
+    return Check(name, method, path, expect, ctype)
+
+
+def parse(raw: Any, *, builtin: bool = False, origin: Optional[str] = None) -> Sidecar:
+    """Read one manifest. `origin` is the reserved origin (the node's own,
+    the schema's) that only a built-in may declare under."""
+    if not isinstance(raw, Mapping):
+        raise ManifestError('a sidecar manifest is a JSON object')
+    if raw.get('sidecar') != VERSION:
+        raise ManifestError('"sidecar": %d is the manifest version this node reads' % VERSION)
+    extra = set(raw) - _KEYS
+    if extra:
+        raise ManifestError('unknown keys: %s' % ', '.join(sorted(extra)))
+    name = raw.get('name')
+    if not isinstance(name, str) or not _NAME.match(name):
+        raise ManifestError('name must be a short lowercase identifier (it is the config block and the systemd unit)')
+    cap = raw.get('capability')
+    if not isinstance(cap, str) or not _CAPABILITY.match(cap):
+        raise ManifestError('capability must be https://<origin>/<name>/<major>.<minor>')
+    if origin and not builtin and origin_mod.reserved(cap, origin):
+        raise ManifestError('capability %s is under %s, which only a built-in manifest may declare; '
+                            'use your own origin' % (cap, origin))
+    tag = raw.get('endpoint_tag', name + '-endpoint')
+    if not isinstance(tag, str) or not _TAG.match(tag):
+        raise ManifestError('endpoint_tag must be a short lowercase identifier')
+    checks_raw = raw.get('checks')
+    if not isinstance(checks_raw, list) or not checks_raw:
+        raise ManifestError('checks must be a non-empty list: a sidecar ships its own self-tests')
+    checks = tuple(_check(c, i) for i, c in enumerate(checks_raw))
+    seen = [c.name for c in checks]
+    if len(set(seen)) != len(seen):
+        raise ManifestError('check names repeat: %s' % ', '.join(sorted({n for n in seen if seen.count(n) > 1})))
+    docs = raw.get('docs', '')
+    if not isinstance(docs, str):
+        raise ManifestError('docs must be a URL string')
+    mount = raw.get('mount', '')
+    if not isinstance(mount, str) or (mount and not _MOUNT.match(mount)):
+        raise ManifestError('mount must be a single path segment such as /git (the proxy fronts the sidecar there)')
+    pricing, blob_route = _pricing(raw.get('pricing'))
+    run = _run(raw.get('run'), name)
+    return Sidecar(name, cap, tag, checks, docs, builtin, mount, pricing, blob_route, run)
+
+
+def _run(raw: Any, name: str) -> Optional[Run]:
+    if raw is None:
+        return None
+    if not isinstance(raw, Mapping):
+        raise ManifestError('run must be an object {image, port[, data, env, cmd]}')
+    extra = set(raw) - _RUN_KEYS
+    if extra:
+        raise ManifestError('run has unknown keys: %s' % ', '.join(sorted(extra)))
+    image = raw.get('image')
+    if not isinstance(image, str) or not _IMAGE.match(image):
+        raise ManifestError('run.image must be a container image reference such as ghcr.io/org/name:tag')
+    port = raw.get('port')
+    if not isinstance(port, int) or isinstance(port, bool) or not 1024 <= port <= 65535:
+        raise ManifestError('run.port must be an unprivileged TCP port (1024-65535); the host publishes it on loopback')
+    data = raw.get('data', '/var/lib/meshnode-' + name)
+    if not isinstance(data, str) or not data.startswith('/') or any(ch.isspace() for ch in data):
+        raise ManifestError('run.data must be an absolute path; the container sees it as /data')
+    env_raw = raw.get('env', {})
+    if not isinstance(env_raw, Mapping) or not all(
+            isinstance(k, str) and _ENV_KEY.match(k) and isinstance(v, str) for k, v in env_raw.items()):
+        raise ManifestError('run.env must map UPPER_CASE names to strings')
+    cmd = raw.get('cmd', '')
+    if not isinstance(cmd, str):
+        raise ManifestError('run.cmd must be a string: the arguments after the image')
+    return Run(image, port, data, tuple(sorted(env_raw.items())), cmd)
+
+
+def _pricing(raw: Any) -> tuple[str, Optional[str]]:
+    if raw is None:
+        return 'none', None
+    if not isinstance(raw, Mapping):
+        raise ManifestError('pricing must be an object {mode[, blob_route]}')
+    extra = set(raw) - _PRICING_KEYS
+    if extra:
+        raise ManifestError('pricing has unknown keys: %s' % ', '.join(sorted(extra)))
+    mode = raw.get('mode', 'none')
+    if mode not in PRICING_MODES:
+        raise ManifestError('pricing.mode must be one of %s' % '/'.join(PRICING_MODES))
+    route = raw.get('blob_route')
+    if mode == 'blob':
+        if not isinstance(route, str) or not route.startswith('^'):
+            raise ManifestError('pricing.blob_route is required for mode blob: an anchored regex whose first group is the sha256')
+        try:
+            if re.compile(route).groups < 1:
+                raise ManifestError('pricing.blob_route needs a capturing group for the sha256')
+        except re.error as e:
+            raise ManifestError('pricing.blob_route is not a regex: %s' % e) from e
+    elif route is not None:
+        raise ManifestError('pricing.blob_route only applies to mode blob')
+    return mode, route if mode == 'blob' else None
+
+
+def load_file(path: str, **kw: Any) -> Sidecar:
+    with open(path, 'r', encoding='utf-8') as f:
+        try:
+            raw = json.load(f)
+        except ValueError as e:
+            raise ManifestError('%s: not JSON (%s)' % (path, e)) from e
+    try:
+        return parse(raw, **kw)
+    except ManifestError as e:
+        raise ManifestError('%s: %s' % (path, e)) from e
+
+
+def builtins(directory: str = BUILTIN_DIR) -> tuple[Sidecar, ...]:
+    """The manifests shipped with the node, in file-name order - that order
+    is the order their capabilities appear on the descriptor."""
+    out = []
+    for fn in sorted(os.listdir(directory)):
+        if fn.endswith('.json') and not fn.startswith('.'):
+            out.append(load_file(os.path.join(directory, fn), builtin=True))
+    return tuple(out)

node/planes/services/sidecars/manifests/10-grasp.json

@@ -0,0 +1,37 @@
+{
+  "sidecar": 1,
+  "name": "grasp",
+  "capability": "https://fleetmesh.org/grasp/1.0",
+  "endpoint_tag": "grasp-endpoint",
+  "checks": [
+    {
+      "name": "http",
+      "method": "HEAD",
+      "path": "/"
+    },
+    {
+      "name": "git-smart-http",
+      "method": "GET",
+      "path": "/fleetmesh-selftest/none.git/info/refs?service=git-upload-pack",
+      "expect": [
+        200,
+        401,
+        402,
+        404
+      ],
+      "content_type": "application/x-git-upload-pack-advertisement"
+    }
+  ],
+  "mount": "/git",
+  "pricing": {
+    "mode": "session"
+  },
+  "run": {
+    "image": "ghcr.io/nostr-protocol/ngit-grasp:latest",
+    "port": 7861,
+    "env": {
+      "GRASP_LISTEN": "0.0.0.0:7861",
+      "GRASP_DATA_DIR": "/data"
+    }
+  }
+}

node/planes/services/sidecars/manifests/20-ncontainer.json

@@ -0,0 +1,31 @@
+{
+  "sidecar": 1,
+  "name": "ncontainer",
+  "capability": "https://fleetmesh.org/ncontainer/1.0",
+  "endpoint_tag": "ncontainer-endpoint",
+  "checks": [
+    {
+      "name": "http",
+      "method": "HEAD",
+      "path": "/"
+    },
+    {
+      "name": "oci-v2",
+      "method": "GET",
+      "path": "/v2/",
+      "expect": [
+        200,
+        401
+      ]
+    }
+  ],
+  "mount": "/oci",
+  "run": {
+    "image": "ghcr.io/ncontainer/ncontainer-gateway:latest",
+    "port": 7862,
+    "env": {
+      "NCONTAINER_LISTEN": "0.0.0.0:7862",
+      "NCONTAINER_DATA_DIR": "/data"
+    }
+  }
+}

node/planes/services/sidecars/manifests/30-magpie.json

@@ -0,0 +1,28 @@
+{
+  "sidecar": 1,
+  "name": "magpie",
+  "capability": "https://fleetmesh.org/magpie/1.0",
+  "endpoint_tag": "magpie-endpoint",
+  "checks": [
+    {
+      "name": "http",
+      "method": "HEAD",
+      "path": "/"
+    },
+    {
+      "name": "files-api",
+      "method": "HEAD",
+      "path": "/api/v1/files/0000000000000000000000000000000000000000000000000000000000000000/content"
+    }
+  ],
+  "mount": "/magpie",
+  "pricing": {
+    "mode": "blob",
+    "blob_route": "^/api/v1/files/([0-9a-f]{64})/content$"
+  },
+  "run": {
+    "image": "magpie:latest",
+    "port": 7863,
+    "cmd": "serve --data /data --listen 0.0.0.0:7863 --allow-public"
+  }
+}

node/planes/services/sidecars/types.py

@@ -0,0 +1,43 @@
+"""Types and records for the Sidecars compartment."""
+from __future__ import annotations
+
+from typing import Any, Mapping, Optional, TypedDict
+
+
+class CheckDict(TypedDict, total=False):
+    name: str
+    method: str
+    path: str
+    expect: list[int]
+    content_type: str
+
+
+class RunDict(TypedDict, total=False):
+    image: str
+    port: int
+    data: str
+    env: Mapping[str, str]
+    cmd: str
+
+
+class PricingDict(TypedDict, total=False):
+    mode: str
+    blob_route: str
+
+
+class SidecarDict(TypedDict, total=False):
+    sidecar: int
+    name: str
+    capability: str
+    endpoint_tag: str
+    checks: list[CheckDict]
+    docs: str
+    mount: str
+    pricing: PricingDict
+    run: RunDict
+
+
+class SidecarStatusReport(TypedDict):
+    verified: Optional[bool]
+    last_checked: Optional[int]
+    checks: list[Mapping[str, Any]]

node/planes/services/video/__init__.py

@@ -0,0 +1,106 @@
+"""Fleetmesh Video Delivery & High-Demand Transit Routing subsystem.
+
+Composes:
+- `VideoStreamPackager` (Phase 2): CMAF/LL-HLS segment packaging, `RhioBaoTree`
+  1024-byte chunked Merkle verification, `p2panda` CBOR manifest logs,
+  cross-protocol stream announcements, and `PaidFetch` / `CrossProtocolSettlementGateway`
+  HTTP 402 commercial settlement.
+- `TransitReflectorManager` & `MoQFrame` (Phase 3): Automated `Capacity` shed
+  handovers (`narrow_scope` / `amend_down` vs. `restore_below`), HTTP 307 viewer
+  redirection, MoQ ring-buffer reflection over FIPS port 30820 and Iroh UDP,
+  and Rhio NATS JetStream cluster fanout.
+- `TransitLedgerStore` (Phase 4): Record-Then-Answer SQLite persistence for
+  bilateral hash-chained `TransitReceipt` IOUs, soft debt ceiling degradation
+  (`degraded-audio-only`), and bartered utility repayments (`blob_shoring`,
+  `verified_compute`, `dvm`, `mediation`).
+- `VideoStreamCommandProvider` & `register_stream_cli` (Phase 5): CLI commands
+  for starting streams, inspecting transit debt, and settling bartered IOUs.
+"""
+from __future__ import annotations
+
+from .barter import (
+    BarterHookResult,
+    BarterSettlementBridge,
+    normalize_peer_pubkey,
+)
+from .cli import (
+    VideoStreamCommandProvider,
+    cmd_stream,
+    register_stream_cli,
+    run_stream_cli,
+)
+from .compartment import (
+    VideoConfig,
+    VideoTransitCompartment,
+)
+from .ingest import (
+    CMAFBoxInfo,
+    IngestSegmentReport,
+    LiveMediaIngestBridge,
+    SpoolPollReport,
+    make_synthetic_cmaf_init,
+    make_synthetic_cmaf_segment,
+    parse_cmaf_boxes,
+)
+from .ledger import (
+    DEFAULT_UTILITY_BYTES_RATES,
+    VALID_UTILITY_KINDS,
+    BarterCreditRow,
+    DebtSummary,
+    OutboundChainState,
+    TransitLedgerStore,
+    UtilityKind,
+)
+from .packager import (
+    SegmentMultiAddress,
+    StreamStateRecord,
+    VideoStreamPackager,
+)
+from .player import (
+    mount_player_routes,
+    render_directory_html,
+    render_player_html,
+)
+from .transit import (
+    MoQFrame,
+    MoQFrameDict,
+    ReflectionResult,
+    TransitReflectorManager,
+)
+
+__all__ = [
+    'BarterHookResult',
+    'BarterSettlementBridge',
+    'normalize_peer_pubkey',
+    'VideoConfig',
+    'VideoTransitCompartment',
+    'CMAFBoxInfo',
+    'IngestSegmentReport',
+    'SpoolPollReport',
+    'parse_cmaf_boxes',
+    'make_synthetic_cmaf_init',
+    'make_synthetic_cmaf_segment',
+    'LiveMediaIngestBridge',
+    'VideoStreamCommandProvider',
+    'cmd_stream',
+    'register_stream_cli',
+    'run_stream_cli',
+    'UtilityKind',
+    'VALID_UTILITY_KINDS',
+    'DEFAULT_UTILITY_BYTES_RATES',
+    'BarterCreditRow',
+    'DebtSummary',
+    'OutboundChainState',
+    'TransitLedgerStore',
+    'StreamStateRecord',
+    'SegmentMultiAddress',
+    'VideoStreamPackager',
+    'render_directory_html',
+    'render_player_html',
+    'mount_player_routes',
+    'MoQFrameDict',
+    'ReflectionResult',
+    'MoQFrame',
+    'TransitReflectorManager',
+]
+