fleetmesh-node 0.2.0__py3-none-any.whl

node/planes/services/video/transit.py

@@ -0,0 +1,478 @@
+"""Transit Reflector Manager, Capacity Shed Handover & MoQ Ring Buffer (Phase 3).
+
+Coordinates:
+- Automated origin-to-supernode handover driven by `planes.peering.capacity`
+  shed transitions (`narrow_scope` / `amend_down` vs. `restore_below`).
+- Real-time Media-over-QUIC (MoQ) / FSP datagram reflection on port 30820 with
+  a bounded drop-on-congestion ring buffer.
+- Soft debt ceiling degradation (`degraded-audio-only`) when a pro-bono origin
+  exceeds `max_unsettled_transit_bytes`.
+- Rhio NATS JetStream cluster fanout (`fleetmesh.video.<stream_id>`).
+"""
+from __future__ import annotations
+
+import collections
+import dataclasses
+import hashlib
+import threading
+import time
+from typing import Any, Callable, Deque, Mapping, Optional, Sequence, TypedDict
+
+from fleetmesh_core.crypto import crypto
+from fleetmesh_core.wire.cbor import cbor_decode, cbor_encode
+from fleetmesh_core.wire.video_transit import (
+    COUNTER_TRANSIT_BYTES_OUT,
+    PORT_VIDEO_STREAM,
+    TransitCloseBody,
+    TransitGrantBody,
+    TransitMode,
+    TransitReceiptBody,
+    TransitSolicitationBody,
+    receipt_link,
+)
+from .ledger import TransitLedgerStore
+from .packager import VideoStreamPackager
+
+
+class MoQFrameDict(TypedDict, total=False):
+    stream_id: str
+    broadcaster: str
+    group_id: int
+    object_id: int
+    track: str
+    is_keyframe: bool
+    is_audio_only: bool
+    payload: bytes
+    sha256: str
+    timestamp: int
+    sig: str
+
+
+class ReflectionResult(TypedDict):
+    accepted: bool
+    forwarded: bool
+    degraded_audio_only: bool
+    dropped_reason: str
+    bytes_fanned_out: int
+    ring_depth: int
+    rhio_published: bool
+
+
+@dataclasses.dataclass(frozen=True)
+class MoQFrame:
+    """Real-time interactive media frame for FIPS FSP (port 30820) and Iroh UDP streams."""
+    stream_id: str
+    broadcaster: str
+    group_id: int
+    object_id: int
+    track: str = 'video'
+    is_keyframe: bool = False
+    is_audio_only: bool = False
+    payload: bytes = b''
+    sha256: str = ''
+    timestamp: int = 0
+    sig: str = ''
+
+    def compute_sha256(self) -> str:
+        return hashlib.sha256(self.payload).hexdigest()
+
+    def signing_digest(self) -> bytes:
+        meta = {
+            'stream_id': self.stream_id,
+            'broadcaster': self.broadcaster.lower(),
+            'group_id': int(self.group_id),
+            'object_id': int(self.object_id),
+            'track': self.track,
+            'is_keyframe': bool(self.is_keyframe),
+            'is_audio_only': bool(self.is_audio_only),
+            'sha256': self.sha256.lower() or self.compute_sha256(),
+            'timestamp': int(self.timestamp),
+        }
+        return hashlib.sha256(crypto.canonical(meta)).digest()
+
+    def sign(self, sk_hex: str) -> 'MoQFrame':
+        digest_hex = self.sha256.lower() or self.compute_sha256()
+        with_hash = dataclasses.replace(self, sha256=digest_hex)
+        sig_hex = crypto.schnorr_sign(with_hash.signing_digest(), sk_hex)
+        return dataclasses.replace(with_hash, sig=sig_hex)
+
+    def verify_integrity(self) -> bool:
+        """Verify payload SHA-256 and broadcaster BIP-340 Schnorr signature.
+        Required security boundary for unauthenticated UDP rungs (IrohLiveEngine).
+        """
+        expected_hash = self.compute_sha256()
+        if self.sha256 and self.sha256.lower() != expected_hash:
+            return False
+        if not self.sig or not self.broadcaster or len(self.sig) != 128:
+            return False
+        try:
+            return crypto.schnorr_verify(self.signing_digest(), self.broadcaster.lower(), self.sig)
+        except (ValueError, TypeError):
+            return False
+
+    def to_cbor(self) -> bytes:
+        d: MoQFrameDict = {
+            'stream_id': self.stream_id,
+            'broadcaster': self.broadcaster.lower(),
+            'group_id': int(self.group_id),
+            'object_id': int(self.object_id),
+            'track': self.track,
+            'is_keyframe': bool(self.is_keyframe),
+            'is_audio_only': bool(self.is_audio_only),
+            'payload': self.payload,
+            'sha256': self.sha256.lower() or self.compute_sha256(),
+            'timestamp': int(self.timestamp),
+            'sig': self.sig.lower(),
+        }
+        return cbor_encode(d, canonical=True)
+
+    @classmethod
+    def from_cbor(cls, data: bytes) -> 'MoQFrame':
+        decoded, _ = cbor_decode(data)
+        if not isinstance(decoded, Mapping):
+            raise ValueError('Decoded MoQFrame CBOR is not a mapping')
+        payload_raw = decoded.get('payload', b'')
+        payload_bytes = bytes(payload_raw) if isinstance(payload_raw, (bytes, bytearray)) else b''
+        return cls(
+            stream_id=str(decoded.get('stream_id', '')),
+            broadcaster=str(decoded.get('broadcaster', '')).lower(),
+            group_id=int(decoded.get('group_id', 0)),
+            object_id=int(decoded.get('object_id', 0)),
+            track=str(decoded.get('track', 'video')),
+            is_keyframe=bool(decoded.get('is_keyframe', False)),
+            is_audio_only=bool(decoded.get('is_audio_only', False)),
+            payload=payload_bytes,
+            sha256=str(decoded.get('sha256', '')).lower(),
+            timestamp=int(decoded.get('timestamp', 0)),
+            sig=str(decoded.get('sig', '')).lower(),
+        )
+
+
+class TransitReflectorManager:
+    """Manages automated capacity shed handovers, MoQ ring-buffer reflection,
+    Rhio NATS JetStream cluster fanout, and bilateral transit metering.
+    """
+
+    def __init__(
+        self,
+        ledger: Optional[TransitLedgerStore] = None,
+        packager: Optional[VideoStreamPackager] = None,
+        ring_capacity: int = 64,
+        rhio_client: Optional[Any] = None,
+        fips_transport: Optional[Any] = None,
+        iroh_engine: Optional[Any] = None,
+        clock: Optional[Callable[[], int]] = None,
+        strain_ledger: Optional[Any] = None,
+    ) -> None:
+        # An injected clock, defaulting to wall time, so a suite can move time
+        # at the boundaries this code decides on. The compartment passes its own.
+        self._clock: Callable[[], int] = clock or (lambda: int(time.time()))
+        # MoQ frames a transport would not take, per transport. A dropped
+        # frame in live video is ordinary and not worth a refusal each, but
+        # these were discarded entirely, so a subscriber whose transport was
+        # wholly broken looked exactly like one receiving everything.
+        self.frames_dropped: dict[str, int] = {}
+        self.ledger = ledger or TransitLedgerStore(':memory:')
+        self.packager = packager
+        self.ring_capacity = max(4, int(ring_capacity))
+        self.rhio_client = rhio_client
+        self.fips_transport = fips_transport
+        self.iroh_engine = iroh_engine
+        self.strain_ledger = strain_ledger
+
+        self._lock = threading.RLock()
+        # stream_id -> active TransitGrantBody
+        self._active_handovers: dict[str, TransitGrantBody] = {}
+        # stream_id -> bounded drop-on-congestion ring buffer of MoQFrames
+        self._rings: dict[str, Deque[MoQFrame]] = {}
+        # grant_id -> cumulative transit_bytes_out metered by reflector
+        self._metered_bytes: dict[str, int] = {}
+
+    # ---- Handshake Helpers --------------------------------------------------
+
+    def build_solicitation(
+        self,
+        stream_id: str,
+        broadcaster: str,
+        bitrate_kbps: int = 4500,
+        max_transit_bytes: int = 2 * 1024 * 1024 * 1024,
+        mode: TransitMode = 'pro-bono',
+        codecs: Sequence[str] = ('avc1.64001f', 'mp4a.40.2'),
+        now: Optional[int] = None,
+    ) -> TransitSolicitationBody:
+        now_val = self._clock() if now is None else int(now)
+        return {
+            'stream_id': stream_id,
+            'broadcaster': broadcaster.lower(),
+            'bitrate_kbps': int(bitrate_kbps),
+            'max_transit_bytes': int(max_transit_bytes),
+            'mode': mode,
+            'codecs': list(codecs),
+            'at': now_val,
+        }
+
+    def accept_solicitation(
+        self,
+        solicitation: TransitSolicitationBody,
+        operator_pk: str,
+        operator_sk: str,
+        reflector_endpoints: Sequence[str],
+        window_seconds: int = 600,
+        receipt_cadence_seconds: int = 60,
+        max_unsettled_transit_bytes: int = 500 * 1024 * 1024,
+        now: Optional[int] = None,
+    ) -> TransitGrantBody:
+        """Operator supernode accepts a transit solicitation and issues a signed grant."""
+        return self.ledger.issue_grant(
+            stream_id=solicitation['stream_id'],
+            broadcaster=solicitation['broadcaster'],
+            operator_pk=operator_pk,
+            operator_sk=operator_sk,
+            window_seconds=window_seconds,
+            receipt_cadence_seconds=receipt_cadence_seconds,
+            max_unsettled_transit_bytes=max_unsettled_transit_bytes,
+            reflector_endpoints=reflector_endpoints,
+            mode=solicitation.get('mode', 'pro-bono'),
+            now=now,
+        )
+
+    # ---- Capacity Shed Ladder Handover Automation ---------------------------
+
+    def is_handover_active(self, stream_id: str) -> bool:
+        """Return True if `stream_id` currently has an active transit reflector handover."""
+        with self._lock:
+            return stream_id in self._active_handovers
+
+    def active_sessions(self) -> dict[str, TransitGrantBody]:
+        """Return a snapshot of active stream_id -> TransitGrantBody handovers."""
+        with self._lock:
+            return dict(self._active_handovers)
+
+    def engage_handover(self, stream_id: str, grant: TransitGrantBody) -> list[str]:
+        """Engage transit handover for `stream_id` using `grant`.
+        Updates VideoStreamPackager's active reflector endpoints so HTTP requests
+        receive 307 Temporary Redirects and manifests advertise the reflector.
+        """
+        self.ledger.save_grant(grant, status='live')
+        endpoints = list(grant.get('reflector_endpoints', []))
+        with self._lock:
+            self._active_handovers[stream_id] = grant
+        if self.packager is not None:
+            try:
+                self.packager.update_transit_endpoints(stream_id, endpoints)
+            except KeyError:
+                pass
+        return endpoints
+
+    def release_handover(
+        self,
+        stream_id: str,
+        window: int,
+        sk_hex: str,
+        reason: str = 'capacity-restored',
+        now: Optional[int] = None,
+    ) -> Optional[ tuple[TransitReceiptBody, TransitCloseBody]]:
+        """Release transit handover once Capacity pressure drops below `restore_below`.
+        Emits a final cumulative TransitReceiptBody and TransitCloseBody, and
+        clears redirect endpoints on the origin packager.
+        """
+        now_val = self._clock() if now is None else int(now)
+        with self._lock:
+            grant = self._active_handovers.pop(stream_id, None)
+            if grant is None:
+                return None
+            grant_id = grant['grant_id']
+            total_bytes = self._metered_bytes.get(grant_id, 0)
+
+        receipt, head = self.ledger.record_outbound_receipt(
+            grant_id=grant_id,
+            stream_id=stream_id,
+            window=window,
+            transit_bytes_out=total_bytes,
+            sk_hex=sk_hex,
+            now=now_val,
+        )
+        close_body: TransitCloseBody = {
+            'grant': grant_id,
+            'stream_id': stream_id,
+            'window': int(window),
+            'final_receipt_head': head,
+            'counters': {COUNTER_TRANSIT_BYTES_OUT: total_bytes},
+            'reason': reason,
+            'at': now_val,
+        }
+        self.ledger.close_grant(grant_id)
+        with self._lock:
+            if not hasattr(self, '_last_release'):
+                self._last_release: dict[str, tuple[TransitReceiptBody, TransitCloseBody]] = {}
+            self._last_release[stream_id] = (receipt, close_body)
+        if self.packager is not None:
+            try:
+                self.packager.update_transit_endpoints(stream_id, ())
+            except KeyError:
+                pass
+        return receipt, close_body
+
+    def evaluate_capacity_handover(
+        self,
+        capacity_comp: Any,
+        stream_id: str,
+        standby_grant: TransitGrantBody,
+        window: int,
+        sk_hex: str,
+        now: Optional[int] = None,
+    ) -> str:
+        """Inspect the node's Capacity compartment shed state (`narrow_scope` /
+        `amend_down`) and automatically engage or release transit handover.
+        Returns 'engaged', 'released', 'active', or 'idle'.
+        """
+        shed_active = bool(
+            (hasattr(capacity_comp, 'narrowing') and capacity_comp.narrowing())
+            or (hasattr(capacity_comp, 'amending') and capacity_comp.amending())
+        )
+        with self._lock:
+            currently_handed_over = stream_id in self._active_handovers
+
+        if shed_active and not currently_handed_over:
+            self.engage_handover(stream_id, standby_grant)
+            return 'engaged'
+        elif not shed_active and currently_handed_over:
+            self.release_handover(stream_id, window=window, sk_hex=sk_hex, now=now)
+            return 'released'
+        return 'active' if currently_handed_over else 'idle'
+
+    # ---- Real-Time MoQ Ring Buffer Reflection & Cluster Fanout --------------
+
+    def get_ring_frames(self, stream_id: str) -> list[MoQFrame]:
+        with self._lock:
+            ring = self._rings.get(stream_id)
+            return list(ring) if ring is not None else []
+
+    def reflect_moq_frame(
+        self,
+        frame: MoQFrame,
+        grant_id: Optional[str] = None,
+        subscribers: Sequence[str] = (),
+    ) -> ReflectionResult:
+        """Verify, buffer, and fan out a real-time MoQ frame across FIPS (port 30820),
+        Iroh UDP streams, and Rhio NATS JetStream cluster subjects.
+        """
+        # 1. Cryptographic integrity check at the ingress door
+        if not frame.verify_integrity():
+            if self.strain_ledger is not None and hasattr(self.strain_ledger, 'record'):
+                self.strain_ledger.record('invalid_moq_frame', peer=frame.broadcaster)
+            return {
+                'accepted': False,
+                'forwarded': False,
+                'degraded_audio_only': False,
+                'dropped_reason': 'invalid_frame_signature_or_digest',
+                'bytes_fanned_out': 0,
+                'ring_depth': len(self.get_ring_frames(frame.stream_id)),
+                'rhio_published': False,
+            }
+
+        # 2. Check soft debt ceiling (`max_unsettled_transit_bytes`)
+        summary = self.ledger.debt_summary(frame.broadcaster, grant_id=grant_id)
+        degraded = summary['degraded_audio_only']
+        if degraded and not frame.is_audio_only:
+            return {
+                'accepted': True,
+                'forwarded': False,
+                'degraded_audio_only': True,
+                'dropped_reason': 'soft_debt_ceiling_audio_only_degradation',
+                'bytes_fanned_out': 0,
+                'ring_depth': len(self.get_ring_frames(frame.stream_id)),
+                'rhio_published': False,
+            }
+
+        # 3. Append to bounded drop-on-congestion ring buffer
+        with self._lock:
+            ring = self._rings.get(frame.stream_id)
+            if ring is None:
+                ring = collections.deque(maxlen=self.ring_capacity)
+                self._rings[frame.stream_id] = ring
+            ring.append(frame)
+            ring_depth = len(ring)
+
+        # 4. Meter outbound bytes across subscribers
+        fanout_multiplier = max(1, len(subscribers))
+        bytes_fanned = len(frame.payload) * fanout_multiplier
+        if grant_id is not None:
+            with self._lock:
+                self._metered_bytes[grant_id] = self._metered_bytes.get(grant_id, 0) + bytes_fanned
+
+        # 5. Fan out to FIPS native FSP port 30820 and Iroh UDP engine if attached
+        cbor_frame = frame.to_cbor()
+        for sub_peer in subscribers:
+            if self.fips_transport is not None and hasattr(self.fips_transport, 'send_datagram'):
+                try:
+                    self.fips_transport.send_datagram(sub_peer, cbor_frame, port=PORT_VIDEO_STREAM)
+                except Exception:
+                    self.frames_dropped['fips'] = self.frames_dropped.get('fips', 0) + 1
+            if self.iroh_engine is not None and hasattr(self.iroh_engine, 'send_stream_chunk'):
+                try:
+                    self.iroh_engine.send_stream_chunk(sub_peer, cbor_frame)
+                except Exception:
+                    self.frames_dropped['iroh'] = self.frames_dropped.get('iroh', 0) + 1
+
+        # 6. Fan out to Rhio NATS JetStream cluster (`fleetmesh.video.<stream_id>`)
+        rhio_published = False
+        if self.rhio_client is not None and hasattr(self.rhio_client, 'publish'):
+            try:
+                res = self.rhio_client.publish(
+                    f"fleetmesh.video.{frame.stream_id}",
+                    {
+                        'stream_id': frame.stream_id,
+                        'group_id': frame.group_id,
+                        'object_id': frame.object_id,
+                        'track': frame.track,
+                        'is_audio_only': frame.is_audio_only,
+                        'sha256': frame.sha256,
+                        'size': len(frame.payload),
+                        'timestamp': frame.timestamp,
+                    },
+                )
+                rhio_published = bool(res.get('ok', True)) if isinstance(res, Mapping) else True
+            except Exception:
+                rhio_published = False
+
+        return {
+            'accepted': True,
+            'forwarded': True,
+            'degraded_audio_only': degraded,
+            'dropped_reason': '',
+            'bytes_fanned_out': bytes_fanned,
+            'ring_depth': ring_depth,
+            'rhio_published': rhio_published,
+        }
+
+    def emit_metered_receipt(
+        self,
+        grant_id: str,
+        stream_id: str,
+        window: int,
+        sk_hex: str,
+        now: Optional[int] = None,
+    ) -> tuple[TransitReceiptBody, str]:
+        """Produce and persist a signed cumulative TransitReceiptBody for all
+        bytes metered under `grant_id` so far.
+        """
+        with self._lock:
+            total = self._metered_bytes.get(grant_id, 0)
+        return self.ledger.record_outbound_receipt(
+            grant_id=grant_id,
+            stream_id=stream_id,
+            window=window,
+            transit_bytes_out=total,
+            sk_hex=sk_hex,
+            now=now,
+        )
+
+
+__all__ = [
+    'MoQFrameDict',
+    'ReflectionResult',
+    'MoQFrame',
+    'TransitReflectorManager',
+    'receipt_link',
+]

node/planes/store/__init__.py

@@ -0,0 +1,24 @@
+"""Store plane: sync, retention, projected, blob_wire, blob_replication."""
+from . import blob_replication, blob_wire, projected, retention, sync
+from .blob_replication.compartment import Replication
+from .blob_wire.core import BlobWire, BlobWireConfig
+from .projected.compartment import ProjectedDB, ProjectedDBConfig
+from .retention.compartment import Retention, RetentionConfig
+from .sync.core import SyncConfig, SyncPeer
+
+__all__ = [
+    'BlobWire',
+    'BlobWireConfig',
+    'ProjectedDB',
+    'ProjectedDBConfig',
+    'Replication',
+    'Retention',
+    'RetentionConfig',
+    'SyncConfig',
+    'SyncPeer',
+    'blob_replication',
+    'blob_wire',
+    'projected',
+    'retention',
+    'sync',
+]

node/planes/store/blob_replication/__init__.py

@@ -0,0 +1,7 @@
+"""Blossom replication: keep N copies of every pinned blob on other nodes.
+
+One-module compartment: everything is in compartment.py.
+"""
+from .compartment import Replication
+
+__all__ = ['Replication']