fleetmesh-node 0.2.0__py3-none-any.whl

node/planes/services/compute/compartment.py

@@ -0,0 +1,681 @@
+"""The Compute subsystem: node-side lifecycle, offer publication and job handling.
+
+Owns its wire half: `compute/wire/` (ComputeWire, ComputeSession, the
+compute/1.0 and /1.1 messages) is constructed here, and wire imports
+executors/jobs from this package - one direction, no cycle.
+"""
+from __future__ import annotations
+
+import os
+from typing import Any, Callable, Mapping, Optional
+
+from fleetmesh_core.crypto import crypto
+from fleetmesh_core.wire.interfaces import NostrEvent
+from node import channels as ch_mod, compartment as compartment_mod, metrics as mx_mod
+from node.compartment import stores as st
+from node.compartment import Command, Need
+from node.config.parameter import Parameter
+from node.config import ComputeConfig
+from .common import ComputeError, ASK_KIND
+from ..compute_wire import ComputeWire
+from ..compute_wire.messages import PIURI as COMPUTE_PIURI, PIURI_V11 as COMPUTE_PIURI_V11
+from .market import (
+    build_ask,
+    build_offer,
+    match_ask_to_offers,
+    match_offer_to_asks,
+    parse_ask,
+    parse_offer,
+    validate_offer,
+)
+from .executors import ExecutorRegistry
+from .jobs import build_receipt, run_job
+from .load import ComputeLoadMonitor
+
+# -------------------------------------------------------------------------
+# Compute Compartment
+# -------------------------------------------------------------------------
+class Compute(compartment_mod.Compartment):
+    optional: bool = True
+    enabled_by: Optional[str] = 'compute'
+    """The compute/1.0 compartment of meshnode."""
+
+    name = 'compute'
+    piuris: tuple[str, ...] = (COMPUTE_PIURI, COMPUTE_PIURI_V11)
+    reads = ('compute', 'executors', 'relays')
+    config_cls = ComputeConfig
+    NEEDS = (
+        Need('ks', 'ks'),
+        Need('blobs', 'stores.blobs'),
+    )
+    COMMANDS = (
+        Command(
+            key='show',
+            name='Compute Status',
+            description='display active compute offers, asks, and receipts',
+            tags=('compute', 'status'),
+            permission='compute/show',
+            run='state',
+            reply='dict',
+        ),
+        Command(
+            key='offer',
+            name='Publish Offer',
+            description='publish a kind 11803 compute offer',
+            tags=('compute', 'offer'),
+            parameters=(
+                Parameter(key='engine', name='engine', description='execution engine', type=str, default='', required=True),
+                Parameter(key='msat_per_mfuel', name='msat_per_mfuel', description='price in msat per megabytes fuel', type=object, default=0, required=True),
+            ),
+            permission='compute/offer',
+            run='_cmd_offer',
+            reply='dict',
+        ),
+        Command(
+            key='ask',
+            name='Publish Ask',
+            description='publish a kind 30810 confidential compute ask',
+            tags=('compute', 'ask'),
+            parameters=(
+                Parameter(key='id', name='id', description='ask id', type=str, default='', required=True),
+                Parameter(key='job', name='job', description='job payload', type=dict, default=dict, required=True),
+                Parameter(key='bid', name='bid', description='bid amount', type=object, default=0, required=True),
+                Parameter(key='executor', name='executor', description='executor type', type=str, default='wasm', required=False),
+                Parameter(key='verify', name='verify', description='verification mode', type=str, default='none', required=False),
+            ),
+            permission='compute/ask',
+            run='_cmd_ask',
+            reply='dict',
+        ),
+        Command(
+            key='run',
+            name='Run Job',
+            description='execute a job payload and return signed kind 30811 receipt',
+            tags=('compute', 'run'),
+            parameters=(
+                Parameter(key='job', name='job', description='job payload', type=dict, default=dict, required=True),
+                Parameter(key='buyer', name='buyer', description='buyer public key', type=str, default='', required=True),
+                Parameter(key='preimage', name='preimage', description='payment preimage', type=str, default=None, required=False),
+                Parameter(key='paid_msat', name='paid_msat', description='paid amount in millisatoshis', type=int, default=None, required=False),
+                Parameter(key='payment_hash', name='payment_hash', description='payment hash', type=str, default=None, required=False),
+            ),
+            permission='compute/run',
+            run='_cmd_run',
+            reply='dict',
+        ),
+        Command(
+            key='submit',
+            name='Submit or Reflect Job',
+            description='execute a job locally when CPU headroom permits or reflect to a less-loaded mesh peer',
+            tags=('compute', 'submit', 'reflection'),
+            parameters=(
+                Parameter(key='job', name='job', description='job payload', type=dict, default=dict, required=True),
+                Parameter(key='buyer', name='buyer', description='buyer public key', type=str, default='', required=True),
+                Parameter(key='bid', name='bid', description='maximum bid in msat per mfuel', type=int, default=0, required=False),
+                Parameter(key='force_reflect', name='force_reflect', description='force reflection to a remote peer if available', type=bool, default=False, required=False),
+            ),
+            permission='compute/submit',
+            run='_cmd_submit',
+            reply='dict',
+        ),
+        Command(
+            key='load',
+            name='Compute Load Snapshot',
+            description='display live host CPU utilization, active worker slots, and reflection status',
+            tags=('compute', 'load', 'metrics'),
+            permission='compute/load',
+            run='_cmd_load',
+            reply='dict',
+        ),
+        Command(
+            key='receipts',
+            name='Job Receipts',
+            description='list compute job receipts held',
+            tags=('compute', 'receipts'),
+            permission='compute/receipts',
+            run='receipts_cmd',
+            reply='list',
+        ),
+        Command(
+            key='book',
+            name='Order Book',
+            description='display active order book offers and asks',
+            tags=('compute', 'book'),
+            permission='compute/book',
+            run='book_cmd',
+            reply='dict',
+        ),
+        Command(
+            key='match',
+            name='Match Orders',
+            description='find matching counterparties in order book',
+            tags=('compute', 'match'),
+            parameters=(
+                Parameter(key='ask', name='ask', description='ask id or event', type=object, default=None, required=False),
+                Parameter(key='offer', name='offer', description='offer id or event', type=object, default=None, required=False),
+            ),
+            permission='compute/match',
+            run='match_cmd',
+            reply='list',
+        ),
+        Command(
+            key='index_offer',
+            name='Index Offer',
+            description='index an external kind 11803 offer',
+            tags=('compute', 'offer'),
+            parameters=(
+                Parameter(key='offer', name='offer', description='kind 11803 offer to index', type=object, default=dict, required=True),
+            ),
+            permission='compute/index_offer',
+            run='index_offer_cmd',
+            reply='bool',
+        ),
+        Command(
+            key='index_ask',
+            name='Index Ask',
+            description='index an external kind 30810 ask',
+            tags=('compute', 'ask'),
+            parameters=(
+                Parameter(key='ask', name='ask', description='kind 30810 ask to index', type=object, default=dict, required=True),
+            ),
+            permission='compute/index_ask',
+            run='index_ask_cmd',
+            reply='bool',
+        ),
+        Command(
+            key='models',
+            name='Compute Models',
+            description='list available AI/LLM models and inference features',
+            tags=('compute', 'models'),
+            permission='compute/models',
+            run='_cmd_models',
+            reply='dict',
+        ),
+        Command(
+            key='stream',
+            name='Stream Inference',
+            description='run a streaming AI/LLM inference query and return token deltas',
+            tags=('compute', 'stream'),
+            parameters=(
+                Parameter(key='prompt', name='prompt', description='prompt text', type=str, default='', required=True),
+                Parameter(key='model', name='model', description='model identifier', type=str, default=None, required=False),
+            ),
+            permission='compute/stream',
+            run='_cmd_stream',
+            reply='dict',
+        ),
+        Command(
+            key='link_provider',
+            name='Link AI Provider',
+            description='link an external or local AI/LLM inference provider to this meshnode',
+            tags=('compute', 'models'),
+            parameters=(
+                Parameter(key='provider_id', name='provider_id', description='provider preset id (openrouter, claude, openai, gemini, copilot, routstr, payperquery, ollama, vllm)', type=str, default='openrouter', required=True),
+                Parameter(key='endpoint', name='endpoint', description='custom base endpoint URL', type=str, default=None, required=False),
+                Parameter(key='default_model', name='default_model', description='default model ID', type=str, default=None, required=False),
+                Parameter(key='api_key', name='api_key', description='operator API key or bearer secret', type=str, default=None, required=False),
+                Parameter(key='cashu_token', name='cashu_token', description='operator Cashu token for Routstr/PPQ', type=str, default=None, required=False),
+                Parameter(key='rate_msat_per_token', name='rate_msat_per_token', description='msat rate per token', type=int, default=None, required=False),
+                Parameter(key='enabled', name='enabled', description='whether provider is active', type=bool, default=True, required=False),
+                Parameter(key='set_default', name='set_default', description='set as primary provider', type=bool, default=True, required=False),
+            ),
+            permission='compute/link_provider',
+            run='_cmd_link_provider',
+            reply='dict',
+        ),
+    )
+    STORES = (
+        st.Store(
+            key='offers',
+            description='offers published by this node, newest last',
+            bound=1024,
+            kind=st.RING,
+        ),
+        st.Store(
+            key='asks',
+            description='active asks placed by this node, by ask_id',
+            bound=1024,
+            evict=st.LRU,
+            kind=st.TABLE,
+            keyed_by='ask',
+        ),
+        st.Store(
+            key='order_book_offers',
+            description='indexed offers from mesh peers, by offer_id',
+            bound=4096,
+            evict=st.LRU,
+            kind=st.TABLE,
+            keyed_by='offer',
+        ),
+        st.Store(
+            key='order_book_asks',
+            description='indexed asks from mesh peers, by ask_id',
+            bound=4096,
+            evict=st.LRU,
+            kind=st.TABLE,
+            keyed_by='ask',
+        ),
+        st.Store(
+            key='receipts_held',
+            description='job completion receipts held by this node, newest last',
+            bound=1024,
+            kind=st.RING,
+        ),
+    )
+    CHANNELS = (
+        ch_mod.Channel('job_executed', 'A compute task execution completed and receipt signed',
+                       level=ch_mod.NOTE, say='compute job {job_id} executed for buyer {buyer}',
+                       tags=('compute', 'execution')),
+        ch_mod.Channel('job_reflected', 'A compute task was reflected to a less-loaded mesh peer',
+                       level=ch_mod.INFO, say='compute job {job_id} reflected to peer {peer} ({reason})',
+                       tags=('compute', 'reflection')),
+        ch_mod.Channel('offer_published', 'A compute engine capability offer was published',
+                       level=ch_mod.NOTE, say='compute offer published for engine {engine}',
+                       tags=('compute', 'market')),
+        ch_mod.Channel('ask_published', 'A confidential compute ask was published to the mesh',
+                       level=ch_mod.NOTE, say='compute ask {ask_id} published bid {bid}',
+                       tags=('compute', 'market')),
+    )
+    METRICS = (
+        mx_mod.Metric(name='compute_jobs_executed', description='Compute task executions completed.',
+                      kind=mx_mod.COUNTER, counts='job_executed', tags=('compute', 'execution')),
+        mx_mod.Metric(name='compute_jobs_reflected', description='Compute jobs reflected to less-loaded mesh peers.',
+                      kind=mx_mod.COUNTER, counts='job_reflected', tags=('compute', 'reflection')),
+        mx_mod.Metric(name='compute_offers_published', description='Compute engine offers published.',
+                      kind=mx_mod.COUNTER, counts='offer_published', tags=('compute', 'market')),
+        mx_mod.Metric(name='compute_asks_published', description='Compute asks published to mesh.',
+                      kind=mx_mod.COUNTER, counts='ask_published', tags=('compute', 'market')),
+    )
+
+    def __init__(self, node: Any = None, registry: Optional[ExecutorRegistry] = None,
+                 ks: Optional[Any] = None, blobs: Optional[Any] = None,
+                 clock: Optional[Callable[[], int]] = None,
+                 load_monitor: Optional[ComputeLoadMonitor] = None,
+                 remote_dispatcher: Optional[Callable[[Mapping[str, Any], Mapping[str, Any]], Mapping[str, Any]]] = None) -> None:
+        super().__init__(node, clock)
+        # What this compartment takes from the node, named: the keystore that
+        # signs offers/asks/receipts and the blob store a job reads from.
+        # Suites that pass a fake node and nothing else still get them off it.
+        self.ks: Optional[Any] = ks if ks is not None else getattr(node, 'ks', None)
+        self.blobs: Optional[Any] = blobs if blobs is not None else getattr(node, 'blobs', None)
+        self.offers: st.Ring[NostrEvent]
+        self.asks: st.Table[str, NostrEvent]
+        self.order_book_offers: st.Table[str, NostrEvent]
+        self.order_book_asks: st.Table[str, NostrEvent]
+        self.receipts_held: st.Ring[Mapping[str, Any]]
+        self.jobs_executed: int = 0
+        self.load_monitor: ComputeLoadMonitor = load_monitor or ComputeLoadMonitor()
+        self.remote_dispatcher: Optional[Callable[[Mapping[str, Any], Mapping[str, Any]], Mapping[str, Any]]] = remote_dispatcher
+        # Operators may enable fixture executors through cfg['compute']
+        # ['mock_mode'] (for demos on a box with no LLM backend); tests pass
+        # a registry directly. Neither happens by default.
+        if registry is None:
+            node_dir = getattr(node, 'dir', None)
+            registry = ExecutorRegistry(blob_store=self.blobs,
+                                        mock_mode=self.settings().mock_mode,
+                                        media_cache_dir=os.path.join(node_dir, 'media-cache') if node_dir else None)
+        self.registry: ExecutorRegistry = registry
+        self.compute_wire = getattr(node, 'compute_wire', None) or ComputeWire(node=node, ks=self.ks)
+
+    def handle(self, inbound: Any) -> Optional[tuple[Any, ...]]:
+        return self.compute_wire.handle(inbound)
+
+    def index_offer(self, offer: NostrEvent) -> bool:
+        probs = validate_offer(offer)
+        if probs:
+            raise ComputeError(f"invalid offer event: {', '.join(probs)}")
+        self.order_book_offers[offer['id']] = offer
+        return True
+
+    def index_ask(self, ask: NostrEvent) -> bool:
+        if ask.get('kind') != ASK_KIND:
+            raise ComputeError("not a kind 30810 ask")
+        ask_id = parse_ask(ask)['ask_id']
+        self.order_book_asks[ask_id or ask['id']] = ask
+        return True
+
+    def find_offers_for_ask(
+        self,
+        ask: NostrEvent | Mapping[str, Any] | str,
+        trust_weights: Optional[Mapping[str, float]] = None,
+    ) -> list[Mapping[str, Any]]:
+        ask_ev = self.order_book_asks.get(ask) if isinstance(ask, str) else ask
+        if not ask_ev:
+            if isinstance(ask, str) and ask in self.asks:
+                ask_ev = self.asks[ask]
+            else:
+                raise ComputeError(f"ask not found: {ask}")
+        return match_ask_to_offers(ask_ev, list(self.order_book_offers.values()), now=self.clock(), trust_weights=trust_weights)
+
+    def find_asks_for_offer(
+        self,
+        offer: NostrEvent | Mapping[str, Any] | str,
+    ) -> list[Mapping[str, Any]]:
+        off_ev = self.order_book_offers.get(offer) if isinstance(offer, str) else offer
+        if not off_ev:
+            if isinstance(offer, str):
+                for o in self.offers:
+                    if o.get('id') == offer:
+                        off_ev = o
+                        break
+            if not off_ev:
+                raise ComputeError(f"offer not found: {offer}")
+        return match_offer_to_asks(off_ev, list(self.order_book_asks.values()), now=self.clock())
+
+    def state(self) -> Mapping[str, Any]:
+        snap = self.load_monitor.snapshot()
+        return {
+            'offers_published': len(self.offers),
+            'active_asks': len(self.asks),
+            'indexed_offers': len(self.order_book_offers),
+            'indexed_asks': len(self.order_book_asks),
+            'receipts_held': len(self.receipts_held),
+            'jobs_executed': self.jobs_executed,
+            'jobs_reflected': snap['jobs_reflected'],
+            'total_fuel_used': snap['total_fuel_used'],
+            'cpu_utilization_ratio': snap['cpu_utilization_ratio'],
+            'cpu_load_pct': snap['cpu_load_pct'],
+            'active_jobs': snap['active_jobs'],
+            'max_concurrent_jobs': snap['max_concurrent_jobs'],
+            'free_slots': snap['free_slots'],
+            'available_engines': self.registry.runnable_engines(),
+            'registered_engines': list(self.registry.executors.keys()),
+            'active_sessions': len(self.compute_wire.sessions),
+        }
+
+    def trouble(self) -> list[str]:
+        out: list[str] = []
+        recent_errs = self.refusals.within(self.TROUBLE_WINDOW, self.clock(), kind='compute')
+        if recent_errs:
+            out.append('%d compute failure(s) in the last 30 minutes' % len(recent_errs))
+        return out
+
+    def receipts_cmd(self) -> list[Mapping[str, Any]]:
+        return list(self.receipts_held)
+
+    def book_cmd(self) -> Mapping[str, Any]:
+        return {
+            'offers_count': len(self.order_book_offers),
+            'asks_count': len(self.order_book_asks),
+            'offers': [parse_offer(o) for o in self.order_book_offers.values()],
+            'asks': [parse_ask(a) for a in self.order_book_asks.values()],
+        }
+
+    def index_offer_cmd(self, args: Mapping[str, Any]) -> bool:
+        return self.index_offer(args['offer'])
+
+    def index_ask_cmd(self, args: Mapping[str, Any]) -> bool:
+        return self.index_ask(args['ask'])
+
+    def match_cmd(self, args: Mapping[str, Any]) -> list[Any]:
+        if args.get('ask') is not None:
+            return self.find_offers_for_ask(args['ask'])
+        if args.get('offer') is not None:
+            return self.find_asks_for_offer(args['offer'])
+        raise ComputeError("match requires 'ask' or 'offer' argument")
+
+    def _require_keystore(self) -> Any:
+        """The node keystore, or a ComputeError when this subsystem is detached."""
+        if self.ks is None:
+            raise ComputeError('no node keystore available')
+        return self.ks
+
+    def _cmd_load(self) -> Mapping[str, Any]:
+        return self.load_monitor.snapshot()
+
+    def _cmd_offer(self, args: Mapping[str, Any]) -> Mapping[str, Any]:
+        ks = self._require_keystore()
+        engine = args['engine']
+        msat = str(args['msat_per_mfuel'])
+        runtimes = None
+        executor_obj = self.registry.get(engine)
+        if executor_obj and executor_obj.wasi_profile_digest:
+            runtimes = [[engine, '1.0', executor_obj.wasi_profile_digest]]
+        limits = self.load_monitor.offer_limits()
+        now_ts = self.clock()
+        ev = build_offer(
+            ks,
+            executors=[[engine, 'msat_per_mfuel', msat]],
+            limits=limits,
+            runtimes=runtimes,
+            now=now_ts,
+            expiration=now_ts + 86400,
+        )
+        self.offers.append(ev)
+        self.order_book_offers[ev['id']] = ev
+        self.say('offer_published', engine=engine)
+        return {'id': ev['id'], 'kind': ev['kind'], 'tags': ev['tags']}
+
+    def _cmd_ask(self, args: Mapping[str, Any]) -> Mapping[str, Any]:
+        ks = self._require_keystore()
+        ask_id = args['id']
+        job = args['job']
+        bid = int(args['bid'])
+        now_ts = self.clock()
+        ev = build_ask(
+            buyer_sk=ks.node_sk,
+            ask_id=ask_id,
+            job_payload=job,
+            executor=args.get('executor', str(job.get('executor', 'wasm'))),
+            bid_currency='msat_per_mfuel',
+            bid_price=bid,
+            deadline=now_ts + 3600,
+            fuel_max=int(job.get('fuel_max', 2500000000)),
+            verify_mode=args.get('verify', 'none'),
+            now=now_ts,
+            expiration=now_ts + 7200,
+        )
+        self.asks[ask_id] = ev
+        self.order_book_asks[ask_id] = ev
+        self.say('ask_published', ask_id=str(ask_id)[:16], bid=bid)
+        return {'id': ev['id'], 'kind': ev['kind'], 'commit': ev['tags'][1][1]}
+
+    def _cmd_run(self, args: Mapping[str, Any]) -> Mapping[str, Any]:
+        ks = self._require_keystore()
+        job = dict(args['job'])
+        if 'blob_store' not in job and self.blobs is not None:
+            job['blob_store'] = self.blobs
+        buyer_pk = args['buyer']
+        with self.load_monitor.job_slot():
+            try:
+                res = run_job(job, registry=self.registry)
+            except Exception as e:
+                self.refusals.refuse('compute', 'execution failed', detail={'error': str(e)})
+                raise ComputeError('job execution failed: %s' % (e,))
+
+        self.load_monitor.record_fuel(int(res.get('fuel_used') or 0))
+        paid_msat = int(job.get('paid_msat') or args.get('paid_msat') or 0)
+        payment_hash = job.get('payment_hash') or args.get('payment_hash') or ('0' * 64)
+
+        blinding = crypto.sha256(os.urandom(32))
+        receipt_payload = {
+            'ask': job.get('ask', '0' * 64),
+            'module': (job.get('module') or {}).get('sha256', '0' * 64),
+            'input': (job.get('input') or {}).get('sha256', '0' * 64),
+            'result': res['result_hash'],
+            'result_cid': res.get('result_cid'),
+            'fuel_used': res['fuel_used'],
+            'paid_msat': paid_msat,
+            'payment_hash': payment_hash,
+            'executor': {
+                'engine': res.get('engine', 'wasm'),
+                'wasi_profile_digest': res.get('wasi_profile_digest', '0' * 64),
+            },
+            'verify': job.get('verify', {'mode': 'none'}),
+            'blinding': blinding,
+        }
+        if args.get('preimage'):
+            receipt_payload['preimage'] = args['preimage']
+
+        ev = build_receipt(ks.node_sk, buyer_pk, receipt_payload, blinding)
+        self.receipts_held.append({'receipt': ev, 'payload': receipt_payload, 'buyer': str(buyer_pk)})
+        self.jobs_executed += 1
+        self.say(
+            'job_executed',
+            job_id=str(job.get('ask', 'job'))[:16],
+            buyer=str(buyer_pk),
+            fuel=int(res['fuel_used']),
+        )
+        return {
+            'result_hash': res['result_hash'],
+            'result_cid': res.get('result_cid'),
+            'raw_output': res.get('raw_output'),
+            'counters': res.get('counters', {}),
+            'receipt_id': ev['id'],
+            'receipt': ev,
+            'fuel_used': res['fuel_used'],
+            'paid_msat': paid_msat,
+            'payment_hash': payment_hash,
+            'worker_pubkey': ks.node_pk,
+            'reflected': False,
+        }
+
+    def submit_or_reflect(
+        self,
+        job: Mapping[str, Any],
+        buyer_pk: str,
+        *,
+        bid_msat: int = 0,
+        force_reflect: bool = False,
+        remote_dispatcher: Optional[Callable[[Mapping[str, Any], Mapping[str, Any]], Mapping[str, Any]]] = None,
+    ) -> Mapping[str, Any]:
+        """Execute a job locally if CPU/slot headroom permits, or reflect it to the least-loaded mesh peer."""
+        ks = self._require_keystore()
+        should_reflect, reason = self.load_monitor.should_reflect_job()
+        if force_reflect:
+            should_reflect, reason = True, 'forced_reflection'
+
+        if should_reflect:
+            ask_id = str(job.get('ask') or crypto.sha256(os.urandom(32)))
+            exec_engine = str(job.get('executor') or job.get('engine') or 'wasm')
+            v_obj = job.get('verify')
+            v_mode = str(v_obj.get('mode', 'delivery')) if isinstance(v_obj, Mapping) else 'delivery'
+            if v_mode not in ('none', 'delivery', 'sample', 'dual', 'proof', 'schema-bounds', 'tee-attestation', 'quasi-deterministic'):
+                v_mode = 'delivery'
+            job_with_ask = dict(job, ask=ask_id)
+            self._cmd_ask({
+                'id': ask_id,
+                'job': job_with_ask,
+                'bid': bid_msat,
+                'executor': exec_engine,
+                'verify': v_mode,
+            })
+            candidates = [
+                m for m in self.find_offers_for_ask(ask_id)
+                if m.get('seller_pubkey') != ks.node_pk
+            ]
+            if candidates:
+                best = candidates[0]
+                peer_pk = str(best['seller_pubkey'])
+                self.load_monitor.record_reflection()
+                self.say('job_reflected', job_id=ask_id[:16], peer=peer_pk[:16], reason=reason)
+                if hasattr(self.compute_wire, 'open_buyer_session'):
+                    self.compute_wire.open_buyer_session(ask_id, peer_pk, job_with_ask)
+                dispatcher = remote_dispatcher or self.remote_dispatcher
+                if callable(dispatcher):
+                    remote_res = dict(dispatcher(best, job_with_ask))
+                    remote_res.setdefault('reflected', True)
+                    remote_res.setdefault('reflect_reason', reason)
+                    remote_res.setdefault('worker_pubkey', peer_pk)
+                    remote_res.setdefault('offer_id', best['offer_id'])
+                    remote_res.setdefault('ask_id', ask_id)
+                    return remote_res
+                return {
+                    'reflected': True,
+                    'reflect_reason': reason,
+                    'ask_id': ask_id,
+                    'offer_id': best['offer_id'],
+                    'worker_pubkey': peer_pk,
+                    'candidate': best,
+                    'candidates': candidates,
+                }
+            self.asks.pop(ask_id, None)
+            self.order_book_asks.pop(ask_id, None)
+
+        res = dict(self._cmd_run({'job': job, 'buyer': buyer_pk}))
+        res['reflected'] = False
+        res['reflect_reason'] = reason
+        return res
+
+    def _cmd_submit(self, args: Mapping[str, Any]) -> Mapping[str, Any]:
+        return self.submit_or_reflect(
+            args['job'],
+            args['buyer'],
+            bid_msat=int(args.get('bid') or 0),
+            force_reflect=bool(args.get('force_reflect', False)),
+        )
+
+    def _cmd_models(self) -> Mapping[str, Any]:
+        llm_exec = self.registry.get('llm')
+        router = getattr(self.registry, 'provider_router', None)
+        active_prov = router.get_provider() if router is not None else None
+        default_m = (
+            active_prov.default_model
+            if active_prov is not None
+            else getattr(llm_exec, 'default_model', 'meta-llama/Llama-3.3-70B-Instruct')
+        )
+        endpoint = (
+            active_prov.endpoint
+            if active_prov is not None
+            else getattr(llm_exec, 'endpoint', 'http://127.0.0.1:8000/v1')
+        )
+        avail: list[str] = [
+            default_m,
+            'deepseek-ai/DeepSeek-R1',
+            'meta-llama/Llama-3.1-8B-Instruct',
+        ]
+        if router is not None:
+            for p in router.list_public_providers():
+                for m in p.get('models', []):
+                    if m not in avail:
+                        avail.append(m)
+        return {
+            'default_model': default_m,
+            'endpoint': endpoint,
+            'default_provider': router.default_provider if router is not None else 'vllm',
+            'providers': router.list_public_providers() if router is not None else [],
+            'available_models': avail,
+            'features': ['streaming', 'tool-calling', 'grammar-json', 'schema-bounds', 'multi-provider'],
+        }
+
+    def _cmd_link_provider(self, args: Mapping[str, Any]) -> Mapping[str, Any]:
+        from .executors.providers import LinkedProviderConfig
+        cfg = LinkedProviderConfig.from_preset(
+            str(args.get('provider_id') or 'openrouter'),
+            endpoint=args.get('endpoint'),
+            default_model=args.get('default_model'),
+            api_key=args.get('api_key'),
+            cashu_token=args.get('cashu_token'),
+            rate_msat_per_token=args.get('rate_msat_per_token'),
+            enabled=bool(args.get('enabled', True)),
+        )
+        self.registry.configure_llm_provider(cfg, set_default=bool(args.get('set_default', True)))
+        return cfg.to_public_dict()
+
+    def _cmd_stream(self, args: Mapping[str, Any]) -> Mapping[str, Any]:
+        prompt = args['prompt']
+        model = args.get('model')
+        job = {
+            'executor': 'llm',
+            'prompt': prompt,
+            'stream': True,
+        }
+        if model:
+            job['model'] = model
+        with self.load_monitor.job_slot():
+            try:
+                res = run_job(job, registry=self.registry)
+            except Exception as e:
+                self.refusals.refuse('compute', 'stream execution failed', detail={'error': str(e)})
+                raise ComputeError('stream execution failed: %s' % (e,))
+        self.load_monitor.record_fuel(int(res.get('fuel_used') or 0))
+        raw = res['raw_output']
+        out_text = raw.decode('utf-8', errors='replace') if isinstance(raw, bytes) else str(raw)
+        return {
+            'model': model or getattr(self.registry.get('llm'), 'default_model', 'meta-llama/Llama-3.3-70B-Instruct'),
+            'output': out_text,
+            'result_hash': res['result_hash'],
+            'fuel_used': res['fuel_used'],
+            'counters': res.get('counters', {}),
+            'verification_mode': res.get('verification_mode', 'delivery'),
+        }