fleetmesh-node 0.2.0__py3-none-any.whl

node/planes/services/compute/executors/nondet.py

@@ -0,0 +1,283 @@
+"""Non-deterministic executors: LLM inference, Bacalhau jobs and agent runs.
+
+These call out over HTTP (or run in mock mode) and are settled on evidence
+rather than reproduction; see docs/non-deterministic-compute.md.
+"""
+from __future__ import annotations
+
+import json
+import time
+from typing import Any, Mapping, Optional
+
+from fleetmesh_core.crypto import crypto
+from node.transport import netfetch
+from ..common import ComputeError, LLM_MAX_RESPONSE_BYTES
+from ..sandbox import ComputeLimits
+from .base import ComputeExecutionResult, ComputeExecutor, ComputeQuote
+from .providers import (
+    PROVIDER_PRESETS,
+    LinkedProviderConfig,
+    OperatorProviderRouter,
+    ProviderPreset,
+)
+
+class LlmExecutor(ComputeExecutor):
+    """Non-deterministic AI/LLM inference driver with grammar-guided decoding.
+
+    Supports OpenAI-compatible and Anthropic HTTP endpoints (OpenRouter, Claude,
+    OpenAI, Gemini, Copilot, Routstr, PayPerQuery, vLLM, Ollama, llama-server)
+    with CFG grammar constraints and fallback offline simulation.
+    """
+
+    engine_id: str = 'llm'
+    wasi_profile_digest: str = ''
+
+    def __init__(
+        self,
+        endpoint: str = 'http://127.0.0.1:8000/v1',
+        default_model: str = 'meta-llama/Llama-3.3-70B-Instruct',
+        timeout_seconds: float = 30.0,
+        mock_mode: bool = False,
+        provider_router: Optional[OperatorProviderRouter] = None,
+        api_key: str = '',
+        cashu_token: str = '',
+        provider_id: str = 'vllm',
+        rate_msat_per_token: int = 12,
+    ) -> None:
+        self.endpoint = endpoint.rstrip('/')
+        self.default_model = default_model
+        self.timeout_seconds = timeout_seconds
+        self.mock_mode = mock_mode
+        self.api_key = api_key
+        self.cashu_token = cashu_token
+        self.provider_id = provider_id
+        self.rate_msat_per_token = int(rate_msat_per_token)
+        self.provider_router: OperatorProviderRouter = provider_router or OperatorProviderRouter()
+
+    def _resolve_provider(self, job_payload: Optional[Mapping[str, Any]] = None) -> LinkedProviderConfig:
+        if job_payload is not None:
+            resolved = self.provider_router.resolve_for_job(job_payload)
+            if resolved is not None:
+                return resolved
+        else:
+            resolved = self.provider_router.get_provider()
+            if resolved is not None:
+                return resolved
+        return LinkedProviderConfig(
+            provider_id=self.provider_id,
+            label=self.provider_id.upper(),
+            endpoint=self.endpoint,
+            default_model=self.default_model,
+            models=(self.default_model,),
+            api_key=self.api_key,
+            cashu_token=self.cashu_token,
+            auth_scheme='bearer' if self.api_key else ('cashu' if self.cashu_token else 'none'),
+            wire_format='anthropic_messages' if self.provider_id in ('claude', 'anthropic') else 'openai',
+            rate_msat_per_token=self.rate_msat_per_token,
+            enabled=True,
+        )
+
+    def is_live_runnable(self) -> bool:
+        """Check whether LLM inference can execute on this node."""
+        if self.mock_mode:
+            return True
+        if self.provider_router.runnable_providers():
+            return True
+        return bool(self.endpoint and self.endpoint.startswith(('http://', 'https://')))
+
+    def quote(
+        self,
+        module_spec: Mapping[str, Any],
+        input_spec: Mapping[str, Any],
+        limits: ComputeLimits,
+    ) -> ComputeQuote:
+        merged_spec = dict(module_spec)
+        merged_spec.update(input_spec)
+        provider = self._resolve_provider(merged_spec)
+        rate = provider.rate_msat_per_token if provider else self.rate_msat_per_token
+        max_tokens = int(input_spec.get('max_tokens', limits.max_tokens or 2048))
+        prompt_est = len(str(input_spec.get('prompt', ''))) // 4
+        fuel_est = (prompt_est + max_tokens) * rate
+        return ComputeQuote(
+            fuel_estimate=min(fuel_est, limits.max_fuel),
+            rate_unit='msat_per_token',
+            rate_amount=rate,
+            engine_id=self.engine_id,
+        )
+
+    def execute(
+        self,
+        job_payload: Mapping[str, Any],
+        fuel_cap: int,
+    ) -> ComputeExecutionResult:
+        provider = self._resolve_provider(job_payload)
+        model = job_payload.get('model') or provider.default_model or self.default_model
+        prompt = job_payload.get('prompt')
+        messages = job_payload.get('messages')
+        sampling = job_payload.get('sampling') or {}
+        grammar = job_payload.get('grammar') or {}
+        stream = bool(job_payload.get('stream', False))
+        # The backend endpoint is operator configuration only. An earlier
+        # version let ``job_payload['endpoint']`` override it, which handed a
+        # requester a POST-anywhere primitive against the node's own network
+        # (the default endpoint is loopback, so allow_private must be on).
+        verif_mode = self.verification_mode(job_payload)
+
+        if not messages and prompt is not None:
+            messages = [{'role': 'user', 'content': str(prompt)}]
+        elif not messages:
+            messages = [{'role': 'user', 'content': ''}]
+
+        max_tokens = int(sampling.get('max_tokens', 512))
+        temperature = float(sampling.get('temperature', 0.7))
+        top_p = float(sampling.get('top_p', 1.0))
+        seed = sampling.get('seed')
+
+        url, req_body = provider.format_request(
+            messages=messages,
+            model=model,
+            max_tokens=max_tokens,
+            temperature=temperature,
+            top_p=top_p,
+            seed=int(seed) if seed is not None else None,
+            grammar=grammar,
+            stream=stream,
+        )
+
+        raw_output_text: str = ''
+        prompt_tokens = max(1, sum(len(str(m.get('content', ''))) // 4 for m in messages))
+        completion_tokens = 0
+        gpu_seconds = 0.05
+
+        if self.mock_mode:
+            # Deterministic stand-in for tests and demos. It is only reachable
+            # when the operator constructed the executor with mock_mode=True;
+            # a live node never falls back to it.
+            if '_mock_completion' in job_payload and job_payload['_mock_completion'] is not None:
+                raw_output_text = str(job_payload['_mock_completion'])
+            elif grammar and (grammar.get('type') == 'json_schema' or 'json_schema' in grammar or grammar.get('response_format') == 'json_object'):
+                raw_output_text = json.dumps({
+                    'status': 'completed',
+                    'model': model,
+                    'verified': True,
+                    'result': 'synthesized-conforming-payload',
+                }, sort_keys=True)
+            else:
+                user_msg = messages[-1].get('content', '')
+                raw_output_text = f"Inference completion for [{model}]: {user_msg[:64]}"
+            completion_tokens = max(1, len(raw_output_text) // 4)
+            gpu_seconds = 0.025
+        else:
+            # A backend failure is a failed job, full stop.
+            data_bytes = json.dumps(req_body).encode('utf-8')
+            headers = provider.build_headers(
+                per_query_cashu_token=job_payload.get('_operator_cashu_token'),
+            )
+            start_t = time.time()
+            try:
+                body, _ = netfetch.fetch(
+                    url,
+                    max_bytes=LLM_MAX_RESPONSE_BYTES,
+                    timeout=self.timeout_seconds,
+                    method='POST',
+                    data=data_bytes,
+                    headers=headers,
+                    # The endpoint is operator-configured (loopback vLLM/Ollama or
+                    # operator-linked cloud/eCash provider), never requester-chosen.
+                    allow_private=True,
+                )
+                resp_data = json.loads(body.decode('utf-8'))
+            except netfetch.FetchError as e:
+                raise ComputeError(f"LLM backend request failed: {e}")
+            except (ValueError, UnicodeDecodeError) as e:
+                raise ComputeError(f"LLM backend returned malformed JSON: {e}")
+            if not isinstance(resp_data, dict):
+                raise ComputeError("LLM backend returned a non-object response")
+            try:
+                raw_output_text, prompt_tokens, completion_tokens = provider.parse_response(
+                    resp_data,
+                    default_prompt_tokens=prompt_tokens,
+                )
+            except ValueError as e:
+                raise ComputeError(str(e))
+            gpu_seconds = round(time.time() - start_t, 3)
+
+        total_tokens = prompt_tokens + completion_tokens
+        rate_msat = provider.rate_msat_per_token or 12
+        fuel_used = total_tokens * rate_msat
+        if fuel_used > fuel_cap:
+            raise ComputeError(f"LLM generation exceeded fuel cap ({fuel_used} > {fuel_cap})")
+
+        on_token = job_payload.get('_on_token')
+        if callable(on_token) and raw_output_text:
+            # Emit token chunks to the streaming callback
+            chunk_size = 18
+            for idx in range(0, len(raw_output_text), chunk_size):
+                try:
+                    on_token(raw_output_text[idx:idx + chunk_size])
+                except Exception:
+                    break
+
+        raw_bytes = raw_output_text.encode('utf-8')
+        result_hash = crypto.sha256(raw_bytes)
+        counters = {
+            'prompt_tokens': prompt_tokens,
+            'completion_tokens': completion_tokens,
+            'total_tokens': total_tokens,
+            'gpu_seconds': gpu_seconds,
+            'rate_msat_per_token': rate_msat,
+        }
+
+        return ComputeExecutionResult(
+            result_hash=result_hash,
+            raw_output=raw_bytes,
+            fuel_used=fuel_used,
+            engine_id=self.engine_id,
+            wasi_profile_digest=f"llm:{model}",
+            counters=counters,
+            verification_mode=verif_mode,
+        )
+
+
+from .bacalhau import BacalhauExecutor
+
+
+from .agent import (
+    AgentExecutor,
+    AgentExecutionContext,
+    AgentToolProtocol,
+    AgentToolResult,
+    AgentDecision,
+    AgentBackendProtocol,
+    CalculatorTool,
+    Sha256Tool,
+    RegexSearchTool,
+    FetchBlobTool,
+    SubAskTool,
+    FinishTool,
+    ScriptedAgentBackend,
+    LlmAgentBackend,
+)
+
+__all__ = [
+    'LlmExecutor',
+    'BacalhauExecutor',
+    'AgentExecutor',
+    'AgentExecutionContext',
+    'AgentToolProtocol',
+    'AgentToolResult',
+    'AgentDecision',
+    'AgentBackendProtocol',
+    'CalculatorTool',
+    'Sha256Tool',
+    'RegexSearchTool',
+    'FetchBlobTool',
+    'SubAskTool',
+    'FinishTool',
+    'ScriptedAgentBackend',
+    'LlmAgentBackend',
+    'PROVIDER_PRESETS',
+    'ProviderPreset',
+    'LinkedProviderConfig',
+    'OperatorProviderRouter',
+]