clm-kernel 0.0.0__py3-none-any.whl

clm/layer3/agency/runtime.py

@@ -0,0 +1,524 @@
+"""
+LLM Runtime Module
+
+Provides LLMRuntime (RuntimeExecutor subclass) for executing LLM prompts
+as part of the PTR polyglot runtime system.
+"""
+
+import json
+from typing import Any
+
+from clm.cross_cutting.config.config_constants import LLM_DEFAULT_TIMEOUT_SECS
+from clm.layer0.model.card import MCard
+from clm.layer0.narrative import IO, Either, Left
+from clm.layer1.runtimes.base import RuntimeExecutor
+
+from .config import DEFAULT_PROVIDER, LLMConfig
+from .providers import LLMProvider, MLCLLMProvider, OllamaProvider
+
+# ─────────────────────────────────────────────────────────────────────────────
+# Provider Factory
+# ─────────────────────────────────────────────────────────────────────────────
+
+
+def get_provider(
+    provider_name: str = DEFAULT_PROVIDER,
+    base_url: str = None,
+    timeout: int = LLM_DEFAULT_TIMEOUT_SECS,
+) -> LLMProvider:
+    """
+    Get an LLM provider instance by name.
+
+    Args:
+        provider_name: Provider identifier ('ollama', 'lmstudio', 'openai', etc.)
+        base_url: Optional override for the provider's base URL
+        timeout: Request timeout in seconds
+
+    Returns:
+        LLMProvider instance
+    """
+    providers = {
+        "ollama": lambda: OllamaProvider(base_url, timeout),
+        "mlc-llm": lambda: MLCLLMProvider(base_url, timeout),
+        # Future: 'lmstudio': lambda: LMStudioProvider(base_url, timeout),
+        # Future: 'openai': lambda: OpenAIProvider(base_url, timeout, api_key),
+    }
+
+    factory = providers.get(provider_name)
+    if not factory:
+        raise ValueError(
+            f"Unknown provider: {provider_name}. Available: {list(providers.keys())}"
+        )
+
+    return factory()
+
+
+# ─────────────────────────────────────────────────────────────────────────────
+# LLM Runtime Executor
+# ─────────────────────────────────────────────────────────────────────────────
+
+
+class LLMRuntime(RuntimeExecutor):
+    """
+    LLM runtime executor for the PTR polyglot runtime system.
+
+    Executes LLM prompts via configurable providers (Ollama, LMStudio, OpenAI, etc.)
+    and integrates with the standard RuntimeFactory pattern.
+
+    Usage in PCard concrete section:
+        concrete:
+          runtime: llm
+          provider: ollama
+          model: qwen3:0.6b:27b
+          llm_config:
+            system_prompt: "You are a helpful assistant."
+            temperature: 0.7
+            max_tokens: 1024
+    """
+
+    runtime_name = "llm"
+
+    def __init__(self, provider_name: str = DEFAULT_PROVIDER):
+        super().__init__()
+        self.provider_name = provider_name
+        self._provider: LLMProvider | None = None
+
+    @property
+    def provider(self) -> LLMProvider:
+        """Lazy-load provider on first access."""
+        if self._provider is None:
+            self._provider = get_provider(self.provider_name)
+        return self._provider
+
+    def execute(
+        self, concrete_impl: dict[str, Any], target: MCard, context: dict[str, Any]
+    ) -> Any:
+        """
+        Execute LLM prompt based on concrete implementation.
+
+        Args:
+            concrete_impl: Concrete section from PCard containing llm_config
+            target: Target MCard containing the prompt/input
+            context: Execution context with optional overrides
+
+        Returns:
+            LLM response (string or structured data based on response_format)
+        """
+        # Build configuration from concrete and context
+        config = LLMConfig.from_concrete(concrete_impl, context)
+
+        # Update provider if different from init
+        if config.provider != self.provider_name:
+            self._provider = get_provider(
+                config.provider, config.endpoint_url, config.timeout
+            )
+
+        # Get prompt from target
+        prompt = self._extract_prompt(target, context)
+
+        from .router import IntelligenceRouter
+
+        router = IntelligenceRouter()
+
+        def attempt_execution(cfg: LLMConfig) -> Either[str, Any]:
+            if cfg.provider != self.provider_name:
+                self._provider = get_provider(
+                    cfg.provider, cfg.endpoint_url, cfg.timeout
+                )
+            if cfg.system_prompt:
+                return self._execute_chat(prompt, cfg)
+            else:
+                return self._execute_completion(prompt, cfg)
+
+        # Attempt primary execution
+        result = attempt_execution(config)
+
+        # Intelligence Router Intercept / Fallback
+        if result.is_left():
+            # If it's a generic connection/timeout/failed error, trigger fallback
+            err_msg = str(result.value).lower()
+            if "timeout" in err_msg or "connection" in err_msg or "failed" in err_msg:
+                # Ask router for failover setup
+                fb_provider, fb_model, fb_url, warning = router.handle_failure(
+                    config.provider, Exception(result.value)
+                )
+
+                # Push warning to context for the _post phase to map into VCard
+                if "execution_warnings" not in context:
+                    context["execution_warnings"] = []
+                context["execution_warnings"].append(warning)
+
+                # Reconfigure and retry silently
+                config.provider = fb_provider
+                config.model = fb_model
+                config.endpoint_url = fb_url
+                self.provider_name = fb_provider
+                self._provider = None  # Invalidate cached provider
+                result = attempt_execution(config)
+
+        # Handle result
+        if result.is_left():
+            return f"Error: {result.value}"
+
+        return self._format_response(result.value, config, context)
+
+    def _extract_prompt(self, target: MCard, context: dict) -> str:
+        """Extract prompt from target MCard, enriched with context variables.
+
+        When test cases provide context variables (e.g. a=5, b=10), they are
+        serialised as JSON and appended to the prompt so the LLM can see
+        the actual input values.
+        """
+        content = target.get_content()
+        if isinstance(content, bytes):
+            content = content.decode("utf-8")
+        else:
+            content = str(content)
+
+        # Enrich prompt with context variables (filter out internal/structural keys)
+        ctx_vars = {}
+        excluded_keys = {
+            # LLM config keys
+            "system_prompt",
+            "temperature",
+            "max_tokens",
+            "model",
+            "provider",
+            "response_format",
+            "endpoint_url",
+            "timeout",
+            "assistant_instruction",
+            "operation",
+            # Structural CLM / PCard metadata
+            "runtime",
+            "llm_config",
+            "input_type",
+            "output_type",
+            "process_type",
+            "boundary",
+            "builtin",
+            "config",
+            # Execution infrastructure
+            "balanced",
+            "abstract",
+            "concrete",
+            "db_dir",
+            "pcard_dir",
+            "identities_db",
+            "content_db",
+            "log_db",
+            "execute_cleanup",
+        }
+        for k, v in context.items():
+            if k.startswith("_") or k in excluded_keys:
+                continue
+            try:
+                # Test serializability before inclusion
+                json.dumps(v)
+                ctx_vars[k] = v
+            except (TypeError, ValueError):
+                continue
+
+        if ctx_vars:
+            content = f"{content}\n\nInput: {json.dumps(ctx_vars)}"
+
+        return content
+
+    def _execute_completion(self, prompt: str, config: LLMConfig) -> Either[str, str]:
+        """Execute text completion."""
+        params = config.to_provider_params()
+        return self.provider.complete(prompt, params)
+
+    def _execute_chat(self, prompt: str, config: LLMConfig) -> Either[str, dict]:
+        """Execute chat completion with system prompt."""
+        messages = []
+
+        # Add system prompt
+        if config.system_prompt:
+            messages.append({"role": "system", "content": config.system_prompt})
+
+        # Add user message
+        messages.append({"role": "user", "content": prompt})
+
+        # Add assistant instruction (prefill) if provided
+        if config.assistant_instruction:
+            messages.append(
+                {"role": "assistant", "content": config.assistant_instruction}
+            )
+
+        params = config.to_provider_params()
+        return self.provider.chat(messages, params)
+
+    def _format_response(
+        self, response: Any, config: LLMConfig, context: dict = None
+    ) -> Any:
+        """Format response based on response_format setting and extract thinking."""
+        # Extract content from chat response
+        if isinstance(response, dict):
+            content = response.get("content", response)
+        else:
+            content = response
+
+        # Surface native thinking field from thinking models (e.g. qwen3:0.6b)
+        # Ollama returns thinking as a separate field, distinct from <think> blocks in content
+        if isinstance(response, dict) and "thinking" in response:
+            thinking_native = response["thinking"]
+            if isinstance(thinking_native, str) and thinking_native:
+                if context is not None:
+                    context["thinking_process"] = thinking_native
+
+        # Isolate thinking process for thinking models (e.g. qwen3:0.6b/deepseek)
+        thinking_process = None
+        if isinstance(content, str):
+            think_start = content.find("<think>")
+            think_end = content.find("</think>")
+            if think_start >= 0 and think_end > think_start:
+                thinking_process = content[think_start + 7 : think_end].strip()
+                content = content[:think_start] + content[think_end + 8 :].strip()
+
+                # Expose it in the context dictionary for VCard evaluation/logging
+                if context is not None:
+                    context["thinking_process"] = thinking_process
+
+                # If we were given a response dict, update it incase it is returned unmodified
+                if isinstance(response, dict):
+                    response["thinking_process"] = thinking_process
+                    response["content"] = content
+
+        # Format based on config
+        if config.response_format == "json":
+            try:
+                # Try to parse as JSON
+                if isinstance(content, str):
+                    # Find JSON in response
+                    start = content.find("{")
+                    end = content.rfind("}") + 1
+                    if start >= 0 and end > start:
+                        return json.loads(content[start:end])
+                return content
+            except json.JSONDecodeError:
+                return content
+
+        return content
+
+    def validate_environment(self) -> bool:
+        """Check if the LLM provider is available."""
+        try:
+            return self.provider.validate_connection()
+        except (OSError, ConnectionError, ImportError):
+            return False
+
+    def get_runtime_status(self) -> dict[str, Any]:
+        """Get LLM runtime status with provider info."""
+        try:
+            available = self.validate_environment()
+            models = []
+
+            if available:
+                models_result = self.provider.list_models()
+                if models_result.is_right():
+                    models = models_result.value
+
+            return {
+                "available": available,
+                "version": (
+                    f"{self.provider_name} ({len(models)} models)"
+                    if available
+                    else None
+                ),
+                "provider": self.provider_name,
+                "models": models,
+                "command": (
+                    "ollama" if self.provider_name == "ollama" else self.provider_name
+                ),
+                "details": f"LLM Runtime via {self.provider_name}",
+            }
+        except (OSError, ConnectionError, ImportError, ValueError) as e:
+            return {
+                "available": False,
+                "version": None,
+                "provider": self.provider_name,
+                "models": [],
+                "command": self.provider_name,
+                "details": f"Error: {e}",
+            }
+
+
+# ─────────────────────────────────────────────────────────────────────────────
+# Monadic Interface Functions
+# ─────────────────────────────────────────────────────────────────────────────
+
+
+def prompt_monad(
+    prompt: str, config: LLMConfig = None, **kwargs
+) -> IO[Either[str, str]]:
+    """
+    Create a monadic LLM completion execution.
+
+    Returns IO[Either[Error, Response]] for functional composition.
+
+    Usage:
+        result = prompt_monad("Explain monads", temperature=0.5).unsafe_run()
+        if result.is_right():
+            print(result.value)
+
+    Args:
+        prompt: The input prompt
+        config: Optional LLMConfig (uses defaults if not provided)
+        **kwargs: Override config parameters
+
+    Returns:
+        IO[Either[str, str]]: Monadic wrapper around the LLM call
+    """
+    if config is None:
+        config = LLMConfig(**kwargs) if kwargs else LLMConfig()
+    elif kwargs:
+        # Merge kwargs into config
+        config_dict = config.to_dict()
+        config_dict.update(kwargs)
+        config = LLMConfig.from_dict(config_dict)
+
+    def run() -> Either[str, str]:
+        try:
+            runtime = LLMRuntime(config.provider)
+
+            # Simple wrapper context to reuse the `execute` retry loop if needed natively,
+            # or we manually handle IntelligenceRouter here as well.
+            from .router import IntelligenceRouter
+
+            router = IntelligenceRouter()
+
+            def attempt_execution(cfg):
+                return runtime._execute_completion(prompt, cfg)
+
+            result = attempt_execution(config)
+
+            if result.is_left():
+                err_msg = str(result.value).lower()
+                if (
+                    "timeout" in err_msg
+                    or "connection" in err_msg
+                    or "failed" in err_msg
+                ):
+                    fb_provider, fb_model, fb_url, warning = router.handle_failure(
+                        config.provider, Exception(result.value)
+                    )
+                    config.provider = fb_provider
+                    config.model = fb_model
+                    config.endpoint_url = fb_url
+                    runtime = LLMRuntime(fb_provider)
+                    result = attempt_execution(config)
+
+            return result
+        except (ValueError, TypeError, RuntimeError, ConnectionError, OSError) as e:
+            return Left(f"LLM execution failed: {e}")
+
+    return IO(run)
+
+
+def chat_monad(
+    messages: list[dict[str, str]] = None,
+    prompt: str = None,
+    system_prompt: str = "",
+    config: LLMConfig = None,
+    **kwargs,
+) -> IO[Either[str, dict]]:
+    """
+    Create a monadic LLM chat execution.
+
+    Returns IO[Either[Error, Response]] for functional composition.
+
+    Usage:
+        result = chat_monad(
+            prompt="What is Python?",
+            system_prompt="You are a programming tutor.",
+            model="llama3:latest"
+        ).unsafe_run()
+
+        if result.is_right():
+            print(result.value['content'])
+
+    Args:
+        messages: Optional list of message dicts (overrides prompt/system_prompt)
+        prompt: User prompt (used if messages not provided)
+        system_prompt: System prompt (used if messages not provided)
+        config: Optional LLMConfig
+        **kwargs: Override config parameters
+
+    Returns:
+        IO[Either[str, Dict]]: Monadic wrapper around the chat call
+    """
+    # Build config
+    kwargs.setdefault("system_prompt", system_prompt)
+
+    if config is None:
+        config = LLMConfig(**kwargs) if kwargs else LLMConfig()
+    elif kwargs:
+        config_dict = config.to_dict()
+        config_dict.update(kwargs)
+        config = LLMConfig.from_dict(config_dict)
+
+    # Build messages
+    if messages is None:
+        messages = []
+        if config.system_prompt:
+            messages.append({"role": "system", "content": config.system_prompt})
+        if prompt:
+            messages.append({"role": "user", "content": prompt})
+
+    def run() -> Either[str, dict]:
+        try:
+            from .router import IntelligenceRouter
+
+            router = IntelligenceRouter()
+
+            def attempt_execution(cfg):
+                provider = get_provider(cfg.provider, cfg.endpoint_url, cfg.timeout)
+                params = cfg.to_provider_params()
+                return provider.chat(messages, params)
+
+            result = attempt_execution(config)
+
+            if result.is_left():
+                err_msg = str(result.value).lower()
+                if (
+                    "timeout" in err_msg
+                    or "connection" in err_msg
+                    or "failed" in err_msg
+                ):
+                    fb_provider, fb_model, fb_url, warning = router.handle_failure(
+                        config.provider, Exception(result.value)
+                    )
+                    config.provider = fb_provider
+                    config.model = fb_model
+                    config.endpoint_url = fb_url
+                    result = attempt_execution(config)
+
+            if result.is_right():
+                response = result.value
+                content = response.get("content", "")
+                if isinstance(content, str):
+                    think_start = content.find("<think>")
+                    think_end = content.find("</think>")
+                    if think_start >= 0 and think_end > think_start:
+                        response["thinking_process"] = content[
+                            think_start + 7 : think_end
+                        ].strip()
+                        response["content"] = (
+                            content[:think_start] + content[think_end + 8 :].strip()
+                        )
+
+            return result
+        except (ValueError, TypeError, RuntimeError, ConnectionError, OSError) as e:
+            return Left(f"LLM chat failed: {e}")
+
+    return IO(run)
+
+
+# Register with Layer 1 runtime factory (INV-383-01 / DEL-383-04)
+try:
+    from clm.layer1.runtimes.factory import register_external_executor
+
+    register_external_executor("llm", LLMRuntime)
+except ImportError:
+    pass