openhands-sdk 1.7.3__py3-none-any.whl

openhands/sdk/llm/mixins/non_native_fc.py

@@ -0,0 +1,97 @@
+from __future__ import annotations
+
+from typing import Protocol, TypeGuard
+
+from litellm import ChatCompletionToolParam, Message as LiteLLMMessage
+from litellm.types.utils import Choices, ModelResponse, StreamingChoices
+
+from openhands.sdk.llm.exceptions import LLMNoResponseError
+from openhands.sdk.llm.mixins.fn_call_converter import (
+    STOP_WORDS,
+    convert_fncall_messages_to_non_fncall_messages,
+    convert_non_fncall_messages_to_fncall_messages,
+)
+from openhands.sdk.llm.utils.model_features import get_features
+
+
+class _HostSupports(Protocol):
+    model: str
+    disable_stop_word: bool | None
+    native_tool_calling: bool
+
+
+class NonNativeToolCallingMixin:
+    """Mixin providing prompt-mocked tool-calling support when native FC is off.
+
+    Host requirements:
+    - self.model: str
+    - self.disable_stop_word: bool | None
+    - self.native_tool_calling -> bool
+    """
+
+    def should_mock_tool_calls(
+        self: _HostSupports, tools: list[ChatCompletionToolParam] | None
+    ) -> bool:
+        return bool(tools) and not self.native_tool_calling
+
+    def pre_request_prompt_mock(
+        self: _HostSupports,
+        messages: list[dict],
+        tools: list[ChatCompletionToolParam],
+        kwargs: dict,
+    ) -> tuple[list[dict], dict]:
+        """Convert to non-fncall prompting when native tool-calling is off."""
+        # Skip in-context learning examples for models that understand the format
+        # or have limited context windows
+        add_iclex = not any(
+            s in self.model for s in ("openhands-lm", "devstral", "nemotron")
+        )
+        messages = convert_fncall_messages_to_non_fncall_messages(
+            messages, tools, add_in_context_learning_example=add_iclex
+        )
+        if get_features(self.model).supports_stop_words and not self.disable_stop_word:
+            kwargs = dict(kwargs)
+            kwargs["stop"] = STOP_WORDS
+
+        # Ensure we don't send tool_choice when mocking
+        kwargs.pop("tool_choice", None)
+        return messages, kwargs
+
+    def post_response_prompt_mock(
+        self: _HostSupports,
+        resp: ModelResponse,
+        nonfncall_msgs: list[dict],
+        tools: list[ChatCompletionToolParam],
+    ) -> ModelResponse:
+        if len(resp.choices) < 1:
+            raise LLMNoResponseError(
+                "Response choices is less than 1 (seen in some providers). Resp: "
+                + str(resp)
+            )
+
+        def _all_choices(
+            items: list[Choices | StreamingChoices],
+        ) -> TypeGuard[list[Choices]]:
+            return all(isinstance(c, Choices) for c in items)
+
+        if not _all_choices(resp.choices):
+            raise AssertionError(
+                "Expected non-streaming Choices when post-processing mocked tools"
+            )
+
+        # Preserve provider-specific reasoning fields before conversion
+        orig_msg = resp.choices[0].message
+        non_fn_message: dict = orig_msg.model_dump()
+        fn_msgs: list[dict] = convert_non_fncall_messages_to_fncall_messages(
+            nonfncall_msgs + [non_fn_message], tools
+        )
+        last: dict = fn_msgs[-1]
+
+        for name in ("reasoning_content", "provider_specific_fields"):
+            val = getattr(orig_msg, name, None)
+            if not val:
+                continue
+            last[name] = val
+
+        resp.choices[0].message = LiteLLMMessage.model_validate(last)
+        return resp

openhands/sdk/llm/options/__init__.py

@@ -0,0 +1 @@
+# options package for LLM parameter selection helpers

openhands/sdk/llm/options/chat_options.py

@@ -0,0 +1,93 @@
+from __future__ import annotations
+
+from typing import Any
+
+from openhands.sdk.llm.options.common import apply_defaults_if_absent
+from openhands.sdk.llm.utils.model_features import get_features
+
+
+def select_chat_options(
+    llm, user_kwargs: dict[str, Any], has_tools: bool
+) -> dict[str, Any]:
+    """Behavior-preserving extraction of _normalize_call_kwargs.
+
+    This keeps the exact provider-aware mappings and precedence.
+    """
+    # First pass: apply simple defaults without touching user-supplied values
+    defaults: dict[str, Any] = {
+        "top_k": llm.top_k,
+        "top_p": llm.top_p,
+        "temperature": llm.temperature,
+        # OpenAI-compatible param is `max_completion_tokens`
+        "max_completion_tokens": llm.max_output_tokens,
+    }
+    out = apply_defaults_if_absent(user_kwargs, defaults)
+
+    # Azure -> uses max_tokens instead
+    if llm.model.startswith("azure"):
+        if "max_completion_tokens" in out:
+            out["max_tokens"] = out.pop("max_completion_tokens")
+
+    # If user didn't set extra_headers, propagate from llm config
+    if llm.extra_headers is not None and "extra_headers" not in out:
+        out["extra_headers"] = dict(llm.extra_headers)
+
+    # Reasoning-model quirks
+    if get_features(llm.model).supports_reasoning_effort:
+        # LiteLLM automatically handles reasoning_effort for all models, including
+        # Claude Opus 4.5 (maps to output_config and adds beta header automatically)
+        if llm.reasoning_effort is not None:
+            out["reasoning_effort"] = llm.reasoning_effort
+
+        # All reasoning models ignore temp/top_p
+        out.pop("temperature", None)
+        out.pop("top_p", None)
+
+        # Gemini 2.5-pro default to low if not set
+        if "gemini-2.5-pro" in llm.model:
+            if llm.reasoning_effort in {None, "none"}:
+                out["reasoning_effort"] = "low"
+
+    # Extended thinking models
+    if get_features(llm.model).supports_extended_thinking:
+        if llm.extended_thinking_budget:
+            out["thinking"] = {
+                "type": "enabled",
+                "budget_tokens": llm.extended_thinking_budget,
+            }
+            # Enable interleaved thinking
+            # Merge default header with any user-provided headers; user wins on conflict
+            existing = out.get("extra_headers") or {}
+            out["extra_headers"] = {
+                "anthropic-beta": "interleaved-thinking-2025-05-14",
+                **existing,
+            }
+            # Fix litellm behavior
+            out["max_tokens"] = llm.max_output_tokens
+        # Anthropic models ignore temp/top_p
+        out.pop("temperature", None)
+        out.pop("top_p", None)
+
+    # Mistral / Gemini safety
+    if llm.safety_settings:
+        ml = llm.model.lower()
+        if "mistral" in ml or "gemini" in ml:
+            out["safety_settings"] = llm.safety_settings
+
+    # Tools: if not using native, strip tool_choice so we don't confuse providers
+    if not has_tools:
+        out.pop("tools", None)
+        out.pop("tool_choice", None)
+
+    # Send prompt_cache_retention only if model supports it
+    if (
+        get_features(llm.model).supports_prompt_cache_retention
+        and llm.prompt_cache_retention
+    ):
+        out["prompt_cache_retention"] = llm.prompt_cache_retention
+
+    # Pass through user-provided extra_body unchanged
+    if llm.litellm_extra_body:
+        out["extra_body"] = llm.litellm_extra_body
+
+    return out

openhands/sdk/llm/options/common.py

@@ -0,0 +1,19 @@
+from __future__ import annotations
+
+from typing import Any
+
+
+def apply_defaults_if_absent(
+    user_kwargs: dict[str, Any], defaults: dict[str, Any]
+) -> dict[str, Any]:
+    """Return a new dict with defaults applied when keys are absent.
+
+    - Pure and deterministic; does not mutate inputs
+    - Only applies defaults when the key is missing and default is not None
+    - Does not alter user-provided values
+    """
+    out = dict(user_kwargs)
+    for key, value in defaults.items():
+        if key not in out and value is not None:
+            out[key] = value
+    return out

openhands/sdk/llm/options/responses_options.py

@@ -0,0 +1,67 @@
+from __future__ import annotations
+
+from typing import Any
+
+from openhands.sdk.llm.options.common import apply_defaults_if_absent
+from openhands.sdk.llm.utils.model_features import get_features
+
+
+def select_responses_options(
+    llm,
+    user_kwargs: dict[str, Any],
+    *,
+    include: list[str] | None,
+    store: bool | None,
+) -> dict[str, Any]:
+    """Behavior-preserving extraction of _normalize_responses_kwargs."""
+    # Apply defaults for keys that are not forced by policy
+    out = apply_defaults_if_absent(
+        user_kwargs,
+        {
+            "max_output_tokens": llm.max_output_tokens,
+        },
+    )
+
+    # Enforce sampling/tool behavior for Responses path
+    out["temperature"] = 1.0
+    out["tool_choice"] = "auto"
+
+    # If user didn't set extra_headers, propagate from llm config
+    if llm.extra_headers is not None and "extra_headers" not in out:
+        out["extra_headers"] = dict(llm.extra_headers)
+
+    # Store defaults to False (stateless) unless explicitly provided
+    if store is not None:
+        out["store"] = bool(store)
+    else:
+        out.setdefault("store", False)
+
+    # Include encrypted reasoning only when the user enables it on the LLM,
+    # and only for stateless calls (store=False). Respect user choice.
+    include_list = list(include) if include is not None else []
+
+    if not out.get("store", False) and llm.enable_encrypted_reasoning:
+        if "reasoning.encrypted_content" not in include_list:
+            include_list.append("reasoning.encrypted_content")
+    if include_list:
+        out["include"] = include_list
+
+    # Include reasoning effort only if explicitly set
+    if llm.reasoning_effort:
+        out["reasoning"] = {"effort": llm.reasoning_effort}
+        # Optionally include summary if explicitly set (requires verified org)
+        if llm.reasoning_summary:
+            out["reasoning"]["summary"] = llm.reasoning_summary
+
+    # Send prompt_cache_retention only if model supports it
+    if (
+        get_features(llm.model).supports_prompt_cache_retention
+        and llm.prompt_cache_retention
+    ):
+        out["prompt_cache_retention"] = llm.prompt_cache_retention
+
+    # Pass through user-provided extra_body unchanged
+    if llm.litellm_extra_body:
+        out["extra_body"] = llm.litellm_extra_body
+
+    return out

openhands/sdk/llm/router/__init__.py

@@ -0,0 +1,10 @@
+from openhands.sdk.llm.router.base import RouterLLM
+from openhands.sdk.llm.router.impl.multimodal import MultimodalRouter
+from openhands.sdk.llm.router.impl.random import RandomRouter
+
+
+__all__ = [
+    "RouterLLM",
+    "RandomRouter",
+    "MultimodalRouter",
+]

openhands/sdk/llm/router/base.py

@@ -0,0 +1,117 @@
+from abc import abstractmethod
+from collections.abc import Sequence
+
+from pydantic import (
+    Field,
+    field_validator,
+    model_validator,
+)
+
+from openhands.sdk.llm.llm import LLM
+from openhands.sdk.llm.llm_response import LLMResponse
+from openhands.sdk.llm.message import Message
+from openhands.sdk.llm.streaming import TokenCallbackType
+from openhands.sdk.logger import get_logger
+from openhands.sdk.tool.tool import ToolDefinition
+
+
+logger = get_logger(__name__)
+
+
+class RouterLLM(LLM):
+    """
+    Base class for multiple LLM acting as a unified LLM.
+    This class provides a foundation for implementing model routing by
+    inheriting from LLM, allowing routers to work with multiple underlying
+    LLM models while presenting a unified LLM interface to consumers.
+    Key features:
+    - Works with multiple LLMs configured via llms_for_routing
+    - Delegates all other operations/properties to the selected LLM
+    - Provides routing interface through select_llm() method
+    """
+
+    router_name: str = Field(default="base_router", description="Name of the router")
+    llms_for_routing: dict[str, LLM] = Field(
+        default_factory=dict
+    )  # Mapping of LLM name to LLM instance for routing
+    active_llm: LLM | None = Field(
+        default=None, description="Currently selected LLM instance"
+    )
+
+    @field_validator("llms_for_routing")
+    @classmethod
+    def validate_llms_not_empty(cls, v):
+        if not v:
+            raise ValueError(
+                "llms_for_routing cannot be empty - at least one LLM must be provided"
+            )
+        return v
+
+    def completion(
+        self,
+        messages: list[Message],
+        tools: Sequence[ToolDefinition] | None = None,
+        return_metrics: bool = False,
+        add_security_risk_prediction: bool = False,
+        on_token: TokenCallbackType | None = None,
+        **kwargs,
+    ) -> LLMResponse:
+        """
+        This method intercepts completion calls and routes them to the appropriate
+        underlying LLM based on the routing logic implemented in select_llm().
+        """
+        # Select appropriate LLM
+        selected_model = self.select_llm(messages)
+        self.active_llm = self.llms_for_routing[selected_model]
+
+        logger.info(f"RouterLLM routing to {selected_model}...")
+
+        # Delegate to selected LLM
+        return self.active_llm.completion(
+            messages=messages,
+            tools=tools,
+            _return_metrics=return_metrics,
+            add_security_risk_prediction=add_security_risk_prediction,
+            on_token=on_token,
+            **kwargs,
+        )
+
+    @abstractmethod
+    def select_llm(self, messages: list[Message]) -> str:
+        """Select which LLM to use based on messages and events.
+
+        This method implements the core routing logic for the RouterLLM.
+        Subclasses should analyze the provided messages to determine which
+        LLM from llms_for_routing is most appropriate for handling the request.
+
+        Args:
+            messages: List of messages in the conversation that can be used
+                     to inform the routing decision.
+
+        Returns:
+            The key/name of the LLM to use from llms_for_routing dictionary.
+        """
+
+    def __getattr__(self, name):
+        """Delegate other attributes/methods to the active LLM."""
+        fallback_llm = next(iter(self.llms_for_routing.values()))
+        logger.info(f"RouterLLM: No active LLM, using first LLM for attribute '{name}'")
+        return getattr(fallback_llm, name)
+
+    def __str__(self) -> str:
+        """String representation of the router."""
+        return f"{self.__class__.__name__}(llms={list(self.llms_for_routing.keys())})"
+
+    @model_validator(mode="before")
+    @classmethod
+    def set_placeholder_model(cls, data):
+        """Guarantee `model` exists before LLM base validation runs."""
+        if not isinstance(data, dict):
+            return data
+        d = dict(data)
+
+        # In router, we don't need a model name to be specified
+        if "model" not in d or not d["model"]:
+            d["model"] = d.get("router_name", "router")
+
+        return d

openhands/sdk/llm/router/impl/multimodal.py

@@ -0,0 +1,76 @@
+from typing import ClassVar
+
+from pydantic import model_validator
+
+from openhands.sdk.llm.message import Message
+from openhands.sdk.llm.router.base import RouterLLM
+from openhands.sdk.logger import get_logger
+
+
+logger = get_logger(__name__)
+
+
+class MultimodalRouter(RouterLLM):
+    """
+    A RouterLLM implementation that routes requests based on multimodal content
+    (e.g., images) and token limits. If any message contains multimodal content
+    or if the token limit of the secondary model is exceeded, it routes to the
+    primary model. Otherwise, it routes to the secondary model.
+
+    Note: The primary model is expected to support multimodal content, while
+    the secondary model is typically a text-only model with a lower context window.
+    """
+
+    router_name: str = "multimodal_router"
+
+    PRIMARY_MODEL_KEY: ClassVar[str] = "primary"
+    SECONDARY_MODEL_KEY: ClassVar[str] = "secondary"
+
+    def select_llm(self, messages: list[Message]) -> str:
+        """Select LLM based on multimodal content and token limits."""
+        route_to_primary = False
+
+        # Check for multimodal content in messages
+        for message in messages:
+            if message.contains_image:
+                logger.info(
+                    "Multimodal content detected in messages. "
+                    "Routing to the primary model."
+                )
+                route_to_primary = True
+
+        # Check if `messages` exceeds context window of the secondary model
+        # Assuming the secondary model has a lower context window limit
+        # compared to the primary model
+        secondary_llm = self.llms_for_routing.get(self.SECONDARY_MODEL_KEY)
+        if secondary_llm and (
+            secondary_llm.max_input_tokens
+            and secondary_llm.get_token_count(messages) > secondary_llm.max_input_tokens
+        ):
+            logger.warning(
+                f"Messages having {secondary_llm.get_token_count(messages)} tokens, exceeded secondary model's max input tokens ({secondary_llm.max_input_tokens} tokens). "  # noqa: E501
+                "Routing to the primary model."
+            )
+            route_to_primary = True
+
+        if route_to_primary:
+            logger.info("Routing to the primary model...")
+            return self.PRIMARY_MODEL_KEY
+        else:
+            logger.info("Routing to the secondary model...")
+            return self.SECONDARY_MODEL_KEY
+
+    @model_validator(mode="after")
+    def _validate_llms_for_routing(self) -> "MultimodalRouter":
+        """Ensure required models are present in llms_for_routing."""
+        if self.PRIMARY_MODEL_KEY not in self.llms_for_routing:
+            raise ValueError(
+                f"Primary LLM key '{self.PRIMARY_MODEL_KEY}' not found"
+                " in llms_for_routing."
+            )
+        if self.SECONDARY_MODEL_KEY not in self.llms_for_routing:
+            raise ValueError(
+                f"Secondary LLM key '{self.SECONDARY_MODEL_KEY}' not found"
+                " in llms_for_routing."
+            )
+        return self

openhands/sdk/llm/router/impl/random.py

@@ -0,0 +1,22 @@
+import random
+
+from openhands.sdk.llm.message import Message
+from openhands.sdk.llm.router.base import RouterLLM
+from openhands.sdk.logger import get_logger
+
+
+logger = get_logger(__name__)
+
+
+class RandomRouter(RouterLLM):
+    """
+    A simple implementation of RouterLLM that randomly selects an LLM from
+    llms_for_routing for each completion request.
+    """
+
+    router_name: str = "random_router"
+
+    def select_llm(self, messages: list[Message]) -> str:  # noqa: ARG002
+        selected_llm_name = random.choice(list(self.llms_for_routing.keys()))
+        logger.info(f"Randomly selected LLM: {selected_llm_name}")
+        return selected_llm_name

openhands/sdk/llm/streaming.py

@@ -0,0 +1,9 @@
+from collections.abc import Callable
+
+from litellm.types.utils import ModelResponseStream
+
+
+# Type alias for stream chunks
+LLMStreamChunk = ModelResponseStream
+
+TokenCallbackType = Callable[[LLMStreamChunk], None]