openhands-sdk 1.7.3__py3-none-any.whl

openhands/sdk/llm/utils/metrics.py

@@ -0,0 +1,312 @@
+import copy
+import time
+from typing import final
+
+from pydantic import BaseModel, Field, field_validator, model_validator
+
+
+class Cost(BaseModel):
+    model: str
+    cost: float = Field(ge=0.0, description="Cost must be non-negative")
+    timestamp: float = Field(default_factory=time.time)
+
+    @field_validator("cost")
+    @classmethod
+    def validate_cost(cls, v: float) -> float:
+        if v < 0:
+            raise ValueError("Cost cannot be negative")
+        return v
+
+
+class ResponseLatency(BaseModel):
+    """Metric tracking the round-trip time per completion call."""
+
+    model: str
+    latency: float = Field(ge=0.0, description="Latency must be non-negative")
+    response_id: str
+
+    @field_validator("latency")
+    @classmethod
+    def validate_latency(cls, v: float) -> float:
+        return max(0.0, v)
+
+
+class TokenUsage(BaseModel):
+    """Metric tracking detailed token usage per completion call."""
+
+    model: str = Field(default="")
+    prompt_tokens: int = Field(
+        default=0, ge=0, description="Prompt tokens must be non-negative"
+    )
+    completion_tokens: int = Field(
+        default=0, ge=0, description="Completion tokens must be non-negative"
+    )
+    cache_read_tokens: int = Field(
+        default=0, ge=0, description="Cache read tokens must be non-negative"
+    )
+    cache_write_tokens: int = Field(
+        default=0, ge=0, description="Cache write tokens must be non-negative"
+    )
+    reasoning_tokens: int = Field(
+        default=0, ge=0, description="Reasoning tokens must be non-negative"
+    )
+    context_window: int = Field(
+        default=0, ge=0, description="Context window must be non-negative"
+    )
+    per_turn_token: int = Field(
+        default=0, ge=0, description="Per turn tokens must be non-negative"
+    )
+    response_id: str = Field(default="")
+
+    def __add__(self, other: "TokenUsage") -> "TokenUsage":
+        """Add two TokenUsage instances together."""
+        return TokenUsage(
+            model=self.model,
+            prompt_tokens=self.prompt_tokens + other.prompt_tokens,
+            completion_tokens=self.completion_tokens + other.completion_tokens,
+            cache_read_tokens=self.cache_read_tokens + other.cache_read_tokens,
+            cache_write_tokens=self.cache_write_tokens + other.cache_write_tokens,
+            reasoning_tokens=self.reasoning_tokens + other.reasoning_tokens,
+            context_window=max(self.context_window, other.context_window),
+            per_turn_token=other.per_turn_token,
+            response_id=self.response_id,
+        )
+
+
+class MetricsSnapshot(BaseModel):
+    """A snapshot of metrics at a point in time.
+
+    Does not include lists of individual costs, latencies, or token usages.
+    """
+
+    model_name: str = Field(default="default", description="Name of the model")
+    accumulated_cost: float = Field(
+        default=0.0, ge=0.0, description="Total accumulated cost, must be non-negative"
+    )
+    max_budget_per_task: float | None = Field(
+        default=None, description="Maximum budget per task"
+    )
+    accumulated_token_usage: TokenUsage | None = Field(
+        default=None, description="Accumulated token usage across all calls"
+    )
+
+
+@final
+class Metrics(MetricsSnapshot):
+    """Metrics class can record various metrics during running and evaluation.
+    We track:
+      - accumulated_cost and costs
+      - max_budget_per_task (budget limit)
+      - A list of ResponseLatency
+      - A list of TokenUsage (one per call).
+    """
+
+    costs: list[Cost] = Field(
+        default_factory=list, description="List of individual costs"
+    )
+    response_latencies: list[ResponseLatency] = Field(
+        default_factory=list, description="List of response latencies"
+    )
+    token_usages: list[TokenUsage] = Field(
+        default_factory=list, description="List of token usage records"
+    )
+
+    @field_validator("accumulated_cost")
+    @classmethod
+    def validate_accumulated_cost(cls, v: float) -> float:
+        if v < 0:
+            raise ValueError("Total cost cannot be negative.")
+        return v
+
+    @model_validator(mode="after")
+    def initialize_accumulated_token_usage(self) -> "Metrics":
+        if self.accumulated_token_usage is None:
+            self.accumulated_token_usage = TokenUsage(
+                model=self.model_name,
+                prompt_tokens=0,
+                completion_tokens=0,
+                cache_read_tokens=0,
+                cache_write_tokens=0,
+                reasoning_tokens=0,
+                context_window=0,
+                response_id="",
+            )
+        return self
+
+    def get_snapshot(self) -> MetricsSnapshot:
+        """Get a snapshot of the current metrics without the detailed lists."""
+        return MetricsSnapshot(
+            model_name=self.model_name,
+            accumulated_cost=self.accumulated_cost,
+            max_budget_per_task=self.max_budget_per_task,
+            accumulated_token_usage=copy.deepcopy(self.accumulated_token_usage)
+            if self.accumulated_token_usage
+            else None,
+        )
+
+    def add_cost(self, value: float) -> None:
+        if value < 0:
+            raise ValueError("Added cost cannot be negative.")
+        self.accumulated_cost += value
+        self.costs.append(Cost(cost=value, model=self.model_name))
+
+    def add_response_latency(self, value: float, response_id: str) -> None:
+        self.response_latencies.append(
+            ResponseLatency(
+                latency=max(0.0, value), model=self.model_name, response_id=response_id
+            )
+        )
+
+    def add_token_usage(
+        self,
+        prompt_tokens: int,
+        completion_tokens: int,
+        cache_read_tokens: int,
+        cache_write_tokens: int,
+        context_window: int,
+        response_id: str,
+        reasoning_tokens: int = 0,
+    ) -> None:
+        """Add a single usage record."""
+        # Token each turn for calculating context usage.
+        per_turn_token = prompt_tokens + completion_tokens
+
+        usage = TokenUsage(
+            model=self.model_name,
+            prompt_tokens=prompt_tokens,
+            completion_tokens=completion_tokens,
+            cache_read_tokens=cache_read_tokens,
+            cache_write_tokens=cache_write_tokens,
+            reasoning_tokens=reasoning_tokens,
+            context_window=context_window,
+            per_turn_token=per_turn_token,
+            response_id=response_id,
+        )
+        self.token_usages.append(usage)
+
+        # Update accumulated token usage using the __add__ operator
+        new_usage = TokenUsage(
+            model=self.model_name,
+            prompt_tokens=prompt_tokens,
+            completion_tokens=completion_tokens,
+            cache_read_tokens=cache_read_tokens,
+            cache_write_tokens=cache_write_tokens,
+            reasoning_tokens=reasoning_tokens,
+            context_window=context_window,
+            per_turn_token=per_turn_token,
+            response_id="",
+        )
+        if self.accumulated_token_usage is None:
+            self.accumulated_token_usage = new_usage
+        else:
+            self.accumulated_token_usage = self.accumulated_token_usage + new_usage
+
+    def merge(self, other: "Metrics") -> None:
+        """Merge 'other' metrics into this one."""
+        self.accumulated_cost += other.accumulated_cost
+
+        # Keep the max_budget_per_task from other if it's set and this one isn't
+        if self.max_budget_per_task is None and other.max_budget_per_task is not None:
+            self.max_budget_per_task = other.max_budget_per_task
+
+        self.costs += other.costs
+        self.token_usages += other.token_usages
+        self.response_latencies += other.response_latencies
+
+        # Merge accumulated token usage using the __add__ operator
+        if self.accumulated_token_usage is None:
+            self.accumulated_token_usage = other.accumulated_token_usage
+        elif other.accumulated_token_usage is not None:
+            self.accumulated_token_usage = (
+                self.accumulated_token_usage + other.accumulated_token_usage
+            )
+
+    def get(self) -> dict:
+        """Return the metrics in a dictionary."""
+        return {
+            "accumulated_cost": self.accumulated_cost,
+            "max_budget_per_task": self.max_budget_per_task,
+            "accumulated_token_usage": self.accumulated_token_usage.model_dump()
+            if self.accumulated_token_usage
+            else None,
+            "costs": [cost.model_dump() for cost in self.costs],
+            "response_latencies": [
+                latency.model_dump() for latency in self.response_latencies
+            ],
+            "token_usages": [usage.model_dump() for usage in self.token_usages],
+        }
+
+    def log(self) -> str:
+        """Log the metrics."""
+        metrics = self.get()
+        logs = ""
+        for key, value in metrics.items():
+            logs += f"{key}: {value}\n"
+        return logs
+
+    def deep_copy(self) -> "Metrics":
+        """Create a deep copy of the Metrics object."""
+        return copy.deepcopy(self)
+
+    def diff(self, baseline: "Metrics") -> "Metrics":
+        """Calculate the difference between current metrics and a baseline.
+
+        This is useful for tracking metrics for specific operations like delegates.
+
+        Args:
+            baseline: A metrics object representing the baseline state
+
+        Returns:
+            A new Metrics object containing only the differences since the baseline
+        """
+        result = Metrics(model_name=self.model_name)
+
+        # Calculate cost difference
+        result.accumulated_cost = self.accumulated_cost - baseline.accumulated_cost
+
+        # Include only costs that were added after the baseline
+        if baseline.costs:
+            last_baseline_timestamp = baseline.costs[-1].timestamp
+            result.costs = [
+                cost for cost in self.costs if cost.timestamp > last_baseline_timestamp
+            ]
+        else:
+            result.costs = self.costs.copy()
+
+        # Include only response latencies that were added after the baseline
+        result.response_latencies = self.response_latencies[
+            len(baseline.response_latencies) :
+        ]
+
+        # Include only token usages that were added after the baseline
+        result.token_usages = self.token_usages[len(baseline.token_usages) :]
+
+        # Calculate accumulated token usage difference
+        base_usage = baseline.accumulated_token_usage
+        current_usage = self.accumulated_token_usage
+
+        if current_usage is not None and base_usage is not None:
+            result.accumulated_token_usage = TokenUsage(
+                model=self.model_name,
+                prompt_tokens=current_usage.prompt_tokens - base_usage.prompt_tokens,
+                completion_tokens=current_usage.completion_tokens
+                - base_usage.completion_tokens,
+                cache_read_tokens=current_usage.cache_read_tokens
+                - base_usage.cache_read_tokens,
+                cache_write_tokens=current_usage.cache_write_tokens
+                - base_usage.cache_write_tokens,
+                reasoning_tokens=current_usage.reasoning_tokens
+                - base_usage.reasoning_tokens,
+                context_window=current_usage.context_window,
+                per_turn_token=0,
+                response_id="",
+            )
+        elif current_usage is not None:
+            result.accumulated_token_usage = current_usage
+        else:
+            result.accumulated_token_usage = None
+
+        return result
+
+    def __repr__(self) -> str:
+        return f"Metrics({self.get()}"

openhands/sdk/llm/utils/model_features.py

@@ -0,0 +1,192 @@
+from dataclasses import dataclass
+
+
+def model_matches(model: str, patterns: list[str]) -> bool:
+    """Return True if any pattern appears as a substring in the raw model name.
+
+    Matching semantics:
+    - Case-insensitive substring search on full raw model string
+    """
+    raw = (model or "").strip().lower()
+    for pat in patterns:
+        token = pat.strip().lower()
+        if token in raw:
+            return True
+    return False
+
+
+def apply_ordered_model_rules(model: str, rules: list[str]) -> bool:
+    """Apply ordered include/exclude model rules to determine final support.
+
+    Rules semantics:
+    - Each entry is a substring token. '!' prefix marks an exclude rule.
+    - Case-insensitive substring matching against the raw model string.
+    - Evaluated in order; the last matching rule wins.
+    - If no rule matches, returns False.
+    """
+    raw = (model or "").strip().lower()
+    decided: bool | None = None
+    for rule in rules:
+        token = rule.strip().lower()
+        if not token:
+            continue
+        is_exclude = token.startswith("!")
+        core = token[1:] if is_exclude else token
+        if core and core in raw:
+            decided = not is_exclude
+    return bool(decided)
+
+
+@dataclass(frozen=True)
+class ModelFeatures:
+    supports_reasoning_effort: bool
+    supports_extended_thinking: bool
+    supports_prompt_cache: bool
+    supports_stop_words: bool
+    supports_responses_api: bool
+    force_string_serializer: bool
+    send_reasoning_content: bool
+    supports_prompt_cache_retention: bool
+
+
+# Model lists capturing current behavior. Keep entries lowercase.
+
+REASONING_EFFORT_MODELS: list[str] = [
+    # Mirror main behavior exactly (no unintended expansion)
+    "o1-2024-12-17",
+    "o1",
+    "o3",
+    "o3-2025-04-16",
+    "o3-mini-2025-01-31",
+    "o3-mini",
+    "o4-mini",
+    "o4-mini-2025-04-16",
+    "gemini-2.5-flash",
+    "gemini-2.5-pro",
+    # OpenAI GPT-5 family (includes mini variants)
+    "gpt-5",
+    # Anthropic Opus 4.5
+    "claude-opus-4-5",
+    # Nova 2 Lite
+    "nova-2-lite",
+]
+
+EXTENDED_THINKING_MODELS: list[str] = [
+    # Anthropic model family
+    # We did not include sonnet 3.7 and 4 here as they don't brings
+    # significant performance improvements for agents
+    "claude-sonnet-4-5",
+    "claude-haiku-4-5",
+]
+
+PROMPT_CACHE_MODELS: list[str] = [
+    "claude-3-7-sonnet",
+    "claude-sonnet-3-7-latest",
+    "claude-3-5-sonnet",
+    "claude-3-5-haiku",
+    "claude-3-haiku-20240307",
+    "claude-3-opus-20240229",
+    "claude-sonnet-4",
+    "claude-opus-4",
+    # Anthropic Haiku 4.5 variants (dash only; official IDs use hyphens)
+    "claude-haiku-4-5",
+    "claude-opus-4-5",
+]
+
+# Models that support a top-level prompt_cache_retention parameter
+# Source: OpenAI Prompt Caching docs (extended retention), which list:
+#   - gpt-5.2
+#   - gpt-5.1
+#   - gpt-5.1-codex
+#   - gpt-5.1-codex-mini
+#   - gpt-5.1-chat-latest
+#   - gpt-5
+#   - gpt-5-codex
+#   - gpt-4.1
+# Use ordered include/exclude rules (last wins) to naturally express exceptions.
+PROMPT_CACHE_RETENTION_MODELS: list[str] = [
+    # Broad allow for GPT-5 family and GPT-4.1 (covers gpt-5.2 and variants)
+    "gpt-5",
+    "gpt-4.1",
+    # Exclude all mini variants by default
+    "!mini",
+    # Re-allow the explicitly documented supported mini variant
+    "gpt-5.1-codex-mini",
+]
+
+SUPPORTS_STOP_WORDS_FALSE_MODELS: list[str] = [
+    # o-series families don't support stop words
+    "o1",
+    "o3",
+    # grok-4 specific model name (basename)
+    "grok-4-0709",
+    "grok-code-fast-1",
+    # DeepSeek R1 family
+    "deepseek-r1-0528",
+]
+
+# Models that should use the OpenAI Responses API path by default
+RESPONSES_API_MODELS: list[str] = [
+    # OpenAI GPT-5 family (includes mini variants)
+    "gpt-5",
+    # OpenAI Codex (uses Responses API)
+    "codex-mini-latest",
+]
+
+# Models that require string serializer for tool messages
+# These models don't support structured content format [{"type":"text","text":"..."}]
+# and need plain strings instead
+# NOTE: model_matches uses case-insensitive substring matching, not globbing.
+#       Keep these entries as bare substrings without wildcards.
+FORCE_STRING_SERIALIZER_MODELS: list[str] = [
+    "deepseek",  # e.g., DeepSeek-V3.2-Exp
+    "glm",  # e.g., GLM-4.5 / GLM-4.6
+    # Kimi K2-Instruct requires string serialization only on Groq
+    "groq/kimi-k2-instruct",  # explicit provider-prefixed IDs
+    # MiniMax-M2 via OpenRouter rejects array content with
+    # "Input should be a valid string" for ChatCompletionToolMessage.content
+    "openrouter/minimax",
+]
+
+# Models that we should send full reasoning content
+# in the message input
+SEND_REASONING_CONTENT_MODELS: list[str] = [
+    "kimi-k2-thinking",
+    "openrouter/minimax-m2",  # MiniMax-M2 via OpenRouter (interleaved thinking)
+    "deepseek/deepseek-reasoner",
+]
+
+
+def get_features(model: str) -> ModelFeatures:
+    """Get model features."""
+    return ModelFeatures(
+        supports_reasoning_effort=model_matches(model, REASONING_EFFORT_MODELS),
+        supports_extended_thinking=model_matches(model, EXTENDED_THINKING_MODELS),
+        supports_prompt_cache=model_matches(model, PROMPT_CACHE_MODELS),
+        supports_stop_words=not model_matches(model, SUPPORTS_STOP_WORDS_FALSE_MODELS),
+        supports_responses_api=model_matches(model, RESPONSES_API_MODELS),
+        force_string_serializer=model_matches(model, FORCE_STRING_SERIALIZER_MODELS),
+        send_reasoning_content=model_matches(model, SEND_REASONING_CONTENT_MODELS),
+        # Extended prompt_cache_retention support follows ordered include/exclude rules.
+        supports_prompt_cache_retention=apply_ordered_model_rules(
+            model, PROMPT_CACHE_RETENTION_MODELS
+        ),
+    )
+
+
+# Default temperature mapping.
+# Each entry: (pattern, default_temperature)
+DEFAULT_TEMPERATURE_MODELS: list[tuple[str, float]] = [
+    ("kimi-k2-thinking", 1.0),
+]
+
+
+def get_default_temperature(model: str) -> float:
+    """Return the default temperature for a given model pattern.
+
+    Uses case-insensitive substring matching via model_matches.
+    """
+    for pattern, value in DEFAULT_TEMPERATURE_MODELS:
+        if model_matches(model, [pattern]):
+            return value
+    return 0.0

openhands/sdk/llm/utils/model_info.py

@@ -0,0 +1,90 @@
+import time
+from functools import lru_cache
+from logging import getLogger
+
+import httpx
+from litellm.types.utils import ModelInfo
+from litellm.utils import get_model_info
+from pydantic import SecretStr
+
+
+logger = getLogger(__name__)
+
+
+@lru_cache
+def _get_model_info_from_litellm_proxy(
+    secret_api_key: SecretStr | str | None,
+    base_url: str,
+    model: str,
+    cache_key: int | None = None,
+):
+    logger.debug(f"Get model_info_from_litellm_proxy:{cache_key}")
+    try:
+        headers = {}
+        if isinstance(secret_api_key, SecretStr):
+            secret_api_key = secret_api_key.get_secret_value()
+        if secret_api_key:
+            headers["Authorization"] = f"Bearer {secret_api_key}"
+
+        response = httpx.get(f"{base_url}/v1/model/info", headers=headers)
+        data = response.json().get("data", [])
+        current = next(
+            (
+                info
+                for info in data
+                if info["model_name"] == model.removeprefix("litellm_proxy/")
+            ),
+            None,
+        )
+        if current:
+            model_info = current.get("model_info")
+            logger.debug(f"Got model info from litellm proxy: {model_info}")
+            return model_info
+    except Exception as e:
+        logger.debug(
+            f"Error fetching model info from proxy: {e}",
+            exc_info=True,
+            stack_info=True,
+        )
+
+
+def get_litellm_model_info(
+    secret_api_key: SecretStr | str | None, base_url: str | None, model: str
+) -> ModelInfo | None:
+    # Try to get model info via openrouter or litellm proxy first
+    try:
+        if model.startswith("openrouter"):
+            model_info = get_model_info(model)
+            if model_info:
+                return model_info
+    except Exception as e:
+        logger.debug(f"get_model_info(openrouter) failed: {e}")
+
+    if model.startswith("litellm_proxy/") and base_url:
+        # Use the current hour as a cache key - only refresh hourly
+        cache_key = int(time.time() / 3600)
+
+        model_info = _get_model_info_from_litellm_proxy(
+            secret_api_key=secret_api_key,
+            base_url=base_url,
+            model=model,
+            cache_key=cache_key,
+        )
+        if model_info:
+            return model_info
+
+    # Fallbacks: try base name variants
+    try:
+        model_info = get_model_info(model.split(":")[0])
+        if model_info:
+            return model_info
+    except Exception:
+        pass
+    try:
+        model_info = get_model_info(model.split("/")[-1])
+        if model_info:
+            return model_info
+    except Exception:
+        pass
+
+    return None

openhands/sdk/llm/utils/model_prompt_spec.py

@@ -0,0 +1,98 @@
+"""Utilities for detecting model families and variants.
+
+These helpers allow prompts and other systems to tailor behavior for specific
+LLM providers while keeping naming heuristics centralized.
+"""
+
+from __future__ import annotations
+
+from pydantic import BaseModel, ConfigDict
+
+
+class ModelPromptSpec(BaseModel):
+    """Detected prompt metadata for a given model configuration."""
+
+    model_config = ConfigDict(frozen=True)
+
+    family: str | None = None
+    variant: str | None = None
+
+
+_MODEL_FAMILY_PATTERNS: dict[str, tuple[str, ...]] = {
+    "openai_gpt": (
+        "gpt-",
+        "o1",
+        "o3",
+        "o4",
+    ),
+    "anthropic_claude": ("claude",),
+    "google_gemini": ("gemini",),
+    "meta_llama": ("llama",),
+    "mistral": ("mistral",),
+    "deepseek": ("deepseek",),
+    "alibaba_qwen": ("qwen",),
+}
+
+# Ordered heuristics to pick the most specific variant available for a family.
+_MODEL_VARIANT_PATTERNS: dict[str, tuple[tuple[str, tuple[str, ...]], ...]] = {
+    "openai_gpt": (
+        ("gpt-5-codex", ("gpt-5-codex", "gpt-5.1-codex")),
+        ("gpt-5", ("gpt-5", "gpt-5.1")),
+    ),
+}
+
+
+def _normalize(name: str | None) -> str:
+    return (name or "").strip().lower()
+
+
+def _match_family(model_name: str) -> str | None:
+    normalized = _normalize(model_name)
+    if not normalized:
+        return None
+
+    for family, patterns in _MODEL_FAMILY_PATTERNS.items():
+        if any(pattern in normalized for pattern in patterns):
+            return family
+    return None
+
+
+def _match_variant(
+    family: str,
+    model_name: str,
+    canonical_name: str | None = None,
+) -> str | None:
+    patterns = _MODEL_VARIANT_PATTERNS.get(family)
+    if not patterns:
+        return None
+
+    # Choose canonical_name if available, otherwise fall back to model_name
+    candidate = _normalize(canonical_name) or _normalize(model_name)
+    if not candidate:
+        return None
+
+    for variant, substrings in patterns:
+        if any(sub in candidate for sub in substrings):
+            return variant
+
+    return None
+
+
+def get_model_prompt_spec(
+    model_name: str,
+    canonical_name: str | None = None,
+) -> ModelPromptSpec:
+    """Return family and variant prompt metadata for the given identifiers."""
+
+    family = _match_family(model_name)
+    if family is None and canonical_name:
+        family = _match_family(canonical_name)
+
+    variant = None
+    if family is not None:
+        variant = _match_variant(family, model_name, canonical_name)
+
+    return ModelPromptSpec(family=family, variant=variant)
+
+
+__all__ = ["ModelPromptSpec", "get_model_prompt_spec"]