openhands 0.0.0__py3-none-any.whl → 1.0.1__py3-none-any.whl

openhands/sdk/llm/utils/metrics.py

@@ -1,311 +0,0 @@
-import copy
-import time
-from typing import Optional
-
-from pydantic import BaseModel, Field, field_validator, model_validator
-
-
-class Cost(BaseModel):
-    model: str
-    cost: float = Field(ge=0.0, description="Cost must be non-negative")
-    timestamp: float = Field(default_factory=time.time)
-
-    @field_validator("cost")
-    @classmethod
-    def validate_cost(cls, v: float) -> float:
-        if v < 0:
-            raise ValueError("Cost cannot be negative")
-        return v
-
-
-class ResponseLatency(BaseModel):
-    """Metric tracking the round-trip time per completion call."""
-
-    model: str
-    latency: float = Field(ge=0.0, description="Latency must be non-negative")
-    response_id: str
-
-    @field_validator("latency")
-    @classmethod
-    def validate_latency(cls, v: float) -> float:
-        return max(0.0, v)
-
-
-class TokenUsage(BaseModel):
-    """Metric tracking detailed token usage per completion call."""
-
-    model: str = Field(default="")
-    prompt_tokens: int = Field(
-        default=0, ge=0, description="Prompt tokens must be non-negative"
-    )
-    completion_tokens: int = Field(
-        default=0, ge=0, description="Completion tokens must be non-negative"
-    )
-    cache_read_tokens: int = Field(
-        default=0, ge=0, description="Cache read tokens must be non-negative"
-    )
-    cache_write_tokens: int = Field(
-        default=0, ge=0, description="Cache write tokens must be non-negative"
-    )
-    reasoning_tokens: int = Field(
-        default=0, ge=0, description="Reasoning tokens must be non-negative"
-    )
-    context_window: int = Field(
-        default=0, ge=0, description="Context window must be non-negative"
-    )
-    per_turn_token: int = Field(
-        default=0, ge=0, description="Per turn tokens must be non-negative"
-    )
-    response_id: str = Field(default="")
-
-    def __add__(self, other: "TokenUsage") -> "TokenUsage":
-        """Add two TokenUsage instances together."""
-        return TokenUsage(
-            model=self.model,
-            prompt_tokens=self.prompt_tokens + other.prompt_tokens,
-            completion_tokens=self.completion_tokens + other.completion_tokens,
-            cache_read_tokens=self.cache_read_tokens + other.cache_read_tokens,
-            cache_write_tokens=self.cache_write_tokens + other.cache_write_tokens,
-            reasoning_tokens=self.reasoning_tokens + other.reasoning_tokens,
-            context_window=max(self.context_window, other.context_window),
-            per_turn_token=other.per_turn_token,
-            response_id=self.response_id,
-        )
-
-
-class MetricsSnapshot(BaseModel):
-    """A snapshot of metrics at a point in time.
-
-    Does not include lists of individual costs, latencies, or token usages.
-    """
-
-    model_name: str = Field(default="default", description="Name of the model")
-    accumulated_cost: float = Field(
-        default=0.0, ge=0.0, description="Total accumulated cost, must be non-negative"
-    )
-    max_budget_per_task: Optional[float] = Field(
-        default=None, description="Maximum budget per task"
-    )
-    accumulated_token_usage: Optional[TokenUsage] = Field(
-        default=None, description="Accumulated token usage across all calls"
-    )
-
-
-class Metrics(MetricsSnapshot):
-    """Metrics class can record various metrics during running and evaluation.
-    We track:
-      - accumulated_cost and costs
-      - max_budget_per_task (budget limit)
-      - A list of ResponseLatency
-      - A list of TokenUsage (one per call).
-    """
-
-    costs: list[Cost] = Field(
-        default_factory=list, description="List of individual costs"
-    )
-    response_latencies: list[ResponseLatency] = Field(
-        default_factory=list, description="List of response latencies"
-    )
-    token_usages: list[TokenUsage] = Field(
-        default_factory=list, description="List of token usage records"
-    )
-
-    @field_validator("accumulated_cost")
-    @classmethod
-    def validate_accumulated_cost(cls, v: float) -> float:
-        if v < 0:
-            raise ValueError("Total cost cannot be negative.")
-        return v
-
-    @model_validator(mode="after")
-    def initialize_accumulated_token_usage(self) -> "Metrics":
-        if self.accumulated_token_usage is None:
-            self.accumulated_token_usage = TokenUsage(
-                model=self.model_name,
-                prompt_tokens=0,
-                completion_tokens=0,
-                cache_read_tokens=0,
-                cache_write_tokens=0,
-                reasoning_tokens=0,
-                context_window=0,
-                response_id="",
-            )
-        return self
-
-    def get_snapshot(self) -> MetricsSnapshot:
-        """Get a snapshot of the current metrics without the detailed lists."""
-        return MetricsSnapshot(
-            model_name=self.model_name,
-            accumulated_cost=self.accumulated_cost,
-            max_budget_per_task=self.max_budget_per_task,
-            accumulated_token_usage=copy.deepcopy(self.accumulated_token_usage)
-            if self.accumulated_token_usage
-            else None,
-        )
-
-    def add_cost(self, value: float) -> None:
-        if value < 0:
-            raise ValueError("Added cost cannot be negative.")
-        self.accumulated_cost += value
-        self.costs.append(Cost(cost=value, model=self.model_name))
-
-    def add_response_latency(self, value: float, response_id: str) -> None:
-        self.response_latencies.append(
-            ResponseLatency(
-                latency=max(0.0, value), model=self.model_name, response_id=response_id
-            )
-        )
-
-    def add_token_usage(
-        self,
-        prompt_tokens: int,
-        completion_tokens: int,
-        cache_read_tokens: int,
-        cache_write_tokens: int,
-        context_window: int,
-        response_id: str,
-        reasoning_tokens: int = 0,
-    ) -> None:
-        """Add a single usage record."""
-        # Token each turn for calculating context usage.
-        per_turn_token = prompt_tokens + completion_tokens
-
-        usage = TokenUsage(
-            model=self.model_name,
-            prompt_tokens=prompt_tokens,
-            completion_tokens=completion_tokens,
-            cache_read_tokens=cache_read_tokens,
-            cache_write_tokens=cache_write_tokens,
-            reasoning_tokens=reasoning_tokens,
-            context_window=context_window,
-            per_turn_token=per_turn_token,
-            response_id=response_id,
-        )
-        self.token_usages.append(usage)
-
-        # Update accumulated token usage using the __add__ operator
-        new_usage = TokenUsage(
-            model=self.model_name,
-            prompt_tokens=prompt_tokens,
-            completion_tokens=completion_tokens,
-            cache_read_tokens=cache_read_tokens,
-            cache_write_tokens=cache_write_tokens,
-            reasoning_tokens=reasoning_tokens,
-            context_window=context_window,
-            per_turn_token=per_turn_token,
-            response_id="",
-        )
-        if self.accumulated_token_usage is None:
-            self.accumulated_token_usage = new_usage
-        else:
-            self.accumulated_token_usage = self.accumulated_token_usage + new_usage
-
-    def merge(self, other: "Metrics") -> None:
-        """Merge 'other' metrics into this one."""
-        self.accumulated_cost += other.accumulated_cost
-
-        # Keep the max_budget_per_task from other if it's set and this one isn't
-        if self.max_budget_per_task is None and other.max_budget_per_task is not None:
-            self.max_budget_per_task = other.max_budget_per_task
-
-        self.costs += other.costs
-        self.token_usages += other.token_usages
-        self.response_latencies += other.response_latencies
-
-        # Merge accumulated token usage using the __add__ operator
-        if self.accumulated_token_usage is None:
-            self.accumulated_token_usage = other.accumulated_token_usage
-        elif other.accumulated_token_usage is not None:
-            self.accumulated_token_usage = (
-                self.accumulated_token_usage + other.accumulated_token_usage
-            )
-
-    def get(self) -> dict:
-        """Return the metrics in a dictionary."""
-        return {
-            "accumulated_cost": self.accumulated_cost,
-            "max_budget_per_task": self.max_budget_per_task,
-            "accumulated_token_usage": self.accumulated_token_usage.model_dump()
-            if self.accumulated_token_usage
-            else None,
-            "costs": [cost.model_dump() for cost in self.costs],
-            "response_latencies": [
-                latency.model_dump() for latency in self.response_latencies
-            ],
-            "token_usages": [usage.model_dump() for usage in self.token_usages],
-        }
-
-    def log(self) -> str:
-        """Log the metrics."""
-        metrics = self.get()
-        logs = ""
-        for key, value in metrics.items():
-            logs += f"{key}: {value}\n"
-        return logs
-
-    def deep_copy(self) -> "Metrics":
-        """Create a deep copy of the Metrics object."""
-        return copy.deepcopy(self)
-
-    def diff(self, baseline: "Metrics") -> "Metrics":
-        """Calculate the difference between current metrics and a baseline.
-
-        This is useful for tracking metrics for specific operations like delegates.
-
-        Args:
-            baseline: A metrics object representing the baseline state
-
-        Returns:
-            A new Metrics object containing only the differences since the baseline
-        """
-        result = Metrics(model_name=self.model_name)
-
-        # Calculate cost difference
-        result.accumulated_cost = self.accumulated_cost - baseline.accumulated_cost
-
-        # Include only costs that were added after the baseline
-        if baseline.costs:
-            last_baseline_timestamp = baseline.costs[-1].timestamp
-            result.costs = [
-                cost for cost in self.costs if cost.timestamp > last_baseline_timestamp
-            ]
-        else:
-            result.costs = self.costs.copy()
-
-        # Include only response latencies that were added after the baseline
-        result.response_latencies = self.response_latencies[
-            len(baseline.response_latencies) :
-        ]
-
-        # Include only token usages that were added after the baseline
-        result.token_usages = self.token_usages[len(baseline.token_usages) :]
-
-        # Calculate accumulated token usage difference
-        base_usage = baseline.accumulated_token_usage
-        current_usage = self.accumulated_token_usage
-
-        if current_usage is not None and base_usage is not None:
-            result.accumulated_token_usage = TokenUsage(
-                model=self.model_name,
-                prompt_tokens=current_usage.prompt_tokens - base_usage.prompt_tokens,
-                completion_tokens=current_usage.completion_tokens
-                - base_usage.completion_tokens,
-                cache_read_tokens=current_usage.cache_read_tokens
-                - base_usage.cache_read_tokens,
-                cache_write_tokens=current_usage.cache_write_tokens
-                - base_usage.cache_write_tokens,
-                reasoning_tokens=current_usage.reasoning_tokens
-                - base_usage.reasoning_tokens,
-                context_window=current_usage.context_window,
-                per_turn_token=0,
-                response_id="",
-            )
-        elif current_usage is not None:
-            result.accumulated_token_usage = current_usage
-        else:
-            result.accumulated_token_usage = None
-
-        return result
-
-    def __repr__(self) -> str:
-        return f"Metrics({self.get()}"

openhands/sdk/llm/utils/model_features.py

@@ -1,153 +0,0 @@
-from dataclasses import dataclass
-from fnmatch import fnmatch
-
-
-def normalize_model_name(model: str) -> str:
-    """Normalize a model string to a canonical, comparable name.
-
-    Strategy:
-    - Trim whitespace
-    - Lowercase
-    - If there is a '/', keep only the basename after the last '/'
-      (handles prefixes like openrouter/, litellm_proxy/, anthropic/, etc.)
-      and treat ':' inside that basename as an Ollama-style variant tag to be removed
-    - There is no provider:model form; providers, when present, use 'provider/model'
-    - Drop a trailing "-gguf" suffix if present
-    - If basename starts with a known vendor prefix followed by '.', drop that prefix
-      (e.g., 'anthropic.claude-*' -> 'claude-*')
-    """
-    raw = (model or "").strip().lower()
-    if "/" in raw:
-        name = raw.split("/")[-1]
-        if ":" in name:
-            # Drop Ollama-style variant tag in basename
-            name = name.split(":", 1)[0]
-    else:
-        # No '/', keep the whole raw name (we do not support provider:model)
-        name = raw
-
-    # Drop common vendor prefixes embedded in the basename (bedrock style), once.
-    # Keep this list small and explicit to avoid accidental over-matching.
-    vendor_prefixes = {
-        "anthropic",
-        "meta",
-        "cohere",
-        "mistral",
-        "ai21",
-        "amazon",
-    }
-    if "." in name:
-        vendor, rest = name.split(".", 1)
-        if vendor in vendor_prefixes and rest:
-            name = rest
-
-    if name.endswith("-gguf"):
-        name = name[: -len("-gguf")]
-    return name
-
-
-def model_matches(model: str, patterns: list[str]) -> bool:
-    """Return True if the model matches any of the glob patterns.
-
-    If a pattern contains a '/', it is treated as provider-qualified and matched
-    against the full, lowercased model string (including provider prefix).
-    Otherwise, it is matched against the normalized basename.
-    """
-    raw = (model or "").strip().lower()
-    name = normalize_model_name(model)
-    for pat in patterns:
-        pat_l = pat.lower()
-        if "/" in pat_l:
-            if fnmatch(raw, pat_l):
-                return True
-        else:
-            if fnmatch(name, pat_l):
-                return True
-    return False
-
-
-@dataclass(frozen=True)
-class ModelFeatures:
-    supports_function_calling: bool
-    supports_reasoning_effort: bool
-    supports_prompt_cache: bool
-    supports_stop_words: bool
-
-
-# Pattern tables capturing current behavior. Keep patterns lowercase.
-FUNCTION_CALLING_PATTERNS: list[str] = [
-    # Anthropic families
-    "claude-3-7-sonnet*",
-    "claude-3.7-sonnet*",
-    "claude-sonnet-3-7-latest",
-    "claude-3-5-sonnet*",
-    "claude-3.5-haiku*",
-    "claude-3-5-haiku*",
-    "claude-sonnet-4*",
-    "claude-opus-4*",
-    # OpenAI families
-    "gpt-4o*",
-    "gpt-4.1",
-    "gpt-5*",
-    # o-series (keep exact o1 support per existing list)
-    "o1-2024-12-17",
-    "o3*",
-    "o4-mini*",
-    # Google Gemini
-    "gemini-2.5-pro*",
-    # Others
-    "kimi-k2-0711-preview",
-    "kimi-k2-instruct",
-    "qwen3-coder*",
-    "qwen3-coder-480b-a35b-instruct",
-]
-
-REASONING_EFFORT_PATTERNS: list[str] = [
-    # Mirror main behavior exactly (no unintended expansion)
-    "o1-2024-12-17",
-    "o1",
-    "o3",
-    "o3-2025-04-16",
-    "o3-mini-2025-01-31",
-    "o3-mini",
-    "o4-mini",
-    "o4-mini-2025-04-16",
-    "gemini-2.5-flash",
-    "gemini-2.5-pro",
-    "gpt-5",
-    "gpt-5-2025-08-07",
-]
-
-PROMPT_CACHE_PATTERNS: list[str] = [
-    "claude-3-7-sonnet*",
-    "claude-3.7-sonnet*",
-    "claude-sonnet-3-7-latest",
-    "claude-3-5-sonnet*",
-    "claude-3-5-haiku*",
-    "claude-3.5-haiku*",
-    "claude-3-haiku-20240307*",
-    "claude-3-opus-20240229*",
-    "claude-sonnet-4*",
-    "claude-opus-4*",
-]
-
-SUPPORTS_STOP_WORDS_FALSE_PATTERNS: list[str] = [
-    # o1 family doesn't support stop words
-    "o1*",
-    # grok-4 specific model name (basename)
-    "grok-4-0709",
-    "grok-code-fast-1",
-    # DeepSeek R1 family
-    "deepseek-r1-0528*",
-]
-
-
-def get_features(model: str) -> ModelFeatures:
-    return ModelFeatures(
-        supports_function_calling=model_matches(model, FUNCTION_CALLING_PATTERNS),
-        supports_reasoning_effort=model_matches(model, REASONING_EFFORT_PATTERNS),
-        supports_prompt_cache=model_matches(model, PROMPT_CACHE_PATTERNS),
-        supports_stop_words=not model_matches(
-            model, SUPPORTS_STOP_WORDS_FALSE_PATTERNS
-        ),
-    )

openhands/sdk/llm/utils/retry_mixin.py

@@ -1,122 +0,0 @@
-from typing import Any, Callable, Iterable, cast
-
-from tenacity import (
-    RetryCallState,
-    retry,
-    retry_if_exception_type,
-    stop_after_attempt,
-    wait_exponential,
-)
-
-from openhands.sdk.llm.exceptions import LLMNoResponseError
-from openhands.sdk.logger import get_logger
-
-
-logger = get_logger(__name__)
-
-# Helpful alias for listener signature: (attempt_number, max_retries) -> None
-RetryListener = Callable[[int, int], None]
-
-
-class RetryMixin:
-    """Mixin class for retry logic."""
-
-    def retry_decorator(
-        self,
-        num_retries: int = 5,
-        retry_exceptions: tuple[type[BaseException], ...] = (LLMNoResponseError,),
-        retry_min_wait: int = 8,
-        retry_max_wait: int = 64,
-        retry_multiplier: float = 2.0,
-        retry_listener: RetryListener | None = None,
-    ) -> Callable[[Callable[..., Any]], Callable[..., Any]]:
-        """
-        Create a LLM retry decorator with customizable parameters.
-        This is used for 429 errors, and a few other exceptions in LLM classes.
-        """
-
-        def before_sleep(retry_state: RetryCallState) -> None:
-            # Log first (also validates outcome as part of logging)
-            self.log_retry_attempt(retry_state)
-
-            if retry_listener is not None:
-                retry_listener(retry_state.attempt_number, num_retries)
-
-            # If there is no outcome or no exception, nothing to tweak.
-            if retry_state.outcome is None:
-                return
-            exc = retry_state.outcome.exception()
-            if exc is None:
-                return
-
-            # Only adjust temperature for LLMNoResponseError
-            if isinstance(exc, LLMNoResponseError):
-                kwargs = getattr(retry_state, "kwargs", None)
-                if isinstance(kwargs, dict):
-                    current_temp = kwargs.get("temperature", 0)
-                    if current_temp == 0:
-                        kwargs["temperature"] = 1.0
-                        logger.warning(
-                            "LLMNoResponseError with temperature=0, "
-                            "setting temperature to 1.0 for next attempt."
-                        )
-                    else:
-                        logger.warning(
-                            f"LLMNoResponseError with temperature={current_temp}, "
-                            "keeping original temperature"
-                        )
-
-        retry_decorator: Callable[[Callable[..., Any]], Callable[..., Any]] = retry(
-            before_sleep=before_sleep,
-            stop=stop_after_attempt(num_retries),
-            reraise=True,
-            retry=retry_if_exception_type(retry_exceptions),
-            wait=wait_exponential(
-                multiplier=retry_multiplier,
-                min=retry_min_wait,
-                max=retry_max_wait,
-            ),
-        )
-        return retry_decorator
-
-    def log_retry_attempt(self, retry_state: RetryCallState) -> None:
-        """Log retry attempts."""
-
-        if retry_state.outcome is None:
-            logger.error(
-                "retry_state.outcome is None. "
-                "This should not happen, please check the retry logic."
-            )
-            return
-
-        exc = retry_state.outcome.exception()
-        if exc is None:
-            logger.error("retry_state.outcome.exception() returned None.")
-            return
-
-        # Try to get max attempts from the stop condition if present
-        max_attempts: int | None = None
-        retry_obj = getattr(retry_state, "retry_object", None)
-        stop_condition = getattr(retry_obj, "stop", None)
-        if stop_condition is not None:
-            # stop_any has .stops, single stop does not
-            stops: Iterable[Any]
-            if hasattr(stop_condition, "stops"):
-                stops = stop_condition.stops  # type: ignore[attr-defined]
-            else:
-                stops = [stop_condition]
-            for stop_func in stops:
-                if hasattr(stop_func, "max_attempts"):
-                    max_attempts = getattr(stop_func, "max_attempts")
-                    break
-
-        # Attach dynamic fields for downstream consumers (keep existing behavior)
-        setattr(cast(Any, exc), "retry_attempt", retry_state.attempt_number)
-        if max_attempts is not None:
-            setattr(cast(Any, exc), "max_retries", max_attempts)
-
-        logger.error(
-            "%s. Attempt #%d | You can customize retry values in the configuration.",
-            exc,
-            retry_state.attempt_number,
-        )