multi-forge 0.2.0__py3-none-any.whl

forge/proxy/data_models.py

@@ -0,0 +1,338 @@
+"""Pydantic models for API request/response validation.
+
+Defines data models for the proxy API, including models for:
+- Content blocks (text, images, tool use)
+- Messages
+- API requests and responses
+- Model name mapping between Claude and Gemini
+"""
+
+import logging
+from typing import Any, Dict, List, Literal, Optional, Union
+
+from pydantic import BaseModel, Field, model_validator
+
+from forge.config import config, is_openai_model
+
+logger = logging.getLogger(__name__)
+
+
+def _detect_tier(values: dict) -> dict:
+    """Detect Claude tier (haiku/sonnet/opus) from model name in request dict.
+
+    Sets `original_model_name`, `tier`, and `has_explicit_tier` fields.
+    Used by model_validator(mode="before") on request models.
+    """
+    if isinstance(values, dict) and "model" in values:
+        model_name = values["model"]
+        values["original_model_name"] = model_name
+
+        model_lower = model_name.lower()
+        if "haiku" in model_lower:
+            values["tier"] = "haiku"
+            values["has_explicit_tier"] = True
+        elif "sonnet" in model_lower:
+            values["tier"] = "sonnet"
+            values["has_explicit_tier"] = True
+        elif "opus" in model_lower:
+            values["tier"] = "opus"
+            values["has_explicit_tier"] = True
+        else:
+            values["tier"] = None
+            values["has_explicit_tier"] = False
+
+    return values
+
+
+class CacheControl(BaseModel):
+    """Cache control directive for prompt caching (Anthropic API).
+
+    The "ephemeral" type indicates content should be cached for the session.
+    Only affects Anthropic/Bedrock models — other providers cache automatically
+    or don't support the field.
+    """
+
+    type: Literal["ephemeral"] = "ephemeral"
+
+
+class ContentBlockText(BaseModel):
+    type: Literal["text"]
+    text: str
+    cache_control: Optional[CacheControl] = None
+
+
+class ContentBlockImageSource(BaseModel):
+    type: Literal["base64"]
+    media_type: str
+    data: str
+
+
+class ContentBlockImage(BaseModel):
+    type: Literal["image"]
+    source: ContentBlockImageSource
+
+
+class ContentBlockToolUse(BaseModel):
+    type: Literal["tool_use"]
+    id: str
+    name: str
+    input: Dict[str, Any]
+
+
+class ContentBlockToolResult(BaseModel):
+    type: Literal["tool_result"]
+    tool_use_id: str
+    content: Union[str, List[Dict[str, Any]]]
+    is_error: Optional[bool] = False
+
+
+class ContentBlockThinking(BaseModel):
+    """Anthropic extended thinking block (sent in conversation history on --resume)."""
+
+    type: Literal["thinking"]
+    thinking: str = ""
+    signature: Optional[str] = None
+
+
+class ContentBlockRedactedThinking(BaseModel):
+    """Anthropic redacted thinking block (opaque, sent back for continuity)."""
+
+    type: Literal["redacted_thinking"]
+    data: str = ""
+
+
+ContentBlock = Union[
+    ContentBlockText,
+    ContentBlockImage,
+    ContentBlockToolUse,
+    ContentBlockToolResult,
+    ContentBlockThinking,
+    ContentBlockRedactedThinking,
+]
+
+
+class SystemContent(BaseModel):
+    type: Literal["text"]
+    text: str
+    cache_control: Optional[CacheControl] = None
+
+
+class Message(BaseModel):
+    role: Literal["user", "assistant"]
+    content: Union[str, List[ContentBlock]]
+
+
+class ToolInputSchema(BaseModel):
+    type: Literal["object"] = "object"
+    properties: Dict[str, Any]
+    required: Optional[List[str]] = None
+
+
+class ToolDefinition(BaseModel):
+    name: str
+    description: Optional[str] = None
+    input_schema: ToolInputSchema
+
+
+class MessagesRequest(BaseModel):
+    model: str  # Raw client-supplied model string; mapped in handler after config reload
+    messages: List[Message]
+    system: Optional[Union[str, List[SystemContent]]] = None
+    max_tokens: int = Field(ge=1)
+    metadata: Optional[Dict[str, Any]] = None
+    stop_sequences: Optional[List[str]] = None
+    stream: Optional[bool] = False
+    temperature: Optional[float] = None
+    top_p: Optional[float] = None
+    top_k: Optional[int] = None
+    # Reasoning/thinking overrides (explicit request overrides are allowed)
+    reasoning_effort: Optional[str] = None
+    verbosity: Optional[str] = None
+    thinking: Optional[Dict[str, Any]] = None
+    tools: Optional[List[ToolDefinition]] = None
+    tool_choice: Optional[Dict[str, Any]] = None
+    original_model_name: Optional[str] = None  # Internal field to store original name pre-mapping
+    tier: Optional[str] = None  # Internal field to store detected tier (haiku/sonnet/opus)
+    has_explicit_tier: bool = False  # Whether tier was explicit in model name (not defaulted)
+
+    @model_validator(mode="before")
+    @classmethod
+    def store_original_model(cls, values):
+        return _detect_tier(values)
+
+
+class TokenCountRequest(BaseModel):
+    model: str  # Raw client-supplied model string; mapped in handler after config reload
+    messages: List[Message]
+    system: Optional[Union[str, List[SystemContent]]] = None
+    original_model_name: Optional[str] = None  # Internal field
+    tier: Optional[str] = None  # Internal field to store detected tier (haiku/sonnet/opus)
+    has_explicit_tier: bool = False  # Whether tier was explicit in model name
+
+    @model_validator(mode="before")
+    @classmethod
+    def store_original_model_token_count(cls, values):
+        return _detect_tier(values)
+
+
+class TokenCountResponse(BaseModel):
+    input_tokens: int
+
+
+class Usage(BaseModel):
+    input_tokens: int
+    output_tokens: int
+
+
+class MessagesResponse(BaseModel):
+    id: str
+    type: Literal["message"] = "message"
+    role: Literal["assistant"] = "assistant"
+    model: str  # Original Anthropic model name
+    content: List[ContentBlock]
+    stop_reason: Optional[Literal["end_turn", "max_tokens", "stop_sequence", "tool_use", "content_filtered"]] = None
+    stop_sequence: Optional[str] = None
+    usage: Usage
+
+
+def map_model_name(anthropic_model_name: str) -> str:
+    """Map Anthropic model names (haiku, sonnet, opus) to backend models.
+
+    Uses unified config for model mappings. Handles:
+    - Pass-through for known backend models (openai/, vertex_ai/, gemini/)
+    - Mapping Anthropic-style names to current provider's tier equivalents
+    - Default provider fallback for ambiguous names
+
+    Returns:
+        The mapped model name for the backend provider.
+    """
+    original = anthropic_model_name
+    preferred = config.proxy.preferred_provider or None
+
+    def _normalize(name: str) -> str:
+        n = name.strip().lower().split("@", 1)[0]
+        for prefix in ("anthropic/", "openai/", "gemini/"):
+            if n.startswith(prefix):
+                n = n[len(prefix) :]
+                break
+        return n
+
+    def _anthropic_flavor(name: str) -> str | None:
+        if "haiku" in name:
+            return "haiku"
+        if "sonnet" in name:
+            return "sonnet"
+        if "opus" in name:
+            return "opus"
+        return None
+
+    def _is_openai(name: str) -> bool:
+        return is_openai_model(name)
+
+    def _is_gemini(name: str) -> bool:
+        # Use unified config for Gemini model detection
+        known = {
+            config.proxy.gemini.tiers.haiku.lower(),
+            config.proxy.gemini.tiers.sonnet.lower(),
+            config.proxy.gemini.tiers.opus.lower(),
+        }
+        return name.startswith("gemini-") or name in known
+
+    def _is_litellm(name: str) -> bool:
+        """Check if model name is a LiteLLM model (has provider prefix)."""
+        return "/" in name and any(
+            name.startswith(prefix)
+            for prefix in [
+                "openai/",
+                "anthropic/",
+                "vertex_ai/",
+                "bedrock/",
+                "replicate/",
+                "together_ai/",
+                "gemini/",  # Local LiteLLM with Google GenAI SDK
+            ]
+        )
+
+    def _get_provider_models(provider_name: str) -> dict[str, str]:
+        """Get tier->model mappings from unified config."""
+        provider = config.proxy.get_provider(provider_name)
+        return {
+            "haiku": provider.tiers.haiku,
+            "sonnet": provider.tiers.sonnet,
+            "opus": provider.tiers.opus,
+            "default": provider.tiers.sonnet,
+        }
+
+    name = _normalize(original)
+    flavor = _anthropic_flavor(name)
+
+    # OpenRouter: pass-through model IDs as-is (OpenRouter handles routing)
+    if preferred == "openrouter":
+        if "/" in original:
+            logger.info(f"Using OpenRouter model: '{original}' (pass-through)")
+            return original
+
+        # Map Anthropic flavors to OpenRouter tier models
+        provider_models = _get_provider_models("openrouter")
+        if flavor:
+            mapped = provider_models[flavor]
+            logger.info(f"Mapping '{original}' ({flavor.title()}) -> OpenRouter '{mapped}'")
+            return mapped
+
+        mapped = provider_models["default"]
+        logger.warning(f"Unknown model '{original}' with provider preference 'openrouter', defaulting to '{mapped}'")
+        return mapped
+
+    # Forced provider: symmetric handling for OpenAI, Gemini, and LiteLLM
+    if preferred in ("openai", "gemini", "litellm"):
+        target = preferred
+
+        # Pass-through if already the target provider
+        if (
+            (target == "openai" and _is_openai(name))
+            or (target == "gemini" and _is_gemini(name))
+            or (target == "litellm" and _is_litellm(original))
+        ):
+            # Return original for LiteLLM to preserve the provider prefix
+            result = original if target == "litellm" else name
+            logger.info(f"Using {target} model: '{result}' (provider preference: {target})")
+            return result
+
+        # Map Anthropic flavors to the target provider
+        provider_models = _get_provider_models(target)
+        if flavor:
+            mapped = provider_models[flavor]
+            logger.info(f"Mapping '{original}' ({flavor.title()}) -> {target.title()} '{mapped}'")
+            return mapped
+
+        # Otherwise default to target provider's default
+        mapped = provider_models["default"]
+        logger.warning(
+            f"Unknown/other model '{original}' with provider preference '{target}', defaulting to '{mapped}'"
+        )
+        return mapped
+
+    # No forced provider: pass-through known provider models
+    if _is_litellm(original):
+        logger.info(f"Detected LiteLLM model: '{original}'")
+        return original
+    if _is_openai(name):
+        logger.info(f"Detected OpenAI model: '{original}' -> '{name}'")
+        return name
+    if _is_gemini(name):
+        logger.info(f"Detected Gemini model: '{original}' -> '{name}'")
+        return name
+
+    # Anthropic or unknown: map Anthropic by flavor, else default to Gemini
+    target = "gemini"
+    provider_models = _get_provider_models(target)
+    if flavor:
+        mapped = provider_models[flavor]
+        logger.info(f"Mapping '{original}' ({flavor.title()}) -> {target.title()} '{mapped}'")
+        return mapped
+
+    # Fail-closed: reject completely unknown models rather than silently routing to default
+    raise ValueError(
+        f"Unrecognized model '{original}'. Cannot route to backend. "
+        "Check model name or configure a mapping in the proxy template."
+    )

forge/proxy/error_hints.py

@@ -0,0 +1,92 @@
+"""Error hint enrichment for client-side tool failures.
+
+Appends targeted hints to tool_result error content before forwarding
+to the LLM, helping non-Claude models recover from common mistakes.
+"""
+
+from typing import Optional
+
+# Sentinel prefix to prevent double-appending hints
+_HINT_PREFIX = "\n\nHINT: "
+
+# Each rule: (tool_name_or_None, list_of_required_substrings, hint_text)
+# tool_name=None means match any tool. First matching rule wins.
+_HINT_RULES: list[tuple[Optional[str], list[str], str]] = [
+    # Edit: no-op (old_string == new_string) -- 57% of all failures
+    (
+        "Edit",
+        ["old_string and new_string are exactly the same"],
+        "Edit requires old_string \u2260 new_string. To view code, use Read instead of Edit.",
+    ),
+    # Edit: not unique match
+    (
+        "Edit",
+        ["matches", "replace_all is false"],
+        "Include more surrounding context in old_string to uniquely identify the target, or set replace_all=true.",
+    ),
+    # Bash: ruff F401 unused import
+    (
+        "Bash",
+        ["F401", "imported but unused"],
+        "Remove the unused import(s) listed above, then retry.",
+    ),
+    # Bash: ruff F811 redefinition of unused name
+    (
+        "Bash",
+        ["F811", "redefinition of unused"],
+        "Remove the duplicate definition listed above, then retry.",
+    ),
+    # TaskOutput: hallucinated task ID
+    (
+        "TaskOutput",
+        ["No task found with ID"],
+        (
+            "Task IDs are short hex strings returned by run_in_background. "
+            "Do not append file extensions. If not found, stop retrying the same ID."
+        ),
+    ),
+    # Read: invalid pages parameter (non-PDF files)
+    (
+        "Read",
+        ["Invalid pages parameter"],
+        "pages is only for PDF files. For non-PDF files, omit pages entirely. Retry with only file_path.",
+    ),
+    # Read: file not found
+    (
+        "Read",
+        ["File does not exist"],
+        "Verify the absolute file path is correct. Use Glob to search for the file.",
+    ),
+    # --- Fallback rules (tool_name=None) for when _find_tool_name() fails ---
+    (
+        None,
+        ["old_string and new_string are exactly the same"],
+        "Edit requires old_string \u2260 new_string. To view code, use Read instead of Edit.",
+    ),
+    (
+        None,
+        ["No task found with ID"],
+        (
+            "Task IDs are short hex strings returned by run_in_background. "
+            "Do not append file extensions. If not found, stop retrying the same ID."
+        ),
+    ),
+]
+
+
+def enrich_error_content(tool_name: Optional[str], error_content: str) -> str:
+    """Append a HINT to error content if a known failure pattern matches.
+
+    First matching rule wins. Returns original content unchanged if no match.
+    """
+    if _HINT_PREFIX in error_content:
+        return error_content
+
+    for rule_tool, required_substrings, hint_text in _HINT_RULES:
+        if rule_tool is not None and tool_name != rule_tool:
+            continue
+
+        if all(substr in error_content for substr in required_substrings):
+            return error_content + _HINT_PREFIX + hint_text
+
+    return error_content

forge/proxy/metrics.py

@@ -0,0 +1,222 @@
+"""In-memory per-proxy runtime metrics.
+
+Each proxy process maintains a single ProxyMetrics instance that accumulates
+request counts, token usage (including cached and failed), and latency.
+Metrics reset on proxy restart — this is expected and correct since each
+proxy is a separate subprocess.
+
+Exposed via GET / (runtime truth endpoint) and ``forge proxy metrics`` CLI.
+"""
+
+from __future__ import annotations
+
+import threading
+import time
+from dataclasses import dataclass, field
+
+from forge.core.state import now_iso
+
+
+@dataclass
+class TierTokens:
+    """Per-tier (or per-model) token breakdown with latency tracking."""
+
+    input_tokens: int = 0
+    output_tokens: int = 0
+    cached_tokens: int = 0
+    total_latency_ms: float = 0.0
+    request_count: int = 0  # for avg latency (separate from requests_by_tier for reset clarity)
+    estimated_cost_micros: int = 0  # microdollars (1 USD = 1_000_000)
+
+    def to_dict(self) -> dict[str, object]:
+        avg = round(self.total_latency_ms / self.request_count, 1) if self.request_count > 0 else 0.0
+        return {
+            "input_tokens": self.input_tokens,
+            "output_tokens": self.output_tokens,
+            "cached_tokens": self.cached_tokens,
+            "avg_latency_ms": avg,
+            "estimated_cost_usd": round(self.estimated_cost_micros / 1_000_000, 6),
+            "estimated_cost_micros": self.estimated_cost_micros,
+        }
+
+
+@dataclass
+class ProxyMetrics:
+    """Thread-safe in-memory metrics for a single proxy process.
+
+    All counter updates go through ``record_request()`` under a single lock.
+    The lock hold time is microseconds (dict increments only), so contention
+    with uvicorn's async event loop or thread pool workers is negligible.
+    """
+
+    # Timestamps
+    started_at: str = field(default_factory=now_iso)
+    _started_mono: float = field(default_factory=time.monotonic)
+
+    # Counters
+    total_requests: int = 0
+    total_streaming: int = 0
+    total_failures: int = 0
+
+    # Token accounting (success + failure)
+    total_input_tokens: int = 0
+    total_output_tokens: int = 0
+    total_cached_tokens: int = 0
+
+    # Failed request tokens (wasted spend)
+    failed_input_tokens: int = 0
+    failed_output_tokens: int = 0
+
+    # Cost estimates (microdollars, 1 USD = 1_000_000)
+    total_cost_micros: int = 0
+    failed_cost_micros: int = 0
+
+    # Per-tier breakdown
+    requests_by_tier: dict[str, int] = field(default_factory=dict)
+    tokens_by_tier: dict[str, TierTokens] = field(default_factory=dict)
+
+    # Per-model breakdown (actual_model_id, for cost comparison)
+    requests_by_model: dict[str, int] = field(default_factory=dict)
+    tokens_by_model: dict[str, TierTokens] = field(default_factory=dict)
+
+    # Failure classification (error_type, not HTTP status — streaming is always 200)
+    failures_by_type: dict[str, int] = field(default_factory=dict)
+
+    # Activity
+    last_request_at: str | None = None
+
+    # Lock
+    _lock: threading.Lock = field(default_factory=threading.Lock, repr=False)
+
+    def record_request(
+        self,
+        *,
+        tier: str,
+        model: str,
+        input_tokens: int,
+        output_tokens: int,
+        cached_tokens: int,
+        latency_ms: float,
+        streaming: bool,
+        failed: bool,
+        error_type: str | None = None,
+        cost_micros: int = 0,
+    ) -> None:
+        """Record a completed request. All fields updated atomically under lock."""
+        with self._lock:
+            self.total_requests += 1
+            if streaming:
+                self.total_streaming += 1
+
+            # Tokens (always, success + failure)
+            self.total_input_tokens += input_tokens
+            self.total_output_tokens += output_tokens
+            self.total_cached_tokens += cached_tokens
+
+            # Cost
+            self.total_cost_micros += cost_micros
+
+            # Per-tier
+            self.requests_by_tier[tier] = self.requests_by_tier.get(tier, 0) + 1
+            tier_tokens = self.tokens_by_tier.get(tier)
+            if tier_tokens is None:
+                tier_tokens = TierTokens()
+                self.tokens_by_tier[tier] = tier_tokens
+            tier_tokens.input_tokens += input_tokens
+            tier_tokens.output_tokens += output_tokens
+            tier_tokens.cached_tokens += cached_tokens
+            tier_tokens.total_latency_ms += latency_ms
+            tier_tokens.request_count += 1
+            tier_tokens.estimated_cost_micros += cost_micros
+
+            # Per-model
+            self.requests_by_model[model] = self.requests_by_model.get(model, 0) + 1
+            model_tokens = self.tokens_by_model.get(model)
+            if model_tokens is None:
+                model_tokens = TierTokens()
+                self.tokens_by_model[model] = model_tokens
+            model_tokens.input_tokens += input_tokens
+            model_tokens.output_tokens += output_tokens
+            model_tokens.cached_tokens += cached_tokens
+            model_tokens.total_latency_ms += latency_ms
+            model_tokens.request_count += 1
+            model_tokens.estimated_cost_micros += cost_micros
+
+            # Failures
+            if failed:
+                self.total_failures += 1
+                self.failed_input_tokens += input_tokens
+                self.failed_output_tokens += output_tokens
+                self.failed_cost_micros += cost_micros
+                if error_type:
+                    self.failures_by_type[error_type] = self.failures_by_type.get(error_type, 0) + 1
+
+            # Activity
+            self.last_request_at = now_iso()
+
+    def snapshot(self) -> dict:
+        """Return a JSON-serializable dict of all metrics plus derived values."""
+        with self._lock:
+            total = self.total_requests
+            uptime = time.monotonic() - self._started_mono
+
+            return {
+                "started_at": self.started_at,
+                "uptime_seconds": round(uptime, 1),
+                "total_requests": total,
+                "total_streaming": self.total_streaming,
+                "total_failures": self.total_failures,
+                "tokens": {
+                    "input": self.total_input_tokens,
+                    "output": self.total_output_tokens,
+                    "cached": self.total_cached_tokens,
+                    "failed_input": self.failed_input_tokens,
+                    "failed_output": self.failed_output_tokens,
+                },
+                "cache_hit_rate": (
+                    round(self.total_cached_tokens / self.total_input_tokens * 100, 1)
+                    if self.total_input_tokens > 0
+                    else 0.0
+                ),
+                "by_tier": {
+                    tier: {"requests": self.requests_by_tier.get(tier, 0), **tokens.to_dict()}
+                    for tier, tokens in self.tokens_by_tier.items()
+                },
+                "by_model": {
+                    model: {"requests": self.requests_by_model.get(model, 0), **tokens.to_dict()}
+                    for model, tokens in self.tokens_by_model.items()
+                },
+                "failures_by_type": dict(self.failures_by_type),
+                "costs": {
+                    "total_usd": round(self.total_cost_micros / 1_000_000, 6),
+                    "failed_usd": round(self.failed_cost_micros / 1_000_000, 6),
+                    "total_micros": self.total_cost_micros,
+                    "failed_micros": self.failed_cost_micros,
+                },
+                "last_request_at": self.last_request_at,
+            }
+
+    def reset(self) -> None:
+        """Zero all counters. Preserves started_at for uptime. For test isolation."""
+        with self._lock:
+            self.total_requests = 0
+            self.total_streaming = 0
+            self.total_failures = 0
+            self.total_input_tokens = 0
+            self.total_output_tokens = 0
+            self.total_cached_tokens = 0
+            self.failed_input_tokens = 0
+            self.failed_output_tokens = 0
+            self.total_cost_micros = 0
+            self.failed_cost_micros = 0
+            self.requests_by_tier.clear()
+            self.tokens_by_tier.clear()
+            self.requests_by_model.clear()
+            self.tokens_by_model.clear()
+            self.failures_by_type.clear()
+            self.last_request_at = None
+
+
+# Module-level singleton — one per proxy process.
+# Matches existing patterns: client_factory and PROXY_ID in server.py are also module globals.
+proxy_metrics = ProxyMetrics()