hud-python 0.4.45__py3-none-any.whl → 0.5.1__py3-none-any.whl

hud/environment/utils/formats.py

@@ -0,0 +1,215 @@
+"""Tool format parsing and conversion for OpenAI, Claude, Gemini, and MCP."""
+
+from __future__ import annotations
+
+import json
+from enum import Enum, auto
+from typing import Any
+
+from hud.types import MCPToolCall, MCPToolResult
+
+__all__ = [
+    "ToolFormat",
+    "format_result",
+    "parse_tool_call",
+    "parse_tool_calls",
+    "result_to_string",
+]
+
+
+class ToolFormat(Enum):
+    """Detected tool call format."""
+
+    OPENAI = auto()  # function.arguments as JSON string
+    CLAUDE = auto()  # type="tool_use", input as dict
+    GEMINI = auto()  # functionCall with args
+    MCP = auto()  # name + arguments
+
+
+# -----------------------------------------------------------------------------
+# Parsing
+# -----------------------------------------------------------------------------
+
+
+def _to_dict(obj: Any) -> dict[str, Any]:
+    """Convert object to dict for uniform processing."""
+    if isinstance(obj, dict):
+        return obj
+    if hasattr(obj, "model_dump"):
+        return obj.model_dump()
+    if hasattr(obj, "__dict__"):
+        return vars(obj)
+    raise ValueError(f"Cannot convert {type(obj).__name__} to dict")
+
+
+def _parse_json_args(args: Any) -> dict[str, Any]:
+    """Parse arguments, handling JSON strings."""
+    if not args:
+        return {}
+    if isinstance(args, str):
+        try:
+            return json.loads(args)
+        except json.JSONDecodeError:
+            return {}
+    return args
+
+
+def parse_tool_call(call: Any, **kwargs: Any) -> tuple[MCPToolCall, ToolFormat]:
+    """Parse any tool call format into (MCPToolCall, ToolFormat).
+
+    Supports:
+        - String (tool name only, or with kwargs)
+        - Tuple: (name,), (name, args), (name, args, id)
+        - MCPToolCall
+        - OpenAI: {function: {name, arguments}, id}
+        - Claude: {type: "tool_use", name, input, id}
+        - Gemini: {functionCall: {name, args}} or {name, args}
+        - Generic: {name, arguments}
+
+    Args:
+        call: Tool call in any supported format.
+        **kwargs: Additional arguments (merged when call is a string).
+
+    Returns:
+        Tuple of (MCPToolCall, ToolFormat) for the parsed call.
+
+    Raises:
+        ValueError: If format is unrecognized.
+    """
+    # Primitives
+    if isinstance(call, str):
+        return MCPToolCall(name=call, arguments=kwargs or {}), ToolFormat.MCP
+
+    if isinstance(call, tuple):
+        tc = MCPToolCall(name=call[0], arguments=call[1] if len(call) > 1 else {})
+        if len(call) > 2:
+            tc.id = call[2]
+        return tc, ToolFormat.MCP
+
+    if isinstance(call, MCPToolCall):
+        return call, ToolFormat.MCP
+
+    # Convert to dict
+    d = _to_dict(call)
+
+    # OpenAI: {function: {name, arguments}, id}
+    if "function" in d:
+        f = _to_dict(d["function"]) if not isinstance(d["function"], dict) else d["function"]
+        tc = MCPToolCall(name=f["name"], arguments=_parse_json_args(f.get("arguments")))
+        if d.get("id"):
+            tc.id = d["id"]
+        return tc, ToolFormat.OPENAI
+
+    # Claude: {type: "tool_use", name, input, id}
+    if d.get("type") == "tool_use":
+        tc = MCPToolCall(name=d["name"], arguments=d.get("input") or {})
+        if d.get("id"):
+            tc.id = d["id"]
+        return tc, ToolFormat.CLAUDE
+
+    # Gemini: {functionCall: {name, args}} or {name, args}
+    if "functionCall" in d:
+        fc = d["functionCall"]
+        return MCPToolCall(name=fc["name"], arguments=fc.get("args") or {}), ToolFormat.GEMINI
+
+    if "args" in d and "name" in d and "arguments" not in d:
+        return MCPToolCall(name=d["name"], arguments=d.get("args") or {}), ToolFormat.GEMINI
+
+    # Generic: {name, arguments/input}
+    if "name" in d:
+        tc = MCPToolCall(name=d["name"], arguments=d.get("arguments") or d.get("input") or {})
+        if d.get("id"):
+            tc.id = d["id"]
+        return tc, ToolFormat.MCP
+
+    raise ValueError(f"Unrecognized tool call format: {list(d.keys())}")
+
+
+def _is_tool_block(item: Any) -> bool:
+    """Check if item is a tool call (not text/other content)."""
+    t = item.get("type") if isinstance(item, dict) else getattr(item, "type", None)
+    return t is None or t in ("tool_use", "function")
+
+
+def parse_tool_calls(calls: Any) -> list[tuple[MCPToolCall, ToolFormat]]:
+    """Parse multiple tool calls, filtering non-tool content (e.g. Claude TextBlock).
+
+    Args:
+        calls: Single call or list of calls in any format.
+
+    Returns:
+        List of (MCPToolCall, ToolFormat) tuples.
+    """
+    if calls is None:
+        return []
+    if not isinstance(calls, list):
+        try:
+            return [parse_tool_call(calls)]
+        except ValueError:
+            return []
+
+    results = []
+    for item in calls:
+        if not _is_tool_block(item):
+            continue
+        try:
+            results.append(parse_tool_call(item))
+        except ValueError:
+            continue
+    return results
+
+
+# -----------------------------------------------------------------------------
+# Result Formatting
+# -----------------------------------------------------------------------------
+
+
+def result_to_string(result: MCPToolResult) -> str:
+    """Convert MCPToolResult content to string.
+
+    Args:
+        result: MCP tool result with content blocks.
+
+    Returns:
+        String representation of the result content.
+    """
+    if not result.content:
+        return ""
+    parts = []
+    for block in result.content:
+        if (text := getattr(block, "text", None)) is not None:
+            parts.append(str(text))
+        elif (data := getattr(block, "data", None)) is not None:
+            parts.append(f"[binary: {len(data)} bytes]")
+    return "\n".join(parts)
+
+
+def format_result(result: MCPToolResult, tc: MCPToolCall, fmt: ToolFormat) -> Any:
+    """Format MCPToolResult based on the input format.
+
+    Args:
+        result: MCP tool result.
+        tc: Original tool call (for id/name).
+        fmt: Target format.
+
+    Returns:
+        OpenAI: {"role": "tool", "tool_call_id": ..., "content": ...}
+        Claude: {"type": "tool_result", "tool_use_id": ..., "content": ..., "is_error"?: bool}
+        Gemini: {"functionResponse": {"name": ..., "response": {"result": ...}}}
+        MCP: MCPToolResult unchanged
+    """
+    content = result_to_string(result)
+
+    if fmt == ToolFormat.OPENAI:
+        return {"role": "tool", "tool_call_id": tc.id, "content": content}
+
+    if fmt == ToolFormat.CLAUDE:
+        r: dict[str, Any] = {"type": "tool_result", "tool_use_id": tc.id, "content": content}
+        if result.isError:
+            r["is_error"] = True
+        return r
+
+    if fmt == ToolFormat.GEMINI:
+        return {"functionResponse": {"name": tc.name, "response": {"result": content}}}
+
+    return result  # MCP format - return as-is

hud/environment/utils/schema.py

@@ -0,0 +1,171 @@
+"""Schema utilities for tool definitions."""
+
+from __future__ import annotations
+
+import logging
+from typing import Any
+
+__all__ = [
+    "ensure_strict_schema",
+    "json_type_to_python",
+    "schema_to_pydantic",
+    "validate_openai_schema",
+]
+
+logger = logging.getLogger(__name__)
+
+
+def ensure_strict_schema(schema: dict[str, Any]) -> dict[str, Any]:
+    """Ensure a JSON schema is compatible with OpenAI's strict mode.
+
+    OpenAI strict mode requires:
+    - additionalProperties: false on all objects
+    - All properties must be in required
+
+    Args:
+        schema: Original JSON schema.
+
+    Returns:
+        Modified schema for strict mode.
+    """
+    schema = dict(schema)
+
+    if schema.get("type") == "object":
+        schema["additionalProperties"] = False
+
+        if "properties" in schema:
+            # All properties must be required
+            schema["required"] = list(schema["properties"].keys())
+
+            # Recursively process nested objects
+            for prop_schema in schema["properties"].values():
+                if isinstance(prop_schema, dict):
+                    _ensure_strict_recursive(prop_schema)
+
+    return schema
+
+
+def _ensure_strict_recursive(schema: dict[str, Any]) -> None:
+    """Recursively apply strict mode to nested schemas."""
+    if schema.get("type") == "object":
+        schema["additionalProperties"] = False
+        if "properties" in schema:
+            schema["required"] = list(schema["properties"].keys())
+            for prop_schema in schema["properties"].values():
+                if isinstance(prop_schema, dict):
+                    _ensure_strict_recursive(prop_schema)
+
+    elif schema.get("type") == "array" and "items" in schema:
+        if isinstance(schema["items"], dict):
+            _ensure_strict_recursive(schema["items"])
+
+
+def schema_to_pydantic(name: str, schema: dict[str, Any]) -> type:
+    """Convert JSON schema to a Pydantic model.
+
+    Args:
+        name: Model name (used for class name).
+        schema: JSON schema with properties.
+
+    Returns:
+        Dynamically created Pydantic model class.
+    """
+    from pydantic import Field, create_model
+
+    properties = schema.get("properties", {})
+    required = set(schema.get("required", []))
+
+    fields = {}
+    for prop_name, prop_schema in properties.items():
+        prop_type = json_type_to_python(prop_schema.get("type", "string"))
+        default = ... if prop_name in required else None
+        description = prop_schema.get("description", "")
+        fields[prop_name] = (prop_type, Field(default=default, description=description))
+
+    return create_model(f"{name}Input", **fields)
+
+
+def json_type_to_python(json_type: str) -> type:
+    """Map JSON schema type to Python type.
+
+    Args:
+        json_type: JSON schema type string.
+
+    Returns:
+        Corresponding Python type.
+    """
+    mapping = {
+        "string": str,
+        "integer": int,
+        "number": float,
+        "boolean": bool,
+        "array": list,
+        "object": dict,
+    }
+    return mapping.get(json_type, str)
+
+
+def validate_openai_schema(
+    schema: dict[str, Any],
+    tool_name: str = "unknown",
+    path: str = "",
+) -> list[str]:
+    """Validate a JSON schema for OpenAI API compatibility.
+
+    OpenAI's API has specific requirements for tool schemas:
+    - Arrays must have 'items' (not 'prefixItems' which tuples generate)
+    - Certain schema features like 'prefixItems' are not supported
+
+    Args:
+        schema: JSON schema to validate.
+        tool_name: Name of the tool (for error messages).
+        path: Current path in schema (for error context).
+
+    Returns:
+        List of validation error messages. Empty if valid.
+    """
+    errors: list[str] = []
+
+    if not isinstance(schema, dict):
+        return errors
+
+    # Check for prefixItems (generated by tuple types)
+    if "prefixItems" in schema:
+        errors.append(
+            f"Tool '{tool_name}' has 'prefixItems' at {path or 'root'} "
+            "(likely from tuple type). Use list[Model] instead of tuple."
+        )
+
+    # Check arrays have 'items'
+    if schema.get("type") == "array" and "items" not in schema and "prefixItems" not in schema:
+        errors.append(
+            f"Tool '{tool_name}' has array at {path or 'root'} without 'items'. "
+            "OpenAI requires 'items' for array schemas."
+        )
+
+    # Recursively check nested schemas
+    # Check properties
+    if "properties" in schema:
+        for prop_name, prop_schema in schema["properties"].items():
+            prop_path = f"{path}.{prop_name}" if path else prop_name
+            errors.extend(validate_openai_schema(prop_schema, tool_name, prop_path))
+
+    # Check items
+    if "items" in schema and isinstance(schema["items"], dict):
+        items_path = f"{path}[items]" if path else "[items]"
+        errors.extend(validate_openai_schema(schema["items"], tool_name, items_path))
+
+    # Check anyOf/oneOf/allOf
+    for key in ("anyOf", "oneOf", "allOf"):
+        if key in schema:
+            for i, sub_schema in enumerate(schema[key]):
+                sub_path = f"{path}.{key}[{i}]" if path else f"{key}[{i}]"
+                errors.extend(validate_openai_schema(sub_schema, tool_name, sub_path))
+
+    # Check $defs (definitions)
+    if "$defs" in schema:
+        for def_name, def_schema in schema["$defs"].items():
+            def_path = f"$defs.{def_name}"
+            errors.extend(validate_openai_schema(def_schema, tool_name, def_path))
+
+    return errors

hud/environment/utils/tool_wrappers.py

@@ -0,0 +1,113 @@
+"""Shared tool wrapper utilities for agent framework integrations."""
+
+from __future__ import annotations
+
+import json
+from typing import TYPE_CHECKING, Any
+
+if TYPE_CHECKING:
+    from collections.abc import Callable
+
+    import mcp.types as mcp_types
+
+__all__ = [
+    "create_async_tool_fn",
+    "create_sync_tool_fn",
+    "create_tool_fns",
+    "stringify_result",
+]
+
+
+def stringify_result(result: Any) -> str:
+    """Convert a tool result to string format.
+
+    Args:
+        result: The tool result (str, dict, or other).
+
+    Returns:
+        String representation of the result.
+    """
+    if isinstance(result, str):
+        return result
+    return json.dumps(result) if result else ""
+
+
+def create_async_tool_fn(
+    env: Any,
+    tool_name: str,
+    description: str | None = None,
+) -> Callable[..., Any]:
+    """Create an async function that calls a tool on the environment.
+
+    Args:
+        env: Environment with call_tool method.
+        tool_name: Name of the tool to call.
+        description: Optional description for the function docstring.
+
+    Returns:
+        Async function that calls the tool and returns string result.
+    """
+
+    async def async_fn(**kwargs: Any) -> str:
+        result = await env.call_tool(tool_name, **kwargs)
+        return stringify_result(result)
+
+    async_fn.__name__ = tool_name
+    async_fn.__doc__ = description or f"Tool: {tool_name}"
+    return async_fn
+
+
+def create_sync_tool_fn(
+    env: Any,
+    tool_name: str,
+    description: str | None = None,
+) -> Callable[..., Any]:
+    """Create a sync function that calls a tool on the environment.
+
+    This handles the complexity of running async code from sync context,
+    including when already in an async event loop.
+
+    Args:
+        env: Environment with call_tool method.
+        tool_name: Name of the tool to call.
+        description: Optional description for the function docstring.
+
+    Returns:
+        Sync function that calls the tool and returns string result.
+    """
+    import asyncio
+
+    def sync_fn(**kwargs: Any) -> str:
+        loop = asyncio.get_event_loop()
+        if loop.is_running():
+            import concurrent.futures
+
+            with concurrent.futures.ThreadPoolExecutor() as executor:
+                future = executor.submit(asyncio.run, env.call_tool(tool_name, **kwargs))
+                result = future.result()
+        else:
+            result = loop.run_until_complete(env.call_tool(tool_name, **kwargs))
+
+        return stringify_result(result)
+
+    sync_fn.__name__ = tool_name
+    sync_fn.__doc__ = description or f"Tool: {tool_name}"
+    return sync_fn
+
+
+def create_tool_fns(
+    env: Any,
+    tool: mcp_types.Tool,
+) -> tuple[Callable[..., str], Callable[..., Any]]:
+    """Create both sync and async functions for a tool.
+
+    Args:
+        env: Environment with call_tool method.
+        tool: MCP tool definition.
+
+    Returns:
+        Tuple of (sync_fn, async_fn).
+    """
+    sync_fn = create_sync_tool_fn(env, tool.name, tool.description)
+    async_fn = create_async_tool_fn(env, tool.name, tool.description)
+    return sync_fn, async_fn

hud/eval/__init__.py

@@ -0,0 +1,67 @@
+"""HUD Eval - Evaluation context and management.
+
+This module provides:
+- Task: A runnable evaluation unit (from env())
+- EvalContext: Environment with evaluation tracking (trace_id, reward, etc.)
+- eval(): Standalone context manager for task-based evaluation
+
+Usage:
+    # Using env() to create Task
+    env = Environment("my-env").connect_hub("browser")
+
+    async with env() as ctx:
+        await ctx.call_tool("navigate", url="...")
+
+    async with env("checkout", user_id="alice") as ctx:
+        await agent.run(ctx.prompt)
+
+    # Standalone with task slugs
+    async with hud.eval("my-org/task:1") as ctx:
+        await agent.run(ctx)
+
+    # Orchestrated with Task objects
+    tasks = [env("checkout", user_id="alice"), env("checkout", user_id="bob")]
+    async with hud.eval(tasks, variants={"model": ["gpt-4o"]}, group=4) as ctx:
+        await agent.run(ctx.prompt)
+
+    # Blank eval for manual reward
+    async with hud.eval() as ctx:
+        ctx.reward = compute_reward()
+"""
+
+from __future__ import annotations
+
+from typing import TYPE_CHECKING
+
+# Auto-instrument httpx on import
+import hud.eval.instrument  # noqa: F401
+
+# run_eval is safe to import (uses lazy imports internally)
+from hud.eval.manager import run_eval
+
+# Task is safe to import
+from hud.eval.task import Task
+
+# Utils for v4 format handling
+from hud.eval.utils import build_env_from_v4, is_v4_format, validate_v4_task
+
+if TYPE_CHECKING:
+    from hud.eval.context import EvalContext
+
+__all__ = [
+    "EvalContext",
+    "Task",
+    "build_env_from_v4",
+    "is_v4_format",
+    "run_eval",
+    "validate_v4_task",
+]
+
+
+def __getattr__(name: str) -> object:
+    """Lazy import EvalContext to avoid circular imports."""
+    if name == "EvalContext":
+        from hud.eval.context import EvalContext
+
+        return EvalContext
+    raise AttributeError(f"module 'hud.eval' has no attribute {name!r}")