hud-python 0.4.45__py3-none-any.whl → 0.5.13__py3-none-any.whl

hud/telemetry/tests/test_instrument.py

@@ -0,0 +1,401 @@
+from __future__ import annotations
+
+from dataclasses import dataclass
+
+import pytest
+
+from hud.telemetry.instrument import _serialize_value, instrument
+
+
+def test_serialize_value_simple_types():
+    """Test _serialize_value with simple types."""
+    assert _serialize_value("string") == "string"
+    assert _serialize_value(42) == 42
+    assert _serialize_value(3.14) == 3.14
+    assert _serialize_value(True) is True
+    assert _serialize_value(None) is None
+
+
+def test_serialize_value_list():
+    """Test _serialize_value with lists."""
+    result = _serialize_value([1, 2, 3])
+    assert result == [1, 2, 3]
+
+
+def test_serialize_value_list_truncation():
+    """Test _serialize_value truncates long lists."""
+    long_list = list(range(20))
+    result = _serialize_value(long_list, max_items=5)
+    assert len(result) == 5
+    assert result == [0, 1, 2, 3, 4]
+
+
+def test_serialize_value_tuple():
+    """Test _serialize_value with tuples."""
+    result = _serialize_value((1, 2, 3))
+    assert result == [1, 2, 3]  # Converted to list by JSON
+
+
+def test_serialize_value_tuple_truncation():
+    """Test _serialize_value truncates long tuples."""
+    long_tuple = tuple(range(20))
+    result = _serialize_value(long_tuple, max_items=5)
+    assert len(result) == 5
+
+
+def test_serialize_value_dict():
+    """Test _serialize_value with dicts."""
+    result = _serialize_value({"key": "value"})
+    assert result == {"key": "value"}
+
+
+def test_serialize_value_dict_truncation():
+    """Test _serialize_value truncates large dicts."""
+    large_dict = {f"key{i}": i for i in range(20)}
+    result = _serialize_value(large_dict, max_items=5)
+    assert len(result) == 5
+
+
+def test_serialize_value_complex_object():
+    """Test _serialize_value with custom objects."""
+
+    @dataclass
+    class CustomObj:
+        name: str
+        value: int
+
+    obj = CustomObj(name="test", value=42)
+    result = _serialize_value(obj)
+    assert isinstance(result, dict)
+    assert result["name"] == "test"
+    assert result["value"] == 42
+
+
+def test_serialize_value_fallback():
+    """Test _serialize_value fallback for non-serializable objects."""
+
+    class WeirdObj:
+        def __init__(self):
+            raise Exception("Can't access")
+
+    obj = WeirdObj.__new__(WeirdObj)
+    result = _serialize_value(obj)
+    # The result is a string representation of the object
+    assert isinstance(result, str)
+    assert "WeirdObj" in result
+
+
+@pytest.mark.asyncio
+async def test_instrument_async_basic():
+    """Test instrument decorator on async function."""
+
+    @instrument
+    async def test_func(x: int, y: int) -> int:
+        return x + y
+
+    result = await test_func(2, 3)
+    assert result == 5
+
+
+@pytest.mark.asyncio
+async def test_instrument_async_with_params():
+    """Test instrument with custom parameters."""
+
+    @instrument(name="custom_name", category="custom_type")
+    async def test_func(x: int) -> int:
+        return x * 2
+
+    result = await test_func(5)
+    assert result == 10
+
+
+@pytest.mark.asyncio
+async def test_instrument_async_with_exception():
+    """Test instrument handles exceptions."""
+
+    @instrument
+    async def test_func():
+        raise ValueError("Test error")
+
+    with pytest.raises(ValueError, match="Test error"):
+        await test_func()
+
+
+@pytest.mark.asyncio
+async def test_instrument_async_no_record_args():
+    """Test instrument with record_args=False."""
+
+    @instrument(record_args=False)
+    async def test_func(x: int) -> int:
+        return x
+
+    result = await test_func(42)
+    assert result == 42
+
+
+@pytest.mark.asyncio
+async def test_instrument_async_no_record_result():
+    """Test instrument with record_result=False."""
+
+    @instrument(record_result=False)
+    async def test_func() -> str:
+        return "test"
+
+    result = await test_func()
+    assert result == "test"
+
+
+@pytest.mark.asyncio
+async def test_instrument_async_with_category():
+    """Test instrument with custom category."""
+
+    @instrument(category="agent")
+    async def test_func() -> int:
+        return 42
+
+    result = await test_func()
+    assert result == 42
+
+
+def test_instrument_sync_basic():
+    """Test instrument decorator on sync function."""
+
+    @instrument
+    def test_func(x: int, y: int) -> int:
+        return x + y
+
+    result = test_func(2, 3)
+    assert result == 5
+
+
+def test_instrument_sync_with_params():
+    """Test instrument on sync function with parameters."""
+
+    @instrument(name="sync_custom", category="sync_type")
+    def test_func(x: int) -> int:
+        return x * 2
+
+    result = test_func(5)
+    assert result == 10
+
+
+def test_instrument_sync_with_exception():
+    """Test instrument handles exceptions in sync functions."""
+
+    @instrument
+    def test_func():
+        raise ValueError("Sync error")
+
+    with pytest.raises(ValueError, match="Sync error"):
+        test_func()
+
+
+def test_instrument_sync_no_record_args():
+    """Test instrument sync with record_args=False."""
+
+    @instrument(record_args=False)
+    def test_func(x: int) -> int:
+        return x
+
+    result = test_func(42)
+    assert result == 42
+
+
+def test_instrument_sync_no_record_result():
+    """Test instrument sync with record_result=False."""
+
+    @instrument(record_result=False)
+    def test_func() -> str:
+        return "test"
+
+    result = test_func()
+    assert result == "test"
+
+
+def test_instrument_sync_with_category():
+    """Test instrument sync with custom category."""
+
+    @instrument(category="tool")
+    def test_func() -> int:
+        return 42
+
+    result = test_func()
+    assert result == 42
+
+
+def test_instrument_already_instrumented():
+    """Test that instrumenting already instrumented function is skipped."""
+
+    @instrument
+    def test_func():
+        return "original"
+
+    # Try to instrument again
+    test_func2 = instrument(test_func)
+
+    # Should be the same function
+    assert test_func2 is test_func
+
+
+def test_instrument_marks_as_instrumented():
+    """Test that instrument marks functions correctly."""
+
+    @instrument
+    def test_func():
+        return True
+
+    assert hasattr(test_func, "_hud_instrumented")
+    assert test_func._hud_instrumented is True
+    assert hasattr(test_func, "_hud_original")
+
+
+@pytest.mark.asyncio
+async def test_instrument_async_complex_result():
+    """Test instrument with complex result object."""
+
+    @instrument
+    async def test_func() -> dict:
+        return {"nested": {"data": [1, 2, 3]}, "count": 3}
+
+    result = await test_func()
+    assert result["count"] == 3
+
+
+def test_instrument_sync_complex_result():
+    """Test instrument sync with complex result."""
+
+    @dataclass
+    class Result:
+        value: int
+        name: str
+
+    @instrument
+    def test_func() -> Result:
+        return Result(value=42, name="test")
+
+    result = test_func()
+    assert result.value == 42
+
+
+@pytest.mark.asyncio
+async def test_instrument_async_with_self_param():
+    """Test instrument properly handles 'self' parameter."""
+
+    class TestClass:
+        @instrument
+        async def method(self, x: int) -> int:
+            return x * 2
+
+    obj = TestClass()
+    result = await obj.method(5)
+    assert result == 10
+
+
+def test_instrument_sync_with_cls_param():
+    """Test instrument properly handles 'cls' parameter."""
+
+    class TestClass:
+        @classmethod
+        @instrument
+        def method(cls, x: int) -> int:
+            return x * 3
+
+    result = TestClass.method(4)
+    assert result == 12
+
+
+@pytest.mark.asyncio
+async def test_instrument_async_serialization_error():
+    """Test instrument handles serialization errors gracefully."""
+
+    class UnserializableArg:
+        def __getattribute__(self, name):
+            raise Exception("Can't serialize")
+
+    @instrument
+    async def test_func(arg):
+        return "success"
+
+    # Should not raise, just skip serialization
+    result = await test_func(UnserializableArg())
+    assert result == "success"
+
+
+def test_instrument_function_without_signature():
+    """Test instrument on functions without inspectable signature."""
+    # Built-in functions don't have signatures
+    instrumented_len = instrument(len)
+    result = instrumented_len([1, 2, 3])
+    assert result == 3
+
+
+@pytest.mark.asyncio
+async def test_instrument_async_result_serialization_error():
+    """Test instrument handles result serialization errors."""
+
+    class UnserializableResult:
+        def __iter__(self):
+            raise Exception("Can't iterate")
+
+    @instrument
+    async def test_func():
+        return UnserializableResult()
+
+    # Should not raise, just skip result recording
+    result = await test_func()
+    assert isinstance(result, UnserializableResult)
+
+
+def test_instrument_without_parentheses():
+    """Test using @instrument without parentheses."""
+
+    @instrument
+    def test_func(x: int) -> int:
+        return x + 1
+
+    assert test_func(5) == 6
+
+
+def test_instrument_with_parentheses():
+    """Test using @instrument() with parentheses."""
+
+    @instrument()
+    def test_func(x: int) -> int:
+        return x + 1
+
+    assert test_func(5) == 6
+
+
+@pytest.mark.asyncio
+async def test_instrument_async_with_defaults():
+    """Test instrument with function that has default arguments."""
+
+    @instrument
+    async def test_func(x: int, y: int = 10) -> int:
+        return x + y
+
+    assert await test_func(5) == 15
+    assert await test_func(5, 20) == 25
+
+
+def test_instrument_sync_with_kwargs():
+    """Test instrument with keyword arguments."""
+
+    @instrument
+    def test_func(x: int, **kwargs) -> dict:
+        return {"x": x, **kwargs}
+
+    result = test_func(1, a=2, b=3)
+    assert result == {"x": 1, "a": 2, "b": 3}
+
+
+@pytest.mark.asyncio
+async def test_instrument_async_with_varargs():
+    """Test instrument with *args."""
+
+    @instrument
+    async def test_func(*args) -> int:
+        return sum(args)
+
+    result = await test_func(1, 2, 3, 4)
+    assert result == 10

hud/tools/__init__.py

@@ -4,6 +4,7 @@ from __future__ import annotations
 
 from typing import TYPE_CHECKING, Any
 
+from .agent import AgentTool
 from .base import BaseHub, BaseTool
 from .bash import BashTool
 from .edit import EditTool
@@ -12,17 +13,26 @@ from .response import ResponseTool
 from .submit import SubmitTool
 
 if TYPE_CHECKING:
-    from .computer import AnthropicComputerTool, HudComputerTool, OpenAIComputerTool
+    from .computer import (
+        AnthropicComputerTool,
+        GeminiComputerTool,
+        HudComputerTool,
+        OpenAIComputerTool,
+        QwenComputerTool,
+    )
 
 __all__ = [
+    "AgentTool",
     "AnthropicComputerTool",
     "BaseHub",
     "BaseTool",
     "BashTool",
     "EditTool",
+    "GeminiComputerTool",
     "HudComputerTool",
     "OpenAIComputerTool",
     "PlaywrightTool",
+    "QwenComputerTool",
     "ResponseTool",
     "SubmitTool",
 ]
@@ -30,7 +40,13 @@ __all__ = [
 
 def __getattr__(name: str) -> Any:
     """Lazy import computer tools to avoid importing pyautogui unless needed."""
-    if name in ("AnthropicComputerTool", "HudComputerTool", "OpenAIComputerTool"):
+    if name in (
+        "AnthropicComputerTool",
+        "HudComputerTool",
+        "OpenAIComputerTool",
+        "GeminiComputerTool",
+        "QwenComputerTool",
+    ):
         from . import computer
 
         return getattr(computer, name)

hud/tools/agent.py

@@ -0,0 +1,223 @@
+"""AgentTool - run a Task with an agent as a tool."""
+
+from __future__ import annotations
+
+import inspect
+from typing import TYPE_CHECKING, Any, Union, get_args, get_origin
+
+from fastmcp.tools.tool import FunctionTool, ToolResult
+from mcp.types import TextContent
+
+from hud.tools.base import BaseTool
+
+if TYPE_CHECKING:
+    from hud.agents.base import MCPAgent
+    from hud.eval.task import Task
+
+__all__ = ["AgentTool"]
+
+
+def _is_eval_only(param: inspect.Parameter) -> bool:
+    """Check if param is eval-only: has None default AND None in type union.
+
+    Handles both runtime types and string annotations (PEP 563).
+    """
+    # Must have default of None
+    if param.default is not None:
+        return False
+    if param.annotation is inspect.Parameter.empty:
+        return False
+
+    annotation = param.annotation
+
+    # Handle string annotations (from __future__ annotations or quoted)
+    if isinstance(annotation, str):
+        # Check if it looks like "X | None", "Union[X, None]", or "Optional[X]"
+        return (
+            "| None" in annotation
+            or "None |" in annotation
+            or "Optional[" in annotation
+            or ("Union[" in annotation and "None" in annotation)
+        )
+
+    # Handle runtime type annotations
+    origin = get_origin(annotation)
+
+    # Union types (X | None or Union[X, None])
+    if origin is Union:
+        return type(None) in get_args(annotation)
+
+    # For Python 3.10+ union syntax at runtime (types.UnionType)
+    try:
+        import types
+
+        if isinstance(annotation, types.UnionType):
+            return type(None) in get_args(annotation)
+    except (ImportError, AttributeError):
+        pass
+
+    return False
+
+
+class AgentTool(BaseTool):
+    """Tool that runs a Task template with an agent.
+
+    Parameters with `| None = None` are eval-only and hidden from the tool schema.
+
+    Example:
+        ```python
+        @env.scenario()
+        async def investigate(
+            issue_id: str,  # Required - orchestrator sees
+            expected_cause: str | None = None,  # Eval only - hidden
+        ):
+            yield {"task": f"Investigate {issue_id}"}
+
+
+        seer = AgentTool(env("investigate"), model="ft:seer-v2")
+        ```
+    """
+
+    def __init__(
+        self,
+        task: Task,
+        *,
+        model: str | None = None,
+        agent: type[MCPAgent] | None = None,
+        agent_params: dict[str, Any] | None = None,
+        name: str | None = None,
+        description: str | None = None,
+        trace: bool = False,
+    ) -> None:
+        if not model and agent is None:
+            raise ValueError("Must provide either 'model' or 'agent'")
+        if model and agent is not None:
+            raise ValueError("Cannot provide both 'model' and 'agent'")
+
+        self._task = task
+        self._model = model
+        self._agent_cls = agent
+        self._agent_params = agent_params or {}
+        self._trace = trace
+
+        # Get visible params from scenario function
+        self._visible_params: set[str] = set()
+        self._param_schema: dict[str, Any] = {
+            "type": "object",
+            "properties": {},
+            "required": [],
+        }
+
+        if task.env and task.scenario:
+            scenario_fn = task.env._scenarios.get(task.scenario)
+            if scenario_fn:
+                sig = inspect.signature(scenario_fn)
+                visible = {name: p for name, p in sig.parameters.items() if not _is_eval_only(p)}
+                self._visible_params = set(visible.keys())
+                self._param_schema = self._build_schema(visible)
+
+        tool_name = name or task.scenario or "agent_tool"
+        tool_desc = description or f"Run scenario: {task.scenario}"
+
+        super().__init__(name=tool_name, description=tool_desc)
+
+    def _build_schema(self, params: dict[str, inspect.Parameter]) -> dict[str, Any]:
+        """Build JSON schema using Pydantic TypeAdapter."""
+        from pydantic import TypeAdapter
+
+        properties: dict[str, Any] = {}
+        required: list[str] = []
+
+        for name, param in params.items():
+            if param.annotation is not inspect.Parameter.empty:
+                try:
+                    # Handle string annotations
+                    annotation = param.annotation
+                    if isinstance(annotation, str):
+                        # Try to evaluate the annotation
+                        try:
+                            annotation = eval(annotation)  # noqa: S307
+                        except Exception:
+                            # Fall back to string type but don't skip required handling
+                            annotation = None
+
+                    if annotation is not None:
+                        adapter = TypeAdapter(annotation)
+                        properties[name] = adapter.json_schema()
+                    else:
+                        properties[name] = {"type": "string"}
+                except Exception:
+                    properties[name] = {"type": "string"}
+            else:
+                properties[name] = {"type": "string"}
+
+            if param.default is inspect.Parameter.empty:
+                required.append(name)
+            elif param.default is not None:
+                properties[name]["default"] = param.default
+
+        return {"type": "object", "properties": properties, "required": required}
+
+    @property
+    def mcp(self) -> FunctionTool:
+        """Get as FastMCP FunctionTool with filtered schema."""
+        if not hasattr(self, "_mcp_tool"):
+            # Directly instantiate FunctionTool with our callable and schema
+            # This bypasses from_function's signature parsing
+            self._mcp_tool = FunctionTool(
+                name=self.name,
+                description=self.description or "",
+                parameters=self._param_schema,
+                fn=self._execute_with_args,
+            )
+        return self._mcp_tool
+
+    async def _execute_with_args(self, **kwargs: Any) -> ToolResult:
+        """Internal executor that FastMCP calls with parsed arguments."""
+        return await self(**kwargs)
+
+    async def __call__(self, **kwargs: Any) -> ToolResult:
+        """Execute the task with a fresh agent."""
+        from hud.eval.context import get_current_trace_id
+        from hud.eval.manager import run_eval
+        from hud.telemetry.instrument import instrument
+
+        # Filter to visible params only
+        filtered = {k: v for k, v in kwargs.items() if k in self._visible_params}
+
+        # Merge with template args
+        base_args = self._task.args or {}
+        task = self._task.model_copy(update={"args": {**base_args, **filtered}})
+
+        # Use parent trace if available (for hierarchical agents)
+        parent_trace_id = get_current_trace_id()
+
+        # If nested (has parent), skip subagent's enter/exit registration
+        # Tool calls are still recorded via the shared trace_id's context
+        is_nested = parent_trace_id is not None
+
+        # Trace if explicitly requested AND not nested (nested uses parent trace)
+        should_trace = self._trace and not is_nested
+
+        # Wrap execution with instrumentation to mark as subagent
+        # Platform uses category="subagent" to detect and render subagent tool calls
+        @instrument(category="subagent", name=self.name)
+        async def _run_subagent() -> ToolResult:
+            async with run_eval(
+                task,
+                trace=should_trace,
+                trace_id=parent_trace_id,
+                quiet=True,
+            ) as ctx:
+                if self._model:
+                    from hud.agents import create_agent
+
+                    agent = create_agent(self._model, **self._agent_params)
+                else:
+                    agent = self._agent_cls.create(**self._agent_params)  # type: ignore
+
+                result = await agent.run(ctx)
+                content = result.content if hasattr(result, "content") and result.content else ""
+                return ToolResult(content=[TextContent(type="text", text=content)])
+
+        return await _run_subagent()