hud-python 0.4.20__py3-none-any.whl → 0.4.22__py3-none-any.whl

hud/tools/grounding/tests/test_grounded_tool.py

@@ -0,0 +1,196 @@
+from __future__ import annotations
+
+from dataclasses import dataclass
+from typing import Any
+
+import mcp.types as types
+import pytest
+
+from hud.tools.grounding.grounded_tool import GroundedComputerTool
+from hud.types import MCPToolCall, MCPToolResult
+
+
+@dataclass
+class FakeResult:
+    content: list[types.ContentBlock]
+    isError: bool = False
+    structuredContent: dict | None = None
+
+
+class FakeMCPClient:
+    """Fake MCP client that implements AgentMCPClient protocol."""
+
+    _initialized: bool
+
+    def __init__(self) -> None:
+        self.calls: list[tuple[str, dict[str, Any]]] = []
+        self._initialized = False
+
+    @property
+    def mcp_config(self) -> dict[str, dict[str, Any]]:
+        return {"test": {"command": "echo", "args": ["test"]}}
+
+    @property
+    def is_connected(self) -> bool:
+        return self._initialized
+
+    async def initialize(self, mcp_config: dict[str, dict[str, Any]] | None = None) -> None:
+        self._initialized = True
+
+    async def list_tools(self) -> list[types.Tool]:
+        return [types.Tool(name="computer", description="Test tool", inputSchema={})]
+
+    async def call_tool(self, tool_call: MCPToolCall) -> MCPToolResult:
+        self.calls.append((tool_call.name, tool_call.arguments or {}))
+        return MCPToolResult(content=[types.TextContent(text="ok", type="text")], isError=False)
+
+    async def shutdown(self) -> None:
+        self._initialized = False
+
+
+class FakeGrounder:
+    """Fake grounder that implements Grounder interface."""
+
+    def __init__(self, coords: tuple[int, int] | None = (10, 20)) -> None:
+        self.coords = coords
+        self.calls: list[tuple[str, str]] = []
+
+    async def predict_click(
+        self, *, image_b64: str, instruction: str, max_retries: int = 3
+    ) -> tuple[int, int] | None:
+        self.calls.append((image_b64[:10], instruction))
+        return self.coords
+
+
+def _png_b64() -> str:
+    # 1x1 transparent PNG base64 (valid minimal image)
+    return (
+        "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR4nGMAAQAABQAB"
+        "J2n0mQAAAABJRU5ErkJggg=="
+    )
+
+
+@pytest.mark.asyncio
+async def test_click_action_grounds_and_calls_mcp() -> None:
+    client = FakeMCPClient()
+    grounder = FakeGrounder(coords=(123, 456))
+    tool = GroundedComputerTool(grounder=grounder, mcp_client=client)  # type: ignore
+
+    blocks = await tool(
+        action="click",
+        element_description="red button",
+        screenshot_b64=_png_b64(),
+        button="left",
+    )
+
+    assert isinstance(blocks, list)
+    # Grounder called once
+    assert len(grounder.calls) == 1
+    # MCP called with resolved coordinates
+    assert client.calls == [("computer", {"action": "click", "x": 123, "y": 456, "button": "left"})]
+
+
+@pytest.mark.asyncio
+async def test_move_and_scroll_require_element_description_and_screenshot() -> None:
+    client = FakeMCPClient()
+    grounder = FakeGrounder(coords=(5, 6))
+    tool = GroundedComputerTool(grounder=grounder, mcp_client=client)  # type: ignore
+
+    # Missing element_description
+    with pytest.raises(Exception) as ei:
+        await tool(action="move", screenshot_b64=_png_b64())
+    assert "element_description is required" in str(ei.value)
+
+    # Missing screenshot
+    with pytest.raises(Exception) as ei2:
+        await tool(action="scroll", element_description="list", scroll_y=100)
+    assert "No screenshot available" in str(ei2.value)
+
+
+@pytest.mark.asyncio
+async def test_drag_grounds_both_points_and_calls_mcp() -> None:
+    client = FakeMCPClient()
+    grounder = FakeGrounder(coords=(10, 20))
+    tool = GroundedComputerTool(grounder=grounder, mcp_client=client)  # type: ignore
+
+    await tool(
+        action="drag",
+        start_element_description="source",
+        end_element_description="target",
+        screenshot_b64=_png_b64(),
+        button="left",
+    )
+
+    # Two grounding calls (start and end)
+    assert len(grounder.calls) == 2
+    # Drag path contains two points, same coords from fake grounder
+    name, args = client.calls[0]
+    assert name == "computer"
+    assert args["action"] == "drag"
+    assert args["button"] == "left"
+    assert args["path"] == [(10, 20), (10, 20)]
+
+
+@pytest.mark.asyncio
+async def test_drag_requires_both_descriptions_and_screenshot() -> None:
+    client = FakeMCPClient()
+    grounder = FakeGrounder()
+    tool = GroundedComputerTool(grounder=grounder, mcp_client=client)  # type: ignore
+
+    with pytest.raises(Exception) as ei:
+        await tool(action="drag", start_element_description="a", screenshot_b64=_png_b64())
+    assert "start_element_description and end_element_description" in str(ei.value)
+
+    with pytest.raises(Exception) as ei2:
+        await tool(
+            action="drag",
+            start_element_description="a",
+            end_element_description="b",
+        )
+    assert "No screenshot available" in str(ei2.value)
+
+
+@pytest.mark.asyncio
+async def test_direct_actions_bypass_grounding_and_call_mcp() -> None:
+    client = FakeMCPClient()
+    grounder = FakeGrounder()
+    tool = GroundedComputerTool(grounder=grounder, mcp_client=client)  # type: ignore
+
+    # Actions that bypass grounding
+    for action, extra in [
+        ("screenshot", {}),
+        ("type", {"text": "hello"}),
+        ("keypress", {"keys": ["ctrl", "a"]}),
+        ("wait", {}),
+        ("get_current_url", {}),
+        ("get_dimensions", {}),
+        ("get_environment", {}),
+    ]:
+        client.calls.clear()
+        _ = await tool(action=action, **extra)
+        assert client.calls and client.calls[0][0] == "computer"
+        assert client.calls[0][1]["action"] == action
+    # Grounder not invoked for these
+    assert grounder.calls == []
+
+
+@pytest.mark.asyncio
+async def test_unsupported_action_raises() -> None:
+    client = FakeMCPClient()
+    grounder = FakeGrounder()
+    tool = GroundedComputerTool(grounder=grounder, mcp_client=client)  # type: ignore
+
+    with pytest.raises(Exception) as ei:
+        await tool(action="zoom")
+    assert "Unsupported action" in str(ei.value)
+
+
+@pytest.mark.asyncio
+async def test_grounding_failure_propagates_as_error() -> None:
+    client = FakeMCPClient()
+    grounder = FakeGrounder(coords=None)
+    tool = GroundedComputerTool(grounder=grounder, mcp_client=client)  # type: ignore
+
+    with pytest.raises(Exception) as ei:
+        await tool(action="click", element_description="x", screenshot_b64=_png_b64())
+    assert "Could not locate element" in str(ei.value)

hud/tools/submit.py

@@ -0,0 +1,66 @@
+from __future__ import annotations
+
+import logging
+
+from mcp.types import ContentBlock, TextContent
+
+from .response import ResponseTool
+
+logger = logging.getLogger(__name__)
+
+
+# Global submission storage
+_SUBMISSION: str | None = None
+
+
+def set_submission(value: str | None) -> None:
+    global _SUBMISSION
+    _SUBMISSION = value
+
+
+def get_submission() -> str | None:
+    return _SUBMISSION
+
+
+class SubmitTool(ResponseTool):
+    """Lifecycle tool to submit the agent's final answer for evaluation.
+
+    Accepts either a `response` string or a `messages` list and stores the
+    submission as a plain string, accessible via `get_submission()`.
+    Priority: The last text content in `messages` (if provided) overrides `response`.
+    """
+
+    name: str = "response"
+    title: str = "Submit Tool"
+    description: str = "Submit the agent's final response for later evaluation"
+
+    async def __call__(
+        self, response: str | None = None, messages: list[ContentBlock] | None = None
+    ) -> list[ContentBlock]:
+        # 1) If messages provided, take the last text block
+        # chosen: str | None = None
+
+        # if messages:
+        #     # Gather all text blocks
+        #     text_blocks: list[str] = []
+        #     for block in messages:
+        #         try:
+        #             if isinstance(block, TextContent):
+        #                 text_blocks.append(str(block.text))
+        #         except Exception:
+        #             logger.debug("SubmitTool skipped non-text block: %s", block)
+        #             continue
+        #     if text_blocks:
+        #         chosen = text_blocks[-1]
+
+        # # 2) Otherwise use `response` as-is
+        # if chosen is None and response is not None:
+        #     chosen = response
+
+        set_submission(response)
+
+        # Echo back what we stored
+        blocks: list[ContentBlock] = []
+        if response:
+            blocks.append(TextContent(text=response, type="text"))
+        return blocks

hud/tools/tests/test_playwright_tool.py

@@ -52,7 +52,7 @@ class TestPlaywrightTool:
             assert any(isinstance(b, TextContent) for b in blocks)
             # The actual call includes wait_until parameter with a Field object
             mock_page.goto.assert_called_once()
-            args, kwargs = mock_page.goto.call_args
+            args, _kwargs = mock_page.goto.call_args
             assert args[0] == "https://example.com"
             mock_ensure.assert_called_once()
 

hud/tools/tests/test_tools_init.py

@@ -33,7 +33,7 @@ class TestToolsInit:
         """Test lazy import with invalid attribute name."""
         import hud.tools as tools_module
 
-        with pytest.raises(AttributeError, match="module '.*' has no attribute 'InvalidTool'"):
+        with pytest.raises(AttributeError, match=r"module '.*' has no attribute 'InvalidTool'"):
             _ = tools_module.InvalidTool
 
     def test_direct_imports_available(self):

hud/tools/tests/test_utils.py

@@ -58,7 +58,7 @@ class TestRun:
         mock_proc.communicate = AsyncMock(return_value=(b"processed", b""))
 
         with patch("asyncio.create_subprocess_shell", return_value=mock_proc):
-            return_code, stdout, stderr = await run("cat", input="test input")
+            return_code, stdout, _stderr = await run("cat", input="test input")
 
             assert return_code == 0
             assert stdout == "processed"
@@ -91,7 +91,7 @@ class TestRun:
         ):
             mock_wait_for.return_value = (b"done", b"")
 
-            return_code, stdout, stderr = await run("sleep 1", timeout=5.0)
+            _return_code, _stdout, _stderr = await run("sleep 1", timeout=5.0)
 
             # Check that wait_for was called with the correct timeout
             mock_wait_for.assert_called_once()

hud/types.py

@@ -15,7 +15,20 @@ class MCPToolCall(CallToolRequestParams):
     id: str = Field(default_factory=lambda: str(uuid.uuid4()))  # Unique identifier for reference
 
     def __str__(self) -> str:
-        """Format tool call with Rich markup for HUD design."""
+        """Format tool call as plain text."""
+        args_str = ""
+        if self.arguments:
+            try:
+                args_str = json.dumps(self.arguments, separators=(",", ":"))
+                if len(args_str) > 60:
+                    args_str = args_str[:57] + "..."
+            except (TypeError, ValueError):
+                args_str = str(self.arguments)[:60]
+
+        return f"→ {self.name}({args_str})"
+
+    def __rich__(self) -> str:
+        """Rich representation with color formatting."""
         from hud.utils.design import design
 
         return design.format_tool_call(self.name, self.arguments)
@@ -24,10 +37,8 @@ class MCPToolCall(CallToolRequestParams):
 class MCPToolResult(CallToolResult):
     """A tool result."""
 
-    def __str__(self) -> str:
-        """Format tool result with Rich markup for HUD design - compact version."""
-        from hud.utils.design import design
-
+    def _get_content_summary(self) -> str:
+        """Extract a summary of the content."""
         # Extract content summary
         content_summary = ""
         if self.content:
@@ -49,6 +60,23 @@ class MCPToolResult(CallToolResult):
             except (TypeError, ValueError):
                 content_summary = str(self.structuredContent)
 
+        return content_summary
+
+    def __str__(self) -> str:
+        """Format tool result as plain text for compatibility."""
+        content_summary = self._get_content_summary()
+
+        # Plain text format with unicode symbols
+        if self.isError:
+            return f"✗ {content_summary}"
+        else:
+            return f"✓ {content_summary}"
+
+    def __rich__(self) -> str:
+        """Rich representation with color formatting."""
+        from hud.utils.design import design
+
+        content_summary = self._get_content_summary()
         return design.format_tool_result(content_summary, self.isError)
 
 

hud/utils/agent_factories.py

@@ -0,0 +1,86 @@
+"""Factory functions for creating agents compatible with run_dataset."""
+
+from __future__ import annotations
+
+from typing import Any
+
+from openai import AsyncOpenAI
+
+from hud.agents.grounded_openai import GroundedOpenAIChatAgent
+from hud.agents.openai_chat_generic import GenericOpenAIChatAgent
+from hud.tools.grounding import GrounderConfig
+
+
+def create_openai_agent(**kwargs: Any) -> GenericOpenAIChatAgent:
+    """Factory for GenericOpenAIChatAgent with run_dataset compatibility.
+
+    Args:
+        api_key: OpenAI API key
+        base_url: Optional custom API endpoint
+        model_name: Model to use (e.g., "gpt-4o-mini")
+        **kwargs: Additional arguments passed to GenericOpenAIChatAgent
+
+    Returns:
+        Configured GenericOpenAIChatAgent instance
+
+    Example:
+        >>> from hud.datasets import run_dataset
+        >>> from hud.utils.agent_factories import create_openai_agent
+        >>> results = await run_dataset(
+        ...     "My Eval",
+        ...     "hud-evals/SheetBench-50",
+        ...     create_openai_agent,
+        ...     {"api_key": "your-key", "model_name": "gpt-4o-mini"},
+        ... )
+    """
+    api_key = kwargs.pop("api_key", None)
+    base_url = kwargs.pop("base_url", None)
+
+    openai_client = AsyncOpenAI(api_key=api_key, base_url=base_url)
+
+    return GenericOpenAIChatAgent(openai_client=openai_client, **kwargs)
+
+
+def create_grounded_agent(**kwargs: Any) -> GroundedOpenAIChatAgent:
+    """Factory for GroundedOpenAIChatAgent with run_dataset compatibility.
+
+    Args:
+        api_key: OpenAI API key for planning model
+        base_url: Optional custom API endpoint for planning model
+        model_name: Planning model to use (e.g., "gpt-4o-mini")
+        grounder_api_key: API key for grounding model
+        grounder_api_base: API base URL for grounding model (default: OpenRouter)
+        grounder_model: Grounding model to use (default: qwen/qwen-2.5-vl-7b-instruct)
+        **kwargs: Additional arguments passed to GroundedOpenAIChatAgent
+
+    Returns:
+        Configured GroundedOpenAIChatAgent instance
+
+    Example:
+        >>> from hud.datasets import run_dataset
+        >>> from hud.utils.agent_factories import create_grounded_agent
+        >>> results = await run_dataset(
+        ...     "Grounded Eval",
+        ...     dataset,
+        ...     create_grounded_agent,
+        ...     {
+        ...         "api_key": "openai-key",
+        ...         "grounder_api_key": "openrouter-key",
+        ...         "model_name": "gpt-4o-mini",
+        ...     },
+        ... )
+    """
+    api_key = kwargs.pop("api_key", None)
+    base_url = kwargs.pop("base_url", None)
+    grounder_api_key = kwargs.pop("grounder_api_key", None)
+    grounder_api_base = kwargs.pop("grounder_api_base", "https://openrouter.ai/api/v1")
+    grounder_model = kwargs.pop("grounder_model", "qwen/qwen-2.5-vl-7b-instruct")
+
+    openai_client = AsyncOpenAI(api_key=api_key, base_url=base_url)
+    grounder_config = GrounderConfig(
+        api_base=grounder_api_base, model=grounder_model, api_key=grounder_api_key
+    )
+
+    return GroundedOpenAIChatAgent(
+        openai_client=openai_client, grounder_config=grounder_config, **kwargs
+    )

hud/utils/design.py

@@ -257,6 +257,63 @@ class HUDDesign:
         else:
             console.print(f"  [cyan]{command}[/cyan]")
 
+    # Exception rendering utilities
+    def render_support_hint(self, stderr: bool = True) -> None:
+        """Render a standard support message for users encountering issues."""
+        support = (
+            "If this looks like an issue with the sdk, please make a github issue at "
+            "https://github.com/hud-evals/hud-python/issues"
+        )
+        self.info(support, stderr=stderr)
+
+    def render_exception(self, error: BaseException, *, stderr: bool = True) -> None:
+        """Render exceptions consistently using the HUD design system.
+
+        - Shows exception type and message
+        - Displays structured hints if present on the exception (e.g., HudException.hints)
+        - Prints a link to open an issue for SDK problems
+        """
+        try:
+            from hud.shared.exceptions import HudRequestError  # lazy import
+        except Exception:
+            # Keep type available for isinstance guards below without import-time dependency
+            HudRequestError = tuple()  # type: ignore
+
+        # Header with exception type
+        ex_type = type(error).__name__
+        message = getattr(error, "message", "") or str(error) or ex_type
+        self.error(f"{ex_type}: {message}", stderr=stderr)
+
+        # Specialized details for request errors
+        if isinstance(error, HudRequestError):  # type: ignore[arg-type]
+            details: dict[str, str] = {}
+            status_code = getattr(error, "status_code", None)
+            if status_code is not None:
+                details["Status"] = str(status_code)
+            response_text = getattr(error, "response_text", None)
+            if response_text:
+                # Limit very long responses
+                trimmed = response_text[:500] + ("..." if len(response_text) > 500 else "")
+                details["Response"] = trimmed
+            response_json = getattr(error, "response_json", None)
+            if response_json and not details.get("Response"):
+                details["Response JSON"] = str(response_json)
+            if details:
+                self.key_value_table(details, show_header=False, stderr=stderr)
+
+        # Structured hints, if available
+        hints = getattr(error, "hints", None)
+        if hints:
+            try:
+                from hud.shared.hints import render_hints  # lazy import
+
+                render_hints(hints, design=self)
+            except Exception as render_error:
+                self.debug(f"Failed to render hints: {render_error}")
+
+        # Standard support hint
+        self.render_support_hint(stderr=stderr)
+
     @property
     def console(self) -> Console:
         """Get the stderr console for direct access when needed."""

hud/utils/mcp.py

@@ -76,4 +76,10 @@ def setup_hud_telemetry(
             MCPConfigPatch(headers={"Run-Id": run_id}, meta={"run_id": run_id}),
         )
 
+    if settings.api_key:
+        patch_mcp_config(
+            mcp_config,
+            MCPConfigPatch(headers={"Authorization": f"Bearer {settings.api_key}"}),
+        )
+
     return auto_trace_cm

hud/utils/pretty_errors.py

@@ -0,0 +1,68 @@
+from __future__ import annotations
+
+import asyncio
+import logging
+import sys
+from typing import Any
+
+from hud.utils.design import design
+
+logger = logging.getLogger(__name__)
+
+
+def _render_and_fallback(exc_type: type[BaseException], value: BaseException, tb: Any) -> None:
+    """Render exceptions via HUD design, then delegate to default excepthook.
+
+    Only formats for HudException family or when running in a TTY; otherwise,
+    defers to the default handler to avoid swallowing useful tracebacks in code.
+    """
+    # First, print the full traceback
+    sys.__excepthook__(exc_type, value, tb)
+
+    # Then print our formatted error
+    try:
+        from hud.shared.exceptions import HudException  # lazy import
+
+        if isinstance(value, HudException):
+            # Flush stderr to ensure traceback is printed first
+            sys.stderr.flush()
+            # Add separator and render our formatted error
+            design.console.print("")
+            design.render_exception(value)
+    except Exception:
+        # If rendering fails for any reason, silently continue
+        logger.warning("Failed to render exception: %s, %s, %s", exc_type, value, tb)
+
+
+def _async_exception_handler(loop: asyncio.AbstractEventLoop, context: dict[str, Any]) -> None:
+    exc = context.get("exception")
+    msg = context.get("message")
+    try:
+        if exc is not None:
+            design.render_exception(exc)
+        elif msg:
+            design.error(msg)
+            design.render_support_hint()
+    except Exception:
+        logger.warning("Failed to render exception: %s, %s, %s", exc, msg, context)
+
+    # Delegate to default handler
+    loop.default_exception_handler(context)
+
+
+def install_pretty_errors() -> None:
+    """Install global pretty error handlers for sync and async exceptions."""
+    sys.excepthook = _render_and_fallback
+    try:
+        # Try to get the running loop first
+        loop = asyncio.get_running_loop()
+        loop.set_exception_handler(_async_exception_handler)
+    except RuntimeError:
+        # No running loop, try to create one
+        try:
+            loop = asyncio.new_event_loop()
+            loop.set_exception_handler(_async_exception_handler)
+        except Exception:
+            logger.warning("No running loop, could not set exception handler")
+    except Exception:
+        logger.warning("No running loop, could not set exception handler")

hud/utils/tests/test_version.py

@@ -5,4 +5,4 @@ def test_import():
     """Test that the package can be imported."""
     import hud
 
-    assert hud.__version__ == "0.4.20"
+    assert hud.__version__ == "0.4.22"

hud/version.py

@@ -4,4 +4,4 @@ Version information for the HUD SDK.
 
 from __future__ import annotations
 
-__version__ = "0.4.20"
+__version__ = "0.4.22"

{hud_python-0.4.20.dist-info → hud_python-0.4.22.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: hud-python
-Version: 0.4.20
+Version: 0.4.22
 Summary: SDK for the HUD platform.
 Project-URL: Homepage, https://github.com/hud-evals/hud-python
 Project-URL: Bug Tracker, https://github.com/hud-evals/hud-python/issues
@@ -38,6 +38,7 @@ Requires-Python: <3.14,>=3.11
 Requires-Dist: httpx<1,>=0.23.0
 Requires-Dist: hud-fastmcp-python-sdk>=0.1.2
 Requires-Dist: hud-mcp-python-sdk>=3.13.2
+Requires-Dist: hud-mcp-use-python-sdk>=2.3.16
 Requires-Dist: opentelemetry-api>=1.34.1
 Requires-Dist: opentelemetry-exporter-otlp-proto-http>=1.34.1
 Requires-Dist: opentelemetry-instrumentation-mcp>=0.44.1
@@ -56,7 +57,6 @@ Provides-Extra: agent
 Requires-Dist: anthropic; extra == 'agent'
 Requires-Dist: datasets>=2.14.0; extra == 'agent'
 Requires-Dist: dotenv>=0.9.9; extra == 'agent'
-Requires-Dist: hud-mcp-use-python-sdk>=2.3.13; extra == 'agent'
 Requires-Dist: ipykernel; extra == 'agent'
 Requires-Dist: ipython<9; extra == 'agent'
 Requires-Dist: jupyter-client; extra == 'agent'
@@ -70,7 +70,6 @@ Provides-Extra: agents
 Requires-Dist: anthropic; extra == 'agents'
 Requires-Dist: datasets>=2.14.0; extra == 'agents'
 Requires-Dist: dotenv>=0.9.9; extra == 'agents'
-Requires-Dist: hud-mcp-use-python-sdk>=2.3.13; extra == 'agents'
 Requires-Dist: ipykernel; extra == 'agents'
 Requires-Dist: ipython<9; extra == 'agents'
 Requires-Dist: jupyter-client; extra == 'agents'
@@ -85,7 +84,6 @@ Requires-Dist: aiodocker>=0.24.0; extra == 'dev'
 Requires-Dist: anthropic; extra == 'dev'
 Requires-Dist: datasets>=2.14.0; extra == 'dev'
 Requires-Dist: dotenv>=0.9.9; extra == 'dev'
-Requires-Dist: hud-mcp-use-python-sdk>=2.3.13; extra == 'dev'
 Requires-Dist: inspect-ai>=0.3.80; extra == 'dev'
 Requires-Dist: ipykernel; extra == 'dev'
 Requires-Dist: ipython<9; extra == 'dev'