hud-python 0.4.21__py3-none-any.whl → 0.4.23__py3-none-any.whl

hud/agents/base.py

@@ -11,7 +11,7 @@ from typing import TYPE_CHECKING, Any, ClassVar, Literal
 import mcp.types as types
 
 from hud.types import AgentResponse, MCPToolCall, MCPToolResult, Trace
-from hud.utils.design import HUDDesign
+from hud.utils.hud_console import HUDConsole
 from hud.utils.mcp import MCPConfigPatch, patch_mcp_config, setup_hud_telemetry
 
 if TYPE_CHECKING:
@@ -37,7 +37,7 @@ class MCPAgent(ABC):
       and automatic marking of lifecycle tools (setup/evaluate) from a `Task`.
     - Messaging: system prompt handling, optional inclusion of setup output on
       the first turn, and control over initial screenshots.
-    - Telemetry & UX: standardized logging/printing via `HUDDesign` and optional
+    - Telemetry & UX: standardized logging/printing via `HUDConsole` and optional
       automatic tracing (`auto_trace`).
 
     Subclasses implement provider-specific formatting and response fetching
@@ -92,11 +92,11 @@ class MCPAgent(ABC):
         self._auto_created_client = False  # Track if we created the client
 
         self.model_name = model_name
-        self.design = HUDDesign(logger=logger)
+        self.console = HUDConsole(logger=logger)
 
         # Set verbose mode if requested
         if verbose:
-            self.design.set_verbose(True)
+            self.console.set_verbose(True)
 
         # Filtering
         self.allowed_tools = allowed_tools
@@ -131,7 +131,7 @@ class MCPAgent(ABC):
 
             self.mcp_client = MCPClient(mcp_config=task.mcp_config)
             self._auto_created_client = True
-            self.design.info_log("Auto-created MCPClient from task.mcp_config")
+            self.console.info_log("Auto-created MCPClient from task.mcp_config")
 
         # Ensure we have a client
         if self.mcp_client is None:
@@ -168,7 +168,7 @@ class MCPAgent(ABC):
         await self._filter_tools()
 
         num_tools = len(self._available_tools)
-        self.design.success_log(
+        self.console.success_log(
             f"Agent initialized with {num_tools} available tools (after filtering)"
         )
 
@@ -243,7 +243,7 @@ class MCPAgent(ABC):
 
             # Execute the setup tool and append the initial observation to the context
             if task.setup_tool is not None:
-                self.design.progress_log(f"Setting up tool phase: {task.setup_tool}")
+                self.console.progress_log(f"Setting up tool phase: {task.setup_tool}")
                 results = await self.call_tools(task.setup_tool)
                 if any(result.isError for result in results):
                     raise RuntimeError(f"{results}")
@@ -257,7 +257,7 @@ class MCPAgent(ABC):
             prompt_result = await self._run_context(start_context, max_steps=max_steps)
 
         except Exception as e:
-            self.design.error_log(f"Task execution failed: {e}")
+            self.console.error_log(f"Task execution failed: {e}")
             # Create an error result but don't return yet - we still want to evaluate
             prompt_result = Trace(reward=0.0, done=True, content=str(e), isError=True)
             prompt_result.populate_from_context()
@@ -265,7 +265,7 @@ class MCPAgent(ABC):
         # Always evaluate if we have a prompt result and evaluate tool
         if prompt_result is not None and task.evaluate_tool is not None:
             try:
-                self.design.progress_log(f"Evaluating tool phase: {task.evaluate_tool}")
+                self.console.progress_log(f"Evaluating tool phase: {task.evaluate_tool}")
                 results = await self.call_tools(task.evaluate_tool)
 
                 if any(result.isError for result in results):
@@ -288,7 +288,7 @@ class MCPAgent(ABC):
                             prompt_result.content = eval_content
 
             except Exception as e:
-                self.design.error_log(f"Evaluation phase failed: {e}")
+                self.console.error_log(f"Evaluation phase failed: {e}")
                 # Continue with the prompt result even if evaluation failed
 
         return (
@@ -319,21 +319,21 @@ class MCPAgent(ABC):
 
             # Add initial context
             messages.extend(await self.format_message(context))
-            self.design.debug(f"Messages: {messages}")
+            self.console.debug(f"Messages: {messages}")
 
             step_count = 0
             while max_steps == -1 or step_count < max_steps:
                 step_count += 1
                 if max_steps == -1:
-                    self.design.debug(f"Step {step_count} (unlimited)")
+                    self.console.debug(f"Step {step_count} (unlimited)")
                 else:
-                    self.design.debug(f"Step {step_count}/{max_steps}")
+                    self.console.debug(f"Step {step_count}/{max_steps}")
 
                 try:
                     # 1. Get model response
                     response = await self.get_response(messages)
 
-                    self.design.debug(f"Agent:\n{response}")
+                    self.console.debug(f"Agent:\n{response}")
 
                     # Check if we should stop
                     if response.done or not response.tool_calls:
@@ -345,16 +345,16 @@ class MCPAgent(ABC):
                                     response.content
                                 )
                             except Exception as e:
-                                self.design.warning_log(f"ResponseAgent failed: {e}")
+                                self.console.warning_log(f"ResponseAgent failed: {e}")
                         if decision == "STOP":
                             # Try to submit response through lifecycle tool
                             await self._maybe_submit_response(response, messages)
 
-                            self.design.debug("Stopping execution")
+                            self.console.debug("Stopping execution")
                             final_response = response
                             break
                         else:
-                            self.design.debug("Continuing execution")
+                            self.console.debug("Continuing execution")
                             messages.extend(await self.format_message(decision))
                             continue
 
@@ -376,21 +376,21 @@ class MCPAgent(ABC):
                     for call, result in zip(tool_calls, tool_results, strict=False):
                         step_info += f"\n{call}\n{result}"
 
-                    self.design.info_log(step_info)
+                    self.console.info_log(step_info)
 
                 except Exception as e:
-                    self.design.error_log(f"Step failed: {e}")
+                    self.console.error_log(f"Step failed: {e}")
                     error = str(e)
                     break
 
         except KeyboardInterrupt:
-            self.design.warning_log("Agent execution interrupted by user")
+            self.console.warning_log("Agent execution interrupted by user")
             error = "Interrupted by user"
         except asyncio.CancelledError:
-            self.design.warning_log("Agent execution cancelled")
+            self.console.warning_log("Agent execution cancelled")
             error = "Cancelled"
         except Exception as e:
-            self.design.error_log(f"Unexpected error: {e}")
+            self.console.error_log(f"Unexpected error: {e}")
             error = str(e)
 
         # Build result
@@ -431,17 +431,17 @@ class MCPAgent(ABC):
         results: list[MCPToolResult] = []
         for tc in tool_call:
             try:
-                self.design.debug(f"Calling tool: {tc}")
+                self.console.debug(f"Calling tool: {tc}")
                 results.append(await self.mcp_client.call_tool(tc))
             except TimeoutError as e:
-                self.design.error_log(f"Tool execution timed out: {e}")
+                self.console.error_log(f"Tool execution timed out: {e}")
                 try:
                     await self.mcp_client.shutdown()
                 except Exception as close_err:
-                    self.design.debug(f"Failed to close MCP client cleanly: {close_err}")
+                    self.console.debug(f"Failed to close MCP client cleanly: {close_err}")
                 raise
             except Exception as e:
-                self.design.error_log(f"Tool execution failed: {e}")
+                self.console.error_log(f"Tool execution failed: {e}")
                 results.append(_format_error_result(str(e)))
         return results
 
@@ -573,7 +573,7 @@ class MCPAgent(ABC):
 
             # Add to lifecycle tools if found
             if response_tool_name and response_tool_name not in self.lifecycle_tools:
-                self.design.debug(f"Auto-detected '{response_tool_name}' tool as a lifecycle tool")
+                self.console.debug(f"Auto-detected '{response_tool_name}' tool as a lifecycle tool")
                 self.response_tool_name = response_tool_name
                 self.lifecycle_tools.append(response_tool_name)
 
@@ -597,7 +597,7 @@ class MCPAgent(ABC):
             messages: The current message history (will be modified in-place)
         """
         if self.response_tool_name:
-            self.design.debug(f"Calling response lifecycle tool: {self.response_tool_name}")
+            self.console.debug(f"Calling response lifecycle tool: {self.response_tool_name}")
             try:
                 # Call the response tool with the agent's response
                 response_tool_call = MCPToolCall(
@@ -612,9 +612,9 @@ class MCPAgent(ABC):
                 messages.extend(response_messages)
 
                 # Mark the task as done
-                self.design.debug("Response lifecycle tool executed, marking task as done")
+                self.console.debug("Response lifecycle tool executed, marking task as done")
             except Exception as e:
-                self.design.error_log(f"Response lifecycle tool failed: {e}")
+                self.console.error_log(f"Response lifecycle tool failed: {e}")
 
     async def _setup_config(self, mcp_config: dict[str, dict[str, Any]]) -> None:
         """Inject metadata into the metadata of the initialize request."""
@@ -668,9 +668,9 @@ class MCPAgent(ABC):
         if self._auto_trace_cm:
             try:
                 self._auto_trace_cm.__exit__(None, None, None)
-                self.design.debug("Closed auto-created trace")
+                self.console.debug("Closed auto-created trace")
             except Exception as e:
-                self.design.warning_log(f"Failed to close auto-created trace: {e}")
+                self.console.warning_log(f"Failed to close auto-created trace: {e}")
             finally:
                 self._auto_trace_cm = None
 
@@ -678,9 +678,9 @@ class MCPAgent(ABC):
         if self._auto_created_client and self.mcp_client:
             try:
                 await self.mcp_client.shutdown()
-                self.design.debug("Closed auto-created MCPClient")
+                self.console.debug("Closed auto-created MCPClient")
             except Exception as e:
-                self.design.warning_log(f"Failed to close auto-created client: {e}")
+                self.console.warning_log(f"Failed to close auto-created client: {e}")
             finally:
                 self.mcp_client = None
                 self._auto_created_client = False
@@ -713,13 +713,13 @@ class MCPAgent(ABC):
         if self._is_connection_error(e):
             msg = self._get_connection_error_message(e)
             # Always show connection errors, not just when logging is enabled
-            self.design.error(f"❌ {msg}")
-            self.design.info("💡 Make sure the MCP server is started before running the agent.")
+            self.console.error(f"❌ {msg}")
+            self.console.info("💡 Make sure the MCP server is started before running the agent.")
 
             # For localhost, provide specific instructions
             error_str = str(e).lower()
             if "localhost" in error_str or "127.0.0.1" in error_str:
-                self.design.info("   Run 'hud dev' in another terminal to start the MCP server")
+                self.console.info("   Run 'hud dev' in another terminal to start the MCP server")
 
             raise RuntimeError(msg) from e
         raise

hud/agents/claude.py

@@ -364,16 +364,21 @@ class ClaudeAgent(MCPAgent):
         messages_cached = copy.deepcopy(messages)
 
         # Mark last user message with cache control
-        if messages_cached and messages_cached[-1].get("role") == "user":
+        if (
+            messages_cached
+            and isinstance(messages_cached[-1], dict)
+            and messages_cached[-1].get("role") == "user"
+        ):
             last_content = messages_cached[-1]["content"]
             # Content is formatted to be list of ContentBlock in format_blocks and format_message
             if isinstance(last_content, list):
                 for block in last_content:
-                    # Only add cache control to block types that support it
-                    block_type = block.get("type")
-                    if block_type in ["text", "image", "tool_use", "tool_result"]:
-                        cache_control: BetaCacheControlEphemeralParam = {"type": "ephemeral"}
-                        block["cache_control"] = cache_control  # type: ignore[reportGeneralTypeIssues]
+                    # Only add cache control to dict-like block types that support it
+                    if isinstance(block, dict):
+                        block_type = block.get("type")
+                        if block_type in ["text", "image", "tool_use", "tool_result"]:
+                            cache_control: BetaCacheControlEphemeralParam = {"type": "ephemeral"}
+                            block["cache_control"] = cache_control  # type: ignore[reportGeneralTypeIssues]
 
         return messages_cached
 

hud/agents/grounded_openai.py

@@ -0,0 +1,282 @@
+"""Grounded OpenAI agent that separates visual grounding from reasoning."""
+
+from __future__ import annotations
+
+import json
+from typing import Any, ClassVar
+
+from hud import instrument
+from hud.tools.grounding import GroundedComputerTool, Grounder, GrounderConfig
+from hud.types import AgentResponse, MCPToolCall, MCPToolResult
+
+from .openai_chat_generic import GenericOpenAIChatAgent
+
+
+class GroundedOpenAIChatAgent(GenericOpenAIChatAgent):
+    """OpenAI agent that uses a separate grounding model for element detection.
+
+    This agent:
+    - Exposes only a synthetic "computer" tool to the planning model
+    - Intercepts tool calls to ground element descriptions to coordinates
+    - Converts grounded results to real computer tool calls
+    - Maintains screenshot state for grounding operations
+
+    The architecture separates concerns:
+    - Planning model (GPT-4o etc) focuses on high-level reasoning
+    - Grounding model (Qwen2-VL etc) handles visual element detection
+    """
+
+    metadata: ClassVar[dict[str, Any]] = {}
+
+    def __init__(
+        self,
+        *,
+        grounder_config: GrounderConfig,
+        model_name: str = "gpt-4o-mini",
+        allowed_tools: list[str] | None = None,
+        append_setup_output: bool = False,
+        system_prompt: str | None = None,
+        **kwargs: Any,
+    ) -> None:
+        """Initialize the grounded OpenAI agent.
+
+        Args:
+            grounder_config: Configuration for the grounding model
+            openai_client: OpenAI client for the planning model
+            model: Name of the OpenAI model to use for planning (e.g., "gpt-4o", "gpt-4o-mini")
+            real_computer_tool_name: Name of the actual computer tool to execute
+            **kwargs: Additional arguments passed to GenericOpenAIChatAgent
+        """
+        # Set defaults for grounded agent
+        if allowed_tools is None:
+            allowed_tools = ["computer"]
+
+        if system_prompt is None:
+            system_prompt = (
+                "You are a helpful AI assistant that can control the computer "
+                "through visual interaction.\n\n"
+                "IMPORTANT: Always explain your reasoning and observations before taking actions:\n"
+                "1. First, describe what you see on the screen\n"
+                "2. Explain what you plan to do and why\n"
+                "3. Then use the computer tool with natural language descriptions\n\n"
+                "For example:\n"
+                "- 'I can see a login form with username and password fields. "
+                "I need to click on the username field first.'\n"
+                "- 'There's a blue submit button at the bottom. "
+                "I'll click on it to submit the form.'\n"
+                "- 'I notice a red close button in the top right corner. "
+                "I'll click it to close this dialog.'\n\n"
+                "Use descriptive element descriptions like:\n"
+                "- Colors: 'red button', 'blue link', 'green checkmark'\n"
+                "- Position: 'top right corner', 'bottom of the page', 'left sidebar'\n"
+                "- Text content: 'Submit button', 'Login link', 'Cancel option'\n"
+                "- Element type: 'text field', 'dropdown menu', 'checkbox'"
+            )
+
+        super().__init__(
+            model_name=model_name,
+            allowed_tools=allowed_tools,
+            append_setup_output=append_setup_output,
+            system_prompt=system_prompt,
+            **kwargs,
+        )
+
+        self.grounder = Grounder(grounder_config)
+        self.grounded_tool = None
+
+    async def initialize(self, task: Any = None) -> None:
+        """Initialize the agent and create the grounded tool with mcp_client."""
+        # Call parent initialization first
+        await super().initialize(task)
+
+        if self.mcp_client is None:
+            raise ValueError("mcp_client must be initialized before creating grounded tool")
+        self.grounded_tool = GroundedComputerTool(
+            grounder=self.grounder, mcp_client=self.mcp_client, computer_tool_name="computer"
+        )
+
+    def get_tool_schemas(self) -> list[Any]:
+        """Override to expose only the synthetic grounded tool.
+
+        The planning model only sees the synthetic "computer" tool,
+        which is provided by the grounded tool itself.
+
+        Returns:
+            List containing only the grounded computer tool schema
+        """
+        if self.grounded_tool is None:
+            return []
+        return [self.grounded_tool.get_openai_tool_schema()]
+
+    @instrument(
+        span_type="agent",
+        record_args=False,
+        record_result=True,
+    )
+    async def get_response(self, messages: Any) -> AgentResponse:
+        """Get response from the planning model and handle grounded tool calls.
+
+        This method:
+        1. Calls the planning model with the grounded tool schema
+        2. Executes any tool calls directly through the grounded tool
+        3. Returns the response
+
+        Args:
+            messages: Conversation messages
+
+        Returns:
+            AgentResponse with either content or tool calls for MCP execution
+        """
+        tool_schemas = self.get_tool_schemas()
+
+        # Take initial screenshot and add to messages if this is the first turn
+        has_image = any(
+            isinstance(m.get("content"), list)
+            and any(
+                block.get("type") == "image_url"
+                for block in m["content"]
+                if isinstance(block, dict)
+            )
+            for m in messages
+            if isinstance(m.get("content"), list)
+        )
+
+        if not has_image:
+            if self.mcp_client is None:
+                raise ValueError("mcp_client is not initialized")
+            screenshot_result = await self.mcp_client.call_tool(
+                MCPToolCall(name="computer", arguments={"action": "screenshot"})
+            )
+
+            for block in screenshot_result.content:
+                # Check for ImageContent type from MCP
+                if hasattr(block, "data") and hasattr(block, "mimeType"):
+                    mime_type = getattr(block, "mimeType", "image/png")
+                    data = getattr(block, "data", "")
+                    messages.append(
+                        {
+                            "role": "user",
+                            "content": [
+                                {
+                                    "type": "image_url",
+                                    "image_url": {"url": f"data:{mime_type};base64,{data}"},
+                                }
+                            ],
+                        }
+                    )
+                    break
+
+        protected_keys = {"model", "messages", "tools", "parallel_tool_calls"}
+        extra = {k: v for k, v in (self.completion_kwargs or {}).items() if k not in protected_keys}
+
+        response = await self.oai.chat.completions.create(
+            model=self.model_name,
+            messages=messages,
+            tools=tool_schemas,
+            parallel_tool_calls=False,
+            **extra,
+        )
+
+        choice = response.choices[0]
+        msg = choice.message
+
+        assistant_msg: dict[str, Any] = {"role": "assistant"}
+        if msg.content:
+            assistant_msg["content"] = msg.content
+        if msg.tool_calls:
+            assistant_msg["tool_calls"] = msg.tool_calls
+
+        messages.append(assistant_msg)
+
+        self.conversation_history = messages.copy()
+
+        if not msg.tool_calls:
+            return AgentResponse(
+                content=msg.content or "",
+                tool_calls=[],
+                done=choice.finish_reason in ("stop", "length"),
+                raw=response,
+            )
+
+        tc = msg.tool_calls[0]
+
+        if tc.function.name != "computer":
+            return AgentResponse(
+                content=f"Error: Model called unexpected tool '{tc.function.name}'",
+                tool_calls=[],
+                done=True,
+                raw=response,
+            )
+
+        # Parse the arguments
+        try:
+            args = json.loads(tc.function.arguments or "{}")
+        except json.JSONDecodeError:
+            return AgentResponse(
+                content="Error: Invalid tool arguments", tool_calls=[], done=True, raw=response
+            )
+
+        tool_call = MCPToolCall(name="computer", arguments=args, id=tc.id)
+
+        return AgentResponse(
+            content=msg.content or "", tool_calls=[tool_call], done=False, raw=response
+        )
+
+    async def call_tools(
+        self, tool_call: MCPToolCall | list[MCPToolCall] | None = None
+    ) -> list[MCPToolResult]:
+        """Override call_tools to intercept computer tool calls.
+
+        Execute them through grounded tool.
+        """
+        if tool_call is None:
+            return []
+
+        if isinstance(tool_call, MCPToolCall):
+            tool_call = [tool_call]
+
+        results: list[MCPToolResult] = []
+        for tc in tool_call:
+            if tc.name == "computer":
+                # Execute through grounded tool instead of MCP
+                try:
+                    # Extract latest screenshot from conversation history
+                    screenshot_b64 = None
+                    for m in reversed(self.conversation_history):
+                        if m.get("role") == "user" and isinstance(m.get("content"), list):
+                            for block in m["content"]:
+                                if (
+                                    isinstance(block, dict)
+                                    and block.get("type") == "image_url"
+                                    and isinstance(block.get("image_url"), dict)
+                                ):
+                                    url = block["image_url"].get("url", "")
+                                    if url.startswith("data:"):
+                                        screenshot_b64 = (
+                                            url.split(",", 1)[1] if "," in url else None
+                                        )
+                                        break
+                            if screenshot_b64:
+                                break
+
+                    # Pass screenshot to grounded tool
+                    args_with_screenshot = dict(tc.arguments) if tc.arguments else {}
+                    if screenshot_b64:
+                        args_with_screenshot["screenshot_b64"] = screenshot_b64
+
+                    if self.grounded_tool is None:
+                        raise ValueError("Grounded tool is not initialized")
+                    content_blocks = await self.grounded_tool(**args_with_screenshot)
+                    results.append(MCPToolResult(content=content_blocks, isError=False))
+                except Exception as e:
+                    # Create error result
+                    from mcp.types import TextContent
+
+                    error_content = TextContent(text=str(e), type="text")
+                    results.append(MCPToolResult(content=[error_content], isError=True))
+            else:
+                # For non-computer tools, use parent implementation
+                parent_results = await super().call_tools(tc)
+                results.extend(parent_results)
+
+        return results

hud/agents/misc/response_agent.py

@@ -16,7 +16,7 @@ class ResponseAgent:
     based on the agent's final response message.
     """
 
-    def __init__(self, api_key: str | None = None) -> None:
+    def __init__(self, api_key: str | None = None, model: str = "gpt-4o") -> None:
         self.api_key = api_key or settings.openai_api_key or os.environ.get("OPENAI_API_KEY")
         if not self.api_key:
             raise ValueError(
@@ -24,6 +24,7 @@ class ResponseAgent:
             )
 
         self.client = AsyncOpenAI(api_key=self.api_key)
+        self.model = model
 
         self.system_prompt = """
         You are an assistant that helps determine the appropriate response to an agent's message.
@@ -54,7 +55,7 @@ class ResponseAgent:
         """
         try:
             response = await self.client.chat.completions.create(
-                model="gpt-5-nano",
+                model=self.model,
                 messages=[
                     {"role": "system", "content": self.system_prompt},
                     {

hud/agents/openai.py

@@ -204,7 +204,7 @@ class OperatorAgent(MCPAgent):
                         break
 
                 if not latest_screenshot:
-                    self.design.warning_log("No screenshot provided for response to action")
+                    self.console.warning_log("No screenshot provided for response to action")
                     return AgentResponse(
                         content="No screenshot available for next action",
                         tool_calls=[],
@@ -327,7 +327,7 @@ class OperatorAgent(MCPAgent):
                 for content in result.content:
                     if isinstance(content, types.TextContent):
                         # Don't add error text as input_text, just track it
-                        self.design.error_log(f"Tool error: {content.text}")
+                        self.console.error_log(f"Tool error: {content.text}")
                     elif isinstance(content, types.ImageContent):
                         # Even error results might have images
                         latest_screenshot = content.data

hud/agents/openai_chat_generic.py

@@ -17,7 +17,7 @@ from __future__ import annotations
 
 import json
 import logging
-from typing import TYPE_CHECKING, Any, cast
+from typing import TYPE_CHECKING, Any, ClassVar, cast
 
 import mcp.types as types
 
@@ -36,6 +36,8 @@ logger = logging.getLogger(__name__)
 class GenericOpenAIChatAgent(MCPAgent):
     """MCP-enabled agent that speaks the OpenAI *chat.completions* protocol."""
 
+    metadata: ClassVar[dict[str, Any]] = {}
+
     def __init__(
         self,
         *,

hud/agents/tests/test_client.py

@@ -200,7 +200,12 @@ class TestMCPClient:
         # Calling a non-existent tool should return an error result
         result = await client.call_tool(name="nonexistent", arguments={})
         assert result.isError is True
-        assert "Tool 'nonexistent' not found" in result.content[0].text
+        # Check that the error message is in the text content
+        text_content = ""
+        for content in result.content:
+            if isinstance(content, types.TextContent):
+                text_content += content.text
+        assert "Tool 'nonexistent' not found" in text_content
 
     @pytest.mark.asyncio
     async def test_get_telemetry_data(self, mock_telemetry, mock_mcp_use_client):