hud-python 0.4.45__py3-none-any.whl → 0.5.13__py3-none-any.whl

hud/agents/grounded_openai.py

@@ -3,96 +3,91 @@
 from __future__ import annotations
 
 import json
-from typing import Any, ClassVar
+from typing import TYPE_CHECKING, Any, ClassVar
+
+from pydantic import ConfigDict, field_validator
 
-from hud import instrument
 from hud.tools.grounding import GroundedComputerTool, Grounder, GrounderConfig
 from hud.types import AgentResponse, MCPToolCall, MCPToolResult
-
-from .openai_chat_generic import GenericOpenAIChatAgent
-
-
-class GroundedOpenAIChatAgent(GenericOpenAIChatAgent):
-    """OpenAI agent that uses a separate grounding model for element detection.
-
-    This agent:
-    - Exposes only a synthetic "computer" tool to the planning model
-    - Intercepts tool calls to ground element descriptions to coordinates
-    - Converts grounded results to real computer tool calls
-    - Maintains screenshot state for grounding operations
-
-    The architecture separates concerns:
-    - Planning model (GPT-4o etc) focuses on high-level reasoning
-    - Grounding model (Qwen2-VL etc) handles visual element detection
-    """
-
-    metadata: ClassVar[dict[str, Any]] = {}
-
-    def __init__(
-        self,
-        *,
-        grounder_config: GrounderConfig,
-        model_name: str = "gpt-4o-mini",
-        allowed_tools: list[str] | None = None,
-        append_setup_output: bool = False,
-        system_prompt: str | None = None,
-        **kwargs: Any,
-    ) -> None:
-        """Initialize the grounded OpenAI agent.
-
-        Args:
-            grounder_config: Configuration for the grounding model
-            openai_client: OpenAI client for the planning model
-            model: Name of the OpenAI model to use for planning (e.g., "gpt-4o", "gpt-4o-mini")
-            real_computer_tool_name: Name of the actual computer tool to execute
-            **kwargs: Additional arguments passed to GenericOpenAIChatAgent
-        """
-        # Set defaults for grounded agent
-        if allowed_tools is None:
-            allowed_tools = ["computer"]
-
-        if system_prompt is None:
-            system_prompt = (
-                "You are a helpful AI assistant that can control the computer "
-                "through visual interaction.\n\n"
-                "IMPORTANT: Always explain your reasoning and observations before taking actions:\n"
-                "1. First, describe what you see on the screen\n"
-                "2. Explain what you plan to do and why\n"
-                "3. Then use the computer tool with natural language descriptions\n\n"
-                "For example:\n"
-                "- 'I can see a login form with username and password fields. "
-                "I need to click on the username field first.'\n"
-                "- 'There's a blue submit button at the bottom. "
-                "I'll click on it to submit the form.'\n"
-                "- 'I notice a red close button in the top right corner. "
-                "I'll click it to close this dialog.'\n\n"
-                "Use descriptive element descriptions like:\n"
-                "- Colors: 'red button', 'blue link', 'green checkmark'\n"
-                "- Position: 'top right corner', 'bottom of the page', 'left sidebar'\n"
-                "- Text content: 'Submit button', 'Login link', 'Cancel option'\n"
-                "- Element type: 'text field', 'dropdown menu', 'checkbox'"
-            )
-
-        super().__init__(
-            model_name=model_name,
-            allowed_tools=allowed_tools,
-            append_setup_output=append_setup_output,
-            system_prompt=system_prompt,
-            **kwargs,
-        )
-
-        self.grounder = Grounder(grounder_config)
-        self.grounded_tool = None
-
-    async def initialize(self, task: Any = None) -> None:
-        """Initialize the agent and create the grounded tool with mcp_client."""
-        # Call parent initialization first
-        await super().initialize(task)
-
-        if self.mcp_client is None:
-            raise ValueError("mcp_client must be initialized before creating grounded tool")
+from hud.utils.types import with_signature
+
+if TYPE_CHECKING:
+    from hud.types import BaseAgentConfig
+from .base import BaseCreateParams
+from .openai_chat import OpenAIChatAgent, OpenAIChatConfig
+
+DEFAULT_GROUNDED_PROMPT = (
+    "You are a helpful AI assistant that can control the computer through visual "
+    "interaction.\n\n"
+    "IMPORTANT: Always explain your reasoning and observations before taking actions:\n"
+    "1. First, describe what you see on the screen.\n"
+    "2. Explain what you plan to do and why.\n"
+    "3. Then use the computer tool with natural language descriptions.\n\n"
+    "Use descriptive element descriptions:\n"
+    '- Colors ("red button", "blue link")\n'
+    '- Position ("top right corner", "left sidebar")\n'
+    '- Text content ("Submit button", "Login link")\n'
+    '- Element type ("text field", "dropdown")'
+)
+
+
+class GroundedOpenAIConfig(OpenAIChatConfig):
+    """Configuration for grounded OpenAI chat agent."""
+
+    model_config = ConfigDict(arbitrary_types_allowed=True)
+
+    grounder_config: GrounderConfig
+    model: str = "gpt-4o-mini"
+    allowed_tools: list[str] | None = None  # Default set in validator
+    append_setup_output: bool = False
+    system_prompt: str | None = DEFAULT_GROUNDED_PROMPT
+
+    @field_validator("grounder_config", mode="before")
+    @classmethod
+    def _coerce_grounder_config(cls, value: GrounderConfig | dict[str, Any]) -> GrounderConfig:
+        if isinstance(value, GrounderConfig):
+            return value
+        if isinstance(value, dict):
+            return GrounderConfig(**value)
+
+    @field_validator("allowed_tools", mode="before")
+    @classmethod
+    def _default_allowed_tools(cls, value: list[str] | None) -> list[str] | None:
+        if value is None:
+            return ["computer"]
+        return value
+
+
+class GroundedOpenAICreateParams(BaseCreateParams, GroundedOpenAIConfig):
+    pass
+
+
+class GroundedOpenAIChatAgent(OpenAIChatAgent):
+    """OpenAI chat agent that pipes 'computer' tool calls through a vision grounder."""
+
+    metadata: ClassVar[dict[str, Any] | None] = None
+    config_cls: ClassVar[type[BaseAgentConfig]] = GroundedOpenAIConfig
+
+    @with_signature(GroundedOpenAICreateParams)
+    @classmethod
+    def create(cls, **kwargs: Any) -> GroundedOpenAIChatAgent:  # pyright: ignore[reportIncompatibleMethodOverride]
+        from .base import MCPAgent
+
+        return MCPAgent.create.__func__(cls, **kwargs)  # type: ignore[return-value]
+
+    def __init__(self, params: GroundedOpenAICreateParams | None = None, **kwargs: Any) -> None:
+        super().__init__(params, **kwargs)  # type: ignore[arg-type]
+        self.config: GroundedOpenAIConfig  # type: ignore[assignment]
+
+        self.grounder = Grounder(self.config.grounder_config)
+        self.grounded_tool: GroundedComputerTool | None = None
+
+    def _on_tools_ready(self) -> None:
+        """Create the grounded tool after context is bound."""
+        if self.ctx is None:
+            raise ValueError("ctx must be set before creating grounded tool")
         self.grounded_tool = GroundedComputerTool(
-            grounder=self.grounder, mcp_client=self.mcp_client, computer_tool_name="computer"
+            grounder=self.grounder, ctx=self.ctx, computer_tool_name="computer"
         )
 
     def get_tool_schemas(self) -> list[Any]:
@@ -108,11 +103,6 @@ class GroundedOpenAIChatAgent(GenericOpenAIChatAgent):
             return []
         return [self.grounded_tool.get_openai_tool_schema()]
 
-    @instrument(
-        span_type="agent",
-        record_args=False,
-        record_result=True,
-    )
     async def get_response(self, messages: Any) -> AgentResponse:
         """Get response from the planning model and handle grounded tool calls.
 
@@ -142,11 +132,9 @@ class GroundedOpenAIChatAgent(GenericOpenAIChatAgent):
         )
 
         if not has_image:
-            if self.mcp_client is None:
-                raise ValueError("mcp_client is not initialized")
-            screenshot_result = await self.mcp_client.call_tool(
-                MCPToolCall(name="computer", arguments={"action": "screenshot"})
-            )
+            if self.ctx is None:
+                raise ValueError("ctx is not initialized")
+            screenshot_result = await self.ctx.call_tool(("computer", {"action": "screenshot"}))
 
             for block in screenshot_result.content:
                 # Check for ImageContent type from MCP
@@ -169,8 +157,8 @@ class GroundedOpenAIChatAgent(GenericOpenAIChatAgent):
         protected_keys = {"model", "messages", "tools", "parallel_tool_calls"}
         extra = {k: v for k, v in (self.completion_kwargs or {}).items() if k not in protected_keys}
 
-        response = await self.oai.chat.completions.create(
-            model=self.model_name,
+        response = await self.oai.chat.completions.create(  # type: ignore
+            model=self.config.model,
             messages=messages,
             tools=tool_schemas,
             parallel_tool_calls=False,
@@ -193,6 +181,7 @@ class GroundedOpenAIChatAgent(GenericOpenAIChatAgent):
         if not msg.tool_calls:
             return AgentResponse(
                 content=msg.content or "",
+                reasoning=msg.reasoning_content,
                 tool_calls=[],
                 done=choice.finish_reason in ("stop", "length"),
                 raw=response,
@@ -203,6 +192,7 @@ class GroundedOpenAIChatAgent(GenericOpenAIChatAgent):
         if tc.function.name != "computer":
             return AgentResponse(
                 content=f"Error: Model called unexpected tool '{tc.function.name}'",
+                reasoning=msg.reasoning_content,
                 tool_calls=[],
                 done=True,
                 raw=response,
@@ -213,13 +203,21 @@ class GroundedOpenAIChatAgent(GenericOpenAIChatAgent):
             args = json.loads(tc.function.arguments or "{}")
         except json.JSONDecodeError:
             return AgentResponse(
-                content="Error: Invalid tool arguments", tool_calls=[], done=True, raw=response
+                content="Error: Invalid tool arguments",
+                reasoning=msg.reasoning_content,
+                tool_calls=[],
+                done=True,
+                raw=response,
             )
 
         tool_call = MCPToolCall(name="computer", arguments=args, id=tc.id)
 
         return AgentResponse(
-            content=msg.content or "", tool_calls=[tool_call], done=False, raw=response
+            content=msg.content or "",
+            reasoning=msg.reasoning_content,
+            tool_calls=[tool_call],
+            done=False,
+            raw=response,
         )
 
     async def call_tools(

hud/agents/misc/integration_test_agent.py

@@ -1,41 +1,72 @@
 from __future__ import annotations
 
-from typing import Any
+from typing import TYPE_CHECKING, Any, ClassVar
 
-from hud.agents.base import MCPAgent, find_reward
-from hud.types import AgentResponse, Task, Trace
+from hud.agents.base import MCPAgent
+from hud.types import AgentResponse, BaseAgentConfig, Trace
+
+if TYPE_CHECKING:
+    from hud.eval.context import EvalContext
 
 
 class IntegrationTestRunner(MCPAgent):
+    """Special agent that runs integration tests by executing tools directly.
+
+    Unlike regular agents, this doesn't run an LLM loop - it executes
+    integration_test_tool and evaluate_tool in sequence to verify tool behavior.
+    """
+
+    metadata: ClassVar[dict[str, Any] | None] = {}
+    config_cls: ClassVar[type[BaseAgentConfig]] = BaseAgentConfig
+
     def __init__(self, **kwargs: Any) -> None:
         kwargs["auto_trace"] = False
         super().__init__(**kwargs)
-        self.metadata = {}
 
-    async def run(self, task: Task, max_steps: int = 10) -> Trace:
+    async def run(
+        self,
+        ctx: EvalContext,
+        *,
+        max_steps: int = 10,
+    ) -> Trace:
+        """Run integration test by executing tools directly.
+
+        The EvalContext should have integration_test_tool and evaluate_tool
+        configured in its metadata or environment setup.
+        """
+        from hud.eval.context import EvalContext
+
+        if not isinstance(ctx, EvalContext):
+            raise TypeError(f"ctx must be EvalContext, got {type(ctx).__name__}")
+
+        self.ctx = ctx
+
         try:
-            # Initialize using base to set up client and telemetry correctly
-            await self.initialize(task)
+            # Initialize tools from context
+            if not self._initialized:
+                await self._initialize_from_ctx(ctx)
 
-            # Validate task shape
-            if not getattr(task, "integration_test_tool", None):
+            self.console.info(f"Full system prompt: {self.system_prompt}")
+
+            # For integration tests, we expect the context's environment to have
+            # _setup_calls, _integration_test_calls, and _evaluate_calls configured
+            env = ctx
+
+            # Run integration test tool (stored in environment metadata or separate list)
+            integration_test_calls = getattr(env, "_integration_test_calls", [])
+            if not integration_test_calls:
                 raise ValueError(
-                    "--integration-test requires task.integration_test_tool (single call)"
+                    "--integration-test requires integration_test_tool to be configured"
                 )
-            elif not getattr(task, "evaluate_tool", None):
-                raise ValueError("--integration-test requires task.evaluate_tool (single call)")
-
-            if task.setup_tool:
-                _ = await self.call_tools(task.setup_tool)
 
-            _ = await self.call_tools(task.integration_test_tool)
-            evaluate_result = await self.call_tools(task.evaluate_tool)
+            for name, args in integration_test_calls:
+                await ctx.call_tool((name, args))
 
-            reward = float(find_reward(evaluate_result[0])) if evaluate_result else 0.0
+            # The evaluate phase runs automatically when ctx exits,
+            # but we can also get the reward from ctx.reward after
+            return Trace(done=True, reward=ctx.reward or 0.0, info={})
 
-            return Trace(done=True, reward=reward, info={})
         finally:
-            # Ensure resources are cleaned up so the CLI can exit cleanly
             await self._cleanup()
 
     # Stub implementations to satisfy abstract base class; not used in --integration-test path

hud/agents/misc/response_agent.py

@@ -1,14 +1,37 @@
 from __future__ import annotations
 
-import os
+import logging
 from typing import Literal
 
 from openai import AsyncOpenAI
 
 from hud.settings import settings
+from hud.telemetry import instrument
+
+logger = logging.getLogger(__name__)
 
 ResponseType = Literal["STOP", "CONTINUE"]
 
+DEFAULT_SYSTEM_PROMPT = """\
+You are an assistant that helps determine the appropriate response to an agent's message.
+
+You will receive messages from an agent that is performing tasks for a user.
+Your job is to analyze these messages and respond with one of the following:
+
+- STOP: If the agent indicates it has successfully completed a task or is stuck,
+  struggling or says it cannot complete the task, even if phrased as a question
+  like "I have entered the right values into this form. Would you like me to do
+  anything else?" or "Here is the website. Is there any other information you
+  need?" or if the agent has strongly determined it wants to stop the task like
+  "The task is infeasible. Can I help you with something else?"
+
+- CONTINUE: If the agent is asking for clarification before proceeding with a task
+  like "I'm about to clear cookies from this website. Would you like me to proceed?"
+  or "I've entered the right values into this form. Would you like me to continue
+  with the rest of the task?"
+
+Respond ONLY with one of these two options."""
+
 
 class ResponseAgent:
     """
@@ -17,49 +40,36 @@ class ResponseAgent:
     """
 
     def __init__(
-        self, api_key: str | None = None, model: str = "gpt-4o", system_prompt: str | None = None
+        self,
+        model: str = "gpt-4o",
+        system_prompt: str | None = None,
     ) -> None:
         """
         Initialize the ResponseAgent.
 
         Args:
-            api_key: The API key to use for the OpenAI client
-            model: The model to use for the OpenAI client (default: "gpt-4o")
-            system_prompt: The system prompt to use for the OpenAI client
+            model: The model to use via HUD inference gateway (default: "gpt-4o").
+                   Supports any model available through inference.hud.ai.
+            system_prompt: Optional custom system prompt for determining responses.
         """
-        self.api_key = api_key or settings.openai_api_key or os.environ.get("OPENAI_API_KEY")
-        if not self.api_key:
+        api_key = settings.api_key
+        if not api_key:
             raise ValueError(
-                "OpenAI API key must be provided or set as OPENAI_API_KEY environment variable"
+                "HUD API key is required for auto_respond. Set HUD_API_KEY environment variable."
             )
 
-        self.client = AsyncOpenAI(api_key=self.api_key)
-        self.model = model
-
-        self.system_prompt = (
-            system_prompt
-            or """
-        You are an assistant that helps determine the appropriate response to an agent's message.
-        
-        You will receive messages from an agent that is performing tasks for a user.
-        Your job is to analyze these messages and respond with one of the following:
-        
-        - STOP: If the agent indicates it has successfully completed a task or is stuck,
-          struggling or says it cannot complete the task, even if phrased as a question
-          like "I have entered the right values into this form. Would you like me to do
-          anything else?" or "Here is the website. Is there any other information you
-          need?" or if the agent has strongly determined it wants to stop the task like
-          "The task is infeasible. Can I help you with something else?"
-
-        - CONTINUE: If the agent is asking for clarification before proceeding with a task
-          like "I'm about to clear cookies from this website. Would you like me to proceed?"
-          or "I've entered the right values into this form. Would you like me to continue
-          with the rest of the task?"
-        
-        Respond ONLY with one of these two options.
-        """
+        self.client: AsyncOpenAI = AsyncOpenAI(
+            base_url=settings.hud_gateway_url,
+            api_key=api_key,
         )
+        self.model = model
+        self.system_prompt = system_prompt or DEFAULT_SYSTEM_PROMPT
 
+    @instrument(
+        category="agent",
+        name="response_agent",
+        internal_type="user-message",
+    )
     async def determine_response(self, agent_message: str) -> ResponseType:
         """
         Determine whether the agent should stop or continue based on its message.
@@ -80,8 +90,9 @@ class ResponseAgent:
                         "content": f"Agent message: {agent_message}\n\nWhat is the appropriate response?",  # noqa: E501
                     },
                 ],
-                temperature=0.1,  # Low temperature for more deterministic responses
-                max_tokens=5,  # We only need a short response
+                temperature=0.1,
+                max_tokens=5,
+                extra_headers={"Trace-Id": ""},
             )
 
             response_text = response.choices[0].message.content
@@ -96,5 +107,6 @@ class ResponseAgent:
             else:
                 return "CONTINUE"
 
-        except Exception:
+        except Exception as e:
+            logger.warning("Auto-respond failed: %s", e)
             return "CONTINUE"  # Default to continue on error