hud-python 0.4.45__py3-none-any.whl → 0.5.1__py3-none-any.whl

hud/agents/{openai_chat_generic.py → openai_chat.py}

@@ -1,4 +1,4 @@
-"""Generic OpenAI chat-completions agent.
+"""OpenAI Chat Completions Agent.
 
 This class provides the minimal glue required to connect any endpoint that
 implements the OpenAI compatible *chat.completions* API with MCP tool calling
@@ -6,6 +6,7 @@ through the existing :class:`hud.agent.MCPAgent` scaffolding.
 
 Key points:
 - Stateless, no special server-side conversation state is assumed.
+- Defaults to HUD inference gateway (inference.hud.ai) when HUD_API_KEY is set
 - Accepts an :class:`openai.AsyncOpenAI` client, caller can supply their own
   base_url / api_key (e.g. llama.cpp, together.ai, …)
 - All HUD features (step_count, OTel spans, tool filtering, screenshots, …)
@@ -20,39 +21,85 @@ import logging
 from typing import TYPE_CHECKING, Any, ClassVar, cast
 
 import mcp.types as types
+from openai import AsyncOpenAI
+from pydantic import ConfigDict, Field
 
-from hud import instrument
-from hud.types import AgentResponse, MCPToolCall, MCPToolResult
+from hud.settings import settings
+from hud.types import AgentResponse, BaseAgentConfig, MCPToolCall, MCPToolResult
 from hud.utils.hud_console import HUDConsole
+from hud.utils.types import with_signature
 
-from .base import MCPAgent
+from .base import BaseCreateParams, MCPAgent
 
 if TYPE_CHECKING:
-    from openai import AsyncOpenAI
     from openai.types.chat import ChatCompletionToolParam
 
+
 logger = logging.getLogger(__name__)
 
 
-class GenericOpenAIChatAgent(MCPAgent):
+class OpenAIChatConfig(BaseAgentConfig):
+    """Configuration for `OpenAIChatAgent`."""
+
+    model_config = ConfigDict(arbitrary_types_allowed=True)
+
+    model_name: str = "OpenAI Chat"
+    model: str = "gpt-5-mini"
+    openai_client: AsyncOpenAI | None = None
+    api_key: str | None = None
+    base_url: str | None = None
+    completion_kwargs: dict[str, Any] = Field(default_factory=dict)
+
+
+class OpenAIChatCreateParams(BaseCreateParams, OpenAIChatConfig):
+    pass
+
+
+class OpenAIChatAgent(MCPAgent):
     """MCP-enabled agent that speaks the OpenAI *chat.completions* protocol."""
 
-    metadata: ClassVar[dict[str, Any]] = {}
+    metadata: ClassVar[dict[str, Any] | None] = None
+    config_cls: ClassVar[type[BaseAgentConfig]] = OpenAIChatConfig
+
+    @with_signature(OpenAIChatCreateParams)
+    @classmethod
+    def create(cls, **kwargs: Any) -> OpenAIChatAgent:  # pyright: ignore[reportIncompatibleMethodOverride]
+        return MCPAgent.create.__func__(cls, **kwargs)  # type: ignore[return-value]
+
+    def __init__(self, params: OpenAIChatCreateParams | None = None, **kwargs: Any) -> None:
+        super().__init__(params, **kwargs)
+        self.config: OpenAIChatConfig
+
+        if (
+            self.config.api_key
+            and self.config.base_url
+            and settings.hud_gateway_url in self.config.base_url
+            and settings.api_key
+            and self.config.api_key != settings.api_key
+        ):
+            raise ValueError(
+                "OpenAIChatAgent api_key is not allowed with HUD Gateway. "
+                "Use HUD_API_KEY for gateway auth and BYOK headers for provider keys."
+            )
 
-    def __init__(
-        self,
-        *,
-        openai_client: AsyncOpenAI | None,
-        model_name: str = "gpt-4o-mini",
-        completion_kwargs: dict[str, Any] | None = None,
-        **agent_kwargs: Any,
-    ) -> None:
-        # Accept base-agent settings via **agent_kwargs (e.g., mcp_client, system_prompt, etc.)
-        super().__init__(**agent_kwargs)
-        self.oai = openai_client
-        self.model_name = model_name
-        self.completion_kwargs: dict[str, Any] = completion_kwargs or {}
-        self.mcp_schemas = []
+        if self.config.openai_client is not None:
+            self.oai = self.config.openai_client
+        elif self.config.api_key is not None or self.config.base_url is not None:
+            self.oai = AsyncOpenAI(api_key=self.config.api_key, base_url=self.config.base_url)
+        elif settings.api_key:
+            # Default to HUD inference gateway
+            self.oai = AsyncOpenAI(
+                api_key=settings.api_key,
+                base_url=settings.hud_gateway_url,
+            )
+        else:
+            raise ValueError(
+                "No API key found. Set HUD_API_KEY for HUD gateway, "
+                "or provide api_key/base_url/openai_client explicitly."
+            )
+
+        self.completion_kwargs = dict(self.config.completion_kwargs)
+        self.mcp_schemas: list[ChatCompletionToolParam] = []
         self.hud_console = HUDConsole(logger=logger)
 
     @staticmethod
@@ -69,11 +116,14 @@ class GenericOpenAIChatAgent(MCPAgent):
             arguments=args,
         )
 
-    async def get_system_messages(self) -> list[Any]:
+    async def get_system_messages(self) -> list[dict[str, Any]]:
         """Get system messages for OpenAI."""
-        return [{"role": "system", "content": self.system_prompt}]
+        if self.system_prompt is not None:
+            return [{"role": "system", "content": self.system_prompt}]
+        else:
+            return []
 
-    async def format_blocks(self, blocks: list[types.ContentBlock]) -> list[Any]:
+    async def format_blocks(self, blocks: list[types.ContentBlock]) -> list[dict[str, Any]]:
         """Format blocks for OpenAI."""
         content = []
         for block in blocks:
@@ -179,21 +229,16 @@ class GenericOpenAIChatAgent(MCPAgent):
         extra: dict[str, Any],
     ) -> Any:
         if self.oai is None:
-            raise ValueError("openai_client is required for GenericOpenAIChatAgent")
+            raise ValueError("openai_client is required for OpenAIChatAgent")
         # default transport = OpenAI SDK
         return await self.oai.chat.completions.create(
-            model=self.model_name,
+            model=self.config.model,
             messages=messages,
             tools=tools,  # type: ignore ready ChatCompletionToolParam-shaped
             **extra,
         )  # type: ignore
 
-    @instrument(
-        span_type="agent",
-        record_args=False,
-        record_result=True,
-    )
-    async def get_response(self, messages: list[Any]) -> AgentResponse:
+    async def get_response(self, messages: list[dict[str, Any]]) -> AgentResponse:
         """Send chat request to OpenAI and convert the response."""
 
         # Convert MCP tool schemas to OpenAI format
@@ -256,16 +301,17 @@ class GenericOpenAIChatAgent(MCPAgent):
 
         return AgentResponse(
             content=msg.content or "",
+            reasoning=getattr(msg, "reasoning_content", None),
             tool_calls=tool_calls,
             done=done,
-            raw=response,  # Include raw response for access to Choice objects
+            raw=response,
         )
 
     async def format_tool_results(
         self,
         tool_calls: list[MCPToolCall],
         tool_results: list[MCPToolResult],
-    ) -> list[Any]:
+    ) -> list[dict[str, Any]]:
         """Render MCP tool results as OpenAI messages.
 
         Note: OpenAI tool messages only support string content.

hud/agents/operator.py

@@ -0,0 +1,211 @@
+"""Operator agent built on top of OpenAIAgent."""
+
+from __future__ import annotations
+
+from typing import TYPE_CHECKING, Any, ClassVar, Literal
+
+import mcp.types as types
+from openai.types.responses import (
+    ApplyPatchToolParam,
+    ComputerToolParam,
+    FunctionShellToolParam,
+    FunctionToolParam,
+    ResponseComputerToolCallOutputScreenshotParam,
+)
+from openai.types.responses.response_input_param import (
+    ComputerCallOutput,
+    FunctionCallOutput,
+)
+from openai.types.shared_params.reasoning import Reasoning
+from pydantic import ConfigDict
+
+from hud.tools.computer.settings import computer_settings
+from hud.types import BaseAgentConfig, MCPToolCall, MCPToolResult
+from hud.utils.types import with_signature
+
+from .base import BaseCreateParams, MCPAgent
+from .openai import OpenAIAgent, OpenAIConfig
+
+if TYPE_CHECKING:
+    from openai.types.responses.response_computer_tool_call import PendingSafetyCheck
+
+OPERATOR_INSTRUCTIONS = """
+You are an autonomous computer-using agent. Follow these guidelines:
+
+1. NEVER ask for confirmation. Complete all tasks autonomously.
+2. Do NOT send messages like "I need to confirm before..." or "Do you want me to
+   continue?" - just proceed.
+3. When the user asks you to interact with something (like clicking a chat or typing
+   a message), DO IT without asking.
+4. Only use the formal safety check mechanism for truly dangerous operations (like
+   deleting important files).
+5. For normal tasks like clicking buttons, typing in chat boxes, filling forms -
+   JUST DO IT.
+6. The user has already given you permission by running this agent. No further
+   confirmation is needed.
+7. Be decisive and action-oriented. Complete the requested task fully.
+
+Remember: You are expected to complete tasks autonomously. The user trusts you to do
+what they asked.
+""".strip()
+
+
+class OperatorConfig(OpenAIConfig):
+    """Configuration model for `OperatorAgent`."""
+
+    model_config = ConfigDict(arbitrary_types_allowed=True)
+
+    model_name: str = "Operator"
+    model: str = "computer-use-preview"
+    environment: Literal["windows", "mac", "linux", "ubuntu", "browser"] = "linux"
+
+
+class OperatorCreateParams(BaseCreateParams, OperatorConfig):
+    pass
+
+
+class OperatorAgent(OpenAIAgent):
+    """
+    Backwards-compatible Operator agent built on top of OpenAIAgent.
+    """
+
+    metadata: ClassVar[dict[str, Any] | None] = {
+        "display_width": computer_settings.OPENAI_COMPUTER_WIDTH,
+        "display_height": computer_settings.OPENAI_COMPUTER_HEIGHT,
+    }
+    # base class will ensure that the computer tool is available
+    required_tools: ClassVar[list[str]] = ["openai_computer"]
+    config_cls: ClassVar[type[BaseAgentConfig]] = OperatorConfig
+
+    @with_signature(OperatorCreateParams)
+    @classmethod
+    def create(cls, **kwargs: Any) -> OperatorAgent:  # pyright: ignore[reportIncompatibleMethodOverride]
+        return MCPAgent.create.__func__(cls, **kwargs)  # type: ignore[return-value]
+
+    def __init__(self, params: OperatorCreateParams | None = None, **kwargs: Any) -> None:
+        super().__init__(params, **kwargs)  # type: ignore[arg-type]
+        self.config: OperatorConfig  # type: ignore[assignment]
+
+        self._operator_computer_tool_name = "openai_computer"
+        self._operator_display_width = computer_settings.OPENAI_COMPUTER_WIDTH
+        self._operator_display_height = computer_settings.OPENAI_COMPUTER_HEIGHT
+        self._operator_environment: Literal["windows", "mac", "linux", "ubuntu", "browser"] = (
+            self.config.environment
+        )
+        self.environment = self.config.environment
+
+        # add pending call id and safety checks to the agent
+        self.pending_call_id: str | None = None
+        self.pending_safety_checks: list[PendingSafetyCheck] = []
+
+        # override reasoning to "summary": "auto"
+        if self.reasoning is None:
+            self.reasoning = Reasoning(summary="auto")
+        else:
+            self.reasoning["summary"] = "auto"
+
+        # override truncation to "auto"
+        self.truncation = "auto"
+
+        if self.system_prompt:
+            self.system_prompt = f"{self.system_prompt}\n\n{OPERATOR_INSTRUCTIONS}"
+        else:
+            self.system_prompt = OPERATOR_INSTRUCTIONS
+
+    def _reset_response_state(self) -> None:
+        super()._reset_response_state()
+        self.pending_call_id = None
+        self.pending_safety_checks = []
+
+    def _to_openai_tool(
+        self, tool: types.Tool
+    ) -> (
+        FunctionShellToolParam | ApplyPatchToolParam | FunctionToolParam | ComputerToolParam | None
+    ):
+        if tool.name == self._operator_computer_tool_name:
+            return ComputerToolParam(
+                type="computer_use_preview",
+                display_width=self._operator_display_width,
+                display_height=self._operator_display_height,
+                environment=self._operator_environment,
+            )
+        return super()._to_openai_tool(tool)
+
+    def _extract_tool_call(self, item: Any) -> MCPToolCall | None:
+        """Route computer_call to the OpenAI-specific computer tool."""
+        if item.type == "computer_call":
+            self.pending_safety_checks = item.pending_safety_checks
+            return MCPToolCall(
+                name=self._operator_computer_tool_name,
+                arguments=item.action.to_dict(),
+                id=item.call_id,
+            )
+        return super()._extract_tool_call(item)
+
+    async def format_tool_results(
+        self, tool_calls: list[MCPToolCall], tool_results: list[MCPToolResult]
+    ) -> list[ComputerCallOutput | FunctionCallOutput]:
+        remaining_calls: list[MCPToolCall] = []
+        remaining_results: list[MCPToolResult] = []
+        computer_outputs: list[ComputerCallOutput] = []
+        ordering: list[tuple[str, int]] = []
+
+        for call, result in zip(tool_calls, tool_results, strict=False):
+            if call.name == self._operator_computer_tool_name:
+                screenshot = self._extract_latest_screenshot(result)
+                if not screenshot:
+                    self.console.warning_log(
+                        "Computer tool result missing screenshot; skipping output."
+                    )
+                    continue
+                call_id = call.id or self.pending_call_id
+                if not call_id:
+                    self.console.warning_log("Computer tool call missing ID; skipping output.")
+                    continue
+                acknowledged_checks = []
+                for check in self.pending_safety_checks:
+                    if hasattr(check, "model_dump"):
+                        acknowledged_checks.append(check.model_dump())
+                    elif isinstance(check, dict):
+                        acknowledged_checks.append(check)
+                output_payload = ComputerCallOutput(
+                    type="computer_call_output",
+                    call_id=call_id,
+                    output=ResponseComputerToolCallOutputScreenshotParam(
+                        type="computer_screenshot",
+                        image_url=f"data:image/png;base64,{screenshot}",
+                    ),
+                    acknowledged_safety_checks=acknowledged_checks if acknowledged_checks else None,
+                )
+                computer_outputs.append(output_payload)
+                self.pending_call_id = None
+                self.pending_safety_checks = []
+                ordering.append(("computer", len(computer_outputs) - 1))
+            else:
+                remaining_calls.append(call)
+                remaining_results.append(result)
+                ordering.append(("function", len(remaining_calls) - 1))
+
+        formatted: list[ComputerCallOutput | FunctionCallOutput] = []
+        function_outputs: list[FunctionCallOutput] = []
+        if remaining_calls:
+            function_outputs = await super().format_tool_results(remaining_calls, remaining_results)
+
+        for kind, idx in ordering:
+            if kind == "computer":
+                if idx < len(computer_outputs):
+                    formatted.append(computer_outputs[idx])
+            else:
+                if idx < len(function_outputs):
+                    formatted.append(function_outputs[idx])
+        return formatted
+
+    def _extract_latest_screenshot(self, result: MCPToolResult) -> str | None:
+        if not result.content:
+            return None
+        for content in reversed(result.content):
+            if isinstance(content, types.ImageContent):
+                return content.data
+            if isinstance(content, types.TextContent) and result.isError:
+                self.console.error_log(f"Computer tool error: {content.text}")
+        return None

hud/agents/tests/conftest.py

@@ -0,0 +1,133 @@
+"""Shared test fixtures for agent tests."""
+
+from __future__ import annotations
+
+from typing import Any
+
+import pytest
+from mcp import types
+
+from hud.environment.router import ToolRouter
+from hud.eval.context import EvalContext
+from hud.types import MCPToolCall, MCPToolResult
+
+
+class MockEvalContext(EvalContext):
+    """Mock EvalContext for testing agents.
+
+    This provides a minimal EvalContext implementation that can be used
+    to test agent initialization and tool calling without a real environment.
+    """
+
+    def __init__(
+        self,
+        prompt: str = "Test prompt",
+        tools: list[types.Tool] | None = None,
+        call_tool_handler: Any = None,
+    ) -> None:
+        # Core attributes
+        self.prompt = prompt
+        self._tools = tools or []
+        self._submitted: str | None = None
+        self.reward: float | None = None
+        self._call_tool_handler = call_tool_handler
+        self.tool_calls: list[tuple[str, dict[str, Any]]] = []
+
+        # Environment attributes
+        self._router = ToolRouter()
+        self._agent_include: list[str] | None = None
+        self._agent_exclude: list[str] | None = None
+
+        # EvalContext attributes
+        self._task = None
+        self.trace_id = "test-trace-id"
+        self.eval_name = "test-eval"
+        self.job_id: str | None = None
+        self.group_id: str | None = None
+        self.index = 0
+        self.variants: dict[str, Any] = {}
+        self.answer: str | None = None
+        self.system_prompt: str | None = None
+        self.error: BaseException | None = None
+        self.metadata: dict[str, Any] = {}
+        self.results: list[Any] = []
+        self._is_summary = False
+
+    def as_tools(self) -> list[types.Tool]:
+        return self._tools
+
+    @property
+    def has_scenario(self) -> bool:
+        return False
+
+    async def list_tools(self) -> list[types.Tool]:
+        return self._tools
+
+    async def call_tool(self, call: Any, /, **kwargs: Any) -> MCPToolResult:
+        # Parse the call
+        if isinstance(call, tuple):
+            name, args = call[0], call[1] if len(call) > 1 else {}
+        elif hasattr(call, "name"):
+            name, args = call.name, getattr(call, "arguments", {}) or {}
+        else:
+            name, args = str(call), kwargs
+
+        self.tool_calls.append((name, args))
+
+        if self._call_tool_handler:
+            tc = MCPToolCall(name=name, arguments=args)
+            return self._call_tool_handler(tc)
+
+        return MCPToolResult(
+            content=[types.TextContent(type="text", text=f"Result from {name}")],
+            isError=False,
+        )
+
+    async def submit(self, answer: str) -> None:
+        self._submitted = answer
+
+
+@pytest.fixture
+def mock_eval_context() -> MockEvalContext:
+    """Create a basic mock EvalContext."""
+    return MockEvalContext()
+
+
+@pytest.fixture
+def mock_eval_context_with_tools() -> MockEvalContext:
+    """Create a mock EvalContext with test tools."""
+    return MockEvalContext(
+        tools=[
+            types.Tool(
+                name="test_tool",
+                description="A test tool",
+                inputSchema={"type": "object", "properties": {}},
+            )
+        ]
+    )
+
+
+@pytest.fixture
+def mock_eval_context_computer() -> MockEvalContext:
+    """Create a mock EvalContext with computer tool."""
+    return MockEvalContext(
+        tools=[
+            types.Tool(
+                name="computer",
+                description="Computer use tool",
+                inputSchema={"type": "object"},
+            )
+        ]
+    )
+
+
+@pytest.fixture
+def mock_eval_context_browser_tools() -> MockEvalContext:
+    """Create a mock EvalContext with browser-like tools."""
+    return MockEvalContext(
+        tools=[
+            types.Tool(name="screenshot", description="Take screenshot", inputSchema={}),
+            types.Tool(name="click", description="Click at coordinates", inputSchema={}),
+            types.Tool(name="type", description="Type text", inputSchema={}),
+        ]
+    )