hud-python 0.3.4__py3-none-any.whl → 0.4.0__py3-none-any.whl

hud/agent/operator.py

@@ -1,268 +0,0 @@
-import json
-import logging
-import os
-from typing import Any, Literal, cast
-
-from openai import AsyncOpenAI
-from openai.types.responses import (
-    ToolParam,
-    ResponseInputParam,
-    ResponseInputItemParam,
-    ResponseOutputMessage,
-    ResponseComputerToolCall,
-    ResponseOutputText,
-)
-
-from hud.adapters import Adapter
-from hud.agent.base import Agent
-from hud.adapters.operator import OperatorAdapter
-from hud.types import Gym
-from hud.utils.common import Observation
-from hud.settings import settings
-from hud.adapters.common.types import LogType
-
-logger = logging.getLogger(__name__)
-
-
-class OperatorAgent(Agent[AsyncOpenAI, dict[str, Any]]):
-    """
-    An agent implementation using OpenAI's Computer Use API.
-
-    This agent interacts with HUD environments using OpenAI's Computer Use API
-    through the OperatorAdapter which converts actions to the format expected by HUD.
-    """
-
-    transfer_gyms: dict[Gym, Gym] = {"qa": "hud-browser"}
-
-    def __init__(
-        self,
-        client: AsyncOpenAI | None = None,
-        model: str = "computer-use-preview",
-        environment: Literal["windows", "mac", "linux", "browser"] = "browser",
-        adapter: Adapter | None = None,
-        max_iterations: int = 8,
-        name: str | None = None,
-    ):
-        """
-        Initialize the OperatorAgent.
-
-        Args:
-            client: The AsyncOpenAI client for API calls (optional, created automatically if not provided)
-            model: The model to use for computer use
-            environment: The environment type (windows, mac, linux, browser)
-            adapter: The adapter to use for preprocessing and postprocessing
-            max_iterations: Maximum number of iterations for the agent
-            name: The name of the agent
-        """
-        # Initialize client if not provided
-        if client is None:
-            # Get API key from settings
-            api_key = settings.openai_api_key
-            if not api_key:
-                raise ValueError(
-                    "OpenAI API key not found in settings or environment variables. Set OPENAI_API_KEY."
-                )
-
-            # Create asynchronous client
-            client = AsyncOpenAI(api_key=api_key)
-
-        adapter = adapter or OperatorAdapter()
-
-        if name is None:
-            name = f"openai-{model}"
-
-        super().__init__(client=client, adapter=adapter, name=name)
-
-        self.model = model
-        self.environment = environment
-        self.max_iterations = max_iterations
-
-        # Default dimensions
-        self.width = 1024
-        self.height = 768
-
-        # Update dimensions if adapter is provided
-        if self.adapter:
-            self.width = self.adapter.agent_width
-            self.height = self.adapter.agent_height
-
-        # Message history and state tracking
-        self.last_response_id = None
-        self.pending_call_id = None
-        self.initial_prompt = None
-        self.pending_safety_checks = []
-
-        self.base_system_prompt = """
-        You are an autonomous computer-using agent. Follow these guidelines:
-
-        1. Be decisive and complete tasks without asking for confirmation unless absolutely necessary.
-        2. If you need user confirmation for safety-critical actions, use the formal safety check mechanism.
-        3. Do NOT ask questions like "Should I proceed?" or "Would you like me to continue?" - just proceed with the task.
-        4. When you find what you're looking for (e.g., a file to upload), proceed with the action directly.
-        5. Only stop when the task is fully complete or if you encounter an error that prevents completion.
-        6. Trust that the user wants you to complete the entire task they've requested.
-
-        Remember: You wave been given permission to complete the requested task autonomously.
-        """
-
-        self.task_run_id = None
-
-    async def fetch_response(self, observation: Observation) -> tuple[list[dict[str, Any]], bool]:
-        """
-        Fetch a response from the model based on the observation.
-
-        Args:
-            observation: The preprocessed observation
-
-        Returns:
-            tuple[list[dict[str, Any]], bool, list[LogType] | None]: A tuple containing the list of raw actions,
-                                             boolean indicating if the agent believes the task is complete.
-        """
-        if not self.client:
-            raise ValueError("Client is required")
-
-        # Define the computer use tool with correct type using cast
-        computer_tool = cast(
-            ToolParam,
-            {
-                "type": "computer_use_preview",
-                "display_width": self.width,
-                "display_height": self.height,
-                "environment": self.environment,
-            },
-        )
-
-        # Process the observation based on whether it's the first one or a response to an action
-        if self.pending_call_id is None:  # and self.last_response_id is None:
-            # This is the first observation, store and send the prompt
-            self.initial_prompt = observation.text
-
-            # Create the initial request following the required structure
-            input_content: list[dict[str, Any]] = [
-                {"type": "input_text", "text": observation.text or ""}
-            ]
-
-            # Add screenshot if present
-            if observation.screenshot:
-                input_content.append(
-                    {
-                        "type": "input_image",
-                        "image_url": f"data:image/png;base64,{observation.screenshot}",
-                    }
-                )
-
-            # Structure the input correctly for the API using cast
-            input_param = cast(ResponseInputParam, [{"role": "user", "content": input_content}])
-
-            response = await self.client.responses.create(
-                model=self.model,
-                tools=[computer_tool],
-                input=input_param,
-                instructions=self.base_system_prompt,
-                truncation="auto",
-                reasoning={"summary": "auto"},
-            )
-
-        else:
-            if not observation.screenshot:
-                logger.warning("No screenshot provided for response to action")
-                return [], True
-
-            # Create a response to the previous action with the new screenshot
-            input_param_followup = cast(
-                ResponseInputParam,
-                [
-                    cast(
-                        ResponseInputItemParam,
-                        {
-                            "call_id": self.pending_call_id,
-                            "type": "computer_call_output",
-                            "output": {
-                                "type": "input_image",
-                                "image_url": f"data:image/png;base64,{observation.screenshot}",
-                            },
-                            "acknowledged_safety_checks": self.pending_safety_checks,
-                        },
-                    )
-                ],
-            )
-            self.pending_safety_checks = []
-
-            response = await self.client.responses.create(
-                model=self.model,
-                previous_response_id=self.last_response_id,
-                tools=[computer_tool],
-                input=input_param_followup,
-                truncation="auto",
-            )
-
-        # Store the response ID for the next call
-        self.last_response_id = response.id
-
-        # Process the response to extract actions or final text
-        actions = []
-        done = True  # Assume done unless a computer call is found
-        final_text_response = ""
-
-        self.pending_call_id = None
-
-        # Check for computer calls first
-        computer_calls = [
-            item
-            for item in response.output
-            if isinstance(item, ResponseComputerToolCall) and item.type == "computer_call"
-        ]
-
-        if computer_calls:
-            # If computer calls exist, process them and set done=False
-            done = False
-            for computer_call in computer_calls:
-                self.pending_call_id = computer_call.call_id
-                action = computer_call.action
-                self.pending_safety_checks = computer_call.pending_safety_checks
-                actions.append(action.model_dump())  # Convert Pydantic model to dict
-                # logger.info(f"Computer call action: {action}")
-        else:
-            # No computer calls, check for a final text message
-            # logger.info("No computer call found. Checking for final message.")
-            # logger.info(response.output)
-            for item in response.output:
-                if isinstance(item, ResponseOutputMessage) and item.type == "message":
-                    # Extract text from content blocks within the message
-                    full_text = "".join(
-                        [c.text for c in item.content if isinstance(c, ResponseOutputText)]
-                    )
-                    if full_text:
-                        final_text_response = full_text
-                        # logger.info(f"Final text message: {final_text_response}")
-                        break  # Stop after finding the first text message
-
-            # If we found final text, package it as a 'response' action
-            if final_text_response:
-                if (
-                    "the task is infeasible" in final_text_response.lower()
-                ):  # Custom action for OSWorld
-                    done = True
-                    actions = [{"type": "custom", "action": "FAIL"}]
-                else:
-                    actions = [{"type": "response", "text": final_text_response}]
-                    done = True
-            else:
-                logger.info("No computer calls and no final text message found.")
-            # Keep done = True, actions remains empty
-
-        reasoning = ""
-        for item in response.output:
-            if item.type == "reasoning" and item.summary:
-                reasoning += f"Thinking: {item.summary[0].text}\n"
-            elif item.type == "message":
-                for content in item.content:
-                    if isinstance(content, ResponseOutputText):
-                        reasoning += f"{content.text}\n"
-
-        # add reasoning to the actions
-        for action in actions:
-            action["reasoning"] = reasoning
-            action["logs"] = response.model_dump()  # type: ignore[assignment]
-
-        return actions, done

hud/agent/tests/__init__.py

@@ -1 +0,0 @@
-# Tests for hud.agent module

hud/agent/tests/test_base.py

@@ -1,202 +0,0 @@
-from __future__ import annotations
-
-from unittest.mock import MagicMock, patch
-from typing import Any
-
-import pytest
-
-from hud.agent.base import Agent
-from hud.adapters import Adapter
-from hud.adapters.common.types import ClickAction, Point
-from hud.utils.common import Observation
-
-
-class ConcreteAgent(Agent[Any, dict[str, Any]]):
-    """Concrete implementation of Agent for testing."""
-
-    def __init__(self, client: Any = None, adapter: Adapter | None = None):
-        super().__init__(client, adapter)
-        self.mock_responses = []
-        self.call_count = 0
-
-    async def fetch_response(self, observation: Observation) -> tuple[list[dict[str, Any]], bool]:
-        """Mock implementation that returns predefined responses."""
-        if self.call_count < len(self.mock_responses):
-            actions, done = self.mock_responses[self.call_count]
-            self.call_count += 1
-            return actions, done
-        return [], True
-
-
-class TestAgentBase:
-    """Test the base Agent class."""
-
-    @pytest.fixture
-    def mock_client(self):
-        """Mock client for testing."""
-        return MagicMock()
-
-    @pytest.fixture
-    def mock_adapter(self):
-        """Mock adapter for testing."""
-        adapter = MagicMock(spec=Adapter)
-        adapter.rescale.return_value = "rescaled_screenshot"
-        adapter.adapt_list.return_value = [ClickAction(point=Point(x=100, y=200))]
-        return adapter
-
-    @pytest.fixture
-    def agent_with_adapter(self, mock_client, mock_adapter):
-        """Agent with both client and adapter."""
-        return ConcreteAgent(client=mock_client, adapter=mock_adapter)
-
-    @pytest.fixture
-    def agent_without_adapter(self, mock_client):
-        """Agent with client but no adapter."""
-        return ConcreteAgent(client=mock_client, adapter=None)
-
-    def test_init_with_client_and_adapter(self, mock_client, mock_adapter):
-        """Test agent initialization with client and adapter."""
-        agent = ConcreteAgent(client=mock_client, adapter=mock_adapter)
-        assert agent.client == mock_client
-        assert agent.adapter == mock_adapter
-
-    def test_init_with_none_values(self):
-        """Test agent initialization with None values."""
-        agent = ConcreteAgent(client=None, adapter=None)
-        assert agent.client is None
-        assert agent.adapter is None
-
-    def test_preprocess_without_adapter(self, agent_without_adapter):
-        """Test preprocess when no adapter is available."""
-        observation = Observation(text="test", screenshot="screenshot_data")
-        result = agent_without_adapter.preprocess(observation)
-
-        # Should return original observation unchanged
-        assert result == observation
-        assert result.text == "test"
-        assert result.screenshot == "screenshot_data"
-
-    def test_preprocess_without_screenshot(self, agent_with_adapter):
-        """Test preprocess when no screenshot is available."""
-        observation = Observation(text="test", screenshot=None)
-        result = agent_with_adapter.preprocess(observation)
-
-        # Should return original observation unchanged
-        assert result == observation
-        assert result.text == "test"
-        assert result.screenshot is None
-
-    def test_preprocess_with_adapter_and_screenshot(self, agent_with_adapter, mock_adapter):
-        """Test preprocess with adapter and screenshot (covers missing lines 48-55)."""
-        observation = Observation(text="test", screenshot="original_screenshot")
-        result = agent_with_adapter.preprocess(observation)
-
-        # Should create new observation with rescaled screenshot
-        mock_adapter.rescale.assert_called_once_with("original_screenshot")
-        assert result.text == "test"
-        assert result.screenshot == "rescaled_screenshot"
-        # Should be a new object, not the original
-        assert result is not observation
-
-    def test_postprocess_without_adapter(self, agent_without_adapter):
-        """Test postprocess when no adapter is available (covers missing lines 82-85)."""
-        actions = [{"type": "click", "x": 100, "y": 200}]
-
-        with pytest.raises(ValueError, match="Cannot postprocess actions without an adapter"):
-            agent_without_adapter.postprocess(actions)
-
-    def test_postprocess_with_adapter(self, agent_with_adapter, mock_adapter):
-        """Test postprocess with adapter."""
-        actions = [{"type": "click", "x": 100, "y": 200}]
-        result = agent_with_adapter.postprocess(actions)
-
-        mock_adapter.adapt_list.assert_called_once_with(actions)
-        assert len(result) == 1
-        assert isinstance(result[0], ClickAction)
-
-    @pytest.mark.asyncio
-    async def test_predict_without_verbose(self, agent_with_adapter):
-        """Test predict method without verbose logging."""
-        observation = Observation(text="test", screenshot="screenshot")
-        agent_with_adapter.mock_responses = [([{"type": "click", "x": 100, "y": 200}], False)]
-
-        actions, done = await agent_with_adapter.predict(observation, verbose=False)
-
-        assert len(actions) == 1
-        assert isinstance(actions[0], ClickAction)
-        assert done is False
-
-    @pytest.mark.asyncio
-    @patch("hud.agent.base.logger")
-    async def test_predict_with_verbose_logging(self, mock_logger, agent_with_adapter):
-        """Test predict method with verbose logging (covers missing lines 100-116)."""
-        observation = Observation(text="test", screenshot="screenshot")
-        agent_with_adapter.mock_responses = [([{"type": "click", "x": 100, "y": 200}], True)]
-
-        actions, done = await agent_with_adapter.predict(observation, verbose=True)
-
-        # Verify verbose logging was called
-        mock_logger.info.assert_any_call("Predicting action...")
-        mock_logger.info.assert_any_call("Raw action: %s", [{"type": "click", "x": 100, "y": 200}])
-
-        assert len(actions) == 1
-        assert isinstance(actions[0], ClickAction)
-        assert done is True
-
-    @pytest.mark.asyncio
-    async def test_predict_without_adapter_returns_raw_actions(self, agent_without_adapter):
-        """Test predict without adapter returns raw actions."""
-        observation = Observation(text="test", screenshot=None)
-        raw_actions = [{"type": "click", "x": 100, "y": 200}]
-        agent_without_adapter.mock_responses = [(raw_actions, True)]
-
-        actions, done = await agent_without_adapter.predict(observation, verbose=False)
-
-        # Should return raw actions, not processed ones
-        assert actions == raw_actions
-        assert done is True
-
-    @pytest.mark.asyncio
-    async def test_predict_with_empty_actions(self, agent_with_adapter):
-        """Test predict when fetch_response returns empty actions."""
-        observation = Observation(text="test", screenshot="screenshot")
-        agent_with_adapter.mock_responses = [([], True)]
-
-        actions, done = await agent_with_adapter.predict(observation, verbose=False)
-
-        # Should return empty actions without calling adapter
-        assert actions == []
-        assert done is True
-
-    @pytest.mark.asyncio
-    async def test_predict_full_pipeline(self, agent_with_adapter, mock_adapter):
-        """Test the complete predict pipeline with all stages."""
-        # Set up observation with screenshot that will be rescaled
-        observation = Observation(text="test input", screenshot="original_screenshot")
-        raw_actions = [{"type": "click", "x": 150, "y": 250}]
-        agent_with_adapter.mock_responses = [(raw_actions, False)]
-
-        actions, done = await agent_with_adapter.predict(observation, verbose=True)
-
-        # Verify all stages were called
-        # Stage 1: Preprocessing
-        mock_adapter.rescale.assert_called_once_with("original_screenshot")
-
-        # Stage 3: Postprocessing
-        mock_adapter.adapt_list.assert_called_once_with(raw_actions)
-
-        assert len(actions) == 1
-        assert isinstance(actions[0], ClickAction)
-        assert done is False
-
-    @pytest.mark.asyncio
-    async def test_predict_integration_without_screenshot(self, agent_with_adapter):
-        """Test predict integration when observation has no screenshot."""
-        observation = Observation(text="test input", screenshot=None)
-        raw_actions = [{"type": "response", "text": "Task completed"}]
-        agent_with_adapter.mock_responses = [(raw_actions, True)]
-
-        actions, done = await agent_with_adapter.predict(observation, verbose=False)
-
-        assert len(actions) == 1
-        assert done is True

hud/env/__init__.py

@@ -1,11 +0,0 @@
-from __future__ import annotations
-
-from . import docker_client, environment, local_docker_client, remote_client, remote_docker_client
-
-__all__ = [
-    "docker_client",
-    "environment",
-    "local_docker_client",
-    "remote_client",
-    "remote_docker_client",
-]

hud/env/client.py

@@ -1,35 +0,0 @@
-from __future__ import annotations
-
-from abc import ABC, abstractmethod
-from typing import TYPE_CHECKING, Any
-
-from pydantic import BaseModel
-
-if TYPE_CHECKING:
-    from hud.types import EnvironmentStatus
-    from hud.utils.config import FunctionConfig
-
-
-class Client(BaseModel, ABC):
-    """
-    Base class for all environment clients.
-    """
-
-    @abstractmethod
-    async def invoke(self, config: FunctionConfig) -> Any:
-        """
-        Invoke the environment with the given config.
-        """
-
-    @abstractmethod
-    async def get_status(self) -> EnvironmentStatus:
-        """
-        Get the current status of the environment.
-        """
-
-    @abstractmethod
-    async def close(self) -> None:
-        """
-        Close the environment and clean up any resources.
-        This method should be called when the environment is no longer needed.
-        """