hud-python 0.1.5__py3-none-any.whl → 0.2.1__py3-none-any.whl

hud/__init__.py

@@ -4,19 +4,23 @@ HUD Gym SDK - A Python SDK for interacting with HUD environments.
 
 from __future__ import annotations
 
-from hud.client import HUDClient
-from hud.environment import Environment, EvalSet, Observation, TaskResult
-from hud.gym import Gym
-from hud.run import Run
+from . import agent, env, gym, settings, task, taskset, types, utils
+from .job import create_job, job, load_job
+from .taskset import load_taskset
 
-__version__ = "0.1.5"
+__version__ = "0.2.1"
 
 __all__ = [
-    "Environment",
-    "EvalSet",
-    "Gym",
-    "HUDClient",
-    "Observation",
-    "Run",
-    "TaskResult",
+    "agent",
+    "create_job",
+    "env",
+    "gym",
+    "job",
+    "load_job",
+    "load_taskset",
+    "settings",
+    "task",
+    "taskset",
+    "types",
+    "utils",
 ]

hud/adapters/__init__.py

@@ -1,5 +1,7 @@
 from __future__ import annotations
 
-from .common import Adapter
+from .claude import ClaudeAdapter
+from .common import CLA, Adapter
+from .operator import OperatorAdapter
 
-__all__ = ["Adapter"]
+__all__ = ["CLA", "Adapter", "ClaudeAdapter", "OperatorAdapter"]

hud/adapters/claude/adapter.py

@@ -13,6 +13,7 @@ from hud.adapters.common.types import (
     Point,
     PositionFetch,
     PressAction,
+    ResponseAction,
     ScreenshotFetch,
     ScrollAction,
     TypeAction,
@@ -21,7 +22,10 @@ from hud.adapters.common.types import (
 
 
 class ClaudeAdapter(Adapter):
-    KEY_MAP: ClassVar[dict[str, CLAKey]] = {"Return": "enter"}
+    KEY_MAP: ClassVar[dict[str, CLAKey]] = {
+        "Return": "enter",
+        "Super": "win",
+        }
 
     def __init__(self) -> None:
         super().__init__()
@@ -31,7 +35,6 @@ class ClaudeAdapter(Adapter):
     def _map_key(self, key: str) -> CLAKey:
         """Map a key to its standardized form."""
         return self.KEY_MAP.get(key, key.lower())  # type: ignore
-
     def convert(self, data: Any) -> CLA:
         try:
             action_type = data.get("action")
@@ -152,6 +155,10 @@ class ClaudeAdapter(Adapter):
             elif action_type == "wait":
                 assert "duration" in data
                 return WaitAction(time=data["duration"])
+
+            elif action_type == "response":
+                return ResponseAction(text=data.get("text", ""))
+
             else:
                 raise ValueError(f"Unsupported action type: {action_type}")
         except AssertionError:

hud/adapters/common/adapter.py

@@ -2,16 +2,16 @@ from __future__ import annotations
 
 from typing import TYPE_CHECKING, Any
 
+import numpy as np
 from PIL import Image
 from pydantic import TypeAdapter, ValidationError
 
 from .types import CLA
 
 if TYPE_CHECKING:
-    import numpy as np  # type: ignore
-    from typing_extensions import TypeAlias, TypeIs
+    from typing_extensions import TypeIs
 
-    ImageType: TypeAlias = "np.ndarray[Any, Any] | Image.Image | str | None"
+ImageType = np.ndarray[Any, Any] | Image.Image | str | None
 
 
 def _is_numpy_array(observation: Any) -> TypeIs[np.ndarray]:
@@ -61,7 +61,7 @@ class Adapter:
             observation: Image data, which can be:
                 - numpy array
                 - PIL Image
-                - base64 string (PNG)
+                - base64 string (PNG) # TODO: JPG
 
         Returns:
             Base64-encoded string of the resized image (PNG format)
@@ -146,7 +146,7 @@ class Adapter:
 
         return processed_action
 
-    def adapt(self, action: Any) -> dict[str, Any]:
+    def adapt(self, action: Any) -> CLA:
         # any preprocessing steps
         action = self.preprocess(action)
 
@@ -154,14 +154,15 @@ class Adapter:
         action = self.convert(action)
         self.memory.append(action)
 
-        # convert to json
+        # convert to json and apply coordinate rescaling
         action_dict = self.json(action)
-
-        # apply coordinate rescaling
         rescaled_action = self.postprocess_action(action_dict)
-        return rescaled_action
 
-    def adapt_list(self, actions: list[Any]) -> list[dict[str, Any]]:
+        # convert back to CLA
+        return TypeAdapter(CLA).validate_python(rescaled_action)
+
+    def adapt_list(self, actions: list[Any]) -> list[CLA]:
         if not isinstance(actions, list):
             raise ValueError("Please provide a list of actions")
+        
         return [self.adapt(action) for action in actions]

hud/adapters/common/types.py

@@ -1,6 +1,6 @@
 from __future__ import annotations
 
-from typing import Annotated, Literal, Union
+from typing import Annotated, Literal
 
 from pydantic import BaseModel, Field
 
@@ -32,10 +32,23 @@ class PressAction(CLAAction):
     keys: list[CLAKey]
 
 
+# KEYDOWN ACTION for key presses/hotkeys
+class KeyDownAction(CLAAction):
+    type: Literal["keydown"] = "keydown"
+    keys: list[CLAKey]
+
+
+# KEYUP ACTION for key presses/hotkeys
+class KeyUpAction(CLAAction):
+    type: Literal["keyup"] = "keyup"
+    keys: list[CLAKey]
+
+
 # TYPE ACTION for text typing
 class TypeAction(CLAAction):
     type: Literal["type"] = "type"
     text: str
+    selector: str | None = None
     enter_after: bool | None = False
 
 
@@ -69,6 +82,12 @@ class DragAction(CLAAction):
     hold_keys: list[CLAKey] | None = None
 
 
+# RESPONSE ACTION from agent
+class ResponseAction(CLAAction):
+    type: Literal["response"] = "response"
+    text: str # The final textual response from the agent
+
+
 # SCREENSHOT ACTION
 class ScreenshotFetch(CLAAction):
     type: Literal["screenshot"] = "screenshot"
@@ -82,20 +101,22 @@ class CustomAction(CLAAction):
     type: Literal["custom"] = "custom"
     action: str
 
+
 # Union of all possible actions
 CLA = Annotated[
-    Union[
-        ClickAction,
-        PressAction,
-        TypeAction,
-        ScrollAction,
-        MoveAction,
-        WaitAction,
-        DragAction,
-        ScreenshotFetch,
-        PositionFetch,
-        CustomAction,
-    ],
+    ClickAction
+    | PressAction
+    | KeyDownAction
+    | KeyUpAction
+    | TypeAction
+    | ResponseAction
+    | ScrollAction
+    | MoveAction
+    | WaitAction
+    | DragAction
+    | CustomAction
+    | ScreenshotFetch
+    | PositionFetch,
     Field(discriminator="type"),
 ]
 

hud/adapters/operator/__init__.py

@@ -0,0 +1,5 @@
+from __future__ import annotations
+
+from .adapter import OperatorAdapter
+
+__all__ = ["OperatorAdapter"]

hud/adapters/operator/adapter.py

@@ -0,0 +1,97 @@
+from __future__ import annotations
+
+from typing import Any, ClassVar
+
+from hud.adapters.common import CLA, Adapter
+from hud.adapters.common.types import (
+    CLAKey,
+    ClickAction,
+    DragAction,
+    MoveAction,
+    Point,
+    PressAction,
+    ResponseAction,
+    ScreenshotFetch,
+    ScrollAction,
+    TypeAction,
+    WaitAction,
+)
+
+
+class OperatorAdapter(Adapter):
+    KEY_MAP: ClassVar[dict[str, CLAKey]] = {
+        "Return": "enter",
+        "ArrowUp": "up",
+        "ArrowDown": "down",
+        "ArrowLeft": "left",
+        "ArrowRight": "right",
+    }
+    
+    def __init__(self) -> None:
+        super().__init__()
+        # OpenAI Computer Use default dimensions
+        self.agent_width = 1024
+        self.agent_height = 768
+        
+    def _map_key(self, key: str) -> CLAKey:
+        """Map a key to its standardized form."""
+        return self.KEY_MAP.get(key, key.lower())  # type: ignore
+        
+    def convert(self, data: Any) -> CLA:
+        """Convert a Computer Use action to a HUD action"""
+        try:
+            action_type = data.get("type")
+            
+            if action_type == "click":
+                x, y = data.get("x", 0), data.get("y", 0)
+                button = data.get("button", "left")
+                return ClickAction(point=Point(x=x, y=y), button=button)
+                
+            elif action_type == "double_click":
+                x, y = data.get("x", 0), data.get("y", 0)
+                return ClickAction(
+                    point=Point(x=x, y=y),
+                    button="left",
+                    pattern=[100]
+                )
+                
+            elif action_type == "scroll":
+                x, y = data.get("x", 0), data.get("y", 0)
+                scroll_x = data.get("scroll_x", 0)
+                scroll_y = data.get("scroll_y", 0)
+                return ScrollAction(
+                    point=Point(x=x, y=y),
+                    scroll=Point(x=scroll_x, y=scroll_y)
+                )
+                
+            elif action_type == "type":
+                text = data.get("text", "")
+                return TypeAction(text=text, enter_after=False)
+                
+            elif action_type == "wait":
+                ms = data.get("ms", 1000)
+                return WaitAction(time=ms)
+                
+            elif action_type == "move":
+                x, y = data.get("x", 0), data.get("y", 0)
+                return MoveAction(point=Point(x=x, y=y))
+                
+            elif action_type == "keypress":
+                keys = data.get("keys", [])
+                return PressAction(keys=[self._map_key(k) for k in keys])
+                
+            elif action_type == "drag":
+                path = data.get("path", [])
+                points = [Point(x=p.get("x", 0), y=p.get("y", 0)) for p in path]
+                return DragAction(path=points)
+            
+            elif action_type == "screenshot":
+                return ScreenshotFetch()
+            
+            elif action_type == "response":
+                return ResponseAction(text=data.get("text", ""))
+            else:
+                raise ValueError(f"Unsupported action type: {action_type}")
+                
+        except Exception as e:
+            raise ValueError(f"Invalid action: {data}. Error: {e!s}") from e

hud/agent/__init__.py

@@ -0,0 +1,7 @@
+from .base import Agent
+from .claude import ClaudeAgent
+from .operator import OperatorAgent
+
+from hud.adapters import OperatorAdapter, ClaudeAdapter
+
+__all__ = ["Agent", "ClaudeAgent", "OperatorAgent", "OperatorAdapter", "ClaudeAdapter"]

hud/agent/base.py

@@ -0,0 +1,109 @@
+from abc import ABC, abstractmethod
+from typing import Sequence, TypeVar, Generic
+
+from hud.adapters import Adapter, CLA
+from hud.env.environment import Observation
+
+# Generic type for different client types (Anthropic, OpenAI, etc.)
+ClientT = TypeVar('ClientT')
+ActionT = TypeVar('ActionT')
+
+class Agent(Generic[ClientT, ActionT], ABC):
+    """
+    Base class for all agents.
+    
+    Implements a three-stage prediction process:
+    1. preprocess - Prepare observation data (e.g., rescale screenshot)
+    2. fetch_response - Make API calls to get model response
+    3. postprocess - Convert model actions to HUD format
+    
+    Subclasses only need to implement the fetch_response method.
+    """
+    
+    def __init__(self, client: ClientT | None = None, adapter: Adapter | None = None):
+        """
+        Initialize the agent.
+        
+        Args:
+            client: The client to use for API calls
+            adapter: The adapter to use for preprocessing and postprocessing
+        """
+        self.client = client
+        self.adapter = adapter
+    
+    def preprocess(self, observation: Observation) -> Observation:
+        """
+        Preprocess the observation before sending to the model.
+        
+        Args:
+            observation: The raw observation from the environment
+            
+        Returns:
+            Observation: The processed observation ready for the model
+        """
+        if not self.adapter or not observation.screenshot:
+            return observation
+            
+        # Create a new observation with the rescaled screenshot
+        processed_obs = Observation(
+            text=observation.text,
+            screenshot=self.adapter.rescale(observation.screenshot)
+        )
+        return processed_obs
+    
+    @abstractmethod
+    async def fetch_response(self, observation: Observation) -> tuple[list[ActionT], bool]:
+        """
+        Fetch a response from the model based on the observation.
+        
+        Args:
+            observation: The preprocessed observation
+            
+        Returns:
+            tuple[list[ActionT], bool]: A tuple containing the list of raw actions and a
+                                       boolean indicating if the agent believes it has
+                                       completed the task
+        """
+        pass
+    
+    def postprocess(self, actions: list[ActionT]) -> list[CLA]:
+        """
+        Convert model actions to HUD actions.
+        
+        Args:
+            actions: The raw actions from the model
+            
+        Returns:
+            Sequence[CLA]: The actions converted to HUD format
+        """
+        if not self.adapter:
+            raise ValueError("Cannot postprocess actions without an adapter")
+        
+        return self.adapter.adapt_list(actions)
+    
+    async def predict(self, observation: Observation) -> tuple[list[CLA] | list[ActionT], bool]:
+        """
+        Predict the next action based on the observation.
+        
+        Implements the full three-stage prediction process.
+        
+        Args:
+            observation: The observation from the environment
+            
+        Returns:
+            tuple[list[CLA] | list[ActionT], bool]: A tuple containing the list of actions and a boolean
+                                                       indicating if the agent believes it has completed the task
+        """
+        # Stage 1: Preprocess the observation
+        processed_obs = self.preprocess(observation)
+        
+        # Stage 2: Fetch response from the model
+        actions, done = await self.fetch_response(processed_obs)
+        
+        # Stage 3: Postprocess the actions if we have an adapter
+        if self.adapter and actions:
+            hud_actions = self.postprocess(actions)
+            return hud_actions, done
+            
+        # If no adapter, return actions as is
+        return actions, done

hud/agent/claude.py

@@ -0,0 +1,207 @@
+import logging
+import os
+from typing import Any, cast
+
+from anthropic import AsyncAnthropic
+from anthropic.types.beta import (
+    BetaMessageParam,
+    BetaToolResultBlockParam,
+    BetaToolComputerUse20250124Param,
+    BetaTextBlockParam,
+    BetaImageBlockParam,
+)
+
+from hud.adapters import Adapter
+from hud.agent.base import Agent
+from hud.adapters.claude import ClaudeAdapter
+from hud.env.environment import Observation
+from hud.settings import settings
+
+logger = logging.getLogger(__name__)
+
+def base64_to_content_block(base64: str) -> BetaImageBlockParam:
+    return {
+        "type": "image",
+        "source": {
+            "type": "base64",
+            "media_type": "image/png",
+            "data": base64
+        }
+    }
+
+def text_to_content_block(text: str) -> BetaTextBlockParam:
+    return {
+        "type": "text",
+        "text": text
+    }
+
+def tool_use_content_block(tool_use_id: str, content: list[BetaTextBlockParam | BetaImageBlockParam]) -> BetaToolResultBlockParam:
+    return {
+        "type": "tool_result",
+        "tool_use_id": tool_use_id,
+        "content": content
+    }
+
+# Claude's Computer Use Tool definition
+COMPUTER_TOOL: BetaToolComputerUse20250124Param = {
+    "type": "computer_20250124", 
+    "name": "computer", 
+    "display_width_px": 1024, 
+    "display_height_px": 768
+}
+
+class ClaudeAgent(Agent[AsyncAnthropic, Any]):
+    """
+    An agent implementation using Anthropic's Claude API with Computer Use.
+    
+    This agent interacts with HUD environments using Claude's Computer Use API
+    through the ClaudeAdapter which converts actions to the format expected by HUD.
+    """
+    
+    def __init__(
+        self, 
+        client: AsyncAnthropic | None = None,
+        adapter: Adapter | None = None,
+        model: str = "claude-3-7-sonnet-20250219",
+        max_tokens: int = 4096,
+        max_iterations: int = 10,
+    ):
+        """
+        Initialize the ClaudeAgent.
+        
+        Args:
+            client: The AsyncAnthropic client for API calls (optional, created automatically if not provided)
+            adapter: The adapter to use for preprocessing and postprocessing
+            model: The Claude model to use
+            max_tokens: Maximum tokens for Claude's response
+            max_iterations: Maximum number of iterations for the agent
+        """
+        # Initialize client if not provided
+        if client is None:
+            # Get API key from settings
+            api_key = settings.anthropic_api_key
+            if not api_key:
+                raise ValueError("Anthropic API key not found in settings or environment variables. Set ANTHROPIC_API_KEY.")
+            
+            # Create client
+            client = AsyncAnthropic(api_key=api_key)
+
+        adapter = adapter or ClaudeAdapter()
+            
+        super().__init__(client=client, adapter=adapter)
+        
+        self.model = model
+        self.max_tokens = max_tokens
+        self.max_iterations = max_iterations
+        
+        # Default dimensions - will be updated if adapter is provided
+        self.width_px = 1024
+        self.height_px = 768
+        
+        # Update dimensions if adapter is provided
+        if self.adapter:
+            self.width_px = self.adapter.agent_width
+            self.height_px = self.adapter.agent_height
+        
+        # Message history
+        self.messages: list[BetaMessageParam] = []
+        self.pending_computer_use_tool_id = None
+
+    async def fetch_response(self, observation: Observation) -> tuple[list[Any], bool]:
+        """
+        Fetch a response from Claude based on the observation.
+        
+        Args:
+            observation: The preprocessed observation
+            
+        Returns:
+            tuple[list[Any], bool]: A tuple containing the list of raw actions and a
+                                   boolean indicating if the agent believes the task is complete
+        """
+        if not self.client:
+            raise ValueError("Client is required")
+            
+        # Prepare the user content for Claude
+        user_content: list[BetaImageBlockParam | BetaTextBlockParam | BetaToolResultBlockParam] = []
+
+        # Add text instruction if present
+        if observation.text:
+            logger.info("Adding text to user content: %s", observation.text)
+            user_content.append(text_to_content_block(str(observation.text)))
+        
+        # Add screenshot if present
+        if observation.screenshot:
+            logger.info("Adding screenshot to user content")
+            if not self.pending_computer_use_tool_id:
+                logger.info("Adding screenshot to user content, no tool id")
+                user_content.append(base64_to_content_block(observation.screenshot))
+            else:
+                logger.info("Adding screenshot to user content, tool id: %s", self.pending_computer_use_tool_id)
+                user_content.append(
+                    tool_use_content_block(
+                        self.pending_computer_use_tool_id, 
+                        [base64_to_content_block(observation.screenshot)]
+                    )
+                )
+                self.pending_computer_use_tool_id = None
+
+        # Add the user content to the messages
+        self.messages.append(cast(BetaMessageParam, {
+            "role": "user",
+            "content": user_content,
+        }))
+
+        # Call Claude API using async client
+        response = await self.client.beta.messages.create(
+            model=self.model,
+            max_tokens=self.max_tokens,
+            messages=self.messages,
+            tools=[COMPUTER_TOOL],
+            betas=["computer-use-2025-01-24"],
+            tool_choice={"type": "auto", "disable_parallel_tool_use": True}
+        )
+
+        # Add Claude's response to the conversation history
+        response_content = response.content
+        self.messages.append(cast(BetaMessageParam, {
+            "role": "assistant",
+            "content": response_content,
+        }))
+
+        # Process tool use
+        actions: list[Any] = []
+        done = True  # Assume we're done unless we find a tool use
+        
+        for block in response_content:
+            logger.info("Processing block: %s", block)
+            if block.type == "tool_use":
+                logger.info("Processing tool use: %s", block)
+                assert block.name == "computer"
+                
+                # Store the raw action
+                actions.append(block.input)
+                self.pending_computer_use_tool_id = block.id
+                
+                # If we found a tool use, we're not done
+                done = False
+                break
+
+        # If no tool use action was found, check for a final text response
+        if not actions and done:
+            final_text_response = ""
+            for block in response_content:
+                if block.type == "text":
+                    final_text_response += block.text
+            
+            if final_text_response.strip():
+                logger.info(f"No tool use found. Using final text as response: {final_text_response}")
+                actions = [{
+                    "action": "response", 
+                    "text": final_text_response.strip()
+                }]
+                # Keep done = True
+            else:
+                 logger.info("No tool use and no final text block found.")
+                 # Keep done = True, actions remains empty
+            
+        return actions, done