hud-python 0.2.0__py3-none-any.whl → 0.2.2__py3-none-any.whl

hud/__init__.py

@@ -5,10 +5,10 @@ HUD Gym SDK - A Python SDK for interacting with HUD environments.
 from __future__ import annotations
 
 from . import agent, env, gym, settings, task, taskset, types, utils
-from .job import create_job, job, load_job
+from .job import create_job, job, load_job, run_job
 from .taskset import load_taskset
 
-__version__ = "0.2.0"
+__version__ = "0.2.2"
 
 __all__ = [
     "agent",
@@ -18,6 +18,7 @@ __all__ = [
     "job",
     "load_job",
     "load_taskset",
+    "run_job",
     "settings",
     "task",
     "taskset",

hud/adapters/__init__.py

@@ -2,6 +2,7 @@ from __future__ import annotations
 
 from .claude import ClaudeAdapter
 from .common import CLA, Adapter
+from .common.types import ResponseAction
 from .operator import OperatorAdapter
 
-__all__ = ["CLA", "Adapter", "ClaudeAdapter", "OperatorAdapter"]
+__all__ = ["CLA", "Adapter", "ClaudeAdapter", "OperatorAdapter", "ResponseAction"]

hud/adapters/claude/adapter.py

@@ -13,6 +13,7 @@ from hud.adapters.common.types import (
     Point,
     PositionFetch,
     PressAction,
+    ResponseAction,
     ScreenshotFetch,
     ScrollAction,
     TypeAction,
@@ -21,7 +22,14 @@ from hud.adapters.common.types import (
 
 
 class ClaudeAdapter(Adapter):
-    KEY_MAP: ClassVar[dict[str, CLAKey]] = {"Return": "enter"}
+    KEY_MAP: ClassVar[dict[str, CLAKey]] = {
+        "return": "enter",
+        "super": "win",
+        "super_l": "win",
+        "super_r": "win",
+        "right shift": "shift",
+        "left shift": "shift",
+    }
 
     def __init__(self) -> None:
         super().__init__()
@@ -30,7 +38,8 @@ class ClaudeAdapter(Adapter):
 
     def _map_key(self, key: str) -> CLAKey:
         """Map a key to its standardized form."""
-        return self.KEY_MAP.get(key, key.lower())  # type: ignore
+        return self.KEY_MAP.get(key.lower(), key.lower())  # type: ignore
+    
     def convert(self, data: Any) -> CLA:
         try:
             action_type = data.get("action")
@@ -151,6 +160,10 @@ class ClaudeAdapter(Adapter):
             elif action_type == "wait":
                 assert "duration" in data
                 return WaitAction(time=data["duration"])
+
+            elif action_type == "response":
+                return ResponseAction(text=data.get("text", ""))
+
             else:
                 raise ValueError(f"Unsupported action type: {action_type}")
         except AssertionError:

hud/adapters/common/types.py

@@ -20,7 +20,6 @@ class Point(BaseModel):
 class ClickAction(CLAAction):
     type: Literal["click"] = "click"
     point: Point | None = None
-    selector: str | None = None
     button: Literal["left", "right", "wheel", "back", "forward"] = "left"
     pattern: list[int] | None = None  # [delay_1, delay_2, ...]
     hold_keys: list[CLAKey] | None = None
@@ -48,7 +47,6 @@ class KeyUpAction(CLAAction):
 class TypeAction(CLAAction):
     type: Literal["type"] = "type"
     text: str
-    selector: str | None = None
     enter_after: bool | None = False
 
 
@@ -64,7 +62,6 @@ class ScrollAction(CLAAction):
 class MoveAction(CLAAction):
     type: Literal["move"] = "move"
     point: Point | None = None
-    selector: str | None = None
     offset: Point | None = None
 
 
@@ -82,6 +79,12 @@ class DragAction(CLAAction):
     hold_keys: list[CLAKey] | None = None
 
 
+# RESPONSE ACTION from agent
+class ResponseAction(CLAAction):
+    type: Literal["response"] = "response"
+    text: str # The final textual response from the agent
+
+
 # SCREENSHOT ACTION
 class ScreenshotFetch(CLAAction):
     type: Literal["screenshot"] = "screenshot"
@@ -103,6 +106,7 @@ CLA = Annotated[
     | KeyDownAction
     | KeyUpAction
     | TypeAction
+    | ResponseAction
     | ScrollAction
     | MoveAction
     | WaitAction

hud/adapters/operator/adapter.py

@@ -10,6 +10,7 @@ from hud.adapters.common.types import (
     MoveAction,
     Point,
     PressAction,
+    ResponseAction,
     ScreenshotFetch,
     ScrollAction,
     TypeAction,
@@ -19,11 +20,11 @@ from hud.adapters.common.types import (
 
 class OperatorAdapter(Adapter):
     KEY_MAP: ClassVar[dict[str, CLAKey]] = {
-        "Return": "enter",
-        "ArrowUp": "up",
-        "ArrowDown": "down",
-        "ArrowLeft": "left",
-        "ArrowRight": "right",
+        "return": "enter",
+        "arrowup": "up",
+        "arrowdown": "down",
+        "arrowleft": "left",
+        "arrowright": "right",
     }
     
     def __init__(self) -> None:
@@ -34,7 +35,7 @@ class OperatorAdapter(Adapter):
         
     def _map_key(self, key: str) -> CLAKey:
         """Map a key to its standardized form."""
-        return self.KEY_MAP.get(key, key.lower())  # type: ignore
+        return self.KEY_MAP.get(key.lower(), key.lower())  # type: ignore
         
     def convert(self, data: Any) -> CLA:
         """Convert a Computer Use action to a HUD action"""
@@ -86,6 +87,9 @@ class OperatorAdapter(Adapter):
             
             elif action_type == "screenshot":
                 return ScreenshotFetch()
+            
+            elif action_type == "response":
+                return ResponseAction(text=data.get("text", ""))
             else:
                 raise ValueError(f"Unsupported action type: {action_type}")
                 

hud/agent/__init__.py

@@ -1,7 +1,8 @@
 from .base import Agent
 from .claude import ClaudeAgent
 from .operator import OperatorAgent
+from .langchain import LangchainAgent
 
 from hud.adapters import OperatorAdapter, ClaudeAdapter
 
-__all__ = ["Agent", "ClaudeAgent", "OperatorAgent", "OperatorAdapter", "ClaudeAdapter"]
+__all__ = ["Agent", "ClaudeAgent", "OperatorAgent", "OperatorAdapter", "ClaudeAdapter", "LangchainAgent"]

hud/agent/claude.py

@@ -11,7 +11,7 @@ from anthropic.types.beta import (
     BetaImageBlockParam,
 )
 
-
+from hud.adapters import Adapter
 from hud.agent.base import Agent
 from hud.adapters.claude import ClaudeAdapter
 from hud.env.environment import Observation
@@ -61,7 +61,7 @@ class ClaudeAgent(Agent[AsyncAnthropic, Any]):
     def __init__(
         self, 
         client: AsyncAnthropic | None = None,
-        adapter: ClaudeAdapter | None = None,
+        adapter: Adapter | None = None,
         model: str = "claude-3-7-sonnet-20250219",
         max_tokens: int = 4096,
         max_iterations: int = 10,
@@ -85,6 +85,8 @@ class ClaudeAgent(Agent[AsyncAnthropic, Any]):
             
             # Create client
             client = AsyncAnthropic(api_key=api_key)
+
+        adapter = adapter or ClaudeAdapter()
             
         super().__init__(client=client, adapter=adapter)
         
@@ -184,4 +186,22 @@ class ClaudeAgent(Agent[AsyncAnthropic, Any]):
                 done = False
                 break
 
+        # If no tool use action was found, check for a final text response
+        if not actions and done:
+            final_text_response = ""
+            for block in response_content:
+                if block.type == "text":
+                    final_text_response += block.text
+            
+            if final_text_response.strip():
+                logger.info(f"No tool use found. Using final text as response: {final_text_response}")
+                actions = [{
+                    "action": "response", 
+                    "text": final_text_response.strip()
+                }]
+                # Keep done = True
+            else:
+                 logger.info("No tool use and no final text block found.")
+                 # Keep done = True, actions remains empty
+            
         return actions, done

hud/agent/langchain.py

@@ -0,0 +1,198 @@
+import logging
+from typing import Any, Generic, List, Optional, TypeVar, Union, cast
+
+# Langchain imports
+from langchain_core.language_models import BaseLanguageModel
+from langchain_core.messages import AIMessage, BaseMessage, HumanMessage, SystemMessage
+from langchain_core.runnables import Runnable, RunnableSerializable
+from pydantic import Field, BaseModel
+
+# HUD imports
+from hud.adapters import Adapter
+from hud.agent.base import Agent
+from hud.env.environment import Observation
+from hud.adapters.common.types import (
+    CLA,
+    ClickAction,
+    TypeAction,
+    ScrollAction,
+    MoveAction,
+    DragAction,
+    PressAction,
+    KeyDownAction,
+    KeyUpAction,
+    WaitAction,
+    ResponseAction,
+    CustomAction,
+    # Exclude ScreenshotFetch, PositionFetch as they are internal
+)
+
+logger = logging.getLogger(__name__)
+
+# Define a Pydantic Union type representing exactly ONE possible CLA action
+# This is what we'll ask the Langchain model to output.
+SingleCLAction = Union[
+    ClickAction,
+    TypeAction,
+    ScrollAction,
+    MoveAction,
+    DragAction,
+    PressAction,
+    KeyDownAction,
+    KeyUpAction,
+    WaitAction,
+    ResponseAction,
+]
+
+# Define a Pydantic model to wrap the single action, potentially making it
+# easier for the LLM to consistently output the desired structure.
+class StepAction(BaseModel):
+    """Wrapper model requesting a single concrete CLA action from the Langchain model."""
+    action: SingleCLAction = Field(..., description="The single CLA action to perform for this step.")
+
+# Generic Type for the Langchain Model/Runnable
+# Allows flexibility in what the user provides (model, chain, etc.)
+# Bound to BaseLanguageModel as .with_structured_output is expected
+LangchainModelOrRunnable = TypeVar("LangchainModelOrRunnable", bound=BaseLanguageModel)
+
+class LangchainAgent(Agent[LangchainModelOrRunnable, Any], Generic[LangchainModelOrRunnable]):
+    """
+    An agent that uses an arbitrary Langchain model or runnable, leveraging
+    Langchain's structured output capabilities to produce a single CLA action per step.
+    """
+
+    def __init__(
+        self,
+        langchain_model: LangchainModelOrRunnable,
+        adapter: Optional[Adapter] = None,
+        system_prompt: str | None = None,
+    ):
+        """
+        Initialize the LangchainAgent.
+
+        Args:
+            langchain_model: The Langchain language model or runnable chain to use.
+                             Must support asynchronous invocation (`ainvoke`) and
+                             `.with_structured_output()`.
+            adapter: An optional HUD adapter. If provided, it will be used for
+                     preprocessing observations (rescaling) and postprocessing
+                     the single CLA action (coordinate rescaling).
+            system_prompt: An optional system prompt to guide the Langchain model.
+                           If None, a default prompt encouraging single CLA output is used.
+        """
+        super().__init__(client=langchain_model, adapter=adapter) # Store model as 'client'
+        self.langchain_model = langchain_model # Also store with specific name
+
+        self.system_prompt_str = system_prompt or self._get_default_system_prompt()
+        self.history: List[BaseMessage] = []
+
+    def _get_default_system_prompt(self) -> str:
+        # TODO: Refine this prompt based on testing.
+        # It needs to strongly encourage outputting *only* the StepAction structure.
+        return (
+            "You are an agent interacting with a computer environment (either a web browser or an OS desktop). "
+            "Your goal is to follow the user's instructions based on the provided text and screenshot observations."
+            "For each step, you must choose exactly ONE action to perform from the available CLA action types."
+            "Output your chosen action using the provided 'StepAction' tool/function."
+            "If you believe the task is complete based on the user's prompt and the observations, use the 'ResponseAction'."
+        )
+
+    async def fetch_response(self, observation: Observation) -> tuple[CLA | None, bool]:
+        """
+        Fetches a response from the configured Langchain model, expecting a single
+        structured CLA action.
+
+        Args:
+            observation: The preprocessed observation (screenshot potentially rescaled by adapter).
+
+        Returns:
+            A tuple containing:
+            - A list with a single dictionary representing the raw CLA action (before adapter postprocessing).
+            - A boolean indicating if the agent chose ResponseAction (task completion).
+        """
+        # 1. Format observation into Langchain message(s)
+        human_content: List[Union[str, dict]] = []
+        if observation.text:
+            human_content.append(observation.text)
+        if observation.screenshot:
+            # Assuming the Langchain model/chain can handle base64 images
+            # This might need adjustment based on the specific model used.
+            human_content.append({
+                "type": "image_url",
+                "image_url": {
+                    "url": f"data:image/png;base64,{observation.screenshot}"
+                }
+            })
+        
+        if not human_content:
+             logger.warning("LangchainAgent received an observation with no text or screenshot.")
+             # Decide how to handle empty observation - perhaps return no action?
+             return [], False # Or raise an error?
+
+        current_human_message = HumanMessage(content=human_content)
+
+        # 2. Prepare message history for the model
+        messages_for_llm: List[BaseMessage] = [
+            SystemMessage(content=self.system_prompt_str),
+            *self.history,
+            current_human_message,
+        ]
+
+        # 3. Configure structured output
+        # We ask for the StepAction wrapper, which contains the actual SingleCLAAction
+        # Explicitly use method="function_calling" to handle schemas with default values
+        structured_llm = self.langchain_model.with_structured_output(
+            schema=StepAction, 
+            method="function_calling"
+        )
+
+        # 4. Invoke Langchain model asynchronously
+        try:
+            ai_response_structured = await structured_llm.ainvoke(messages_for_llm)
+        except Exception as e:
+            logger.error(f"Langchain model invocation failed: {e}", exc_info=True)
+            # Decide how to handle LLM errors - maybe retry or return empty action?
+            return [], False
+
+        # 5. Process the structured response
+        is_done = False
+        ai_message_content_for_history = "" # For storing in history
+
+        if isinstance(ai_response_structured, StepAction):
+            # Successfully got the wrapper, extract the actual action
+            actual_action = ai_response_structured.action
+            ai_message_content_for_history = actual_action.model_dump()
+            if isinstance(actual_action, ResponseAction):
+                is_done = True
+                logger.info(f"LangchainAgent determined task is done with response: {actual_action.text[:100]}...")
+            else:
+                 logger.info(f"LangchainAgent produced action: {type(actual_action).__name__}")
+
+        else:
+            logger.warning(
+                f"Langchain model did not return the expected StepAction structure. "
+                f"Received type: {type(ai_response_structured)}. Value: {ai_response_structured!r}"
+            )
+             # Attempt to add raw response to history for debugging
+            if isinstance(ai_response_structured, BaseMessage):
+                 ai_message_content_for_history = ai_response_structured.content
+            elif isinstance(ai_response_structured, str):
+                 ai_message_content_for_history = ai_response_structured
+            else:
+                 ai_message_content_for_history = repr(ai_response_structured)
+            # Return no action as we didn't get the expected structure
+            return [], False
+
+        # 6. Update history
+        self.history.append(current_human_message)
+        # Add the AI response (containing the structured action dict) to history
+        # Convert dict to string representation for AIMessage content
+        self.history.append(AIMessage(content=repr(ai_message_content_for_history)))
+        # TODO: Consider history truncation/summarization if it grows too long
+
+        if actual_action:
+            # Return the single action dictionary within a list
+            return [actual_action], is_done
+        else:
+            # Should ideally not happen if structure validation worked, but as a fallback
+            return [], is_done 

hud/agent/operator.py

@@ -9,9 +9,11 @@ from openai.types.responses import (
     ResponseInputParam,
     ResponseInputItemParam,
     ResponseOutputMessage,
-    ResponseComputerToolCall
+    ResponseComputerToolCall,
+    ResponseOutputText
 )
 
+from hud.adapters import Adapter
 from hud.agent.base import Agent
 from hud.adapters.operator import OperatorAdapter
 from hud.env.environment import Observation
@@ -32,7 +34,7 @@ class OperatorAgent(Agent[OpenAI, dict[str, Any]]):
         client: OpenAI | None = None,
         model: str = "computer-use-preview",
         environment: Literal["windows", "mac", "linux", "browser"] = "windows",
-        adapter: OperatorAdapter | None = None,
+        adapter: Adapter | None = None,
         max_iterations: int = 8
     ):
         """
@@ -54,6 +56,8 @@ class OperatorAgent(Agent[OpenAI, dict[str, Any]]):
             
             # Create synchronous client
             client = OpenAI(api_key=api_key)
+
+        adapter = adapter or OperatorAdapter()
         
         super().__init__(client=client, adapter=adapter)
         
@@ -74,7 +78,7 @@ class OperatorAgent(Agent[OpenAI, dict[str, Any]]):
         self.last_response_id = None
         self.pending_call_id = None
         self.initial_prompt = None
-    
+
     async def fetch_response(self, observation: Observation) -> tuple[list[dict[str, Any]], bool]:
         """
         Fetch a response from the model based on the observation.
@@ -158,33 +162,47 @@ class OperatorAgent(Agent[OpenAI, dict[str, Any]]):
         # Store the response ID for the next call
         self.last_response_id = response.id
         
-        # Process the response to extract computer calls
+        # Process the response to extract actions or final text
         actions = []
-        done = True  # Assume we're done unless we find a computer call
-        
-        # Loop through all items in the output to find computer_call items
+        done = True  # Assume done unless a computer call is found
+        final_text_response = ""
+
+        # Check for computer calls first
         computer_calls = [
             item for item in response.output 
             if isinstance(item, ResponseComputerToolCall) and item.type == "computer_call"
         ]
         
         if computer_calls:
-            # Extract the computer calls and mark that we're not done
+            # If computer calls exist, process them and set done=False
             done = False
-            
-            # Process all computer calls
             for computer_call in computer_calls:
                 self.pending_call_id = computer_call.call_id
                 action = computer_call.action
-                actions.append(action.model_dump())
-                
-                # Log the action
+                actions.append(action.model_dump()) # Convert Pydantic model to dict
                 logger.info(f"Computer call action: {action}")
         else:
-            # If there are no computer calls, print some debug info
-            logger.info("No computer call found in the response. Either complete or error.")
+            # No computer calls, check for a final text message
+            logger.info("No computer call found. Checking for final message.")
+            logger.info(response.output)
             for item in response.output:
                 if isinstance(item, ResponseOutputMessage) and item.type == "message":
-                    logger.info(f"Message: {item.content}")
-        
+                    # Extract text from content blocks within the message
+                    full_text = "".join([c.text for c in item.content if isinstance(c, ResponseOutputText)])
+                    if full_text:
+                        final_text_response = full_text
+                        logger.info(f"Final text message: {final_text_response}")
+                        break # Stop after finding the first text message
+            
+            # If we found final text, package it as a 'response' action
+            if final_text_response:
+                actions = [{
+                    "type": "response",
+                    "text": final_text_response
+                }]
+                # Keep done = True
+            else:
+                logger.info("No computer calls and no final text message found.")
+                # Keep done = True, actions remains empty
+
         return actions, done 

hud/env/docker_client.py

@@ -215,7 +215,7 @@ class DockerClient(Client):
                 raise ValueError("Could not find package name in pyproject.toml")
             logger.info("Installing %s in /root/controller", self._package_name)
             result = await self.execute(
-                ["bash", "-c", "cd /root/controller && pip install -e ."],
+                ["bash", "-c", "cd /root/controller && pip install -e . --break-system-packages"],
                 timeout=60,
             )
             if result["stdout"]: