hud-python 0.2.1__py3-none-any.whl → 0.2.3__py3-none-any.whl

hud/agent/base.py

@@ -2,108 +2,108 @@ from abc import ABC, abstractmethod
 from typing import Sequence, TypeVar, Generic
 
 from hud.adapters import Adapter, CLA
-from hud.env.environment import Observation
+from hud.utils.common import Observation
 
 # Generic type for different client types (Anthropic, OpenAI, etc.)
-ClientT = TypeVar('ClientT')
-ActionT = TypeVar('ActionT')
+ClientT = TypeVar("ClientT")
+ActionT = TypeVar("ActionT")
+
 
 class Agent(Generic[ClientT, ActionT], ABC):
     """
     Base class for all agents.
-    
+
     Implements a three-stage prediction process:
     1. preprocess - Prepare observation data (e.g., rescale screenshot)
     2. fetch_response - Make API calls to get model response
     3. postprocess - Convert model actions to HUD format
-    
+
     Subclasses only need to implement the fetch_response method.
     """
-    
+
     def __init__(self, client: ClientT | None = None, adapter: Adapter | None = None):
         """
         Initialize the agent.
-        
+
         Args:
             client: The client to use for API calls
             adapter: The adapter to use for preprocessing and postprocessing
         """
         self.client = client
         self.adapter = adapter
-    
+
     def preprocess(self, observation: Observation) -> Observation:
         """
         Preprocess the observation before sending to the model.
-        
+
         Args:
             observation: The raw observation from the environment
-            
+
         Returns:
             Observation: The processed observation ready for the model
         """
         if not self.adapter or not observation.screenshot:
             return observation
-            
+
         # Create a new observation with the rescaled screenshot
         processed_obs = Observation(
-            text=observation.text,
-            screenshot=self.adapter.rescale(observation.screenshot)
+            text=observation.text, screenshot=self.adapter.rescale(observation.screenshot)
         )
         return processed_obs
-    
+
     @abstractmethod
     async def fetch_response(self, observation: Observation) -> tuple[list[ActionT], bool]:
         """
         Fetch a response from the model based on the observation.
-        
+
         Args:
             observation: The preprocessed observation
-            
+
         Returns:
             tuple[list[ActionT], bool]: A tuple containing the list of raw actions and a
                                        boolean indicating if the agent believes it has
                                        completed the task
         """
         pass
-    
+
     def postprocess(self, actions: list[ActionT]) -> list[CLA]:
         """
         Convert model actions to HUD actions.
-        
+
         Args:
             actions: The raw actions from the model
-            
+
         Returns:
             Sequence[CLA]: The actions converted to HUD format
         """
         if not self.adapter:
             raise ValueError("Cannot postprocess actions without an adapter")
-        
+
         return self.adapter.adapt_list(actions)
-    
+
     async def predict(self, observation: Observation) -> tuple[list[CLA] | list[ActionT], bool]:
         """
         Predict the next action based on the observation.
-        
+
         Implements the full three-stage prediction process.
-        
+
         Args:
             observation: The observation from the environment
-            
+
         Returns:
             tuple[list[CLA] | list[ActionT], bool]: A tuple containing the list of actions and a boolean
                                                        indicating if the agent believes it has completed the task
         """
         # Stage 1: Preprocess the observation
         processed_obs = self.preprocess(observation)
-        
+
         # Stage 2: Fetch response from the model
         actions, done = await self.fetch_response(processed_obs)
-        
+
         # Stage 3: Postprocess the actions if we have an adapter
         if self.adapter and actions:
             hud_actions = self.postprocess(actions)
             return hud_actions, done
-            
+
         # If no adapter, return actions as is
-        return actions, done
+        return actions, done

hud/agent/claude.py

@@ -1,5 +1,4 @@
 import logging
-import os
 from typing import Any, cast
 
 from anthropic import AsyncAnthropic
@@ -14,52 +13,48 @@ from anthropic.types.beta import (
 from hud.adapters import Adapter
 from hud.agent.base import Agent
 from hud.adapters.claude import ClaudeAdapter
-from hud.env.environment import Observation
+from hud.utils.common import Observation
 from hud.settings import settings
 
 logger = logging.getLogger(__name__)
 
+
 def base64_to_content_block(base64: str) -> BetaImageBlockParam:
     return {
         "type": "image",
-        "source": {
-            "type": "base64",
-            "media_type": "image/png",
-            "data": base64
-        }
+        "source": {"type": "base64", "media_type": "image/png", "data": base64},
     }
 
+
 def text_to_content_block(text: str) -> BetaTextBlockParam:
-    return {
-        "type": "text",
-        "text": text
-    }
+    return {"type": "text", "text": text}
+
+
+def tool_use_content_block(
+    tool_use_id: str, content: list[BetaTextBlockParam | BetaImageBlockParam]
+) -> BetaToolResultBlockParam:
+    return {"type": "tool_result", "tool_use_id": tool_use_id, "content": content}
 
-def tool_use_content_block(tool_use_id: str, content: list[BetaTextBlockParam | BetaImageBlockParam]) -> BetaToolResultBlockParam:
-    return {
-        "type": "tool_result",
-        "tool_use_id": tool_use_id,
-        "content": content
-    }
 
 # Claude's Computer Use Tool definition
 COMPUTER_TOOL: BetaToolComputerUse20250124Param = {
-    "type": "computer_20250124", 
-    "name": "computer", 
-    "display_width_px": 1024, 
-    "display_height_px": 768
+    "type": "computer_20250124",
+    "name": "computer",
+    "display_width_px": 1024,
+    "display_height_px": 768,
 }
 
+
 class ClaudeAgent(Agent[AsyncAnthropic, Any]):
     """
     An agent implementation using Anthropic's Claude API with Computer Use.
-    
+
     This agent interacts with HUD environments using Claude's Computer Use API
     through the ClaudeAdapter which converts actions to the format expected by HUD.
     """
-    
+
     def __init__(
-        self, 
+        self,
         client: AsyncAnthropic | None = None,
         adapter: Adapter | None = None,
         model: str = "claude-3-7-sonnet-20250219",
@@ -68,7 +63,7 @@ class ClaudeAgent(Agent[AsyncAnthropic, Any]):
     ):
         """
         Initialize the ClaudeAgent.
-        
+
         Args:
             client: The AsyncAnthropic client for API calls (optional, created automatically if not provided)
             adapter: The adapter to use for preprocessing and postprocessing
@@ -81,28 +76,30 @@ class ClaudeAgent(Agent[AsyncAnthropic, Any]):
             # Get API key from settings
             api_key = settings.anthropic_api_key
             if not api_key:
-                raise ValueError("Anthropic API key not found in settings or environment variables. Set ANTHROPIC_API_KEY.")
-            
+                raise ValueError(
+                    "Anthropic API key not found in settings or environment variables. Set ANTHROPIC_API_KEY."
+                )
+
             # Create client
             client = AsyncAnthropic(api_key=api_key)
 
         adapter = adapter or ClaudeAdapter()
-            
+
         super().__init__(client=client, adapter=adapter)
-        
+
         self.model = model
         self.max_tokens = max_tokens
         self.max_iterations = max_iterations
-        
+
         # Default dimensions - will be updated if adapter is provided
         self.width_px = 1024
         self.height_px = 768
-        
+
         # Update dimensions if adapter is provided
         if self.adapter:
             self.width_px = self.adapter.agent_width
             self.height_px = self.adapter.agent_height
-        
+
         # Message history
         self.messages: list[BetaMessageParam] = []
         self.pending_computer_use_tool_id = None
@@ -110,17 +107,17 @@ class ClaudeAgent(Agent[AsyncAnthropic, Any]):
     async def fetch_response(self, observation: Observation) -> tuple[list[Any], bool]:
         """
         Fetch a response from Claude based on the observation.
-        
+
         Args:
             observation: The preprocessed observation
-            
+
         Returns:
             tuple[list[Any], bool]: A tuple containing the list of raw actions and a
                                    boolean indicating if the agent believes the task is complete
         """
         if not self.client:
             raise ValueError("Client is required")
-            
+
         # Prepare the user content for Claude
         user_content: list[BetaImageBlockParam | BetaTextBlockParam | BetaToolResultBlockParam] = []
 
@@ -128,7 +125,7 @@ class ClaudeAgent(Agent[AsyncAnthropic, Any]):
         if observation.text:
             logger.info("Adding text to user content: %s", observation.text)
             user_content.append(text_to_content_block(str(observation.text)))
-        
+
         # Add screenshot if present
         if observation.screenshot:
             logger.info("Adding screenshot to user content")
@@ -136,20 +133,28 @@ class ClaudeAgent(Agent[AsyncAnthropic, Any]):
                 logger.info("Adding screenshot to user content, no tool id")
                 user_content.append(base64_to_content_block(observation.screenshot))
             else:
-                logger.info("Adding screenshot to user content, tool id: %s", self.pending_computer_use_tool_id)
+                logger.info(
+                    "Adding screenshot to user content, tool id: %s",
+                    self.pending_computer_use_tool_id,
+                )
                 user_content.append(
                     tool_use_content_block(
-                        self.pending_computer_use_tool_id, 
-                        [base64_to_content_block(observation.screenshot)]
+                        self.pending_computer_use_tool_id,
+                        [base64_to_content_block(observation.screenshot)],
                     )
                 )
                 self.pending_computer_use_tool_id = None
 
         # Add the user content to the messages
-        self.messages.append(cast(BetaMessageParam, {
-            "role": "user",
-            "content": user_content,
-        }))
+        self.messages.append(
+            cast(
+                BetaMessageParam,
+                {
+                    "role": "user",
+                    "content": user_content,
+                },
+            )
+        )
 
         # Call Claude API using async client
         response = await self.client.beta.messages.create(
@@ -158,30 +163,35 @@ class ClaudeAgent(Agent[AsyncAnthropic, Any]):
             messages=self.messages,
             tools=[COMPUTER_TOOL],
             betas=["computer-use-2025-01-24"],
-            tool_choice={"type": "auto", "disable_parallel_tool_use": True}
+            tool_choice={"type": "auto", "disable_parallel_tool_use": True},
         )
 
         # Add Claude's response to the conversation history
         response_content = response.content
-        self.messages.append(cast(BetaMessageParam, {
-            "role": "assistant",
-            "content": response_content,
-        }))
+        self.messages.append(
+            cast(
+                BetaMessageParam,
+                {
+                    "role": "assistant",
+                    "content": response_content,
+                },
+            )
+        )
 
         # Process tool use
         actions: list[Any] = []
         done = True  # Assume we're done unless we find a tool use
-        
+
         for block in response_content:
             logger.info("Processing block: %s", block)
             if block.type == "tool_use":
                 logger.info("Processing tool use: %s", block)
                 assert block.name == "computer"
-                
+
                 # Store the raw action
                 actions.append(block.input)
                 self.pending_computer_use_tool_id = block.id
-                
+
                 # If we found a tool use, we're not done
                 done = False
                 break
@@ -192,16 +202,15 @@ class ClaudeAgent(Agent[AsyncAnthropic, Any]):
             for block in response_content:
                 if block.type == "text":
                     final_text_response += block.text
-            
+
             if final_text_response.strip():
-                logger.info(f"No tool use found. Using final text as response: {final_text_response}")
-                actions = [{
-                    "action": "response", 
-                    "text": final_text_response.strip()
-                }]
+                logger.info(
+                    f"No tool use found. Using final text as response: {final_text_response}"
+                )
+                actions = [{"action": "response", "text": final_text_response.strip()}]
                 # Keep done = True
             else:
-                 logger.info("No tool use and no final text block found.")
-                 # Keep done = True, actions remains empty
-            
+                logger.info("No tool use and no final text block found.")
+                # Keep done = True, actions remains empty
+
         return actions, done

hud/agent/langchain.py

@@ -0,0 +1,204 @@
+import logging
+from typing import Any, Generic, List, Optional, TypeVar, Union, cast
+
+# Langchain imports
+from langchain_core.language_models import BaseLanguageModel
+from langchain_core.messages import AIMessage, BaseMessage, HumanMessage, SystemMessage
+from langchain_core.runnables import Runnable, RunnableSerializable
+from pydantic import Field, BaseModel
+
+# HUD imports
+from hud.adapters import Adapter
+from hud.agent.base import Agent
+from hud.utils.common import Observation
+from hud.adapters.common.types import (
+    ClickAction,
+    TypeAction,
+    ScrollAction,
+    MoveAction,
+    DragAction,
+    PressAction,
+    KeyDownAction,
+    KeyUpAction,
+    WaitAction,
+    ResponseAction,
+    CustomAction,
+    # Exclude ScreenshotFetch, PositionFetch as they are internal
+)
+
+logger = logging.getLogger(__name__)
+
+# Define a Pydantic Union type representing exactly ONE possible CLA action
+# This is what we'll ask the Langchain model to output.
+SingleCLAction = Union[
+    ClickAction,
+    TypeAction,
+    ScrollAction,
+    MoveAction,
+    DragAction,
+    PressAction,
+    KeyDownAction,
+    KeyUpAction,
+    WaitAction,
+    ResponseAction,
+]
+
+
+# Define a Pydantic model to wrap the single action, potentially making it
+# easier for the LLM to consistently output the desired structure.
+class StepAction(BaseModel):
+    """Wrapper model requesting a single concrete CLA action from the Langchain model."""
+
+    action: SingleCLAction = Field(
+        ..., description="The single CLA action to perform for this step."
+    )
+
+
+# Generic Type for the Langchain Model/Runnable
+# Allows flexibility in what the user provides (model, chain, etc.)
+# Bound to BaseLanguageModel as .with_structured_output is expected
+LangchainModelOrRunnable = TypeVar("LangchainModelOrRunnable", bound=BaseLanguageModel)
+
+
+class LangchainAgent(Agent[LangchainModelOrRunnable, Any], Generic[LangchainModelOrRunnable]):
+    """
+    An agent that uses an arbitrary Langchain model or runnable, leveraging
+    Langchain's structured output capabilities to produce a single CLA action per step.
+    """
+
+    def __init__(
+        self,
+        langchain_model: LangchainModelOrRunnable,
+        adapter: Optional[Adapter] = None,
+        system_prompt: str | None = None,
+    ):
+        """
+        Initialize the LangchainAgent.
+
+        Args:
+            langchain_model: The Langchain language model or runnable chain to use.
+                             Must support asynchronous invocation (`ainvoke`) and
+                             `.with_structured_output()`.
+            adapter: An optional HUD adapter. If provided, it will be used for
+                     preprocessing observations (rescaling) and postprocessing
+                     the single CLA action (coordinate rescaling).
+            system_prompt: An optional system prompt to guide the Langchain model.
+                           If None, a default prompt encouraging single CLA output is used.
+        """
+        super().__init__(client=langchain_model, adapter=adapter)  # Store model as 'client'
+        self.langchain_model = langchain_model  # Also store with specific name
+
+        self.system_prompt_str = system_prompt or self._get_default_system_prompt()
+        self.history: List[BaseMessage] = []
+
+    def _get_default_system_prompt(self) -> str:
+        # TODO: Refine this prompt based on testing.
+        # It needs to strongly encourage outputting *only* the StepAction structure.
+        return (
+            "You are an agent interacting with a computer environment (either a web browser or an OS desktop). "
+            "Your goal is to follow the user's instructions based on the provided text and screenshot observations."
+            "For each step, you must choose exactly ONE action to perform from the available CLA action types."
+            "Output your chosen action using the provided 'StepAction' tool/function."
+            "If you believe the task is complete based on the user's prompt and the observations, use the 'ResponseAction'."
+        )
+
+    async def fetch_response(self, observation: Observation) -> tuple[list[dict], bool]:
+        """
+        Fetches a response from the configured Langchain model, expecting a single
+        structured CLA action.
+
+        Args:
+            observation: The preprocessed observation (screenshot potentially rescaled by adapter).
+
+        Returns:
+            A tuple containing:
+            - A list with a single dictionary representing the raw CLA action (before adapter postprocessing).
+            - A boolean indicating if the agent chose ResponseAction (task completion).
+        """
+        # 1. Format observation into Langchain message(s)
+        human_content: List[Union[str, dict]] = []
+        if observation.text:
+            human_content.append(observation.text)
+        if observation.screenshot:
+            # Assuming the Langchain model/chain can handle base64 images
+            # This might need adjustment based on the specific model used.
+            human_content.append(
+                {
+                    "type": "image_url",
+                    "image_url": {"url": f"data:image/png;base64,{observation.screenshot}"},
+                }
+            )
+
+        if not human_content:
+            logger.warning("LangchainAgent received an observation with no text or screenshot.")
+            # Decide how to handle empty observation - perhaps return no action?
+            return [], False  # Or raise an error?
+
+        current_human_message = HumanMessage(content=human_content)
+
+        # 2. Prepare message history for the model
+        messages_for_llm: List[BaseMessage] = [
+            SystemMessage(content=self.system_prompt_str),
+            *self.history,
+            current_human_message,
+        ]
+
+        # 3. Configure structured output
+        # We ask for the StepAction wrapper, which contains the actual SingleCLAAction
+        # Explicitly use method="function_calling" to handle schemas with default values
+        structured_llm = self.langchain_model.with_structured_output(
+            schema=StepAction, method="function_calling"
+        )
+
+        # 4. Invoke Langchain model asynchronously
+        try:
+            ai_response_structured = await structured_llm.ainvoke(messages_for_llm)
+        except Exception as e:
+            logger.error(f"Langchain model invocation failed: {e}", exc_info=True)
+            # Decide how to handle LLM errors - maybe retry or return empty action?
+            return [], False
+
+        # 5. Process the structured response
+        is_done = False
+        ai_message_content_for_history = ""  # For storing in history
+
+        if isinstance(ai_response_structured, StepAction):
+            # Successfully got the wrapper, extract the actual action
+            actual_action = ai_response_structured.action
+            ai_message_content_for_history = actual_action.model_dump()
+            if isinstance(actual_action, ResponseAction):
+                is_done = True
+                logger.info(
+                    f"LangchainAgent determined task is done with response: {actual_action.text[:100]}..."
+                )
+            else:
+                logger.info(f"LangchainAgent produced action: {type(actual_action).__name__}")
+
+        else:
+            logger.warning(
+                f"Langchain model did not return the expected StepAction structure. "
+                f"Received type: {type(ai_response_structured)}. Value: {ai_response_structured!r}"
+            )
+            # Attempt to add raw response to history for debugging
+            if isinstance(ai_response_structured, BaseMessage):
+                ai_message_content_for_history = ai_response_structured.content
+            elif isinstance(ai_response_structured, str):
+                ai_message_content_for_history = ai_response_structured
+            else:
+                ai_message_content_for_history = repr(ai_response_structured)
+            # Return no action as we didn't get the expected structure
+            return [], False
+
+        # 6. Update history
+        self.history.append(current_human_message)
+        # Add the AI response (containing the structured action dict) to history
+        # Convert dict to string representation for AIMessage content
+        self.history.append(AIMessage(content=repr(ai_message_content_for_history)))
+        # TODO: Consider history truncation/summarization if it grows too long
+
+        if actual_action:
+            # Return the single action dictionary within a list
+            return [actual_action.model_dump()], is_done
+        else:
+            # Should ideally not happen if structure validation worked, but as a fallback
+            return [], is_done