hud-python 0.3.4__py3-none-any.whl → 0.4.0__py3-none-any.whl

hud/agent/claude.py

@@ -1,271 +0,0 @@
-import copy
-import logging
-from typing import Any, cast
-
-from anthropic import AsyncAnthropic, BadRequestError
-from anthropic.types.beta import (
-    BetaMessageParam,
-    BetaToolResultBlockParam,
-    BetaToolComputerUse20250124Param,
-    BetaTextBlockParam,
-    BetaImageBlockParam,
-    BetaCacheControlEphemeralParam,
-)
-
-from hud.adapters import Adapter
-from hud.agent.base import Agent
-from hud.adapters.claude import ClaudeAdapter
-from hud.types import Gym
-from hud.utils.common import Observation
-from hud.settings import settings
-from hud.adapters.common.types import LogType
-
-logger = logging.getLogger(__name__)
-
-
-def base64_to_content_block(base64: str) -> BetaImageBlockParam:
-    return {
-        "type": "image",
-        "source": {"type": "base64", "media_type": "image/png", "data": base64},
-    }
-
-
-def text_to_content_block(text: str) -> BetaTextBlockParam:
-    return {"type": "text", "text": text}
-
-
-def tool_use_content_block(
-    tool_use_id: str, content: list[BetaTextBlockParam | BetaImageBlockParam]
-) -> BetaToolResultBlockParam:
-    return {"type": "tool_result", "tool_use_id": tool_use_id, "content": content}
-
-
-# Claude's Computer Use Tool definition
-COMPUTER_TOOL: BetaToolComputerUse20250124Param = {
-    "type": "computer_20250124",
-    "name": "computer",
-    "display_width_px": 1024,
-    "display_height_px": 768,
-}
-
-
-class ClaudeAgent(Agent[AsyncAnthropic, Any]):
-    """
-    An agent implementation using Anthropic's Claude API with Computer Use.
-
-    This agent interacts with HUD environments using Claude's Computer Use API
-    through the ClaudeAdapter which converts actions to the format expected by HUD.
-    """
-
-    transfer_gyms: dict[Gym, Gym] = {"qa": "hud-browser"}
-
-    def __init__(
-        self,
-        client: AsyncAnthropic | None = None,
-        adapter: Adapter | None = None,
-        model: str = "claude-3-7-sonnet-20250219",
-        max_tokens: int = 4096,
-        max_iterations: int = 10,
-        name: str | None = None,
-    ):
-        """
-        Initialize the ClaudeAgent.
-
-        Args:
-            client: The AsyncAnthropic client for API calls (optional, created automatically if not provided)
-            adapter: The adapter to use for preprocessing and postprocessing
-            model: The Claude model to use
-            max_tokens: Maximum tokens for Claude's response
-            max_iterations: Maximum number of iterations for the agent
-            name: The name of the agent
-        """
-        # Initialize client if not provided
-        if client is None:
-            # Get API key from settings
-            api_key = settings.anthropic_api_key
-            if not api_key:
-                raise ValueError(
-                    "Anthropic API key not found in settings or environment variables. Set ANTHROPIC_API_KEY."
-                )
-
-            # Create client
-            client = AsyncAnthropic(api_key=api_key)
-
-        adapter = adapter or ClaudeAdapter()
-
-        if name is None:
-            name = model
-
-        super().__init__(client=client, adapter=adapter, name=name)
-
-        self.model = model
-        self.max_tokens = max_tokens
-        self.max_iterations = max_iterations
-
-        # Default dimensions - will be updated if adapter is provided
-        self.width_px = 1024
-        self.height_px = 768
-
-        # Update dimensions if adapter is provided
-        if self.adapter:
-            self.width_px = self.adapter.agent_width
-            self.height_px = self.adapter.agent_height
-
-        # Message history
-        self.messages: list[BetaMessageParam] = []
-        self.pending_computer_use_tool_id = None
-
-    async def fetch_response(self, observation: Observation) -> tuple[list[Any], bool]:
-        """
-        Fetch a response from Claude based on the observation.
-
-        Args:
-            observation: The preprocessed observation
-
-        Returns:
-            tuple[list[Any], bool, list[str | dict[str, Any]] | None]: A tuple containing the list of raw actions,
-                                   boolean indicating if the agent believes the task is complete, and a list of strings or dictionaries of logs.
-        """
-        if not self.client:
-            raise ValueError("Client is required")
-
-        if not observation.text and not observation.screenshot:
-            raise ValueError("Observation must contain either text or screenshot")
-
-        # Prepare the user content for Claude
-        user_content: list[BetaImageBlockParam | BetaTextBlockParam | BetaToolResultBlockParam] = []
-
-        # Add text instruction if present
-        if observation.text:
-            # logger.info("Adding text to user content: %s", observation.text)
-            user_content.append(text_to_content_block(str(observation.text)))
-
-        # Add screenshot if present
-        if observation.screenshot:
-            # logger.info("Adding screenshot to user content")
-            if not self.pending_computer_use_tool_id:
-                # logger.info("Adding screenshot to user content, no tool id")
-                user_content.append(base64_to_content_block(observation.screenshot))
-            else:
-                # logger.info(
-                #    "Adding screenshot to user content, tool id: %s",
-                #    self.pending_computer_use_tool_id,
-                # )
-                user_content.append(
-                    tool_use_content_block(
-                        self.pending_computer_use_tool_id,
-                        [base64_to_content_block(observation.screenshot)],
-                    )
-                )
-                self.pending_computer_use_tool_id = None
-
-        # Add the user content to the messages
-        self.messages.append(
-            cast(
-                BetaMessageParam,
-                {
-                    "role": "user",
-                    "content": user_content,
-                },
-            )
-        )
-
-        # Call Claude API using async client, truncating 50 messages at a time if needed
-        while True:
-            # first, make a copy and add prompt caching to the last message
-            messages_cached = copy.deepcopy(self.messages)
-            # Mark last user message with cache control for prompt caching
-            last_msg = messages_cached[-1]
-            if last_msg.get("role") == "user":
-                last_content = last_msg["content"]
-                if isinstance(last_content, list):
-                    for block in last_content:
-                        if (
-                            not block["type"] == "thinking"
-                            and not block["type"] == "redacted_thinking"
-                        ):
-                            cache_control: BetaCacheControlEphemeralParam = {"type": "ephemeral"}
-                            block["cache_control"] = cache_control
-
-            try:
-                response = await self.client.beta.messages.create(
-                    model=self.model,
-                    max_tokens=self.max_tokens,
-                    messages=messages_cached,
-                    tools=[COMPUTER_TOOL],
-                    betas=["computer-use-2025-01-24"],
-                    tool_choice={"type": "auto", "disable_parallel_tool_use": True},
-                )
-            except BadRequestError as e:
-                if e.message.startswith("prompt is too long"):
-                    logger.warning(
-                        f"Prompt is too long, removing the first 50 messages except for the first user message: {e.message}"
-                    )
-                    self.messages = [self.messages[0]] + self.messages[50:]
-                    continue
-                else:
-                    raise e
-
-            # break out of the while loop if we get a response
-            break
-
-        # Add Claude's response to the conversation history
-        response_content = response.content
-        self.messages.append(
-            cast(
-                BetaMessageParam,
-                {
-                    "role": "assistant",
-                    "content": response_content,
-                },
-            )
-        )
-
-        # Process tool use
-        actions: list[Any] = []
-        done = True  # Assume we're done unless we find a tool use
-
-        for block in response_content:
-            # logger.info("Processing block: %s", block)
-            if block.type == "tool_use":
-                # logger.info("Processing tool use: %s", block)
-                assert block.name == "computer"
-
-                # Store the raw action
-                actions.append(block.input)
-                self.pending_computer_use_tool_id = block.id
-
-                # If we found a tool use, we're not done
-                done = False
-                break
-
-        # If no tool use action was found, check for a final text response
-        if len(actions) == 0 and done:
-            final_text_response = ""
-            for block in response_content:
-                if block.type == "text":
-                    final_text_response += block.text
-
-            if final_text_response.strip():
-                # logger.info(
-                #    f"No tool use found. Using final text as response: {final_text_response}"
-                # )
-                actions = [{"action": "response", "text": final_text_response.strip()}]
-                done = True
-            # else:
-            # logger.info("No tool use and no final text block found.")
-            # Keep done = True, actions remains empty
-
-        reasoning = ""
-        for block in response_content:
-            if block.type == "thinking":
-                reasoning += f"Thinking: {block.thinking}\n"
-            elif block.type == "text":
-                reasoning += block.text
-
-        # add reasoning to the actions
-        for action in actions:
-            action["reasoning"] = reasoning
-            action["logs"] = response.model_dump()
-
-        return actions, done

hud/agent/langchain.py

@@ -1,215 +0,0 @@
-import logging
-from typing import Any, Generic, List, Optional, TypeVar, Union, cast
-
-# Langchain imports
-from langchain_core.language_models import BaseLanguageModel
-from langchain_core.messages import AIMessage, BaseMessage, HumanMessage, SystemMessage
-from langchain_core.runnables import Runnable, RunnableSerializable
-from pydantic import Field, BaseModel
-
-# HUD imports
-from hud.adapters import Adapter
-from hud.agent.base import Agent
-from hud.types import Gym
-from hud.utils.common import Observation
-from hud.adapters.common.types import (
-    ClickAction,
-    TypeAction,
-    ScrollAction,
-    MoveAction,
-    DragAction,
-    PressAction,
-    KeyDownAction,
-    KeyUpAction,
-    WaitAction,
-    ResponseAction,
-    CustomAction,
-    LogType,
-    # Exclude ScreenshotFetch, PositionFetch as they are internal
-)
-
-logger = logging.getLogger(__name__)
-
-# Define a Pydantic Union type representing exactly ONE possible CLA action
-# This is what we'll ask the Langchain model to output.
-SingleCLAction = Union[
-    ClickAction,
-    TypeAction,
-    ScrollAction,
-    MoveAction,
-    DragAction,
-    PressAction,
-    KeyDownAction,
-    KeyUpAction,
-    WaitAction,
-    ResponseAction,
-]
-
-
-# Define a Pydantic model to wrap the single action, potentially making it
-# easier for the LLM to consistently output the desired structure.
-class StepAction(BaseModel):
-    """Wrapper model requesting a single concrete CLA action from the Langchain model."""
-
-    action: SingleCLAction = Field(
-        ..., description="The single CLA action to perform for this step."
-    )
-
-
-# Generic Type for the Langchain Model/Runnable
-# Allows flexibility in what the user provides (model, chain, etc.)
-# Bound to BaseLanguageModel as .with_structured_output is expected
-LangchainModelOrRunnable = TypeVar("LangchainModelOrRunnable", bound=BaseLanguageModel)
-
-
-class LangchainAgent(Agent[LangchainModelOrRunnable, Any], Generic[LangchainModelOrRunnable]):
-    """
-    An agent that uses an arbitrary Langchain model or runnable, leveraging
-    Langchain's structured output capabilities to produce a single CLA action per step.
-    """
-
-    transfer_gyms: dict[Gym, Gym] = {"qa": "hud-browser"}
-
-    def __init__(
-        self,
-        langchain_model: LangchainModelOrRunnable,
-        adapter: Optional[Adapter] = None,
-        system_prompt: str | None = None,
-        name: str | None = None,
-    ):
-        """
-        Initialize the LangchainAgent.
-
-        Args:
-            langchain_model: The Langchain language model or runnable chain to use.
-                             Must support asynchronous invocation (`ainvoke`) and
-                             `.with_structured_output()`.
-            adapter: An optional HUD adapter. If provided, it will be used for
-                     preprocessing observations (rescaling) and postprocessing
-                     the single CLA action (coordinate rescaling).
-            system_prompt: An optional system prompt to guide the Langchain model.
-                           If None, a default prompt encouraging single CLA output is used.
-        """
-        super().__init__(
-            client=langchain_model, adapter=adapter, name=name
-        )  # Store model as 'client'
-        self.langchain_model = langchain_model  # Also store with specific name
-
-        self.system_prompt_str = system_prompt or self._get_default_system_prompt()
-        self.history: List[BaseMessage] = []
-
-    def _get_default_system_prompt(self) -> str:
-        # TODO: Refine this prompt based on testing.
-        # It needs to strongly encourage outputting *only* the StepAction structure.
-        return (
-            "You are an agent interacting with a computer environment (either a web browser or an OS desktop). "
-            "Your goal is to follow the user's instructions based on the provided text and screenshot observations."
-            "For each step, you must choose exactly ONE action to perform from the available CLA action types."
-            "Output your chosen action using the provided 'StepAction' tool/function."
-            "If you believe the task is complete based on the user's prompt and the observations, use the 'ResponseAction'."
-        )
-
-    async def fetch_response(
-        self, observation: Observation
-    ) -> tuple[list[dict | SingleCLAction], bool]:
-        """
-        Fetches a response from the configured Langchain model, expecting a single
-        structured CLA action.
-
-        Args:
-            observation: The preprocessed observation (screenshot potentially rescaled by adapter).
-
-        Returns:
-            A tuple containing:
-            - A list with a single dictionary representing the raw CLA action (before adapter postprocessing).
-            - A boolean indicating if the agent chose ResponseAction (task completion).
-        """
-        # 1. Format observation into Langchain message(s)
-        human_content: List[Union[str, dict]] = []
-        if observation.text:
-            human_content.append(observation.text)
-        if observation.screenshot:
-            # Assuming the Langchain model/chain can handle base64 images
-            # This might need adjustment based on the specific model used.
-            human_content.append(
-                {
-                    "type": "image_url",
-                    "image_url": {"url": f"data:image/png;base64,{observation.screenshot}"},
-                }
-            )
-
-        if not human_content:
-            logger.warning("LangchainAgent received an observation with no text or screenshot.")
-            # Decide how to handle empty observation - perhaps return no action?
-            return [], False
-
-        current_human_message = HumanMessage(content=human_content)
-
-        # 2. Prepare message history for the model
-        messages_for_llm: List[BaseMessage] = [
-            SystemMessage(content=self.system_prompt_str),
-            *self.history,
-            current_human_message,
-        ]
-
-        # 3. Configure structured output
-        # We ask for the StepAction wrapper, which contains the actual SingleCLAAction
-        # Explicitly use method="function_calling" to handle schemas with default values
-        structured_llm = self.langchain_model.with_structured_output(
-            schema=StepAction, method="function_calling"
-        )
-
-        # 4. Invoke Langchain model asynchronously
-        try:
-            ai_response_structured = await structured_llm.ainvoke(messages_for_llm)
-        except Exception as e:
-            logger.error(f"Langchain model invocation failed: {e}", exc_info=True)
-            # Decide how to handle LLM errors - maybe retry or return empty action?
-            return [], False
-
-        # 5. Process the structured response
-        is_done = False
-        ai_message_content_for_history = ""  # For storing in history
-
-        if isinstance(ai_response_structured, StepAction):
-            # Successfully got the wrapper, extract the actual action
-            actual_action = ai_response_structured.action
-            ai_message_content_for_history = actual_action.model_dump()
-            if isinstance(actual_action, ResponseAction):
-                is_done = True
-                # logger.info(
-                #     f"LangchainAgent determined task is done with response: {actual_action.text[:100]}..."
-                # )
-            # else:
-            #     logger.info(f"LangchainAgent produced action: {type(actual_action).__name__}")
-
-        else:
-            logger.warning(
-                f"Langchain model did not return the expected StepAction structure. "
-                f"Received type: {type(ai_response_structured)}. Value: {ai_response_structured!r}"
-            )
-            # Attempt to add raw response to history for debugging
-            if isinstance(ai_response_structured, BaseMessage):
-                ai_message_content_for_history = ai_response_structured.content
-            elif isinstance(ai_response_structured, str):
-                ai_message_content_for_history = ai_response_structured
-            else:
-                ai_message_content_for_history = repr(ai_response_structured)
-            # Return no action as we didn't get the expected structure
-            return [], False
-
-        # 6. Update history
-        self.history.append(current_human_message)
-        # Add the AI response (containing the structured action dict) to history
-        # Convert dict to string representation for AIMessage content
-        self.history.append(AIMessage(content=repr(ai_message_content_for_history)))
-        # TODO: Consider history truncation/summarization if it grows too long
-
-        if actual_action:
-            actual_action = actual_action.model_dump()
-            # Return the single action dictionary within a list
-            actual_action["logs"] = ai_message_content_for_history
-            return [actual_action], is_done
-        else:
-            # Should ideally not happen if structure validation worked, but as a fallback
-            return [], is_done

hud/agent/misc/__init__.py

@@ -1,3 +0,0 @@
-from .response_agent import ResponseAgent
-
-__all__ = ["ResponseAgent"]