hud-python 0.2.10__py3-none-any.whl → 0.3.1__py3-none-any.whl

hud/tools/__init__.py

@@ -0,0 +1,34 @@
+"""HUD tools for computer control, file editing, and bash commands."""
+
+from __future__ import annotations
+
+from typing import TYPE_CHECKING, Any
+
+from .base import ToolError, ToolResult, tool_result_to_content_blocks
+from .bash import BashTool
+from .edit import EditTool
+from .playwright_tool import PlaywrightTool
+
+if TYPE_CHECKING:
+    from .computer import AnthropicComputerTool, HudComputerTool, OpenAIComputerTool
+
+__all__ = [
+    "AnthropicComputerTool",
+    "BashTool",
+    "EditTool",
+    "HudComputerTool",
+    "OpenAIComputerTool",
+    "PlaywrightTool",
+    "ToolError",
+    "ToolResult",
+    "tool_result_to_content_blocks",
+]
+
+
+def __getattr__(name: str) -> Any:
+    """Lazy import computer tools to avoid importing pyautogui unless needed."""
+    if name in ("AnthropicComputerTool", "HudComputerTool", "OpenAIComputerTool"):
+        from . import computer
+
+        return getattr(computer, name)
+    raise AttributeError(f"module '{__name__}' has no attribute '{name}'")

hud/tools/base.py

@@ -0,0 +1,65 @@
+from __future__ import annotations
+
+from dataclasses import dataclass, fields, replace
+from typing import Any
+
+from mcp.types import ImageContent, TextContent
+
+
+@dataclass(kw_only=True, frozen=True)
+class ToolResult:
+    """Represents the result of a tool execution."""
+
+    output: str | None = None
+    error: str | None = None
+    base64_image: str | None = None
+    system: str | None = None
+
+    def __bool__(self) -> bool:
+        return any(getattr(self, field.name) for field in fields(self))
+
+    def __add__(self, other: ToolResult) -> ToolResult:
+        def combine_fields(
+            field: str | None, other_field: str | None, concatenate: bool = True
+        ) -> str | None:
+            if field and other_field:
+                if concatenate:
+                    return field + other_field
+                raise ValueError("Cannot combine tool results")
+            return field or other_field
+
+        return ToolResult(
+            output=combine_fields(self.output, other.output),
+            error=combine_fields(self.error, other.error),
+            base64_image=combine_fields(self.base64_image, other.base64_image, False),
+            system=combine_fields(self.system, other.system),
+        )
+
+    def replace(self, **kwargs: Any) -> ToolResult:
+        """Returns a new ToolResult with the given fields replaced."""
+        return replace(self, **kwargs)
+
+
+# Legacy alias for backward compatibility
+CLIResult = ToolResult
+
+
+class ToolError(Exception):
+    """An error raised by a tool."""
+
+
+# Legacy alias for backward compatibility
+CLIError = ToolError
+
+
+def tool_result_to_content_blocks(result: ToolResult) -> list[ImageContent | TextContent]:
+    """Convert a ToolResult to MCP content blocks."""
+    blocks = []
+
+    if result.output:
+        blocks.append(TextContent(text=result.output, type="text"))
+    if result.error:
+        blocks.append(TextContent(text=result.error, type="text"))
+    if result.base64_image:
+        blocks.append(ImageContent(data=result.base64_image, mimeType="image/png", type="image"))
+    return blocks

hud/tools/bash.py

@@ -0,0 +1,137 @@
+from __future__ import annotations
+
+import asyncio
+import os
+import sys
+from typing import Any
+
+from .base import CLIResult, ToolError, ToolResult
+
+
+class _BashSession:
+    """A session of a bash shell."""
+
+    _started: bool
+    _process: asyncio.subprocess.Process
+
+    command: str = "/bin/bash"
+    _output_delay: float = 0.2  # seconds
+    _timeout: float = 120.0  # seconds
+    _sentinel: str = "<<exit>>"
+
+    def __init__(self) -> None:
+        self._started = False
+        self._timed_out = False
+
+    async def start(self) -> None:
+        if self._started:
+            await asyncio.sleep(0)
+            return
+
+        # Platform-specific subprocess creation
+        kwargs = {
+            "shell": True,
+            "bufsize": 0,
+            "stdin": asyncio.subprocess.PIPE,
+            "stdout": asyncio.subprocess.PIPE,
+            "stderr": asyncio.subprocess.PIPE,
+        }
+
+        # Only use setsid on Unix-like systems
+        if sys.platform != "win32":
+            kwargs["preexec_fn"] = os.setsid
+
+        self._process = await asyncio.create_subprocess_shell(self.command, **kwargs)
+
+        self._started = True
+
+    def stop(self) -> None:
+        """Terminate the bash shell."""
+        if not self._started:
+            raise ToolError("Session has not started.")
+        if self._process.returncode is not None:
+            return
+        self._process.terminate()
+
+    async def run(self, command: str) -> CLIResult:
+        """Execute a command in the bash shell."""
+        if not self._started:
+            raise ToolError("Session has not started.")
+        if self._process.returncode is not None:
+            await asyncio.sleep(0)
+            return ToolResult(
+                system="tool must be restarted",
+                error=f"bash has exited with returncode {self._process.returncode}",
+            )
+        if self._timed_out:
+            raise ToolError(
+                f"timed out: bash did not return in {self._timeout} seconds and must be restarted",
+            ) from None
+
+        if self._process.stdin is None:
+            raise ToolError("stdin is None")
+        if self._process.stdout is None:
+            raise ToolError("stdout is None")
+        if self._process.stderr is None:
+            raise ToolError("stderr is None")
+
+        # Send command to the process
+        self._process.stdin.write(command.encode() + f"; echo '{self._sentinel}'\n".encode())
+        await self._process.stdin.drain()
+
+        # Read output from the process, until the sentinel is found
+        sentinel_line = f"{self._sentinel}\n"
+        sentinel_bytes = sentinel_line.encode()
+
+        try:
+            raw_out: bytes = await asyncio.wait_for(
+                self._process.stdout.readuntil(sentinel_bytes),
+                timeout=self._timeout,
+            )
+            output = raw_out.decode()[: -len(sentinel_line)]
+        except (TimeoutError, asyncio.LimitOverrunError):
+            self._timed_out = True
+            raise ToolError(
+                f"timed out: bash did not return in {self._timeout} seconds and must be restarted",
+            ) from None
+
+        # Attempt non-blocking stderr fetch (may return empty)
+        try:
+            error_bytes = await asyncio.wait_for(self._process.stderr.read(), timeout=0.01)
+            error = error_bytes.decode().rstrip("\n")
+        except TimeoutError:
+            error = ""
+
+        return CLIResult(output=output, error=error)
+
+
+class BashTool:
+    """
+    A tool that allows the agent to run bash commands.
+    The tool parameters are defined by Anthropic and are not editable.
+    """
+
+    _session: _BashSession | None
+
+    def __init__(self) -> None:
+        self._session = None
+
+    async def __call__(
+        self, command: str | None = None, restart: bool = False, **kwargs: Any
+    ) -> ToolResult:
+        if restart:
+            if self._session:
+                self._session.stop()
+            self._session = _BashSession()
+            await self._session.start()
+
+            return ToolResult(system="tool has been restarted.")
+
+        if self._session is None:
+            self._session = _BashSession()
+            await self._session.start()
+
+        if command is not None:
+            return await self._session.run(command)
+
+        raise ToolError("no command provided.")

hud/tools/computer/__init__.py

@@ -0,0 +1,13 @@
+"""Computer control tools for different agent APIs."""
+
+from __future__ import annotations
+
+from .anthropic import AnthropicComputerTool
+from .hud import HudComputerTool
+from .openai import OpenAIComputerTool
+
+__all__ = [
+    "AnthropicComputerTool",
+    "HudComputerTool",
+    "OpenAIComputerTool",
+]

hud/tools/computer/anthropic.py

@@ -0,0 +1,411 @@
+# flake8: noqa: B008
+from __future__ import annotations
+
+import logging
+from typing import TYPE_CHECKING, Literal, cast
+
+from mcp import ErrorData, McpError
+from mcp.types import INTERNAL_ERROR, INVALID_PARAMS, ImageContent, TextContent
+from pydantic import Field
+
+from hud.tools.base import ToolResult, tool_result_to_content_blocks
+
+from .hud import HudComputerTool
+
+if TYPE_CHECKING:
+    from anthropic.types.beta import BetaToolComputerUse20250124Param
+
+logger = logging.getLogger(__name__)
+
+# Map Anthropic key names to CLA standard keys
+ANTHROPIC_TO_CLA_KEYS = {
+    # Common variations
+    "Return": "enter",
+    "Escape": "escape",
+    "ArrowUp": "up",
+    "ArrowDown": "down",
+    "ArrowLeft": "left",
+    "ArrowRight": "right",
+    "Backspace": "backspace",
+    "Delete": "delete",
+    "Tab": "tab",
+    "Space": "space",
+    "Control": "ctrl",
+    "Alt": "alt",
+    "Shift": "shift",
+    "Meta": "win",  # Windows key
+    "Command": "cmd",  # macOS
+    "Super": "win",  # Linux
+    "PageUp": "pageup",
+    "PageDown": "pagedown",
+    "Home": "home",
+    "End": "end",
+    "Insert": "insert",
+    "F1": "f1",
+    "F2": "f2",
+    "F3": "f3",
+    "F4": "f4",
+    "F5": "f5",
+    "F6": "f6",
+    "F7": "f7",
+    "F8": "f8",
+    "F9": "f9",
+    "F10": "f10",
+    "F11": "f11",
+    "F12": "f12",
+}
+
+
+class AnthropicComputerTool(HudComputerTool):
+    """
+    Anthropic Computer Use tool for interacting with the computer.
+    """
+
+    name: str = "computer"
+    api_type: str = "computer_20250124"
+
+    def __init__(
+        self,
+        width: int = 1400,
+        height: int = 850,
+        display_num: int | None = None,
+        platform_type: Literal["auto", "xdo", "pyautogui"] = "auto",
+        rescale_images: bool = False,
+    ) -> None:
+        """
+        Initialize with Anthropic's default dimensions.
+
+        Args:
+            width: Target width for rescaling (default: 1024 for Anthropic)
+            height: Target height for rescaling (default: 768 for Anthropic)
+            display_num: X display number
+            platform_type: Which executor to use:
+                - "auto": Automatically detect based on platform
+                - "xdo": Use XDOExecutor (Linux/X11 only)
+                - "pyautogui": Use PyAutoGUIExecutor (cross-platform)
+            rescale_images: If True, rescale screenshots. If False, only rescale action coordinates
+        """
+        super().__init__(
+            width=width,
+            height=height,
+            display_num=display_num,
+            platform_type=platform_type,
+            rescale_images=rescale_images,
+        )
+
+    def to_params(self) -> BetaToolComputerUse20250124Param:
+        """Convert to Anthropic tool parameters."""
+        return cast(
+            "BetaToolComputerUse20250124Param",
+            {
+                "type": self.api_type,
+                "name": self.name,
+                "display_width_px": self.width,
+                "display_height_px": self.height,
+            },
+        )
+
+    def _map_anthropic_key_to_cla(self, key: str) -> str:
+        """Map Anthropic key name to CLA standard key."""
+        # Handle key combinations like "ctrl+a"
+        if "+" in key:
+            parts = key.split("+")
+            mapped_parts = []
+            for part in parts:
+                # Try exact match first, then case-insensitive
+                mapped = ANTHROPIC_TO_CLA_KEYS.get(
+                    part, ANTHROPIC_TO_CLA_KEYS.get(part.capitalize(), part.lower())
+                )
+                mapped_parts.append(mapped)
+            return "+".join(mapped_parts)
+        else:
+            # Single key - try exact match first, then case-insensitive
+            return ANTHROPIC_TO_CLA_KEYS.get(
+                key, ANTHROPIC_TO_CLA_KEYS.get(key.capitalize(), key.lower())
+            )
+
+    async def __call__(
+        self,
+        action: str = Field(..., description="The action to perform on the computer"),
+        coordinate: list[int] | tuple[int, int] | None = Field(
+            None, description="The coordinate to interact with on the computer [x, y]"
+        ),
+        text: str | None = Field(
+            None, description="The text to type on the computer or key to press"
+        ),
+        start_coordinate: list[int] | tuple[int, int] | None = Field(
+            None, description="The starting coordinate for drag actions [x, y]"
+        ),
+        scroll_direction: str | None = Field(
+            None, description="The direction to scroll (up, down, left, right)"
+        ),
+        scroll_amount: int | None = Field(None, description="The amount to scroll"),
+        duration: float | None = Field(None, description="The duration of the action in seconds"),
+        take_screenshot_on_click: bool = Field(
+            True, description="Whether to take a screenshot after clicking"
+        ),
+    ) -> list[ImageContent | TextContent]:
+        """
+        Handle Anthropic Computer Use API calls.
+
+        This converts Anthropic's action format to HudComputerTool's format.
+
+        Returns:
+            List of MCP content blocks
+        """
+        logger.info("AnthropicComputerTool received action: %s", action)
+
+        # Convert lists to tuples if needed
+        coord_tuple = None
+        if coordinate:
+            coord_tuple = tuple(coordinate) if isinstance(coordinate, list) else coordinate
+
+        start_coord_tuple = None
+        if start_coordinate:
+            start_coord_tuple = (
+                tuple(start_coordinate) if isinstance(start_coordinate, list) else start_coordinate
+            )
+
+        # Map Anthropic actions to HudComputerTool actions
+        if action == "screenshot":
+            screenshot_base64 = await self.executor.screenshot()
+            if screenshot_base64:
+                # Rescale screenshot if requested
+                screenshot_base64 = await self._rescale_screenshot(screenshot_base64)
+                result = ToolResult(base64_image=screenshot_base64)
+            else:
+                result = ToolResult(error="Failed to take screenshot")
+
+        elif action == "left_click" or action == "click":
+            if coord_tuple and len(coord_tuple) >= 2:
+                scaled_x, scaled_y = self._scale_coordinates(coord_tuple[0], coord_tuple[1])
+                logger.info("Scaled coordinates: %s, %s", scaled_x, scaled_y)
+                result = await self.executor.click(x=scaled_x, y=scaled_y)
+            else:
+                result = await self.executor.click()
+
+        elif action == "double_click":
+            if coord_tuple and len(coord_tuple) >= 2:
+                # Use pattern for double-click
+                scaled_x, scaled_y = self._scale_coordinates(coord_tuple[0], coord_tuple[1])
+                result = await self.executor.click(x=scaled_x, y=scaled_y, pattern=[100])
+            else:
+                result = await self.executor.click(pattern=[100])
+
+        elif action == "triple_click":
+            if coord_tuple and len(coord_tuple) >= 2:
+                # Use pattern for triple-click
+                scaled_x, scaled_y = self._scale_coordinates(coord_tuple[0], coord_tuple[1])
+                result = await self.executor.click(x=scaled_x, y=scaled_y, pattern=[100, 100])
+            else:
+                result = await self.executor.click(pattern=[100, 100])
+
+        elif action == "right_click":
+            if coord_tuple and len(coord_tuple) >= 2:
+                scaled_x, scaled_y = self._scale_coordinates(coord_tuple[0], coord_tuple[1])
+                result = await self.executor.click(x=scaled_x, y=scaled_y, button="right")
+            else:
+                result = await self.executor.click(button="right")
+
+        elif action == "middle_click":
+            if coord_tuple and len(coord_tuple) >= 2:
+                scaled_x, scaled_y = self._scale_coordinates(coord_tuple[0], coord_tuple[1])
+                result = await self.executor.click(x=scaled_x, y=scaled_y, button="middle")
+            else:
+                result = await self.executor.click(button="middle")
+
+        elif action == "mouse_move" or action == "move":
+            if coord_tuple and len(coord_tuple) >= 2:
+                scaled_x, scaled_y = self._scale_coordinates(coord_tuple[0], coord_tuple[1])
+                result = await self.executor.move(x=scaled_x, y=scaled_y)
+            else:
+                raise McpError(
+                    ErrorData(code=INVALID_PARAMS, message="coordinate is required for mouse_move")
+                )
+
+        elif action == "type":
+            if text:
+                result = await self.executor.type(text=text)
+            else:
+                raise McpError(ErrorData(code=INVALID_PARAMS, message="text is required for type"))
+
+        elif action == "key":
+            if text:
+                # Anthropic sends single key or combo like "ctrl+a"
+                # Map to CLA standard key format
+                mapped_key = self._map_anthropic_key_to_cla(text)
+                result = await self.executor.press(keys=[mapped_key])
+            else:
+                raise McpError(ErrorData(code=INVALID_PARAMS, message="text is required for key"))
+
+        elif action == "scroll":
+            # Original implementation validates scroll_direction and scroll_amount
+            if scroll_direction not in ["up", "down", "left", "right"]:
+                raise McpError(
+                    ErrorData(
+                        code=INVALID_PARAMS,
+                        message="scroll_direction must be 'up', 'down', 'left', or 'right'",
+                    )
+                )
+
+            if scroll_amount is None or scroll_amount < 0:
+                raise McpError(
+                    ErrorData(
+                        code=INVALID_PARAMS, message="scroll_amount must be a non-negative int"
+                    )
+                )
+
+            # Convert direction to scroll amounts
+            scroll_x = None
+            scroll_y = None
+            if scroll_direction == "down":
+                scroll_y = scroll_amount
+            elif scroll_direction == "up":
+                scroll_y = -scroll_amount
+            elif scroll_direction == "right":
+                scroll_x = scroll_amount
+            elif scroll_direction == "left":
+                scroll_x = -scroll_amount
+
+            if coord_tuple and len(coord_tuple) >= 2:
+                scaled_x, scaled_y = self._scale_coordinates(coord_tuple[0], coord_tuple[1])
+                result = await self.executor.scroll(
+                    x=scaled_x, y=scaled_y, scroll_x=scroll_x, scroll_y=scroll_y
+                )
+            else:
+                result = await self.executor.scroll(scroll_x=scroll_x, scroll_y=scroll_y)
+
+        elif action == "left_click_drag" or action == "drag":
+            # Anthropic sends drag with start and end coordinates
+            if coord_tuple and len(coord_tuple) >= 2:
+                if start_coord_tuple and len(start_coord_tuple) >= 2:
+                    # Full drag path
+                    path = [
+                        (start_coord_tuple[0], start_coord_tuple[1]),
+                        (coord_tuple[0], coord_tuple[1]),
+                    ]
+                    scaled_path = self._scale_path(path)
+                    result = await self.executor.drag(path=scaled_path)
+                else:
+                    # Just end coordinate, drag from current position
+                    # Original spec allows this
+                    current_pos = [(0, 0), (coord_tuple[0], coord_tuple[1])]  # Simplified
+                    scaled_path = self._scale_path(current_pos)
+                    result = await self.executor.drag(path=scaled_path)
+            else:
+                raise McpError(
+                    ErrorData(
+                        code=INVALID_PARAMS, message="coordinate is required for left_click_drag"
+                    )
+                )
+
+        elif action == "wait":
+            # Original spec expects duration in seconds
+            if duration is None:
+                raise McpError(
+                    ErrorData(code=INVALID_PARAMS, message="duration is required for wait")
+                )
+            if duration < 0:
+                raise McpError(
+                    ErrorData(code=INVALID_PARAMS, message="duration must be non-negative")
+                )
+            if duration > 100:
+                raise McpError(ErrorData(code=INVALID_PARAMS, message="duration is too long"))
+
+            # Convert seconds to milliseconds for HudComputerTool
+            result = await self.executor.wait(time=int(duration * 1000))
+
+        elif action == "hold_key":
+            # Original spec has hold_key action
+            if text is None:
+                raise McpError(
+                    ErrorData(code=INVALID_PARAMS, message="text is required for hold_key")
+                )
+            if duration is None:
+                raise McpError(
+                    ErrorData(code=INVALID_PARAMS, message="duration is required for hold_key")
+                )
+            if duration < 0:
+                raise McpError(
+                    ErrorData(code=INVALID_PARAMS, message="duration must be non-negative")
+                )
+            if duration > 100:
+                raise McpError(ErrorData(code=INVALID_PARAMS, message="duration is too long"))
+
+            # Hold key action
+            result = await self.executor.hold_key(key=text, duration=duration)
+
+        elif action == "left_mouse_down":
+            # These don't accept coordinates in original spec
+            if coord_tuple is not None:
+                raise McpError(
+                    ErrorData(
+                        code=INVALID_PARAMS,
+                        message="coordinate is not accepted for left_mouse_down",
+                    )
+                )
+            # Use generic mouse_down method
+            result = await self.executor.mouse_down(button="left")
+
+        elif action == "left_mouse_up":
+            # These don't accept coordinates in original spec
+            if coord_tuple is not None:
+                raise McpError(
+                    ErrorData(
+                        code=INVALID_PARAMS, message="coordinate is not accepted for left_mouse_up"
+                    )
+                )
+            # Use generic mouse_up method
+            result = await self.executor.mouse_up(button="left")
+
+        elif action == "cursor_position":
+            result = await self.executor.position()
+
+        else:
+            # Unknown action
+            raise McpError(ErrorData(code=INTERNAL_ERROR, message=f"Invalid action: {action}"))
+
+        # Rescale screenshot in result if present
+        if isinstance(result, ToolResult) and result.base64_image and self.rescale_images:
+            rescaled_image = await self._rescale_screenshot(result.base64_image)
+            result = result.replace(base64_image=rescaled_image)
+
+        # Handle screenshot for actions that need it
+        screenshot_actions = {
+            "screenshot",
+            "left_click",
+            "click",
+            "double_click",
+            "triple_click",
+            "right_click",
+            "middle_click",
+            "mouse_move",
+            "move",
+            "type",
+            "key",
+            "scroll",
+            "left_click_drag",
+            "drag",
+            "wait",
+            "hold_key",
+            "left_mouse_down",
+            "left_mouse_up",
+        }
+
+        if (
+            action in screenshot_actions
+            and action != "screenshot"
+            and take_screenshot_on_click
+            and isinstance(result, ToolResult)
+            and not result.base64_image
+        ):
+            screenshot_base64 = await self.executor.screenshot()
+            if screenshot_base64:
+                # Rescale screenshot if requested
+                screenshot_base64 = await self._rescale_screenshot(screenshot_base64)
+                result = ToolResult(
+                    output=result.output, error=result.error, base64_image=screenshot_base64
+                )
+
+        # Convert to content blocks
+        return tool_result_to_content_blocks(result)