hud-python 0.4.45__py3-none-any.whl → 0.5.1__py3-none-any.whl

hud/tools/computer/openai.py

@@ -9,7 +9,7 @@ from mcp.types import INTERNAL_ERROR, INVALID_PARAMS, ContentBlock, TextContent
 from pydantic import Field
 
 from hud.tools.computer.settings import computer_settings
-from hud.tools.types import ContentResult
+from hud.tools.types import ContentResult, Coordinate
 
 from .hud import HudComputerTool
 
@@ -18,6 +18,7 @@ if TYPE_CHECKING:
 
 logger = logging.getLogger(__name__)
 
+
 # Map OpenAI key names to CLA standard keys
 OPENAI_TO_CLA_KEYS = {
     # Common variations
@@ -95,14 +96,26 @@ class OpenAIComputerTool(HudComputerTool):
         # OpenAI uses lowercase key names
         return OPENAI_TO_CLA_KEYS.get(key.lower(), key.lower())
 
-    async def __call__(
+    async def __call__(  # type: ignore[override]
         self,
-        type: str = Field(..., description="The action type to perform"),
+        type: Literal[
+            "screenshot",
+            "click",
+            "double_click",
+            "scroll",
+            "type",
+            "wait",
+            "move",
+            "keypress",
+            "drag",
+            "response",
+            "custom",
+        ] = Field(..., description="The action type to perform"),
         # Coordinate parameters
         x: int | None = Field(None, description="X coordinate for click/move/scroll actions"),
         y: int | None = Field(None, description="Y coordinate for click/move/scroll actions"),
         # Button parameter
-        button: str | None = Field(
+        button: Literal["left", "right", "middle", "back", "forward"] | None = Field(
             None, description="Mouse button for click actions (left, right, middle, wheel)"
         ),
         # Text parameter
@@ -115,7 +128,7 @@ class OpenAIComputerTool(HudComputerTool):
         # Key press parameter
         keys: list[str] | None = Field(None, description="Keys to press"),
         # Drag parameter
-        path: list[dict[str, int]] | None = Field(
+        path: list[Coordinate] | None = Field(
             None, description="Path for drag actions as list of {x, y} dicts"
         ),
         # Custom action parameter
@@ -131,11 +144,6 @@ class OpenAIComputerTool(HudComputerTool):
         """
         logger.info("OpenAIComputerTool received type: %s", type)
 
-        # Map button names
-        button_map = {"wheel": "middle"}
-        if button:
-            button = button_map.get(button, button)
-
         # Process based on action type
         if type == "screenshot":
             screenshot_base64 = await self.executor.screenshot()
@@ -227,17 +235,8 @@ class OpenAIComputerTool(HudComputerTool):
                     )
                 )
 
-            # Convert path from list of dicts to list of tuples
-            drag_path = []
-            for point in path:
-                if "x" in point and "y" in point:
-                    drag_path.append((point["x"], point["y"]))
-                else:
-                    raise McpError(
-                        ErrorData(
-                            code=INVALID_PARAMS, message="Each point in path must have x and y"
-                        )
-                    )
+            # Convert path from list of Coordinate objects to list of tuples
+            drag_path = [(point.x, point.y) for point in path]
 
             scaled_path = self._scale_path(drag_path)
             result = await self.executor.drag(path=scaled_path)

hud/tools/computer/qwen.py

@@ -0,0 +1,434 @@
+# flake8: noqa: B008
+from __future__ import annotations
+
+import logging
+import re
+from typing import TYPE_CHECKING, Any, Literal
+
+from mcp import ErrorData, McpError
+from mcp.types import INTERNAL_ERROR, INVALID_PARAMS, ContentBlock
+from pydantic import Field
+
+from hud.tools.types import ContentResult
+
+from .hud import HudComputerTool
+from .settings import computer_settings
+
+if TYPE_CHECKING:
+    from hud.tools.executors.base import BaseExecutor
+
+logger = logging.getLogger(__name__)
+
+
+class QwenComputerTool(HudComputerTool):
+    """
+    Qwen Computer Use tool for interacting with the computer.
+    """
+
+    name: str = "computer_use"
+    api_type: str = "computer_use"
+
+    def __init__(
+        self,
+        # Define within environment based on platform
+        executor: BaseExecutor | None = None,
+        platform_type: Literal["auto", "xdo", "pyautogui"] = "auto",
+        display_num: int | None = None,
+        # Overrides for what dimensions the agent thinks it operates in
+        width: int = computer_settings.QWEN_COMPUTER_WIDTH,
+        height: int = computer_settings.QWEN_COMPUTER_HEIGHT,
+        rescale_images: bool = computer_settings.QWEN_RESCALE_IMAGES,
+        # What the agent sees as the tool's name, title, and description
+        name: str | None = None,
+        title: str | None = None,
+        description: str | None = None,
+        **kwargs: Any,
+    ) -> None:
+        """
+        Initialize with Qwen's default dimensions.
+
+        Args:
+            width: Target width for rescaling (None = use environment width)
+            height: Target height for rescaling (None = use environment height)
+            rescale_images: If True, rescale screenshots. If False, only rescale action coordinates
+            name: Tool name for MCP registration (auto-generated from class name if not provided)
+            title: Human-readable display name for the tool (auto-generated from class name)
+            description: Tool description (auto-generated from docstring if not provided)
+        """
+        # Store dimensions for description
+        self.display_width_px = width
+        self.display_height_px = height
+
+        # Build custom description with resolution info
+        custom_description = (
+            description
+            or f"""
+Use a mouse and keyboard to interact with a computer, and take screenshots.
+* This is an interface to a desktop GUI. You do not have access to a terminal or
+applications menu. You must click on desktop icons to start applications.
+* Some applications may take time to start or process actions, so you may need to
+wait and take successive screenshots to see the results of your actions. E.g. if you
+click on Firefox and a window doesn't open, try wait and taking another screenshot.
+* The screen's resolution is {width}x{height}.
+* Whenever you intend to move the cursor to click on an element like an icon, you
+should consult a screenshot to determine the coordinates of the element before
+moving the cursor.
+* If you tried clicking on a program or link but it failed to load, even after
+waiting, try adjusting your cursor position so that the tip of the cursor visually
+falls on the element that you want to click.
+* Make sure to click any buttons, links, icons, etc with the cursor tip in the
+center of the element. Don't click boxes on their edges.
+""".strip()
+        )
+
+        super().__init__(
+            executor=executor,
+            platform_type=platform_type,
+            display_num=display_num,
+            width=width,
+            height=height,
+            rescale_images=rescale_images,
+            name=name or "qwen_computer",
+            title=title or "Qwen Computer Tool",
+            description=custom_description,
+            **kwargs,
+        )
+
+    def to_params(self) -> dict:
+        """Convert to Qwen tool parameters."""
+        return {
+            "type": self.api_type,
+            "name": self.name,
+            "display_width_px": self.display_width_px,
+            "display_height_px": self.display_height_px,
+            "description": self.description,
+            "parameters": {
+                "properties": {
+                    "action": {
+                        "description": """
+The action to perform. The available actions are:
+* `key`: Performs key down presses on the arguments passed in order, then performs
+key releases in reverse order.
+* `type`: Type a string of text on the keyboard.
+* `mouse_move`: Move the cursor to a specified (x, y) pixel coordinate on the
+screen.
+* `left_click`: Click the left mouse button at a specified (x, y) pixel coordinate
+on the screen.
+* `left_click_drag`: Click and drag the cursor to a specified (x, y) pixel
+coordinate on the screen.
+* `right_click`: Click the right mouse button at a specified (x, y) pixel
+coordinate on the screen.
+* `middle_click`: Click the middle mouse button at a specified (x, y) pixel
+coordinate on the screen.
+* `double_click`: Double-click the left mouse button at a specified (x, y) pixel
+coordinate on the screen.
+* `triple_click`: Triple-click the left mouse button at a specified (x, y) pixel
+coordinate on the screen.
+* `scroll`: Performs a scroll of the mouse scroll wheel.
+* `hscroll`: Performs a horizontal scroll.
+* `wait`: Wait specified seconds for the change to happen.
+* `terminate`: Terminate the current task and report its completion status
+(NOT SUPPORTED).
+* `answer`: Answer a question (NOT SUPPORTED).
+""".strip(),
+                        "enum": [
+                            "key",
+                            "type",
+                            "mouse_move",
+                            "left_click",
+                            "left_click_drag",
+                            "right_click",
+                            "middle_click",
+                            "double_click",
+                            "triple_click",
+                            "scroll",
+                            "hscroll",
+                            "wait",
+                            "terminate",
+                            "answer",
+                        ],
+                        "type": "string",
+                    },
+                    "keys": {
+                        "description": "Required only by `action=key`.",
+                        "type": "array",
+                    },
+                    "text": {
+                        "description": "Required only by `action=type` and `action=answer`.",
+                        "type": "string",
+                    },
+                    "coordinate": {
+                        "description": (
+                            "(x, y): The x (pixels from the left edge) and y "
+                            "(pixels from the top edge) coordinates to move the mouse to."
+                        ),
+                        "type": "array",
+                    },
+                    "pixels": {
+                        "description": (
+                            "The amount of scrolling to perform. Positive values scroll up, "
+                            "negative values scroll down. Required only by `action=scroll` "
+                            "and `action=hscroll`."
+                        ),
+                        "type": "number",
+                    },
+                    "time": {
+                        "description": "The seconds to wait. Required only by `action=wait`.",
+                        "type": "number",
+                    },
+                    "status": {
+                        "description": (
+                            "The status of the task. Required only by `action=terminate`."
+                        ),
+                        "type": "string",
+                        "enum": ["success", "failure"],
+                    },
+                },
+                "required": ["action"],
+                "type": "object",
+            },
+        }
+
+    async def __call__(
+        self,
+        action: str = Field(..., description="The action to perform on the computer"),
+        keys: list[str] | None = Field(None, description="Keys for key action"),
+        text: str | None = Field(None, description="Text to type"),
+        coordinate: list[int] | None = Field(
+            None, description="The coordinate to interact with on the computer [x, y]"
+        ),
+        pixels: int | None = Field(None, description="Pixels to scroll"),
+        time: float | None = Field(None, description="Time to wait in seconds"),
+        status: str | None = Field(None, description="Status for terminate action"),
+    ) -> list[ContentBlock]:
+        """
+        Handle Qwen Computer Use API calls.
+
+        This converts Qwen's action format to HudComputerTool's format.
+
+        Returns:
+            List of MCP content blocks
+        """
+        logger.info("QwenComputerTool received action: %s", action)
+
+        # Handle non-computer actions that should raise errors
+        if action == "terminate":
+            raise McpError(
+                ErrorData(
+                    code=INVALID_PARAMS,
+                    message=(
+                        "terminate action is not supported for computer control. This is a no-op."
+                    ),
+                )
+            )
+
+        if action == "answer":
+            raise McpError(
+                ErrorData(
+                    code=INVALID_PARAMS,
+                    message="answer action is not supported for computer control. This is a no-op.",
+                )
+            )
+
+        # Convert lists to tuples if needed
+        coord_tuple = None
+        if coordinate:
+            coord_tuple = tuple(coordinate) if isinstance(coordinate, list) else coordinate
+
+        # Map Qwen actions to HudComputerTool actions
+        if action == "left_click":
+            if coord_tuple and len(coord_tuple) >= 2:
+                scaled_x, scaled_y = self._scale_coordinates(coord_tuple[0], coord_tuple[1])
+                logger.info("Scaled coordinates: %s, %s", scaled_x, scaled_y)
+                result = await self.executor.click(x=scaled_x, y=scaled_y)
+            else:
+                raise McpError(
+                    ErrorData(code=INVALID_PARAMS, message="coordinate is required for left_click")
+                )
+
+        elif action == "double_click":
+            if coord_tuple and len(coord_tuple) >= 2:
+                # Use pattern for double-click
+                scaled_x, scaled_y = self._scale_coordinates(coord_tuple[0], coord_tuple[1])
+                result = await self.executor.click(x=scaled_x, y=scaled_y, pattern=[100])
+            else:
+                raise McpError(
+                    ErrorData(
+                        code=INVALID_PARAMS, message="coordinate is required for double_click"
+                    )
+                )
+
+        elif action == "triple_click":
+            if coord_tuple and len(coord_tuple) >= 2:
+                # Use pattern for triple-click (simulated as double-click)
+                scaled_x, scaled_y = self._scale_coordinates(coord_tuple[0], coord_tuple[1])
+                # Note: triple-click simulated as double-click as per requirement
+                result = await self.executor.click(x=scaled_x, y=scaled_y, pattern=[100])
+            else:
+                raise McpError(
+                    ErrorData(
+                        code=INVALID_PARAMS, message="coordinate is required for triple_click"
+                    )
+                )
+
+        elif action == "right_click":
+            if coord_tuple and len(coord_tuple) >= 2:
+                scaled_x, scaled_y = self._scale_coordinates(coord_tuple[0], coord_tuple[1])
+                result = await self.executor.click(x=scaled_x, y=scaled_y, button="right")
+            else:
+                raise McpError(
+                    ErrorData(code=INVALID_PARAMS, message="coordinate is required for right_click")
+                )
+
+        elif action == "middle_click":
+            if coord_tuple and len(coord_tuple) >= 2:
+                scaled_x, scaled_y = self._scale_coordinates(coord_tuple[0], coord_tuple[1])
+                result = await self.executor.click(x=scaled_x, y=scaled_y, button="middle")
+            else:
+                raise McpError(
+                    ErrorData(
+                        code=INVALID_PARAMS, message="coordinate is required for middle_click"
+                    )
+                )
+
+        elif action == "mouse_move":
+            if coord_tuple and len(coord_tuple) >= 2:
+                scaled_x, scaled_y = self._scale_coordinates(coord_tuple[0], coord_tuple[1])
+                result = await self.executor.move(x=scaled_x, y=scaled_y)
+            else:
+                raise McpError(
+                    ErrorData(code=INVALID_PARAMS, message="coordinate is required for mouse_move")
+                )
+
+        elif action == "type":
+            if text:
+                result = await self.executor.write(text=text)
+            else:
+                raise McpError(ErrorData(code=INVALID_PARAMS, message="text is required for type"))
+
+        elif action == "key":
+            if keys:
+                # Qwen sends an array of keys to press
+                result = await self.executor.press(keys=keys)
+            else:
+                raise McpError(ErrorData(code=INVALID_PARAMS, message="keys is required for key"))
+
+        elif action == "scroll":
+            if pixels is None:
+                raise McpError(
+                    ErrorData(code=INVALID_PARAMS, message="pixels is required for scroll")
+                )
+
+            # Qwen's pixels: positive scrolls up, negative scrolls down
+            # HUD's scroll_y: positive scrolls down, negative scrolls up
+            # So we need to negate the value
+            scroll_y = -pixels
+
+            if coord_tuple and len(coord_tuple) >= 2:
+                scaled_x, scaled_y = self._scale_coordinates(coord_tuple[0], coord_tuple[1])
+                result = await self.executor.scroll(x=scaled_x, y=scaled_y, scroll_y=scroll_y)
+            else:
+                result = await self.executor.scroll(scroll_y=scroll_y)
+
+        elif action == "hscroll":
+            if pixels is None:
+                raise McpError(
+                    ErrorData(code=INVALID_PARAMS, message="pixels is required for hscroll")
+                )
+
+            # For horizontal scroll, positive values scroll right, negative scroll left
+            scroll_x = pixels
+
+            if coord_tuple and len(coord_tuple) >= 2:
+                scaled_x, scaled_y = self._scale_coordinates(coord_tuple[0], coord_tuple[1])
+                result = await self.executor.scroll(x=scaled_x, y=scaled_y, scroll_x=scroll_x)
+            else:
+                result = await self.executor.scroll(scroll_x=scroll_x)
+
+        elif action == "left_click_drag":
+            if coord_tuple and len(coord_tuple) >= 2:
+                # For drag, we need a path. Qwen provides the end coordinate.
+                # We'll get the current position and drag from there to the target
+                current_pos = await self.executor.position()
+                if isinstance(current_pos, ContentResult) and current_pos.output:
+                    # Parse the position from the output
+                    match = re.search(r"x=(\d+), y=(\d+)", current_pos.output)
+                    if match:
+                        # Current position is in screen coordinates
+                        screen_start_x, screen_start_y = int(match.group(1)), int(match.group(2))
+                        # End position is in agent coordinates, needs scaling
+                        scaled_end_x, scaled_end_y = self._scale_coordinates(
+                            coord_tuple[0], coord_tuple[1]
+                        )
+                        # Create path in screen coordinates
+                        path = [(screen_start_x, screen_start_y), (scaled_end_x, scaled_end_y)]
+                        # Path is already in screen coordinates, no need to scale again
+                        result = await self.executor.drag(path=path)
+                    else:
+                        raise McpError(
+                            ErrorData(
+                                code=INTERNAL_ERROR, message="Failed to parse current position"
+                            )
+                        )
+                else:
+                    raise McpError(
+                        ErrorData(code=INTERNAL_ERROR, message="Failed to get current position")
+                    )
+            else:
+                raise McpError(
+                    ErrorData(
+                        code=INVALID_PARAMS, message="coordinate is required for left_click_drag"
+                    )
+                )
+
+        elif action == "wait":
+            if time is None:
+                raise McpError(ErrorData(code=INVALID_PARAMS, message="time is required for wait"))
+            if time < 0:
+                raise McpError(ErrorData(code=INVALID_PARAMS, message="time must be non-negative"))
+
+            # Convert seconds to milliseconds for HudComputerTool
+            result = await self.executor.wait(time=int(time * 1000))
+
+        else:
+            # Unknown action
+            raise McpError(ErrorData(code=INTERNAL_ERROR, message=f"Invalid action: {action}"))
+
+        # Rescale screenshot in result if present
+        if isinstance(result, ContentResult) and result.base64_image and self.rescale_images:
+            rescaled_image = await self._rescale_screenshot(result.base64_image)
+            result.base64_image = rescaled_image
+
+        # Auto-add screenshot for interactive actions
+        interactive_actions = {
+            "left_click",
+            "double_click",
+            "triple_click",
+            "right_click",
+            "middle_click",
+            "mouse_move",
+            "type",
+            "key",
+            "scroll",
+            "hscroll",
+            "left_click_drag",
+        }
+
+        if (
+            action in interactive_actions
+            and isinstance(result, ContentResult)
+            and not result.base64_image
+        ):
+            screenshot_base64 = await self.executor.screenshot()
+            if screenshot_base64:
+                # Rescale screenshot if requested
+                screenshot_base64 = await self._rescale_screenshot(screenshot_base64)
+                result = ContentResult(
+                    # note: we suppress the output since it's not useful
+                    output="",
+                    error=result.error,
+                    base64_image=screenshot_base64,
+                )
+
+        # Convert to content blocks
+        return result.to_content_blocks()

hud/tools/computer/settings.py

@@ -62,6 +62,17 @@ class ComputerSettings(BaseSettings):
         validation_alias="OPENAI_COMPUTER_HEIGHT",
     )
 
+    QWEN_COMPUTER_WIDTH: int = Field(
+        default=700,
+        description="Width of the display to use for the Qwen computer tools",
+        validation_alias="QWEN_COMPUTER_WIDTH",
+    )
+    QWEN_COMPUTER_HEIGHT: int = Field(
+        default=448,
+        description="Height of the display to use for the Qwen computer tools",
+        validation_alias="QWEN_COMPUTER_HEIGHT",
+    )
+
     HUD_RESCALE_IMAGES: bool = Field(
         default=False,
         description="Whether to rescale images to the agent width and height",
@@ -77,6 +88,32 @@ class ComputerSettings(BaseSettings):
         description="Whether to rescale images to the agent width and height",
         validation_alias="OPENAI_RESCALE_IMAGES",
     )
+    QWEN_RESCALE_IMAGES: bool = Field(
+        default=True,
+        description="Whether to rescale images to the agent width and height",
+        validation_alias="QWEN_RESCALE_IMAGES",
+    )
+
+    GEMINI_COMPUTER_WIDTH: int = Field(
+        default=1440,
+        description="Width of the display to use for the Gemini computer tools",
+        validation_alias="GEMINI_COMPUTER_WIDTH",
+    )
+    GEMINI_COMPUTER_HEIGHT: int = Field(
+        default=900,
+        description="Height of the display to use for the Gemini computer tools",
+        validation_alias="GEMINI_COMPUTER_HEIGHT",
+    )
+    GEMINI_RESCALE_IMAGES: bool = Field(
+        default=True,
+        description="Whether to rescale images to the agent width and height",
+        validation_alias="GEMINI_RESCALE_IMAGES",
+    )
+    GEMINI_MAX_RECENT_TURN_WITH_SCREENSHOTS: int = Field(
+        default=3,
+        description="Maximum number of recent turns to keep screenshots for in Gemini agent",
+        validation_alias="GEMINI_MAX_RECENT_TURN_WITH_SCREENSHOTS",
+    )
 
 
 computer_settings = ComputerSettings()

hud/tools/edit.py

@@ -1,16 +1,13 @@
-from __future__ import annotations
-
 from collections import defaultdict
 from pathlib import Path
-from typing import TYPE_CHECKING, Any, Literal, get_args
+from typing import Literal, get_args
+
+from mcp.types import ContentBlock
 
 from .base import BaseTool
 from .types import ContentResult, ToolError
 from .utils import maybe_truncate, run
 
-if TYPE_CHECKING:
-    from mcp.types import ContentBlock
-
 Command = Literal[
     "view",
     "create",
@@ -56,7 +53,6 @@ class EditTool(BaseTool):
         old_str: str | None = None,
         new_str: str | None = None,
         insert_line: int | None = None,
-        **kwargs: Any,
     ) -> list[ContentBlock]:
         _path = Path(path)
         self.validate_path(command, _path)

hud/tools/executors/base.py

@@ -280,7 +280,7 @@ class BaseExecutor:
 
     # ===== Utility Actions =====
 
-    async def wait(self, time: int) -> ContentResult:
+    async def wait(self, time: int, take_screenshot: bool = True) -> ContentResult:
         """
         Wait for specified time.
 
@@ -289,7 +289,9 @@ class BaseExecutor:
         """
         duration_seconds = time / 1000.0
         await asyncio.sleep(duration_seconds)
-        return ContentResult(output=f"Waited {time}ms")
+        # take screenshot
+        screenshot = await self.screenshot() if take_screenshot else None
+        return ContentResult(output=f"Waited {time}ms", base64_image=screenshot)
 
     async def screenshot(self) -> str | None:
         """

hud/tools/executors/pyautogui.py

@@ -31,7 +31,7 @@ def _get_pyautogui() -> Any | None:
             try:
                 from hud.tools.computer import computer_settings
 
-                os.environ["DISPLAY"] = str(computer_settings.DISPLAY_NUM)
+                os.environ["DISPLAY"] = f":{computer_settings.DISPLAY_NUM}"
             except (ImportError, AttributeError):
                 os.environ["DISPLAY"] = ":0"