hud-python 0.4.45__py3-none-any.whl → 0.5.1__py3-none-any.whl

hud/tools/base.py

@@ -1,5 +1,6 @@
 from __future__ import annotations
 
+import logging
 from abc import ABC, abstractmethod
 from typing import TYPE_CHECKING, Any, cast
 
@@ -8,7 +9,7 @@ from fastmcp import FastMCP
 from hud.tools.types import ContentBlock, EvaluationResult
 
 if TYPE_CHECKING:
-    from collections.abc import Callable
+    from collections.abc import Awaitable, Callable
 
     from fastmcp.tools import FunctionTool
     from fastmcp.tools.tool import Tool, ToolResult
@@ -16,6 +17,8 @@ if TYPE_CHECKING:
 # Basic result types for tools
 BaseResult = list[ContentBlock] | EvaluationResult
 
+logger = logging.getLogger(__name__)
+
 
 class BaseTool(ABC):
     """
@@ -58,6 +61,10 @@ class BaseTool(ABC):
         self.title = title or self.__class__.__name__.replace("Tool", "").replace("_", " ").title()
         self.description = description or (self.__doc__.strip() if self.__doc__ else None)
         self.meta = meta
+        self._callbacks: dict[
+            str,
+            list[Callable[..., Awaitable[Any]]],
+        ] = {}  # {"event_name": [callback_functions]}
 
         # Expose attributes FastMCP expects when registering an instance directly
         self.__name__ = self.name  # FastMCP uses fn.__name__ if name param omitted
@@ -100,13 +107,48 @@ class BaseTool(ABC):
             )
         return self._mcp_tool
 
+    def add_callback(self, event_type: str, callback: Callable[..., Awaitable[Any]]) -> None:
+        """Register a callback function for specific event
+
+        Args:
+            event_type: (Required) Specific event name to trigger callback
+                        e.g. "after_click", "before_navigate"
+            callback: (Required) Async function to call. Must be defined by `async def f(...)`
+        """
+        if event_type not in self._callbacks:
+            self._callbacks[event_type] = []
+        self._callbacks[event_type].append(callback)
+
+    def remove_callback(self, event_type: str, callback: Callable[..., Awaitable[Any]]) -> None:
+        """Remove a registered callback
+        Args:
+            event_type: (Required) Specific event name to trigger callback
+                        e.g. "after_click", "before_navigate"
+            callback: (Required) Function to remove from callback list.
+        """
+        if (event_type in self._callbacks) and (callback in self._callbacks[event_type]):
+            self._callbacks[event_type].remove(callback)
+
+    async def _trigger_callbacks(self, event_type: str, **kwargs: Any) -> None:
+        """Trigger all registered callback functions of an event type"""
+        callback_list = self._callbacks.get(event_type, [])
+        for callback in callback_list:
+            try:
+                await callback(**kwargs)
+            except Exception as e:
+                logger.warning("Callback failed for %s: %s", event_type, e)
+
 
 # Prefix for internal tool names
 _INTERNAL_PREFIX = "int_"
 
 
 class BaseHub(FastMCP):
-    """A composition-friendly FastMCP server that holds an internal tool dispatcher."""
+    """A composition-friendly FastMCP server that holds an internal tool dispatcher.
+
+    Note: BaseHub can be used standalone or to wrap existing routers. For the newer
+    FastAPI-like pattern, consider using HiddenRouter from hud.server instead.
+    """
 
     env: Any
 
@@ -129,6 +171,10 @@ class BaseHub(FastMCP):
             Optional long-lived environment object. Stored on the server
             instance (``layer.env``) and therefore available to every request
             via ``ctx.fastmcp.env``.
+        title:
+            Optional title for the dispatcher tool.
+        description:
+            Optional description for the dispatcher tool.
         meta:
             Metadata to include in MCP tool listing.
         """
@@ -370,8 +416,12 @@ class BaseHub(FastMCP):
                 }
 
     # Override _list_tools to hide internal tools when mounted
-    async def _list_tools(self) -> list[Tool]:
-        """Override _list_tools to hide internal tools when mounted."""
+    async def _list_tools(self, context: Any = None) -> list[Tool]:
+        """Override _list_tools to hide internal tools when mounted.
+
+        Args:
+            context: MiddlewareContext passed by FastMCP (optional for backwards compat)
+        """
         return [
             tool
             for key, tool in self._tool_manager._tools.items()

hud/tools/bash.py

@@ -3,7 +3,7 @@ from __future__ import annotations
 import asyncio
 import os
 import sys
-from typing import TYPE_CHECKING, Any
+from typing import TYPE_CHECKING
 
 from .base import BaseTool
 from .types import ContentResult, ToolError
@@ -140,7 +140,7 @@ class BashTool(BaseTool):
         self.env = value
 
     async def __call__(
-        self, command: str | None = None, restart: bool = False, **kwargs: Any
+        self, command: str | None = None, restart: bool = False
     ) -> list[ContentBlock]:
         if restart:
             if self.session:

hud/tools/computer/__init__.py

@@ -3,13 +3,17 @@
 from __future__ import annotations
 
 from .anthropic import AnthropicComputerTool
+from .gemini import GeminiComputerTool
 from .hud import HudComputerTool
 from .openai import OpenAIComputerTool
+from .qwen import QwenComputerTool
 from .settings import computer_settings
 
 __all__ = [
     "AnthropicComputerTool",
+    "GeminiComputerTool",
     "HudComputerTool",
     "OpenAIComputerTool",
+    "QwenComputerTool",
     "computer_settings",
 ]

hud/tools/computer/anthropic.py

@@ -141,13 +141,13 @@ class AnthropicComputerTool(HudComputerTool):
     async def __call__(
         self,
         action: str = Field(..., description="The action to perform on the computer"),
-        coordinate: list[int] | tuple[int, int] | None = Field(
+        coordinate: list[int] | None = Field(
             None, description="The coordinate to interact with on the computer [x, y]"
         ),
         text: str | None = Field(
             None, description="The text to type on the computer or key to press"
         ),
-        start_coordinate: list[int] | tuple[int, int] | None = Field(
+        start_coordinate: list[int] | None = Field(
             None, description="The starting coordinate for drag actions [x, y]"
         ),
         scroll_direction: str | None = Field(

hud/tools/computer/gemini.py

@@ -0,0 +1,385 @@
+from __future__ import annotations
+
+import logging
+import platform
+from typing import TYPE_CHECKING, Any, Literal
+
+from mcp import ErrorData, McpError
+from mcp.types import INVALID_PARAMS, ContentBlock
+from pydantic import Field
+
+from hud.tools.types import ContentResult
+
+from .hud import HudComputerTool
+from .settings import computer_settings
+
+if TYPE_CHECKING:
+    from hud.tools.executors.base import BaseExecutor
+
+logger = logging.getLogger(__name__)
+
+
+ACTION_FIELD = Field(..., description="Gemini Computer Use action to perform")
+X_FIELD = Field(None, description="X coordinate (pixels in agent space)")
+Y_FIELD = Field(None, description="Y coordinate (pixels in agent space)")
+TEXT_FIELD = Field(None, description="Text to type")
+PRESS_ENTER_FIELD = Field(None, description="Whether to press Enter after typing (type_text_at)")
+CLEAR_BEFORE_TYPING_FIELD = Field(
+    None, description="Whether to select-all before typing (type_text_at)"
+)
+DIRECTION_FIELD = Field(None, description="Scroll direction for scroll_document/scroll_at")
+MAGNITUDE_FIELD = Field(None, description="Scroll magnitude (pixels in agent space)")
+URL_FIELD = Field(None, description="Target URL for navigate")
+KEYS_FIELD = Field(None, description="Keys for key_combination")
+DESTINATION_X_FIELD = Field(None, description="Destination X for drag_and_drop (agent space)")
+DESTINATION_Y_FIELD = Field(None, description="Destination Y for drag_and_drop (agent space)")
+TAKE_SCREENSHOT_ON_CLICK_FIELD = Field(
+    True, description="Whether to include a screenshot for interactive actions"
+)
+
+
+class GeminiComputerTool(HudComputerTool):
+    """
+    Gemini Computer Use tool for interacting with a computer via MCP.
+
+    Maps Gemini's predefined function names (open_web_browser, click_at, hover_at,
+    type_text_at, scroll_document, scroll_at, wait_5_seconds, go_back, go_forward,
+    search, navigate, key_combination, drag_and_drop) to executor actions.
+    """
+
+    def __init__(
+        self,
+        # Define within environment based on platform
+        executor: BaseExecutor | None = None,
+        platform_type: Literal["auto", "xdo", "pyautogui"] = "auto",
+        display_num: int | None = None,
+        # Overrides for what dimensions the agent thinks it operates in
+        width: int = computer_settings.GEMINI_COMPUTER_WIDTH,
+        height: int = computer_settings.GEMINI_COMPUTER_HEIGHT,
+        rescale_images: bool = computer_settings.GEMINI_RESCALE_IMAGES,
+        # What the agent sees as the tool's name, title, and description
+        name: str | None = None,
+        title: str | None = None,
+        description: str | None = None,
+        **kwargs: Any,
+    ) -> None:
+        """
+        Initialize with Gemini's default dimensions.
+        """
+        super().__init__(
+            executor=executor,
+            platform_type=platform_type,
+            display_num=display_num,
+            width=width,
+            height=height,
+            rescale_images=rescale_images,
+            name=name or "gemini_computer",
+            title=title or "Gemini Computer Tool",
+            description=description or "Control computer with mouse, keyboard, and screenshots",
+            **kwargs,
+        )
+
+    async def __call__(
+        self,
+        action: str = ACTION_FIELD,
+        # Common coordinates
+        x: int | None = X_FIELD,
+        y: int | None = Y_FIELD,
+        # Text input
+        text: str | None = TEXT_FIELD,
+        press_enter: bool | None = PRESS_ENTER_FIELD,
+        clear_before_typing: bool | None = CLEAR_BEFORE_TYPING_FIELD,
+        # Scroll parameters
+        direction: Literal["up", "down", "left", "right"] | None = DIRECTION_FIELD,
+        magnitude: int | None = MAGNITUDE_FIELD,
+        # Navigation
+        url: str | None = URL_FIELD,
+        # Key combos
+        keys: list[str] | str | None = KEYS_FIELD,
+        # Drag parameters
+        destination_x: int | None = DESTINATION_X_FIELD,
+        destination_y: int | None = DESTINATION_Y_FIELD,
+        # Behavior
+        take_screenshot_on_click: bool = TAKE_SCREENSHOT_ON_CLICK_FIELD,
+    ) -> list[ContentBlock]:
+        """
+        Handle Gemini Computer Use API calls by mapping to executor actions.
+
+        Returns:
+            List of MCP content blocks
+        """
+        logger.info("GeminiComputerTool received action: %s", action)
+
+        # Helper to finalize ContentResult: rescale if requested and ensure URL metadata
+        async def _finalize(
+            result: ContentResult, requested_url: str | None = None
+        ) -> list[ContentBlock]:
+            if result.base64_image and self.rescale_images:
+                try:
+                    result.base64_image = await self._rescale_screenshot(result.base64_image)
+                except Exception as e:
+                    logger.warning("Failed to rescale screenshot: %s", e)
+            # Always include URL metadata if provided; otherwise default to about:blank
+            result.url = requested_url or result.url or "about:blank"
+            return result.to_content_blocks()
+
+        # Scale coordinates helper
+        def _scale(xv: int | None, yv: int | None) -> tuple[int | None, int | None]:
+            return self._scale_coordinates(xv, yv)
+
+        # Gemini emits coordinates/magnitudes in a 0-1000 normalized space.
+        def _denormalize(value: float | None, axis: Literal["x", "y"]) -> int | None:
+            if value is None:
+                return None
+            try:
+                numeric = float(value)
+            except (TypeError, ValueError):
+                try:
+                    return int(value)  # type: ignore[arg-type]
+                except (TypeError, ValueError):
+                    return None
+
+            # Treat values within the normalized range (including defaults like 800).
+            if 0 <= numeric <= 1000:
+                target = self.width if axis == "x" else self.height
+                numeric = numeric / 1000 * target
+
+            return round(numeric)
+
+        def _scale_distance(value: int | None, axis: Literal["x", "y"]) -> int | None:
+            if value is None:
+                return None
+            scale = self.scale_x if axis == "x" else self.scale_y
+            if scale != 1.0:
+                return round(value / scale)
+            return value
+
+        # Map actions
+        if action == "open_web_browser":
+            screenshot = await self.executor.screenshot()
+            if screenshot:
+                result = ContentResult(base64_image=screenshot, url="about:blank")
+            else:
+                result = ContentResult(error="Failed to take screenshot", url="about:blank")
+            return await _finalize(result)
+
+        elif action == "click_at":
+            if x is None or y is None:
+                raise McpError(ErrorData(code=INVALID_PARAMS, message="x and y are required"))
+            dx = _denormalize(x, "x")
+            dy = _denormalize(y, "y")
+            sx, sy = _scale(dx, dy)
+            result = await self.executor.click(x=sx, y=sy)
+            return await _finalize(result)
+
+        elif action == "hover_at":
+            if x is None or y is None:
+                raise McpError(ErrorData(code=INVALID_PARAMS, message="x and y are required"))
+            dx = _denormalize(x, "x")
+            dy = _denormalize(y, "y")
+            sx, sy = _scale(dx, dy)
+            result = await self.executor.move(x=sx, y=sy)
+            return await _finalize(result)
+
+        elif action == "type_text_at":
+            if x is None or y is None:
+                raise McpError(ErrorData(code=INVALID_PARAMS, message="x and y are required"))
+            if text is None:
+                raise McpError(ErrorData(code=INVALID_PARAMS, message="text is required"))
+
+            dx = _denormalize(x, "x")
+            dy = _denormalize(y, "y")
+            sx, sy = _scale(dx, dy)
+
+            # Focus the field
+            await self.executor.move(x=sx, y=sy, take_screenshot=False)
+            await self.executor.click(x=sx, y=sy, take_screenshot=False)
+
+            # Clear existing text if requested
+            if clear_before_typing is None or clear_before_typing:
+                is_mac = platform.system().lower() == "darwin"
+                combo = ["cmd", "a"] if is_mac else ["ctrl", "a"]
+                await self.executor.press(keys=combo, take_screenshot=False)
+                delete_key = "backspace" if is_mac else "delete"
+                await self.executor.press(keys=[delete_key], take_screenshot=False)
+
+            # Type (optionally press enter after)
+            result = await self.executor.write(text=text, enter_after=bool(press_enter))
+            return await _finalize(result)
+
+        elif action == "scroll_document":
+            if direction is None:
+                raise McpError(ErrorData(code=INVALID_PARAMS, message="direction is required"))
+            # Default magnitude similar to reference implementation
+            mag = magnitude if magnitude is not None else 800
+            # Convert to environment units while preserving sign
+            if direction in ("down", "up"):
+                distance = _denormalize(mag, "y")
+                if distance is None:
+                    raise McpError(
+                        ErrorData(
+                            code=INVALID_PARAMS, message="Unable to determine scroll magnitude"
+                        )
+                    )
+                distance = _scale_distance(distance, "y")
+                if distance is None:
+                    raise McpError(
+                        ErrorData(
+                            code=INVALID_PARAMS,
+                            message="Unable to determine scroll magnitude",
+                        )
+                    )
+                scroll_y = distance if direction == "down" else -distance
+                scroll_x = None
+            elif direction in ("right", "left"):
+                distance = _denormalize(mag, "x")
+                if distance is None:
+                    raise McpError(
+                        ErrorData(
+                            code=INVALID_PARAMS, message="Unable to determine scroll magnitude"
+                        )
+                    )
+                distance = _scale_distance(distance, "x")
+                if distance is None:
+                    raise McpError(
+                        ErrorData(
+                            code=INVALID_PARAMS,
+                            message="Unable to determine scroll magnitude",
+                        )
+                    )
+                scroll_x = distance if direction == "right" else -distance
+                scroll_y = None
+            else:
+                raise McpError(
+                    ErrorData(code=INVALID_PARAMS, message=f"Invalid direction: {direction}")
+                )
+            result = await self.executor.scroll(scroll_x=scroll_x, scroll_y=scroll_y)
+            return await _finalize(result)
+
+        elif action == "scroll_at":
+            if direction is None:
+                raise McpError(ErrorData(code=INVALID_PARAMS, message="direction is required"))
+            if x is None or y is None:
+                raise McpError(ErrorData(code=INVALID_PARAMS, message="x and y are required"))
+            mag = magnitude if magnitude is not None else 800
+            dx = _denormalize(x, "x")
+            dy = _denormalize(y, "y")
+            sx, sy = _scale(dx, dy)
+            if direction in ("down", "up"):
+                distance = _denormalize(mag, "y")
+                if distance is None:
+                    raise McpError(
+                        ErrorData(
+                            code=INVALID_PARAMS, message="Unable to determine scroll magnitude"
+                        )
+                    )
+                distance = _scale_distance(distance, "y")
+                if distance is None:
+                    raise McpError(
+                        ErrorData(
+                            code=INVALID_PARAMS,
+                            message="Unable to determine scroll magnitude",
+                        )
+                    )
+                scroll_y = distance if direction == "down" else -distance
+                scroll_x = None
+            elif direction in ("right", "left"):
+                distance = _denormalize(mag, "x")
+                if distance is None:
+                    raise McpError(
+                        ErrorData(
+                            code=INVALID_PARAMS, message="Unable to determine scroll magnitude"
+                        )
+                    )
+                distance = _scale_distance(distance, "x")
+                if distance is None:
+                    raise McpError(
+                        ErrorData(
+                            code=INVALID_PARAMS,
+                            message="Unable to determine scroll magnitude",
+                        )
+                    )
+                scroll_x = distance if direction == "right" else -distance
+                scroll_y = None
+            else:
+                raise McpError(
+                    ErrorData(code=INVALID_PARAMS, message=f"Invalid direction: {direction}")
+                )
+            result = await self.executor.scroll(x=sx, y=sy, scroll_x=scroll_x, scroll_y=scroll_y)
+            return await _finalize(result)
+
+        elif action == "wait_5_seconds":
+            result = await self.executor.wait(time=5000)
+            return await _finalize(result)
+
+        elif action == "go_back":
+            is_mac = platform.system().lower() == "darwin"
+            combo = ["cmd", "["] if is_mac else ["alt", "left"]
+            result = await self.executor.press(keys=combo)
+            return await _finalize(result)
+
+        elif action == "go_forward":
+            is_mac = platform.system().lower() == "darwin"
+            combo = ["cmd", "]"] if is_mac else ["alt", "right"]
+            result = await self.executor.press(keys=combo)
+            return await _finalize(result)
+
+        elif action == "search":
+            # Best-effort navigate to a default search page
+            target = url or "https://www.google.com"
+            is_mac = platform.system().lower() == "darwin"
+            await self.executor.press(
+                keys=["cmd", "l"] if is_mac else ["ctrl", "l"], take_screenshot=False
+            )
+            result = await self.executor.write(text=target, enter_after=True)
+            return await _finalize(result, requested_url=target)
+
+        elif action == "navigate":
+            if not url:
+                raise McpError(ErrorData(code=INVALID_PARAMS, message="url is required"))
+            is_mac = platform.system().lower() == "darwin"
+            await self.executor.press(
+                keys=["cmd", "l"] if is_mac else ["ctrl", "l"], take_screenshot=False
+            )
+            result = await self.executor.write(text=url, enter_after=True)
+            return await _finalize(result, requested_url=url)
+
+        elif action == "key_combination":
+            if keys is None:
+                raise McpError(ErrorData(code=INVALID_PARAMS, message="keys is required"))
+            if isinstance(keys, str):
+                # Accept formats like "ctrl+c" or "ctrl+shift+t"
+                key_list = [k.strip() for k in keys.split("+") if k.strip()]
+            else:
+                key_list = keys
+            result = await self.executor.press(keys=key_list)
+            return await _finalize(result)
+
+        elif action == "drag_and_drop":
+            if x is None or y is None or destination_x is None or destination_y is None:
+                raise McpError(
+                    ErrorData(
+                        code=INVALID_PARAMS,
+                        message="x, y, destination_x, and destination_y are required",
+                    )
+                )
+            sx_norm = _denormalize(x, "x")
+            sy_norm = _denormalize(y, "y")
+            dx_norm = _denormalize(destination_x, "x")
+            dy_norm = _denormalize(destination_y, "y")
+            sx, sy = _scale(sx_norm, sy_norm)
+            dx_scaled, dy_scaled = _scale(dx_norm, dy_norm)
+            # Build a two-point path
+            path = []  # type: list[tuple[int, int]]
+            if (
+                sx is not None
+                and sy is not None
+                and dx_scaled is not None
+                and dy_scaled is not None
+            ):
+                path = [(sx, sy), (dx_scaled, dy_scaled)]
+            result = await self.executor.drag(path=path)
+            return await _finalize(result)
+
+        else:
+            raise McpError(ErrorData(code=INVALID_PARAMS, message=f"Unknown action: {action}"))

hud/tools/computer/hud.py

@@ -13,7 +13,7 @@ from hud.tools.base import BaseTool
 from hud.tools.executors.base import BaseExecutor
 from hud.tools.executors.pyautogui import PyAutoGUIExecutor
 from hud.tools.executors.xdo import XDOExecutor
-from hud.tools.types import ContentResult, ToolError
+from hud.tools.types import ContentResult, Coordinate, ToolError
 
 from .settings import computer_settings
 
@@ -231,7 +231,23 @@ class HudComputerTool(BaseTool):
 
     async def __call__(
         self,
-        action: str = Field(..., description="The action name (click, press, write, move, etc.)"),
+        action: Literal[
+            "click",
+            "press",
+            "keydown",
+            "keyup",
+            "write",
+            "scroll",
+            "move",
+            "wait",
+            "drag",
+            "response",
+            "screenshot",
+            "position",
+            "hold_key",
+            "mouse_down",
+            "mouse_up",
+        ] = Field(..., description="The action name (click, press, write, move, etc.)"),
         # Click parameters
         x: int | None = Field(None, description="X coordinate for click/move/scroll actions"),
         y: int | None = Field(None, description="Y coordinate for click/move/scroll actions"),
@@ -254,8 +270,8 @@ class HudComputerTool(BaseTool):
         offset_x: int | None = Field(None, description="X offset for relative move"),
         offset_y: int | None = Field(None, description="Y offset for relative move"),
         # Drag parameters
-        path: list[tuple[int, int]] | None = Field(
-            None, description="Path for drag actions as list of (x, y) coordinates"
+        path: list[Coordinate] | None = Field(
+            None, description="Path for drag actions as list of {x, y} coordinates"
         ),
         # Wait parameter
         time: int | None = Field(None, description="Time in milliseconds for wait action"),
@@ -332,8 +348,9 @@ class HudComputerTool(BaseTool):
             elif action == "drag":
                 if path is None:
                     raise ToolError("path parameter is required for drag")
-                # Scale path from client space to screen space
-                scaled_path = self._scale_path(path)
+                # Convert Coordinate objects to tuples and scale from client space to screen space
+                path_tuples = [(point.x, point.y) for point in path]
+                scaled_path = self._scale_path(path_tuples)
                 result = await self.executor.drag(
                     path=scaled_path, pattern=pattern, hold_keys=hold_keys
                 )