hud-python 0.3.0__py3-none-any.whl → 0.3.1__py3-none-any.whl

hud/tools/executors/pyautogui.py

@@ -5,28 +5,56 @@ import base64
 import logging
 import os
 from io import BytesIO
-from typing import Literal
+from typing import Any, Literal
 
-if "DISPLAY" not in os.environ:
-    try:
-        from hud.settings import settings
+from hud.tools.base import ToolResult
 
-        os.environ["DISPLAY"] = settings.display
-    except (ImportError, AttributeError):
-        os.environ["DISPLAY"] = ":0"
+from .base import BaseExecutor
 
-try:
-    import pyautogui
+logger = logging.getLogger(__name__)
 
-    PYAUTOGUI_AVAILABLE = True
-except ImportError:
-    PYAUTOGUI_AVAILABLE = False
+# Lazy loading for pyautogui
+_pyautogui = None
+_pyautogui_available = None
 
-from hud.tools.base import ToolResult
 
-from .base import BaseExecutor
+def _get_pyautogui() -> Any | None:
+    """Lazily import and return pyautogui module."""
+    global _pyautogui, _pyautogui_available
+
+    if _pyautogui_available is False:
+        return None
+
+    if _pyautogui is None:
+        # Set display if not already set
+        if "DISPLAY" not in os.environ:
+            try:
+                from hud.settings import settings
+
+                os.environ["DISPLAY"] = settings.display
+            except (ImportError, AttributeError):
+                os.environ["DISPLAY"] = ":0"
+
+        try:
+            import pyautogui
+
+            _pyautogui = pyautogui
+            _pyautogui_available = True
+
+            # Configure PyAutoGUI settings
+            _pyautogui.FAILSAFE = False  # Disable fail-safe feature
+            _pyautogui.PAUSE = 0.1  # Small pause between actions
+        except ImportError:
+            _pyautogui_available = False
+            logger.warning("PyAutoGUI is not available")
+            return None
+        except Exception as e:
+            _pyautogui_available = False
+            logger.warning("Failed to initialize PyAutoGUI: %s", e)
+            return None
+
+    return _pyautogui
 
-logger = logging.getLogger(__name__)
 
 # Map CLA standard keys to PyAutoGUI keys (only where they differ)
 CLA_TO_PYAUTOGUI = {
@@ -58,12 +86,17 @@ class PyAutoGUIExecutor(BaseExecutor):
             display_num: X display number (used only on Linux, ignored on Windows/macOS)
         """
         super().__init__(display_num)
-
+        self._pyautogui = None
         logger.info("PyAutoGUIExecutor initialized")
 
-        # Configure PyAutoGUI settings
-        pyautogui.FAILSAFE = False  # Disable fail-safe feature
-        pyautogui.PAUSE = 0.1  # Small pause between actions
+    @property
+    def pyautogui(self) -> Any:
+        """Get the pyautogui module, importing it lazily if needed."""
+        if self._pyautogui is None:
+            self._pyautogui = _get_pyautogui()
+            if self._pyautogui is None:
+                raise RuntimeError("PyAutoGUI is not available")
+        return self._pyautogui
 
     def _map_key(self, key: str) -> str:
         """Map CLA standard key to PyAutoGUI key."""
@@ -90,7 +123,8 @@ class PyAutoGUIExecutor(BaseExecutor):
         Returns:
             True if PyAutoGUI is available and functional, False otherwise
         """
-        if not PYAUTOGUI_AVAILABLE:
+        pyautogui = _get_pyautogui()
+        if not pyautogui:
             return False
 
         try:
@@ -109,7 +143,7 @@ class PyAutoGUIExecutor(BaseExecutor):
         """
         try:
             # Take screenshot using PyAutoGUI
-            screenshot = pyautogui.screenshot()
+            screenshot = self.pyautogui.screenshot()
 
             # Convert to base64
             buffer = BytesIO()
@@ -131,13 +165,13 @@ class PyAutoGUIExecutor(BaseExecutor):
         """
         if keys:
             for key in keys:
-                pyautogui.keyDown(key)
+                self.pyautogui.keyDown(key)
 
     def _release_keys(self, keys: list[str] | None) -> None:
         """Release held keys."""
         if keys:
             for key in reversed(keys):  # Release in reverse order
-                pyautogui.keyUp(key)
+                self.pyautogui.keyUp(key)
 
     # ===== CLA Action Implementations =====
 
@@ -172,17 +206,17 @@ class PyAutoGUIExecutor(BaseExecutor):
                     interval = pattern[0] / 1000.0 if pattern else 0.1  # Convert ms to seconds
 
                     if x is not None and y is not None:
-                        pyautogui.click(
+                        self.pyautogui.click(
                             x=x, y=y, clicks=clicks, interval=interval, button=button_name
                         )
                     else:
-                        pyautogui.click(clicks=clicks, interval=interval, button=button_name)
+                        self.pyautogui.click(clicks=clicks, interval=interval, button=button_name)
                 else:
                     # Single click
                     if x is not None and y is not None:
-                        pyautogui.click(x=x, y=y, button=button_name)
+                        self.pyautogui.click(x=x, y=y, button=button_name)
                     else:
-                        pyautogui.click(button=button_name)
+                        self.pyautogui.click(button=button_name)
             finally:
                 # Release held keys
                 self._release_keys(hold_keys)
@@ -210,10 +244,10 @@ class PyAutoGUIExecutor(BaseExecutor):
         try:
             # Convert delay from milliseconds to seconds for PyAutoGUI
             interval = delay / 1000.0
-            pyautogui.typewrite(text, interval=interval)
+            self.pyautogui.typewrite(text, interval=interval)
 
             if enter_after:
-                pyautogui.press("enter")
+                self.pyautogui.press("enter")
 
             result = ToolResult(
                 output=f"Typed: '{text}'" + (" and pressed Enter" if enter_after else "")
@@ -237,12 +271,12 @@ class PyAutoGUIExecutor(BaseExecutor):
             # Handle key combinations (e.g., "ctrl+c")
             if "+" in key_sequence:
                 keys = key_sequence.split("+")
-                pyautogui.hotkey(*keys)
+                self.pyautogui.hotkey(*keys)
                 result = ToolResult(output=f"Pressed hotkey: {key_sequence}")
             else:
                 # Map common key names from xdotool to PyAutoGUI
                 key = key_sequence.lower()
-                pyautogui.press(CLA_TO_PYAUTOGUI.get(key, key))
+                self.pyautogui.press(CLA_TO_PYAUTOGUI.get(key, key))
                 result = ToolResult(output=f"Pressed key: {key_sequence}")
 
             if take_screenshot:
@@ -265,7 +299,7 @@ class PyAutoGUIExecutor(BaseExecutor):
 
             # Handle single key or combination
             if len(mapped_keys) == 1 and "+" not in mapped_keys[0]:
-                pyautogui.press(mapped_keys[0])
+                self.pyautogui.press(mapped_keys[0])
                 result = ToolResult(output=f"Pressed key: {keys[0]}")
             else:
                 # For combinations, use hotkey
@@ -275,7 +309,7 @@ class PyAutoGUIExecutor(BaseExecutor):
                         hotkey_parts.extend(key.split("+"))
                     else:
                         hotkey_parts.append(key)
-                pyautogui.hotkey(*hotkey_parts)
+                self.pyautogui.hotkey(*hotkey_parts)
                 result = ToolResult(output=f"Pressed hotkey: {'+'.join(keys)}")
 
             if take_screenshot:
@@ -296,7 +330,7 @@ class PyAutoGUIExecutor(BaseExecutor):
             # Map CLA keys to PyAutoGUI keys
             mapped_keys = self._map_keys(keys)
             for key in mapped_keys:
-                pyautogui.keyDown(key)
+                self.pyautogui.keyDown(key)
 
             result = ToolResult(output=f"Keys down: {', '.join(keys)}")
 
@@ -318,7 +352,7 @@ class PyAutoGUIExecutor(BaseExecutor):
             # Map CLA keys to PyAutoGUI keys
             mapped_keys = self._map_keys(keys)
             for key in reversed(mapped_keys):  # Release in reverse order
-                pyautogui.keyUp(key)
+                self.pyautogui.keyUp(key)
 
             result = ToolResult(output=f"Keys up: {', '.join(keys)}")
 
@@ -347,7 +381,7 @@ class PyAutoGUIExecutor(BaseExecutor):
         try:
             # Move to position if specified
             if x is not None and y is not None:
-                pyautogui.moveTo(x, y)
+                self.pyautogui.moveTo(x, y)
 
             # Hold keys if specified
             self._hold_keys_context(hold_keys)
@@ -358,14 +392,14 @@ class PyAutoGUIExecutor(BaseExecutor):
                 # Perform vertical scroll
                 if scroll_y and scroll_y != 0:
                     # PyAutoGUI: positive = up, negative = down (opposite of our convention)
-                    pyautogui.scroll(-scroll_y)
+                    self.pyautogui.scroll(-scroll_y)
                     msg_parts.append(f"vertically by {scroll_y}")
 
                 # Perform horizontal scroll (if supported)
                 if scroll_x and scroll_x != 0:
                     # PyAutoGUI horizontal scroll might not work on all platforms
                     try:
-                        pyautogui.hscroll(scroll_x)
+                        self.pyautogui.hscroll(scroll_x)
                         msg_parts.append(f"horizontally by {scroll_x}")
                     except AttributeError:
                         # hscroll not available
@@ -409,13 +443,13 @@ class PyAutoGUIExecutor(BaseExecutor):
         try:
             if x is not None and y is not None:
                 # Absolute move
-                pyautogui.moveTo(x, y, duration=0.1)
+                self.pyautogui.moveTo(x, y, duration=0.1)
                 result = ToolResult(output=f"Moved mouse to ({x}, {y})")
             elif offset_x is not None or offset_y is not None:
                 # Relative move
                 offset_x = offset_x or 0
                 offset_y = offset_y or 0
-                pyautogui.moveRel(xOffset=offset_x, yOffset=offset_y, duration=0.1)
+                self.pyautogui.moveRel(xOffset=offset_x, yOffset=offset_y, duration=0.1)
                 result = ToolResult(output=f"Moved mouse by offset ({offset_x}, {offset_y})")
             else:
                 return ToolResult(output="No move coordinates specified")
@@ -450,25 +484,25 @@ class PyAutoGUIExecutor(BaseExecutor):
             try:
                 # Move to start
                 start_x, start_y = path[0]
-                pyautogui.moveTo(start_x, start_y)
+                self.pyautogui.moveTo(start_x, start_y)
 
                 # Handle multi-point drag
                 if len(path) == 2:
                     # Simple drag
                     end_x, end_y = path[1]
-                    pyautogui.dragTo(end_x, end_y, duration=0.5, button="left")
+                    self.pyautogui.dragTo(end_x, end_y, duration=0.5, button="left")
                     result = ToolResult(
                         output=f"Dragged from ({start_x}, {start_y}) to ({end_x}, {end_y})"
                     )
                 else:
                     # Multi-point drag
-                    pyautogui.mouseDown(button="left")
+                    self.pyautogui.mouseDown(button="left")
                     for i, (x, y) in enumerate(path[1:], 1):
                         duration = 0.1
                         if pattern and i - 1 < len(pattern):
                             duration = pattern[i - 1] / 1000.0  # Convert ms to seconds
-                        pyautogui.moveTo(x, y, duration=duration)
-                    pyautogui.mouseUp(button="left")
+                        self.pyautogui.moveTo(x, y, duration=duration)
+                    self.pyautogui.mouseUp(button="left")
 
                     result = ToolResult(output=f"Dragged along {len(path)} points")
 
@@ -507,7 +541,7 @@ class PyAutoGUIExecutor(BaseExecutor):
             }  # Fallback for unsupported
             button_name = button_map.get(button, "left")
 
-            pyautogui.mouseDown(button=button_name)
+            self.pyautogui.mouseDown(button=button_name)
             result = ToolResult(output=f"Mouse down: {button} button")
 
             if take_screenshot:
@@ -539,7 +573,7 @@ class PyAutoGUIExecutor(BaseExecutor):
             }  # Fallback for unsupported
             button_name = button_map.get(button, "left")
 
-            pyautogui.mouseUp(button=button_name)
+            self.pyautogui.mouseUp(button=button_name)
             result = ToolResult(output=f"Mouse up: {button} button")
 
             if take_screenshot:
@@ -559,9 +593,9 @@ class PyAutoGUIExecutor(BaseExecutor):
         try:
             # Map CLA key to PyAutoGUI key
             mapped_key = self._map_key(key)
-            pyautogui.keyDown(mapped_key)
+            self.pyautogui.keyDown(mapped_key)
             await asyncio.sleep(duration)
-            pyautogui.keyUp(mapped_key)
+            self.pyautogui.keyUp(mapped_key)
 
             result = ToolResult(output=f"Held key '{key}' for {duration} seconds")
 
@@ -579,7 +613,7 @@ class PyAutoGUIExecutor(BaseExecutor):
     async def position(self) -> ToolResult:
         """Get current cursor position."""
         try:
-            x, y = pyautogui.position()
+            x, y = self.pyautogui.position()
             return ToolResult(output=f"Mouse position: ({x}, {y})")
         except Exception as e:
             return ToolResult(error=str(e))

hud/tools/executors/tests/test_pyautogui_executor.py

@@ -7,7 +7,10 @@ from unittest.mock import AsyncMock, MagicMock, patch
 import pytest
 
 from hud.tools.base import ToolResult
-from hud.tools.executors.pyautogui import PYAUTOGUI_AVAILABLE, PyAutoGUIExecutor
+from hud.tools.executors.pyautogui import PyAutoGUIExecutor
+
+# Check if pyautogui is available for test skipping
+PYAUTOGUI_AVAILABLE = PyAutoGUIExecutor.is_available()
 
 
 class TestPyAutoGUIExecutor:

hud/tools/playwright_tool.py

@@ -21,8 +21,9 @@ logger = logging.getLogger(__name__)
 class PlaywrightTool:
     """Playwright tool for web automation."""
 
-    def __init__(self) -> None:
+    def __init__(self, cdp_url: str | None = None) -> None:
         super().__init__()
+        self._cdp_url = cdp_url
         self._playwright = None
         self._browser: Browser | None = None
         self._context: BrowserContext | None = None
@@ -46,9 +47,6 @@ class PlaywrightTool:
             None, description="CSS selector for element (for click, type, wait_for_element actions)"
         ),
         text: str | None = Field(None, description="Text to type (for type action)"),
-        path: str | None = Field(
-            None, description="File path to save screenshot (for screenshot action)"
-        ),
         wait_for_load_state: Literal["commit", "domcontentloaded", "load", "networkidle"]
         | None = Field(
             None,
@@ -74,7 +72,7 @@ class PlaywrightTool:
                 result = await self.navigate(url, wait_for_load_state or "networkidle")
 
             elif action == "screenshot":
-                result = await self.screenshot(path)
+                result = await self.screenshot()
 
             elif action == "click":
                 if selector is None:
@@ -119,17 +117,13 @@ class PlaywrightTool:
             # Convert dict result to ToolResult
             if isinstance(result, dict):
                 if result.get("success"):
-                    if "screenshot" in result:
-                        # Return screenshot as image content
-                        tool_result = ToolResult(
-                            output=result.get("message", ""), base64_image=result["screenshot"]
-                        )
-                    else:
-                        tool_result = ToolResult(output=result.get("message", ""))
+                    tool_result = ToolResult(output=result.get("message", ""))
                 else:
                     tool_result = ToolResult(error=result.get("error", "Unknown error"))
-            else:
+            elif isinstance(result, ToolResult):
                 tool_result = result
+            else:
+                tool_result = ToolResult(output=str(result))
 
             # Convert result to content blocks
             return tool_result_to_content_blocks(tool_result)
@@ -143,10 +137,14 @@ class PlaywrightTool:
     async def _ensure_browser(self) -> None:
         """Ensure browser is launched and ready."""
         if self._browser is None or not self._browser.is_connected():
-            logger.info("Launching Playwright browser...")
+            if self._cdp_url:
+                logger.info("Connecting to remote browser via CDP: %s", self._cdp_url)
+            else:
+                logger.info("Launching Playwright browser...")
 
-            # Ensure DISPLAY is set
-            os.environ["DISPLAY"] = os.environ.get("DISPLAY", ":1")
+            # Ensure DISPLAY is set (only needed for local browser)
+            if not self._cdp_url:
+                os.environ["DISPLAY"] = os.environ.get("DISPLAY", ":1")
 
             if self._playwright is None:
                 try:
@@ -158,37 +156,56 @@ class PlaywrightTool:
                         "Playwright is not installed. Please install with: pip install playwright"
                     ) from None
 
-            self._browser = await self._playwright.chromium.launch(
-                headless=False,
-                args=[
-                    "--no-sandbox",
-                    "--disable-dev-shm-usage",
-                    "--disable-gpu",
-                    "--disable-web-security",
-                    "--disable-features=IsolateOrigins,site-per-process",
-                    "--disable-blink-features=AutomationControlled",
-                    "--window-size=1920,1080",
-                    "--window-position=0,0",
-                    "--start-maximized",
-                    "--disable-background-timer-throttling",
-                    "--disable-backgrounding-occluded-windows",
-                    "--disable-renderer-backgrounding",
-                    "--disable-features=TranslateUI",
-                    "--disable-ipc-flooding-protection",
-                    "--disable-default-apps",
-                    "--no-first-run",
-                    "--disable-sync",
-                    "--no-default-browser-check",
-                ],
-            )
-
-            if self._browser is None:
-                raise RuntimeError("Browser failed to initialize")
-
-            self._context = await self._browser.new_context(
-                viewport={"width": 1920, "height": 1080},
-                ignore_https_errors=True,
-            )
+            # Connect via CDP URL or launch local browser
+            if self._cdp_url:
+                # Connect to remote browser via CDP
+                self._browser = await self._playwright.chromium.connect_over_cdp(self._cdp_url)
+
+                if self._browser is None:
+                    raise RuntimeError("Failed to connect to remote browser")
+
+                # Use existing context or create new one
+                contexts = self._browser.contexts
+                if contexts:
+                    self._context = contexts[0]
+                else:
+                    self._context = await self._browser.new_context(
+                        viewport={"width": 1920, "height": 1080},
+                        ignore_https_errors=True,
+                    )
+            else:
+                # Launch local browser
+                self._browser = await self._playwright.chromium.launch(
+                    headless=False,
+                    args=[
+                        "--no-sandbox",
+                        "--disable-dev-shm-usage",
+                        "--disable-gpu",
+                        "--disable-web-security",
+                        "--disable-features=IsolateOrigins,site-per-process",
+                        "--disable-blink-features=AutomationControlled",
+                        "--window-size=1920,1080",
+                        "--window-position=0,0",
+                        "--start-maximized",
+                        "--disable-background-timer-throttling",
+                        "--disable-backgrounding-occluded-windows",
+                        "--disable-renderer-backgrounding",
+                        "--disable-features=TranslateUI",
+                        "--disable-ipc-flooding-protection",
+                        "--disable-default-apps",
+                        "--no-first-run",
+                        "--disable-sync",
+                        "--no-default-browser-check",
+                    ],
+                )
+
+                if self._browser is None:
+                    raise RuntimeError("Browser failed to initialize")
+
+                self._context = await self._browser.new_context(
+                    viewport={"width": 1920, "height": 1080},
+                    ignore_https_errors=True,
+                )
 
             if self._context is None:
                 raise RuntimeError("Browser context failed to initialize")
@@ -234,35 +251,24 @@ class PlaywrightTool:
                 "message": f"Failed to navigate to {url}: {e}",
             }
 
-    async def screenshot(self, path: str | None = None) -> dict[str, Any]:
+    async def screenshot(self) -> ToolResult:
         """Take a screenshot of the current page.
 
-        Args:
-            path: Optional path to save screenshot
-
         Returns:
-            Dict with screenshot result
+            ToolResult with base64_image
         """
         await self._ensure_browser()
 
         try:
-            if path:
-                await self.page.screenshot(path=path, full_page=True)
-                return {"success": True, "path": path, "message": f"Screenshot saved to {path}"}
-            else:
-                # Return base64 encoded screenshot
-                screenshot_bytes = await self.page.screenshot(full_page=True)
-                import base64
-
-                screenshot_b64 = base64.b64encode(screenshot_bytes).decode()
-                return {
-                    "success": True,
-                    "screenshot": screenshot_b64,
-                    "message": "Screenshot captured",
-                }
+            # Always return base64 encoded screenshot as ToolResult
+            screenshot_bytes = await self.page.screenshot(full_page=True)
+            import base64
+
+            screenshot_b64 = base64.b64encode(screenshot_bytes).decode()
+            return ToolResult(base64_image=screenshot_b64)
         except Exception as e:
             logger.error("Screenshot failed: %s", e)
-            return {"success": False, "error": str(e), "message": f"Failed to take screenshot: {e}"}
+            return ToolResult(error=f"Failed to take screenshot: {e}")
 
     async def click(self, selector: str) -> dict[str, Any]:
         """Click an element by selector.

hud/tools/tests/test_edit.py

@@ -3,6 +3,7 @@
 from __future__ import annotations
 
 import os
+import sys
 import tempfile
 from pathlib import Path
 from unittest.mock import AsyncMock, patch
@@ -37,8 +38,14 @@ class TestEditTool:
         """Test validate_path when file doesn't exist for non-create commands."""
         tool = EditTool()
 
+        # Use a platform-appropriate absolute path
+        if sys.platform == "win32":
+            nonexistent_path = Path("C:\\nonexistent\\file.txt")
+        else:
+            nonexistent_path = Path("/nonexistent/file.txt")
+
         with pytest.raises(ToolError) as exc_info:
-            tool.validate_path("view", Path("/nonexistent/file.txt"))
+            tool.validate_path("view", nonexistent_path)
 
         assert "does not exist" in str(exc_info.value)
 

hud/tools/tests/test_tools.py

@@ -2,6 +2,7 @@ from __future__ import annotations
 
 import asyncio
 import inspect
+import sys
 
 import pytest
 from mcp.types import ImageContent, TextContent
@@ -72,6 +73,7 @@ async def test_bash_tool_restart_and_no_command():
 
 
 @pytest.mark.asyncio
+@pytest.mark.skipif(sys.platform == "win32", reason="EditTool uses Unix commands")
 async def test_edit_tool_flow(tmp_path):
     file_path = tmp_path / "demo.txt"
 
@@ -106,6 +108,7 @@ async def test_base_executor_simulation():
 
 
 @pytest.mark.asyncio
+@pytest.mark.skipif(sys.platform == "win32", reason="EditTool uses Unix commands")
 async def test_edit_tool_view(tmp_path):
     # Create a temporary file
     p = tmp_path / "sample.txt"

hud/trajectory.py

@@ -3,7 +3,6 @@ from __future__ import annotations
 
 import datetime
 
-from IPython.display import HTML, Markdown, display
 from pydantic import BaseModel, Field
 
 from .adapters.common.types import LogType
@@ -30,6 +29,11 @@ class Trajectory(BaseModel):
     trajectory: list[TrajectoryStep] = Field(default_factory=list)
 
     def display(self) -> None:
+        try:
+            from IPython.display import HTML, Markdown, display
+        except ImportError:
+            raise ImportError("IPython is required for trajectory display") from None
+
         trajectory_start_timestamp_str = self.trajectory[0].start_timestamp
         t_start_dt = (
             datetime.datetime.fromisoformat(trajectory_start_timestamp_str.replace("Z", "+00:00"))

hud/utils/tests/test_version.py

@@ -5,4 +5,4 @@ def test_import():
     """Test that the package can be imported."""
     import hud
 
-    assert hud.__version__ == "0.3.0"
+    assert hud.__version__ == "0.3.1"

hud/version.py

@@ -4,4 +4,4 @@ Version information for the HUD SDK.
 
 from __future__ import annotations
 
-__version__ = "0.3.0"
+__version__ = "0.3.1"