agentscope-runtime 1.0.4a1__py3-none-any.whl → 1.0.5.post1__py3-none-any.whl

agentscope_runtime/sandbox/box/gui/gui_sandbox.py

@@ -8,7 +8,7 @@ from urllib.parse import urljoin, urlencode
 from ...utils import build_image_uri, get_platform
 from ...registry import SandboxRegistry
 from ...enums import SandboxType
-from ...box.base import BaseSandbox
+from ...box.base import BaseSandbox, BaseSandboxAsync
 from ...constant import TIMEOUT
 
 logger = logging.getLogger(__name__)
@@ -34,6 +34,34 @@ class GUIMixin:
         )
 
 
+class AsyncGUIMixin:
+    async def get_desktop_url_async(self):
+        # Check sandbox health asynchronously
+        is_healthy = await self.manager_api.check_health_async(
+            identity=self.sandbox_id,
+        )
+        if not is_healthy:
+            raise RuntimeError(f"Sandbox {self.sandbox_id} is not healthy")
+
+        # Retrieve container information asynchronously
+        info = await self.get_info_async()
+
+        # Default local VNC path and remote VNC relay path
+        path = "/vnc/vnc_lite.html"
+        remote_path = "/vnc/vnc_relay.html"
+        params = {"password": info["runtime_token"]}
+
+        # If base_url is not set, construct the local URL
+        if self.base_url is None:
+            return urljoin(info["url"], path) + "?" + urlencode(params)
+
+        # Construct the remote URL with sandbox ID and VNC relay path
+        return (
+            f"{self.base_url}/desktop/{self.sandbox_id}{remote_path}"
+            f"?{urlencode(params)}"
+        )
+
+
 @SandboxRegistry.register(
     build_image_uri("runtime-sandbox-gui"),
     sandbox_type=SandboxType.GUI,
@@ -121,3 +149,91 @@ class GuiSandbox(GUIMixin, BaseSandbox):
             payload["text"] = text
 
         return self.call_tool("computer", payload)
+
+
+@SandboxRegistry.register(
+    build_image_uri("runtime-sandbox-gui"),
+    sandbox_type=SandboxType.GUI_ASYNC,
+    security_level="high",
+    timeout=TIMEOUT,
+    description="GUI Sandbox (Async)",
+)
+class GuiSandboxAsync(GUIMixin, AsyncGUIMixin, BaseSandboxAsync):
+    def __init__(  # pylint: disable=useless-parent-delegation
+        self,
+        sandbox_id: Optional[str] = None,
+        timeout: int = 3000,
+        base_url: Optional[str] = None,
+        bearer_token: Optional[str] = None,
+        sandbox_type: SandboxType = SandboxType.GUI_ASYNC,
+    ):
+        super().__init__(
+            sandbox_id,
+            timeout,
+            base_url,
+            bearer_token,
+            sandbox_type,
+        )
+        # Architecture compatibility warning
+        if get_platform() == "linux/arm64":
+            logger.warning(
+                "\nCompatibility Notice: This GUI Sandbox may have issues on "
+                "arm64 CPU architectures, due to the computer-use-mcp not "
+                "providing linux/arm64 compatibility. It has been tested "
+                "on Apple M4 chips with Rosetta enabled. However, on M1, M2, "
+                "and M3 chips, Chromium browser might crash due to the missing "
+                "SSE3 instruction set.",
+            )
+
+    async def computer_use(
+        self,
+        action: str,
+        coordinate: Optional[Union[List[float], Tuple[float, float]]] = None,
+        text: Optional[str] = None,
+    ):
+        """
+        Asynchronously use mouse and keyboard to interact with a desktop GUI.
+
+        This method interfaces with the sandbox's GUI environment.
+        You do not have access to a terminal or applications menu;
+        interaction is performed by clicking on desktop icons or using
+        keyboard shortcuts.
+
+        Guidelines:
+            * Prefer keyboard shortcuts where possible over cursor actions.
+            * If visual keyboard hints (two-letter boxes) are shown, typing
+              those letters will click the element — use this where possible.
+            * Applications or actions may require waiting; e.g., repeat
+              screenshots if windows don’t open immediately.
+            * Always determine cursor coordinates using screenshots before moving
+              the cursor to click on elements.
+            * If clicks fail to load content, try adjusting cursor coordinates to
+              center on the target element.
+            * Click with the cursor tip centered on elements, not on edges.
+
+        Args:
+            action (str): The action to perform. Options include:
+                * `key` — Press a key or key combination.
+                * `type` — Type a string of text.
+                * `get_cursor_position` — Get current cursor coordinates (x, y).
+                * `mouse_move` — Move cursor to given (x, y).
+                * `left_click` — Left mouse click.
+                * `left_click_drag` — Click and drag to given (x, y).
+                * `right_click` — Right mouse click.
+                * `middle_click` — Middle mouse click.
+                * `double_click` — Double left mouse click.
+                * `get_screenshot` — Capture screen screenshot.
+            coordinate (list[float] | tuple[float, float], optional):
+                Pixel coordinates (x from left edge, y from top edge).
+            text (str, optional): String to type, or key-combination for `key` action.
+
+        Returns:
+            Any: Tool execution result from the sandbox.
+        """
+        payload = {"action": action}
+        if coordinate is not None:
+            payload["coordinate"] = coordinate
+        if text is not None:
+            payload["text"] = text
+
+        return await self.call_tool_async("computer", payload)

agentscope_runtime/sandbox/box/mobile/__init__.py

@@ -1,4 +1,4 @@
 # -*- coding: utf-8 -*-
-from .mobile_sandbox import MobileSandbox
+from .mobile_sandbox import MobileSandbox, MobileSandboxAsync
 
-__all__ = ["MobileSandbox"]
+__all__ = ["MobileSandbox", "MobileSandboxAsync"]

agentscope_runtime/sandbox/box/mobile/mobile_sandbox.py

@@ -5,7 +5,7 @@ import platform
 from typing import Optional, List, Union
 from urllib.parse import urlencode, urljoin
 
-from agentscope_runtime.sandbox.box.sandbox import Sandbox
+from ..sandbox import Sandbox, SandboxAsync
 
 from ...utils import build_image_uri
 from ...registry import SandboxRegistry
@@ -15,6 +15,107 @@ from ...constant import TIMEOUT
 logger = logging.getLogger(__name__)
 
 
+class HostPrerequisiteError(Exception):
+    """Exception raised when host prerequisites
+    for MobileSandbox are not met."""
+
+
+def _check_host_readiness() -> None:
+    logger.info(
+        "Performing one-time host environment check for MobileSandbox...",
+    )
+
+    architecture = platform.machine().lower()
+    if architecture in ("aarch64", "arm64"):
+        logger.warning(
+            "\n======================== WARNING ========================\n"
+            "ARM64/aarch64 architecture detected (e.g., Apple M-series).\n"
+            "Running this mobile sandbox on a non-x86_64 host may lead \n"
+            " to unexpected compatibility or performance issues.\n"
+            "=========================================================",
+        )
+
+    os_type = platform.system()
+    if os_type == "Linux":
+        try:
+            result = subprocess.run(
+                ["lsmod"],
+                capture_output=True,
+                text=True,
+                check=True,
+            )
+            loaded_modules = result.stdout
+        except (FileNotFoundError, subprocess.CalledProcessError):
+            loaded_modules = ""
+            logger.warning(
+                "Could not execute 'lsmod' to verify kernel modules.",
+            )
+
+        if "binder_linux" not in loaded_modules:
+            error_message = (
+                "\n========== HOST PREREQUISITE FAILED ==========\n"
+                "MobileSandbox requires specific kernel modules"
+                " that appear to be missing or not loaded.\n\n"
+                "To fix this, please run the following commands"
+                " on your Linux host:\n\n"
+                "## Install required kernel modules\n"
+                "sudo apt update"
+                " && sudo apt install -y linux-modules-extra-`uname -r`\n"
+                "sudo modprobe binder_linux"
+                ' devices="binder,hwbinder,vndbinder"\n'
+                "## (Optional) Load the ashmem driver for older kernels\n"
+                "sudo modprobe ashmem_linux\n"
+                "=================================================="
+            )
+            raise HostPrerequisiteError(error_message)
+
+    if os_type == "Windows":
+        try:
+            result = subprocess.run(
+                ["wsl", "lsmod"],
+                capture_output=True,
+                text=True,
+                check=True,
+                encoding="utf-8",
+            )
+            loaded_modules = result.stdout
+        except (FileNotFoundError, subprocess.CalledProcessError):
+            loaded_modules = ""
+            logger.warning(
+                "Could not execute 'wsl lsmod' to verify kernel modules.",
+            )
+
+        if "binder_linux" not in loaded_modules:
+            error_message = (
+                "\n========== HOST PREREQUISITE FAILED ==========\n"
+                "MobileSandbox on Windows requires Docker Desktop "
+                "with the WSL 2 backend.\n"
+                "The required kernel modules seem to be missing "
+                "in your WSL 2 environment.\n\n"
+                "To fix this, please follow these steps:\n\n"
+                "1. **Ensure Docker Desktop is using WSL 2**:\n"
+                "   - Open Docker Desktop -> Settings -> General.\n"
+                "   - Make sure 'Use the WSL 2 based engine' "
+                "is checked.\n\n"
+                "2. **Ensure WSL is installed and updated**:\n"
+                "   - Open PowerShell or Command Prompt "
+                "as Administrator.\n"
+                "   - Run: wsl --install\n"
+                "   - Run: wsl --update\n"
+                "   (An update usually installs a recent Linux kernel "
+                "with the required modules.)\n\n"
+                "3. **Verify manually (Optional)**:\n"
+                "   - After updating, run 'wsl lsmod | findstr binder' "
+                "in your terminal.\n"
+                "   - If it shows 'binder_linux', "
+                "the issue should be resolved.\n"
+                "=================================================="
+            )
+            raise HostPrerequisiteError(error_message)
+
+    logger.info("Host environment check passed.")
+
+
 class MobileMixin:
     @property
     def mobile_url(self):
@@ -41,9 +142,41 @@ class MobileMixin:
         )
 
 
-class HostPrerequisiteError(Exception):
-    """Exception raised when host prerequisites
-    for MobileSandbox are not met."""
+class AsyncMobileMixin:
+    async def get_mobile_url_async(self):
+        """
+        Asynchronously retrieve the mobile VNC/websockify connection URL.
+
+        Returns:
+            str: Fully qualified URL to access the mobile sandbox UI remotely.
+
+        Raises:
+            RuntimeError: If the sandbox is not healthy.
+        """
+        # Check health asynchronously
+        is_healthy = await self.manager_api.check_health_async(
+            identity=self.sandbox_id,
+        )
+        if not is_healthy:
+            raise RuntimeError(f"Sandbox {self.sandbox_id} is not healthy")
+
+        # Get container info asynchronously
+        info = await self.get_info_async()
+
+        # Local path and remote path (currently the same)
+        path = "/websockify/"
+        remote_path = "/websockify/"
+        params = {"password": info["runtime_token"]}
+
+        # Local URL if base_url is not set
+        if self.base_url is None:
+            return urljoin(info["url"], path) + "?" + urlencode(params)
+
+        # Remote URL
+        return (
+            f"{self.base_url}/desktop/{self.sandbox_id}{remote_path}"
+            f"?{urlencode(params)}"
+        )
 
 
 @SandboxRegistry.register(
@@ -66,7 +199,7 @@ class MobileSandbox(MobileMixin, Sandbox):
         sandbox_type: SandboxType = SandboxType.MOBILE,
     ):
         if base_url is None and not self.__class__._host_check_done:
-            self._check_host_readiness()
+            _check_host_readiness()
             self.__class__._host_check_done = True
 
         super().__init__(
@@ -77,101 +210,6 @@ class MobileSandbox(MobileMixin, Sandbox):
             sandbox_type,
         )
 
-    def _check_host_readiness(self) -> None:
-        logger.info(
-            "Performing one-time host environment check for MobileSandbox...",
-        )
-
-        architecture = platform.machine().lower()
-        if architecture in ("aarch64", "arm64"):
-            logger.warning(
-                "\n======================== WARNING ========================\n"
-                "ARM64/aarch64 architecture detected (e.g., Apple M-series).\n"
-                "Running this mobile sandbox on a non-x86_64 host may lead \n"
-                " to unexpected compatibility or performance issues.\n"
-                "=========================================================",
-            )
-
-        os_type = platform.system()
-        if os_type == "Linux":
-            try:
-                result = subprocess.run(
-                    ["lsmod"],
-                    capture_output=True,
-                    text=True,
-                    check=True,
-                )
-                loaded_modules = result.stdout
-            except (FileNotFoundError, subprocess.CalledProcessError):
-                loaded_modules = ""
-                logger.warning(
-                    "Could not execute 'lsmod' to verify kernel modules.",
-                )
-
-            if "binder_linux" not in loaded_modules:
-                error_message = (
-                    "\n========== HOST PREREQUISITE FAILED ==========\n"
-                    "MobileSandbox requires specific kernel modules"
-                    " that appear to be missing or not loaded.\n\n"
-                    "To fix this, please run the following commands"
-                    " on your Linux host:\n\n"
-                    "## Install required kernel modules\n"
-                    "sudo apt update"
-                    " && sudo apt install -y linux-modules-extra-`uname -r`\n"
-                    "sudo modprobe binder_linux"
-                    ' devices="binder,hwbinder,vndbinder"\n'
-                    "## (Optional) Load the ashmem driver for older kernels\n"
-                    "sudo modprobe ashmem_linux\n"
-                    "=================================================="
-                )
-                raise HostPrerequisiteError(error_message)
-
-        if os_type == "Windows":
-            try:
-                result = subprocess.run(
-                    ["wsl", "lsmod"],
-                    capture_output=True,
-                    text=True,
-                    check=True,
-                    encoding="utf-8",
-                )
-                loaded_modules = result.stdout
-            except (FileNotFoundError, subprocess.CalledProcessError):
-                loaded_modules = ""
-                logger.warning(
-                    "Could not execute 'wsl lsmod' to verify kernel modules.",
-                )
-
-            if "binder_linux" not in loaded_modules:
-                error_message = (
-                    "\n========== HOST PREREQUISITE FAILED ==========\n"
-                    "MobileSandbox on Windows requires Docker Desktop "
-                    "with the WSL 2 backend.\n"
-                    "The required kernel modules seem to be missing "
-                    "in your WSL 2 environment.\n\n"
-                    "To fix this, please follow these steps:\n\n"
-                    "1. **Ensure Docker Desktop is using WSL 2**:\n"
-                    "   - Open Docker Desktop -> Settings -> General.\n"
-                    "   - Make sure 'Use the WSL 2 based engine' "
-                    "is checked.\n\n"
-                    "2. **Ensure WSL is installed and updated**:\n"
-                    "   - Open PowerShell or Command Prompt "
-                    "as Administrator.\n"
-                    "   - Run: wsl --install\n"
-                    "   - Run: wsl --update\n"
-                    "   (An update usually installs a recent Linux kernel "
-                    "with the required modules.)\n\n"
-                    "3. **Verify manually (Optional)**:\n"
-                    "   - After updating, run 'wsl lsmod | findstr binder' "
-                    "in your terminal.\n"
-                    "   - If it shows 'binder_linux', "
-                    "the issue should be resolved.\n"
-                    "=================================================="
-                )
-                raise HostPrerequisiteError(error_message)
-
-        logger.info("Host environment check passed.")
-
     def adb_use(
         self,
         action: str,
@@ -288,3 +326,112 @@ class MobileSandbox(MobileMixin, Sandbox):
     def mobile_get_screenshot(self):
         """Take a screenshot of the current device screen."""
         return self.call_tool("adb", {"action": "get_screenshot"})
+
+
+@SandboxRegistry.register(
+    build_image_uri("runtime-sandbox-mobile"),
+    sandbox_type=SandboxType.MOBILE_ASYNC,
+    security_level="high",
+    timeout=TIMEOUT,
+    description="Mobile Sandbox (Async)",
+    runtime_config={"privileged": True},
+)
+class MobileSandboxAsync(MobileMixin, AsyncMobileMixin, SandboxAsync):
+    _host_check_done = False
+
+    def __init__(
+        self,
+        sandbox_id: Optional[str] = None,
+        timeout: int = 3000,
+        base_url: Optional[str] = None,
+        bearer_token: Optional[str] = None,
+        sandbox_type: SandboxType = SandboxType.MOBILE_ASYNC,
+    ):
+        if base_url is None and not self.__class__._host_check_done:
+            _check_host_readiness()
+            self.__class__._host_check_done = True
+
+        super().__init__(
+            sandbox_id,
+            timeout,
+            base_url,
+            bearer_token,
+            sandbox_type,
+        )
+
+    async def adb_use(
+        self,
+        action: str,
+        coordinate: Optional[List[int]] = None,
+        start: Optional[List[int]] = None,
+        end: Optional[List[int]] = None,
+        duration: Optional[int] = None,
+        code: Optional[Union[int, str]] = None,
+        text: Optional[str] = None,
+    ):
+        """
+        Asynchronously execute a general-purpose ADB action.
+        """
+        payload = {"action": action}
+        if coordinate is not None:
+            payload["coordinate"] = coordinate
+        if start is not None:
+            payload["start"] = start
+        if end is not None:
+            payload["end"] = end
+        if duration is not None:
+            payload["duration"] = duration
+        if code is not None:
+            payload["code"] = code
+        if text is not None:
+            payload["text"] = text
+
+        return await self.call_tool_async("adb", payload)
+
+    async def mobile_get_screen_resolution(self):
+        """Asynchronously get the screen resolution."""
+        return await self.call_tool_async(
+            "adb",
+            {"action": "get_screen_resolution"},
+        )
+
+    async def mobile_tap(self, coordinate: List[int]):
+        """Asynchronously tap specific screen coordinates."""
+        return await self.call_tool_async(
+            "adb",
+            {"action": "tap", "coordinate": coordinate},
+        )
+
+    async def mobile_swipe(
+        self,
+        start: List[int],
+        end: List[int],
+        duration: Optional[int] = None,
+    ):
+        """Asynchronously perform a swipe gesture."""
+        payload = {
+            "action": "swipe",
+            "start": start,
+            "end": end,
+        }
+        if duration is not None:
+            payload["duration"] = duration
+        return await self.call_tool_async("adb", payload)
+
+    async def mobile_input_text(self, text: str):
+        """Asynchronously input text into the focused UI element."""
+        return await self.call_tool_async(
+            "adb",
+            {"action": "input_text", "text": text},
+        )
+
+    async def mobile_key_event(self, code: Union[int, str]):
+        """Asynchronously send a key event to the device."""
+        return await self.call_tool_async(
+            "adb",
+            {"action": "key_event", "code": code},
+        )
+
+    async def mobile_get_screenshot(self):
+        """Asynchronously take a screenshot."""
+        return await self.call_tool_async("adb", {"action": "get_screenshot"})