minitap-mobile-use 0.0.1.dev0__py3-none-any.whl

minitap/mobile_use/servers/stop_servers.py

@@ -0,0 +1,215 @@
+import sys
+import time
+from typing import List
+
+import psutil
+import requests
+from minitap.mobile_use.servers.config import server_settings
+from minitap.mobile_use.servers.device_hardware_bridge import DEVICE_HARDWARE_BRIDGE_PORT
+from minitap.mobile_use.utils.logger import get_server_logger
+
+logger = get_server_logger()
+
+
+def find_processes_by_name(name: str) -> List[psutil.Process]:
+    """Find all processes with the given name."""
+    processes = []
+    for proc in psutil.process_iter(["pid", "name", "cmdline"]):
+        try:
+            if name.lower() in proc.info["name"].lower():
+                processes.append(proc)
+            elif proc.info["cmdline"] and any(name in cmd for cmd in proc.info["cmdline"]):
+                processes.append(proc)
+        except (psutil.NoSuchProcess, psutil.AccessDenied):
+            continue
+    return processes
+
+
+def find_processes_by_port(port: int) -> List[psutil.Process]:
+    processes = []
+    for proc in psutil.process_iter(["pid", "name"]):
+        try:
+            for conn in proc.net_connections():
+                if conn.laddr.port == port and conn.status == psutil.CONN_LISTEN:
+                    processes.append(proc)
+                    break
+        except (psutil.NoSuchProcess, psutil.AccessDenied, psutil.ZombieProcess):
+            continue
+    return processes
+
+
+def stop_process_gracefully(process: psutil.Process, timeout: int = 5) -> bool:
+    try:
+        if not process.is_running():
+            logger.success(f"Process {process.pid} ({process.name()}) already terminated")
+            return True
+
+        logger.info(f"Stopping process {process.pid} ({process.name()})")
+
+        process.terminate()
+
+        try:
+            process.wait(timeout=timeout)
+            logger.success(f"Process {process.pid} terminated gracefully")
+            return True
+        except psutil.TimeoutExpired:
+            logger.warning(f"Process {process.pid} didn't terminate gracefully, force killing...")
+            try:
+                process.kill()
+                process.wait(timeout=2)
+                logger.success(f"Process {process.pid} force killed")
+                return True
+            except psutil.NoSuchProcess:
+                logger.success(f"Process {process.pid} already terminated during force kill")
+                return True
+
+    except psutil.NoSuchProcess:
+        logger.success(f"Process {process.pid} no longer exists (already terminated)")
+        return True
+    except (psutil.AccessDenied, psutil.ZombieProcess) as e:
+        logger.warning(f"Cannot stop process {process.pid}: {e}")
+        return False
+    except Exception as e:
+        logger.error(f"Unexpected error stopping process {process.pid}: {e}")
+        return False
+
+
+def check_service_health(port: int, service_name: str) -> bool:
+    try:
+        if port == server_settings.DEVICE_SCREEN_API_PORT:
+            response = requests.get(f"http://localhost:{port}/health", timeout=2)
+        elif port == DEVICE_HARDWARE_BRIDGE_PORT:
+            response = requests.get(f"http://localhost:{port}/api/banner-message", timeout=2)
+        else:
+            return False
+
+        if response.status_code == 200:
+            logger.warning(f"{service_name} is still responding on port {port}")
+            return True
+    except requests.exceptions.RequestException:
+        pass
+
+    return False
+
+
+def stop_device_screen_api() -> bool:
+    logger.info("Stopping Device Screen API...")
+    api_port = server_settings.DEVICE_SCREEN_API_PORT
+
+    if not check_service_health(api_port, "Device Screen API"):
+        logger.success("Device Screen API is not running")
+        return True
+
+    # Find processes by port
+    processes = find_processes_by_port(api_port)
+
+    # Also find by process name/command
+    uvicorn_processes = find_processes_by_name("uvicorn")
+    python_processes = find_processes_by_name("device_screen_api.py")
+
+    all_processes = list(set(processes + uvicorn_processes + python_processes))
+
+    if not all_processes:
+        logger.warning("No Device Screen API processes found, but service is still responding")
+        # Still try to verify if service actually stops
+        time.sleep(1)
+        if not check_service_health(api_port, "Device Screen API"):
+            logger.success("Device Screen API stopped successfully (was orphaned)")
+            return True
+        return False
+
+    # Stop all processes
+    for proc in all_processes:
+        stop_process_gracefully(proc)
+
+    # Verify service is stopped
+    time.sleep(1)
+    if check_service_health(api_port, "Device Screen API"):
+        logger.error("Device Screen API is still running after stop attempt")
+        return False
+
+    logger.success("Device Screen API stopped successfully")
+    return True
+
+
+def stop_device_hardware_bridge() -> bool:
+    logger.info("Stopping Device Hardware Bridge...")
+
+    if not check_service_health(DEVICE_HARDWARE_BRIDGE_PORT, "Maestro Studio"):
+        logger.success("Device Hardware Bridge is not running")
+        return True
+
+    processes = find_processes_by_port(DEVICE_HARDWARE_BRIDGE_PORT)
+
+    maestro_processes = find_processes_by_name("maestro")
+
+    all_processes = list(set(processes + maestro_processes))
+
+    if not all_processes:
+        logger.warning("No Device Hardware Bridge processes found, but service is still responding")
+        # Still try to verify if service actually stops
+        time.sleep(1)
+        if not check_service_health(DEVICE_HARDWARE_BRIDGE_PORT, "Maestro Studio"):
+            logger.success("Device Hardware Bridge stopped successfully (was orphaned)")
+            return True
+        return False
+
+    for proc in all_processes:
+        stop_process_gracefully(proc)
+
+    time.sleep(1)
+    if check_service_health(DEVICE_HARDWARE_BRIDGE_PORT, "Maestro Studio"):
+        logger.error("Device Hardware Bridge is still running after stop attempt")
+        return False
+
+    logger.success("Device Hardware Bridge stopped successfully")
+    return True
+
+
+def stop_servers(
+    device_screen_api: bool = False, device_hardware_bridge: bool = False
+) -> tuple[bool, bool]:
+    """Stop all servers and return (api_success, bridge_success).
+
+    Args:
+        device_screen_api: If True, stop the Device Screen API
+        device_hardware_bridge: If True, stop the Device Hardware Bridge
+
+    Returns:
+        Tuple of (api_stopped, bridge_stopped) booleans
+    """
+    api_success = stop_device_screen_api() if device_screen_api else True
+    bridge_success = stop_device_hardware_bridge() if device_hardware_bridge else True
+
+    if api_success and bridge_success:
+        logger.success("All servers stopped successfully")
+    elif api_success:
+        logger.warning("Device Screen API stopped, but Device Hardware Bridge had issues")
+    elif bridge_success:
+        logger.warning("Device Hardware Bridge stopped, but Device Screen API had issues")
+    else:
+        logger.error("Failed to stop both servers")
+
+    return api_success, bridge_success
+
+
+def main():
+    """Main function to stop all servers."""
+    api_success, bridge_success = stop_servers(device_screen_api=True, device_hardware_bridge=True)
+    if api_success and bridge_success:
+        return 0
+    elif api_success or bridge_success:
+        return 1
+    else:
+        return 2
+
+
+if __name__ == "__main__":
+    try:
+        sys.exit(main())
+    except KeyboardInterrupt:
+        logger.warning("\nStop operation interrupted by user")
+        sys.exit(130)
+    except Exception as e:
+        logger.error(f"Unexpected error: {e}")
+        sys.exit(1)

minitap/mobile_use/servers/utils.py

@@ -0,0 +1,11 @@
+import psutil
+
+
+def is_port_in_use(port: int):
+    for conn in psutil.net_connections():
+        if conn.status == psutil.CONN_LISTEN and conn.laddr:
+            if hasattr(conn.laddr, "port") and conn.laddr.port == port:
+                return True
+            elif isinstance(conn.laddr, tuple) and len(conn.laddr) >= 2 and conn.laddr[1] == port:
+                return True
+    return False

minitap/mobile_use/services/accessibility.py

@@ -0,0 +1,100 @@
+import asyncio
+import logging
+import re
+
+# Set up basic logging
+logging.basicConfig(level=logging.INFO)
+logger = logging.getLogger(__name__)
+
+
+async def run_subprocess(command: str) -> tuple[str, str]:
+    """
+    Executes a shell command in a subprocess.
+
+    Args:
+        command: The command to execute.
+
+    Returns:
+        A tuple containing the stdout and stderr of the command.
+    """
+    process = await asyncio.create_subprocess_shell(
+        command,
+        stdout=asyncio.subprocess.PIPE,
+        stderr=asyncio.subprocess.PIPE,
+    )
+    stdout, stderr = await process.communicate()
+    return stdout.decode(errors="ignore"), stderr.decode(errors="ignore")
+
+
+async def get_accessibility_tree(device_id: str | None = None) -> str:
+    """
+    Retrieves the UI accessibility tree from an Android device as an XML string.
+
+    This function uses `uiautomator` to dump the current UI hierarchy.
+
+    Args:
+        device_id: The optional ID of the target device. If not provided,
+                   the command will run on the only connected device.
+
+    Returns:
+        The UI hierarchy as an XML string.
+        Returns an empty string if the command fails.
+    """
+    adb_command = "adb"
+    if device_id:
+        adb_command = f"adb -s {device_id}"
+
+    # The '/dev/tty' trick is used to get the raw XML output directly.
+    # On some devices, '/dev/null' or a temporary file might be needed.
+    command = f"{adb_command} shell uiautomator dump /dev/tty"
+
+    logger.info(f"Executing command: {command}")
+
+    try:
+        stdout, stderr = await run_subprocess(command)
+
+        if "UI hierchary dumped to" in stderr:  # Mispelling is in the original tool
+            # The XML is often in stdout, but sometimes mixed with stderr
+            # We'll clean it up to ensure we only get the XML part.
+            xml_output = re.sub(r"UI hierchary dumped to.*", "", stderr, flags=re.DOTALL).strip()
+            if not xml_output.startswith("<?xml"):
+                xml_output = stdout
+
+            # Clean up potential non-XML text at the beginning
+            xml_start_index = xml_output.find("<?xml")
+            if xml_start_index != -1:
+                return xml_output[xml_start_index:].strip()
+            else:
+                logger.error("Could not find XML content in the output.")
+                return ""
+
+        elif "ERROR" in stderr:
+            logger.error(f"Failed to get accessibility tree: {stderr.strip()}")
+            return ""
+
+        return stdout.strip()
+
+    except Exception as e:
+        logger.error(f"An exception occurred while getting the accessibility tree: {e}")
+        return ""
+
+
+# Example of how to run this function
+async def main():
+    print("Attempting to retrieve accessibility tree from the connected device...")
+    # You can specify a device_id like "emulator-5554" if you have multiple devices
+    accessibility_tree = await get_accessibility_tree()
+
+    if accessibility_tree:
+        print("\n--- Accessibility Tree XML ---")
+        print(accessibility_tree)
+        print("\n----------------------------")
+    else:
+        print("\nFailed to retrieve the accessibility tree.")
+
+
+if __name__ == "__main__":
+    # To run this example, save it as a Python file (e.g., `get_tree.py`)
+    # and run `python get_tree.py` in your terminal.
+    # Make sure you have an Android device connected with ADB enabled.
+    asyncio.run(main())

minitap/mobile_use/services/llm.py

@@ -0,0 +1,143 @@
+import logging
+from typing import Awaitable, Callable, Literal, TypeVar
+from typing_extensions import overload
+
+from langchain_google_genai import ChatGoogleGenerativeAI
+from langchain_openai import ChatOpenAI
+from minitap.mobile_use.config import (
+    AgentNode,
+    AgentNodeWithFallback,
+    LLMUtilsNode,
+    LLMWithFallback,
+    settings,
+)
+from minitap.mobile_use.context import MobileUseContext
+
+logger = logging.getLogger(__name__)
+
+
+def get_google_llm(
+    model_name: str = "gemini-2.5-pro",
+    temperature: float = 0.7,
+) -> ChatGoogleGenerativeAI:
+    assert settings.GOOGLE_API_KEY is not None
+    client = ChatGoogleGenerativeAI(
+        model=model_name,
+        max_tokens=None,
+        temperature=temperature,
+        api_key=settings.GOOGLE_API_KEY,
+        max_retries=2,
+    )
+    return client
+
+
+def get_openai_llm(
+    model_name: str = "o3",
+    temperature: float = 1,
+) -> ChatOpenAI:
+    assert settings.OPENAI_API_KEY is not None
+    client = ChatOpenAI(
+        model=model_name,
+        api_key=settings.OPENAI_API_KEY,
+        temperature=temperature,
+    )
+    return client
+
+
+def get_openrouter_llm(model_name: str, temperature: float = 1):
+    assert settings.OPEN_ROUTER_API_KEY is not None
+    client = ChatOpenAI(
+        model=model_name,
+        temperature=temperature,
+        api_key=settings.OPEN_ROUTER_API_KEY,
+        base_url="https://openrouter.ai/api/v1",
+    )
+    return client
+
+
+def get_grok_llm(model_name: str, temperature: float = 1) -> ChatOpenAI:
+    assert settings.XAI_API_KEY is not None
+    client = ChatOpenAI(
+        model=model_name,
+        api_key=settings.XAI_API_KEY,
+        temperature=temperature,
+        base_url="https://api.x.ai/v1",
+    )
+    return client
+
+
+@overload
+def get_llm(
+    ctx: MobileUseContext,
+    name: AgentNodeWithFallback,
+    *,
+    use_fallback: bool = False,
+    temperature: float = 1,
+): ...
+
+
+@overload
+def get_llm(
+    ctx: MobileUseContext,
+    name: AgentNode,
+    *,
+    temperature: float = 1,
+): ...
+
+
+@overload
+def get_llm(
+    ctx: MobileUseContext,
+    name: LLMUtilsNode,
+    *,
+    is_utils: Literal[True],
+    temperature: float = 1,
+): ...
+
+
+def get_llm(
+    ctx: MobileUseContext,
+    name: AgentNode | LLMUtilsNode | AgentNodeWithFallback,
+    is_utils: bool = False,
+    use_fallback: bool = False,
+    temperature: float = 1,
+):
+    llm = (
+        ctx.llm_config.get_utils(name)  # type: ignore
+        if is_utils
+        else ctx.llm_config.get_agent(name)  # type: ignore
+    )
+    if use_fallback:
+        if isinstance(llm, LLMWithFallback):
+            llm = llm.fallback
+        else:
+            raise ValueError("LLM has no fallback!")
+    if llm.provider == "openai":
+        return get_openai_llm(llm.model, temperature)
+    elif llm.provider == "google":
+        return get_google_llm(llm.model, temperature)
+    elif llm.provider == "openrouter":
+        return get_openrouter_llm(llm.model, temperature)
+    elif llm.provider == "xai":
+        return get_grok_llm(llm.model, temperature)
+    else:
+        raise ValueError(f"Unsupported provider: {llm.provider}")
+
+
+T = TypeVar("T")
+
+
+async def with_fallback(
+    main_call: Callable[[], Awaitable[T]],
+    fallback_call: Callable[[], Awaitable[T]],
+    none_should_fallback: bool = True,
+) -> T:
+    try:
+        result = await main_call()
+        if result is None and none_should_fallback:
+            logger.warning("Main LLM inference returned None. Falling back...")
+            return await fallback_call()
+        return result
+    except Exception as e:
+        logger.warning(f"❗ Main LLM inference failed: {e}. Falling back...")
+        return await fallback_call()

minitap/mobile_use/tools/index.py

@@ -0,0 +1,54 @@
+from langchain_core.tools import BaseTool
+from minitap.mobile_use.context import MobileUseContext
+from minitap.mobile_use.tools.mobile.back import back_wrapper
+from minitap.mobile_use.tools.mobile.copy_text_from import copy_text_from_wrapper
+from minitap.mobile_use.tools.mobile.erase_text import erase_text_wrapper
+from minitap.mobile_use.tools.mobile.input_text import input_text_wrapper
+from minitap.mobile_use.tools.mobile.launch_app import launch_app_wrapper
+from minitap.mobile_use.tools.mobile.list_packages import list_packages_wrapper
+from minitap.mobile_use.tools.mobile.long_press_on import long_press_on_wrapper
+from minitap.mobile_use.tools.mobile.open_link import open_link_wrapper
+from minitap.mobile_use.tools.mobile.paste_text import paste_text_wrapper
+from minitap.mobile_use.tools.mobile.press_key import press_key_wrapper
+
+# from minitap.mobile_use.tools.mobile.run_flow import run_flow_wrapper
+from minitap.mobile_use.tools.mobile.stop_app import stop_app_wrapper
+from minitap.mobile_use.tools.mobile.swipe import swipe_wrapper
+from minitap.mobile_use.tools.mobile.take_screenshot import take_screenshot_wrapper
+from minitap.mobile_use.tools.mobile.tap import tap_wrapper
+from minitap.mobile_use.tools.mobile.wait_for_animation_to_end import (
+    wait_for_animation_to_end_wrapper,
+)
+from minitap.mobile_use.tools.tool_wrapper import ToolWrapper
+
+EXECUTOR_WRAPPERS_TOOLS = [
+    back_wrapper,
+    open_link_wrapper,
+    tap_wrapper,
+    long_press_on_wrapper,
+    swipe_wrapper,
+    take_screenshot_wrapper,
+    # run_flow_wrapper, # To decomment when subflow is implemented
+    copy_text_from_wrapper,
+    input_text_wrapper,
+    list_packages_wrapper,
+    launch_app_wrapper,
+    stop_app_wrapper,
+    paste_text_wrapper,
+    erase_text_wrapper,
+    press_key_wrapper,
+    wait_for_animation_to_end_wrapper,
+]
+
+
+def get_tools_from_wrappers(ctx: MobileUseContext, wrappers: list[ToolWrapper]) -> list[BaseTool]:
+    """Get the tools from the wrappers."""
+    return [wrapper.tool_fn_getter(ctx) for wrapper in wrappers]
+
+
+def get_tool_wrapper_from_name(name: str) -> ToolWrapper | None:
+    """Get the tool wrapper from the name."""
+    for wrapper in EXECUTOR_WRAPPERS_TOOLS:
+        if wrapper.tool_fn_getter.__name__ == f"get_{name}_tool":
+            return wrapper
+    return None

minitap/mobile_use/tools/mobile/back.py

@@ -0,0 +1,52 @@
+from typing import Optional
+
+from langchain_core.messages import ToolMessage
+from langchain_core.tools import tool
+from langchain_core.tools.base import InjectedToolCallId
+from langgraph.types import Command
+from minitap.mobile_use.controllers.mobile_command_controller import back as back_controller
+from minitap.mobile_use.tools.tool_wrapper import ExecutorMetadata, ToolWrapper
+from typing_extensions import Annotated
+from minitap.mobile_use.context import MobileUseContext
+from minitap.mobile_use.graph.state import State
+from langgraph.prebuilt import InjectedState
+
+
+def get_back_tool(ctx: MobileUseContext):
+    @tool
+    def back(
+        tool_call_id: Annotated[str, InjectedToolCallId],
+        state: Annotated[State, InjectedState],
+        agent_thought: str,
+        executor_metadata: Optional[ExecutorMetadata],
+    ):
+        """Navigates to the previous screen. (Only works on Android for the moment)"""
+        output = back_controller(ctx=ctx)
+        has_failed = output is not None
+        tool_message = ToolMessage(
+            tool_call_id=tool_call_id,
+            content=back_wrapper.on_failure_fn() if has_failed else back_wrapper.on_success_fn(),
+            additional_kwargs={"error": output} if has_failed else {},
+        )
+        return Command(
+            update=back_wrapper.handle_executor_state_fields(
+                ctx=ctx,
+                state=state,
+                executor_metadata=executor_metadata,
+                tool_message=tool_message,
+                is_failure=has_failed,
+                updates={
+                    "agents_thoughts": [agent_thought],
+                    "messages": [tool_message],
+                },
+            ),
+        )
+
+    return back
+
+
+back_wrapper = ToolWrapper(
+    tool_fn_getter=get_back_tool,
+    on_success_fn=lambda: "Navigated to the previous screen.",
+    on_failure_fn=lambda: "Failed to navigate to the previous screen.",
+)

minitap/mobile_use/tools/mobile/copy_text_from.py

@@ -0,0 +1,77 @@
+from typing import Optional
+
+from langchain_core.messages import ToolMessage
+from langchain_core.tools import tool
+from langchain_core.tools.base import InjectedToolCallId
+from langgraph.types import Command
+from minitap.mobile_use.controllers.mobile_command_controller import SelectorRequest
+from minitap.mobile_use.controllers.mobile_command_controller import (
+    copy_text_from as copy_text_from_controller,
+)
+from minitap.mobile_use.tools.tool_wrapper import ExecutorMetadata, ToolWrapper
+from pydantic import Field
+from typing_extensions import Annotated
+from minitap.mobile_use.context import MobileUseContext
+from minitap.mobile_use.graph.state import State
+from langgraph.prebuilt import InjectedState
+
+
+def get_copy_text_from_tool(ctx: MobileUseContext):
+    @tool
+    def copy_text_from(
+        tool_call_id: Annotated[str, InjectedToolCallId],
+        state: Annotated[State, InjectedState],
+        agent_thought: str,
+        executor_metadata: Optional[ExecutorMetadata],
+        selector_request: SelectorRequest = Field(
+            ..., description="The selector to copy text from"
+        ),
+    ):
+        """
+        Copies text from a UI element identified by the given selector and stores it in memory.
+
+        The copied text can be:
+        - Pasted later using `pasteText`
+        - Accessed in JavaScript via `maestro.copiedText`
+
+        Example Usage:
+            - launchApp
+            - copyTextFrom: { id: "someId" }
+            - tapOn: { id: "searchFieldId" }
+            - pasteText
+
+        See the Selectors documentation for supported selector types.
+        """
+        output = copy_text_from_controller(ctx=ctx, selector_request=selector_request)
+        has_failed = output is not None
+        tool_message = ToolMessage(
+            tool_call_id=tool_call_id,
+            content=copy_text_from_wrapper.on_failure_fn(selector_request)
+            if has_failed
+            else copy_text_from_wrapper.on_success_fn(selector_request),
+            additional_kwargs={"error": output} if has_failed else {},
+        )
+        return Command(
+            update=copy_text_from_wrapper.handle_executor_state_fields(
+                ctx=ctx,
+                state=state,
+                executor_metadata=executor_metadata,
+                tool_message=tool_message,
+                is_failure=has_failed,
+                updates={
+                    "agents_thoughts": [agent_thought],
+                    "messages": [tool_message],
+                },
+            ),
+        )
+
+    return copy_text_from
+
+
+copy_text_from_wrapper = ToolWrapper(
+    tool_fn_getter=get_copy_text_from_tool,
+    on_success_fn=lambda selector_request: (
+        f'Text copied successfully from selector "{selector_request}".'
+    ),
+    on_failure_fn=lambda selector_request: f"Failed to copy text from selector {selector_request}.",
+)