minitap-mobile-use 0.0.1.dev0__py3-none-any.whl

minitap/mobile_use/tools/mobile/erase_text.py

@@ -0,0 +1,124 @@
+from typing import Optional
+
+from langchain_core.messages import ToolMessage
+from langchain_core.tools import tool
+from langchain_core.tools.base import InjectedToolCallId
+from langgraph.prebuilt import InjectedState
+from langgraph.types import Command
+from typing_extensions import Annotated
+
+from minitap.mobile_use.controllers.mobile_command_controller import (
+    ScreenDataResponse,
+    WaitTimeout,
+    get_screen_data,
+    wait_for_animation_to_end,
+)
+from minitap.mobile_use.controllers.mobile_command_controller import (
+    erase_text as erase_text_controller,
+)
+from minitap.mobile_use.graph.state import State
+from minitap.mobile_use.tools.tool_wrapper import ExecutorMetadata, ToolWrapper
+from minitap.mobile_use.utils.ui_hierarchy import find_element_by_resource_id
+from minitap.mobile_use.context import MobileUseContext
+
+
+def get_erase_text_tool(ctx: MobileUseContext):
+    @tool
+    def erase_text(
+        tool_call_id: Annotated[str, InjectedToolCallId],
+        state: Annotated[State, InjectedState],
+        agent_thought: str,
+        input_text_resource_id: str,
+        executor_metadata: Optional[ExecutorMetadata],
+        nb_chars: Optional[int] = None,
+    ):
+        """
+        Erases up to `nb_chars` characters from the currently selected text field (default: 50).
+
+        iOS Note:
+            This may be flaky on iOS. As a workaround:
+                - long_press_on("<input id>")
+                - tap_on("Select All")
+                - erase_text()
+
+        Matches 'clearText' in search.
+        """
+        # value of text key from input_text_ressource_id
+        latest_ui_hierarchy = state.latest_ui_hierarchy
+        previous_text_value = None
+        new_text_value = None
+        nb_char_erased = -1
+        if latest_ui_hierarchy is not None:
+            text_input_element = find_element_by_resource_id(
+                ui_hierarchy=latest_ui_hierarchy, resource_id=input_text_resource_id
+            )
+            if text_input_element:
+                previous_text_value = text_input_element.get("text", None)
+
+        output = erase_text_controller(ctx=ctx, nb_chars=nb_chars)
+        has_failed = output is not None
+
+        wait_for_animation_to_end(ctx=ctx, timeout=WaitTimeout.MEDIUM)
+
+        screen_data: ScreenDataResponse = get_screen_data(screen_api_client=ctx.screen_api_client)
+        latest_ui_hierarchy = screen_data.elements
+
+        if not has_failed and latest_ui_hierarchy is not None:
+            text_input_element = find_element_by_resource_id(
+                ui_hierarchy=latest_ui_hierarchy, resource_id=input_text_resource_id
+            )
+            if text_input_element:
+                new_text_value = text_input_element.get("text", None)
+
+        if previous_text_value is not None and new_text_value is not None:
+            if previous_text_value == new_text_value:
+                has_failed = True
+                output = (
+                    "Unable to erase text: text is very likely a placeholder."
+                    " Thus, assuming the text input is empty."
+                )
+            else:
+                nb_char_erased = len(previous_text_value) - len(new_text_value)
+        tool_message = ToolMessage(
+            tool_call_id=tool_call_id,
+            content=erase_text_wrapper.on_failure_fn(output)
+            if has_failed
+            else erase_text_wrapper.on_success_fn(
+                nb_char_erased=nb_char_erased, new_text_value=new_text_value
+            ),
+            additional_kwargs={"error": output} if has_failed else {},
+        )
+
+        return Command(
+            update=erase_text_wrapper.handle_executor_state_fields(
+                ctx=ctx,
+                state=state,
+                executor_metadata=executor_metadata,
+                tool_message=tool_message,
+                is_failure=has_failed,
+                updates={
+                    "agents_thoughts": [agent_thought],
+                    "messages": [tool_message],
+                },
+            ),
+        )
+
+    return erase_text
+
+
+def format_success_message(nb_char_erased: int, new_text_value: Optional[str]):
+    output = ""
+    if nb_char_erased == -1:
+        output = "Text erased successfully."
+    else:
+        output = f"Text erased successfully. {nb_char_erased} characters were erased."
+    if new_text_value is not None:
+        output += f" New text in the input is {new_text_value}."
+    return output
+
+
+erase_text_wrapper = ToolWrapper(
+    tool_fn_getter=get_erase_text_tool,
+    on_success_fn=format_success_message,
+    on_failure_fn=lambda output: "Failed to erase text. " + output,
+)

minitap/mobile_use/tools/mobile/input_text.py

@@ -0,0 +1,74 @@
+from typing import Optional
+
+from langchain_core.messages import ToolMessage
+from langchain_core.tools import tool
+from langchain_core.tools.base import InjectedToolCallId
+from langgraph.types import Command
+from minitap.mobile_use.controllers.mobile_command_controller import (
+    input_text as input_text_controller,
+)
+from minitap.mobile_use.tools.tool_wrapper import ExecutorMetadata, ToolWrapper
+from typing_extensions import Annotated
+from minitap.mobile_use.graph.state import State
+from langgraph.prebuilt import InjectedState
+from minitap.mobile_use.context import MobileUseContext
+
+
+def get_input_text_tool(ctx: MobileUseContext):
+    @tool
+    def input_text(
+        tool_call_id: Annotated[str, InjectedToolCallId],
+        state: Annotated[State, InjectedState],
+        agent_thought: str,
+        executor_metadata: Optional[ExecutorMetadata],
+        text: str,
+    ):
+        """
+        Inputs the specified text into the UI (works even if no field is focused).
+
+        Example:
+            - inputText: "Hello World"
+
+        Notes:
+        - Unicode not supported on Android.
+
+        Random Input Options:
+            - inputRandomEmail
+            - inputRandomPersonName
+            - inputRandomNumber (with optional 'length', default 8)
+            - inputRandomText (with optional 'length', default 8)
+
+        Tip:
+            Use `copyTextFrom` to reuse generated inputs in later steps.
+        """
+        output = input_text_controller(ctx=ctx, text=text)
+        has_failed = output is not None
+        tool_message = ToolMessage(
+            tool_call_id=tool_call_id,
+            content=input_text_wrapper.on_failure_fn(text)
+            if has_failed
+            else input_text_wrapper.on_success_fn(text),
+            additional_kwargs={"error": output} if has_failed else {},
+        )
+        return Command(
+            update=input_text_wrapper.handle_executor_state_fields(
+                ctx=ctx,
+                state=state,
+                executor_metadata=executor_metadata,
+                tool_message=tool_message,
+                is_failure=has_failed,
+                updates={
+                    "agents_thoughts": [agent_thought],
+                    "messages": [tool_message],
+                },
+            ),
+        )
+
+    return input_text
+
+
+input_text_wrapper = ToolWrapper(
+    tool_fn_getter=get_input_text_tool,
+    on_success_fn=lambda text: f"Successfully typed {text}",
+    on_failure_fn=lambda text: f"Failed to input text {text}",
+)

minitap/mobile_use/tools/mobile/launch_app.py

@@ -0,0 +1,59 @@
+from typing import Optional
+
+from langchain_core.messages import ToolMessage
+from langchain_core.tools import tool
+from langchain_core.tools.base import InjectedToolCallId
+from langgraph.types import Command
+from minitap.mobile_use.controllers.mobile_command_controller import (
+    launch_app as launch_app_controller,
+)
+from minitap.mobile_use.tools.tool_wrapper import ExecutorMetadata, ToolWrapper
+from typing_extensions import Annotated
+from minitap.mobile_use.context import MobileUseContext
+from minitap.mobile_use.graph.state import State
+from langgraph.prebuilt import InjectedState
+
+
+def get_launch_app_tool(ctx: MobileUseContext):
+    @tool
+    def launch_app(
+        tool_call_id: Annotated[str, InjectedToolCallId],
+        state: Annotated[State, InjectedState],
+        agent_thought: str,
+        executor_metadata: Optional[ExecutorMetadata],
+        package_name: str,
+    ):
+        """
+        Launch an application on the device using the package name on Android, bundle id on iOS.
+        """
+        output = launch_app_controller(ctx=ctx, package_name=package_name)
+        has_failed = output is not None
+        tool_message = ToolMessage(
+            tool_call_id=tool_call_id,
+            content=launch_app_wrapper.on_failure_fn(package_name)
+            if has_failed
+            else launch_app_wrapper.on_success_fn(package_name),
+            additional_kwargs={"error": output} if has_failed else {},
+        )
+        return Command(
+            update=launch_app_wrapper.handle_executor_state_fields(
+                ctx=ctx,
+                state=state,
+                executor_metadata=executor_metadata,
+                tool_message=tool_message,
+                is_failure=has_failed,
+                updates={
+                    "agents_thoughts": [agent_thought],
+                    "messages": [tool_message],
+                },
+            ),
+        )
+
+    return launch_app
+
+
+launch_app_wrapper = ToolWrapper(
+    tool_fn_getter=get_launch_app_tool,
+    on_success_fn=lambda package_name: f"App {package_name} launched successfully.",
+    on_failure_fn=lambda package_name: f"Failed to launch app {package_name}.",
+)

minitap/mobile_use/tools/mobile/list_packages.py

@@ -0,0 +1,78 @@
+from typing import Optional
+
+from langchain_core.messages import ToolMessage
+from langchain_core.tools import tool
+from langchain_core.tools.base import InjectedToolCallId
+from langgraph.prebuilt import InjectedState
+from langgraph.types import Command
+from minitap.mobile_use.agents.hopper.hopper import HopperOutput, hopper
+from minitap.mobile_use.context import MobileUseContext
+from minitap.mobile_use.controllers.platform_specific_commands_controller import (
+    list_packages as list_packages_command,
+)
+from minitap.mobile_use.graph.state import State
+from minitap.mobile_use.tools.tool_wrapper import ExecutorMetadata, ToolWrapper
+from typing_extensions import Annotated
+
+
+def get_list_packages_tool(ctx: MobileUseContext):
+    @tool
+    async def list_packages(
+        tool_call_id: Annotated[str, InjectedToolCallId],
+        state: Annotated[State, InjectedState],
+        agent_thought: str,
+        executor_metadata: Optional[ExecutorMetadata],
+    ) -> Command:
+        """
+        Lists all the applications on the device.
+        Outputs the full package names list (android) or bundle ids list (IOS).
+        """
+        output: str = list_packages_command(ctx=ctx)
+        has_failed = False
+
+        try:
+            hopper_output: HopperOutput = await hopper(
+                ctx=ctx,
+                initial_goal=state.initial_goal,
+                messages=state.messages,
+                data=output,
+            )
+            tool_message = ToolMessage(
+                tool_call_id=tool_call_id,
+                content=list_packages_wrapper.on_success_fn()
+                + ": "
+                + hopper_output.step
+                + ": "
+                + hopper_output.output,
+            )
+        except Exception as e:
+            print("Failed to extract insights from data: " + str(e))
+            tool_message = ToolMessage(
+                tool_call_id=tool_call_id,
+                content=list_packages_wrapper.on_failure_fn(),
+                additional_kwargs={"output": output},
+            )
+            has_failed = True
+
+        return Command(
+            update=list_packages_wrapper.handle_executor_state_fields(
+                ctx=ctx,
+                state=state,
+                executor_metadata=executor_metadata,
+                tool_message=tool_message,
+                is_failure=has_failed,
+                updates={
+                    "agents_thoughts": [agent_thought],
+                    "messages": [tool_message],
+                },
+            ),
+        )
+
+    return list_packages
+
+
+list_packages_wrapper = ToolWrapper(
+    tool_fn_getter=get_list_packages_tool,
+    on_success_fn=lambda: "Packages listed successfully.",
+    on_failure_fn=lambda: "Failed to list packages.",
+)

minitap/mobile_use/tools/mobile/long_press_on.py

@@ -0,0 +1,62 @@
+from typing import Optional
+
+from langchain_core.messages import ToolMessage
+from langchain_core.tools import tool
+from langchain_core.tools.base import InjectedToolCallId
+from langgraph.prebuilt import InjectedState
+from langgraph.types import Command
+from minitap.mobile_use.context import MobileUseContext
+from minitap.mobile_use.controllers.mobile_command_controller import SelectorRequest
+from minitap.mobile_use.controllers.mobile_command_controller import (
+    long_press_on as long_press_on_controller,
+)
+from minitap.mobile_use.graph.state import State
+from minitap.mobile_use.tools.tool_wrapper import ExecutorMetadata, ToolWrapper
+from typing_extensions import Annotated
+
+
+def get_long_press_on_tool(ctx: MobileUseContext):
+    @tool
+    def long_press_on(
+        tool_call_id: Annotated[str, InjectedToolCallId],
+        state: Annotated[State, InjectedState],
+        agent_thought: str,
+        executor_metadata: Optional[ExecutorMetadata],
+        selector_request: SelectorRequest,
+        index: Optional[int] = None,
+    ):
+        """
+        Long press on a UI element identified by the given selector.
+        An index can be specified to select a specific element if multiple are found.
+        """
+        output = long_press_on_controller(ctx=ctx, selector_request=selector_request, index=index)
+        has_failed = output is not None
+        tool_message = ToolMessage(
+            tool_call_id=tool_call_id,
+            content=long_press_on_wrapper.on_failure_fn()
+            if has_failed
+            else long_press_on_wrapper.on_success_fn(),
+            additional_kwargs={"error": output} if has_failed else {},
+        )
+        return Command(
+            update=long_press_on_wrapper.handle_executor_state_fields(
+                ctx=ctx,
+                state=state,
+                executor_metadata=executor_metadata,
+                tool_message=tool_message,
+                is_failure=has_failed,
+                updates={
+                    "agents_thoughts": [agent_thought],
+                    "messages": [tool_message],
+                },
+            ),
+        )
+
+    return long_press_on
+
+
+long_press_on_wrapper = ToolWrapper(
+    tool_fn_getter=get_long_press_on_tool,
+    on_success_fn=lambda: "Long press on is successful.",
+    on_failure_fn=lambda: "Failed to long press on.",
+)

minitap/mobile_use/tools/mobile/open_link.py

@@ -0,0 +1,59 @@
+from typing import Optional
+
+from langchain_core.messages import ToolMessage
+from langchain_core.tools import tool
+from langchain_core.tools.base import InjectedToolCallId
+from langgraph.prebuilt import InjectedState
+from langgraph.types import Command
+from minitap.mobile_use.context import MobileUseContext
+from minitap.mobile_use.controllers.mobile_command_controller import (
+    open_link as open_link_controller,
+)
+from minitap.mobile_use.graph.state import State
+from minitap.mobile_use.tools.tool_wrapper import ExecutorMetadata, ToolWrapper
+from typing_extensions import Annotated
+
+
+def get_open_link_tool(ctx: MobileUseContext):
+    @tool
+    def open_link(
+        tool_call_id: Annotated[str, InjectedToolCallId],
+        state: Annotated[State, InjectedState],
+        agent_thought: str,
+        executor_metadata: Optional[ExecutorMetadata],
+        url: str,
+    ):
+        """
+        Open a link on a device (i.e. a deep link).
+        """
+        output = open_link_controller(ctx=ctx, url=url)
+        has_failed = output is not None
+        tool_message = ToolMessage(
+            tool_call_id=tool_call_id,
+            content=open_link_wrapper.on_failure_fn()
+            if has_failed
+            else open_link_wrapper.on_success_fn(url),
+            additional_kwargs={"error": output} if has_failed else {},
+        )
+        return Command(
+            update=open_link_wrapper.handle_executor_state_fields(
+                ctx=ctx,
+                state=state,
+                executor_metadata=executor_metadata,
+                tool_message=tool_message,
+                is_failure=has_failed,
+                updates={
+                    "agents_thoughts": [agent_thought],
+                    "messages": [tool_message],
+                },
+            ),
+        )
+
+    return open_link
+
+
+open_link_wrapper = ToolWrapper(
+    tool_fn_getter=get_open_link_tool,
+    on_success_fn=lambda url: f"Link {url} opened successfully.",
+    on_failure_fn=lambda: "Failed to open link.",
+)

minitap/mobile_use/tools/mobile/paste_text.py

@@ -0,0 +1,66 @@
+from typing import Optional
+
+from langchain_core.messages import ToolMessage
+from langchain_core.tools import tool
+from langchain_core.tools.base import InjectedToolCallId
+from langgraph.types import Command
+from minitap.mobile_use.context import MobileUseContext
+from minitap.mobile_use.controllers.mobile_command_controller import (
+    paste_text as paste_text_controller,
+)
+from minitap.mobile_use.graph.state import State
+from langgraph.prebuilt import InjectedState
+from minitap.mobile_use.tools.tool_wrapper import ExecutorMetadata, ToolWrapper
+from typing_extensions import Annotated
+
+
+def get_paste_text_tool(ctx: MobileUseContext):
+    @tool
+    def paste_text(
+        tool_call_id: Annotated[str, InjectedToolCallId],
+        state: Annotated[State, InjectedState],
+        agent_thought: str,
+        executor_metadata: Optional[ExecutorMetadata],
+    ):
+        """
+        Pastes text previously copied via `copyTextFrom` into the currently focused field.
+
+        Note:
+            The text field must be focused before using this command.
+
+        Example:
+            - copyTextFrom: { id: "someId" }
+            - tapOn: { id: "searchFieldId" }
+            - pasteText
+        """
+        output = paste_text_controller(ctx=ctx)
+        has_failed = output is not None
+        tool_message = ToolMessage(
+            tool_call_id=tool_call_id,
+            content=paste_text_wrapper.on_failure_fn()
+            if has_failed
+            else paste_text_wrapper.on_success_fn(),
+            additional_kwargs={"error": output} if has_failed else {},
+        )
+        return Command(
+            update=paste_text_wrapper.handle_executor_state_fields(
+                ctx=ctx,
+                state=state,
+                executor_metadata=executor_metadata,
+                tool_message=tool_message,
+                is_failure=has_failed,
+                updates={
+                    "agents_thoughts": [agent_thought],
+                    "messages": [tool_message],
+                },
+            ),
+        )
+
+    return paste_text
+
+
+paste_text_wrapper = ToolWrapper(
+    tool_fn_getter=get_paste_text_tool,
+    on_success_fn=lambda: "Text pasted successfully.",
+    on_failure_fn=lambda: "Failed to paste text.",
+)

minitap/mobile_use/tools/mobile/press_key.py

@@ -0,0 +1,58 @@
+from typing import Optional
+
+from langchain_core.messages import ToolMessage
+from langchain_core.tools import tool
+from langchain_core.tools.base import InjectedToolCallId
+from langgraph.prebuilt import InjectedState
+from langgraph.types import Command
+from minitap.mobile_use.context import MobileUseContext
+from minitap.mobile_use.controllers.mobile_command_controller import Key
+from minitap.mobile_use.controllers.mobile_command_controller import (
+    press_key as press_key_controller,
+)
+from minitap.mobile_use.graph.state import State
+from minitap.mobile_use.tools.tool_wrapper import ExecutorMetadata, ToolWrapper
+from typing_extensions import Annotated
+
+
+def get_press_key_tool(ctx: MobileUseContext):
+    @tool
+    def press_key(
+        tool_call_id: Annotated[str, InjectedToolCallId],
+        state: Annotated[State, InjectedState],
+        agent_thought: str,
+        executor_metadata: Optional[ExecutorMetadata],
+        key: Key,
+    ):
+        """Press a key on the device."""
+        output = press_key_controller(ctx=ctx, key=key)
+        has_failed = output is not None
+        tool_message = ToolMessage(
+            tool_call_id=tool_call_id,
+            content=press_key_wrapper.on_failure_fn(key)
+            if has_failed
+            else press_key_wrapper.on_success_fn(key),
+            additional_kwargs={"error": output} if has_failed else {},
+        )
+        return Command(
+            update=press_key_wrapper.handle_executor_state_fields(
+                ctx=ctx,
+                state=state,
+                executor_metadata=executor_metadata,
+                tool_message=tool_message,
+                is_failure=has_failed,
+                updates={
+                    "agents_thoughts": [agent_thought],
+                    "messages": [tool_message],
+                },
+            ),
+        )
+
+    return press_key
+
+
+press_key_wrapper = ToolWrapper(
+    tool_fn_getter=get_press_key_tool,
+    on_success_fn=lambda key: f"Key {key.value} pressed successfully.",
+    on_failure_fn=lambda key: f"Failed to press key {key.value}.",
+)

minitap/mobile_use/tools/mobile/run_flow.py

@@ -0,0 +1,57 @@
+from typing import Optional
+from langchain_core.messages import ToolMessage
+from langchain_core.tools import tool
+from langchain_core.tools.base import InjectedToolCallId
+from langgraph.prebuilt import InjectedState
+from langgraph.types import Command
+from minitap.mobile_use.context import MobileUseContext
+from minitap.mobile_use.controllers.mobile_command_controller import run_flow as run_flow_controller
+from minitap.mobile_use.graph.state import State
+from minitap.mobile_use.tools.tool_wrapper import ExecutorMetadata, ToolWrapper
+from typing_extensions import Annotated
+
+
+def get_run_flow_tool(ctx: MobileUseContext):
+    @tool
+    def run_flow(
+        tool_call_id: Annotated[str, InjectedToolCallId],
+        state: Annotated[State, InjectedState],
+        agent_thought: str,
+        flow_steps: list,
+        executor_metadata: Optional[ExecutorMetadata],
+        dry_run: bool = False,
+    ):
+        """
+        Run a flow i.e, a sequence of commands.
+        """
+        output = run_flow_controller(ctx=ctx, flow_steps=flow_steps, dry_run=dry_run)
+        has_failed = output is not None
+        tool_message = ToolMessage(
+            tool_call_id=tool_call_id,
+            content=run_flow_wrapper.on_failure_fn()
+            if has_failed
+            else run_flow_wrapper.on_success_fn(),
+            additional_kwargs={"error": output} if has_failed else {},
+        )
+        return Command(
+            update=run_flow_wrapper.handle_executor_state_fields(
+                ctx=ctx,
+                state=state,
+                executor_metadata=executor_metadata,
+                tool_message=tool_message,
+                is_failure=has_failed,
+                updates={
+                    "agents_thoughts": [agent_thought],
+                    "messages": [tool_message],
+                },
+            ),
+        )
+
+    return run_flow
+
+
+run_flow_wrapper = ToolWrapper(
+    tool_fn_getter=get_run_flow_tool,
+    on_success_fn=lambda: "Flow run successfully.",
+    on_failure_fn=lambda: "Failed to run flow.",
+)