minitap-mobile-use 2.3.0__py3-none-any.whl → 2.5.0__py3-none-any.whl

minitap/mobile_use/tools/mobile/launch_app.py

@@ -1,44 +1,76 @@
+from typing import Annotated
+
 from langchain_core.messages import ToolMessage
 from langchain_core.tools import tool
 from langchain_core.tools.base import InjectedToolCallId
+from langgraph.prebuilt import InjectedState
 from langgraph.types import Command
+
+from minitap.mobile_use.agents.hopper.hopper import HopperOutput, hopper
 from minitap.mobile_use.constants import EXECUTOR_MESSAGES_KEY
+from minitap.mobile_use.context import MobileUseContext
 from minitap.mobile_use.controllers.mobile_command_controller import (
     launch_app as launch_app_controller,
 )
-from minitap.mobile_use.tools.tool_wrapper import ToolWrapper
-from typing import Annotated
-from minitap.mobile_use.context import MobileUseContext
+from minitap.mobile_use.controllers.platform_specific_commands_controller import list_packages
 from minitap.mobile_use.graph.state import State
-from langgraph.prebuilt import InjectedState
+from minitap.mobile_use.tools.tool_wrapper import ToolWrapper
+
+
+async def find_package(ctx: MobileUseContext, app_name: str) -> str | None:
+    """
+    Finds the package name for a given application name.
+    """
+    all_packages = list_packages(ctx=ctx)
+    try:
+        hopper_output: HopperOutput = await hopper(
+            ctx=ctx,
+            request=f"I'm looking for the package name of the following app: '{app_name}'",
+            data=all_packages,
+        )
+        # Assuming hopper_output.output directly contains the package name
+        return hopper_output.output
+    except Exception as e:
+        print(f"Failed to find package for '{app_name}': {e}")
+        return None
 
 
 def get_launch_app_tool(ctx: MobileUseContext):
     @tool
-    def launch_app(
+    async def launch_app(
         tool_call_id: Annotated[str, InjectedToolCallId],
         state: Annotated[State, InjectedState],
+        app_name: str,
         agent_thought: str,
-        package_name: str,
-    ):
+    ) -> Command:
         """
-        Launch an application on the device using the package name on Android, bundle id on iOS.
+        Finds and launches an application on the device using its natural language name.
         """
-        output = launch_app_controller(ctx=ctx, package_name=package_name)
-        has_failed = output is not None
-        tool_message = ToolMessage(
-            tool_call_id=tool_call_id,
-            content=launch_app_wrapper.on_failure_fn(package_name)
-            if has_failed
-            else launch_app_wrapper.on_success_fn(package_name),
-            additional_kwargs={"error": output} if has_failed else {},
-            status="error" if has_failed else "success",
-        )
+        package_name = await find_package(ctx=ctx, app_name=app_name)
+
+        if not package_name:
+            tool_message = ToolMessage(
+                tool_call_id=tool_call_id,
+                content=launch_app_wrapper.on_failure_fn(app_name, "Package not found."),
+                status="error",
+            )
+        else:
+            output = launch_app_controller(ctx=ctx, package_name=package_name)
+            has_failed = output is not None
+            tool_message = ToolMessage(
+                tool_call_id=tool_call_id,
+                content=launch_app_wrapper.on_failure_fn(app_name, output)
+                if has_failed
+                else launch_app_wrapper.on_success_fn(app_name),
+                additional_kwargs={"error": output} if has_failed else {},
+                status="error" if has_failed else "success",
+            )
+
         return Command(
-            update=state.sanitize_update(
+            update=await state.asanitize_update(
                 ctx=ctx,
                 update={
-                    "agents_thoughts": [agent_thought],
+                    "agents_thoughts": [agent_thought, tool_message.content],
                     EXECUTOR_MESSAGES_KEY: [tool_message],
                 },
                 agent="executor",
@@ -50,6 +82,6 @@ def get_launch_app_tool(ctx: MobileUseContext):
 
 launch_app_wrapper = ToolWrapper(
     tool_fn_getter=get_launch_app_tool,
-    on_success_fn=lambda package_name: f"App {package_name} launched successfully.",
-    on_failure_fn=lambda package_name: f"Failed to launch app {package_name}.",
+    on_success_fn=lambda app_name: f"App '{app_name}' launched successfully.",
+    on_failure_fn=lambda app_name, error: f"Failed to launch app '{app_name}': {error}",
 )

minitap/mobile_use/tools/mobile/long_press_on.py

@@ -1,8 +1,11 @@
+from typing import Annotated
+
 from langchain_core.messages import ToolMessage
 from langchain_core.tools import tool
 from langchain_core.tools.base import InjectedToolCallId
 from langgraph.prebuilt import InjectedState
 from langgraph.types import Command
+
 from minitap.mobile_use.constants import EXECUTOR_MESSAGES_KEY
 from minitap.mobile_use.context import MobileUseContext
 from minitap.mobile_use.controllers.mobile_command_controller import SelectorRequest
@@ -11,37 +14,41 @@ from minitap.mobile_use.controllers.mobile_command_controller import (
 )
 from minitap.mobile_use.graph.state import State
 from minitap.mobile_use.tools.tool_wrapper import ToolWrapper
-from typing import Annotated
 
 
 def get_long_press_on_tool(ctx: MobileUseContext):
     @tool
-    def long_press_on(
+    async def long_press_on(
         tool_call_id: Annotated[str, InjectedToolCallId],
         state: Annotated[State, InjectedState],
         agent_thought: str,
         selector_request: SelectorRequest,
         index: int | None = None,
-    ):
+    ) -> Command:
         """
         Long press on a UI element identified by the given selector.
         An index can be specified to select a specific element if multiple are found.
         """
         output = long_press_on_controller(ctx=ctx, selector_request=selector_request, index=index)
         has_failed = output is not None
+
+        agent_outcome = (
+            long_press_on_wrapper.on_failure_fn()
+            if has_failed
+            else long_press_on_wrapper.on_success_fn()
+        )
+
         tool_message = ToolMessage(
             tool_call_id=tool_call_id,
-            content=long_press_on_wrapper.on_failure_fn()
-            if has_failed
-            else long_press_on_wrapper.on_success_fn(),
+            content=agent_outcome,
             additional_kwargs={"error": output} if has_failed else {},
             status="error" if has_failed else "success",
         )
         return Command(
-            update=state.sanitize_update(
+            update=await state.asanitize_update(
                 ctx=ctx,
                 update={
-                    "agents_thoughts": [agent_thought],
+                    "agents_thoughts": [agent_thought, agent_outcome],
                     EXECUTOR_MESSAGES_KEY: [tool_message],
                 },
                 agent="executor",

minitap/mobile_use/tools/mobile/open_link.py

@@ -1,8 +1,11 @@
+from typing import Annotated
+
 from langchain_core.messages import ToolMessage
 from langchain_core.tools import tool
 from langchain_core.tools.base import InjectedToolCallId
 from langgraph.prebuilt import InjectedState
 from langgraph.types import Command
+
 from minitap.mobile_use.constants import EXECUTOR_MESSAGES_KEY
 from minitap.mobile_use.context import MobileUseContext
 from minitap.mobile_use.controllers.mobile_command_controller import (
@@ -10,35 +13,39 @@ from minitap.mobile_use.controllers.mobile_command_controller import (
 )
 from minitap.mobile_use.graph.state import State
 from minitap.mobile_use.tools.tool_wrapper import ToolWrapper
-from typing import Annotated
 
 
 def get_open_link_tool(ctx: MobileUseContext):
     @tool
-    def open_link(
+    async def open_link(
         tool_call_id: Annotated[str, InjectedToolCallId],
         state: Annotated[State, InjectedState],
         agent_thought: str,
         url: str,
-    ):
+    ) -> Command:
         """
         Open a link on a device (i.e. a deep link).
         """
         output = open_link_controller(ctx=ctx, url=url)
         has_failed = output is not None
+
+        agent_outcome = (
+            open_link_wrapper.on_failure_fn()
+            if has_failed
+            else open_link_wrapper.on_success_fn(url)
+        )
+
         tool_message = ToolMessage(
             tool_call_id=tool_call_id,
-            content=open_link_wrapper.on_failure_fn()
-            if has_failed
-            else open_link_wrapper.on_success_fn(url),
+            content=agent_outcome,
             additional_kwargs={"error": output} if has_failed else {},
             status="error" if has_failed else "success",
         )
         return Command(
-            update=state.sanitize_update(
+            update=await state.asanitize_update(
                 ctx=ctx,
                 update={
-                    "agents_thoughts": [agent_thought],
+                    "agents_thoughts": [agent_thought, agent_outcome],
                     EXECUTOR_MESSAGES_KEY: [tool_message],
                 },
                 agent="executor",

minitap/mobile_use/tools/mobile/press_key.py

@@ -1,8 +1,11 @@
+from typing import Annotated
+
 from langchain_core.messages import ToolMessage
 from langchain_core.tools import tool
 from langchain_core.tools.base import InjectedToolCallId
 from langgraph.prebuilt import InjectedState
 from langgraph.types import Command
+
 from minitap.mobile_use.constants import EXECUTOR_MESSAGES_KEY
 from minitap.mobile_use.context import MobileUseContext
 from minitap.mobile_use.controllers.mobile_command_controller import Key
@@ -11,33 +14,37 @@ from minitap.mobile_use.controllers.mobile_command_controller import (
 )
 from minitap.mobile_use.graph.state import State
 from minitap.mobile_use.tools.tool_wrapper import ToolWrapper
-from typing import Annotated
 
 
 def get_press_key_tool(ctx: MobileUseContext):
     @tool
-    def press_key(
+    async def press_key(
         tool_call_id: Annotated[str, InjectedToolCallId],
         state: Annotated[State, InjectedState],
         agent_thought: str,
         key: Key,
-    ):
+    ) -> Command:
         """Press a key on the device."""
         output = press_key_controller(ctx=ctx, key=key)
         has_failed = output is not None
+
+        agent_outcome = (
+            press_key_wrapper.on_failure_fn(key)
+            if has_failed
+            else press_key_wrapper.on_success_fn(key)
+        )
+
         tool_message = ToolMessage(
             tool_call_id=tool_call_id,
-            content=press_key_wrapper.on_failure_fn(key)
-            if has_failed
-            else press_key_wrapper.on_success_fn(key),
+            content=agent_outcome,
             additional_kwargs={"error": output} if has_failed else {},
             status="error" if has_failed else "success",
         )
         return Command(
-            update=state.sanitize_update(
+            update=await state.asanitize_update(
                 ctx=ctx,
                 update={
-                    "agents_thoughts": [agent_thought],
+                    "agents_thoughts": [agent_thought, agent_outcome],
                     EXECUTOR_MESSAGES_KEY: [tool_message],
                 },
                 agent="executor",

minitap/mobile_use/tools/mobile/stop_app.py

@@ -1,43 +1,49 @@
+from typing import Annotated
+
 from langchain_core.messages import ToolMessage
 from langchain_core.tools import tool
 from langchain_core.tools.base import InjectedToolCallId
 from langgraph.prebuilt import InjectedState
 from langgraph.types import Command
+
 from minitap.mobile_use.constants import EXECUTOR_MESSAGES_KEY
 from minitap.mobile_use.context import MobileUseContext
 from minitap.mobile_use.controllers.mobile_command_controller import stop_app as stop_app_controller
 from minitap.mobile_use.graph.state import State
 from minitap.mobile_use.tools.tool_wrapper import ToolWrapper
-from typing import Annotated
 
 
 def get_stop_app_tool(ctx: MobileUseContext):
     @tool
-    def stop_app(
+    async def stop_app(
         tool_call_id: Annotated[str, InjectedToolCallId],
         state: Annotated[State, InjectedState],
         agent_thought: str,
         package_name: str | None = None,
-    ):
+    ) -> Command:
         """
         Stops current application if it is running.
         You can also specify the package name of the app to be stopped.
         """
         output = stop_app_controller(ctx=ctx, package_name=package_name)
         has_failed = output is not None
+
+        agent_outcome = (
+            stop_app_wrapper.on_failure_fn(package_name)
+            if has_failed
+            else stop_app_wrapper.on_success_fn(package_name)
+        )
         tool_message = ToolMessage(
             tool_call_id=tool_call_id,
-            content=stop_app_wrapper.on_failure_fn(package_name)
-            if has_failed
-            else stop_app_wrapper.on_success_fn(package_name),
+            content=agent_outcome,
             additional_kwargs={"error": output} if has_failed else {},
             status="error" if has_failed else "success",
         )
         return Command(
-            update=state.sanitize_update(
+            update=await state.asanitize_update(
                 ctx=ctx,
                 update={
-                    "agents_thoughts": [agent_thought],
+                    "agents_thoughts": [agent_thought, agent_outcome],
                     EXECUTOR_MESSAGES_KEY: [tool_message],
                 },
                 agent="executor",

minitap/mobile_use/tools/mobile/swipe.py

@@ -24,26 +24,32 @@ from minitap.mobile_use.tools.tool_wrapper import CompositeToolWrapper
 
 def get_swipe_tool(ctx: MobileUseContext) -> BaseTool:
     @tool
-    def swipe(
+    async def swipe(
         tool_call_id: Annotated[str, InjectedToolCallId],
         state: Annotated[State, InjectedState],
         agent_thought: str,
         swipe_request: SwipeRequest,
-    ):
+    ) -> Command:
         """Swipes on the screen."""
         output = swipe_controller(ctx=ctx, swipe_request=swipe_request)
         has_failed = output is not None
+
+        agent_outcome = (
+            swipe_wrapper.on_success_fn() if not has_failed else swipe_wrapper.on_failure_fn()
+        )
+
         tool_message = ToolMessage(
             tool_call_id=tool_call_id,
-            content=swipe_wrapper.on_failure_fn() if has_failed else swipe_wrapper.on_success_fn(),
+            content=agent_outcome,
             additional_kwargs={"error": output} if has_failed else {},
             status="error" if has_failed else "success",
         )
+
         return Command(
-            update=state.sanitize_update(
+            update=await state.asanitize_update(
                 ctx=ctx,
                 update={
-                    "agents_thoughts": [agent_thought],
+                    "agents_thoughts": [agent_thought, agent_outcome],
                     EXECUTOR_MESSAGES_KEY: [tool_message],
                 },
                 agent="executor",

minitap/mobile_use/tools/mobile/tap.py

@@ -1,45 +1,132 @@
+from typing import Annotated
+
 from langchain_core.messages import ToolMessage
 from langchain_core.tools import tool
 from langchain_core.tools.base import InjectedToolCallId
 from langgraph.prebuilt import InjectedState
 from langgraph.types import Command
+
 from minitap.mobile_use.constants import EXECUTOR_MESSAGES_KEY
 from minitap.mobile_use.context import MobileUseContext
-from minitap.mobile_use.controllers.mobile_command_controller import SelectorRequest
+from minitap.mobile_use.controllers.mobile_command_controller import (
+    CoordinatesSelectorRequest,
+    IdSelectorRequest,
+    SelectorRequestWithCoordinates,
+    TextSelectorRequest,
+)
 from minitap.mobile_use.controllers.mobile_command_controller import tap as tap_controller
 from minitap.mobile_use.graph.state import State
 from minitap.mobile_use.tools.tool_wrapper import ToolWrapper
-from typing import Annotated
+from minitap.mobile_use.tools.types import Target
+from minitap.mobile_use.utils.logger import get_logger
+
+logger = get_logger(__name__)
 
 
 def get_tap_tool(ctx: MobileUseContext):
     @tool
-    def tap(
+    async def tap(
         tool_call_id: Annotated[str, InjectedToolCallId],
         state: Annotated[State, InjectedState],
         agent_thought: str,
-        selector_request: SelectorRequest,
-        index: int | None = None,
+        target: Target,
     ):
         """
-        Taps on a selector.
-        Index is optional and is used when you have multiple views matching the same selector.
+        Taps on a UI element identified by the 'target' object.
+
+        The 'target' object allows specifying an element by its resource_id
+        (with an optional index), its coordinates, or its text content (with an optional index).
+        The tool uses a fallback strategy, trying the locators in that order.
         """
-        output = tap_controller(ctx=ctx, selector_request=selector_request, index=index)
+        output = {
+            "error": "No valid selector provided or all selectors failed."
+        }  # Default to failure
+        final_selector_info = "N/A"
+
+        # 1. Try with resource_id
+        if target.resource_id:
+            try:
+                selector = IdSelectorRequest(id=target.resource_id)
+                logger.info(
+                    f"Attempting to tap using resource_id: '{target.resource_id}' "
+                    f"at index {target.resource_id_index}"
+                )
+                result = tap_controller(
+                    ctx=ctx, selector_request=selector, index=target.resource_id_index
+                )
+                if result is None:  # Success
+                    output = None
+                    final_selector_info = (
+                        f"resource_id='{target.resource_id}' (index={target.resource_id_index})"
+                    )
+                else:
+                    logger.warning(
+                        f"Tap with resource_id '{target.resource_id}' failed. Error: {result}"
+                    )
+                    output = result
+            except Exception as e:
+                logger.warning(f"Exception during tap with resource_id '{target.resource_id}': {e}")
+                output = {"error": str(e)}
+
+        # 2. If resource_id failed or wasn't provided, try with coordinates
+        if output is not None and target.coordinates:
+            try:
+                center_point = target.coordinates.get_center()
+                selector = SelectorRequestWithCoordinates(
+                    coordinates=CoordinatesSelectorRequest(x=center_point.x, y=center_point.y)
+                )
+                logger.info(
+                    f"Attempting to tap using coordinates: {center_point.x},{center_point.y}"
+                )
+                result = tap_controller(ctx=ctx, selector_request=selector)
+                if result is None:  # Success
+                    output = None
+                    final_selector_info = f"coordinates='{target.coordinates}'"
+                else:
+                    logger.warning(
+                        f"Tap with coordinates '{target.coordinates}' failed. Error: {result}"
+                    )
+                    output = result
+            except Exception as e:
+                logger.warning(f"Exception during tap with coordinates '{target.coordinates}': {e}")
+                output = {"error": str(e)}
+
+        # 3. If coordinates failed or weren't provided, try with text
+        if output is not None and target.text:
+            try:
+                selector = TextSelectorRequest(text=target.text)
+                logger.info(
+                    f"Attempting to tap using text: '{target.text}' at index {target.text_index}"
+                )
+                result = tap_controller(ctx=ctx, selector_request=selector, index=target.text_index)
+                if result is None:  # Success
+                    output = None
+                    final_selector_info = f"text='{target.text}' (index={target.text_index})"
+                else:
+                    logger.warning(f"Tap with text '{target.text}' failed. Error: {result}")
+                    output = result
+            except Exception as e:
+                logger.warning(f"Exception during tap with text '{target.text}': {e}")
+                output = {"error": str(e)}
+
         has_failed = output is not None
+        agent_outcome = (
+            tap_wrapper.on_failure_fn(final_selector_info)
+            if has_failed
+            else tap_wrapper.on_success_fn(final_selector_info)
+        )
+
         tool_message = ToolMessage(
             tool_call_id=tool_call_id,
-            content=tap_wrapper.on_failure_fn(selector_request, index)
-            if has_failed
-            else tap_wrapper.on_success_fn(selector_request, index),
+            content=agent_outcome,
             additional_kwargs={"error": output} if has_failed else {},
             status="error" if has_failed else "success",
         )
         return Command(
-            update=state.sanitize_update(
+            update=await state.asanitize_update(
                 ctx=ctx,
                 update={
-                    "agents_thoughts": [agent_thought],
+                    "agents_thoughts": [agent_thought, agent_outcome],
                     EXECUTOR_MESSAGES_KEY: [tool_message],
                 },
                 agent="executor",
@@ -51,12 +138,7 @@ def get_tap_tool(ctx: MobileUseContext):
 
 tap_wrapper = ToolWrapper(
     tool_fn_getter=get_tap_tool,
-    on_success_fn=(
-        lambda selector_request,
-        index: f"Tap on {selector_request} {'at index {index}' if index else ''} is successful."
-    ),
-    on_failure_fn=(
-        lambda selector_request,
-        index: f"Failed to tap on {selector_request} {'at index {index}' if index else ''}."
-    ),
+    on_success_fn=lambda selector_info: f"Tap on element with {selector_info} was successful.",
+    on_failure_fn=lambda selector_info: "Failed to tap on element. "
+    + f"Last attempt was with {selector_info}.",
 )

minitap/mobile_use/tools/mobile/wait_for_animation_to_end.py

@@ -16,12 +16,12 @@ from typing import Annotated
 
 def get_wait_for_animation_to_end_tool(ctx: MobileUseContext):
     @tool
-    def wait_for_animation_to_end(
+    async def wait_for_animation_to_end(
         tool_call_id: Annotated[str, InjectedToolCallId],
         state: Annotated[State, InjectedState],
         agent_thought: str,
         timeout: WaitTimeout | None,
-    ):
+    ) -> Command:
         """
         Waits for ongoing animations or videos to finish before continuing.
 
@@ -44,7 +44,7 @@ def get_wait_for_animation_to_end_tool(ctx: MobileUseContext):
             status="error" if has_failed else "success",
         )
         return Command(
-            update=state.sanitize_update(
+            update=await state.asanitize_update(
                 ctx=ctx,
                 update={
                     "agents_thoughts": [agent_thought],