minitap-mobile-use 2.1.0__py3-none-any.whl → 2.3.0__py3-none-any.whl

minitap/mobile_use/tools/mobile/input_text.py

@@ -1,6 +1,6 @@
 from __future__ import annotations
 
-from typing import Literal
+from typing import Annotated, Literal
 
 from langchain_core.messages import ToolMessage
 from langchain_core.tools import tool
@@ -8,10 +8,12 @@ from langchain_core.tools.base import InjectedToolCallId
 from langgraph.prebuilt import InjectedState
 from langgraph.types import Command
 from pydantic import BaseModel
-from typing import Annotated
 
 from minitap.mobile_use.constants import EXECUTOR_MESSAGES_KEY
 from minitap.mobile_use.context import MobileUseContext
+from minitap.mobile_use.controllers.mobile_command_controller import (
+    get_screen_data,
+)
 from minitap.mobile_use.controllers.mobile_command_controller import (
     input_text as input_text_controller,
 )
@@ -19,6 +21,11 @@ from minitap.mobile_use.graph.state import State
 from minitap.mobile_use.tools.tool_wrapper import ToolWrapper
 from minitap.mobile_use.tools.utils import focus_element_if_needed, move_cursor_to_end_if_bounds
 from minitap.mobile_use.utils.logger import get_logger
+from minitap.mobile_use.utils.ui_hierarchy import (
+    ElementBounds,
+    find_element_by_resource_id,
+    get_element_text,
+)
 
 logger = get_logger(__name__)
 
@@ -47,7 +54,9 @@ def get_input_text_tool(ctx: MobileUseContext):
         state: Annotated[State, InjectedState],
         agent_thought: str,
         text: str,
-        text_input_resource_id: str,
+        text_input_resource_id: str | None,
+        text_input_coordinates: ElementBounds | None,
+        text_input_text: str | None,
     ):
         """
         Focus a text field and type text into it.
@@ -55,23 +64,83 @@ def get_input_text_tool(ctx: MobileUseContext):
         - Ensure the corresponding element is focused (tap if necessary).
         - If bounds are available, tap near the end to place the cursor at the end.
         - Type the provided `text` using the controller.
+
+        Args:
+            tool_call_id: The ID of the tool call.
+            state: The state of the agent.
+            agent_thought: The thought of the agent.
+            text: The text to type.
+            text_input_resource_id: The resource ID of the text input (if available).
+            text_input_coordinates: The bounds (ElementBounds) of the text input (if available).
+            text_input_text: The current text content of the text input (if available).
         """
-        focused = focus_element_if_needed(ctx=ctx, resource_id=text_input_resource_id)
-        if focused:
-            move_cursor_to_end_if_bounds(ctx=ctx, state=state, resource_id=text_input_resource_id)
+
+        focused = focus_element_if_needed(
+            ctx=ctx,
+            input_resource_id=text_input_resource_id,
+            input_coordinates=text_input_coordinates,
+            input_text=text_input_text,
+        )
+        if not focused:
+            error_message = "Failed to focus the text input element before typing."
+            tool_message = ToolMessage(
+                tool_call_id=tool_call_id,
+                content=input_text_wrapper.on_failure_fn(text, error_message),
+                additional_kwargs={"error": error_message},
+                status="error",
+            )
+            return Command(
+                update=state.sanitize_update(
+                    ctx=ctx,
+                    update={
+                        "agents_thoughts": [agent_thought, error_message],
+                        EXECUTOR_MESSAGES_KEY: [tool_message],
+                    },
+                    agent="executor",
+                ),
+            )
+
+        move_cursor_to_end_if_bounds(
+            ctx=ctx,
+            state=state,
+            text_input_resource_id=text_input_resource_id,
+            text_input_coordinates=text_input_coordinates,
+            text_input_text=text_input_text,
+        )
 
         result = _controller_input_text(ctx=ctx, text=text)
 
         status: Literal["success", "error"] = "success" if result.ok else "error"
-        content_msg = (
-            input_text_wrapper.on_success_fn(text)
+
+        text_input_content = ""
+        if status == "success":
+            if text_input_resource_id is not None:
+                # Verification phase for elements with resource_id
+                screen_data = get_screen_data(screen_api_client=ctx.screen_api_client)
+                state.latest_ui_hierarchy = screen_data.elements
+
+                element = find_element_by_resource_id(
+                    ui_hierarchy=state.latest_ui_hierarchy, resource_id=text_input_resource_id
+                )
+
+                if not element:
+                    result = InputResult(ok=False, error="Element not found")
+
+                if element:
+                    text_input_content = get_element_text(element)
+            else:
+                # For elements without resource_id, skip verification and use direct message
+                pass
+
+        agent_outcome = (
+            input_text_wrapper.on_success_fn(text, text_input_content, text_input_resource_id)
             if result.ok
-            else input_text_wrapper.on_failure_fn(text)
+            else input_text_wrapper.on_failure_fn(text, result.error)
         )
 
         tool_message = ToolMessage(
             tool_call_id=tool_call_id,
-            content=content_msg,
+            content=agent_outcome,
             additional_kwargs={"error": result.error} if not result.ok else {},
             status=status,
         )
@@ -80,7 +149,7 @@ def get_input_text_tool(ctx: MobileUseContext):
             update=state.sanitize_update(
                 ctx=ctx,
                 update={
-                    "agents_thoughts": [agent_thought],
+                    "agents_thoughts": [agent_thought, agent_outcome],
                     EXECUTOR_MESSAGES_KEY: [tool_message],
                 },
                 agent="executor",
@@ -90,8 +159,20 @@ def get_input_text_tool(ctx: MobileUseContext):
     return input_text
 
 
+def _on_input_success(text, text_input_content, text_input_resource_id):
+    """Success message handler for input text operations."""
+    if text_input_resource_id is not None:
+        return (
+            f"Typed {repr(text)}.\n"
+            f"Here is the whole content of input with id {repr(text_input_resource_id)}: "
+            f"{repr(text_input_content)}"
+        )
+    else:
+        return "Typed text, should now verify before moving forward"
+
+
 input_text_wrapper = ToolWrapper(
     tool_fn_getter=get_input_text_tool,
-    on_success_fn=lambda text: f"Successfully typed {text}",
-    on_failure_fn=lambda text: f"Failed to input text {text}",
+    on_success_fn=_on_input_success,
+    on_failure_fn=lambda text, error: f"Failed to input text {repr(text)}. Reason: {error}",
 )

minitap/mobile_use/tools/mobile/paste_text.py

@@ -1,16 +1,22 @@
+from typing import Annotated
+
 from langchain_core.messages import ToolMessage
 from langchain_core.tools import tool
 from langchain_core.tools.base import InjectedToolCallId
+from langgraph.prebuilt import InjectedState
 from langgraph.types import Command
+
 from minitap.mobile_use.constants import EXECUTOR_MESSAGES_KEY
 from minitap.mobile_use.context import MobileUseContext
+from minitap.mobile_use.controllers.mobile_command_controller import (
+    get_screen_data,
+)
 from minitap.mobile_use.controllers.mobile_command_controller import (
     paste_text as paste_text_controller,
 )
 from minitap.mobile_use.graph.state import State
-from langgraph.prebuilt import InjectedState
 from minitap.mobile_use.tools.tool_wrapper import ToolWrapper
-from typing import Annotated
+from minitap.mobile_use.utils.ui_hierarchy import find_element_by_resource_id, get_element_text
 
 
 def get_paste_text_tool(ctx: MobileUseContext):
@@ -19,6 +25,7 @@ def get_paste_text_tool(ctx: MobileUseContext):
         tool_call_id: Annotated[str, InjectedToolCallId],
         state: Annotated[State, InjectedState],
         agent_thought: str,
+        focused_element_resource_id: str,
     ):
         """
         Pastes text previously copied via `copyTextFrom` into the currently focused field.
@@ -32,12 +39,29 @@ def get_paste_text_tool(ctx: MobileUseContext):
             - pasteText
         """
         output = paste_text_controller(ctx=ctx)
+
+        text_input_content = ""
+        screen_data = get_screen_data(screen_api_client=ctx.screen_api_client)
+        state.latest_ui_hierarchy = screen_data.elements
+
+        element = find_element_by_resource_id(
+            ui_hierarchy=state.latest_ui_hierarchy, resource_id=focused_element_resource_id
+        )
+
+        if element:
+            text_input_content = get_element_text(element)
+
         has_failed = output is not None
+
+        agent_outcome = (
+            paste_text_wrapper.on_success_fn(text_input_content)
+            if not has_failed
+            else paste_text_wrapper.on_failure_fn(text_input_content)
+        )
+
         tool_message = ToolMessage(
             tool_call_id=tool_call_id,
-            content=paste_text_wrapper.on_failure_fn()
-            if has_failed
-            else paste_text_wrapper.on_success_fn(),
+            content=agent_outcome,
             additional_kwargs={"error": output} if has_failed else {},
             status="error" if has_failed else "success",
         )
@@ -45,7 +69,7 @@ def get_paste_text_tool(ctx: MobileUseContext):
             update=state.sanitize_update(
                 ctx=ctx,
                 update={
-                    "agents_thoughts": [agent_thought],
+                    "agents_thoughts": [agent_thought, agent_outcome],
                     EXECUTOR_MESSAGES_KEY: [tool_message],
                 },
                 agent="executor",
@@ -57,6 +81,8 @@ def get_paste_text_tool(ctx: MobileUseContext):
 
 paste_text_wrapper = ToolWrapper(
     tool_fn_getter=get_paste_text_tool,
-    on_success_fn=lambda: "Text pasted successfully.",
-    on_failure_fn=lambda: "Failed to paste text.",
+    on_success_fn=lambda input_content: "Text pasted successfully. Here is the actual"
+    + f"content of the text field : {repr(input_content)}",
+    on_failure_fn=lambda input_content: "Failed to paste text."
+    + f"Here is the actual content of the text field : {repr(input_content)}",
 )

minitap/mobile_use/tools/mobile/swipe.py

@@ -1,18 +1,28 @@
+from typing import Annotated
+
 from langchain_core.messages import ToolMessage
 from langchain_core.tools import tool
-from langchain_core.tools.base import InjectedToolCallId
+from langchain_core.tools.base import BaseTool, InjectedToolCallId
 from langgraph.prebuilt import InjectedState
 from langgraph.types import Command
+from pydantic import Field
+
 from minitap.mobile_use.constants import EXECUTOR_MESSAGES_KEY
 from minitap.mobile_use.context import MobileUseContext
-from minitap.mobile_use.controllers.mobile_command_controller import SwipeRequest
+from minitap.mobile_use.controllers.mobile_command_controller import (
+    CoordinatesSelectorRequest,
+    PercentagesSelectorRequest,
+    SwipeDirection,
+    SwipeRequest,
+    SwipeStartEndCoordinatesRequest,
+    SwipeStartEndPercentagesRequest,
+)
 from minitap.mobile_use.controllers.mobile_command_controller import swipe as swipe_controller
 from minitap.mobile_use.graph.state import State
-from minitap.mobile_use.tools.tool_wrapper import ToolWrapper
-from typing import Annotated
+from minitap.mobile_use.tools.tool_wrapper import CompositeToolWrapper
 
 
-def get_swipe_tool(ctx: MobileUseContext):
+def get_swipe_tool(ctx: MobileUseContext) -> BaseTool:
     @tool
     def swipe(
         tool_call_id: Annotated[str, InjectedToolCallId],
@@ -20,9 +30,7 @@ def get_swipe_tool(ctx: MobileUseContext):
         agent_thought: str,
         swipe_request: SwipeRequest,
     ):
-        """
-        Swipes on the screen.
-        """
+        """Swipes on the screen."""
         output = swipe_controller(ctx=ctx, swipe_request=swipe_request)
         has_failed = output is not None
         tool_message = ToolMessage(
@@ -45,8 +53,98 @@ def get_swipe_tool(ctx: MobileUseContext):
     return swipe
 
 
-swipe_wrapper = ToolWrapper(
+def get_composite_swipe_tools(ctx: MobileUseContext) -> list[BaseTool]:
+    """
+    Returns composite swipe tools for use with Vertex AI LLMs.
+    Each tool handles a specific swipe mode to avoid complex Union type issues.
+    """
+
+    @tool
+    def swipe_coordinates(
+        agent_thought: str,
+        tool_call_id: Annotated[str, InjectedToolCallId],
+        state: Annotated[State, InjectedState],
+        start_x: int = Field(description="Start X coordinate in pixels"),
+        start_y: int = Field(description="Start Y coordinate in pixels"),
+        end_x: int = Field(description="End X coordinate in pixels"),
+        end_y: int = Field(description="End Y coordinate in pixels"),
+        duration: int = Field(description="Duration in ms", ge=1, le=10000, default=400),
+    ):
+        """Swipe using pixel coordinates from start position to end position."""
+        swipe_request = SwipeRequest(
+            swipe_mode=SwipeStartEndCoordinatesRequest(
+                start=CoordinatesSelectorRequest(x=start_x, y=start_y),
+                end=CoordinatesSelectorRequest(x=end_x, y=end_y),
+            ),
+            duration=duration,
+        )
+        return get_swipe_tool(ctx=ctx).invoke(
+            input={
+                "tool_call_id": tool_call_id,
+                "state": state,
+                "agent_thought": agent_thought,
+                "swipe_request": swipe_request,
+            }
+        )
+
+    @tool
+    def swipe_percentages(
+        agent_thought: str,
+        tool_call_id: Annotated[str, InjectedToolCallId],
+        state: Annotated[State, InjectedState],
+        start_x_percent: int = Field(description="Start X percent (0-100)", ge=0, le=100),
+        start_y_percent: int = Field(description="Start Y percent (0-100)", ge=0, le=100),
+        end_x_percent: int = Field(description="End X percent (0-100)", ge=0, le=100),
+        end_y_percent: int = Field(description="End Y percent (0-100)", ge=0, le=100),
+        duration: int = Field(description="Duration in ms", ge=1, le=10000, default=400),
+    ):
+        """Swipe using percentage coordinates from start position to end position."""
+        swipe_request = SwipeRequest(
+            swipe_mode=SwipeStartEndPercentagesRequest(
+                start=PercentagesSelectorRequest(
+                    x_percent=start_x_percent, y_percent=start_y_percent
+                ),
+                end=PercentagesSelectorRequest(x_percent=end_x_percent, y_percent=end_y_percent),
+            ),
+            duration=duration,
+        )
+        return get_swipe_tool(ctx=ctx).invoke(
+            input={
+                "tool_call_id": tool_call_id,
+                "state": state,
+                "agent_thought": agent_thought,
+                "swipe_request": swipe_request,
+            }
+        )
+
+    @tool
+    def swipe_direction(
+        agent_thought: str,
+        tool_call_id: Annotated[str, InjectedToolCallId],
+        state: Annotated[State, InjectedState],
+        direction: SwipeDirection,
+        duration: int = Field(description="Duration in ms", ge=1, le=10000, default=400),
+    ):
+        """Swipe in a specific direction across the screen."""
+        swipe_request = SwipeRequest(
+            swipe_mode=direction,
+            duration=duration,
+        )
+        return get_swipe_tool(ctx=ctx).invoke(
+            input={
+                "tool_call_id": tool_call_id,
+                "state": state,
+                "agent_thought": agent_thought,
+                "swipe_request": swipe_request,
+            }
+        )
+
+    return [swipe_coordinates, swipe_percentages, swipe_direction]
+
+
+swipe_wrapper = CompositeToolWrapper(
     tool_fn_getter=get_swipe_tool,
+    composite_tools_fn_getter=get_composite_swipe_tools,
     on_success_fn=lambda: "Swipe is successful.",
     on_failure_fn=lambda: "Failed to swipe.",
 )