minitap-mobile-use 2.0.1__py3-none-any.whl → 2.1.0__py3-none-any.whl

minitap/mobile_use/tools/mobile/clear_text.py

@@ -0,0 +1,277 @@
+from typing import Annotated
+
+from langchain_core.messages import ToolMessage
+from langchain_core.tools import tool
+from langchain_core.tools.base import InjectedToolCallId
+from langgraph.prebuilt import InjectedState
+from langgraph.types import Command
+from pydantic import BaseModel
+
+from minitap.mobile_use.constants import EXECUTOR_MESSAGES_KEY
+from minitap.mobile_use.context import MobileUseContext
+from minitap.mobile_use.controllers.mobile_command_controller import (
+    erase_text as erase_text_controller,
+)
+from minitap.mobile_use.controllers.mobile_command_controller import (
+    get_screen_data,
+)
+from minitap.mobile_use.graph.state import State
+from minitap.mobile_use.tools.tool_wrapper import ToolWrapper
+from minitap.mobile_use.tools.utils import (
+    focus_element_if_needed,
+    move_cursor_to_end_if_bounds,
+)
+from minitap.mobile_use.utils.logger import get_logger
+from minitap.mobile_use.utils.ui_hierarchy import (
+    find_element_by_resource_id,
+    get_element_text,
+    text_input_is_empty,
+)
+
+logger = get_logger(__name__)
+
+MAX_CLEAR_TRIES = 5
+DEFAULT_CHARS_TO_ERASE = 50
+
+
+class ClearTextResult(BaseModel):
+    success: bool
+    error_message: str | None
+    chars_erased: int
+    final_text: str | None
+
+
+class TextClearer:
+    def __init__(self, ctx: MobileUseContext, state: State):
+        self.ctx = ctx
+        self.state = state
+
+    def _refresh_ui_hierarchy(self) -> None:
+        screen_data = get_screen_data(screen_api_client=self.ctx.screen_api_client)
+        self.state.latest_ui_hierarchy = screen_data.elements
+
+    def _get_element_info(self, resource_id: str) -> tuple[object | None, str | None, str | None]:
+        if not self.state.latest_ui_hierarchy:
+            self._refresh_ui_hierarchy()
+
+        if not self.state.latest_ui_hierarchy:
+            return None, None, None
+
+        element = find_element_by_resource_id(
+            ui_hierarchy=self.state.latest_ui_hierarchy, resource_id=resource_id
+        )
+
+        if not element:
+            return None, None, None
+
+        current_text = get_element_text(element)
+        hint_text = get_element_text(element, hint_text=True)
+
+        return element, current_text, hint_text
+
+    def _format_text_with_hint_info(self, text: str | None, hint_text: str | None) -> str | None:
+        if text is None:
+            return None
+
+        is_hint_text = hint_text is not None and hint_text != "" and hint_text == text
+
+        if is_hint_text:
+            return f"{text} (which is the hint text, the input is very likely empty)"
+
+        return text
+
+    def _should_clear_text(self, current_text: str | None, hint_text: str | None) -> bool:
+        return current_text is not None and current_text != "" and current_text != hint_text
+
+    def _prepare_element_for_clearing(self, resource_id: str) -> bool:
+        if not focus_element_if_needed(ctx=self.ctx, resource_id=resource_id):
+            return False
+
+        move_cursor_to_end_if_bounds(ctx=self.ctx, state=self.state, resource_id=resource_id)
+        return True
+
+    def _erase_text_attempt(self, text_length: int) -> str | None:
+        chars_to_erase = text_length + 1
+        logger.info(f"Erasing {chars_to_erase} characters from the input")
+
+        error = erase_text_controller(ctx=self.ctx, nb_chars=chars_to_erase)
+        if error:
+            logger.error(f"Failed to erase text: {error}")
+            return str(error)
+
+        return None
+
+    def _clear_with_retries(
+        self, resource_id: str, initial_text: str, hint_text: str | None
+    ) -> tuple[bool, str | None, int]:
+        current_text = initial_text
+        erased_chars = 0
+
+        for attempt in range(1, MAX_CLEAR_TRIES + 1):
+            logger.info(f"Clear attempt {attempt}/{MAX_CLEAR_TRIES}")
+
+            chars_to_erase = len(current_text) if current_text else DEFAULT_CHARS_TO_ERASE
+            error = self._erase_text_attempt(text_length=chars_to_erase)
+
+            if error:
+                return False, current_text, 0
+            erased_chars += chars_to_erase
+
+            self._refresh_ui_hierarchy()
+            elt = find_element_by_resource_id(
+                ui_hierarchy=self.state.latest_ui_hierarchy or [],
+                resource_id=resource_id,
+            )
+            if elt:
+                current_text = get_element_text(elt)
+                logger.info(f"Current text: {current_text}")
+                if text_input_is_empty(text=current_text, hint_text=hint_text):
+                    break
+
+            move_cursor_to_end_if_bounds(
+                ctx=self.ctx, state=self.state, resource_id=resource_id, elt=elt
+            )
+
+        return True, current_text, erased_chars
+
+    def _create_result(
+        self,
+        success: bool,
+        error_message: str | None,
+        chars_erased: int,
+        final_text: str | None,
+        hint_text: str | None,
+    ) -> ClearTextResult:
+        formatted_final_text = self._format_text_with_hint_info(final_text, hint_text)
+
+        return ClearTextResult(
+            success=success,
+            error_message=error_message,
+            chars_erased=chars_erased,
+            final_text=formatted_final_text,
+        )
+
+    def _handle_no_clearing_needed(
+        self, current_text: str | None, hint_text: str | None
+    ) -> ClearTextResult:
+        return self._create_result(
+            success=True,
+            error_message=None,
+            chars_erased=-1,
+            final_text=current_text,
+            hint_text=hint_text,
+        )
+
+    def _handle_element_not_found(self, resource_id: str, hint_text: str | None) -> ClearTextResult:
+        error = erase_text_controller(ctx=self.ctx)
+        self._refresh_ui_hierarchy()
+
+        _, final_text, _ = self._get_element_info(resource_id)
+
+        return self._create_result(
+            success=error is None,
+            error_message=str(error) if error is not None else None,
+            chars_erased=0,  # Unknown since we don't have initial text
+            final_text=final_text,
+            hint_text=hint_text,
+        )
+
+    def clear_text_by_resource_id(self, resource_id: str) -> ClearTextResult:
+        element, current_text, hint_text = self._get_element_info(resource_id)
+
+        if not element:
+            return self._handle_element_not_found(resource_id, hint_text)
+
+        if not self._should_clear_text(current_text, hint_text):
+            return self._handle_no_clearing_needed(current_text, hint_text)
+
+        if not self._prepare_element_for_clearing(resource_id):
+            return self._create_result(
+                success=False,
+                error_message="Failed to focus element",
+                chars_erased=0,
+                final_text=current_text,
+                hint_text=hint_text,
+            )
+
+        success, final_text, chars_erased = self._clear_with_retries(
+            resource_id=resource_id,
+            initial_text=current_text or "",
+            hint_text=hint_text,
+        )
+
+        error_message = None if success else "Failed to clear text after retries"
+
+        return self._create_result(
+            success=success,
+            error_message=error_message,
+            chars_erased=chars_erased,
+            final_text=final_text,
+            hint_text=hint_text,
+        )
+
+
+def get_clear_text_tool(ctx: MobileUseContext):
+    @tool
+    def clear_text(
+        tool_call_id: Annotated[str, InjectedToolCallId],
+        state: Annotated[State, InjectedState],
+        agent_thought: str,
+        text_input_resource_id: str,
+    ):
+        """
+        Clears all the text from the text field, by focusing it if needed.
+        """
+        clearer = TextClearer(ctx, state)
+        result = clearer.clear_text_by_resource_id(text_input_resource_id)
+
+        content = (
+            clear_text_wrapper.on_failure_fn(result.error_message)
+            if not result.success
+            else clear_text_wrapper.on_success_fn(
+                nb_char_erased=result.chars_erased, new_text_value=result.final_text
+            )
+        )
+
+        tool_message = ToolMessage(
+            tool_call_id=tool_call_id,
+            content=content,
+            additional_kwargs={"error": result.error_message} if not result.success else {},
+            status="error" if not result.success else "success",
+        )
+
+        return Command(
+            update=state.sanitize_update(
+                ctx=ctx,
+                update={
+                    "agents_thoughts": [agent_thought],
+                    EXECUTOR_MESSAGES_KEY: [tool_message],
+                },
+                agent="executor",
+            ),
+        )
+
+    return clear_text
+
+
+def _format_success_message(nb_char_erased: int, new_text_value: str | None) -> str:
+    if nb_char_erased == -1:
+        msg = "No text clearing was needed (the input was already empty)."
+    else:
+        msg = f"Text erased successfully. {nb_char_erased} characters were erased."
+
+    if new_text_value is not None:
+        msg += f" New text in the input is '{new_text_value}'."
+
+    return msg
+
+
+def _format_failure_message(output: str | None) -> str:
+    return "Failed to erase text. " + (str(output) if output else "")
+
+
+clear_text_wrapper = ToolWrapper(
+    tool_fn_getter=get_clear_text_tool,
+    on_success_fn=_format_success_message,
+    on_failure_fn=_format_failure_message,
+)

minitap/mobile_use/tools/mobile/copy_text_from.py

@@ -9,7 +9,7 @@ from minitap.mobile_use.controllers.mobile_command_controller import (
 )
 from minitap.mobile_use.tools.tool_wrapper import ToolWrapper
 from pydantic import Field
-from typing_extensions import Annotated
+from typing import Annotated
 from minitap.mobile_use.context import MobileUseContext
 from minitap.mobile_use.graph.state import State
 from langgraph.prebuilt import InjectedState

minitap/mobile_use/tools/mobile/erase_one_char.py

@@ -0,0 +1,56 @@
+from langchain_core.messages import ToolMessage
+from langchain_core.tools import tool
+from langchain_core.tools.base import InjectedToolCallId
+from langgraph.prebuilt import InjectedState
+from langgraph.types import Command
+from typing import Annotated
+
+from minitap.mobile_use.constants import EXECUTOR_MESSAGES_KEY
+from minitap.mobile_use.context import MobileUseContext
+from minitap.mobile_use.controllers.mobile_command_controller import (
+    erase_text as erase_text_controller,
+)
+from minitap.mobile_use.graph.state import State
+from minitap.mobile_use.tools.tool_wrapper import ToolWrapper
+
+
+def get_erase_one_char_tool(ctx: MobileUseContext):
+    @tool
+    def erase_one_char(
+        tool_call_id: Annotated[str, InjectedToolCallId],
+        state: Annotated[State, InjectedState],
+        agent_thought: str,
+    ):
+        """
+        Erase one character from a text area.
+        It acts the same as pressing backspace a single time.
+        """
+        output = erase_text_controller(ctx=ctx, nb_chars=1)
+        has_failed = output is not None
+        tool_message = ToolMessage(
+            tool_call_id=tool_call_id,
+            content=erase_one_char_wrapper.on_failure_fn()
+            if has_failed
+            else erase_one_char_wrapper.on_success_fn(),
+            additional_kwargs={"error": output} if has_failed else {},
+            status="error" if has_failed else "success",
+        )
+        return Command(
+            update=state.sanitize_update(
+                ctx=ctx,
+                update={
+                    "agents_thoughts": [agent_thought],
+                    EXECUTOR_MESSAGES_KEY: [tool_message],
+                },
+                agent="executor",
+            ),
+        )
+
+    return erase_one_char
+
+
+erase_one_char_wrapper = ToolWrapper(
+    tool_fn_getter=get_erase_one_char_tool,
+    on_success_fn=lambda: "Erased one character successfully.",
+    on_failure_fn=lambda: "Failed to erase one character.",
+)

minitap/mobile_use/tools/mobile/find_packages.py

@@ -9,7 +9,7 @@ from minitap.mobile_use.context import MobileUseContext
 from minitap.mobile_use.controllers.platform_specific_commands_controller import list_packages
 from minitap.mobile_use.graph.state import State
 from minitap.mobile_use.tools.tool_wrapper import ToolWrapper
-from typing_extensions import Annotated
+from typing import Annotated
 
 
 def get_find_packages_tool(ctx: MobileUseContext):

minitap/mobile_use/tools/mobile/input_text.py

@@ -8,28 +8,17 @@ from langchain_core.tools.base import InjectedToolCallId
 from langgraph.prebuilt import InjectedState
 from langgraph.types import Command
 from pydantic import BaseModel
-from typing_extensions import Annotated
+from typing import Annotated
 
 from minitap.mobile_use.constants import EXECUTOR_MESSAGES_KEY
 from minitap.mobile_use.context import MobileUseContext
-from minitap.mobile_use.controllers.mobile_command_controller import (
-    CoordinatesSelectorRequest,
-    IdSelectorRequest,
-    SelectorRequestWithCoordinates,
-    tap,
-)
 from minitap.mobile_use.controllers.mobile_command_controller import (
     input_text as input_text_controller,
 )
 from minitap.mobile_use.graph.state import State
 from minitap.mobile_use.tools.tool_wrapper import ToolWrapper
+from minitap.mobile_use.tools.utils import focus_element_if_needed, move_cursor_to_end_if_bounds
 from minitap.mobile_use.utils.logger import get_logger
-from minitap.mobile_use.utils.ui_hierarchy import (
-    Point,
-    find_element_by_resource_id,
-    get_bounds_for_element,
-    is_element_focused,
-)
 
 logger = get_logger(__name__)
 
@@ -41,71 +30,6 @@ class InputResult(BaseModel):
     error: str | None = None
 
 
-def _focus_element_if_needed(
-    ctx: MobileUseContext,
-    state: State,
-    resource_id: str,
-) -> bool:
-    """
-    Ensures the element identified by `resource_id` is focused.
-    """
-    rich_hierarchy: list[dict] = ctx.hw_bridge_client.get_rich_hierarchy()
-    rich_elt = find_element_by_resource_id(
-        ui_hierarchy=rich_hierarchy,
-        resource_id=resource_id,
-        is_rich_hierarchy=True,
-    )
-    if rich_elt and not is_element_focused(rich_elt):
-        tap(ctx=ctx, selector_request=IdSelectorRequest(id=resource_id))
-        logger.debug(f"Focused (tap) on resource_id={resource_id}")
-        rich_hierarchy = ctx.hw_bridge_client.get_rich_hierarchy()
-        rich_elt = find_element_by_resource_id(
-            ui_hierarchy=rich_hierarchy,
-            resource_id=resource_id,
-            is_rich_hierarchy=True,
-        )
-    if rich_elt and is_element_focused(rich_elt):
-        logger.debug(f"Text input is focused: {resource_id}")
-        return True
-
-    logger.warning(f"Failed to focus resource_id={resource_id}")
-    return False
-
-
-def _move_cursor_to_end_if_bounds(
-    ctx: MobileUseContext,
-    state: State,
-    resource_id: str,
-) -> None:
-    """
-    Best-effort move of the text cursor near the end of the input by tapping the
-    bottom-right area of the focused element (if bounds are available).
-    """
-    elt = find_element_by_resource_id(
-        ui_hierarchy=state.latest_ui_hierarchy or [],
-        resource_id=resource_id,
-    )
-    if not elt:
-        return
-
-    bounds = get_bounds_for_element(elt)
-    if not bounds:
-        return
-
-    logger.debug("Tapping near the end of the input to move the cursor")
-    bottom_right: Point = bounds.get_relative_point(x_percent=0.99, y_percent=0.99)
-    tap(
-        ctx=ctx,
-        selector_request=SelectorRequestWithCoordinates(
-            coordinates=CoordinatesSelectorRequest(
-                x=bottom_right.x,
-                y=bottom_right.y,
-            ),
-        ),
-    )
-    logger.debug(f"Tapped end of input {resource_id} at ({bottom_right.x}, {bottom_right.y})")
-
-
 def _controller_input_text(ctx: MobileUseContext, text: str) -> InputResult:
     """
     Thin wrapper to normalize the controller result.
@@ -132,9 +56,9 @@ def get_input_text_tool(ctx: MobileUseContext):
         - If bounds are available, tap near the end to place the cursor at the end.
         - Type the provided `text` using the controller.
         """
-        focused = _focus_element_if_needed(ctx=ctx, state=state, resource_id=text_input_resource_id)
+        focused = focus_element_if_needed(ctx=ctx, resource_id=text_input_resource_id)
         if focused:
-            _move_cursor_to_end_if_bounds(ctx=ctx, state=state, resource_id=text_input_resource_id)
+            move_cursor_to_end_if_bounds(ctx=ctx, state=state, resource_id=text_input_resource_id)
 
         result = _controller_input_text(ctx=ctx, text=text)
 

minitap/mobile_use/tools/mobile/launch_app.py

@@ -7,7 +7,7 @@ from minitap.mobile_use.controllers.mobile_command_controller import (
     launch_app as launch_app_controller,
 )
 from minitap.mobile_use.tools.tool_wrapper import ToolWrapper
-from typing_extensions import Annotated
+from typing import Annotated
 from minitap.mobile_use.context import MobileUseContext
 from minitap.mobile_use.graph.state import State
 from langgraph.prebuilt import InjectedState

minitap/mobile_use/tools/mobile/long_press_on.py

@@ -1,5 +1,3 @@
-from typing import Optional
-
 from langchain_core.messages import ToolMessage
 from langchain_core.tools import tool
 from langchain_core.tools.base import InjectedToolCallId
@@ -13,7 +11,7 @@ from minitap.mobile_use.controllers.mobile_command_controller import (
 )
 from minitap.mobile_use.graph.state import State
 from minitap.mobile_use.tools.tool_wrapper import ToolWrapper
-from typing_extensions import Annotated
+from typing import Annotated
 
 
 def get_long_press_on_tool(ctx: MobileUseContext):
@@ -23,7 +21,7 @@ def get_long_press_on_tool(ctx: MobileUseContext):
         state: Annotated[State, InjectedState],
         agent_thought: str,
         selector_request: SelectorRequest,
-        index: Optional[int] = None,
+        index: int | None = None,
     ):
         """
         Long press on a UI element identified by the given selector.

minitap/mobile_use/tools/mobile/open_link.py

@@ -10,7 +10,7 @@ from minitap.mobile_use.controllers.mobile_command_controller import (
 )
 from minitap.mobile_use.graph.state import State
 from minitap.mobile_use.tools.tool_wrapper import ToolWrapper
-from typing_extensions import Annotated
+from typing import Annotated
 
 
 def get_open_link_tool(ctx: MobileUseContext):

minitap/mobile_use/tools/mobile/paste_text.py

@@ -10,7 +10,7 @@ from minitap.mobile_use.controllers.mobile_command_controller import (
 from minitap.mobile_use.graph.state import State
 from langgraph.prebuilt import InjectedState
 from minitap.mobile_use.tools.tool_wrapper import ToolWrapper
-from typing_extensions import Annotated
+from typing import Annotated
 
 
 def get_paste_text_tool(ctx: MobileUseContext):

minitap/mobile_use/tools/mobile/press_key.py

@@ -11,7 +11,7 @@ from minitap.mobile_use.controllers.mobile_command_controller import (
 )
 from minitap.mobile_use.graph.state import State
 from minitap.mobile_use.tools.tool_wrapper import ToolWrapper
-from typing_extensions import Annotated
+from typing import Annotated
 
 
 def get_press_key_tool(ctx: MobileUseContext):

minitap/mobile_use/tools/mobile/stop_app.py

@@ -1,5 +1,3 @@
-from typing import Optional
-
 from langchain_core.messages import ToolMessage
 from langchain_core.tools import tool
 from langchain_core.tools.base import InjectedToolCallId
@@ -10,7 +8,7 @@ from minitap.mobile_use.context import MobileUseContext
 from minitap.mobile_use.controllers.mobile_command_controller import stop_app as stop_app_controller
 from minitap.mobile_use.graph.state import State
 from minitap.mobile_use.tools.tool_wrapper import ToolWrapper
-from typing_extensions import Annotated
+from typing import Annotated
 
 
 def get_stop_app_tool(ctx: MobileUseContext):
@@ -19,7 +17,7 @@ def get_stop_app_tool(ctx: MobileUseContext):
         tool_call_id: Annotated[str, InjectedToolCallId],
         state: Annotated[State, InjectedState],
         agent_thought: str,
-        package_name: Optional[str] = None,
+        package_name: str | None = None,
     ):
         """
         Stops current application if it is running.

minitap/mobile_use/tools/mobile/swipe.py

@@ -9,7 +9,7 @@ from minitap.mobile_use.controllers.mobile_command_controller import SwipeReques
 from minitap.mobile_use.controllers.mobile_command_controller import swipe as swipe_controller
 from minitap.mobile_use.graph.state import State
 from minitap.mobile_use.tools.tool_wrapper import ToolWrapper
-from typing_extensions import Annotated
+from typing import Annotated
 
 
 def get_swipe_tool(ctx: MobileUseContext):

minitap/mobile_use/tools/mobile/take_screenshot.py

@@ -11,7 +11,7 @@ from minitap.mobile_use.controllers.mobile_command_controller import (
 from minitap.mobile_use.graph.state import State
 from minitap.mobile_use.tools.tool_wrapper import ToolWrapper
 from minitap.mobile_use.utils.media import compress_base64_jpeg
-from typing_extensions import Annotated
+from typing import Annotated
 
 
 def get_take_screenshot_tool(ctx: MobileUseContext):

minitap/mobile_use/tools/mobile/tap.py

@@ -1,5 +1,3 @@
-from typing import Optional
-
 from langchain_core.messages import ToolMessage
 from langchain_core.tools import tool
 from langchain_core.tools.base import InjectedToolCallId
@@ -11,7 +9,7 @@ from minitap.mobile_use.controllers.mobile_command_controller import SelectorReq
 from minitap.mobile_use.controllers.mobile_command_controller import tap as tap_controller
 from minitap.mobile_use.graph.state import State
 from minitap.mobile_use.tools.tool_wrapper import ToolWrapper
-from typing_extensions import Annotated
+from typing import Annotated
 
 
 def get_tap_tool(ctx: MobileUseContext):
@@ -21,7 +19,7 @@ def get_tap_tool(ctx: MobileUseContext):
         state: Annotated[State, InjectedState],
         agent_thought: str,
         selector_request: SelectorRequest,
-        index: Optional[int] = None,
+        index: int | None = None,
     ):
         """
         Taps on a selector.

minitap/mobile_use/tools/mobile/wait_for_animation_to_end.py

@@ -1,5 +1,3 @@
-from typing import Optional
-
 from langchain_core.messages import ToolMessage
 from langchain_core.tools import tool
 from langchain_core.tools.base import InjectedToolCallId
@@ -13,7 +11,7 @@ from minitap.mobile_use.controllers.mobile_command_controller import (
 )
 from minitap.mobile_use.graph.state import State
 from minitap.mobile_use.tools.tool_wrapper import ToolWrapper
-from typing_extensions import Annotated
+from typing import Annotated
 
 
 def get_wait_for_animation_to_end_tool(ctx: MobileUseContext):
@@ -22,7 +20,7 @@ def get_wait_for_animation_to_end_tool(ctx: MobileUseContext):
         tool_call_id: Annotated[str, InjectedToolCallId],
         state: Annotated[State, InjectedState],
         agent_thought: str,
-        timeout: Optional[WaitTimeout],
+        timeout: WaitTimeout | None,
     ):
         """
         Waits for ongoing animations or videos to finish before continuing.

minitap/mobile_use/tools/tool_wrapper.py

@@ -1,4 +1,4 @@
-from typing import Callable
+from collections.abc import Callable
 
 from langchain_core.tools import BaseTool
 from pydantic import BaseModel