minitap-mobile-use 2.1.0__py3-none-any.whl → 2.3.0__py3-none-any.whl

minitap/mobile_use/tools/test_utils.py

@@ -0,0 +1,351 @@
+import sys
+from unittest.mock import Mock, patch
+
+import pytest
+
+# Mock the problematic langgraph import at module level
+sys.modules["langgraph.prebuilt.chat_agent_executor"] = Mock()
+sys.modules["minitap.mobile_use.graph.state"] = Mock()
+
+from minitap.mobile_use.context import MobileUseContext  # noqa: E402
+from minitap.mobile_use.controllers.mobile_command_controller import (  # noqa: E402
+    IdSelectorRequest,
+    SelectorRequestWithCoordinates,
+)
+from minitap.mobile_use.tools.utils import (  # noqa: E402
+    focus_element_if_needed,
+    move_cursor_to_end_if_bounds,
+)
+from minitap.mobile_use.utils.ui_hierarchy import ElementBounds  # noqa: E402
+
+
+@pytest.fixture
+def mock_context():
+    """Create a mock MobileUseContext for testing."""
+    ctx = Mock(spec=MobileUseContext)
+    ctx.hw_bridge_client = Mock()
+    return ctx
+
+
+@pytest.fixture
+def mock_state():
+    """Create a mock State for testing."""
+    state = Mock()
+    state.latest_ui_hierarchy = []
+    return state
+
+
+@pytest.fixture
+def sample_element():
+    """Create a sample UI element for testing."""
+    return {
+        "resourceId": "com.example:id/text_input",
+        "text": "Sample text",
+        "bounds": {"x": 100, "y": 200, "width": 300, "height": 50},
+        "focused": "false",
+    }
+
+
+@pytest.fixture
+def sample_rich_element():
+    """Create a sample rich UI element for testing."""
+    return {
+        "attributes": {
+            "resource-id": "com.example:id/text_input",
+            "focused": "false",
+            "text": "Sample text",
+        },
+        "children": [],
+    }
+
+
+class TestMoveCursorToEndIfBounds:
+    """Test cases for move_cursor_to_end_if_bounds function."""
+
+    @patch("minitap.mobile_use.tools.utils.tap")
+    @patch("minitap.mobile_use.tools.utils.find_element_by_resource_id")
+    def test_move_cursor_with_resource_id(
+        self, mock_find_element, mock_tap, mock_context, mock_state, sample_element
+    ):
+        """Test moving cursor using resource_id (highest priority)."""
+        mock_state.latest_ui_hierarchy = [sample_element]
+        mock_find_element.return_value = sample_element
+
+        result = move_cursor_to_end_if_bounds(
+            ctx=mock_context,
+            state=mock_state,
+            text_input_resource_id="com.example:id/text_input",
+            text_input_coordinates=None,
+            text_input_text=None,
+        )
+
+        mock_find_element.assert_called_once_with(
+            ui_hierarchy=[sample_element], resource_id="com.example:id/text_input"
+        )
+        mock_tap.assert_called_once()
+        call_args = mock_tap.call_args[1]
+        selector_request = call_args["selector_request"]
+        assert isinstance(selector_request, SelectorRequestWithCoordinates)
+        coords = selector_request.coordinates
+        assert coords.x == 397  # 100 + 300 * 0.99
+        assert coords.y == 249  # 200 + 50 * 0.99
+        assert result == sample_element
+
+    @patch("minitap.mobile_use.tools.utils.tap")
+    @patch("minitap.mobile_use.tools.utils.find_element_by_resource_id")
+    def test_move_cursor_with_coordinates_only(
+        self, mock_find_element, mock_tap, mock_context, mock_state
+    ):
+        """Test moving cursor when only coordinates are provided."""
+        bounds = ElementBounds(x=50, y=150, width=200, height=40)
+
+        result = move_cursor_to_end_if_bounds(
+            ctx=mock_context,
+            state=mock_state,
+            text_input_resource_id=None,
+            text_input_coordinates=bounds,
+            text_input_text=None,
+        )
+
+        mock_find_element.assert_not_called()
+        mock_tap.assert_called_once()
+        call_args = mock_tap.call_args[1]
+        selector_request = call_args["selector_request"]
+        coords = selector_request.coordinates
+        assert coords.x == 248  # 50 + 200 * 0.99
+        assert coords.y == 189  # 150 + 40 * 0.99
+        assert result is None  # No element is returned when using coords directly
+
+    @patch("minitap.mobile_use.tools.utils.tap")
+    @patch("minitap.mobile_use.tools.utils.find_element_by_text")
+    def test_move_cursor_with_text_only_success(
+        self, mock_find_text, mock_tap, mock_context, mock_state, sample_element
+    ):
+        """Test moving cursor when only text is provided and succeeds."""
+        mock_state.latest_ui_hierarchy = [sample_element]
+        mock_find_text.return_value = sample_element
+
+        result = move_cursor_to_end_if_bounds(
+            ctx=mock_context,
+            state=mock_state,
+            text_input_resource_id=None,
+            text_input_coordinates=None,
+            text_input_text="Sample text",
+        )
+
+        mock_find_text.assert_called_once_with([sample_element], "Sample text")
+        mock_tap.assert_called_once()
+        assert result == sample_element
+
+    @patch("minitap.mobile_use.tools.utils.tap")
+    @patch("minitap.mobile_use.tools.utils.find_element_by_text")
+    def test_move_cursor_with_text_only_element_not_found(
+        self, mock_find_text, mock_tap, mock_context, mock_state
+    ):
+        """Test when searching by text finds no element."""
+        mock_state.latest_ui_hierarchy = []
+        mock_find_text.return_value = None
+
+        result = move_cursor_to_end_if_bounds(
+            ctx=mock_context,
+            state=mock_state,
+            text_input_resource_id=None,
+            text_input_coordinates=None,
+            text_input_text="Nonexistent text",
+        )
+
+        mock_tap.assert_not_called()
+        assert result is None
+
+    @patch("minitap.mobile_use.tools.utils.tap")
+    @patch("minitap.mobile_use.tools.utils.find_element_by_text")
+    def test_move_cursor_with_text_only_no_bounds(
+        self, mock_find_text, mock_tap, mock_context, mock_state
+    ):
+        """Test when element is found by text but has no bounds."""
+        element_no_bounds = {"text": "Text without bounds"}
+        mock_state.latest_ui_hierarchy = [element_no_bounds]
+        mock_find_text.return_value = element_no_bounds
+
+        result = move_cursor_to_end_if_bounds(
+            ctx=mock_context,
+            state=mock_state,
+            text_input_resource_id=None,
+            text_input_coordinates=None,
+            text_input_text="Text without bounds",
+        )
+
+        mock_tap.assert_not_called()
+        assert result is None  # Should return None as no action was taken
+
+    @patch("minitap.mobile_use.tools.utils.find_element_by_resource_id")
+    def test_move_cursor_element_not_found_by_id(self, mock_find_element, mock_context, mock_state):
+        """Test when element is not found by resource_id."""
+        mock_find_element.return_value = None
+
+        result = move_cursor_to_end_if_bounds(
+            ctx=mock_context,
+            state=mock_state,
+            text_input_resource_id="com.example:id/nonexistent",
+            text_input_coordinates=None,
+            text_input_text=None,
+        )
+
+        assert result is None
+
+
+class TestFocusElementIfNeeded:
+    """Test cases for focus_element_if_needed function."""
+
+    @patch("minitap.mobile_use.tools.utils.tap")
+    @patch("minitap.mobile_use.tools.utils.find_element_by_resource_id")
+    def test_focus_element_already_focused(
+        self, mock_find_element, mock_tap, mock_context, sample_rich_element
+    ):
+        """Test when element is already focused."""
+        focused_element = sample_rich_element.copy()
+        focused_element["attributes"]["focused"] = "true"
+
+        mock_context.hw_bridge_client.get_rich_hierarchy.return_value = [focused_element]
+        mock_find_element.return_value = focused_element["attributes"]
+
+        result = focus_element_if_needed(
+            ctx=mock_context,
+            input_resource_id="com.example:id/text_input",
+            input_coordinates=None,
+            input_text=None,
+        )
+
+        mock_tap.assert_not_called()
+        assert result is True
+        mock_context.hw_bridge_client.get_rich_hierarchy.assert_called_once()
+
+    @patch("minitap.mobile_use.tools.utils.tap")
+    @patch("minitap.mobile_use.tools.utils.find_element_by_resource_id")
+    def test_focus_element_needs_focus_success(
+        self, mock_find_element, mock_tap, mock_context, sample_rich_element
+    ):
+        """Test when element needs focus and focusing succeeds."""
+        unfocused_element = sample_rich_element
+        focused_element = {
+            "attributes": {
+                "resource-id": "com.example:id/text_input",
+                "focused": "true",
+            },
+            "children": [],
+        }
+
+        mock_context.hw_bridge_client.get_rich_hierarchy.side_effect = [
+            [unfocused_element],
+            [focused_element],
+        ]
+        mock_find_element.side_effect = [
+            unfocused_element["attributes"],
+            focused_element["attributes"],
+        ]
+
+        result = focus_element_if_needed(
+            ctx=mock_context,
+            input_resource_id="com.example:id/text_input",
+            input_coordinates=None,
+            input_text=None,
+        )
+
+        mock_tap.assert_called_once_with(
+            ctx=mock_context,
+            selector_request=IdSelectorRequest(id="com.example:id/text_input"),
+        )
+        assert mock_context.hw_bridge_client.get_rich_hierarchy.call_count == 2
+        assert result is True
+
+    @patch("minitap.mobile_use.tools.utils.tap")
+    @patch("minitap.mobile_use.tools.utils.logger")
+    @patch("minitap.mobile_use.tools.utils.find_element_by_resource_id")
+    def test_focus_id_and_text_mismatch_fallback_to_text(
+        self, mock_find_id, mock_logger, mock_tap, mock_context, sample_rich_element
+    ):
+        """Test fallback when resource_id and text point to different elements."""
+        element_from_id = sample_rich_element["attributes"].copy()
+        element_from_id["text"] = "Different text"
+
+        # L'élément qui sera trouvé par le texte doit avoir des "bounds"
+        element_from_text = sample_rich_element.copy()
+        element_from_text["bounds"] = {"x": 10, "y": 20, "width": 100, "height": 30}
+
+        mock_context.hw_bridge_client.get_rich_hierarchy.return_value = [element_from_text]
+        mock_find_id.return_value = element_from_id
+
+        with patch("minitap.mobile_use.tools.utils.find_element_by_text") as mock_find_text:
+            mock_find_text.return_value = element_from_text  # Trouvé par le texte
+
+            result = focus_element_if_needed(
+                ctx=mock_context,
+                input_resource_id="com.example:id/text_input",
+                input_coordinates=None,
+                input_text="Sample text",  # Le texte correct à rechercher
+            )
+
+            mock_logger.warning.assert_called_once()
+            # Maintenant, tap devrait être appelé car l'élément trouvé a des "bounds"
+            mock_tap.assert_called_once()
+            assert result is True
+
+    @patch("minitap.mobile_use.tools.utils.tap")
+    @patch("minitap.mobile_use.tools.utils.find_element_by_text")
+    def test_focus_fallback_to_text(
+        self, mock_find_text, mock_tap, mock_context, sample_rich_element
+    ):
+        """Test fallback to focusing using text."""
+        # L'élément doit avoir des "bounds" au premier niveau pour
+        # que get_bounds_for_element fonctionne
+        element_with_bounds = sample_rich_element.copy()
+        element_with_bounds["bounds"] = {"x": 10, "y": 20, "width": 100, "height": 30}
+
+        mock_context.hw_bridge_client.get_rich_hierarchy.return_value = [element_with_bounds]
+        mock_find_text.return_value = element_with_bounds
+
+        result = focus_element_if_needed(
+            ctx=mock_context,
+            input_resource_id=None,
+            input_coordinates=None,
+            input_text="Sample text",
+        )
+
+        mock_find_text.assert_called_once()
+        mock_tap.assert_called_once()
+        call_args = mock_tap.call_args[1]
+        selector = call_args["selector_request"]
+        # Vérifie que le tap se fait bien au centre des "bounds"
+        assert selector.coordinates.x == 60  # 10 + 100/2
+        assert selector.coordinates.y == 35  # 20 + 30/2
+        assert result is True
+
+    @patch("minitap.mobile_use.tools.utils.logger")
+    def test_focus_all_locators_fail(self, mock_logger, mock_context):
+        """Test failure when no locator can find an element."""
+        mock_context.hw_bridge_client.get_rich_hierarchy.return_value = []
+
+        # Mock find_element functions to return None
+        with (
+            patch("minitap.mobile_use.tools.utils.find_element_by_resource_id") as mock_find_id,
+            patch("minitap.mobile_use.tools.utils.find_element_by_text") as mock_find_text,
+        ):
+            mock_find_id.return_value = None
+            mock_find_text.return_value = None
+
+            result = focus_element_if_needed(
+                ctx=mock_context,
+                input_resource_id="nonexistent",
+                input_coordinates=None,
+                input_text="nonexistent",
+            )
+
+        mock_logger.error.assert_called_once_with(
+            "Failed to focus element. No valid locator"
+            + "(resource_id, coordinates, or text) succeeded."
+        )
+        assert result is False
+
+
+if __name__ == "__main__":
+    pytest.main([__file__])

minitap/mobile_use/tools/tool_wrapper.py

@@ -2,6 +2,7 @@ from collections.abc import Callable
 
 from langchain_core.tools import BaseTool
 from pydantic import BaseModel
+
 from minitap.mobile_use.context import MobileUseContext
 
 
@@ -9,3 +10,7 @@ class ToolWrapper(BaseModel):
     tool_fn_getter: Callable[[MobileUseContext], BaseTool]
     on_success_fn: Callable[..., str]
     on_failure_fn: Callable[..., str]
+
+
+class CompositeToolWrapper(ToolWrapper):
+    composite_tools_fn_getter: Callable[[MobileUseContext], list[BaseTool]]

minitap/mobile_use/tools/utils.py

@@ -10,38 +10,47 @@ from minitap.mobile_use.controllers.mobile_command_controller import (
 from minitap.mobile_use.graph.state import State
 from minitap.mobile_use.utils.logger import get_logger
 from minitap.mobile_use.utils.ui_hierarchy import (
+    ElementBounds,
     Point,
     find_element_by_resource_id,
     get_bounds_for_element,
+    get_element_text,
     is_element_focused,
 )
 
 logger = get_logger(__name__)
 
 
-def move_cursor_to_end_if_bounds(
-    ctx: MobileUseContext,
-    state: State,
-    resource_id: str,
-    elt: dict | None = None,
-) -> dict | None:
+def find_element_by_text(ui_hierarchy: list[dict], text: str) -> dict | None:
     """
-    Best-effort move of the text cursor near the end of the input by tapping the
-    bottom-right area of the focused element (if bounds are available).
+    Find a UI element by its text content (adapted to both flat and rich hierarchy)
+
+    This function performs a recursive, case-insensitive partial search.
+
+    Args:
+        ui_hierarchy: List of UI element dictionaries.
+        text: The text content to search for.
+
+    Returns:
+        The complete UI element dictionary if found, None otherwise.
     """
-    if not elt:
-        elt = find_element_by_resource_id(
-            ui_hierarchy=state.latest_ui_hierarchy or [],
-            resource_id=resource_id,
-        )
-    if not elt:
-        return
 
-    bounds = get_bounds_for_element(elt)
-    if not bounds:
-        return elt
+    def search_recursive(elements: list[dict]) -> dict | None:
+        for element in elements:
+            if isinstance(element, dict):
+                src = element.get("attributes", element)
+                if text and text.lower() == src.get("text", "").lower():
+                    return element
+                if (children := element.get("children", [])) and (
+                    found := search_recursive(children)
+                ):
+                    return found
+        return None
+
+    return search_recursive(ui_hierarchy)
+
 
-    logger.debug("Tapping near the end of the input to move the cursor")
+def tap_bottom_right_of_element(bounds: ElementBounds, ctx: MobileUseContext):
     bottom_right: Point = bounds.get_relative_point(x_percent=0.99, y_percent=0.99)
     tap(
         ctx=ctx,
@@ -52,35 +61,133 @@ def move_cursor_to_end_if_bounds(
             ),
         ),
     )
-    logger.debug(f"Tapped end of input {resource_id} at ({bottom_right.x}, {bottom_right.y})")
-    return elt
+
+
+def move_cursor_to_end_if_bounds(
+    ctx: MobileUseContext,
+    state: State,
+    text_input_resource_id: str | None,
+    text_input_coordinates: ElementBounds | None,
+    text_input_text: str | None,
+    elt: dict | None = None,
+) -> dict | None:
+    """
+    Best-effort move of the text cursor near the end of the input by tapping the
+    bottom-right area of the focused element (if bounds are available).
+    """
+    if text_input_resource_id:
+        if not elt:
+            elt = find_element_by_resource_id(
+                ui_hierarchy=state.latest_ui_hierarchy or [],
+                resource_id=text_input_resource_id,
+            )
+        if not elt:
+            return
+
+        bounds = get_bounds_for_element(elt)
+        if not bounds:
+            return elt
+
+        logger.debug("Tapping near the end of the input to move the cursor")
+        tap_bottom_right_of_element(bounds=bounds, ctx=ctx)
+        logger.debug(f"Tapped end of input {text_input_resource_id}")
+        return elt
+
+    if text_input_coordinates:
+        tap_bottom_right_of_element(text_input_coordinates, ctx=ctx)
+        logger.debug("Tapped end of input by coordinates")
+        return elt
+
+    if text_input_text:
+        text_elt = find_element_by_text(state.latest_ui_hierarchy or [], text_input_text)
+        if text_elt:
+            bounds = get_bounds_for_element(text_elt)
+            if bounds:
+                tap_bottom_right_of_element(bounds=bounds, ctx=ctx)
+                logger.debug(f"Tapped end of input that had text'{text_input_text}'")
+                return text_elt
+        return None
+
+    return None
 
 
 def focus_element_if_needed(
     ctx: MobileUseContext,
-    resource_id: str,
+    input_resource_id: str | None,
+    input_coordinates: ElementBounds | None,
+    input_text: str | None,
 ) -> bool:
     """
-    Ensures the element identified by `resource_id` is focused.
+    Ensures the element is focused, with a sanity check to prevent trusting misleading IDs.
     """
-    rich_hierarchy: list[dict] = ctx.hw_bridge_client.get_rich_hierarchy()
-    rich_elt = find_element_by_resource_id(
-        ui_hierarchy=rich_hierarchy,
-        resource_id=resource_id,
-        is_rich_hierarchy=True,
-    )
-    if rich_elt and not is_element_focused(rich_elt):
-        tap(ctx=ctx, selector_request=IdSelectorRequest(id=resource_id))
-        logger.debug(f"Focused (tap) on resource_id={resource_id}")
-        rich_hierarchy = ctx.hw_bridge_client.get_rich_hierarchy()
-        rich_elt = find_element_by_resource_id(
-            ui_hierarchy=rich_hierarchy,
-            resource_id=resource_id,
-            is_rich_hierarchy=True,
+    rich_hierarchy = ctx.hw_bridge_client.get_rich_hierarchy()
+
+    elt_from_id = None
+    if input_resource_id:
+        elt_from_id = find_element_by_resource_id(
+            ui_hierarchy=rich_hierarchy, resource_id=input_resource_id, is_rich_hierarchy=True
+        )
+
+    if elt_from_id and input_text:
+        text_from_id_elt = get_element_text(elt_from_id)
+        if not text_from_id_elt or input_text.lower() != text_from_id_elt.lower():
+            logger.warning(
+                f"ID '{input_resource_id}' and text '{input_text}'"
+                + "seem to be on different elements. "
+                "Ignoring the resource_id and falling back to other locators."
+            )
+            elt_from_id = None
+
+    if elt_from_id:
+        if not is_element_focused(elt_from_id):
+            tap(ctx=ctx, selector_request=IdSelectorRequest(id=input_resource_id))  # type: ignore
+            logger.debug(f"Focused (tap) on resource_id={input_resource_id}")
+            rich_hierarchy = ctx.hw_bridge_client.get_rich_hierarchy()
+            elt_from_id = find_element_by_resource_id(
+                ui_hierarchy=rich_hierarchy,
+                resource_id=input_resource_id,  # type: ignore
+                is_rich_hierarchy=True,
+            )
+        if elt_from_id and is_element_focused(elt_from_id):
+            logger.debug(f"Text input is focused: {input_resource_id}")
+            return True
+
+        logger.warning(f"Failed to focus using resource_id='{input_resource_id}'. Fallback...")
+
+    if input_coordinates:
+        relative_point = input_coordinates.get_center()
+        tap(
+            ctx=ctx,
+            selector_request=SelectorRequestWithCoordinates(
+                coordinates=CoordinatesSelectorRequest(
+                    x=relative_point.x,
+                    y=relative_point.y,
+                ),
+            ),
         )
-    if rich_elt and is_element_focused(rich_elt):
-        logger.debug(f"Text input is focused: {resource_id}")
+        logger.debug(f"Tapped on coordinates ({relative_point.x}, {relative_point.y}) to focus.")
         return True
 
-    logger.warning(f"Failed to focus resource_id={resource_id}")
+    if input_text:
+        text_elt = find_element_by_text(rich_hierarchy, input_text)
+        if text_elt:
+            bounds = get_bounds_for_element(text_elt)
+            if bounds:
+                relative_point = bounds.get_center()
+                tap(
+                    ctx=ctx,
+                    selector_request=SelectorRequestWithCoordinates(
+                        coordinates=CoordinatesSelectorRequest(
+                            x=relative_point.x,
+                            y=relative_point.y,
+                        ),
+                    ),
+                )
+                logger.debug(f"Tapped on text element '{input_text}' to focus.")
+                return True
+
+    logger.error(
+        "Failed to focus element. No valid locator"
+        + "(resource_id, coordinates, or text) succeeded."
+    )
     return False

minitap/mobile_use/utils/recorder.py

@@ -45,12 +45,5 @@ def record_interaction(ctx: MobileUseContext, response: BaseMessage):
     return "Screenshot recorded successfully"
 
 
-def log_agent_thought(prefix: str, agent_thought: str):
-    if prefix:
-        prefix = prefix[0].upper() + prefix[1:]
-    else:
-        prefix = "New agent thought"
-    logger.info(
-        f"💭 {Fore.LIGHTMAGENTA_EX + Style.BRIGHT}{prefix}{Style.RESET_ALL}: "
-        f"{Fore.LIGHTMAGENTA_EX}{agent_thought}{Style.RESET_ALL}"
-    )
+def log_agent_thought(agent_thought: str):
+    logger.info(f"💭 {Fore.LIGHTMAGENTA_EX}{agent_thought}{Style.RESET_ALL}")