minitap-mobile-use 2.5.3__py3-none-any.whl → 2.7.0__py3-none-any.whl

minitap/mobile_use/sdk/services/platform.py

@@ -23,6 +23,7 @@ from minitap.mobile_use.sdk.types.platform import (
 )
 from minitap.mobile_use.sdk.types.task import (
     AgentProfile,
+    CloudDevicePlatformTaskRequest,
     ManualTaskConfig,
     PlatformTaskInfo,
     PlatformTaskRequest,
@@ -60,6 +61,10 @@ class PlatformService:
 
     async def create_task_run(self, request: PlatformTaskRequest) -> PlatformTaskInfo:
         try:
+            virtual_mobile_id = None
+            if isinstance(request, CloudDevicePlatformTaskRequest):
+                virtual_mobile_id = request.virtual_mobile_id
+
             # Check if task is a string (fetch from platform) or ManualTaskConfig (create manually)
             if isinstance(request.task, str):
                 # Fetch task from platform
@@ -87,7 +92,11 @@ class PlatformService:
                     thoughts_output_path=request.thoughts_output_path,
                 )
 
-                task_run = await self._create_task_run(task=task, profile=profile)
+                task_run = await self._create_task_run(
+                    task=task,
+                    profile=profile,
+                    virtual_mobile_id=virtual_mobile_id,
+                )
             else:
                 # Create task manually from ManualTaskConfig
                 logger.info(f"Creating manual task with goal: {request.task.goal}")
@@ -113,6 +122,7 @@ class PlatformService:
                 task_run = await self._create_manual_task_run(
                     manual_config=request.task,
                     profile=profile,
+                    virtual_mobile_id=virtual_mobile_id,
                 )
 
             return PlatformTaskInfo(
@@ -244,12 +254,14 @@ class PlatformService:
         self,
         task: TaskResponse,
         profile: LLMProfileResponse,
+        virtual_mobile_id: str | None = None,
     ) -> TaskRunResponse:
         try:
             logger.info(f"Creating task run for task: {task.name}")
             task_run = CreateTaskRunRequest(
                 task_id=task.id,
                 llm_profile_id=profile.id,
+                virtual_mobile_id=virtual_mobile_id,
             )
             response = await self._client.post(url="v1/task-runs", json=task_run.model_dump())
             response.raise_for_status()
@@ -264,6 +276,7 @@ class PlatformService:
         self,
         manual_config: ManualTaskConfig,
         profile: LLMProfileResponse,
+        virtual_mobile_id: str | None = None,
     ) -> TaskRunResponse:
         """
         Create an orphan task run from a manual task configuration.
@@ -277,6 +290,7 @@ class PlatformService:
                 "inputPrompt": manual_config.goal,
                 "outputDescription": manual_config.output_description,
                 "llmProfileId": profile.id,
+                "virtualMobileId": virtual_mobile_id,
             }
 
             response = await self._client.post(url="v1/task-runs/orphan", json=orphan_payload)

minitap/mobile_use/sdk/types/platform.py

@@ -63,6 +63,7 @@ class CreateTaskRunRequest(BaseApiModel):
 
     task_id: str = Field(..., description="ID of the task to run")
     llm_profile_id: str = Field(..., description="LLM profile ID to use")
+    virtual_mobile_id: str | None = Field(None, description="Virtual mobile ID to use")
 
 
 class UpdateTaskRunStatusRequest(BaseApiModel):

minitap/mobile_use/sdk/types/task.py

@@ -2,12 +2,13 @@
 Task-related type definitions for the Mobile-use SDK.
 """
 
+from asyncio import Event
 from collections.abc import Callable, Coroutine
 from datetime import datetime
 from pathlib import Path
 from typing import Any, TypeVar, overload
 
-from pydantic import BaseModel, Field
+from pydantic import BaseModel, ConfigDict, Field
 
 from minitap.mobile_use.config import LLMConfig, get_default_llm_config
 from minitap.mobile_use.constants import RECURSION_LIMIT
@@ -135,6 +136,14 @@ class PlatformTaskRequest[TOutput](TaskRequestBase):
     api_key: str | None = None
 
 
+class CloudDevicePlatformTaskRequest[TOutput](PlatformTaskRequest[TOutput]):
+    model_config = ConfigDict(arbitrary_types_allowed=True)
+
+    task_run_id_available_event: Event = Event()
+    task_run_id: str | None = None
+    virtual_mobile_id: str | None = None
+
+
 class TaskResult(BaseModel):
     """
     Result of a mobile automation task.

minitap/mobile_use/servers/device_hardware_bridge.py

@@ -135,23 +135,30 @@ class DeviceHardwareBridge:
             print(f"[Maestro Studio ERROR]: {line}")
             self.output.append(line)
 
-            if "device offline" in line.lower():
+            lower_line = line.lower()
+
+            # Ignore known benign warnings (common on macOS/JDK 21+)
+            if line.startswith("WARNING:") or (
+                "restricted method" in lower_line
+                or "jansi" in lower_line
+                or "enable-native-access" in lower_line
+                or "java.lang.system::load" in lower_line
+            ):
+                continue
+
+            if "device offline" in lower_line:
                 with self.lock:
                     self.status = BridgeStatus.FAILED
                 if self.process:
                     self.process.kill()
                 break
 
-            if "address already in use" in line.lower():
+            if "address already in use" in lower_line:
                 with self.lock:
                     self.status = BridgeStatus.PORT_IN_USE
                 if self.process:
                     self.process.kill()
                 break
-            else:
-                with self.lock:
-                    if self.status == BridgeStatus.STARTING:
-                        self.status = BridgeStatus.FAILED
 
     def _wait_for_health_check(self, retries=5, delay=2):
         health_url = f"http://localhost:{DEVICE_HARDWARE_BRIDGE_PORT}/api/banner-message"

minitap/mobile_use/services/llm.py

@@ -13,6 +13,7 @@ from minitap.mobile_use.config import (
     AgentNode,
     AgentNodeWithFallback,
     LLMUtilsNode,
+    LLMUtilsNodeWithFallback,
     LLMWithFallback,
     settings,
 )
@@ -169,8 +170,9 @@ def get_llm(
 @overload
 def get_llm(
     ctx: MobileUseContext,
-    name: AgentNode,
+    name: LLMUtilsNode,
     *,
+    is_utils: Literal[True],
     temperature: float = 1,
 ) -> BaseChatModel: ...
 
@@ -178,9 +180,10 @@ def get_llm(
 @overload
 def get_llm(
     ctx: MobileUseContext,
-    name: LLMUtilsNode,
+    name: LLMUtilsNodeWithFallback,
     *,
     is_utils: Literal[True],
+    use_fallback: bool = False,
     temperature: float = 1,
 ) -> BaseChatModel: ...
 

minitap/mobile_use/tools/index.py

@@ -2,10 +2,9 @@ from langchain_core.tools import BaseTool
 
 from minitap.mobile_use.context import MobileUseContext
 from minitap.mobile_use.tools.mobile.back import back_wrapper
-from minitap.mobile_use.tools.mobile.clear_text import clear_text_wrapper
 from minitap.mobile_use.tools.mobile.erase_one_char import erase_one_char_wrapper
-from minitap.mobile_use.tools.mobile.glimpse_screen import glimpse_screen_wrapper
-from minitap.mobile_use.tools.mobile.input_text import input_text_wrapper
+from minitap.mobile_use.tools.mobile.focus_and_clear_text import focus_and_clear_text_wrapper
+from minitap.mobile_use.tools.mobile.focus_and_input_text import focus_and_input_text_wrapper
 from minitap.mobile_use.tools.mobile.launch_app import launch_app_wrapper
 from minitap.mobile_use.tools.mobile.long_press_on import long_press_on_wrapper
 from minitap.mobile_use.tools.mobile.open_link import open_link_wrapper
@@ -13,8 +12,8 @@ from minitap.mobile_use.tools.mobile.press_key import press_key_wrapper
 from minitap.mobile_use.tools.mobile.stop_app import stop_app_wrapper
 from minitap.mobile_use.tools.mobile.swipe import swipe_wrapper
 from minitap.mobile_use.tools.mobile.tap import tap_wrapper
-from minitap.mobile_use.tools.mobile.wait_for_animation_to_end import (
-    wait_for_animation_to_end_wrapper,
+from minitap.mobile_use.tools.mobile.wait_for_delay import (
+    wait_for_delay_wrapper,
 )
 from minitap.mobile_use.tools.tool_wrapper import CompositeToolWrapper, ToolWrapper
 
@@ -24,14 +23,13 @@ EXECUTOR_WRAPPERS_TOOLS = [
     tap_wrapper,
     long_press_on_wrapper,
     swipe_wrapper,
-    glimpse_screen_wrapper,
-    input_text_wrapper,
+    focus_and_input_text_wrapper,
     erase_one_char_wrapper,
     launch_app_wrapper,
     stop_app_wrapper,
-    clear_text_wrapper,
+    focus_and_clear_text_wrapper,
     press_key_wrapper,
-    wait_for_animation_to_end_wrapper,
+    wait_for_delay_wrapper,
 ]
 
 

minitap/mobile_use/tools/mobile/{clear_text.py → focus_and_clear_text.py}

@@ -238,9 +238,9 @@ class TextClearer:
         )
 
 
-def get_clear_text_tool(ctx: MobileUseContext):
+def get_focus_and_clear_text_tool(ctx: MobileUseContext):
     @tool
-    async def clear_text(
+    async def focus_and_clear_text(
         tool_call_id: Annotated[str, InjectedToolCallId],
         state: Annotated[State, InjectedState],
         agent_thought: str,
@@ -255,9 +255,9 @@ def get_clear_text_tool(ctx: MobileUseContext):
         )
 
         agent_outcome = (
-            clear_text_wrapper.on_failure_fn(result.error_message)
+            focus_and_clear_text_wrapper.on_failure_fn(result.error_message)
             if not result.success
-            else clear_text_wrapper.on_success_fn(
+            else focus_and_clear_text_wrapper.on_success_fn(
                 nb_char_erased=result.chars_erased, new_text_value=result.final_text
             )
         )
@@ -280,7 +280,7 @@ def get_clear_text_tool(ctx: MobileUseContext):
             ),
         )
 
-    return clear_text
+    return focus_and_clear_text
 
 
 def _format_success_message(nb_char_erased: int, new_text_value: str | None) -> str:
@@ -299,8 +299,8 @@ def _format_failure_message(output: str | None) -> str:
     return "Failed to erase text. " + (str(output) if output else "")
 
 
-clear_text_wrapper = ToolWrapper(
-    tool_fn_getter=get_clear_text_tool,
+focus_and_clear_text_wrapper = ToolWrapper(
+    tool_fn_getter=get_focus_and_clear_text_tool,
     on_success_fn=_format_success_message,
     on_failure_fn=_format_failure_message,
 )

minitap/mobile_use/tools/mobile/{input_text.py → focus_and_input_text.py}

@@ -42,9 +42,9 @@ def _controller_input_text(ctx: MobileUseContext, text: str) -> InputResult:
     return InputResult(ok=False, error=str(controller_out))
 
 
-def get_input_text_tool(ctx: MobileUseContext):
+def get_focus_and_input_text_tool(ctx: MobileUseContext):
     @tool
-    async def input_text(
+    async def focus_and_input_text(
         tool_call_id: Annotated[str, InjectedToolCallId],
         state: Annotated[State, InjectedState],
         agent_thought: str,
@@ -70,7 +70,7 @@ def get_input_text_tool(ctx: MobileUseContext):
             error_message = "Failed to focus the text input element before typing."
             tool_message = ToolMessage(
                 tool_call_id=tool_call_id,
-                content=input_text_wrapper.on_failure_fn(text, error_message),
+                content=focus_and_input_text_wrapper.on_failure_fn(text, error_message),
                 additional_kwargs={"error": error_message},
                 status="error",
             )
@@ -103,9 +103,9 @@ def get_input_text_tool(ctx: MobileUseContext):
                 text_input_content = get_element_text(element)
 
         agent_outcome = (
-            input_text_wrapper.on_success_fn(text, text_input_content, target.resource_id)
+            focus_and_input_text_wrapper.on_success_fn(text, text_input_content, target.resource_id)
             if result.ok
-            else input_text_wrapper.on_failure_fn(text, result.error)
+            else focus_and_input_text_wrapper.on_failure_fn(text, result.error)
         )
 
         tool_message = ToolMessage(
@@ -126,7 +126,7 @@ def get_input_text_tool(ctx: MobileUseContext):
             ),
         )
 
-    return input_text
+    return focus_and_input_text
 
 
 def _on_input_success(text, text_input_content, text_input_resource_id):
@@ -141,8 +141,8 @@ def _on_input_success(text, text_input_content, text_input_resource_id):
         return "Typed text, should now verify before moving forward"
 
 
-input_text_wrapper = ToolWrapper(
-    tool_fn_getter=get_input_text_tool,
+focus_and_input_text_wrapper = ToolWrapper(
+    tool_fn_getter=get_focus_and_input_text_tool,
     on_success_fn=_on_input_success,
     on_failure_fn=lambda text, error: f"Failed to input text {repr(text)}. Reason: {error}",
 )

minitap/mobile_use/tools/mobile/long_press_on.py

@@ -2,46 +2,158 @@ from typing import Annotated
 
 from langchain_core.messages import ToolMessage
 from langchain_core.tools import tool
-from langchain_core.tools.base import InjectedToolCallId
+from langchain_core.tools.base import BaseTool, InjectedToolCallId
 from langgraph.prebuilt import InjectedState
 from langgraph.types import Command
 
 from minitap.mobile_use.constants import EXECUTOR_MESSAGES_KEY
 from minitap.mobile_use.context import MobileUseContext
-from minitap.mobile_use.controllers.mobile_command_controller import SelectorRequest
+from minitap.mobile_use.controllers.mobile_command_controller import (
+    CoordinatesSelectorRequest,
+    IdSelectorRequest,
+    SelectorRequestWithCoordinates,
+    TextSelectorRequest,
+)
 from minitap.mobile_use.controllers.mobile_command_controller import (
     long_press_on as long_press_on_controller,
 )
 from minitap.mobile_use.graph.state import State
 from minitap.mobile_use.tools.tool_wrapper import ToolWrapper
+from minitap.mobile_use.tools.types import Target
+from minitap.mobile_use.utils.logger import get_logger
 
+logger = get_logger(__name__)
 
-def get_long_press_on_tool(ctx: MobileUseContext):
+
+def get_long_press_on_tool(ctx: MobileUseContext) -> BaseTool:
     @tool
     async def long_press_on(
         tool_call_id: Annotated[str, InjectedToolCallId],
         state: Annotated[State, InjectedState],
         agent_thought: str,
-        selector_request: SelectorRequest,
-        index: int | None = None,
-    ) -> Command:
+        target: Target,
+        duration_ms: int = 1000,
+    ):
         """
-        Long press on a UI element identified by the given selector.
-        An index can be specified to select a specific element if multiple are found.
+        Long presses on a UI element identified by the 'target' object.
+
+        The 'target' object allows specifying an element by its resource_id
+        (with an optional index), its coordinates, or its text content (with an optional index).
+        The tool uses a fallback strategy, trying the locators in that order.
+
+        Args:
+            target: The UI element to long press on (coordinates, resource_id, or text).
+            duration_ms: Duration of the long press in milliseconds. Choose based on interaction:
+                        - 500-800ms: Quick long press (e.g., selecting text, haptic feedback)
+                        - 1000ms (default): Standard long press (most common use case)
+                        - 1500-2000ms: Extended long press (e.g., context menus, special actions)
+                        - 2500ms+: Very long press (e.g., accessibility, advanced gestures)
         """
-        output = long_press_on_controller(ctx=ctx, selector_request=selector_request, index=index)
-        has_failed = output is not None
+        error_obj: dict | None = {
+            "error": "No valid selector provided or all selectors failed."
+        }  # Default to failure
+        latest_selector_info: str | None = None
+
+        # 1. Try with COORDINATES FIRST (visual approach)
+        if target.coordinates:
+            try:
+                center_point = target.coordinates.get_center()
+                selector = SelectorRequestWithCoordinates(
+                    coordinates=CoordinatesSelectorRequest(x=center_point.x, y=center_point.y)
+                )
+                logger.info(
+                    f"Attempting to long press using coordinates: {center_point.x},{center_point.y}"
+                )
+                latest_selector_info = f"coordinates='{target.coordinates}'"
+                result = long_press_on_controller(
+                    ctx=ctx,
+                    selector_request=selector,
+                    ui_hierarchy=state.latest_ui_hierarchy,
+                    long_press_duration=duration_ms,
+                )
+                if result is None:  # Success
+                    error_obj = None
+                else:
+                    logger.warning(
+                        f"Long press with coordinates '{target.coordinates}' failed. "
+                        f"Error: {result}"
+                    )
+                    error_obj = {"error": result} if isinstance(result, str) else result
+            except Exception as e:
+                logger.warning(
+                    f"Exception during long press with coordinates '{target.coordinates}': {e}"
+                )
+                error_obj = {"error": str(e)}
+
+        # 2. If coordinates failed or weren't provided, try with resource_id
+        if error_obj is not None and target.resource_id:
+            try:
+                selector = IdSelectorRequest(id=target.resource_id)
+                logger.info(
+                    f"Attempting to long press using resource_id: '{target.resource_id}' "
+                    f"at index {target.resource_id_index}"
+                )
+                latest_selector_info = (
+                    f"resource_id='{target.resource_id}' (index={target.resource_id_index})"
+                )
+                result = long_press_on_controller(
+                    ctx=ctx,
+                    selector_request=selector,
+                    index=target.resource_id_index,
+                    ui_hierarchy=state.latest_ui_hierarchy,
+                    long_press_duration=duration_ms,
+                )
+                if result is None:  # Success
+                    error_obj = None
+                else:
+                    logger.warning(
+                        f"Long press with resource_id '{target.resource_id}' failed. "
+                        f"Error: {result}"
+                    )
+                    error_obj = {"error": result} if isinstance(result, str) else result
+            except Exception as e:
+                logger.warning(
+                    f"Exception during long press with resource_id '{target.resource_id}': {e}"
+                )
+                error_obj = {"error": str(e)}
+
+        # 3. If resource_id failed or wasn't provided, try with text (last resort)
+        if error_obj is not None and target.text:
+            try:
+                selector = TextSelectorRequest(text=target.text)
+                logger.info(
+                    f"Attempting to long press using text: '{target.text}' "
+                    f"at index {target.text_index}"
+                )
+                latest_selector_info = f"text='{target.text}' (index={target.text_index})"
+                result = long_press_on_controller(
+                    ctx=ctx,
+                    selector_request=selector,
+                    index=target.text_index,
+                    ui_hierarchy=state.latest_ui_hierarchy,
+                    long_press_duration=duration_ms,
+                )
+                if result is None:  # Success
+                    error_obj = None
+                else:
+                    logger.warning(f"Long press with text '{target.text}' failed. Error: {result}")
+                    error_obj = {"error": result} if isinstance(result, str) else result
+            except Exception as e:
+                logger.warning(f"Exception during long press with text '{target.text}': {e}")
+                error_obj = {"error": str(e)}
 
+        has_failed = error_obj is not None
+        final_selector_info = latest_selector_info if latest_selector_info else "N/A"
         agent_outcome = (
-            long_press_on_wrapper.on_failure_fn()
+            long_press_on_wrapper.on_failure_fn(final_selector_info)
             if has_failed
-            else long_press_on_wrapper.on_success_fn()
+            else long_press_on_wrapper.on_success_fn(final_selector_info)
         )
 
         tool_message = ToolMessage(
             tool_call_id=tool_call_id,
             content=agent_outcome,
-            additional_kwargs={"error": output} if has_failed else {},
+            additional_kwargs=error_obj if has_failed else {},
             status="error" if has_failed else "success",
         )
         return Command(
@@ -60,6 +172,9 @@ def get_long_press_on_tool(ctx: MobileUseContext):
 
 long_press_on_wrapper = ToolWrapper(
     tool_fn_getter=get_long_press_on_tool,
-    on_success_fn=lambda: "Long press on is successful.",
-    on_failure_fn=lambda: "Failed to long press on.",
+    on_success_fn=lambda selector_info: (
+        f"Long press on element with {selector_info} was successful."
+    ),
+    on_failure_fn=lambda selector_info: "Failed to long press on element. "
+    + f"Last attempt was with {selector_info}.",
 )

minitap/mobile_use/tools/mobile/swipe.py

@@ -9,15 +9,14 @@ from pydantic import Field
 
 from minitap.mobile_use.constants import EXECUTOR_MESSAGES_KEY
 from minitap.mobile_use.context import MobileUseContext
-from minitap.mobile_use.controllers.mobile_command_controller import (
+from minitap.mobile_use.controllers.mobile_command_controller import swipe as swipe_controller
+from minitap.mobile_use.controllers.types import (
     CoordinatesSelectorRequest,
     PercentagesSelectorRequest,
-    SwipeDirection,
     SwipeRequest,
     SwipeStartEndCoordinatesRequest,
     SwipeStartEndPercentagesRequest,
 )
-from minitap.mobile_use.controllers.mobile_command_controller import swipe as swipe_controller
 from minitap.mobile_use.graph.state import State
 from minitap.mobile_use.tools.tool_wrapper import CompositeToolWrapper
 
@@ -123,29 +122,7 @@ def get_composite_swipe_tools(ctx: MobileUseContext) -> list[BaseTool]:
             }
         )
 
-    @tool
-    def swipe_direction(
-        agent_thought: str,
-        tool_call_id: Annotated[str, InjectedToolCallId],
-        state: Annotated[State, InjectedState],
-        direction: SwipeDirection,
-        duration: int = Field(description="Duration in ms", ge=1, le=10000, default=400),
-    ):
-        """Swipe in a specific direction across the screen."""
-        swipe_request = SwipeRequest(
-            swipe_mode=direction,
-            duration=duration,
-        )
-        return get_swipe_tool(ctx=ctx).invoke(
-            input={
-                "tool_call_id": tool_call_id,
-                "state": state,
-                "agent_thought": agent_thought,
-                "swipe_request": swipe_request,
-            }
-        )
-
-    return [swipe_coordinates, swipe_percentages, swipe_direction]
+    return [swipe_coordinates, swipe_percentages]
 
 
 swipe_wrapper = CompositeToolWrapper(