minitap-mobile-use 2.1.0__py3-none-any.whl → 2.3.0__py3-none-any.whl

minitap/mobile_use/agents/contextor/contextor.py

@@ -1,4 +1,5 @@
 from minitap.mobile_use.agents.executor.utils import is_last_tool_message_take_screenshot
+from minitap.mobile_use.context import MobileUseContext
 from minitap.mobile_use.controllers.mobile_command_controller import get_screen_data
 from minitap.mobile_use.controllers.platform_specific_commands_controller import (
     get_device_date,
@@ -7,7 +8,6 @@ from minitap.mobile_use.controllers.platform_specific_commands_controller import
 from minitap.mobile_use.graph.state import State
 from minitap.mobile_use.utils.decorators import wrap_with_callbacks
 from minitap.mobile_use.utils.logger import get_logger
-from minitap.mobile_use.context import MobileUseContext
 
 logger = get_logger(__name__)
 
@@ -26,7 +26,9 @@ class ContextorNode:
         focused_app_info = get_focused_app_info(self.ctx)
         device_date = get_device_date(self.ctx)
 
-        should_add_screenshot_context = is_last_tool_message_take_screenshot(list(state.messages))
+        should_add_screenshot_context = is_last_tool_message_take_screenshot(
+            list(state.executor_messages)
+        )
 
         return state.sanitize_update(
             ctx=self.ctx,

minitap/mobile_use/agents/cortex/cortex.md

@@ -4,19 +4,40 @@ Your job is to **analyze the current {{ platform }} mobile device state** and pr
 
 You must act like a human brain, responsible for giving instructions to your hands (the **Executor** agent). Therefore, you must act with the same imprecision and uncertainty as a human when performing swipe actions: humans don't know where exactly they are swiping (always prefer percentages of width and height instead of absolute coordinates), they just know they are swiping up or down, left or right, and with how much force (usually amplified compared to what's truly needed - go overboard of sliders for instance).
 
-### Context You Receive:
+### Core Principle: Break Unproductive Cycles
 
-You are provided with:
+Your highest priority is to recognize when you are not making progress. You are in an unproductive cycle if a **sequence of actions brings you back to a previous state without achieving the subgoal.**
 
-- 📱 **Device state**:
+If you detect a cycle, you are **FORBIDDEN** from repeating it. You must pivot your strategy.
 
-  - Latest **UI hierarchy**
-  - (Optional) Latest **screenshot (base64)**. You can query one if you need it by calling the take_screenshot tool. Often, the UI hierarchy is enough to understand what is happening on the screen.
-  - Current **focused app info**
-  - **Screen size** and **device date**
+1.  **Announce the Pivot:** In your `agent_thought`, you must briefly state which workflow is failing and what your new approach is.
 
-- 🧭 **Task context**:
+2.  **Find a Simpler Path:** Abandon the current workflow. Ask yourself: **"How would a human do this if this feature didn't exist?"** This usually means relying on fundamental actions like scrolling, swiping, or navigating through menus manually.
+
+3.  **Retreat as a Last Resort:** If no simpler path exists, declare the subgoal a failure to trigger a replan.
+
+### How to Perceive the Screen: A Two-Sense Approach
+
+To understand the device state, you have two senses, each with its purpose:
+
+1.  **UI Hierarchy (Your sense of "Touch"):**
+    *   **What it is:** A structured list of all elements on the screen.
+    *   **Use it for:** Finding elements by `resource-id`, checking for specific text, and understanding the layout structure.
+    *   **Limitation:** It does NOT tell you what the screen *looks* like. It can be incomplete, and it contains no information about images, colors, or whether an element is visually obscured.
+
+2.  **`glimpse_screen` (Your sense of "Sight"):**
+    *   **What it is:** A tool that provides a real, up-to-date image of the screen.
+    *   **Use it for:** Confirming what is actually visible. This is your source of TRUTH for all visual information (icons, images, element positions, colors).
+    *   **Golden Rule:** When the UI hierarchy is ambiguous, seems incomplete, or when you need to verify a visual detail before acting, **`glimpse_screen` is always the most effective and reliable action.** Never guess what the screen looks like; use your sight to be sure.
+
+  **CRITICAL NOTE ON SIGHT:** The visual information from `glimpse_screen` is **ephemeral**. It is available for **THIS decision turn ONLY**. You MUST extract all necessary information from it IMMEDIATELY, as it will be cleared before the next step.
+### Context You Receive:
+
+- 📱 **Device state**:
+  - Latest **UI hierarchy** and (if available) a **screenshot**.
+  - **CRITICAL NOTE ON SIGHT:** The visual information from `glimpse_screen` is **ephemeral**. It is available for **THIS decision turn ONLY**. You MUST extract all necessary information from it IMMEDIATELY, as it will be cleared before the next step.
 
+- 🧭 **Task context**:
   - The user's **initial goal**
   - The **subgoal plan** with their statuses
   - The **current subgoal** (the one in `PENDING` in the plan)
@@ -27,25 +48,51 @@ You are provided with:
 
 Focus on the **current PENDING subgoal and the next subgoals not yet started**.
 
-1. **Analyze the UI** and environment to understand what action is required.
+**CRITICAL: Before making any decision, you MUST thoroughly analyze the agent thoughts history to:**
+- **Detect patterns of failure or repeated attempts** that suggest the current approach isn't working
+- **Identify contradictions** between what was planned and what actually happened
+- **Spot errors in previous reasoning** that need to be corrected
+- **Learn from successful strategies** used in similar situations
+- **Avoid repeating failed approaches** by recognizing when to change strategy
+
+1. **Analyze the agent thoughts first** - Review all previous agent thoughts to understand:
+   - What strategies have been tried and their outcomes
+   - Any errors or misconceptions in previous reasoning
+   - Patterns that indicate success or failure
+   - Whether the current approach should be continued or modified
+
+2. **Then analyze the UI** and environment to understand what action is required, but always in the context of what the agent thoughts reveal about the situation.
 
-2.1. If some of the subgoals must be **completed** based on your observations, add them to `complete_subgoals_by_ids`. To justify your conclusion, you will fill in the `agent_thought` field based on:
+3. If some of the subgoals must be **completed** based on your observations, add them to `complete_subgoals_by_ids`. To justify your conclusion, you will fill in the `agent_thought` field based on:
 
 - The current UI state
-- Past agent thoughts
-- Recent tool effects
+- **Critical analysis of past agent thoughts and their accuracy**
+- Recent tool effects and whether they matched expectations from agent thoughts
+- **Any corrections needed to previous reasoning or strategy**
+
+
+### The Rule of Element Interaction
+
+**You MUST follow it for every element interaction.**
 
-  2.2. Otherwise, output a **stringified structured set of instructions** that an **Executor agent** can perform on a real mobile device:
+When you target a UI element (for a `tap`, `input_text`, `clear_text`, etc.), you **MUST** provide a comprehensive target object containing every piece of information you can find about it.
+
+*   **1. `resource_id`**: Include this if it is present in the UI hierarchy.
+*   **2. `coordinates`**: Include the full bounds (`x`, `y`, `width`, `height`) if they are available.
+*   **3. `text`**: Include the *current text* content of the element (e.g., "Sign In", "Search...", "First Name").
+
+**This is NOT optional.** Providing all three locators if we have, it is the foundation of the system's reliability. It allows next steps to use a fallback mechanism: if the ID fails, it tries the coordinates, etc. Failing to provide this complete context will lead to action failures.
+
+### Outputting Your Decisions
+
+If you decide to act, output a **valid JSON stringified structured set of instructions** for the Executor.
 
 - These must be **concrete low-level actions**.
-- The executor has the following available tools: **{{ executor_tools_list }}**.
+- The executor has the following available tools: {{ executor_tools_list }}.
 - Your goal is to achieve subgoals **fast** - so you must put as much actions as possible in your instructions to complete all achievable subgoals (based on your observations) in one go.
 - To open URLs/links directly, use the `open_link` tool - it will automatically handle opening in the appropriate browser. It also handles deep links.
 - When you need to open an app, use the `find_packages` low-level action to try and get its name. Then, simply use the `launch_app` low-level action to launch it.
-- If you refer to a UI element or coordinates, specify it clearly (e.g., `resource-id: com.whatsapp:id/search`, `text: "Alice"`, `x: 100, y: 200`).
-- **The structure is up to you**, but it must be valid **JSON stringified output**. You will accompany this output with a **natural-language summary** of your reasoning and approach in your agent thought.
-- **Never use a sequence of `tap` + `input_text` to type into a field. Always use a single `input_text` action** with the correct `resource_id` (this already ensures the element is focused and the cursor is moved to the end).
-- When you want to launch/stop an app, prefer using its package name.
+-   **Always use a single `input_text` action** to type in a field. This tool handles focusing the element and placing the cursor correctly. If the tool feedback indicates verification is needed or shows None/empty content, perform verification before proceeding.
 - **Only reference UI element IDs or visible texts that are explicitly present in the provided UI hierarchy or screenshot. Do not invent, infer, or guess any IDs or texts that are not directly observed**.
 - **For text clearing**: When you need to completely clear text from an input field, always call the `clear_text` tool with the correct resource_id. This tool automatically focuses the element, and ensures the field is emptied. If you notice this tool fails to clear the text, try to long press the input, select all, and call `erase_one_char`.
 
@@ -57,10 +104,12 @@ Focus on the **current PENDING subgoal and the next subgoals not yet started**.
 - **Structured Decisions** _(optional)_:
   A **valid stringified JSON** describing what should be executed **right now** to advance through the subgoals as much as possible.
 
-- **Agent Thought** _(1-2 sentences)_:
-  If there is any information you need to remember for later steps, you must include it here, because only the agent thoughts will be used to produce the final structured output.
+- **Agent Thought** _(2-4 sentences)_:
+  **MANDATORY: Start by analyzing previous agent thoughts** - Did previous reasoning contain errors? Are we repeating failed approaches? What worked before in similar situations?
+  
+  Then explain your current decision based on this analysis. If there is any information you need to remember for later steps, you must include it here, because only the agent thoughts will be used to produce the final structured output.
 
-  This also helps other agents understand your decision and learn from future failures.
+  This also helps other agents understand your decision and learn from future failures. **Explicitly mention if you're correcting a previous error or changing strategy based on agent thoughts analysis.**
   You must also use this field to mention checkpoints when you perform actions without definite ending: for instance "Swiping up to reveal more recipes - last seen recipe was <ID or NAME>, stop when no more".
 
 **Important:** `complete_subgoals_by_ids` and the structured decisions are mutually exclusive: if you provide both, the structured decisions will be ignored. Therefore, you must always prioritize completing subgoals over providing structured decisions.
@@ -76,12 +125,12 @@ Focus on the **current PENDING subgoal and the next subgoals not yet started**.
 #### Structured Decisions:
 
 ```text
-"{\"action\": \"tap\", \"target\": {\"resource_id\": \"com.whatsapp:id/menuitem_search\", \"text\": \"Search\"}}"
+"{\"action\": \"tap\", \"target\": {\"text_input_resource_id\": \"com.whatsapp:id/menuitem_search\", \"text_input_coordinates\": {\"x\": 880, \"y\": 150, \"width\": 120, \"height\": 120}, \"text_input_text\": \"Search\"}}"
 ```
 
 #### Agent Thought:
 
-> I will tap the search icon at the top of the WhatsApp interface to begin searching for Alice.
+> Analysis: No previous attempts, this is a fresh approach. I will tap the search icon to begin searching. I am providing its resource_id, coordinates, and text content to ensure the Executor can find it reliably, following the element rule.
 
 ### Input
 
@@ -94,9 +143,6 @@ Focus on the **current PENDING subgoal and the next subgoals not yet started**.
 **Current Subgoal (what needs to be done right now):**
 {{ current_subgoal }}
 
-**Agent thoughts (previous reasoning, observations about the environment):**
-{{ agents_thoughts }}
-
 **Executor agent feedback on latest UI decisions:**
 
 {{ executor_feedback }}

minitap/mobile_use/agents/cortex/cortex.py

@@ -44,9 +44,8 @@ class CortexNode:
             initial_goal=state.initial_goal,
             subgoal_plan=state.subgoal_plan,
             current_subgoal=get_current_subgoal(state.subgoal_plan),
-            agents_thoughts=state.agents_thoughts,
             executor_feedback=executor_feedback,
-            executor_tools_list=format_tools_list(self.ctx, EXECUTOR_WRAPPERS_TOOLS),
+            executor_tools_list=format_tools_list(ctx=self.ctx, wrappers=EXECUTOR_WRAPPERS_TOOLS),
         )
         messages = [
             SystemMessage(content=system_message),

minitap/mobile_use/agents/executor/executor.md

@@ -28,7 +28,6 @@ and your previous actions, you must:
 {
   "action": "tap",
   "target": {
-    "text": "Alice",
     "resource_id": "com.whatsapp:id/conversation_item"
   }
 }
@@ -39,7 +38,6 @@ and your previous actions, you must:
 Call the `tap_on_element` tool with:
 
 - `resource_id = "com.whatsapp:id/conversation_item"`
-- `text = "Alice"`
 - `agent_thought = "I'm tapping on the chat item labeled 'Alice' to open the conversation."`
 
 ---
@@ -55,10 +53,14 @@ Call the `tap_on_element` tool with:
 
 When using the `input_text` tool:
 
-- **Always provide the `resource_id` of the element** you want to type into.
+- **Provide all available information** from the following optional parameters to identify the text input element:
+  - `text_input_resource_id`: The resource ID of the text input element (when available)
+  - `text_input_coordinates`: The bounds (ElementBounds) of the text input element (when available)
+  - `text_input_text`: The current text content of the text input element (when available)
+
 - The tool will automatically:
 
-  1. **Focus the element first**
+  1. **Focus the element** using the provided identification parameters
   2. **Move the cursor to the end** of the existing text
   3. **Then type the new text**
 

minitap/mobile_use/agents/executor/executor.py

@@ -3,6 +3,8 @@ from pathlib import Path
 from jinja2 import Template
 from langchain_core.messages import HumanMessage, SystemMessage
 from langchain_google_genai import ChatGoogleGenerativeAI
+from langchain_google_vertexai.chat_models import ChatVertexAI
+
 from minitap.mobile_use.constants import EXECUTOR_MESSAGES_KEY
 from minitap.mobile_use.context import MobileUseContext
 from minitap.mobile_use.graph.state import State
@@ -56,7 +58,7 @@ class ExecutorNode:
         }
 
         # ChatGoogleGenerativeAI does not support the "parallel_tool_calls" keyword
-        if not isinstance(llm, ChatGoogleGenerativeAI):
+        if not isinstance(llm, ChatGoogleGenerativeAI | ChatVertexAI):
             llm_bind_tools_kwargs["parallel_tool_calls"] = True
 
         llm = llm.bind_tools(**llm_bind_tools_kwargs)

minitap/mobile_use/agents/executor/utils.py

@@ -1,4 +1,5 @@
 from langchain_core.messages import BaseMessage
+
 from minitap.mobile_use.utils.conversations import is_tool_message
 
 
@@ -7,5 +8,5 @@ def is_last_tool_message_take_screenshot(messages: list[BaseMessage]) -> bool:
         return False
     for msg in messages[::-1]:
         if is_tool_message(msg):
-            return msg.name == "take_screenshot"
+            return msg.name == "glimpse_screen"
     return False

minitap/mobile_use/agents/outputter/test_outputter.py

@@ -1,9 +1,21 @@
-from minitap.mobile_use.agents.outputter.outputter import outputter
-from minitap.mobile_use.config import LLM, OutputConfig
-from minitap.mobile_use.context import MobileUseContext
-from minitap.mobile_use.utils.logger import get_logger
+import sys
+from unittest.mock import AsyncMock, Mock, patch
+
+import pytest
 from pydantic import BaseModel
 
+sys.modules["langgraph.prebuilt.chat_agent_executor"] = Mock()
+sys.modules["minitap.mobile_use.graph.state"] = Mock()
+sys.modules["langchain_google_vertexai"] = Mock()
+sys.modules["langchain_google_genai"] = Mock()
+sys.modules["langchain_openai"] = Mock()
+sys.modules["langchain_cerebras"] = Mock()
+
+from minitap.mobile_use.agents.outputter.outputter import outputter  # noqa: E402
+from minitap.mobile_use.config import LLM, OutputConfig  # noqa: E402
+from minitap.mobile_use.context import MobileUseContext  # noqa: E402
+from minitap.mobile_use.utils.logger import get_logger  # noqa: E402
+
 logger = get_logger(__name__)
 
 
@@ -40,68 +52,118 @@ mocked_state = DummyState(
     ],
 )
 
-mocked_ctx = MobileUseContext(
-    llm_config={
+
+@pytest.fixture
+def mock_context():
+    """Create a properly mocked context with all required fields."""
+    ctx = Mock(spec=MobileUseContext)
+    ctx.llm_config = {
         "executor": LLM(provider="openai", model="gpt-5-nano"),
         "cortex": LLM(provider="openai", model="gpt-5-nano"),
         "planner": LLM(provider="openai", model="gpt-5-nano"),
         "orchestrator": LLM(provider="openai", model="gpt-5-nano"),
-    },
-)  # type: ignore
+    }
+    ctx.device = Mock()
+    ctx.hw_bridge_client = Mock()
+    ctx.screen_api_client = Mock()
+    return ctx
+
+
+@pytest.fixture
+def mock_state():
+    """Create a mock state with test data."""
+    return DummyState(
+        messages=[],
+        initial_goal="Find a green product on my website",
+        agents_thoughts=[
+            "Going on http://superwebsite.fr",
+            "Searching for products",
+            "Filtering by color",
+            "Color 'green' found for a 20 dollars product",
+        ],
+    )
+
+
+@patch("minitap.mobile_use.agents.outputter.outputter.get_llm")
+@pytest.mark.asyncio
+async def test_outputter_with_pydantic_model(mock_get_llm, mock_context, mock_state):
+    """Test outputter with Pydantic model output."""
+    # Mock the structured LLM response
+    mock_structured_llm = AsyncMock()
+    mock_structured_llm.ainvoke.return_value = MockPydanticSchema(
+        color="green", price=20, currency_symbol="$", website_url="http://superwebsite.fr"
+    )
 
+    # Mock the base LLM
+    mock_llm = Mock()
+    mock_llm.with_structured_output.return_value = mock_structured_llm
+    mock_get_llm.return_value = mock_llm
 
-async def test_outputter_with_pydantic_model():
-    logger.info("Starting test_outputter_with_pydantic_model")
     config = OutputConfig(
         structured_output=MockPydanticSchema,
         output_description=None,
     )
 
-    result = await outputter(ctx=mocked_ctx, output_config=config, graph_output=mocked_state)  # type: ignore
-
-    assert isinstance(result, MockPydanticSchema)
-    assert result.color.lower() == "green"
-    logger.success(str(result))
+    result = await outputter(ctx=mock_context, output_config=config, graph_output=mock_state)
 
+    assert isinstance(result, dict)
+    assert result.get("color") == "green"
+
+
+@patch("minitap.mobile_use.agents.outputter.outputter.get_llm")
+@pytest.mark.asyncio
+async def test_outputter_with_dict(mock_get_llm, mock_context, mock_state):
+    """Test outputter with dictionary output."""
+    # Mock the structured LLM response for dict
+    mock_structured_llm = AsyncMock()
+    expected_dict = {
+        "color": "green",
+        "price": 20,
+        "currency_symbol": "$",
+        "website_url": "http://superwebsite.fr",
+    }
+    mock_structured_llm.ainvoke.return_value = expected_dict
+
+    # Mock the base LLM
+    mock_llm = Mock()
+    mock_llm.with_structured_output.return_value = mock_structured_llm
+    mock_get_llm.return_value = mock_llm
 
-async def test_outputter_with_dict():
-    logger.info("Starting test_outputter_with_dict")
     config = OutputConfig(
         structured_output=mock_dict,
         output_description=None,
     )
 
-    result = await outputter(ctx=mocked_ctx, output_config=config, graph_output=mocked_state)  # type: ignore
+    result = await outputter(ctx=mock_context, output_config=config, graph_output=mock_state)
 
     assert isinstance(result, dict)
-    assert result.get("color", None) == "green"
-    assert result.get("price", None) == 20
-    assert result.get("currency_symbol", None) == "$"
-    assert result.get("website_url", None) == "http://superwebsite.fr"
-    logger.success(str(result))
-
+    assert result.get("color") == "green"
+    assert result.get("price") == 20
+    assert result.get("currency_symbol") == "$"
+    assert result.get("website_url") == "http://superwebsite.fr"
+
+
+@patch("minitap.mobile_use.agents.outputter.outputter.get_llm")
+@pytest.mark.asyncio
+async def test_outputter_with_natural_language_output(mock_get_llm, mock_context, mock_state):
+    """Test outputter with natural language description output."""
+    # Mock the LLM response for natural language output (no structured output)
+    mock_llm = AsyncMock()
+    expected_json = '{"color": "green", "price": 20, "currency_symbol": "$", "website_url": "http://superwebsite.fr"}'
+    mock_llm.ainvoke.return_value = Mock(content=expected_json)
+    mock_get_llm.return_value = mock_llm
 
-async def test_outputter_with_natural_language_output():
-    logger.info("Starting test_outputter_with_natural_language_output")
     config = OutputConfig(
         structured_output=None,
-        output_description="A JSON object with a color, \
-        a price, a currency_symbol and a website_url key",
+        output_description=(
+            "A JSON object with a color, a price, a currency_symbol and a website_url key"
+        ),
     )
 
-    result = await outputter(ctx=mocked_ctx, output_config=config, graph_output=mocked_state)  # type: ignore
-    logger.info(str(result))
+    result = await outputter(ctx=mock_context, output_config=config, graph_output=mock_state)
 
     assert isinstance(result, dict)
-    assert result.get("color", None) == "green"
-    assert result.get("price", None) == 20
-    assert result.get("currency_symbol", None) == "$"
-    assert result.get("website_url", None) == "http://superwebsite.fr"
-    logger.success(str(result))
-
-
-if __name__ == "__main__":
-    import asyncio
-
-    asyncio.run(test_outputter_with_pydantic_model())
-    asyncio.run(test_outputter_with_natural_language_output())
+    assert result.get("color") == "green"
+    assert result.get("price") == 20
+    assert result.get("currency_symbol") == "$"
+    assert result.get("website_url") == "http://superwebsite.fr"

minitap/mobile_use/agents/planner/planner.md

@@ -13,7 +13,7 @@ You work like an agile tech lead: defining the key milestones without locking in
    - Don't assume the full UI is visible yet. Plan based on how most mobile apps work, and keep flexibility.
    - List of agents thoughts is empty which is expected, since it is the first plan.
    - Avoid too granular UI actions based tasks (e.g. "tap", "swipe", "copy", "paste") unless explicitly required.
-   - The executor has the following available tools: **{{ executor_tools_list }}**.
+   - The executor has the following available tools: {{ executor_tools_list }}.
      When one of these tools offers a direct shortcut (e.g. `openLink` instead of manually launching a browser and typing a URL), prefer it over decomposed manual steps.
 
 2. **Replanning**

minitap/mobile_use/agents/planner/planner.py

@@ -30,7 +30,10 @@ class PlannerNode:
 
         system_message = Template(
             Path(__file__).parent.joinpath("planner.md").read_text(encoding="utf-8")
-        ).render(platform=self.ctx.device.mobile_platform.value)
+        ).render(
+            platform=self.ctx.device.mobile_platform.value,
+            executor_tools_list=format_tools_list(ctx=self.ctx, wrappers=EXECUTOR_WRAPPERS_TOOLS),
+        )
         human_message = Template(
             Path(__file__).parent.joinpath("human.md").read_text(encoding="utf-8")
         ).render(
@@ -38,7 +41,6 @@ class PlannerNode:
             initial_goal=state.initial_goal,
             previous_plan="\n".join(str(s) for s in state.subgoal_plan),
             agent_thoughts="\n".join(state.agents_thoughts),
-            executor_tools_list=format_tools_list(self.ctx, EXECUTOR_WRAPPERS_TOOLS),
         )
         messages = [
             SystemMessage(content=system_message),

minitap/mobile_use/config.py

@@ -3,7 +3,9 @@ import os
 from pathlib import Path
 from typing import Annotated, Any, Literal
 
+import google.auth
 from dotenv import load_dotenv
+from google.auth.exceptions import DefaultCredentialsError
 from pydantic import BaseModel, Field, SecretStr, ValidationError, model_validator
 from pydantic_settings import BaseSettings
 
@@ -88,7 +90,7 @@ def record_events(output_path: Path | None, events: list[str] | BaseModel | Any)
 
 ### LLM Configuration
 
-LLMProvider = Literal["openai", "google", "openrouter", "xai"]
+LLMProvider = Literal["openai", "google", "openrouter", "xai", "vertexai"]
 LLMUtilsNode = Literal["outputter", "hopper"]
 AgentNode = Literal["planner", "orchestrator", "cortex", "executor"]
 AgentNodeWithFallback = Literal["cortex"]
@@ -98,6 +100,17 @@ DEFAULT_LLM_CONFIG_FILENAME = "llm-config.defaults.jsonc"
 OVERRIDE_LLM_CONFIG_FILENAME = "llm-config.override.jsonc"
 
 
+def validate_vertex_ai_credentials():
+    try:
+        _, project = google.auth.default()
+        if not project:
+            raise Exception("VertexAI requires a Google Cloud project to be set.")
+    except DefaultCredentialsError as e:
+        raise Exception(
+            f"VertexAI requires valid Google Application Default Credentials (ADC): {e}"
+        )
+
+
 class LLM(BaseModel):
     provider: LLMProvider
     model: str
@@ -110,6 +123,8 @@ class LLM(BaseModel):
             case "google":
                 if not settings.GOOGLE_API_KEY:
                     raise Exception(f"{name} requires GOOGLE_API_KEY in .env")
+            case "vertexai":
+                validate_vertex_ai_credentials()
             case "openrouter":
                 if not settings.OPEN_ROUTER_API_KEY:
                     raise Exception(f"{name} requires OPEN_ROUTER_API_KEY in .env")

minitap/mobile_use/controllers/mobile_command_controller.py

@@ -311,9 +311,9 @@ def press_key(ctx: MobileUseContext, key: Key, dry_run: bool = False):
 
 
 class WaitTimeout(Enum):
-    SHORT = 500
-    MEDIUM = 1000
-    LONG = 5000
+    SHORT = "500"
+    MEDIUM = "1000"
+    LONG = "5000"
 
 
 def wait_for_animation_to_end(
@@ -327,7 +327,7 @@ def wait_for_animation_to_end(
 def run_flow_with_wait_for_animation_to_end(
     ctx: MobileUseContext, base_flow: list, dry_run: bool = False
 ):
-    base_flow.append({"waitForAnimationToEnd": {"timeout": WaitTimeout.MEDIUM.value}})
+    base_flow.append({"waitForAnimationToEnd": {"timeout": int(WaitTimeout.MEDIUM.value)}})
     return run_flow(ctx, base_flow, dry_run=dry_run)
 
 

minitap/mobile_use/main.py

@@ -35,9 +35,9 @@ async def run_automation(
     if settings.ADB_HOST:
         config.with_adb_server(host=settings.ADB_HOST, port=settings.ADB_PORT)
     if settings.DEVICE_HARDWARE_BRIDGE_BASE_URL:
-        config.with_hw_bridge_base_url(url=settings.DEVICE_HARDWARE_BRIDGE_BASE_URL)
+        config.with_hw_bridge(url=settings.DEVICE_HARDWARE_BRIDGE_BASE_URL)
     if settings.DEVICE_SCREEN_API_BASE_URL:
-        config.with_screen_api_base_url(url=settings.DEVICE_SCREEN_API_BASE_URL)
+        config.with_screen_api(url=settings.DEVICE_SCREEN_API_BASE_URL)
     if graph_config_callbacks:
         config.with_graph_config_callbacks(graph_config_callbacks)
 

minitap/mobile_use/sdk/agent.py

@@ -5,6 +5,7 @@ import time
 import uuid
 from datetime import datetime
 from pathlib import Path
+from shutil import which
 from types import NoneType
 from typing import TypeVar, overload
 
@@ -41,6 +42,7 @@ from minitap.mobile_use.sdk.types.exceptions import (
     AgentProfileNotFoundError,
     AgentTaskRequestError,
     DeviceNotFoundError,
+    ExecutableNotFoundError,
     ServerStartupError,
 )
 from minitap.mobile_use.sdk.types.task import AgentProfile, Task, TaskRequest, TaskStatus
@@ -81,6 +83,12 @@ class Agent:
         self._tasks = []
         self._tmp_traces_dir = Path(tempfile.gettempdir()) / "mobile-use-traces"
         self._initialized = False
+        self._is_default_hw_bridge = (
+            self._config.servers.hw_bridge_base_url == DEFAULT_HW_BRIDGE_BASE_URL
+        )
+        self._is_default_screen_api = (
+            self._config.servers.screen_api_base_url == DEFAULT_SCREEN_API_BASE_URL
+        )
 
     def init(
         self,
@@ -88,6 +96,11 @@ class Agent:
         retry_count: int = 5,
         retry_wait_seconds: int = 5,
     ):
+        if not which("adb") and not which("xcrun"):
+            raise ExecutableNotFoundError("cli_tools")
+        if self._is_default_hw_bridge and not which("maestro"):
+            raise ExecutableNotFoundError("maestro")
+
         if self._initialized:
             logger.warning("Agent is already initialized. Skipping...")
             return True
@@ -278,13 +291,12 @@ class Agent:
                         )
 
                 if stream_mode == "updates":
-                    for key, value in payload.items():  # type: ignore
+                    for _, value in payload.items():  # type: ignore node name, node output
                         if value and "agents_thoughts" in value:
                             new_thoughts = value["agents_thoughts"]
                             last_item = new_thoughts[-1] if new_thoughts else None
                             if last_item:
                                 log_agent_thought(
-                                    prefix=key,
                                     agent_thought=last_item,
                                 )
 
@@ -434,17 +446,11 @@ class Agent:
         self._hw_bridge_client = DeviceHardwareClient(
             base_url=self._config.servers.hw_bridge_base_url.to_url(),
         )
-        self._is_default_hw_bridge = (
-            self._config.servers.hw_bridge_base_url == DEFAULT_HW_BRIDGE_BASE_URL
-        )
         self._screen_api_client = ScreenApiClient(
             base_url=self._config.servers.screen_api_base_url.to_url(),
             retry_count=retry_count,
             retry_wait_seconds=retry_wait_seconds,
         )
-        self._is_default_screen_api = (
-            self._config.servers.screen_api_base_url == DEFAULT_SCREEN_API_BASE_URL
-        )
 
     def _run_servers(self, device_id: str, platform: DevicePlatform) -> bool:
         if self._is_default_hw_bridge:
@@ -497,7 +503,10 @@ class Agent:
 
     def _check_device_screen_api_health(self) -> bool:
         try:
+            # Required to know if the Screen API is up
             self._screen_api_client.get_with_retry("/health", timeout=5)
+            # Required to know if the Screen API actually receives screenshot from the HW Bridge API
+            self._screen_api_client.get_with_retry("/screen-info", timeout=5)
             return True
         except Exception as e:
             logger.error(f"Device Screen API health check failed: {e}")

minitap/mobile_use/sdk/builders/agent_config_builder.py

@@ -105,7 +105,7 @@ class AgentConfigBuilder:
         self._task_request_defaults = copy.deepcopy(config)
         return self
 
-    def with_hw_bridge_base_url(self, url: str | ApiBaseUrl) -> "AgentConfigBuilder":
+    def with_hw_bridge(self, url: str | ApiBaseUrl) -> "AgentConfigBuilder":
         """
         Set the base URL for the device HW bridge API.
 
@@ -117,7 +117,7 @@ class AgentConfigBuilder:
         self._servers.hw_bridge_base_url = url
         return self
 
-    def with_screen_api_base_url(self, url: str | ApiBaseUrl) -> "AgentConfigBuilder":
+    def with_screen_api(self, url: str | ApiBaseUrl) -> "AgentConfigBuilder":
         """
         Set the base URL for the device screen API.