minitap-mobile-use 2.2.0__py3-none-any.whl → 2.4.0__py3-none-any.whl

minitap/mobile_use/agents/contextor/contextor.py

@@ -1,4 +1,5 @@
 from minitap.mobile_use.agents.executor.utils import is_last_tool_message_take_screenshot
+from minitap.mobile_use.context import MobileUseContext
 from minitap.mobile_use.controllers.mobile_command_controller import get_screen_data
 from minitap.mobile_use.controllers.platform_specific_commands_controller import (
     get_device_date,
@@ -7,7 +8,6 @@ from minitap.mobile_use.controllers.platform_specific_commands_controller import
 from minitap.mobile_use.graph.state import State
 from minitap.mobile_use.utils.decorators import wrap_with_callbacks
 from minitap.mobile_use.utils.logger import get_logger
-from minitap.mobile_use.context import MobileUseContext
 
 logger = get_logger(__name__)
 
@@ -21,14 +21,16 @@ class ContextorNode:
         on_success=lambda _: logger.success("Contextor Agent"),
         on_failure=lambda _: logger.error("Contextor Agent"),
     )
-    def __call__(self, state: State):
+    async def __call__(self, state: State):
         device_data = get_screen_data(self.ctx.screen_api_client)
         focused_app_info = get_focused_app_info(self.ctx)
         device_date = get_device_date(self.ctx)
 
-        should_add_screenshot_context = is_last_tool_message_take_screenshot(list(state.messages))
+        should_add_screenshot_context = is_last_tool_message_take_screenshot(
+            list(state.executor_messages)
+        )
 
-        return state.sanitize_update(
+        return await state.asanitize_update(
             ctx=self.ctx,
             update={
                 "latest_screenshot_base64": device_data.base64

minitap/mobile_use/agents/cortex/cortex.md

@@ -4,19 +4,46 @@ Your job is to **analyze the current {{ platform }} mobile device state** and pr
 
 You must act like a human brain, responsible for giving instructions to your hands (the **Executor** agent). Therefore, you must act with the same imprecision and uncertainty as a human when performing swipe actions: humans don't know where exactly they are swiping (always prefer percentages of width and height instead of absolute coordinates), they just know they are swiping up or down, left or right, and with how much force (usually amplified compared to what's truly needed - go overboard of sliders for instance).
 
-### Context You Receive:
+### Core Principle: Break Unproductive Cycles
 
-You are provided with:
+Your highest priority is to recognize when you are not making progress. You are in an unproductive cycle if a **sequence of actions brings you back to a previous state without achieving the subgoal.**
 
-- 📱 **Device state**:
+If you detect a cycle, you are **FORBIDDEN** from repeating it. You must pivot your strategy.
 
-  - Latest **UI hierarchy**
-  - (Optional) Latest **screenshot (base64)**. You can query one if you need it by calling the take_screenshot tool. Often, the UI hierarchy is enough to understand what is happening on the screen.
-  - Current **focused app info**
-  - **Screen size** and **device date**
+1.  **Announce the Pivot:** In your `agent_thought`, you must briefly state which workflow is failing and what your new approach is.
 
-- 🧭 **Task context**:
+2.  **Find a Simpler Path:** Abandon the current workflow. Ask yourself: **"How would a human do this if this feature didn't exist?"** This usually means relying on fundamental actions like scrolling, swiping, or navigating through menus manually.
+
+3.  **Retreat as a Last Resort:** If no simpler path exists, declare the subgoal a failure to trigger a replan.
+
+### How to Perceive the Screen: A Two-Sense Approach
+
+To understand the device state, you have two senses, each with its purpose:
+
+1.  **UI Hierarchy (Your sense of "Touch"):**
+    *   **What it is:** A structured list of all elements on the screen.
+    *   **Use it for:** Finding elements by `resource-id`, checking for specific text, and understanding the layout structure.
+    *   **Limitation:** It does NOT tell you what the screen *looks* like. It can be incomplete, and it contains no information about images, colors, or whether an element is visually obscured.
+
+2.  **`glimpse_screen` (Your sense of "Sight"):**
+    *   **What it is:** A tool that provides a real, up-to-date image of the screen.
+    *   **Use it for:** Confirming what is actually visible. This is your source of TRUTH for all visual information (icons, images, element positions, colors).
+    *   **Golden Rule:** When the UI hierarchy is ambiguous, seems incomplete, or when you need to verify a visual detail before acting, **`glimpse_screen` is always the most effective and reliable action.** Never guess what the screen looks like; use your sight to be sure.
+
+  **CRITICAL NOTE ON SIGHT:** The visual information from `glimpse_screen` is **ephemeral**. It is available for **THIS decision turn ONLY**. You MUST extract all necessary information from it IMMEDIATELY, as it will be cleared before the next step.
+
+### CRITICAL ACTION DIRECTIVES
+
+- **To open an application, you MUST use the `launch_app` tool.** Provide the natural language name of the app (e.g., "Uber Eats"). Do NOT attempt to open apps manually by swiping to the app drawer and searching. The `launch_app` tool is the fastest and most reliable method.
+- **To open URLs/links, you MUST use the `open_link` tool.** This handles all links, including deep links, correctly.
 
+### Context You Receive:
+
+- 📱 **Device state**:
+  - Latest **UI hierarchy** and (if available) a **screenshot**.
+  - **CRITICAL NOTE ON SIGHT:** The visual information from `glimpse_screen` is **ephemeral**. It is available for **THIS decision turn ONLY**. You MUST extract all necessary information from it IMMEDIATELY, as it will be cleared before the next step.
+
+- 🧭 **Task context**:
   - The user's **initial goal**
   - The **subgoal plan** with their statuses
   - The **current subgoal** (the one in `PENDING` in the plan)
@@ -27,25 +54,70 @@ You are provided with:
 
 Focus on the **current PENDING subgoal and the next subgoals not yet started**.
 
-1. **Analyze the UI** and environment to understand what action is required.
+**CRITICAL: Before making any decision, you MUST thoroughly analyze the agent thoughts history to:**
+- **Detect patterns of failure or repeated attempts** that suggest the current approach isn't working
+- **Identify contradictions** between what was planned and what actually happened
+- **Spot errors in previous reasoning** that need to be corrected
+- **Learn from successful strategies** used in similar situations
+- **Avoid repeating failed approaches** by recognizing when to change strategy
+
+1. **Analyze the agent thoughts first** - Review all previous agent thoughts to understand:
+   - What strategies have been tried and their outcomes
+   - Any errors or misconceptions in previous reasoning
+   - Patterns that indicate success or failure
+   - Whether the current approach should be continued or modified
 
-2.1. If some of the subgoals must be **completed** based on your observations, add them to `complete_subgoals_by_ids`. To justify your conclusion, you will fill in the `agent_thought` field based on:
+2. **Then analyze the UI** and environment to understand what action is required, but always in the context of what the agent thoughts reveal about the situation.
+
+3. If some of the subgoals must be **completed** based on your observations, add them to `complete_subgoals_by_ids`. To justify your conclusion, you will fill in the `agent_thought` field based on:
 
 - The current UI state
-- Past agent thoughts
-- Recent tool effects
+- **Critical analysis of past agent thoughts and their accuracy**
+- Recent tool effects and whether they matched expectations from agent thoughts
+- **Any corrections needed to previous reasoning or strategy**
+
+
+### The Rule of Element Interaction
+
+**You MUST follow it for every element interaction.**
+
+When you target a UI element (for a `tap`, `input_text`, `clear_text`, etc.), you **MUST** provide a comprehensive `target` object containing every piece of information you can find about **that single element**.
 
-  2.2. Otherwise, output a **stringified structured set of instructions** that an **Executor agent** can perform on a real mobile device:
+*   **1. `resource_id`**: Include this if it is present in the UI hierarchy.
+*   **2. `resource_id_index`**: If there are multiple elements with the same `resource_id`, provide the zero-based index of the specific one you are targeting.
+*   **3. `coordinates`**: Include the full bounds (`x`, `y`, `width`, `height`) if they are available.
+*   **4. `text`**: Include the *current text* content of the element (e.g., placeholder text for an input).
+*   **5. `text_index`**: If there are multiple elements with the same `text`, provide the zero-based index of the specific one you are targeting.
+
+**CRITICAL: The index must correspond to its identifier.** `resource_id_index` is only used when targeting by `resource_id`. `text_index` is only used when targeting by `text`. This ensures the fallback logic targets the correct element.
+
+**This is NOT optional.** Providing all locators if we have, it is the foundation of the system's reliability. It allows next steps to use a fallback mechanism: if the ID fails, it tries the coordinates, etc. Failing to provide this complete context will lead to action failures.
+
+### The Rule of Unpredictable Actions
+
+Certain actions have outcomes that can significantly and sometimes unpredictably change the UI. These include:
+- `back`
+- `launch_app`
+- `stop_app`
+- `open_link`
+- `tap` on an element that is clearly for navigation (e.g., a "Back" button, a menu item, a link to another screen).
+
+**CRITICAL RULE: If your decision includes one of these unpredictable actions, it MUST be the only action in your `Structured Decisions` for this turn. Else, use flows to group actions together.**
+
+This is not optional. Failing to isolate these actions will cause the system to act on an outdated understanding of the screen, leading to catastrophic errors. For example, after a `back` command, you MUST wait to see the new screen before deciding what to tap next.
+
+You may only group simple, predictable actions together, such as tapping a text field and then immediately typing into it (`tap` followed by `input_text`).
+
+### Outputting Your Decisions
+
+If you decide to act, output a **valid JSON stringified structured set of instructions** for the Executor.
 
 - These must be **concrete low-level actions**.
 - The executor has the following available tools: {{ executor_tools_list }}.
 - Your goal is to achieve subgoals **fast** - so you must put as much actions as possible in your instructions to complete all achievable subgoals (based on your observations) in one go.
-- To open URLs/links directly, use the `open_link` tool - it will automatically handle opening in the appropriate browser. It also handles deep links.
-- When you need to open an app, use the `find_packages` low-level action to try and get its name. Then, simply use the `launch_app` low-level action to launch it.
-- If you refer to a UI element or coordinates, specify it clearly (e.g., `resource-id: com.whatsapp:id/search`, `text: "Alice"`, `x: 100, y: 200`).
+- If you refer to a UI element or coordinates, specify it clearly (e.g., `resource-id: com.whatsapp:id/search`, `resource-id-index: 0`, `text: "Alice"`, `resource-id-index: 0`, `x: 100, y: 200, width: 100, height: 100`).
 - **The structure is up to you**, but it must be valid **JSON stringified output**. You will accompany this output with a **natural-language summary** of your reasoning and approach in your agent thought.
-- **Never use a sequence of `tap` + `input_text` to type into a field. Always use a single `input_text` action** with the correct `resource_id` (this already ensures the element is focused and the cursor is moved to the end).
-- When you want to launch/stop an app, prefer using its package name.
+-   **Always use a single `input_text` action** to type in a field. This tool handles focusing the element and placing the cursor correctly. If the tool feedback indicates verification is needed or shows None/empty content, perform verification before proceeding.
 - **Only reference UI element IDs or visible texts that are explicitly present in the provided UI hierarchy or screenshot. Do not invent, infer, or guess any IDs or texts that are not directly observed**.
 - **For text clearing**: When you need to completely clear text from an input field, always call the `clear_text` tool with the correct resource_id. This tool automatically focuses the element, and ensures the field is emptied. If you notice this tool fails to clear the text, try to long press the input, select all, and call `erase_one_char`.
 
@@ -57,17 +129,35 @@ Focus on the **current PENDING subgoal and the next subgoals not yet started**.
 - **Structured Decisions** _(optional)_:
   A **valid stringified JSON** describing what should be executed **right now** to advance through the subgoals as much as possible.
 
-- **Agent Thought** _(1-2 sentences)_:
-  If there is any information you need to remember for later steps, you must include it here, because only the agent thoughts will be used to produce the final structured output.
+- **Agent Thought** _(2-4 sentences)_:
+  **MANDATORY: Start by analyzing previous agent thoughts** - Did previous reasoning contain errors? Are we repeating failed approaches? What worked before in similar situations?
+  
+  Then explain your current decision based on this analysis. If there is any information you need to remember for later steps, you must include it here, because only the agent thoughts will be used to produce the final structured output.
 
-  This also helps other agents understand your decision and learn from future failures.
+  This also helps other agents understand your decision and learn from future failures. **Explicitly mention if you're correcting a previous error or changing strategy based on agent thoughts analysis.**
   You must also use this field to mention checkpoints when you perform actions without definite ending: for instance "Swiping up to reveal more recipes - last seen recipe was <ID or NAME>, stop when no more".
 
 **Important:** `complete_subgoals_by_ids` and the structured decisions are mutually exclusive: if you provide both, the structured decisions will be ignored. Therefore, you must always prioritize completing subgoals over providing structured decisions.
 
 ---
 
-### Example
+### Example 1
+
+#### Current Subgoal:
+
+> "Open WhatsApp"
+
+#### Structured Decisions:
+
+```text
+"{\"action\": \"launch_app\", \"app_name\": \"WhatsApp\"}"
+```
+
+#### Agent Thought:
+
+> I need to launch the WhatsApp app. I will use the `launch_app` tool to open it.
+
+### Exemple 2
 
 #### Current Subgoal:
 
@@ -76,12 +166,12 @@ Focus on the **current PENDING subgoal and the next subgoals not yet started**.
 #### Structured Decisions:
 
 ```text
-"{\"action\": \"tap\", \"target\": {\"resource_id\": \"com.whatsapp:id/menuitem_search\", \"text\": \"Search\"}}"
+"[{\"action\": \"tap\", \"target\": {\"resource_id\": \"com.whatsapp:id/menuitem_search\", \"resource_id_index\": 1, \"text\": \"Search\", \"text_index\": 0, \"coordinates\": {\"x\": 880, \"y\": 150, \"width\": 120, \"height\": 120}}}]"
 ```
 
 #### Agent Thought:
 
-> I will tap the search icon at the top of the WhatsApp interface to begin searching for Alice.
+> Analysis: No previous attempts, this is a fresh approach. I will tap the search icon to begin searching. I am providing its resource_id, coordinates, and text content to ensure the Executor can find it reliably, following the element rule.
 
 ### Input
 
@@ -94,9 +184,6 @@ Focus on the **current PENDING subgoal and the next subgoals not yet started**.
 **Current Subgoal (what needs to be done right now):**
 {{ current_subgoal }}
 
-**Agent thoughts (previous reasoning, observations about the environment):**
-{{ agents_thoughts }}
-
 **Executor agent feedback on latest UI decisions:**
 
 {{ executor_feedback }}

minitap/mobile_use/agents/cortex/cortex.py

@@ -16,7 +16,7 @@ from minitap.mobile_use.agents.planner.utils import get_current_subgoal
 from minitap.mobile_use.constants import EXECUTOR_MESSAGES_KEY
 from minitap.mobile_use.context import MobileUseContext
 from minitap.mobile_use.graph.state import State
-from minitap.mobile_use.services.llm import get_llm, with_fallback
+from minitap.mobile_use.services.llm import get_llm, invoke_llm_with_timeout_message, with_fallback
 from minitap.mobile_use.tools.index import EXECUTOR_WRAPPERS_TOOLS, format_tools_list
 from minitap.mobile_use.utils.conversations import get_screenshot_message_for_llm
 from minitap.mobile_use.utils.decorators import wrap_with_callbacks
@@ -44,7 +44,6 @@ class CortexNode:
             initial_goal=state.initial_goal,
             subgoal_plan=state.subgoal_plan,
             current_subgoal=get_current_subgoal(state.subgoal_plan),
-            agents_thoughts=state.agents_thoughts,
             executor_feedback=executor_feedback,
             executor_tools_list=format_tools_list(ctx=self.ctx, wrappers=EXECUTOR_WRAPPERS_TOOLS),
         )
@@ -79,8 +78,12 @@ class CortexNode:
             ctx=self.ctx, name="cortex", use_fallback=True, temperature=1
         ).with_structured_output(CortexOutput)
         response: CortexOutput = await with_fallback(
-            main_call=lambda: llm.ainvoke(messages),
-            fallback_call=lambda: llm_fallback.ainvoke(messages),
+            main_call=lambda: invoke_llm_with_timeout_message(
+                llm.ainvoke(messages), agent_name="Cortex"
+            ),
+            fallback_call=lambda: invoke_llm_with_timeout_message(
+                llm_fallback.ainvoke(messages), agent_name="Cortex (Fallback)"
+            ),
         )  # type: ignore
 
         is_subgoal_completed = (
@@ -91,7 +94,7 @@ class CortexNode:
         if not is_subgoal_completed:
             response.complete_subgoals_by_ids = []
 
-        return state.sanitize_update(
+        return await state.asanitize_update(
             ctx=self.ctx,
             update={
                 "agents_thoughts": [response.agent_thought],

minitap/mobile_use/agents/executor/executor.md

@@ -25,13 +25,7 @@ and your previous actions, you must:
 "I'm tapping on the chat item labeled 'Alice' to open the conversation."
 
 ```json
-{
-  "action": "tap",
-  "target": {
-    "text": "Alice",
-    "resource_id": "com.whatsapp:id/conversation_item"
-  }
-}
+  "[{\"tool_name\": \"tap\", \"arguments\": {\"target\": {\"resource_id\": \"com.whatsapp:id/conversation_item\", \"resource_id_index\": 0, \"text\": \"Alice\", \"text_index\": 0, \"coordinates\": {\"x\": 0, \"y\": 350, \"width\": 1080, \"height\": 80}}}}]"
 ```
 
 **→ Executor Action**:
@@ -39,14 +33,17 @@ and your previous actions, you must:
 Call the `tap_on_element` tool with:
 
 - `resource_id = "com.whatsapp:id/conversation_item"`
+- `resource_id_index = 0`
 - `text = "Alice"`
+- `text_index = 0`
+- `coordinates = {"x": 0, "y": 350, "width": 1080, "height": 80}`
 - `agent_thought = "I'm tapping on the chat item labeled 'Alice' to open the conversation."`
 
 ---
 
 ### ⚙️ Tools
 
-- Tools may include actions like: `tap`, `swipe`, `start_app`, `stop_app`, `find_packages`, `get_current_focus`, etc.
+- Tools may include actions like: `tap`, `swipe`, `launch_app`, `stop_app`, etc.
 - You **must not hardcode tool definitions** here.
 - Just use the right tool based on what the `structured_decisions` requires.
 - The tools are provided dynamically via LangGraph's tool binding mechanism.
@@ -55,13 +52,21 @@ Call the `tap_on_element` tool with:
 
 When using the `input_text` tool:
 
-- **Always provide the `resource_id` of the element** you want to type into.
+- **Provide all available information** in the target object to identify text input element
+  - `resource_id`: The resource ID of the text input element (when available)
+  - `resource_id_index`: The zero-based index of the specific resource ID you are targeting (when available)
+  - `text`: The current text content of the text input element (when available)
+  - `text_index`: The zero-based index of the specific text you are targeting (when available)
+  - `coordinates`: The bounds (ElementBounds) of the text input element (when available)
+
 - The tool will automatically:
 
-  1. **Focus the element first**
+  1. **Focus the element** using the provided identification parameters
   2. **Move the cursor to the end** of the existing text
   3. **Then type the new text**
 
+- **Important**: Special characters and markdown-like escape sequences (e.g., \n, \t, *, _) are not interpreted. For example, typing \n will insert the literal characters \ and n, not a line break.
+
 #### 🔄 Text Clearing Best Practice
 
 When you need to completely clear text from an input field, always use the clear_text tool with the correct resource_id.

minitap/mobile_use/agents/executor/executor.py

@@ -8,7 +8,7 @@ from langchain_google_vertexai.chat_models import ChatVertexAI
 from minitap.mobile_use.constants import EXECUTOR_MESSAGES_KEY
 from minitap.mobile_use.context import MobileUseContext
 from minitap.mobile_use.graph.state import State
-from minitap.mobile_use.services.llm import get_llm
+from minitap.mobile_use.services.llm import get_llm, invoke_llm_with_timeout_message
 from minitap.mobile_use.tools.index import EXECUTOR_WRAPPERS_TOOLS, get_tools_from_wrappers
 from minitap.mobile_use.utils.decorators import wrap_with_callbacks
 from minitap.mobile_use.utils.logger import get_logger
@@ -29,7 +29,7 @@ class ExecutorNode:
         structured_decisions = state.structured_decisions
         if not structured_decisions:
             logger.warning("No structured decisions found.")
-            return state.sanitize_update(
+            return await state.asanitize_update(
                 ctx=self.ctx,
                 update={
                     "agents_thoughts": [
@@ -62,9 +62,10 @@ class ExecutorNode:
             llm_bind_tools_kwargs["parallel_tool_calls"] = True
 
         llm = llm.bind_tools(**llm_bind_tools_kwargs)
-        response = await llm.ainvoke(messages)
-
-        return state.sanitize_update(
+        response = await invoke_llm_with_timeout_message(
+            llm.ainvoke(messages), agent_name="Executor"
+        )
+        return await state.asanitize_update(
             ctx=self.ctx,
             update={
                 "cortex_last_thought": cortex_last_thought,

minitap/mobile_use/agents/executor/utils.py

@@ -1,4 +1,5 @@
 from langchain_core.messages import BaseMessage
+
 from minitap.mobile_use.utils.conversations import is_tool_message
 
 
@@ -7,5 +8,5 @@ def is_last_tool_message_take_screenshot(messages: list[BaseMessage]) -> bool:
         return False
     for msg in messages[::-1]:
         if is_tool_message(msg):
-            return msg.name == "take_screenshot"
+            return msg.name == "glimpse_screen"
     return False

minitap/mobile_use/agents/hopper/hopper.py

@@ -2,10 +2,11 @@ from pathlib import Path
 
 from jinja2 import Template
 from langchain_core.messages import HumanMessage, SystemMessage
-from minitap.mobile_use.context import MobileUseContext
-from minitap.mobile_use.services.llm import get_llm
 from pydantic import BaseModel, Field
 
+from minitap.mobile_use.context import MobileUseContext
+from minitap.mobile_use.services.llm import get_llm, invoke_llm_with_timeout_message
+
 
 class HopperOutput(BaseModel):
     step: str = Field(
@@ -33,7 +34,9 @@ async def hopper(
 
     llm = get_llm(ctx=ctx, name="hopper", is_utils=True, temperature=0)
     structured_llm = llm.with_structured_output(HopperOutput)
-    response: HopperOutput = await structured_llm.ainvoke(messages)  # type: ignore
+    response: HopperOutput = await invoke_llm_with_timeout_message(
+        structured_llm.ainvoke(messages), agent_name="Hopper"
+    )  # type: ignore
     return HopperOutput(
         step=response.step,
         output=response.output,

minitap/mobile_use/agents/orchestrator/orchestrator.py

@@ -15,7 +15,7 @@ from minitap.mobile_use.agents.planner.utils import (
 )
 from minitap.mobile_use.context import MobileUseContext
 from minitap.mobile_use.graph.state import State
-from minitap.mobile_use.services.llm import get_llm
+from minitap.mobile_use.services.llm import get_llm, invoke_llm_with_timeout_message
 from minitap.mobile_use.utils.decorators import wrap_with_callbacks
 from minitap.mobile_use.utils.logger import get_logger
 
@@ -45,14 +45,18 @@ class OrchestratorNode:
                     else f"Starting the next subgoal: {new_subgoal}"
                 )
             ]
-            return _get_state_update(ctx=self.ctx, state=state, thoughts=thoughts, update_plan=True)
+            return await _get_state_update(
+                ctx=self.ctx, state=state, thoughts=thoughts, update_plan=True
+            )
 
         subgoals_to_examine = get_subgoals_by_ids(
             subgoals=state.subgoal_plan,
             ids=state.complete_subgoals_by_ids,
         )
         if len(subgoals_to_examine) <= 0:
-            return _get_state_update(ctx=self.ctx, state=state, thoughts=["No subgoal to examine."])
+            return await _get_state_update(
+                ctx=self.ctx, state=state, thoughts=["No subgoal to examine."]
+            )
 
         system_message = Template(
             Path(__file__).parent.joinpath("orchestrator.md").read_text(encoding="utf-8")
@@ -72,13 +76,16 @@ class OrchestratorNode:
 
         llm = get_llm(ctx=self.ctx, name="orchestrator", temperature=1)
         llm = llm.with_structured_output(OrchestratorOutput)
-        response: OrchestratorOutput = await llm.ainvoke(messages)  # type: ignore
-
+        response: OrchestratorOutput = await invoke_llm_with_timeout_message(
+            llm.ainvoke(messages), agent_name="Orchestrator"
+        )  # type: ignore
         if response.needs_replaning:
             thoughts = [response.reason]
             state.subgoal_plan = fail_current_subgoal(state.subgoal_plan)
             thoughts.append("==== END OF PLAN, REPLANNING ====")
-            return _get_state_update(ctx=self.ctx, state=state, thoughts=thoughts, update_plan=True)
+            return await _get_state_update(
+                ctx=self.ctx, state=state, thoughts=thoughts, update_plan=True
+            )
 
         state.subgoal_plan = complete_subgoals_by_ids(
             subgoals=state.subgoal_plan,
@@ -87,19 +94,25 @@ class OrchestratorNode:
         thoughts = [response.reason]
         if all_completed(state.subgoal_plan):
             logger.success("All the subgoals have been completed successfully.")
-            return _get_state_update(ctx=self.ctx, state=state, thoughts=thoughts, update_plan=True)
+            return await _get_state_update(
+                ctx=self.ctx, state=state, thoughts=thoughts, update_plan=True
+            )
 
         if current_subgoal.id not in response.completed_subgoal_ids:
             # The current subgoal is not yet complete.
-            return _get_state_update(ctx=self.ctx, state=state, thoughts=thoughts, update_plan=True)
+            return await _get_state_update(
+                ctx=self.ctx, state=state, thoughts=thoughts, update_plan=True
+            )
 
         state.subgoal_plan = start_next_subgoal(state.subgoal_plan)
         new_subgoal = get_current_subgoal(state.subgoal_plan)
         thoughts.append(f"==== NEXT SUBGOAL: {new_subgoal} ====")
-        return _get_state_update(ctx=self.ctx, state=state, thoughts=thoughts, update_plan=True)
+        return await _get_state_update(
+            ctx=self.ctx, state=state, thoughts=thoughts, update_plan=True
+        )
 
 
-def _get_state_update(
+async def _get_state_update(
     ctx: MobileUseContext,
     state: State,
     thoughts: list[str],
@@ -111,4 +124,6 @@ def _get_state_update(
     }
     if update_plan:
         update["subgoal_plan"] = state.subgoal_plan
-    return state.sanitize_update(ctx=ctx, update=update, agent="orchestrator")
+        if ctx.on_plan_changes:
+            await ctx.on_plan_changes(state.subgoal_plan, False)
+    return await state.asanitize_update(ctx=ctx, update=update, agent="orchestrator")

minitap/mobile_use/agents/outputter/outputter.py

@@ -3,13 +3,14 @@ from pathlib import Path
 
 from jinja2 import Template
 from langchain_core.messages import BaseMessage, HumanMessage, SystemMessage
+from pydantic import BaseModel
+
 from minitap.mobile_use.config import OutputConfig
 from minitap.mobile_use.context import MobileUseContext
 from minitap.mobile_use.graph.state import State
-from minitap.mobile_use.services.llm import get_llm
+from minitap.mobile_use.services.llm import get_llm, invoke_llm_with_timeout_message
 from minitap.mobile_use.utils.conversations import is_ai_message
 from minitap.mobile_use.utils.logger import get_logger
-from pydantic import BaseModel
 
 logger = get_logger(__name__)
 
@@ -61,7 +62,9 @@ async def outputter(
         if schema is not None:
             structured_llm = llm.with_structured_output(schema)
 
-    response = await structured_llm.ainvoke(messages)  # type: ignore
+    response = await invoke_llm_with_timeout_message(
+        structured_llm.ainvoke(messages), agent_name="Outputter"
+    )  # type: ignore
     if isinstance(response, BaseModel):
         if output_config.output_description and hasattr(response, "content"):
             response = json.loads(response.content)  # type: ignore