minitap-mobile-use 2.2.0__py3-none-any.whl → 2.4.0__py3-none-any.whl

minitap/mobile_use/agents/outputter/test_outputter.py

@@ -1,9 +1,21 @@
-from minitap.mobile_use.agents.outputter.outputter import outputter
-from minitap.mobile_use.config import LLM, OutputConfig
-from minitap.mobile_use.context import MobileUseContext
-from minitap.mobile_use.utils.logger import get_logger
+import sys
+from unittest.mock import AsyncMock, Mock, patch
+
+import pytest
 from pydantic import BaseModel
 
+sys.modules["langgraph.prebuilt.chat_agent_executor"] = Mock()
+sys.modules["minitap.mobile_use.graph.state"] = Mock()
+sys.modules["langchain_google_vertexai"] = Mock()
+sys.modules["langchain_google_genai"] = Mock()
+sys.modules["langchain_openai"] = Mock()
+sys.modules["langchain_cerebras"] = Mock()
+
+from minitap.mobile_use.agents.outputter.outputter import outputter  # noqa: E402
+from minitap.mobile_use.config import LLM, OutputConfig  # noqa: E402
+from minitap.mobile_use.context import MobileUseContext  # noqa: E402
+from minitap.mobile_use.utils.logger import get_logger  # noqa: E402
+
 logger = get_logger(__name__)
 
 
@@ -40,68 +52,118 @@ mocked_state = DummyState(
     ],
 )
 
-mocked_ctx = MobileUseContext(
-    llm_config={
+
+@pytest.fixture
+def mock_context():
+    """Create a properly mocked context with all required fields."""
+    ctx = Mock(spec=MobileUseContext)
+    ctx.llm_config = {
         "executor": LLM(provider="openai", model="gpt-5-nano"),
         "cortex": LLM(provider="openai", model="gpt-5-nano"),
         "planner": LLM(provider="openai", model="gpt-5-nano"),
         "orchestrator": LLM(provider="openai", model="gpt-5-nano"),
-    },
-)  # type: ignore
+    }
+    ctx.device = Mock()
+    ctx.hw_bridge_client = Mock()
+    ctx.screen_api_client = Mock()
+    return ctx
+
+
+@pytest.fixture
+def mock_state():
+    """Create a mock state with test data."""
+    return DummyState(
+        messages=[],
+        initial_goal="Find a green product on my website",
+        agents_thoughts=[
+            "Going on http://superwebsite.fr",
+            "Searching for products",
+            "Filtering by color",
+            "Color 'green' found for a 20 dollars product",
+        ],
+    )
+
+
+@patch("minitap.mobile_use.agents.outputter.outputter.get_llm")
+@pytest.mark.asyncio
+async def test_outputter_with_pydantic_model(mock_get_llm, mock_context, mock_state):
+    """Test outputter with Pydantic model output."""
+    # Mock the structured LLM response
+    mock_structured_llm = AsyncMock()
+    mock_structured_llm.ainvoke.return_value = MockPydanticSchema(
+        color="green", price=20, currency_symbol="$", website_url="http://superwebsite.fr"
+    )
 
+    # Mock the base LLM
+    mock_llm = Mock()
+    mock_llm.with_structured_output.return_value = mock_structured_llm
+    mock_get_llm.return_value = mock_llm
 
-async def test_outputter_with_pydantic_model():
-    logger.info("Starting test_outputter_with_pydantic_model")
     config = OutputConfig(
         structured_output=MockPydanticSchema,
         output_description=None,
     )
 
-    result = await outputter(ctx=mocked_ctx, output_config=config, graph_output=mocked_state)  # type: ignore
-
-    assert isinstance(result, MockPydanticSchema)
-    assert result.color.lower() == "green"
-    logger.success(str(result))
+    result = await outputter(ctx=mock_context, output_config=config, graph_output=mock_state)
 
+    assert isinstance(result, dict)
+    assert result.get("color") == "green"
+
+
+@patch("minitap.mobile_use.agents.outputter.outputter.get_llm")
+@pytest.mark.asyncio
+async def test_outputter_with_dict(mock_get_llm, mock_context, mock_state):
+    """Test outputter with dictionary output."""
+    # Mock the structured LLM response for dict
+    mock_structured_llm = AsyncMock()
+    expected_dict = {
+        "color": "green",
+        "price": 20,
+        "currency_symbol": "$",
+        "website_url": "http://superwebsite.fr",
+    }
+    mock_structured_llm.ainvoke.return_value = expected_dict
+
+    # Mock the base LLM
+    mock_llm = Mock()
+    mock_llm.with_structured_output.return_value = mock_structured_llm
+    mock_get_llm.return_value = mock_llm
 
-async def test_outputter_with_dict():
-    logger.info("Starting test_outputter_with_dict")
     config = OutputConfig(
         structured_output=mock_dict,
         output_description=None,
     )
 
-    result = await outputter(ctx=mocked_ctx, output_config=config, graph_output=mocked_state)  # type: ignore
+    result = await outputter(ctx=mock_context, output_config=config, graph_output=mock_state)
 
     assert isinstance(result, dict)
-    assert result.get("color", None) == "green"
-    assert result.get("price", None) == 20
-    assert result.get("currency_symbol", None) == "$"
-    assert result.get("website_url", None) == "http://superwebsite.fr"
-    logger.success(str(result))
-
+    assert result.get("color") == "green"
+    assert result.get("price") == 20
+    assert result.get("currency_symbol") == "$"
+    assert result.get("website_url") == "http://superwebsite.fr"
+
+
+@patch("minitap.mobile_use.agents.outputter.outputter.get_llm")
+@pytest.mark.asyncio
+async def test_outputter_with_natural_language_output(mock_get_llm, mock_context, mock_state):
+    """Test outputter with natural language description output."""
+    # Mock the LLM response for natural language output (no structured output)
+    mock_llm = AsyncMock()
+    expected_json = '{"color": "green", "price": 20, "currency_symbol": "$", "website_url": "http://superwebsite.fr"}'
+    mock_llm.ainvoke.return_value = Mock(content=expected_json)
+    mock_get_llm.return_value = mock_llm
 
-async def test_outputter_with_natural_language_output():
-    logger.info("Starting test_outputter_with_natural_language_output")
     config = OutputConfig(
         structured_output=None,
-        output_description="A JSON object with a color, \
-        a price, a currency_symbol and a website_url key",
+        output_description=(
+            "A JSON object with a color, a price, a currency_symbol and a website_url key"
+        ),
     )
 
-    result = await outputter(ctx=mocked_ctx, output_config=config, graph_output=mocked_state)  # type: ignore
-    logger.info(str(result))
+    result = await outputter(ctx=mock_context, output_config=config, graph_output=mock_state)
 
     assert isinstance(result, dict)
-    assert result.get("color", None) == "green"
-    assert result.get("price", None) == 20
-    assert result.get("currency_symbol", None) == "$"
-    assert result.get("website_url", None) == "http://superwebsite.fr"
-    logger.success(str(result))
-
-
-if __name__ == "__main__":
-    import asyncio
-
-    asyncio.run(test_outputter_with_pydantic_model())
-    asyncio.run(test_outputter_with_natural_language_output())
+    assert result.get("color") == "green"
+    assert result.get("price") == 20
+    assert result.get("currency_symbol") == "$"
+    assert result.get("website_url") == "http://superwebsite.fr"

minitap/mobile_use/agents/planner/planner.md

@@ -9,12 +9,13 @@ You work like an agile tech lead: defining the key milestones without locking in
    Given the **user's goal**:
 
    - Create a **high-level sequence of subgoals** to complete that goal.
-   - Subgoals should reflect real interactions with mobile UIs (e.g. "Open app", "Tap search bar", "Scroll to item", "Send message to Bob", etc).
+   - Subgoals should reflect real interactions with mobile UIs and describe the intent of the action (e.g., "Open the app to find a contact," "View the image to extract information," "Send a message to Bob confirming the appointment").
+   - Focus on the goal of the interaction, not just the physical action. For example, instead of 'View the receipt,' a better subgoal is 'Open and analyze the receipt to identify transactions.
    - Don't assume the full UI is visible yet. Plan based on how most mobile apps work, and keep flexibility.
-   - List of agents thoughts is empty which is expected, since it is the first plan.
-   - Avoid too granular UI actions based tasks (e.g. "tap", "swipe", "copy", "paste") unless explicitly required.
    - The executor has the following available tools: {{ executor_tools_list }}.
      When one of these tools offers a direct shortcut (e.g. `openLink` instead of manually launching a browser and typing a URL), prefer it over decomposed manual steps.
+   - Ensure that each subgoal prepares the ground for the next. If data needs to be gathered in one step to be used in another, the subgoal should reflect the intent to gather that data.
+
 
 2. **Replanning**
    If you're asked to **revise a previous plan**, you'll also receive:
@@ -27,38 +28,35 @@ You work like an agile tech lead: defining the key milestones without locking in
 
 ### Output
 
-You must output a **list of subgoals (description + optional subgoal ID)**, each representing a clear subgoal.
+You must output a **list of subgoals (description)**, each representing a clear subgoal.
 Each subgoal should be:
 
-- Focused on **realistic mobile interactions**
+- Focused on **purpose-driven mobile interactions** that clearly state the intent
 - Neither too vague nor too granular
 - Sequential (later steps may depend on earlier ones)
 - Don't use loop-like formulation unless necessary (e.g. don't say "repeat this X times", instead reuse the same steps X times as subgoals)
 
-If you're replaning and need to keep a previous subgoal, you **must keep the same subgoal ID**.
-
 ### Examples
 
-#### **Initial Goal**: "Open WhatsApp and send 'I’m running late' to Alice"
+#### **Initial Goal**: "Go on https://tesla.com, and tell me what is the first car being displayed"
 
 **Plan**:
 
-- Open the WhatsApp app (ID: None -> will be generated as a UUID like bc3c362d-f498-4f1a-991e-4a2d1f8c1226)
-- Locate or search for Alice (ID: None)
-- Open the conversation with Alice (ID: None)
-- Type the message "I’m running late" (ID: None)
-- Send the message (ID: None)
+- Open the link https://tesla.com to find information
+- Analyze the home page to identify the first car displayed
 
-#### **Initial Goal**: "Go on https://tesla.com, and tell me what is the first car being displayed"
+#### **Initial Goal**: "Open WhatsApp and send 'I’m running late' to Alice"
 
 **Plan**:
 
-- Open the link https://tesla.com (ID: None)
-- Find the first car displayed on the home page (ID: None)
+- Open the WhatsApp app to find the contact "Alice"
+- Open the conversation with Alice to send a message
+- Type the message "I’m running late" into the message field
+- Send the message
 
 #### **Replanning Example**
 
-**Original Plan**: same as above with IDs set
+**Original Plan**: same as above
 **Agent Thoughts**:
 
 - Couldn't find Alice in recent chats
@@ -67,8 +65,8 @@ If you're replaning and need to keep a previous subgoal, you **must keep the sam
 
 **New Plan**:
 
-- Open WhatsApp (ID: bc3c362d-f498-4f1a-991e-4a2d1f8c1226)
-- Tap the search bar (ID: None)
-- Search for "Alice" (ID: None)
-- Select the correct chat (ID: None)
-- Type and send "I’m running late" (ID: None)
+- Open WhatsApp
+- Tap the search bar to find a contact 
+- Search for "Alice" in the search field
+- Select the correct chat to open the conversation
+- Type and send "I’m running late"

minitap/mobile_use/agents/planner/planner.py

@@ -1,14 +1,13 @@
-import uuid
 from pathlib import Path
 
 from jinja2 import Template
 from langchain_core.messages import HumanMessage, SystemMessage
 
 from minitap.mobile_use.agents.planner.types import PlannerOutput, Subgoal, SubgoalStatus
-from minitap.mobile_use.agents.planner.utils import one_of_them_is_failure
+from minitap.mobile_use.agents.planner.utils import generate_id, one_of_them_is_failure
 from minitap.mobile_use.context import MobileUseContext
 from minitap.mobile_use.graph.state import State
-from minitap.mobile_use.services.llm import get_llm
+from minitap.mobile_use.services.llm import get_llm, invoke_llm_with_timeout_message
 from minitap.mobile_use.tools.index import EXECUTOR_WRAPPERS_TOOLS, format_tools_list
 from minitap.mobile_use.utils.decorators import wrap_with_callbacks
 from minitap.mobile_use.utils.logger import get_logger
@@ -49,11 +48,12 @@ class PlannerNode:
 
         llm = get_llm(ctx=self.ctx, name="planner")
         llm = llm.with_structured_output(PlannerOutput)
-        response: PlannerOutput = await llm.ainvoke(messages)  # type: ignore
-
+        response: PlannerOutput = await invoke_llm_with_timeout_message(
+            llm.ainvoke(messages), agent_name="Planner"
+        )  # type: ignore
         subgoals_plan = [
             Subgoal(
-                id=subgoal.id or str(uuid.uuid4()),
+                id=generate_id(),
                 description=subgoal.description,
                 status=SubgoalStatus.NOT_STARTED,
                 completion_reason=None,
@@ -63,7 +63,10 @@ class PlannerNode:
         logger.info("📜 Generated plan:")
         logger.info("\n".join(str(s) for s in subgoals_plan))
 
-        return state.sanitize_update(
+        if self.ctx.on_plan_changes:
+            await self.ctx.on_plan_changes(subgoals_plan, needs_replan)
+
+        return await state.asanitize_update(
             ctx=self.ctx,
             update={
                 "subgoal_plan": subgoals_plan,

minitap/mobile_use/agents/planner/types.py

@@ -1,11 +1,11 @@
+from datetime import datetime
 from enum import Enum
+from typing import Annotated
 
 from pydantic import BaseModel
-from typing import Annotated
 
 
 class PlannerSubgoalOutput(BaseModel):
-    id: Annotated[str | None, "If not provided, it will be generated"] = None
     description: str
 
 
@@ -27,6 +27,8 @@ class Subgoal(BaseModel):
         str | None, "Reason why the subgoal was completed (failure or success)"
     ] = None
     status: SubgoalStatus
+    started_at: Annotated[datetime | None, "When the subgoal started"] = None
+    ended_at: Annotated[datetime | None, "When the subgoal ended"] = None
 
     def __str__(self):
         status_emoji = "❓"

minitap/mobile_use/agents/planner/utils.py

@@ -1,4 +1,8 @@
+import random
+import string
+
 from minitap.mobile_use.agents.planner.types import Subgoal, SubgoalStatus
+from datetime import datetime, UTC
 
 
 def get_current_subgoal(subgoals: list[Subgoal]) -> Subgoal | None:
@@ -22,6 +26,7 @@ def complete_current_subgoal(subgoals: list[Subgoal]) -> list[Subgoal]:
     if not current_subgoal:
         return subgoals
     current_subgoal.status = SubgoalStatus.SUCCESS
+    current_subgoal.ended_at = datetime.now(UTC)
     return subgoals
 
 
@@ -29,6 +34,7 @@ def complete_subgoals_by_ids(subgoals: list[Subgoal], ids: list[str]) -> list[Su
     for subgoal in subgoals:
         if subgoal.id in ids:
             subgoal.status = SubgoalStatus.SUCCESS
+            subgoal.ended_at = datetime.now(UTC)
     return subgoals
 
 
@@ -37,6 +43,7 @@ def fail_current_subgoal(subgoals: list[Subgoal]) -> list[Subgoal]:
     if not current_subgoal:
         return subgoals
     current_subgoal.status = SubgoalStatus.FAILURE
+    current_subgoal.ended_at = datetime.now(UTC)
     return subgoals
 
 
@@ -53,4 +60,11 @@ def start_next_subgoal(subgoals: list[Subgoal]) -> list[Subgoal]:
     if not next_subgoal:
         return subgoals
     next_subgoal.status = SubgoalStatus.PENDING
+    next_subgoal.started_at = datetime.now(UTC)
     return subgoals
+
+
+def generate_id(length: int = 6) -> str:
+    """Generates a small and distinct random string ID."""
+    chars = string.ascii_lowercase + string.digits
+    return "".join(random.choice(chars) for _ in range(length))

minitap/mobile_use/agents/summarizer/summarizer.py

@@ -13,7 +13,7 @@ class SummarizerNode:
     def __init__(self, ctx: MobileUseContext):
         self.ctx = ctx
 
-    def __call__(self, state: State):
+    async def __call__(self, state: State):
         if len(state.messages) <= MAX_MESSAGES_IN_HISTORY:
             return {}
 
@@ -27,7 +27,7 @@ class SummarizerNode:
                 start_removal = True
             if start_removal and msg.id:
                 remove_messages.append(RemoveMessage(id=msg.id))
-            return state.sanitize_update(
+            return await state.asanitize_update(
                 ctx=self.ctx,
                 update={
                     "messages": remove_messages,

minitap/mobile_use/config.py

@@ -23,8 +23,10 @@ class Settings(BaseSettings):
     GOOGLE_API_KEY: SecretStr | None = None
     XAI_API_KEY: SecretStr | None = None
     OPEN_ROUTER_API_KEY: SecretStr | None = None
+    MINITAP_API_KEY: SecretStr | None = None
 
     OPENAI_BASE_URL: str | None = None
+    MINITAP_API_BASE_URL: str = "https://platform.minitap.ai"
 
     DEVICE_SCREEN_API_BASE_URL: str | None = None
     DEVICE_HARDWARE_BRIDGE_BASE_URL: str | None = None
@@ -90,7 +92,7 @@ def record_events(output_path: Path | None, events: list[str] | BaseModel | Any)
 
 ### LLM Configuration
 
-LLMProvider = Literal["openai", "google", "openrouter", "xai", "vertexai"]
+LLMProvider = Literal["openai", "google", "openrouter", "xai", "vertexai", "minitap"]
 LLMUtilsNode = Literal["outputter", "hopper"]
 AgentNode = Literal["planner", "orchestrator", "cortex", "executor"]
 AgentNodeWithFallback = Literal["cortex"]
@@ -131,6 +133,9 @@ class LLM(BaseModel):
             case "xai":
                 if not settings.XAI_API_KEY:
                     raise Exception(f"{name} requires XAI_API_KEY in .env")
+            case "minitap":
+                if not settings.MINITAP_API_KEY:
+                    raise Exception(f"{name} requires MINITAP_API_KEY in .env")
 
     def __str__(self):
         return f"{self.provider}/{self.model}"

minitap/mobile_use/context.py

@@ -4,17 +4,19 @@ Context variables for global state management.
 Uses ContextVar to avoid prop drilling and maintain clean function signatures.
 """
 
+from collections.abc import Callable, Coroutine
 from enum import Enum
 from pathlib import Path
+from typing import Literal
 
 from adbutils import AdbClient
 from openai import BaseModel
 from pydantic import ConfigDict
-from typing import Literal
 
+from minitap.mobile_use.agents.planner.types import Subgoal
 from minitap.mobile_use.clients.device_hardware_client import DeviceHardwareClient
 from minitap.mobile_use.clients.screen_api_client import ScreenApiClient
-from minitap.mobile_use.config import LLMConfig
+from minitap.mobile_use.config import AgentNode, LLMConfig
 
 
 class DevicePlatform(str, Enum):
@@ -45,18 +47,26 @@ class ExecutionSetup(BaseModel):
     """Execution setup for a task."""
 
     traces_path: Path
-    trace_id: str
+    trace_name: str
+    enable_remote_tracing: bool
+
+
+IsReplan = bool
 
 
 class MobileUseContext(BaseModel):
     model_config = ConfigDict(arbitrary_types_allowed=True)
 
+    trace_id: str
     device: DeviceContext
     hw_bridge_client: DeviceHardwareClient
     screen_api_client: ScreenApiClient
     llm_config: LLMConfig
     adb_client: AdbClient | None = None
     execution_setup: ExecutionSetup | None = None
+    on_agent_thought: Callable[[AgentNode, str], Coroutine] | None = None
+    on_plan_changes: Callable[[list[Subgoal], IsReplan], Coroutine] | None = None
+    minitap_api_key: str | None = None
 
     def get_adb_client(self) -> AdbClient:
         if self.adb_client is None:

minitap/mobile_use/controllers/mobile_command_controller.py

@@ -243,20 +243,6 @@ def input_text(ctx: MobileUseContext, text: str, dry_run: bool = False):
     return run_flow(ctx, [{"inputText": text}], dry_run=dry_run)
 
 
-def copy_text_from(ctx: MobileUseContext, selector_request: SelectorRequest, dry_run: bool = False):
-    copy_text_from_body = selector_request.to_dict()
-    if not copy_text_from_body:
-        error = "Invalid copyTextFrom selector request, could not format yaml"
-        logger.error(error)
-        raise ControllerErrors(error)
-    flow_input = [{"copyTextFrom": copy_text_from_body}]
-    return run_flow(ctx, flow_input, dry_run=dry_run)
-
-
-def paste_text(ctx: MobileUseContext, dry_run: bool = False):
-    return run_flow(ctx, ["pasteText"], dry_run=dry_run)
-
-
 def erase_text(ctx: MobileUseContext, nb_chars: int | None = None, dry_run: bool = False):
     """
     Removes characters from the currently selected textfield (if any)
@@ -333,6 +319,7 @@ def run_flow_with_wait_for_animation_to_end(
 
 if __name__ == "__main__":
     ctx = MobileUseContext(
+        trace_id="trace_id",
         llm_config=initialize_llm_config(),
         device=DeviceContext(
             host_platform="WINDOWS",

minitap/mobile_use/graph/state.py

@@ -54,7 +54,7 @@ class State(AgentStatePydantic):
         take_last,
     ]
 
-    def sanitize_update(
+    async def asanitize_update(
         self,
         ctx: MobileUseContext,
         update: dict,
@@ -72,7 +72,7 @@ class State(AgentStatePydantic):
                 raise ValueError("agents_thoughts must be a str or list[str]")
             if agent is None:
                 raise ValueError("Agent is required when updating the 'agents_thoughts' key")
-            update["agents_thoughts"] = _add_agent_thoughts(
+            update["agents_thoughts"] = await _add_agent_thoughts(
                 ctx=ctx,
                 old=self.agents_thoughts,
                 new=updated_agents_thoughts,
@@ -81,12 +81,16 @@ class State(AgentStatePydantic):
         return update
 
 
-def _add_agent_thoughts(
+async def _add_agent_thoughts(
     ctx: MobileUseContext,
     old: list[str],
     new: list[str],
     agent: AgentNode,
 ) -> list[str]:
+    if ctx.on_agent_thought:
+        for thought in new:
+            await ctx.on_agent_thought(agent, thought)
+
     named_thoughts = [f"[{agent}] {thought}" for thought in new]
     if ctx.execution_setup:
         record_interaction(ctx, response=AIMessage(content=str(named_thoughts)))