minitap-mobile-use 3.3.0__py3-none-any.whl

minitap/mobile_use/agents/outputter/test_outputter.py

@@ -0,0 +1,167 @@
+import sys
+from unittest.mock import AsyncMock, Mock, patch
+
+import pytest
+from pydantic import BaseModel
+
+sys.modules["langgraph.prebuilt.chat_agent_executor"] = Mock()
+sys.modules["minitap.mobile_use.graph.state"] = Mock()
+sys.modules["langchain_google_vertexai"] = Mock()
+sys.modules["langchain_google_genai"] = Mock()
+sys.modules["langchain_openai"] = Mock()
+sys.modules["langchain_cerebras"] = Mock()
+
+from minitap.mobile_use.agents.outputter.outputter import outputter  # noqa: E402
+from minitap.mobile_use.config import LLM, OutputConfig  # noqa: E402
+from minitap.mobile_use.context import MobileUseContext  # noqa: E402
+from minitap.mobile_use.utils.logger import get_logger  # noqa: E402
+
+logger = get_logger(__name__)
+
+
+class MockPydanticSchema(BaseModel):
+    color: str
+    price: float
+    currency_symbol: str
+    website_url: str
+
+
+mock_dict = {
+    "color": "green",
+    "price": 20,
+    "currency_symbol": "$",
+    "website_url": "http://superwebsite.fr",
+}
+
+
+class DummyState:
+    def __init__(self, messages, initial_goal, agents_thoughts):
+        self.messages = messages
+        self.initial_goal = initial_goal
+        self.agents_thoughts = agents_thoughts
+
+
+mocked_state = DummyState(
+    messages=[],
+    initial_goal="Find a green product on my website",
+    agents_thoughts=[
+        "Going on http://superwebsite.fr",
+        "Searching for products",
+        "Filtering by color",
+        "Color 'green' found for a 20 dollars product",
+    ],
+)
+
+
+@pytest.fixture
+def mock_context():
+    """Create a properly mocked context with all required fields."""
+    ctx = Mock(spec=MobileUseContext)
+    ctx.llm_config = {
+        "executor": LLM(provider="openai", model="gpt-5-nano"),
+        "cortex": LLM(provider="openai", model="gpt-5-nano"),
+        "planner": LLM(provider="openai", model="gpt-5-nano"),
+        "orchestrator": LLM(provider="openai", model="gpt-5-nano"),
+    }
+    ctx.device = Mock()
+    return ctx
+
+
+@pytest.fixture
+def mock_state():
+    """Create a mock state with test data."""
+    return DummyState(
+        messages=[],
+        initial_goal="Find a green product on my website",
+        agents_thoughts=[
+            "Going on http://superwebsite.fr",
+            "Searching for products",
+            "Filtering by color",
+            "Color 'green' found for a 20 dollars product",
+        ],
+    )
+
+
+@patch("minitap.mobile_use.agents.outputter.outputter.get_llm")
+@pytest.mark.asyncio
+async def test_outputter_with_pydantic_model(mock_get_llm, mock_context, mock_state):
+    """Test outputter with Pydantic model output."""
+    # Mock the structured LLM response
+    mock_structured_llm = AsyncMock()
+    mock_structured_llm.ainvoke.return_value = MockPydanticSchema(
+        color="green", price=20, currency_symbol="$", website_url="http://superwebsite.fr"
+    )
+
+    # Mock the base LLM
+    mock_llm = Mock()
+    mock_llm.with_structured_output.return_value = mock_structured_llm
+    mock_get_llm.return_value = mock_llm
+
+    config = OutputConfig(
+        structured_output=MockPydanticSchema,
+        output_description=None,
+    )
+
+    result = await outputter(ctx=mock_context, output_config=config, graph_output=mock_state)
+
+    assert isinstance(result, dict)
+    assert result.get("color") == "green"
+
+
+@patch("minitap.mobile_use.agents.outputter.outputter.get_llm")
+@pytest.mark.asyncio
+async def test_outputter_with_dict(mock_get_llm, mock_context, mock_state):
+    """Test outputter with dictionary output."""
+    # Mock the structured LLM response for dict
+    mock_structured_llm = AsyncMock()
+    expected_dict = {
+        "color": "green",
+        "price": 20,
+        "currency_symbol": "$",
+        "website_url": "http://superwebsite.fr",
+    }
+    mock_structured_llm.ainvoke.return_value = expected_dict
+
+    # Mock the base LLM
+    mock_llm = Mock()
+    mock_llm.with_structured_output.return_value = mock_structured_llm
+    mock_get_llm.return_value = mock_llm
+
+    config = OutputConfig(
+        structured_output=mock_dict,
+        output_description=None,
+    )
+
+    result = await outputter(ctx=mock_context, output_config=config, graph_output=mock_state)
+
+    assert isinstance(result, dict)
+    assert result.get("color") == "green"
+    assert result.get("price") == 20
+    assert result.get("currency_symbol") == "$"
+    assert result.get("website_url") == "http://superwebsite.fr"
+
+
+@patch("minitap.mobile_use.agents.outputter.outputter.get_llm")
+@pytest.mark.asyncio
+async def test_outputter_with_natural_language_output(mock_get_llm, mock_context, mock_state):
+    """Test outputter with natural language description output."""
+    # Mock the LLM response for natural language output (no structured output)
+    mock_llm = AsyncMock()
+    expected_json = '{"color": "green", "price": 20, "currency_symbol": "$", "website_url": "http://superwebsite.fr"}'
+    mock_llm.ainvoke.return_value = Mock(content=expected_json)
+    mock_get_llm.return_value = mock_llm
+
+    config = OutputConfig(
+        structured_output=None,
+        output_description=(
+            "A JSON object with a color, a price, a currency_symbol and a website_url key"
+        ),
+    )
+
+    result = await outputter(ctx=mock_context, output_config=config, graph_output=mock_state)
+
+    assert isinstance(result, dict)
+    assert result.get("color") == "green"
+    assert result.get("price") == 20
+    assert result.get("currency_symbol") == "$"
+    assert result.get("website_url") == "http://superwebsite.fr"

minitap/mobile_use/agents/planner/human.md

@@ -0,0 +1,14 @@
+Here is your input.
+
+---
+
+**Action (plan or replan)**: {{ action }}
+
+**Initial Goal**: {{ initial_goal }}
+
+{% if action == "replan" %}
+Relevant only if action is replan:
+
+**Previous Plan**: {{ previous_plan }}
+**Agent Thoughts**: {{ agent_thoughts }}
+{% endif %}

minitap/mobile_use/agents/planner/planner.md

@@ -0,0 +1,126 @@
+## You are the **Planner**
+
+Break down user goals into **sequential subgoals** for {{ platform }} mobile execution.
+
+---
+
+## 🚨 Critical Rules
+
+{% if current_foreground_app %}
+### App Already Open: `{{ current_foreground_app }}`
+**NEVER** create "Open {{ current_foreground_app }}" subgoal. Start with first action INSIDE the app.
+{% endif %}
+{% if locked_app_package %}
+### App Lock: `{{ locked_app_package }}`
+All actions must stay within this app (except OAuth flows).
+{% endif %}
+
+---
+
+## Planning Guidelines
+
+**Subgoals should be:**
+- **Purpose-driven**: "Open conversation with Alice to send message" not just "Tap chat"
+- **Sequential**: Each step prepares the next
+- **Not too granular**: High-level milestones, not button-by-button
+- **No loops**: Instead of "repeat 3 times", write 3 separate subgoals
+- **Self-Correcting**: If the goal has specific formatting constraints (e.g., "add a new line", "sorted list"), include a final subgoal to **verify the result and fix it** if necessary.
+
+**Shortcuts**: Always prefer `launch_app` to open apps (not manual app drawer navigation), `open_link` for URLs.
+{% if video_recording_enabled %}
+### 🎥 Video Recording Pattern
+
+When a goal involves capturing, transcribing, or analyzing video content, follow this execution order:
+
+1. **Start recording BEFORE interacting with the video** — The recording must be active before any click that triggers playback
+2. **Perform video interactions** — Open/play the video and wait for it to complete
+3. **Stop recording AFTER the video ends** — Only stop once playback is fully finished
+
+⚠️ **Critical:** Each recording action (start/stop) must be its own dedicated subgoal. This prevents truncation due to short videos or inference latency.
+
+**Example subgoals for "transcribe video X":**
+1. Navigate to video X location
+2. Start screen recording
+3. Play video X and wait for completion
+4. Stop recording and extract transcription
+{% endif %}
+
+Available tools: {{ executor_tools_list }}
+
+---
+
+## Replanning
+
+When revising a failed plan:
+1. **Keep completed subgoals** - don't restart from scratch
+2. **Use agent thoughts** as source of truth for what happened
+3. **Pivot strategy** based on observations (e.g., use search if scrolling failed)
+4. **Continue from current state**, not from beginning
+
+---
+
+## Output Format
+
+```json
+{
+  "subgoals": [
+    {"description": "First subgoal description"},
+    {"description": "Second subgoal description"}
+  ]
+}
+```
+
+---
+
+## Examples
+
+**Goal:** "Send 'I'm running late' to Alice on WhatsApp"
+
+❌ **Bad subgoals (overlapping/vague):**
+```
+- Open WhatsApp to find Alice  ← What does "find" mean?
+- Open conversation with Alice  ← Might already be done if "find" included tapping
+```
+
+✅ **Good subgoals (atomic, non-overlapping):**
+```
+- Open WhatsApp
+- Navigate to Alice's conversation
+- Send the message "I'm running late"
+```
+
+**Key principle:** Each subgoal = one clear checkpoint. The Cortex decides HOW, the Planner defines WHAT milestone to reach.
+
+---
+
+**Replanning after failure:**
+```
+Original: "Navigate to Alice's conversation" (FAILED)
+Agent thoughts: Alice not in visible chats, search bar available
+
+New plan:
+- Search for "Alice" using search bar
+- Open conversation from search results
+- Send message
+```
+
+**Cross-app data transfer (using note tools):**
+```
+Goal: "Copy the recipe ingredients from RecipeApp and add them to my shopping list in ShoppingApp"
+
+✅ Correct subgoals:
+- Open RecipeApp and navigate to the recipe
+- Save the ingredients list using the `save_note` tool
+- Open ShoppingApp
+- Read the saved note using the `read_note` tool and add items to shopping list
+```
+{% if current_foreground_app %}
+
+**Foreground app already open (`{{ current_foreground_app }}`):**
+```
+Goal: "Send message to Bob"
+
+✅ Correct: Navigate to Bob's chat → Send message
+❌ Wrong: Open WhatsApp → ... (app already open!)
+```
+{% endif %}

minitap/mobile_use/agents/planner/planner.py

@@ -0,0 +1,101 @@
+from pathlib import Path
+
+from jinja2 import Template
+from langchain_core.messages import HumanMessage, SystemMessage
+
+from minitap.mobile_use.agents.planner.types import PlannerOutput, Subgoal, SubgoalStatus
+from minitap.mobile_use.agents.planner.utils import generate_id, one_of_them_is_failure
+from minitap.mobile_use.context import MobileUseContext
+from minitap.mobile_use.controllers.platform_specific_commands_controller import (
+    get_current_foreground_package,
+)
+from minitap.mobile_use.graph.state import State
+from minitap.mobile_use.services.llm import get_llm, invoke_llm_with_timeout_message, with_fallback
+from minitap.mobile_use.tools.index import (
+    EXECUTOR_WRAPPERS_TOOLS,
+    VIDEO_RECORDING_WRAPPERS,
+    format_tools_list,
+)
+from minitap.mobile_use.utils.decorators import wrap_with_callbacks
+from minitap.mobile_use.utils.logger import get_logger
+
+logger = get_logger(__name__)
+
+
+class PlannerNode:
+    def __init__(self, ctx: MobileUseContext):
+        self.ctx = ctx
+
+    @wrap_with_callbacks(
+        before=lambda: logger.info("Starting Planner Agent..."),
+        on_success=lambda _: logger.success("Planner Agent"),
+        on_failure=lambda _: logger.error("Planner Agent"),
+    )
+    async def __call__(self, state: State):
+        needs_replan = one_of_them_is_failure(state.subgoal_plan)
+
+        current_locked_app_package = (
+            self.ctx.execution_setup.get_locked_app_package() if self.ctx.execution_setup else None
+        )
+        current_foreground_app = get_current_foreground_package(self.ctx)
+
+        executor_wrappers = list(EXECUTOR_WRAPPERS_TOOLS)
+        if self.ctx.video_recording_enabled:
+            executor_wrappers.extend(VIDEO_RECORDING_WRAPPERS)
+
+        system_message = Template(
+            Path(__file__).parent.joinpath("planner.md").read_text(encoding="utf-8")
+        ).render(
+            platform=self.ctx.device.mobile_platform.value,
+            executor_tools_list=format_tools_list(ctx=self.ctx, wrappers=executor_wrappers),
+            locked_app_package=current_locked_app_package,
+            current_foreground_app=current_foreground_app,
+            video_recording_enabled=self.ctx.video_recording_enabled,
+        )
+        human_message = Template(
+            Path(__file__).parent.joinpath("human.md").read_text(encoding="utf-8")
+        ).render(
+            action="replan" if needs_replan else "plan",
+            initial_goal=state.initial_goal,
+            previous_plan="\n".join(str(s) for s in state.subgoal_plan),
+            agent_thoughts="\n".join(state.agents_thoughts),
+        )
+        messages = [
+            SystemMessage(content=system_message),
+            HumanMessage(content=human_message),
+        ]
+
+        llm = get_llm(ctx=self.ctx, name="planner").with_structured_output(PlannerOutput)
+        llm_fallback = get_llm(
+            ctx=self.ctx, name="planner", use_fallback=True
+        ).with_structured_output(PlannerOutput)
+        response: PlannerOutput = await with_fallback(
+            main_call=lambda: invoke_llm_with_timeout_message(
+                llm.ainvoke(messages),
+            ),
+            fallback_call=lambda: invoke_llm_with_timeout_message(
+                llm_fallback.ainvoke(messages),
+            ),
+        )  # type: ignore
+        subgoals_plan = [
+            Subgoal(
+                id=generate_id(),
+                description=subgoal.description,
+                status=SubgoalStatus.NOT_STARTED,
+                completion_reason=None,
+            )
+            for subgoal in response.subgoals
+        ]
+        logger.info("📜 Generated plan:")
+        logger.info("\n".join(str(s) for s in subgoals_plan))
+
+        if self.ctx.on_plan_changes:
+            await self.ctx.on_plan_changes(subgoals_plan, needs_replan)
+
+        return await state.asanitize_update(
+            ctx=self.ctx,
+            update={
+                "subgoal_plan": subgoals_plan,
+            },
+            agent="planner",
+        )

minitap/mobile_use/agents/planner/types.py

@@ -0,0 +1,51 @@
+from datetime import datetime
+from enum import Enum
+from typing import Annotated
+
+from pydantic import BaseModel
+
+
+class PlannerSubgoalOutput(BaseModel):
+    description: str
+
+
+class PlannerOutput(BaseModel):
+    subgoals: list[PlannerSubgoalOutput]
+
+
+class SubgoalStatus(Enum):
+    NOT_STARTED = "NOT_STARTED"
+    PENDING = "PENDING"
+    SUCCESS = "SUCCESS"
+    FAILURE = "FAILURE"
+
+
+class Subgoal(BaseModel):
+    id: Annotated[str, "Unique identifier of the subgoal"]
+    description: Annotated[str, "Description of the subgoal"]
+    completion_reason: Annotated[
+        str | None, "Reason why the subgoal was completed (failure or success)"
+    ] = None
+    status: SubgoalStatus
+    started_at: Annotated[datetime | None, "When the subgoal started"] = None
+    ended_at: Annotated[datetime | None, "When the subgoal ended"] = None
+
+    def __str__(self):
+        status_emoji = "❓"
+        match self.status:
+            case SubgoalStatus.SUCCESS:
+                status_emoji = "✅"
+            case SubgoalStatus.FAILURE:
+                status_emoji = "❌"
+            case SubgoalStatus.PENDING:
+                status_emoji = "⏳"
+            case SubgoalStatus.NOT_STARTED:
+                status_emoji = "(not started yet)"
+
+        output = f"- [ID:{self.id}]: {self.description} : {status_emoji}."
+        if self.completion_reason:
+            output += f" Completion reason: {self.completion_reason}"
+        return output
+
+    def __repr__(self):
+        return str(self)

minitap/mobile_use/agents/planner/utils.py

@@ -0,0 +1,70 @@
+import random
+import string
+
+from minitap.mobile_use.agents.planner.types import Subgoal, SubgoalStatus
+from datetime import datetime, UTC
+
+
+def get_current_subgoal(subgoals: list[Subgoal]) -> Subgoal | None:
+    return next((s for s in subgoals if s.status == SubgoalStatus.PENDING), None)
+
+
+def get_subgoals_by_ids(subgoals: list[Subgoal], ids: list[str]) -> list[Subgoal]:
+    return [s for s in subgoals if s.id in ids]
+
+
+def get_next_subgoal(subgoals: list[Subgoal]) -> Subgoal | None:
+    return next((s for s in subgoals if s.status == SubgoalStatus.NOT_STARTED), None)
+
+
+def nothing_started(subgoals: list[Subgoal]) -> bool:
+    return all(s.status == SubgoalStatus.NOT_STARTED for s in subgoals)
+
+
+def complete_current_subgoal(subgoals: list[Subgoal]) -> list[Subgoal]:
+    current_subgoal = get_current_subgoal(subgoals)
+    if not current_subgoal:
+        return subgoals
+    current_subgoal.status = SubgoalStatus.SUCCESS
+    current_subgoal.ended_at = datetime.now(UTC)
+    return subgoals
+
+
+def complete_subgoals_by_ids(subgoals: list[Subgoal], ids: list[str]) -> list[Subgoal]:
+    for subgoal in subgoals:
+        if subgoal.id in ids:
+            subgoal.status = SubgoalStatus.SUCCESS
+            subgoal.ended_at = datetime.now(UTC)
+    return subgoals
+
+
+def fail_current_subgoal(subgoals: list[Subgoal]) -> list[Subgoal]:
+    current_subgoal = get_current_subgoal(subgoals)
+    if not current_subgoal:
+        return subgoals
+    current_subgoal.status = SubgoalStatus.FAILURE
+    current_subgoal.ended_at = datetime.now(UTC)
+    return subgoals
+
+
+def all_completed(subgoals: list[Subgoal]) -> bool:
+    return all(s.status == SubgoalStatus.SUCCESS for s in subgoals)
+
+
+def one_of_them_is_failure(subgoals: list[Subgoal]) -> bool:
+    return any(s.status == SubgoalStatus.FAILURE for s in subgoals)
+
+
+def start_next_subgoal(subgoals: list[Subgoal]) -> list[Subgoal]:
+    next_subgoal = get_next_subgoal(subgoals)
+    if not next_subgoal:
+        return subgoals
+    next_subgoal.status = SubgoalStatus.PENDING
+    next_subgoal.started_at = datetime.now(UTC)
+    return subgoals
+
+
+def generate_id(length: int = 6) -> str:
+    """Generates a small and distinct random string ID."""
+    chars = string.ascii_lowercase + string.digits
+    return "".join(random.choice(chars) for _ in range(length))

minitap/mobile_use/agents/summarizer/summarizer.py

@@ -0,0 +1,35 @@
+from langchain_core.messages import (
+    HumanMessage,
+    RemoveMessage,
+    ToolMessage,
+)
+
+from minitap.mobile_use.constants import MAX_MESSAGES_IN_HISTORY
+from minitap.mobile_use.context import MobileUseContext
+from minitap.mobile_use.graph.state import State
+
+
+class SummarizerNode:
+    def __init__(self, ctx: MobileUseContext):
+        self.ctx = ctx
+
+    async def __call__(self, state: State):
+        if len(state.messages) <= MAX_MESSAGES_IN_HISTORY:
+            return {}
+
+        nb_removal_candidates = len(state.messages) - MAX_MESSAGES_IN_HISTORY
+
+        remove_messages = []
+        start_removal = False
+
+        for msg in reversed(state.messages[:nb_removal_candidates]):
+            if isinstance(msg, ToolMessage | HumanMessage):
+                start_removal = True
+            if start_removal and msg.id:
+                remove_messages.append(RemoveMessage(id=msg.id))
+            return await state.asanitize_update(
+                ctx=self.ctx,
+                update={
+                    "messages": remove_messages,
+                },
+            )

minitap/mobile_use/agents/video_analyzer/__init__.py

@@ -0,0 +1,5 @@
+"""Video analyzer utility for analyzing video content with Gemini models."""
+
+from minitap.mobile_use.agents.video_analyzer.video_analyzer import analyze_video
+
+__all__ = ["analyze_video"]

minitap/mobile_use/agents/video_analyzer/human.md

@@ -0,0 +1,5 @@
+Please analyze the following video recording and respond to my request.
+
+---
+
+**My Request**: {{ prompt }}

minitap/mobile_use/agents/video_analyzer/video_analyzer.md

@@ -0,0 +1,37 @@
+## You are a **Video Analysis Assistant**
+
+You analyze video recordings of mobile device screens and provide accurate, detailed responses based on what you observe.
+
+---
+
+## Your Focus Areas
+
+When analyzing videos, pay attention to:
+
+- **UI elements** and their states (buttons, text fields, toggles, etc.)
+- **Text content** displayed on screen
+- **Actions that occur** (taps, scrolls, transitions, animations)
+- **Notifications or dialogs** that appear
+- **Changes in the interface** over time
+- **Audio content** if present (transcribe speech, describe sounds)
+
+---
+
+## Guidelines
+
+- **Be precise and factual** - Only describe what you can actually see or hear
+- **Note uncertainty** - If you cannot clearly see or determine something, say so
+- **Be thorough** - Capture all relevant details that relate to the user's question
+- **Use timestamps** when describing sequences of events (e.g., "At 0:05, the user taps...")
+- **Structure your response** clearly when there's a lot of information
+
+---
+
+## Response Format
+
+Adapt your response format to the user's request:
+
+- For **transcription requests**: Provide clean, readable text of what was spoken or displayed
+- For **description requests**: Give a chronological narrative of events
+- For **specific questions**: Answer directly and concisely
+- For **extraction requests**: List items clearly (e.g., notifications, text content)