minitap-mobile-use 3.3.0__py3-none-any.whl

minitap/mobile_use/agents/contextor/contextor.md

@@ -0,0 +1,55 @@
+## You are the **Contextor Agent**
+
+Verify app lock compliance. Decide: **relaunch locked app** or **allow deviation**.
+
+---
+
+## Context
+
+- **Locked app:** `{{ locked_app_package }}`
+- **Current app:** `{{ current_app_package }}` ← Different ?
+
+**Default: RELAUNCH.** Only allow deviation with clear justification.
+
+---
+
+## Allow Deviation ONLY If
+
+All conditions met:
+1. **Intentional** - Agent thoughts show explicit plan to use current app
+2. **Necessary** - Current app required for task (not just convenient)
+3. **Valid pattern**: OAuth/login flow, payment, system permissions, SMS/email verification, deep link
+
+## Relaunch If ANY True
+
+- Current app unrelated to task
+- Deviation looks accidental (no intent in agent thoughts)
+- Current app cannot help complete task
+- When in doubt → **RELAUNCH**
+
+---
+
+## Output
+
+```json
+{
+  "should_relaunch_app": true/false,
+  "reasoning": "2-4 sentences explaining decision"
+}
+```
+
+---
+
+## Input
+
+**Task Goal:** {{ task_goal }}
+
+**Subgoal Plan:** {{ subgoal_plan }}
+
+**Locked App:** {{ locked_app_package }}
+
+**Current App:** {{ current_app_package }}
+
+**Agent Thoughts:**
+{% for thought in agents_thoughts %}- {{ thought }}
+{% endfor %}

minitap/mobile_use/agents/contextor/contextor.py

@@ -0,0 +1,175 @@
+from pathlib import Path
+
+from jinja2 import Template
+from langchain_core.messages import HumanMessage, SystemMessage
+
+from minitap.mobile_use.agents.contextor.types import AppLockVerificationOutput, ContextorOutput
+from minitap.mobile_use.agents.planner.types import Subgoal
+from minitap.mobile_use.context import MobileUseContext
+from minitap.mobile_use.controllers.controller_factory import create_device_controller
+from minitap.mobile_use.controllers.platform_specific_commands_controller import (
+    get_current_foreground_package,
+    get_device_date,
+)
+from minitap.mobile_use.graph.state import State
+from minitap.mobile_use.services.llm import get_llm, invoke_llm_with_timeout_message, with_fallback
+from minitap.mobile_use.utils.app_launch_utils import launch_app_with_retries
+from minitap.mobile_use.utils.decorators import wrap_with_callbacks
+from minitap.mobile_use.utils.logger import get_logger
+
+logger = get_logger(__name__)
+
+
+class ContextorNode:
+    def __init__(self, ctx: MobileUseContext):
+        self.ctx = ctx
+
+    @wrap_with_callbacks(
+        before=lambda: logger.info("Starting Contextor Agent"),
+        on_success=lambda _: logger.success("Contextor Agent"),
+        on_failure=lambda _: logger.error("Contextor Agent"),
+    )
+    async def __call__(self, state: State):
+        device_controller = create_device_controller(self.ctx)
+        device_data = await device_controller.get_screen_data()
+        current_app_package = get_current_foreground_package(self.ctx)
+        device_date = get_device_date(self.ctx)
+        agent_outcome: str | None = None
+
+        if self.ctx.execution_setup and self.ctx.execution_setup.app_lock_status:
+            locked_app_package = self.ctx.execution_setup.app_lock_status.locked_app_package
+            should_verify_app_lock = (
+                self.ctx.execution_setup.app_lock_status.locked_app_initial_launch_success
+            )
+            if should_verify_app_lock:
+                if current_app_package:
+                    try:
+                        verification: AppLockVerificationOutput = (
+                            await self._handle_app_lock_verification(
+                                state=state,
+                                current_app_package=current_app_package,
+                                locked_app_package=locked_app_package,
+                            )
+                        )
+                        agent_outcome = verification.to_optional_message()
+                    except Exception as e:
+                        logger.error(f"Failed to verify app lock: {e}")
+                else:
+                    logger.warning(
+                        f"App lock feature is setup for {locked_app_package}, "
+                        "but could not determine current app, skipping"
+                    )
+            else:
+                logger.warning(
+                    f"App lock feature is setup for {locked_app_package}, "
+                    "but initial launch was not successful, skipping"
+                )
+
+        return await state.asanitize_update(
+            ctx=self.ctx,
+            update={
+                "latest_ui_hierarchy": device_data.elements,
+                "latest_screenshot": device_data.base64,
+                "focused_app_info": current_app_package,
+                "screen_size": (device_data.width, device_data.height),
+                "device_date": device_date,
+                "agents_thoughts": [agent_outcome],
+            },
+            agent="contextor",
+        )
+
+    async def _handle_app_lock_verification(
+        self, state: State, current_app_package: str, locked_app_package: str
+    ) -> AppLockVerificationOutput:
+        """Verify app lock compliance and decide whether to relaunch the locked app."""
+        if not self.ctx.execution_setup or not self.ctx.execution_setup.app_lock_status:
+            return AppLockVerificationOutput(
+                package_name=locked_app_package,
+                reasoning="App lock feature is not setup",
+                status="error",
+            )
+
+        app_lock_status = self.ctx.execution_setup.app_lock_status
+        locked_app_package = app_lock_status.locked_app_package
+
+        if current_app_package == locked_app_package:
+            logger.info(f"App lock verified: current app matches locked app ({locked_app_package})")
+            return AppLockVerificationOutput(
+                package_name=locked_app_package,
+                status="already_in_foreground",
+            )
+
+        logger.warning(
+            f"App lock violation detected: expected '{locked_app_package}', "
+            f"but current app is '{current_app_package}'"
+        )
+
+        decision: ContextorOutput = await self._invoke_contextor_llm(
+            initial_goal=state.initial_goal,
+            subgoal_plan=state.subgoal_plan,
+            agents_thoughts=state.agents_thoughts,
+            locked_app_package=locked_app_package,
+            current_app_package=current_app_package,
+        )
+
+        if decision.should_relaunch_app:
+            logger.info(f"Relaunching locked app: {locked_app_package}")
+            success, error = await launch_app_with_retries(self.ctx, app_package=locked_app_package)
+            if not success:
+                logger.error(f"Failed to relaunch {locked_app_package}: {error}")
+                return AppLockVerificationOutput(
+                    package_name=locked_app_package,
+                    reasoning=f"Failed to relaunch app: {error}",
+                    status="error",
+                )
+            return AppLockVerificationOutput(
+                package_name=locked_app_package,
+                reasoning=decision.reasoning,
+                status="relaunched",
+            )
+
+        logger.info(f"Allowing app deviation to: {current_app_package}")
+        return AppLockVerificationOutput(
+            package_name=locked_app_package,
+            reasoning=decision.reasoning,
+            status="allowed_deviation",
+        )
+
+    async def _invoke_contextor_llm(
+        self,
+        initial_goal: str,
+        subgoal_plan: list[Subgoal],
+        agents_thoughts: list[str],
+        locked_app_package: str,
+        current_app_package: str,
+    ) -> ContextorOutput:
+        """Invoke the LLM to decide whether to relaunch the locked app."""
+
+        MAX_AGENTS_THOUGHTS = 25
+
+        system_message = Template(
+            Path(__file__).parent.joinpath("contextor.md").read_text(encoding="utf-8")
+        ).render(
+            task_goal=initial_goal,
+            subgoal_plan="\n".join([str(subgoal) for subgoal in subgoal_plan]),
+            locked_app_package=locked_app_package,
+            current_app_package=current_app_package,
+            agents_thoughts=agents_thoughts[:MAX_AGENTS_THOUGHTS],
+        )
+
+        messages = [
+            SystemMessage(content=system_message),
+            HumanMessage(content="Please make your decision."),
+        ]
+
+        llm = get_llm(ctx=self.ctx, name="contextor").with_structured_output(ContextorOutput)
+        llm_fallback = get_llm(
+            ctx=self.ctx, name="contextor", use_fallback=True
+        ).with_structured_output(ContextorOutput)
+
+        response: ContextorOutput = await with_fallback(
+            main_call=lambda: invoke_llm_with_timeout_message(llm.ainvoke(messages)),
+            fallback_call=lambda: invoke_llm_with_timeout_message(llm_fallback.ainvoke(messages)),
+        )  # type: ignore
+
+        return response

minitap/mobile_use/agents/contextor/types.py

@@ -0,0 +1,36 @@
+from typing import Literal
+
+from pydantic import BaseModel, Field
+
+
+class ContextorOutput(BaseModel):
+    """Output schema for the Contextor agent decision."""
+
+    should_relaunch_app: bool = Field(..., description="Whether to relaunch the locked app")
+    reasoning: str = Field(
+        ..., description="Explanation of why we should or should not relaunch the app"
+    )
+
+
+class AppLockVerificationOutput(BaseModel):
+    package_name: str = Field(..., description="Package name of the app that was verified")
+    reasoning: str | None = Field(default=None, description="Reasoning for the decision")
+    status: Literal["already_in_foreground", "relaunched", "allowed_deviation", "error"] = Field(
+        ..., description="Status of the decision"
+    )
+
+    def to_optional_message(self) -> str | None:
+        msg = f"App {self.package_name}"
+        if self.reasoning:
+            msg = f"{self.reasoning} {msg}"
+        match self.status:
+            case "already_in_foreground":
+                return None
+            case "relaunched":
+                msg += " was successfully relaunched ✅"
+            case "allowed_deviation":
+                msg += " was allowed deviation ⚠️"
+            case "error":
+                msg = f"Could not verify app lock for {self.package_name}."
+
+        return msg

minitap/mobile_use/agents/cortex/cortex.md

@@ -0,0 +1,135 @@
+## You are the **Cortex**
+
+You analyze the {{ platform }} mobile device state and produce structured decisions to achieve subgoals. You are the brain giving instructions to the Executor (your hands).
+
+---
+
+## 🚨 CRITICAL RULES (Read First)
+
+### 1. Analyze Agent Thoughts Before Acting
+Before ANY decision, review agent thoughts history to:
+- Detect **repeated failures** → change strategy, don't retry blindly
+- Spot **contradictions** between plan and reality
+- Learn from what worked/failed
+
+### 2. Never Repeat Failed Actions
+If something failed, understand WHY before trying again. Ask: "How would a human solve this differently?"
+
+### 3. Unpredictable Actions = Isolate Them
+These actions change the screen unpredictably: `back`, `launch_app`, `stop_app`, `open_link`, navigation taps.
+**Rule:** If your decision includes one of these, it MUST be the ONLY action in that turn. Wait to see the new screen before deciding next steps.
+
+### 4. Complete Goals Only on OBSERVED Evidence
+Never mark a goal complete "in advance". Only complete based on executor feedback confirming success.
+
+### 5. Data Fidelity Over "Helpfulness"
+For any data-related task: transcribe content **exactly as-is** unless explicitly told otherwise.
+
+---
+
+## 📱 Perception
+
+You have 2 senses:
+
+| Sense | Use For | Limitation |
+|-------|---------|------------|
+| **UI Hierarchy** | Find elements by resource-id, text, bounds | No visual info (colors, images, obscured elements) |
+| **Screenshot** | Visual context, verify elements are visible, visual cues (badges, colors, icons) | Can't reliably extract precise element coordinates from pixels |
+
+You must combine your 2 senses to cancel out the limitations of each.
+
+---
+
+## 🎯 Element Targeting (MANDATORY)
+
+When targeting ANY element (tap, input, clear...), provide ALL available info:
+
+```json
+{
+  "target": {
+    "resource_id": "com.app:id/button",
+    "resource_id_index": 0,
+    "bounds": {"x": 100, "y": 200, "width": 50, "height": 50},
+    "text": "Submit",
+    "text_index": 0
+  }
+}
+```
+
+- `resource_id_index` = index among elements with same resource_id
+- `text_index` = index among elements with same text
+- This enables **fallback**: if ID fails → tries bounds → tries text
+
+**On tap failure:** "Out of bounds" = stale bounds. "No element found" = screen changed. Adapt, don't retry blindly.
+
+---
+
+## 🔧 Tools & Actions
+
+Available tools: {{ executor_tools_list }}
+
+| Action | Tool | Notes |
+|--------|------|-------|
+| **Open app** | `launch_app` | **ALWAYS use first** with app name (e.g., "WhatsApp"). Only try app drawer manually if launch_app fails. |
+| Open URL | `open_link` | Handles deep links correctly |
+| Type text | `focus_and_input_text` | Focuses + types. Verify if feedback shows empty. To create a blank line between paragraphs, use \n\n. |
+| Clear text | `focus_and_clear_text` | If fails, try: long press → select all → `erase_one_char` |
+
+### Swipe Physics
+Swipe direction "pushes" the screen: **swipe RIGHT → reveals LEFT page** (and vice versa).
+Default to **percentage-based** swipes. Use coordinates only for precise controls (sliders).
+Memory aid: Swipe RIGHT (low→high x) to see LEFT page. Swipe LEFT (high→low x) to see RIGHT page.
+
+### Form Filling
+Before concluding a field is missing, **scroll through the entire form** to verify all fields. If you observed a field earlier but can't find it now, scroll back - don't assume it's gone.
+**Rule:** Never input data into the wrong field if the correct field was previously observed.
+
+{% if locked_app_package %}
+---
+
+## 🔒 App Lock Mode
+
+Session locked to: **{{ locked_app_package }}**
+- Stay within this app
+- Avoid navigating away unless necessary (e.g., OAuth)
+- Contextor agent will relaunch if you leave accidentally
+{% endif %}
+
+---
+
+## 📤 Output Format
+
+| Field | Required | Description |
+|-------|----------|-------------|
+| **complete_subgoals_by_ids** | Optional | IDs of subgoals to mark complete (based on OBSERVED evidence) |
+| **Structured Decisions** | Optional | Valid JSON string of actions to execute |
+| **Decisions Reason** | Required | 2-4 sentences: analyze agent thoughts → explain decision → note strategy changes |
+| **Goals Completion Reason** | Required | Why completing these goals, or "None" |
+
+---
+
+## 📝 Example
+
+**Subgoal:** "Send 'Hello!' to Alice on WhatsApp"
+
+**Context:** Agent thoughts show previous turn typed "Hello!" successfully. UI shows message in field + send button visible.
+
+**Output:**
+```
+complete_subgoals_by_ids: ["subgoal-4-type-message"]
+Structured Decisions: "[{\"action\": \"tap\", \"target\": {\"resource_id\": \"com.whatsapp:id/send\", \"resource_id_index\": 0, \"bounds\": {\"x\": 950, \"y\": 1800, \"width\": 100, \"height\": 100}}}]"
+Decisions Reason: Agent thoughts confirm typing succeeded. Completing typing subgoal based on observed evidence. Now tapping send with full target info.
+Goals Completion Reason: Executor feedback confirmed "Hello!" was entered successfully.
+```
+
+---
+
+## Input
+
+**Initial Goal:** {{ initial_goal }}
+
+**Subgoal Plan:** {{ subgoal_plan }}
+
+**Current Subgoal:** {{ current_subgoal }}
+
+**Executor Feedback:** {{ executor_feedback }}

minitap/mobile_use/agents/cortex/cortex.py

@@ -0,0 +1,152 @@
+import json
+from pathlib import Path
+
+from jinja2 import Template
+from langchain_core.messages import (
+    AIMessage,
+    HumanMessage,
+    RemoveMessage,
+    SystemMessage,
+    ToolMessage,
+)
+from langgraph.graph.message import REMOVE_ALL_MESSAGES
+
+from minitap.mobile_use.agents.cortex.types import CortexOutput
+from minitap.mobile_use.agents.planner.utils import get_current_subgoal
+from minitap.mobile_use.constants import EXECUTOR_MESSAGES_KEY
+from minitap.mobile_use.context import MobileUseContext
+from minitap.mobile_use.controllers.controller_factory import create_device_controller
+from minitap.mobile_use.graph.state import State
+from minitap.mobile_use.services.llm import get_llm, invoke_llm_with_timeout_message, with_fallback
+from minitap.mobile_use.services.telemetry import telemetry
+from minitap.mobile_use.tools.index import (
+    EXECUTOR_WRAPPERS_TOOLS,
+    VIDEO_RECORDING_WRAPPERS,
+    format_tools_list,
+)
+from minitap.mobile_use.utils.conversations import get_screenshot_message_for_llm
+from minitap.mobile_use.utils.decorators import wrap_with_callbacks
+from minitap.mobile_use.utils.logger import get_logger
+
+logger = get_logger(__name__)
+
+
+class CortexNode:
+    def __init__(self, ctx: MobileUseContext):
+        self.ctx = ctx
+
+    @wrap_with_callbacks(
+        before=lambda: logger.info("Starting Cortex Agent..."),
+        on_success=lambda _: logger.success("Cortex Agent"),
+        on_failure=lambda _: logger.error("Cortex Agent"),
+    )
+    async def __call__(self, state: State):
+        executor_feedback = get_executor_agent_feedback(state)
+
+        current_locked_app_package = (
+            self.ctx.execution_setup.get_locked_app_package() if self.ctx.execution_setup else None
+        )
+
+        executor_wrappers = list(EXECUTOR_WRAPPERS_TOOLS)
+        if self.ctx.video_recording_enabled:
+            executor_wrappers.extend(VIDEO_RECORDING_WRAPPERS)
+
+        system_message = Template(
+            Path(__file__).parent.joinpath("cortex.md").read_text(encoding="utf-8")
+        ).render(
+            platform=self.ctx.device.mobile_platform.value,
+            initial_goal=state.initial_goal,
+            subgoal_plan=state.subgoal_plan,
+            current_subgoal=get_current_subgoal(state.subgoal_plan),
+            executor_feedback=executor_feedback,
+            executor_tools_list=format_tools_list(ctx=self.ctx, wrappers=executor_wrappers),
+            locked_app_package=current_locked_app_package,
+        )
+        messages = [
+            SystemMessage(content=system_message),
+            HumanMessage(
+                content="Here are my device info:\n"
+                + self.ctx.device.to_str()
+                + f"Device date: {state.device_date}\n"
+                if state.device_date
+                else "" + f"Focused app info: {state.focused_app_info}\n"
+                if state.focused_app_info
+                else ""
+            ),
+        ]
+        for thought in state.agents_thoughts:
+            messages.append(AIMessage(content=thought))
+
+        if state.latest_ui_hierarchy:
+            ui_hierarchy_dict: list[dict] = state.latest_ui_hierarchy
+            ui_hierarchy_str = json.dumps(ui_hierarchy_dict, indent=2, ensure_ascii=False)
+            messages.append(HumanMessage(content="Here is the UI hierarchy:\n" + ui_hierarchy_str))
+
+        if state.latest_screenshot:
+            controller = create_device_controller(self.ctx)
+            compressed_image_base64 = controller.get_compressed_b64_screenshot(
+                state.latest_screenshot
+            )
+            messages.append(get_screenshot_message_for_llm(compressed_image_base64))
+
+        llm = get_llm(ctx=self.ctx, name="cortex", temperature=1).with_structured_output(
+            CortexOutput
+        )
+        llm_fallback = get_llm(
+            ctx=self.ctx, name="cortex", use_fallback=True, temperature=1
+        ).with_structured_output(CortexOutput)
+        response: CortexOutput = await with_fallback(
+            main_call=lambda: invoke_llm_with_timeout_message(llm.ainvoke(messages)),
+            fallback_call=lambda: invoke_llm_with_timeout_message(llm_fallback.ainvoke(messages)),
+        )  # type: ignore
+
+        EMPTY_STRING_TOKENS = ["{}", "[]", "null", "", "None"]
+
+        if response.decisions in EMPTY_STRING_TOKENS:
+            response.decisions = None
+        if response.goals_completion_reason in EMPTY_STRING_TOKENS:
+            response.goals_completion_reason = None
+
+        thought_parts = []
+        if response.decisions_reason:
+            thought_parts.append(f"Decisions reason: {response.decisions_reason}")
+        if response.goals_completion_reason:
+            thought_parts.append(f"Goals completion reason: {response.goals_completion_reason}")
+
+        agent_thought = "\n\n".join(thought_parts)
+
+        # Capture cortex decision telemetry (only non-sensitive flags)
+        telemetry.capture_cortex_decision(
+            task_id=self.ctx.trace_id,
+            has_decisions=response.decisions is not None,
+            has_goals_completion=response.goals_completion_reason is not None,
+            completed_subgoals_count=len(response.complete_subgoals_by_ids or []),
+        )
+
+        return await state.asanitize_update(
+            ctx=self.ctx,
+            update={
+                "agents_thoughts": [agent_thought],
+                "structured_decisions": response.decisions,
+                "complete_subgoals_by_ids": response.complete_subgoals_by_ids,
+                "latest_ui_hierarchy": None,
+                "latest_screenshot": None,
+                "focused_app_info": None,
+                "device_date": None,
+                # Executor related fields
+                EXECUTOR_MESSAGES_KEY: [RemoveMessage(id=REMOVE_ALL_MESSAGES)],
+                "cortex_last_thought": agent_thought,
+            },
+            agent="cortex",
+        )
+
+
+def get_executor_agent_feedback(state: State) -> str:
+    if state.structured_decisions is None:
+        return "None."
+    executor_tool_messages = [m for m in state.executor_messages if isinstance(m, ToolMessage)]
+    return (
+        f"Latest UI decisions:\n{state.structured_decisions}"
+        + "\n\n"
+        + f"Executor feedback:\n{executor_tool_messages}"
+    )

minitap/mobile_use/agents/cortex/types.py

@@ -0,0 +1,15 @@
+from pydantic import BaseModel, Field
+
+
+class CortexOutput(BaseModel):
+    decisions: str | None = Field(
+        default=None, description="The decisions to be made. A stringified JSON object"
+    )
+    decisions_reason: str | None = Field(default=None, description="The reason for the decisions")
+    goals_completion_reason: str | None = Field(
+        default=None,
+        description="The reason for the goals completion, if there are any goals to be completed.",
+    )
+    complete_subgoals_by_ids: list[str] = Field(
+        default_factory=list, description="List of subgoal IDs to complete"
+    )

minitap/mobile_use/agents/executor/executor.md

@@ -0,0 +1,42 @@
+## You are the **Executor**
+
+Interpret Cortex decisions and execute tools on {{ platform }} mobile device. You are the hands, Cortex is the brain.
+
+---
+
+## Your Job
+
+1. **Parse** structured decisions from Cortex
+2. **Call tools** in the specified order
+3. **Always include `agent_thought`** for each tool - explains WHY (for debugging/tracing)
+
+---
+
+## Example
+
+**Cortex decision:**
+```json
+"[{\"action\": \"tap\", \"target\": {\"resource_id\": \"com.whatsapp:id/chat\", \"text\": \"Alice\", \"bounds\": {\"x\": 100, \"y\": 350, \"width\": 50, \"height\": 50}}}]"
+```
+
+**You execute:**
+```
+tap(target={resource_id: "com.whatsapp:id/chat", text: "Alice", ...}, agent_thought: "Tapping Alice's chat to open conversation")
+```
+
+---
+
+## Tool Notes
+
+| Tool | Notes |
+|------|-------|
+| `focus_and_input_text` | Provide full target info. Auto-focuses + moves cursor to end. Special chars are supported like newlines (use `\n` not `\\n`) as well as UTF-8 characters `行` |
+| `focus_and_clear_text` | Clears entire field. If fails: long press → select all → `erase_one_char` |
+
+---
+
+## Rules
+
+- **Don't reason about strategy** - just execute what Cortex decided
+- **`agent_thought` must be specific** - not generic/vague
+- **Order matters** - tools execute in the order you return them