minitap-mobile-use 3.3.0__py3-none-any.whl

minitap/mobile_use/agents/executor/executor.py

@@ -0,0 +1,87 @@
+from pathlib import Path
+
+from jinja2 import Template
+from langchain_core.messages import HumanMessage, SystemMessage
+from langchain_google_genai import ChatGoogleGenerativeAI
+from langchain_google_vertexai.chat_models import ChatVertexAI
+
+from minitap.mobile_use.constants import EXECUTOR_MESSAGES_KEY
+from minitap.mobile_use.context import MobileUseContext
+from minitap.mobile_use.graph.state import State
+from minitap.mobile_use.services.llm import get_llm, invoke_llm_with_timeout_message, with_fallback
+from minitap.mobile_use.tools.index import (
+    EXECUTOR_WRAPPERS_TOOLS,
+    VIDEO_RECORDING_WRAPPERS,
+    get_tools_from_wrappers,
+)
+from minitap.mobile_use.utils.decorators import wrap_with_callbacks
+from minitap.mobile_use.utils.logger import get_logger
+
+logger = get_logger(__name__)
+
+
+class ExecutorNode:
+    def __init__(self, ctx: MobileUseContext):
+        self.ctx = ctx
+
+    @wrap_with_callbacks(
+        before=lambda: logger.info("Starting Executor Agent..."),
+        on_success=lambda _: logger.success("Executor Agent"),
+        on_failure=lambda _: logger.error("Executor Agent"),
+    )
+    async def __call__(self, state: State):
+        structured_decisions = state.structured_decisions
+        if not structured_decisions:
+            logger.warning("No structured decisions found.")
+            return await state.asanitize_update(
+                ctx=self.ctx,
+                update={
+                    "agents_thoughts": [
+                        "No structured decisions found, I cannot execute anything."
+                    ],
+                },
+                agent="executor",
+            )
+
+        system_message = Template(
+            Path(__file__).parent.joinpath("executor.md").read_text(encoding="utf-8")
+        ).render(platform=self.ctx.device.mobile_platform.value)
+        cortex_last_thought = (
+            state.cortex_last_thought if state.cortex_last_thought else state.agents_thoughts[-1]
+        )
+        messages = [
+            SystemMessage(content=system_message),
+            HumanMessage(content=cortex_last_thought),
+            HumanMessage(content=structured_decisions),
+            *state.executor_messages,
+        ]
+
+        llm = get_llm(ctx=self.ctx, name="executor")
+        llm_fallback = get_llm(ctx=self.ctx, name="executor", use_fallback=True)
+
+        executor_wrappers = list(EXECUTOR_WRAPPERS_TOOLS)
+        if self.ctx.video_recording_enabled:
+            executor_wrappers.extend(VIDEO_RECORDING_WRAPPERS)
+
+        llm_bind_tools_kwargs: dict = {
+            "tools": get_tools_from_wrappers(self.ctx, executor_wrappers),
+        }
+
+        # ChatGoogleGenerativeAI does not support the "parallel_tool_calls" keyword
+        if not isinstance(llm, ChatGoogleGenerativeAI | ChatVertexAI):
+            llm_bind_tools_kwargs["parallel_tool_calls"] = True
+
+        llm = llm.bind_tools(**llm_bind_tools_kwargs)
+        llm_fallback = llm_fallback.bind_tools(**llm_bind_tools_kwargs)
+        response = await with_fallback(
+            main_call=lambda: invoke_llm_with_timeout_message(llm.ainvoke(messages)),
+            fallback_call=lambda: invoke_llm_with_timeout_message(llm_fallback.ainvoke(messages)),
+        )
+        return await state.asanitize_update(
+            ctx=self.ctx,
+            update={
+                "cortex_last_thought": cortex_last_thought,
+                EXECUTOR_MESSAGES_KEY: [response],
+            },
+            agent="executor",
+        )

minitap/mobile_use/agents/executor/tool_node.py

@@ -0,0 +1,152 @@
+import asyncio
+import copy
+from typing import Any, override
+
+from langchain_core.messages import AnyMessage, ToolCall, ToolMessage
+from langchain_core.runnables import RunnableConfig
+from langgraph.prebuilt import ToolNode
+from langgraph.store.base import BaseStore
+from langgraph.types import Command
+from pydantic import BaseModel
+
+from minitap.mobile_use.services.telemetry import telemetry
+from minitap.mobile_use.utils.logger import get_logger
+
+logger = get_logger(__name__)
+
+
+class ExecutorToolNode(ToolNode):
+    """
+    ToolNode that runs tool calls one after the other - not simultaneously.
+    If one error occurs, the remaining tool calls are aborted!
+    """
+
+    def __init__(self, tools, messages_key: str, trace_id: str | None = None):
+        super().__init__(tools=tools, messages_key=messages_key)
+        self._trace_id = trace_id
+
+    @override
+    async def _afunc(
+        self,
+        input: list[AnyMessage] | dict[str, Any] | BaseModel,
+        config: RunnableConfig,
+        *,
+        store: BaseStore | None,
+    ):
+        return await self.__func(is_async=True, input=input, config=config, store=store)
+
+    @override
+    def _func(
+        self,
+        input: list[AnyMessage] | dict[str, Any] | BaseModel,
+        config: RunnableConfig,
+        *,
+        store: BaseStore | None,
+    ) -> Any:
+        loop = asyncio.get_event_loop()
+        return loop.run_until_complete(
+            self.__func(is_async=False, input=input, config=config, store=store)
+        )
+
+    async def __func(
+        self,
+        is_async: bool,
+        input: list[AnyMessage] | dict[str, Any] | BaseModel,
+        config: RunnableConfig,
+        *,
+        store: BaseStore | None,
+    ) -> Any:
+        tool_calls, input_type = self._parse_input(input, store)
+        outputs: list[Command | ToolMessage] = []
+        failed = False
+        for call in tool_calls:
+            if failed:
+                output = self._get_erroneous_command(
+                    call=call,
+                    message="Aborted: a previous tool call failed!",
+                )
+            else:
+                if is_async:
+                    output = await self._arun_one(call, input_type, config)
+                else:
+                    output = self._run_one(call, input_type, config)
+                failed = self._has_tool_call_failed(call, output)
+                if failed is None:
+                    output = self._get_erroneous_command(
+                        call=call,
+                        message=f"Unexpected tool output type: {type(output)}",
+                    )
+                    failed = True
+
+            call_without_state = copy.deepcopy(call)
+            if "args" in call_without_state and "state" in call_without_state["args"]:
+                del call_without_state["args"]["state"]
+            if failed:
+                error_msg = ""
+                try:
+                    if isinstance(output, ToolMessage):
+                        error_msg = output.content
+                    elif isinstance(output, Command):
+                        tool_msg = self._get_tool_message(output)
+                        error_msg = tool_msg.content
+                except Exception:
+                    error_msg = "Could not extract error details"
+
+                logger.info(f"❌ Tool call failed: {call_without_state}")
+                logger.info(f"   Error: {error_msg}")
+
+                # Capture executor action telemetry
+                if self._trace_id:
+                    telemetry.capture_executor_action(
+                        task_id=self._trace_id,
+                        tool_name=call["name"],
+                        success=False,
+                        error=str(error_msg)[:500] if error_msg else None,
+                    )
+            else:
+                logger.info("✅ Tool call succeeded: " + str(call_without_state))
+
+                # Capture executor action telemetry
+                if self._trace_id:
+                    telemetry.capture_executor_action(
+                        task_id=self._trace_id,
+                        tool_name=call["name"],
+                        success=True,
+                    )
+
+            outputs.append(output)
+        return self._combine_tool_outputs(outputs, input_type)  # type: ignore
+
+    def _has_tool_call_failed(
+        self,
+        call: ToolCall,
+        output: ToolMessage | Command,
+    ) -> bool | None:
+        if isinstance(output, ToolMessage):
+            return output.status == "error"
+        if isinstance(output, Command):
+            output_msg = self._get_tool_message(output)
+            return output_msg.status == "error"
+        return None
+
+    def _get_erroneous_command(self, call: ToolCall, message: str) -> Command:
+        tool_message = ToolMessage(
+            name=call["name"], tool_call_id=call["id"], content=message, status="error"
+        )
+        return Command(update={self.messages_key: [tool_message]})
+
+    def _get_tool_message(self, cmd: Command) -> ToolMessage:
+        if isinstance(cmd.update, dict):
+            msg = cmd.update.get(self.messages_key)
+            if isinstance(msg, list):
+                if len(msg) == 0:
+                    raise ValueError("No messages found in command update")
+                if not isinstance(msg[-1], ToolMessage):
+                    raise ValueError("Last message in command update is not a tool message")
+                return msg[-1]
+            elif isinstance(msg, ToolMessage):
+                return msg
+            elif msg is None:
+                raise ValueError(f"Missing '{self.messages_key}' in command update")
+            raise ValueError(f"Unexpected message type in command update: {type(msg)}")
+        raise ValueError("Command update is not a dict")

minitap/mobile_use/agents/hopper/hopper.md

@@ -0,0 +1,15 @@
+## Hopper
+
+Extract relevant information from batch data. **Keep extracted data exactly as-is** - no reformatting.
+
+## Output
+- **found**: `true` if data was found, `false` otherwise
+- **output**: Extracted information if found, `null` otherwise
+- **reason**: Brief explanation of search logic
+
+## Rules
+1. **Search entire input** - may contain hundreds of entries
+2. **For app package lookup**: Match app name (or variations) in package identifier
+   - Common patterns: lowercase app name, company+app, brand name, codenames
+3. **Prefer direct matches** over partial matches
+4. **Return `null`** if not found or if multiple ambiguous matches exist - don't guess

minitap/mobile_use/agents/hopper/hopper.py

@@ -0,0 +1,44 @@
+from pathlib import Path
+
+from jinja2 import Template
+from langchain_core.messages import HumanMessage, SystemMessage
+from pydantic import BaseModel, Field
+
+from minitap.mobile_use.context import MobileUseContext
+from minitap.mobile_use.services.llm import get_llm, invoke_llm_with_timeout_message, with_fallback
+
+
+class HopperOutput(BaseModel):
+    found: bool = Field(description="True if the requested data was found, False otherwise.")
+    output: str | None = Field(description="The extracted data if found, null otherwise.")
+    reason: str = Field(
+        description="A short explanation of what you looked for"
+        + " and how you decided what to extract."
+    )
+
+
+async def hopper(
+    ctx: MobileUseContext,
+    request: str,
+    data: str,
+) -> HopperOutput:
+    print("Starting Hopper Agent", flush=True)
+    system_message = Template(
+        Path(__file__).parent.joinpath("hopper.md").read_text(encoding="utf-8")
+    ).render()
+    messages = [
+        SystemMessage(content=system_message),
+        HumanMessage(content=f"{request}\nHere is the data you must dig:\n{data}"),
+    ]
+
+    llm = get_llm(ctx=ctx, name="hopper", is_utils=True, temperature=0).with_structured_output(
+        HopperOutput
+    )
+    llm_fallback = get_llm(
+        ctx=ctx, name="hopper", is_utils=True, use_fallback=True, temperature=0
+    ).with_structured_output(HopperOutput)
+    response: HopperOutput = await with_fallback(
+        main_call=lambda: invoke_llm_with_timeout_message(llm.ainvoke(messages)),
+        fallback_call=lambda: invoke_llm_with_timeout_message(llm_fallback.ainvoke(messages)),
+    )  # type: ignore
+    return response

minitap/mobile_use/agents/orchestrator/human.md

@@ -0,0 +1,12 @@
+Here is the input for your analysis:
+
+**Initial goal** : {{ initial_goal }}
+
+**Subgoal plan**
+{{ subgoal_plan }}
+
+**Subgoals to examine (provided by the Cortex)**
+{{ subgoals_to_examine }}
+
+**Agent thoughts**
+{{ agent_thoughts }}

minitap/mobile_use/agents/orchestrator/orchestrator.md

@@ -0,0 +1,21 @@
+## You are the **Orchestrator**
+
+Decide what to do next based on {{ platform }} mobile device execution state.
+
+## Input
+- Current **subgoal plan** with statuses
+- **Subgoals to examine** (PENDING/NOT_STARTED)
+- **Agent thoughts** from execution
+- **Initial goal**
+
+## Your Decisions
+
+1. **Mark subgoals complete**: Add finished subgoal IDs to `completed_subgoal_ids`
+2. **Set `needs_replanning = TRUE`** if repeated failures make current plan unworkable
+3. **Fill `reason`**: Final answer if goal complete, or explanation of decisions
+
+## Agent Roles (for context)
+- **Planner**: Creates/updates subgoal plan
+- **Cortex**: Analyzes screen, decides actions (may complete multiple subgoals at once)
+- **Executor**: Executes actions on device
+- **You (Orchestrator)**: Coordinate, track completion, trigger replanning

minitap/mobile_use/agents/orchestrator/orchestrator.py

@@ -0,0 +1,134 @@
+from pathlib import Path
+
+from jinja2 import Template
+from langchain_core.messages import HumanMessage, SystemMessage
+
+from minitap.mobile_use.agents.orchestrator.types import OrchestratorOutput
+from minitap.mobile_use.agents.planner.utils import (
+    all_completed,
+    complete_subgoals_by_ids,
+    fail_current_subgoal,
+    get_current_subgoal,
+    get_subgoals_by_ids,
+    nothing_started,
+    start_next_subgoal,
+)
+from minitap.mobile_use.context import MobileUseContext
+from minitap.mobile_use.graph.state import State
+from minitap.mobile_use.services.llm import get_llm, invoke_llm_with_timeout_message, with_fallback
+from minitap.mobile_use.utils.decorators import wrap_with_callbacks
+from minitap.mobile_use.utils.logger import get_logger
+
+logger = get_logger(__name__)
+
+
+class OrchestratorNode:
+    def __init__(self, ctx: MobileUseContext):
+        self.ctx = ctx
+
+    @wrap_with_callbacks(
+        before=lambda: logger.info("Starting Orchestrator Agent..."),
+        on_success=lambda _: logger.success("Orchestrator Agent"),
+        on_failure=lambda _: logger.error("Orchestrator Agent"),
+    )
+    async def __call__(self, state: State):
+        no_subgoal_started = nothing_started(state.subgoal_plan)
+        current_subgoal = get_current_subgoal(state.subgoal_plan)
+
+        if no_subgoal_started or not current_subgoal:
+            state.subgoal_plan = start_next_subgoal(state.subgoal_plan)
+            new_subgoal = get_current_subgoal(state.subgoal_plan)
+            thoughts = [
+                (
+                    f"Starting the first subgoal: {new_subgoal}"
+                    if no_subgoal_started
+                    else f"Starting the next subgoal: {new_subgoal}"
+                )
+            ]
+            return await _get_state_update(
+                ctx=self.ctx, state=state, thoughts=thoughts, update_plan=True
+            )
+
+        subgoals_to_examine = get_subgoals_by_ids(
+            subgoals=state.subgoal_plan,
+            ids=state.complete_subgoals_by_ids,
+        )
+        if len(subgoals_to_examine) <= 0:
+            return await _get_state_update(
+                ctx=self.ctx, state=state, thoughts=["No subgoal to examine."]
+            )
+
+        system_message = Template(
+            Path(__file__).parent.joinpath("orchestrator.md").read_text(encoding="utf-8")
+        ).render(platform=self.ctx.device.mobile_platform.value)
+        human_message = Template(
+            Path(__file__).parent.joinpath("human.md").read_text(encoding="utf-8")
+        ).render(
+            initial_goal=state.initial_goal,
+            subgoal_plan="\n".join(str(s) for s in state.subgoal_plan),
+            subgoals_to_examine="\n".join(str(s) for s in subgoals_to_examine),
+            agent_thoughts="\n".join(state.agents_thoughts),
+        )
+        messages = [
+            SystemMessage(content=system_message),
+            HumanMessage(content=human_message),
+        ]
+
+        llm = get_llm(ctx=self.ctx, name="orchestrator", temperature=1).with_structured_output(
+            OrchestratorOutput
+        )
+        llm_fallback = get_llm(
+            ctx=self.ctx, name="orchestrator", use_fallback=True, temperature=1
+        ).with_structured_output(OrchestratorOutput)
+        response: OrchestratorOutput = await with_fallback(
+            main_call=lambda: invoke_llm_with_timeout_message(llm.ainvoke(messages)),
+            fallback_call=lambda: invoke_llm_with_timeout_message(llm_fallback.ainvoke(messages)),
+        )  # type: ignore
+        if response.needs_replaning:
+            thoughts = [response.reason]
+            state.subgoal_plan = fail_current_subgoal(state.subgoal_plan)
+            thoughts.append("==== END OF PLAN, REPLANNING ====")
+            return await _get_state_update(
+                ctx=self.ctx, state=state, thoughts=thoughts, update_plan=True
+            )
+
+        state.subgoal_plan = complete_subgoals_by_ids(
+            subgoals=state.subgoal_plan,
+            ids=response.completed_subgoal_ids,
+        )
+        thoughts = [response.reason]
+        if all_completed(state.subgoal_plan):
+            logger.success("All the subgoals have been completed successfully.")
+            return await _get_state_update(
+                ctx=self.ctx, state=state, thoughts=thoughts, update_plan=True
+            )
+
+        if current_subgoal.id not in response.completed_subgoal_ids:
+            # The current subgoal is not yet complete.
+            return await _get_state_update(
+                ctx=self.ctx, state=state, thoughts=thoughts, update_plan=True
+            )
+
+        state.subgoal_plan = start_next_subgoal(state.subgoal_plan)
+        new_subgoal = get_current_subgoal(state.subgoal_plan)
+        thoughts.append(f"==== NEXT SUBGOAL: {new_subgoal} ====")
+        return await _get_state_update(
+            ctx=self.ctx, state=state, thoughts=thoughts, update_plan=True
+        )
+
+
+async def _get_state_update(
+    ctx: MobileUseContext,
+    state: State,
+    thoughts: list[str],
+    update_plan: bool = False,
+):
+    update = {
+        "agents_thoughts": thoughts,
+        "complete_subgoals_by_ids": [],
+    }
+    if update_plan:
+        update["subgoal_plan"] = state.subgoal_plan
+        if ctx.on_plan_changes:
+            await ctx.on_plan_changes(state.subgoal_plan, False)
+    return await state.asanitize_update(ctx=ctx, update=update, agent="orchestrator")

minitap/mobile_use/agents/orchestrator/types.py

@@ -0,0 +1,11 @@
+from typing import Annotated
+
+from pydantic import BaseModel
+
+
+class OrchestratorOutput(BaseModel):
+    completed_subgoal_ids: Annotated[
+        list[str], "IDs of subgoals that can now be marked as complete"
+    ] = []
+    needs_replaning: Annotated[bool, "Whether the orchestrator needs to replan the subgoal plan"]
+    reason: str

minitap/mobile_use/agents/outputter/human.md

@@ -0,0 +1,25 @@
+You are a helpful assistant tasked with generating the final structured output of a multi-agent reasoning process.
+
+## The original goal was:
+{{ initial_goal }}
+
+{% if agents_thoughts %}
+Throughout the reasoning process, the following agent thoughts were collected:
+{% for thought in agents_thoughts %}
+- {{ thought }}
+{% endfor %}
+{% endif %}
+
+{% if last_ai_message %}
+The last message generated by the graph execution was:
+"{{ last_ai_message }}"
+{% endif %}
+
+{% if not structured_output %}
+Please generate a well-structured JSON object based on the following instructions:
+
+> "{{ output_description }}"
+
+Only return the JSON object, with fields matching the description as closely as possible.  
+Do not include explanations or markdown, just the raw JSON.
+{% endif %}

minitap/mobile_use/agents/outputter/outputter.py

@@ -0,0 +1,85 @@
+import json
+from pathlib import Path
+
+from jinja2 import Template
+from langchain_core.messages import BaseMessage, HumanMessage, SystemMessage
+from pydantic import BaseModel
+
+from minitap.mobile_use.config import OutputConfig
+from minitap.mobile_use.context import MobileUseContext
+from minitap.mobile_use.graph.state import State
+from minitap.mobile_use.services.llm import get_llm, invoke_llm_with_timeout_message, with_fallback
+from minitap.mobile_use.utils.conversations import is_ai_message
+from minitap.mobile_use.utils.logger import get_logger
+
+logger = get_logger(__name__)
+
+
+async def outputter(
+    ctx: MobileUseContext, output_config: OutputConfig, graph_output: State
+) -> dict:
+    logger.info("Starting Outputter Agent")
+    last_message = graph_output.messages[-1] if graph_output.messages else None
+
+    system_message = (
+        "You are a helpful assistant tasked with generating "
+        + "the final structured output of a multi-agent reasoning process."
+    )
+    human_message = Template(
+        Path(__file__).parent.joinpath("human.md").read_text(encoding="utf-8")
+    ).render(
+        initial_goal=graph_output.initial_goal,
+        agents_thoughts=graph_output.agents_thoughts,
+        structured_output=output_config.structured_output,
+        output_description=output_config.output_description,
+        last_ai_message=last_message.content
+        if last_message and is_ai_message(message=last_message)
+        else None,
+    )
+
+    messages: list[BaseMessage] = [
+        SystemMessage(content=system_message),
+        HumanMessage(content=human_message),
+    ]
+
+    if output_config.output_description:
+        messages.append(HumanMessage(content=output_config.output_description))
+
+    llm = get_llm(ctx=ctx, name="outputter", is_utils=True, temperature=1)
+    llm_fallback = get_llm(
+        ctx=ctx, name="outputter", is_utils=True, use_fallback=True, temperature=1
+    )
+    structured_llm = llm
+    structured_llm_fallback = llm_fallback
+
+    if output_config.structured_output:
+        schema: dict | type[BaseModel] | None = None
+        so = output_config.structured_output
+
+        if isinstance(so, dict):
+            schema = so
+        elif isinstance(so, BaseModel):
+            schema = type(so)
+        elif isinstance(so, type) and issubclass(so, BaseModel):
+            schema = so
+
+        if schema is not None:
+            structured_llm = llm.with_structured_output(schema)
+            structured_llm_fallback = llm_fallback.with_structured_output(schema)
+
+    response = await with_fallback(
+        main_call=lambda: invoke_llm_with_timeout_message(structured_llm.ainvoke(messages)),
+        fallback_call=lambda: invoke_llm_with_timeout_message(
+            structured_llm_fallback.ainvoke(messages)
+        ),
+    )  # type: ignore
+    if isinstance(response, BaseModel):
+        if output_config.output_description and hasattr(response, "content"):
+            response = json.loads(response.content)  # type: ignore
+            return response
+        return response.model_dump()
+    elif hasattr(response, "content"):
+        return json.loads(response.content)  # type: ignore
+    else:
+        logger.info("Found unknown response type: " + str(type(response)))
+    return response