minitap-mobile-use 0.0.1.dev0__py3-none-any.whl

minitap/mobile_use/agents/orchestrator/types.py

@@ -0,0 +1,14 @@
+from enum import Enum
+
+from pydantic import BaseModel
+
+
+class OrchestratorStatus(Enum):
+    CONTINUE = "continue"
+    RESUME = "resume"
+    REPLAN = "replan"
+
+
+class OrchestratorOutput(BaseModel):
+    status: OrchestratorStatus
+    reason: str

minitap/mobile_use/agents/outputter/human.md

@@ -0,0 +1,25 @@
+You are a helpful assistant tasked with generating the final structured output of a multi-agent reasoning process.
+
+## The original goal was:
+{{ initial_goal }}
+
+{% if agents_thoughts %}
+Throughout the reasoning process, the following agent thoughts were collected:
+{% for thought in agents_thoughts %}
+- {{ thought }}
+{% endfor %}
+{% endif %}
+
+{% if last_ai_message %}
+The last message generated by the graph execution was:
+"{{ last_ai_message }}"
+{% endif %}
+
+{% if not structured_output %}
+Please generate a well-structured JSON object based on the following instructions:
+
+> "{{ output_description }}"
+
+Only return the JSON object, with fields matching the description as closely as possible.  
+Do not include explanations or markdown, just the raw JSON.
+{% endif %}

minitap/mobile_use/agents/outputter/outputter.py

@@ -0,0 +1,75 @@
+import json
+from pathlib import Path
+from typing import Dict, Type, Union
+
+from jinja2 import Template
+from langchain_core.messages import BaseMessage, HumanMessage, SystemMessage
+from minitap.mobile_use.config import OutputConfig
+from minitap.mobile_use.context import MobileUseContext
+from minitap.mobile_use.graph.state import State
+from minitap.mobile_use.services.llm import get_llm
+from minitap.mobile_use.utils.conversations import is_ai_message
+from minitap.mobile_use.utils.logger import get_logger
+from pydantic import BaseModel
+
+logger = get_logger(__name__)
+
+
+async def outputter(
+    ctx: MobileUseContext, output_config: OutputConfig, graph_output: State
+) -> dict:
+    logger.info("Starting Outputter Agent")
+    last_message = graph_output.messages[-1] if graph_output.messages else None
+
+    system_message = (
+        "You are a helpful assistant tasked with generating "
+        + "the final structured output of a multi-agent reasoning process."
+    )
+    human_message = Template(
+        Path(__file__).parent.joinpath("human.md").read_text(encoding="utf-8")
+    ).render(
+        initial_goal=graph_output.initial_goal,
+        agents_thoughts=graph_output.agents_thoughts,
+        structured_output=output_config.structured_output,
+        output_description=output_config.output_description,
+        last_ai_message=last_message.content
+        if last_message and is_ai_message(message=last_message)
+        else None,
+    )
+
+    messages: list[BaseMessage] = [
+        SystemMessage(content=system_message),
+        HumanMessage(content=human_message),
+    ]
+
+    if output_config.output_description:
+        messages.append(HumanMessage(content=output_config.output_description))
+
+    llm = get_llm(ctx=ctx, name="outputter", is_utils=True, temperature=1)
+    structured_llm = llm
+
+    if output_config.structured_output:
+        schema: Union[Dict, Type[BaseModel], None] = None
+        so = output_config.structured_output
+
+        if isinstance(so, dict):
+            schema = so
+        elif isinstance(so, BaseModel):
+            schema = type(so)
+        elif isinstance(so, type) and issubclass(so, BaseModel):
+            schema = so
+
+        if schema is not None:
+            structured_llm = llm.with_structured_output(schema)
+
+    response = await structured_llm.ainvoke(messages)  # type: ignore
+    if isinstance(response, BaseModel):
+        if output_config.output_description and hasattr(response, "content"):
+            response = json.loads(response.content)  # type: ignore
+            return response
+        return response.model_dump()
+    elif hasattr(response, "content"):
+        return json.loads(response.content)  # type: ignore
+    else:
+        logger.info("Found unknown response type: " + str(type(response)))
+    return response

minitap/mobile_use/agents/outputter/test_outputter.py

@@ -0,0 +1,107 @@
+from minitap.mobile_use.agents.outputter.outputter import outputter
+from minitap.mobile_use.config import LLM, OutputConfig
+from minitap.mobile_use.context import MobileUseContext
+from minitap.mobile_use.utils.logger import get_logger
+from pydantic import BaseModel
+
+logger = get_logger(__name__)
+
+
+class MockPydanticSchema(BaseModel):
+    color: str
+    price: float
+    currency_symbol: str
+    website_url: str
+
+
+mock_dict = {
+    "color": "green",
+    "price": 20,
+    "currency_symbol": "$",
+    "website_url": "http://superwebsite.fr",
+}
+
+
+class DummyState:
+    def __init__(self, messages, initial_goal, agents_thoughts):
+        self.messages = messages
+        self.initial_goal = initial_goal
+        self.agents_thoughts = agents_thoughts
+
+
+mocked_state = DummyState(
+    messages=[],
+    initial_goal="Find a green product on my website",
+    agents_thoughts=[
+        "Going on http://superwebsite.fr",
+        "Searching for products",
+        "Filtering by color",
+        "Color 'green' found for a 20 dollars product",
+    ],
+)
+
+mocked_ctx = MobileUseContext(
+    llm_config={
+        "executor": LLM(provider="openai", model="gpt-5-nano"),
+        "cortex": LLM(provider="openai", model="gpt-5-nano"),
+        "planner": LLM(provider="openai", model="gpt-5-nano"),
+        "orchestrator": LLM(provider="openai", model="gpt-5-nano"),
+    },
+)  # type: ignore
+
+
+async def test_outputter_with_pydantic_model():
+    logger.info("Starting test_outputter_with_pydantic_model")
+    config = OutputConfig(
+        structured_output=MockPydanticSchema,
+        output_description=None,
+    )
+
+    result = await outputter(ctx=mocked_ctx, output_config=config, graph_output=mocked_state)  # type: ignore
+
+    assert isinstance(result, MockPydanticSchema)
+    assert result.color.lower() == "green"
+    logger.success(str(result))
+
+
+async def test_outputter_with_dict():
+    logger.info("Starting test_outputter_with_dict")
+    config = OutputConfig(
+        structured_output=mock_dict,
+        output_description=None,
+    )
+
+    result = await outputter(ctx=mocked_ctx, output_config=config, graph_output=mocked_state)  # type: ignore
+
+    assert isinstance(result, dict)
+    assert result.get("color", None) == "green"
+    assert result.get("price", None) == 20
+    assert result.get("currency_symbol", None) == "$"
+    assert result.get("website_url", None) == "http://superwebsite.fr"
+    logger.success(str(result))
+
+
+async def test_outputter_with_natural_language_output():
+    logger.info("Starting test_outputter_with_natural_language_output")
+    config = OutputConfig(
+        structured_output=None,
+        output_description="A JSON object with a color, \
+        a price, a currency_symbol and a website_url key",
+    )
+
+    result = await outputter(ctx=mocked_ctx, output_config=config, graph_output=mocked_state)  # type: ignore
+    logger.info(str(result))
+
+    assert isinstance(result, dict)
+    assert result.get("color", None) == "green"
+    assert result.get("price", None) == 20
+    assert result.get("currency_symbol", None) == "$"
+    assert result.get("website_url", None) == "http://superwebsite.fr"
+    logger.success(str(result))
+
+
+if __name__ == "__main__":
+    import asyncio
+
+    asyncio.run(test_outputter_with_pydantic_model())
+    asyncio.run(test_outputter_with_natural_language_output())

minitap/mobile_use/agents/planner/human.md

@@ -0,0 +1,12 @@
+Here is your input.
+
+---
+
+**Action (plan or replan)**: {{ action }}
+
+**Initial Goal**: {{ initial_goal }}
+
+Relevant only if action is replan:
+
+**Previous Plan**: {{ previous_plan }}
+**Agent Thoughts**: {{ agent_thoughts }}

minitap/mobile_use/agents/planner/planner.md

@@ -0,0 +1,64 @@
+You are the **Planner**.
+Your role is to **break down a user’s goal into a realistic series of subgoals** that can be executed step-by-step on an {{ platform }} **mobile device**.
+
+You work like an agile tech lead: defining the key milestones without locking in details too early. Other agents will handle the specifics later.
+
+### Core Responsibilities
+
+1. **Initial Planning**
+   Given the **user's goal**:
+
+   - Create a **high-level sequence of subgoals** to complete that goal.
+   - Subgoals should reflect real interactions with mobile UIs (e.g. "Open app", "Tap search bar", "Scroll to item", "Send message to Bob", etc).
+   - Don't assume the full UI is visible yet. Plan based on how most mobile apps work, and keep flexibility.
+   - List of agents thoughts is empty which is expected, since it is the first plan.
+   - Don't use precise UI actions when formulating subgoals like "copy", "paste", "tap", "swipe", ... unless explicitly asked in the initial goal.
+
+2. **Replanning**
+   If you're asked to **revise a previous plan**, you'll also receive:
+
+   - The **original plan** (with notes about which subgoals succeeded or failed)
+   - A list of **agent thoughts**, including observations from the device, challenges encountered, and reasoning about what happened
+   - Take into account the agent thoughts/previous plan to update the plan : maybe some steps are not required as we successfully completed them.
+
+   Use these inputs to update the plan: removing dead ends, adapting to what we learned, and suggesting new directions.
+
+### Output
+
+You must output a **list of strings**, each representing a clear subgoal.
+Each subgoal should be:
+
+- Focused on **realistic mobile interactions**
+- Neither too vague nor too granular
+- Sequential (later steps may depend on earlier ones)
+- Don't use loop-like formulation unless necessary (e.g. don't say "repeat this X times", instead reuse the same steps X times as subgoals)
+
+### Examples
+
+#### **Initial Goal**: "Open WhatsApp and send 'I’m running late' to Alice"
+
+**Plan**:
+
+- Open the WhatsApp app
+- Locate or search for Alice
+- Open the conversation with Alice
+- Type the message "I’m running late"
+- Send the message
+
+#### **Replanning Example**
+
+**Original Plan**: same as above
+**Agent Thoughts**:
+
+- Couldn’t find Alice in recent chats
+- Search bar was present on top of the chat screen
+- Keyboard appeared after tapping search
+
+**New Plan**:
+
+- Unlock the phone if needed
+- Open WhatsApp
+- Tap the search bar
+- Search for "Alice"
+- Select the correct chat
+- Type and send "I’m running late"

minitap/mobile_use/agents/planner/planner.py

@@ -0,0 +1,64 @@
+from pathlib import Path
+
+from jinja2 import Template
+from langchain_core.messages import HumanMessage, SystemMessage
+from minitap.mobile_use.agents.planner.types import PlannerOutput, Subgoal, SubgoalStatus
+from minitap.mobile_use.agents.planner.utils import one_of_them_is_failure
+from minitap.mobile_use.context import MobileUseContext
+from minitap.mobile_use.graph.state import State
+from minitap.mobile_use.services.llm import get_llm
+from minitap.mobile_use.utils.decorators import wrap_with_callbacks
+from minitap.mobile_use.utils.logger import get_logger
+
+logger = get_logger(__name__)
+
+
+class PlannerNode:
+    def __init__(self, ctx: MobileUseContext):
+        self.ctx = ctx
+
+    @wrap_with_callbacks(
+        before=lambda: logger.info("Starting Planner Agent..."),
+        on_success=lambda _: logger.success("Planner Agent"),
+        on_failure=lambda _: logger.error("Planner Agent"),
+    )
+    async def __call__(self, state: State):
+        needs_replan = one_of_them_is_failure(state.subgoal_plan)
+
+        system_message = Template(
+            Path(__file__).parent.joinpath("planner.md").read_text(encoding="utf-8")
+        ).render(platform=self.ctx.device.mobile_platform.value)
+        human_message = Template(
+            Path(__file__).parent.joinpath("human.md").read_text(encoding="utf-8")
+        ).render(
+            action="replan" if needs_replan else "plan",
+            initial_goal=state.initial_goal,
+            previous_plan="\n".join(str(s) for s in state.subgoal_plan),
+            agent_thoughts="\n".join(state.agents_thoughts),
+        )
+        messages = [
+            SystemMessage(content=system_message),
+            HumanMessage(content=human_message),
+        ]
+
+        llm = get_llm(ctx=self.ctx, name="planner")
+        llm = llm.with_structured_output(PlannerOutput)
+        response: PlannerOutput = await llm.ainvoke(messages)  # type: ignore
+
+        subgoals_plan = [
+            Subgoal(
+                description=subgoal,
+                status=SubgoalStatus.NOT_STARTED,
+                completion_reason=None,
+            )
+            for subgoal in response.subgoals
+        ]
+        logger.info("📜 Generated plan:")
+        logger.info("\n".join(str(s) for s in subgoals_plan))
+
+        return state.sanitize_update(
+            ctx=self.ctx,
+            update={
+                "subgoal_plan": subgoals_plan,
+            },
+        )

minitap/mobile_use/agents/planner/types.py

@@ -0,0 +1,44 @@
+from enum import Enum
+from typing import Optional
+
+from pydantic import BaseModel
+from typing_extensions import Annotated
+
+
+class PlannerOutput(BaseModel):
+    subgoals: list[str]
+
+
+class SubgoalStatus(Enum):
+    NOT_STARTED = "NOT_STARTED"
+    PENDING = "PENDING"
+    SUCCESS = "SUCCESS"
+    FAILURE = "FAILURE"
+
+
+class Subgoal(BaseModel):
+    description: Annotated[str, "Description of the subgoal"]
+    completion_reason: Annotated[
+        Optional[str], "Reason why the subgoal was completed (failure or success)"
+    ] = None
+    status: SubgoalStatus
+
+    def __str__(self):
+        status_emoji = "❓"
+        match self.status:
+            case SubgoalStatus.SUCCESS:
+                status_emoji = "✅"
+            case SubgoalStatus.FAILURE:
+                status_emoji = "❌"
+            case SubgoalStatus.PENDING:
+                status_emoji = "⏳"
+            case SubgoalStatus.NOT_STARTED:
+                status_emoji = "(not started yet)"
+
+        output = f"- {self.description} : {status_emoji}."
+        if self.completion_reason:
+            output += f" Completion reason: {self.completion_reason}"
+        return output
+
+    def __repr__(self):
+        return str(self)

minitap/mobile_use/agents/planner/utils.py

@@ -0,0 +1,45 @@
+from minitap.mobile_use.agents.planner.types import Subgoal, SubgoalStatus
+
+
+def get_current_subgoal(subgoals: list[Subgoal]) -> Subgoal | None:
+    return next((s for s in subgoals if s.status == SubgoalStatus.PENDING), None)
+
+
+def get_next_subgoal(subgoals: list[Subgoal]) -> Subgoal | None:
+    return next((s for s in subgoals if s.status == SubgoalStatus.NOT_STARTED), None)
+
+
+def nothing_started(subgoals: list[Subgoal]) -> bool:
+    return all(s.status == SubgoalStatus.NOT_STARTED for s in subgoals)
+
+
+def complete_current_subgoal(subgoals: list[Subgoal]) -> list[Subgoal]:
+    current_subgoal = get_current_subgoal(subgoals)
+    if not current_subgoal:
+        return subgoals
+    current_subgoal.status = SubgoalStatus.SUCCESS
+    return subgoals
+
+
+def fail_current_subgoal(subgoals: list[Subgoal]) -> list[Subgoal]:
+    current_subgoal = get_current_subgoal(subgoals)
+    if not current_subgoal:
+        return subgoals
+    current_subgoal.status = SubgoalStatus.FAILURE
+    return subgoals
+
+
+def all_completed(subgoals: list[Subgoal]) -> bool:
+    return all(s.status == SubgoalStatus.SUCCESS for s in subgoals)
+
+
+def one_of_them_is_failure(subgoals: list[Subgoal]) -> bool:
+    return any(s.status == SubgoalStatus.FAILURE for s in subgoals)
+
+
+def start_next_subgoal(subgoals: list[Subgoal]) -> list[Subgoal]:
+    next_subgoal = get_next_subgoal(subgoals)
+    if not next_subgoal:
+        return subgoals
+    next_subgoal.status = SubgoalStatus.PENDING
+    return subgoals

minitap/mobile_use/agents/summarizer/summarizer.py

@@ -0,0 +1,34 @@
+from langchain_core.messages import (
+    HumanMessage,
+    RemoveMessage,
+    ToolMessage,
+)
+from minitap.mobile_use.constants import MAX_MESSAGES_IN_HISTORY
+from minitap.mobile_use.context import MobileUseContext
+from minitap.mobile_use.graph.state import State
+
+
+class SummarizerNode:
+    def __init__(self, ctx: MobileUseContext):
+        self.ctx = ctx
+
+    def __call__(self, state: State):
+        if len(state.messages) <= MAX_MESSAGES_IN_HISTORY:
+            return {}
+
+        nb_removal_candidates = len(state.messages) - MAX_MESSAGES_IN_HISTORY
+
+        remove_messages = []
+        start_removal = False
+
+        for msg in reversed(state.messages[:nb_removal_candidates]):
+            if isinstance(msg, (ToolMessage, HumanMessage)):
+                start_removal = True
+            if start_removal and msg.id:
+                remove_messages.append(RemoveMessage(id=msg.id))
+            return state.sanitize_update(
+                ctx=self.ctx,
+                update={
+                    "messages": remove_messages,
+                },
+            )

minitap/mobile_use/clients/device_hardware_client.py

@@ -0,0 +1,23 @@
+from urllib.parse import urljoin
+
+from minitap.mobile_use.utils.requests_utils import get_session_with_curl_logging
+
+
+class DeviceHardwareClient:
+    def __init__(self, base_url: str):
+        self.base_url = base_url
+        self.session = get_session_with_curl_logging()
+
+    def get(self, path: str, **kwargs):
+        url = urljoin(self.base_url, f"/api/{path.lstrip('/')}")
+        return self.session.get(url, **kwargs)
+
+    def post(self, path: str, **kwargs):
+        url = urljoin(self.base_url, f"/api/{path.lstrip('/')}")
+        return self.session.post(url, **kwargs)
+
+
+def get_client(base_url: str | None = None):
+    if not base_url:
+        base_url = "http://localhost:9999"
+    return DeviceHardwareClient(base_url)

minitap/mobile_use/clients/ios_client.py

@@ -0,0 +1,44 @@
+import json
+import platform
+
+from minitap.mobile_use.utils.shell_utils import run_shell_command_on_host
+
+
+def get_ios_devices() -> tuple[bool, list[str], str]:
+    """
+    Get UDIDs of iOS simulator devices only.
+
+    Returns:
+        A tuple containing:
+        - bool: True if xcrun is available, False otherwise.
+        - list[str]: A list of iOS device UDIDs.
+        - str: An error message if any.
+    """
+    if platform.system() != "Linux":
+        return False, [], "xcrun is only available on macOS."
+
+    try:
+        cmd = ["xcrun", "simctl", "list", "devices", "--json"]
+        output = run_shell_command_on_host(" ".join(cmd))
+        data = json.loads(output)
+
+        serials = []
+        devices_dict = data.get("devices", {})
+
+        for runtime, devices in devices_dict.items():
+            if "ios" in runtime.lower():  # e.g. "com.apple.CoreSimulator.SimRuntime.iOS-17-0"
+                for dev in devices:
+                    if "udid" in dev:
+                        serials.append(dev["udid"])
+
+        return True, serials, ""
+
+    except FileNotFoundError:
+        error_message = (
+            "'xcrun' command not found. Please ensure Xcode Command Line Tools are installed."
+        )
+        return False, [], error_message
+    except json.JSONDecodeError as e:
+        return True, [], f"Failed to parse xcrun output as JSON: {e}"
+    except Exception as e:
+        return True, [], f"Failed to get iOS devices: {e}"

minitap/mobile_use/clients/screen_api_client.py

@@ -0,0 +1,53 @@
+import os
+import time
+from urllib.parse import urljoin
+
+import requests
+from minitap.mobile_use.utils.logger import get_logger
+from minitap.mobile_use.utils.requests_utils import get_session_with_curl_logging
+
+logger = get_logger(__name__)
+
+
+class ScreenApiClient:
+    def __init__(self, base_url: str, retry_count: int = 5, retry_wait_seconds: int = 1):
+        self.base_url = base_url
+        self.session = get_session_with_curl_logging()
+        self.retry_count = retry_count
+        self.retry_wait_seconds = retry_wait_seconds
+
+    def get_with_retry(self, path: str, **kwargs):
+        """
+        Make a GET request to the Screen API with retry logic based on the client configuration.
+        """
+        for attempt in range(self.retry_count):
+            try:
+                response = self.session.get(urljoin(self.base_url, path), **kwargs)
+                if 200 <= response.status_code and response.status_code < 300:
+                    return response
+
+                logger.warning(
+                    f"Received {response.status_code}, attempt {attempt + 1} of {self.retry_count}."
+                    f" Retrying in {self.retry_wait_seconds} seconds..."
+                )
+                time.sleep(self.retry_wait_seconds)
+
+            except requests.exceptions.RequestException as e:
+                if attempt == self.retry_count - 1:
+                    raise e
+                time.sleep(self.retry_wait_seconds)
+
+        raise requests.exceptions.RequestException(
+            f"Failed to get a valid response after {self.retry_count} attempts."
+        )
+
+    def post(self, path: str, **kwargs):
+        return self.session.post(urljoin(self.base_url, path), **kwargs)
+
+
+def get_client(base_url: str | None = None):
+    if not base_url:
+        base_url = "http://localhost:9998"
+    retry_count = int(os.getenv("MOBILE_USE_HEALTH_RETRIES", 5))
+    retry_wait_seconds = int(os.getenv("MOBILE_USE_HEALTH_DELAY", 1))
+    return ScreenApiClient(base_url, retry_count, retry_wait_seconds)