minitap-mobile-use 0.0.1.dev0__py3-none-any.whl

minitap/mobile_use/controllers/mobile_command_controller.py

@@ -0,0 +1,379 @@
+import uuid
+from enum import Enum
+from typing import Annotated, Literal, Optional, Union
+
+import yaml
+from langgraph.types import Command
+from pydantic import BaseModel, BeforeValidator, ConfigDict, Field
+from requests import JSONDecodeError
+
+from minitap.mobile_use.clients.device_hardware_client import DeviceHardwareClient
+from minitap.mobile_use.clients.screen_api_client import ScreenApiClient
+from minitap.mobile_use.context import DeviceContext, DevicePlatform, MobileUseContext
+from minitap.mobile_use.utils.errors import ControllerErrors
+from minitap.mobile_use.utils.logger import get_logger
+
+logger = get_logger(__name__)
+
+
+###### Screen elements retrieval ######
+
+
+class ScreenDataResponse(BaseModel):
+    base64: str
+    elements: list
+    width: int
+    height: int
+    platform: str
+
+
+def get_screen_data(screen_api_client: ScreenApiClient):
+    response = screen_api_client.get_with_retry("/screen-info")
+    return ScreenDataResponse(**response.json())
+
+
+def take_screenshot(ctx: MobileUseContext):
+    return get_screen_data(ctx.screen_api_client).base64
+
+
+class RunFlowRequest(BaseModel):
+    model_config = ConfigDict(extra="forbid")
+    yaml: str
+    dry_run: bool = Field(default=False, alias="dryRun")
+
+
+def run_flow(ctx: MobileUseContext, flow_steps: list, dry_run: bool = False) -> Optional[dict]:
+    """
+    Run a flow i.e, a sequence of commands.
+    Returns None on success, or the response body of the failed command.
+    """
+    logger.info(f"Running flow: {flow_steps}")
+
+    for step in flow_steps:
+        step_yml = yaml.dump(step)
+        payload = RunFlowRequest(yaml=step_yml, dryRun=dry_run).model_dump(by_alias=True)
+        response = ctx.hw_bridge_client.post("run-command", json=payload)
+
+        try:
+            response_body = response.json()
+        except JSONDecodeError:
+            response_body = response.text
+
+        if isinstance(response_body, dict):
+            response_body = {k: v for k, v in response_body.items() if v is not None}
+
+        if response.status_code >= 300:
+            logger.error(f"Tool call failed with status code: {response.status_code}")
+            return {"status_code": response.status_code, "body": response_body}
+
+    logger.success("Tool call completed")
+    return None
+
+
+class CoordinatesSelectorRequest(BaseModel):
+    model_config = ConfigDict(extra="forbid")
+    x: int
+    y: int
+
+    def to_str(self):
+        return f"{self.x}, {self.y}"
+
+
+class PercentagesSelectorRequest(BaseModel):
+    model_config = ConfigDict(extra="forbid")
+    """
+    0%,0%        # top-left corner
+    100%,100%    # bottom-right corner
+    50%,50%      # center
+    """
+
+    x_percent: int
+    y_percent: int
+
+    def to_str(self):
+        return f"{self.x_percent}%, {self.y_percent}%"
+
+
+class IdSelectorRequest(BaseModel):
+    model_config = ConfigDict(extra="forbid")
+    id: str
+
+    def to_dict(self) -> dict[str, str | int]:
+        return {"id": self.id}
+
+
+# Useful to tap on an element when there are multiple views with the same id
+class IdWithTextSelectorRequest(BaseModel):
+    model_config = ConfigDict(extra="forbid")
+    id: str
+    text: str
+
+    def to_dict(self) -> dict[str, str | int]:
+        return {"id": self.id, "text": self.text}
+
+
+class TextSelectorRequest(BaseModel):
+    model_config = ConfigDict(extra="forbid")
+    text: str
+
+    def to_dict(self) -> dict[str, str | int]:
+        return {"text": self.text}
+
+
+class SelectorRequestWithCoordinates(BaseModel):
+    model_config = ConfigDict(extra="forbid")
+    coordinates: CoordinatesSelectorRequest
+
+    def to_dict(self) -> dict[str, str | int]:
+        return {"point": self.coordinates.to_str()}
+
+
+class SelectorRequestWithPercentages(BaseModel):
+    model_config = ConfigDict(extra="forbid")
+    percentages: PercentagesSelectorRequest
+
+    def to_dict(self) -> dict[str, str | int]:
+        return {"point": self.percentages.to_str()}
+
+
+SelectorRequest = Union[
+    IdSelectorRequest,
+    SelectorRequestWithCoordinates,
+    SelectorRequestWithPercentages,
+    TextSelectorRequest,
+    IdWithTextSelectorRequest,
+]
+
+
+def tap(
+    ctx: MobileUseContext,
+    selector_request: SelectorRequest,
+    dry_run: bool = False,
+    index: Optional[int] = None,
+):
+    """
+    Tap on a selector.
+    Index is optional and is used when you have multiple views matching the same selector.
+    """
+    tap_body = selector_request.to_dict()
+    if not tap_body:
+        error = "Invalid tap selector request, could not format yaml"
+        logger.error(error)
+        raise ControllerErrors(error)
+    if index:
+        tap_body["index"] = index
+    flow_input = [{"tapOn": tap_body}]
+    return run_flow_with_wait_for_animation_to_end(ctx, flow_input, dry_run=dry_run)
+
+
+def long_press_on(
+    ctx: MobileUseContext,
+    selector_request: SelectorRequest,
+    dry_run: bool = False,
+    index: Optional[int] = None,
+):
+    long_press_on_body = selector_request.to_dict()
+    if not long_press_on_body:
+        error = "Invalid longPressOn selector request, could not format yaml"
+        logger.error(error)
+        raise ControllerErrors(error)
+    if index:
+        long_press_on_body["index"] = index
+    flow_input = [{"longPressOn": long_press_on_body}]
+    return run_flow_with_wait_for_animation_to_end(ctx, flow_input, dry_run=dry_run)
+
+
+class SwipeStartEndCoordinatesRequest(BaseModel):
+    model_config = ConfigDict(extra="forbid")
+    start: CoordinatesSelectorRequest
+    end: CoordinatesSelectorRequest
+
+    def to_dict(self):
+        return {"start": self.start.to_str(), "end": self.end.to_str()}
+
+
+class SwipeStartEndPercentagesRequest(BaseModel):
+    model_config = ConfigDict(extra="forbid")
+    start: PercentagesSelectorRequest
+    end: PercentagesSelectorRequest
+
+    def to_dict(self):
+        return {"start": self.start.to_str(), "end": self.end.to_str()}
+
+
+SwipeDirection = Annotated[
+    Literal["UP", "DOWN", "LEFT", "RIGHT"],
+    BeforeValidator(lambda v: v.upper() if isinstance(v, str) else v),
+]
+
+
+class SwipeRequest(BaseModel):
+    model_config = ConfigDict(extra="forbid")
+    swipe_mode: SwipeStartEndCoordinatesRequest | SwipeStartEndPercentagesRequest | SwipeDirection
+    duration: Optional[int] = None  # in ms, default is 400ms
+
+    def to_dict(self):
+        res = {}
+        if isinstance(self.swipe_mode, SwipeStartEndCoordinatesRequest):
+            res |= self.swipe_mode.to_dict()
+        elif isinstance(self.swipe_mode, SwipeStartEndPercentagesRequest):
+            res |= self.swipe_mode.to_dict()
+        elif self.swipe_mode in ["UP", "DOWN", "LEFT", "RIGHT"]:
+            res |= {"direction": self.swipe_mode}
+        if self.duration:
+            res |= {"duration": self.duration}
+        return res
+
+
+def swipe(ctx: MobileUseContext, swipe_request: SwipeRequest, dry_run: bool = False):
+    swipe_body = swipe_request.to_dict()
+    if not swipe_body:
+        error = "Invalid swipe selector request, could not format yaml"
+        logger.error(error)
+        raise ControllerErrors(error)
+    flow_input = [{"swipe": swipe_body}]
+    return run_flow_with_wait_for_animation_to_end(ctx, flow_input, dry_run=dry_run)
+
+
+##### Text related commands #####
+
+
+def input_text(ctx: MobileUseContext, text: str, dry_run: bool = False):
+    return run_flow(ctx, [{"inputText": text}], dry_run=dry_run)
+
+
+def copy_text_from(ctx: MobileUseContext, selector_request: SelectorRequest, dry_run: bool = False):
+    copy_text_from_body = selector_request.to_dict()
+    if not copy_text_from_body:
+        error = "Invalid copyTextFrom selector request, could not format yaml"
+        logger.error(error)
+        raise ControllerErrors(error)
+    flow_input = [{"copyTextFrom": copy_text_from_body}]
+    return run_flow(ctx, flow_input, dry_run=dry_run)
+
+
+def paste_text(ctx: MobileUseContext, dry_run: bool = False):
+    return run_flow(ctx, ["pasteText"], dry_run=dry_run)
+
+
+def erase_text(ctx: MobileUseContext, nb_chars: Optional[int] = None, dry_run: bool = False):
+    """
+    Removes characters from the currently selected textfield (if any)
+    Removes 50 characters if nb_chars is not specified.
+    """
+    if nb_chars is None:
+        return run_flow(ctx, ["eraseText"], dry_run=dry_run)
+    return run_flow(ctx, [{"eraseText": nb_chars}], dry_run=dry_run)
+
+
+##### App related commands #####
+
+
+def launch_app(ctx: MobileUseContext, package_name: str, dry_run: bool = False):
+    flow_input = [{"launchApp": package_name}]
+    return run_flow_with_wait_for_animation_to_end(ctx, flow_input, dry_run=dry_run)
+
+
+def stop_app(ctx: MobileUseContext, package_name: Optional[str] = None, dry_run: bool = False):
+    if package_name is None:
+        flow_input = ["stopApp"]
+    else:
+        flow_input = [{"stopApp": package_name}]
+    return run_flow_with_wait_for_animation_to_end(ctx, flow_input, dry_run=dry_run)
+
+
+def open_link(ctx: MobileUseContext, url: str, dry_run: bool = False):
+    flow_input = [{"openLink": url}]
+    return run_flow_with_wait_for_animation_to_end(ctx, flow_input, dry_run=dry_run)
+
+
+##### Key related commands #####
+
+
+def back(ctx: MobileUseContext, dry_run: bool = False):
+    flow_input = ["back"]
+    return run_flow_with_wait_for_animation_to_end(ctx, flow_input, dry_run=dry_run)
+
+
+class Key(Enum):
+    ENTER = "Enter"
+    HOME = "Home"
+    BACK = "Back"
+
+
+def press_key(ctx: MobileUseContext, key: Key, dry_run: bool = False):
+    flow_input = [{"pressKey": key.value}]
+    return run_flow_with_wait_for_animation_to_end(ctx, flow_input, dry_run=dry_run)
+
+
+#### Other commands ####
+
+
+class WaitTimeout(Enum):
+    SHORT = 500
+    MEDIUM = 1000
+    LONG = 5000
+
+
+def wait_for_animation_to_end(
+    ctx: MobileUseContext, timeout: Optional[WaitTimeout] = None, dry_run: bool = False
+):
+    if timeout is None:
+        return run_flow(ctx, ["waitForAnimationToEnd"], dry_run=dry_run)
+    return run_flow(ctx, [{"waitForAnimationToEnd": {"timeout": timeout.value}}], dry_run=dry_run)
+
+
+def run_flow_with_wait_for_animation_to_end(
+    ctx: MobileUseContext, base_flow: list, dry_run: bool = False
+):
+    base_flow.append({"waitForAnimationToEnd": {"timeout": WaitTimeout.MEDIUM.value}})
+    return run_flow(ctx, base_flow, dry_run=dry_run)
+
+
+if __name__ == "__main__":
+    # long press, erase
+    # input_text(text="test")
+    # erase_text()
+    ctx = MobileUseContext(
+        device=DeviceContext(
+            host_platform="LINUX",
+            mobile_platform=DevicePlatform.ANDROID,
+            device_id="emulator-5554",
+            device_width=1080,
+            device_height=1920,
+        ),
+        hw_bridge_client=DeviceHardwareClient("http://localhost:9999"),
+        screen_api_client=ScreenApiClient("http://localhost:9998"),
+    )
+    screen_data = get_screen_data(ctx.screen_api_client)
+    from minitap.mobile_use.graph.state import State
+    from minitap.mobile_use.tools.mobile.erase_text import get_erase_text_tool
+
+    dummy_state = State(
+        latest_ui_hierarchy=screen_data.elements,
+        messages=[],
+        initial_goal="",
+        subgoal_plan=[],
+        latest_screenshot_base64=screen_data.base64,
+        focused_app_info=None,
+        device_date="",
+        structured_decisions=None,
+        executor_retrigger=False,
+        executor_failed=False,
+        executor_messages=[],
+        cortex_last_thought="",
+        agents_thoughts=[],
+    )
+
+    # invoke erase_text tool
+    input_resource_id = "com.google.android.settings.intelligence:id/open_search_view_edit_text"
+    command_output: Command = get_erase_text_tool(ctx=ctx).invoke(
+        {
+            "tool_call_id": uuid.uuid4().hex,
+            "agent_thought": "",
+            "input_text_resource_id": input_resource_id,
+            "state": dummy_state,
+            "executor_metadata": None,
+        }
+    )
+    print(command_output)

minitap/mobile_use/controllers/platform_specific_commands_controller.py

@@ -0,0 +1,74 @@
+from datetime import date
+import json
+from typing import Optional
+
+from adbutils import AdbDevice
+from minitap.mobile_use.utils.logger import MobileUseLogger
+from minitap.mobile_use.utils.shell_utils import run_shell_command_on_host
+from minitap.mobile_use.context import MobileUseContext
+from minitap.mobile_use.context import DevicePlatform
+
+
+def get_adb_device(ctx: MobileUseContext) -> AdbDevice:
+    if ctx.device.mobile_platform != DevicePlatform.ANDROID:
+        raise ValueError("Device is not an Android device")
+    adb = ctx.get_adb_client()
+    device = adb.device(serial=ctx.device.device_id)
+    if not device:
+        raise ConnectionError(f"Device {ctx.device.device_id} not found.")
+    return device
+
+
+def get_first_device(
+    logger: Optional[MobileUseLogger] = None,
+) -> tuple[Optional[str], Optional[DevicePlatform]]:
+    """Gets the first available device."""
+    try:
+        android_output = run_shell_command_on_host("adb devices")
+        lines = android_output.strip().split("\n")
+        for line in lines:
+            if "device" in line and not line.startswith("List of devices"):
+                return line.split()[0], DevicePlatform.ANDROID
+    except RuntimeError as e:
+        if logger:
+            logger.error(f"ADB command failed: {e}")
+        return None, None
+
+    try:
+        ios_output = run_shell_command_on_host("xcrun simctl list devices booted -j")
+        data = json.loads(ios_output)
+        for runtime, devices in data.get("devices", {}).items():
+            if "iOS" not in runtime:
+                continue
+            for device in devices:
+                if device.get("state") == "Booted":
+                    return device["udid"], DevicePlatform.IOS
+    except RuntimeError as e:
+        if logger:
+            logger.error(f"xcrun command failed: {e}")
+
+    return None, None
+
+
+def get_focused_app_info(ctx: MobileUseContext) -> Optional[str]:
+    if ctx.device.mobile_platform == DevicePlatform.IOS:
+        return None
+    device = get_adb_device(ctx)
+    return str(device.shell("dumpsys window | grep -E 'mCurrentFocus|mFocusedApp'"))
+
+
+def get_device_date(ctx: MobileUseContext) -> str:
+    if ctx.device.mobile_platform == DevicePlatform.IOS:
+        return date.today().strftime("%a %b %d %H:%M:%S %Z %Y")
+    device = get_adb_device(ctx)
+    return str(device.shell("date"))
+
+
+def list_packages(ctx: MobileUseContext) -> str:
+    if ctx.device.mobile_platform == DevicePlatform.IOS:
+        cmd = ["xcrun", "simctl", "listapps", "booted", "|", "grep", "CFBundleIdentifier"]
+        return run_shell_command_on_host(" ".join(cmd))
+    else:
+        device = get_adb_device(ctx)
+        cmd = ["pm", "list", "packages", "-f"]
+        return str(device.shell(" ".join(cmd)))

minitap/mobile_use/graph/graph.py

@@ -0,0 +1,149 @@
+from typing import Literal
+
+from langchain_core.messages import (
+    AIMessage,
+)
+from langgraph.constants import END, START
+from langgraph.graph import StateGraph
+from langgraph.graph.state import CompiledStateGraph
+from langgraph.prebuilt import ToolNode
+from minitap.mobile_use.agents.contextor.contextor import ContextorNode
+from minitap.mobile_use.agents.cortex.cortex import CortexNode
+from minitap.mobile_use.agents.executor.executor import ExecutorNode
+from minitap.mobile_use.agents.executor.executor_context_cleaner import (
+    executor_context_cleaner_node,
+)
+from minitap.mobile_use.agents.orchestrator.orchestrator import OrchestratorNode
+from minitap.mobile_use.agents.planner.planner import PlannerNode
+from minitap.mobile_use.agents.planner.utils import (
+    all_completed,
+    get_current_subgoal,
+    one_of_them_is_failure,
+)
+from minitap.mobile_use.agents.summarizer.summarizer import SummarizerNode
+from minitap.mobile_use.context import MobileUseContext
+from minitap.mobile_use.graph.state import State
+from minitap.mobile_use.tools.index import EXECUTOR_WRAPPERS_TOOLS, get_tools_from_wrappers
+from minitap.mobile_use.utils.logger import get_logger
+
+logger = get_logger(__name__)
+
+
+def post_orchestrator_gate(
+    state: State,
+) -> Literal["continue", "replan", "end"]:
+    logger.info("Starting post_orchestrator_gate")
+    if one_of_them_is_failure(state.subgoal_plan):
+        logger.info("One of the subgoals is in failure state, asking to replan")
+        return "replan"
+
+    if all_completed(state.subgoal_plan):
+        logger.info("All subgoals are completed, ending the goal")
+        return "end"
+
+    if not get_current_subgoal(state.subgoal_plan):
+        logger.info("No subgoal running, ending the goal")
+        return "end"
+
+    logger.info("Goal is not achieved, continuing")
+    return "continue"
+
+
+def post_cortex_gate(
+    state: State,
+) -> Literal["continue", "end_subgoal"]:
+    logger.info("Starting post_cortex_gate")
+    if not state.structured_decisions:
+        return "end_subgoal"
+    return "continue"
+
+
+def post_executor_gate(
+    state: State,
+) -> Literal["invoke_tools", "skip"]:
+    logger.info("Starting post_executor_gate")
+    messages = state.messages
+    if not messages:
+        return "skip"
+    last_message = messages[-1]
+
+    if isinstance(last_message, AIMessage):
+        tool_calls = getattr(last_message, "tool_calls", None)
+        if tool_calls and len(tool_calls) > 0:
+            logger.info("🔨👁️  Found tool calls: " + str(tool_calls))
+            return "invoke_tools"
+        else:
+            logger.info("🔨❌ No tool calls found")
+    return "skip"
+
+
+def post_executor_tools_gate(
+    state: State,
+) -> Literal["continue", "failed", "done"]:
+    logger.info("Starting post_executor_tools_gate")
+    if state.executor_failed:
+        return "failed"
+    if state.executor_retrigger:
+        return "continue"
+    return "done"
+
+
+async def get_graph(ctx: MobileUseContext) -> CompiledStateGraph:
+    graph_builder = StateGraph(State)
+
+    ## Define nodes
+    graph_builder.add_node("planner", PlannerNode(ctx))
+    graph_builder.add_node("orchestrator", OrchestratorNode(ctx))
+
+    graph_builder.add_node("contextor", ContextorNode(ctx))
+
+    graph_builder.add_node("cortex", CortexNode(ctx))
+
+    graph_builder.add_node("executor", ExecutorNode(ctx))
+    executor_tool_node = ToolNode(
+        get_tools_from_wrappers(ctx=ctx, wrappers=EXECUTOR_WRAPPERS_TOOLS)
+    )
+    graph_builder.add_node("executor_tools", executor_tool_node)
+
+    graph_builder.add_node("executor_context_cleaner", executor_context_cleaner_node)
+    graph_builder.add_node("summarizer", SummarizerNode(ctx))
+
+    # Linking nodes
+    graph_builder.add_edge(START, "planner")
+    graph_builder.add_edge("planner", "orchestrator")
+    graph_builder.add_conditional_edges(
+        "orchestrator",
+        post_orchestrator_gate,
+        {
+            "continue": "contextor",
+            "replan": "planner",
+            "end": END,
+        },
+    )
+    graph_builder.add_edge("contextor", "cortex")
+    graph_builder.add_conditional_edges(
+        "cortex",
+        post_cortex_gate,
+        {
+            "continue": "executor",
+            "end_subgoal": "orchestrator",
+        },
+    )
+    graph_builder.add_conditional_edges(
+        "executor",
+        post_executor_gate,
+        {"invoke_tools": "executor_tools", "skip": "executor_context_cleaner"},
+    )
+    graph_builder.add_conditional_edges(
+        "executor_tools",
+        post_executor_tools_gate,
+        {
+            "continue": "executor",
+            "done": "executor_context_cleaner",
+            "failed": "executor_context_cleaner",
+        },
+    )
+    graph_builder.add_edge("executor_context_cleaner", "summarizer")
+    graph_builder.add_edge("summarizer", "contextor")
+
+    return graph_builder.compile()

minitap/mobile_use/graph/state.py

@@ -0,0 +1,73 @@
+from langchain_core.messages import AIMessage, AnyMessage
+from langgraph.graph import add_messages
+from langgraph.prebuilt.chat_agent_executor import AgentStatePydantic
+from typing_extensions import Annotated, Optional
+
+from minitap.mobile_use.agents.planner.types import Subgoal
+from minitap.mobile_use.utils.logger import get_logger
+from minitap.mobile_use.utils.recorder import record_interaction
+from minitap.mobile_use.context import MobileUseContext
+
+logger = get_logger(__name__)
+
+
+def take_last(a, b):
+    return b
+
+
+class State(AgentStatePydantic):
+    # planner related keys
+    initial_goal: Annotated[str, "Initial goal given by the user"]
+
+    # orchestrator related keys
+    subgoal_plan: Annotated[list[Subgoal], "The current plan, made of subgoals"]
+
+    # contextor related keys
+    latest_screenshot_base64: Annotated[Optional[str], "Latest screenshot of the device", take_last]
+    latest_ui_hierarchy: Annotated[
+        Optional[list[dict]], "Latest UI hierarchy of the device", take_last
+    ]
+    focused_app_info: Annotated[Optional[str], "Focused app info", take_last]
+    device_date: Annotated[Optional[str], "Date of the device", take_last]
+
+    # cortex related keys
+    structured_decisions: Annotated[
+        Optional[str],
+        "Structured decisions made by the cortex, for the executor to follow",
+        take_last,
+    ]
+
+    # executor related keys
+    executor_retrigger: Annotated[Optional[bool], "Whether the executor must be retriggered"]
+    executor_failed: Annotated[bool, "Whether a tool call made by the executor failed"]
+    executor_messages: Annotated[list[AnyMessage], "Sequential Executor messages", add_messages]
+    cortex_last_thought: Annotated[Optional[str], "Last thought of the cortex for the executor"]
+
+    # common keys
+    agents_thoughts: Annotated[
+        list[str],
+        "All thoughts and reasons that led to actions (why a tool was called, expected outcomes..)",
+    ]
+
+    def sanitize_update(self, ctx: MobileUseContext, update: dict):
+        """
+        Sanitizes the state update to ensure it is valid and apply side effect logic where required.
+        """
+        updated_agents_thoughts: Optional[str | list[str]] = update.get("agents_thoughts", None)
+        if updated_agents_thoughts is not None:
+            if isinstance(updated_agents_thoughts, str):
+                updated_agents_thoughts = [updated_agents_thoughts]
+            elif not isinstance(updated_agents_thoughts, list):
+                raise ValueError("agents_thoughts must be a str or list[str]")
+            update["agents_thoughts"] = _add_agent_thoughts(
+                ctx=ctx,
+                old=self.agents_thoughts,
+                new=updated_agents_thoughts,
+            )
+        return update
+
+
+def _add_agent_thoughts(ctx: MobileUseContext, old: list[str], new: list[str]) -> list[str]:
+    if ctx.execution_setup:
+        record_interaction(ctx, response=AIMessage(content=str(new)))
+    return old + new