minitap-mobile-use 3.3.0__py3-none-any.whl

minitap/mobile_use/controllers/platform_specific_commands_controller.py

@@ -0,0 +1,199 @@
+import asyncio
+from concurrent.futures.thread import ThreadPoolExecutor
+from datetime import date
+from shutil import which
+
+from adbutils import AdbDevice
+
+from minitap.mobile_use.clients.ios_client import (
+    DeviceType,
+    get_all_ios_devices_detailed,
+    get_device_type,
+)
+from minitap.mobile_use.context import DevicePlatform, MobileUseContext
+from minitap.mobile_use.utils.logger import MobileUseLogger, get_logger
+from minitap.mobile_use.utils.shell_utils import run_shell_command_on_host
+
+logger = get_logger(__name__)
+
+
+def get_adb_device(ctx: MobileUseContext) -> AdbDevice:
+    if ctx.device.mobile_platform != DevicePlatform.ANDROID:
+        raise ValueError("Device is not an Android device")
+    adb = ctx.get_adb_client()
+    device = adb.device(serial=ctx.device.device_id)
+    if not device:
+        raise ConnectionError(f"Device {ctx.device.device_id} not found.")
+    return device
+
+
+def get_first_device(
+    logger: MobileUseLogger | None = None,
+    prefer_physical: bool = True,
+) -> tuple[str | None, DevicePlatform | None, DeviceType | None]:
+    """Gets the first available device.
+
+    Args:
+        logger: Optional logger for error messages
+        prefer_physical: If True, prefer physical iOS devices over simulators
+
+    Returns:
+        Tuple of (device_id, platform, device_type) or (None, None, None) if no device found.
+        device_type is only set for iOS devices (SIMULATOR or PHYSICAL).
+    """
+    # Check for Android devices first
+    if which("adb"):
+        try:
+            android_output = run_shell_command_on_host("adb devices")
+            lines = android_output.strip().split("\n")
+            for line in lines:
+                if "device" in line and not line.startswith("List of devices"):
+                    return line.split()[0], DevicePlatform.ANDROID, None
+        except RuntimeError as e:
+            if logger:
+                logger.error(f"ADB command failed: {e}")
+
+    # Check for iOS devices (both simulators and physical)
+    ios_devices = get_all_ios_devices_detailed()
+    if ios_devices:
+        if prefer_physical:
+            # Sort to prefer physical devices
+            ios_devices.sort(key=lambda d: d["type"] != DeviceType.PHYSICAL)
+
+        device = ios_devices[0]
+        if logger:
+            logger.info(
+                f"Selected iOS device: {device['name']} ({device['type'].value}) - {device['udid']}"
+            )
+        return device["udid"], DevicePlatform.IOS, device["type"]
+
+    return None, None, None
+
+
+def get_device_date(ctx: MobileUseContext) -> str:
+    if ctx.device.mobile_platform == DevicePlatform.IOS:
+        return date.today().strftime("%a %b %d %H:%M:%S %Z %Y")
+    device = get_adb_device(ctx)
+    return str(device.shell("date"))
+
+
+def list_packages(ctx: MobileUseContext) -> str:
+    if ctx.device.mobile_platform == DevicePlatform.IOS:
+        udid = ctx.device.device_id
+        device_type = get_device_type(udid)
+
+        if device_type == DeviceType.SIMULATOR:
+            cmd = ["xcrun", "simctl", "listapps", udid, "|", "grep", "CFBundleIdentifier"]
+            return run_shell_command_on_host(" ".join(cmd))
+
+        # Physical device: try ios-deploy first (common with React Native/Cordova)
+        if which("ios-deploy"):
+            cmd = ["ios-deploy", "--id", udid, "--list_bundle_id"]
+            try:
+                output = run_shell_command_on_host(" ".join(cmd))
+                packages = [line.strip() for line in output.strip().split("\n") if line.strip()]
+                return "\n".join(sorted(packages))
+            except Exception as e:
+                logger.debug(f"ios-deploy failed: {e}")
+
+        # Fallback: ideviceinstaller (libimobiledevice)
+        if which("ideviceinstaller"):
+            cmd = ["ideviceinstaller", "-l", "-u", udid]
+            try:
+                output = run_shell_command_on_host(" ".join(cmd))
+                # Parse output: "CFBundleIdentifier, CFBundleVersion, CFBundleDisplayName"
+                lines = output.strip().split("\n")
+                packages = []
+                for line in lines:
+                    if ", " in line:
+                        bundle_id = line.split(", ")[0].strip()
+                        if bundle_id and not bundle_id.startswith("CFBundle"):
+                            packages.append(bundle_id)
+                return "\n".join(sorted(packages))
+            except Exception as e:
+                logger.debug(f"ideviceinstaller failed: {e}")
+
+        logger.warning(
+            "Cannot list apps on physical iOS device. Install ios-deploy "
+            "(npm install -g ios-deploy) or ideviceinstaller (brew install ideviceinstaller)"
+        )
+        return ""
+    else:
+        device = get_adb_device(ctx)
+        # Get full package list with paths
+        cmd = ["pm", "list", "packages", "-f"]
+        raw_output = str(device.shell(" ".join(cmd)))
+
+        # Extract only package names (remove paths and "package:" prefix)
+        # Format: "package:/path/to/app.apk=com.example.app" -> "com.example.app"
+        lines = raw_output.strip().split("\n")
+        packages = []
+        for line in lines:
+            if "=" in line:
+                package_name = line.split("=")[-1].strip()
+                packages.append(package_name)
+
+        return "\n".join(sorted(packages))
+
+
+def get_current_foreground_package(ctx: MobileUseContext) -> str | None:
+    """
+    Get the package name of the currently focused/foreground app.
+
+    Returns only the clean package/bundle name (e.g., 'com.whatsapp'),
+    without any metadata or window information.
+
+    Returns:
+        The package/bundle name, or None if unable to determine
+    """
+    try:
+        if ctx.device.mobile_platform == DevicePlatform.IOS:
+            return _get_ios_foreground_package(ctx)
+
+        device = get_adb_device(ctx)
+        output = str(device.shell("dumpsys window | grep mCurrentFocus"))
+
+        if "mCurrentFocus=" not in output:
+            return None
+
+        segment = output.split("mCurrentFocus=")[-1]
+
+        if "/" in segment:
+            tokens = segment.split()
+            for token in tokens:
+                if "." in token and not token.startswith("Window"):
+                    package = token.split("/")[0]
+                    package = package.rstrip("}")
+                    if package and "." in package:
+                        return package
+
+        return None
+
+    except Exception as e:
+        logger.debug(f"Failed to get current foreground package: {e}")
+        return None
+
+
+def _get_ios_foreground_package(ctx: MobileUseContext) -> str | None:
+    """Get foreground package for iOS devices (simulator or physical)."""
+
+    ios_client = ctx.ios_client
+
+    if not ios_client:
+        return None
+
+    try:
+        # Handle both running and non-running event loops
+        try:
+            asyncio.get_running_loop()
+            # Already in async context - run in separate thread
+            with ThreadPoolExecutor() as pool:
+                app_info = pool.submit(asyncio.run, ios_client.app_current()).result(timeout=10)
+        except RuntimeError:
+            # No running loop - use asyncio.run()
+            app_info = asyncio.run(ios_client.app_current())
+        if app_info and app_info.bundle_id:
+            return app_info.bundle_id
+    except Exception as e:
+        logger.debug(f"Failed to get foreground app: {e}")
+    return None

minitap/mobile_use/controllers/types.py

@@ -0,0 +1,106 @@
+from pydantic import BaseModel, ConfigDict, Field
+
+
+class TapOutput(BaseModel):
+    """Output from tap operations."""
+
+    error: str | None = Field(default=None, description="Error message if tap failed")
+
+
+class Bounds(BaseModel):
+    """Represents the bounds of a UI element."""
+
+    x1: int
+    y1: int
+    x2: int
+    y2: int
+
+    def get_center(self) -> "CoordinatesSelectorRequest":
+        """Get the center point of the bounds."""
+        return CoordinatesSelectorRequest(
+            x=(self.x1 + self.x2) // 2,
+            y=(self.y1 + self.y2) // 2,
+        )
+
+
+class CoordinatesSelectorRequest(BaseModel):
+    model_config = ConfigDict(extra="forbid")
+    x: int
+    y: int
+
+    def to_str(self):
+        return f"{self.x}, {self.y}"
+
+
+class PercentagesSelectorRequest(BaseModel):
+    model_config = ConfigDict(extra="forbid")
+    """
+    0%,0%        # top-left corner
+    100%,100%    # bottom-right corner
+    50%,50%      # center
+    """
+
+    x_percent: int = Field(ge=0, le=100, description="X percentage (0-100)")
+    y_percent: int = Field(ge=0, le=100, description="Y percentage (0-100)")
+
+    def to_str(self):
+        return f"{self.x_percent}%, {self.y_percent}%"
+
+    def to_coords(self, width: int, height: int) -> CoordinatesSelectorRequest:
+        """Convert percentages to pixel coordinates."""
+        x = min(max(int(width * self.x_percent / 100), 0), max(0, width - 1))
+        y = min(max(int(height * self.y_percent / 100), 0), max(0, height - 1))
+        return CoordinatesSelectorRequest(x=x, y=y)
+
+
+class SwipeStartEndCoordinatesRequest(BaseModel):
+    model_config = ConfigDict(extra="forbid")
+    start: CoordinatesSelectorRequest
+    end: CoordinatesSelectorRequest
+
+    def to_dict(self):
+        return {"start": self.start.to_str(), "end": self.end.to_str()}
+
+
+class SwipeStartEndPercentagesRequest(BaseModel):
+    model_config = ConfigDict(extra="forbid")
+    start: PercentagesSelectorRequest
+    end: PercentagesSelectorRequest
+
+    def to_dict(self):
+        return {"start": self.start.to_str(), "end": self.end.to_str()}
+
+    def to_coords(self, width: int, height: int) -> SwipeStartEndCoordinatesRequest:
+        """Convert percentage-based swipe to coordinate-based swipe."""
+        return SwipeStartEndCoordinatesRequest(
+            start=self.start.to_coords(width, height),
+            end=self.end.to_coords(width, height),
+        )
+
+
+class SwipeRequest(BaseModel):
+    """
+    Swipe from start to end position using coordinates or percentages.
+    """
+
+    model_config = ConfigDict(extra="forbid")
+    swipe_mode: SwipeStartEndCoordinatesRequest | SwipeStartEndPercentagesRequest = Field(
+        description="Start and end positions. Use EITHER (x, y) OR (x_percent, y_percent)."
+    )
+    duration: int | None = Field(
+        default=None,
+        description="Swipe duration in ms. If not provided, tool functions default to 400ms.",
+        ge=1,
+        le=10000,
+    )
+
+    def to_dict(self):
+        res = {}
+        if isinstance(
+            self.swipe_mode,
+            SwipeStartEndCoordinatesRequest | SwipeStartEndPercentagesRequest,
+        ):
+            res |= self.swipe_mode.to_dict()
+        if self.duration:
+            res |= {"duration": self.duration}
+        return res

minitap/mobile_use/controllers/unified_controller.py

@@ -0,0 +1,193 @@
+from minitap.mobile_use.context import MobileUseContext
+from minitap.mobile_use.controllers.controller_factory import get_controller
+from minitap.mobile_use.controllers.device_controller import MobileDeviceController
+from minitap.mobile_use.controllers.types import (
+    CoordinatesSelectorRequest,
+    PercentagesSelectorRequest,
+    SwipeRequest,
+    SwipeStartEndCoordinatesRequest,
+    SwipeStartEndPercentagesRequest,
+    TapOutput,
+)
+from minitap.mobile_use.utils.logger import get_logger
+
+logger = get_logger(__name__)
+
+
+class UnifiedMobileController:
+    def __init__(self, ctx: MobileUseContext):
+        self.ctx = ctx
+        self._controller: MobileDeviceController = get_controller(ctx)
+
+    @property
+    def controller(self) -> MobileDeviceController:
+        return self._controller
+
+    async def tap_at(
+        self,
+        x: int,
+        y: int,
+        long_press: bool = False,
+        long_press_duration: int = 1000,
+    ) -> TapOutput:
+        coords = CoordinatesSelectorRequest(x=x, y=y)
+        return await self._controller.tap(coords, long_press, long_press_duration)
+
+    async def tap_percentage(
+        self,
+        x_percent: int,
+        y_percent: int,
+        long_press: bool = False,
+        long_press_duration: int = 1000,
+    ) -> TapOutput:
+        """Tap at percentage-based coordinates (0 to 100)."""
+        coords = PercentagesSelectorRequest(x_percent=x_percent, y_percent=y_percent).to_coords(
+            width=self.ctx.device.device_width,
+            height=self.ctx.device.device_height,
+        )
+        return await self._controller.tap(coords, long_press, long_press_duration)
+
+    async def tap_element(
+        self,
+        resource_id: str | None = None,
+        text: str | None = None,
+        index: int = 0,
+        long_press: bool = False,
+        long_press_duration: int = 1000,
+    ) -> TapOutput:
+        """
+        Tap on a UI element by finding it in the hierarchy.
+
+        Args:
+            resource_id: Android resource ID or iOS element type
+            text: Element text/label/value to match
+            index: Which match to tap if multiple elements match
+            long_press: Whether to perform long press
+            long_press_duration: Duration of long press in milliseconds
+
+        Returns:
+            TapOutput with error field set on failure
+        """
+        # Get UI hierarchy
+        ui_hierarchy = await self._controller.get_ui_hierarchy()
+
+        # Find element
+        element, bounds, error = self._controller.find_element(
+            ui_hierarchy=ui_hierarchy,
+            resource_id=resource_id,
+            text=text,
+            index=index,
+        )
+
+        if error or not bounds:
+            return TapOutput(error=error or "Could not extract bounds for element")
+
+        # Tap at element center
+        center = bounds.get_center()
+        return await self._controller.tap(center, long_press, long_press_duration)
+
+    async def swipe_coords(
+        self,
+        start_x: int,
+        start_y: int,
+        end_x: int,
+        end_y: int,
+        duration: int = 400,
+    ) -> str | None:
+        """Swipe between two coordinate points."""
+        start = CoordinatesSelectorRequest(x=start_x, y=start_y)
+        end = CoordinatesSelectorRequest(x=end_x, y=end_y)
+        return await self._controller.swipe(start, end, duration)
+
+    async def swipe_percentage(
+        self,
+        start_x_percent: int,
+        start_y_percent: int,
+        end_x_percent: int,
+        end_y_percent: int,
+        duration: int = 400,
+    ) -> str | None:
+        """Swipe using percentage-based coordinates (0 to 100)."""
+        start = PercentagesSelectorRequest(
+            x_percent=start_x_percent, y_percent=start_y_percent
+        ).to_coords(
+            width=self.ctx.device.device_width,
+            height=self.ctx.device.device_height,
+        )
+        end = PercentagesSelectorRequest(
+            x_percent=end_x_percent, y_percent=end_y_percent
+        ).to_coords(
+            width=self.ctx.device.device_width,
+            height=self.ctx.device.device_height,
+        )
+        return await self._controller.swipe(start, end, duration)
+
+    async def swipe_request(self, request: SwipeRequest) -> str | None:
+        mode = request.swipe_mode
+
+        if isinstance(mode, SwipeStartEndCoordinatesRequest):
+            return await self._controller.swipe(
+                start=mode.start,
+                end=mode.end,
+                duration=request.duration or 400,
+            )
+        elif isinstance(mode, SwipeStartEndPercentagesRequest):
+            coords = mode.to_coords(
+                width=self.ctx.device.device_width,
+                height=self.ctx.device.device_height,
+            )
+            return await self._controller.swipe(
+                start=coords.start,
+                end=coords.end,
+                duration=request.duration or 400,
+            )
+        else:
+            return "Unsupported swipe mode"
+
+    async def type_text(self, text: str) -> bool:
+        return await self._controller.input_text(text)
+
+    async def take_screenshot(self) -> str:
+        return await self._controller.screenshot()
+
+    async def launch_app(self, package_or_bundle_id: str) -> bool:
+        return await self._controller.launch_app(package_or_bundle_id)
+
+    async def terminate_app(self, package_or_bundle_id: str | None) -> bool:
+        return await self._controller.terminate_app(package_or_bundle_id)
+
+    async def open_url(self, url: str) -> bool:
+        return await self._controller.open_url(url)
+
+    async def go_back(self) -> bool:
+        return await self._controller.press_back()
+
+    async def go_home(self) -> bool:
+        return await self._controller.press_home()
+
+    async def press_enter(self) -> bool:
+        return await self._controller.press_enter()
+
+    async def erase_text(self, nb_chars: int | None = None) -> bool:
+        return await self._controller.erase_text(nb_chars)
+
+    async def get_ui_elements(self) -> list[dict]:
+        return await self._controller.get_ui_hierarchy()
+
+    async def find_element(
+        self,
+        resource_id: str | None = None,
+        text: str | None = None,
+        index: int = 0,
+    ) -> tuple[dict | None, str | None]:
+        ui_hierarchy = await self._controller.get_ui_hierarchy()
+        element, bounds, error = self._controller.find_element(
+            ui_hierarchy=ui_hierarchy,
+            resource_id=resource_id,
+            text=text,
+            index=index,
+        )
+        return element, error
+
+    async def cleanup(self) -> None:
+        await self._controller.cleanup()

minitap/mobile_use/graph/graph.py

@@ -0,0 +1,160 @@
+from collections.abc import Sequence
+from typing import Literal
+
+from langchain_core.messages import AIMessage
+from langgraph.constants import END, START
+from langgraph.graph import StateGraph
+from langgraph.graph.state import CompiledStateGraph
+
+from minitap.mobile_use.agents.contextor.contextor import ContextorNode
+from minitap.mobile_use.agents.cortex.cortex import CortexNode
+from minitap.mobile_use.agents.executor.executor import ExecutorNode
+from minitap.mobile_use.agents.executor.tool_node import ExecutorToolNode
+from minitap.mobile_use.agents.orchestrator.orchestrator import OrchestratorNode
+from minitap.mobile_use.agents.planner.planner import PlannerNode
+from minitap.mobile_use.agents.planner.utils import (
+    all_completed,
+    get_current_subgoal,
+    one_of_them_is_failure,
+)
+from minitap.mobile_use.agents.summarizer.summarizer import SummarizerNode
+from minitap.mobile_use.constants import EXECUTOR_MESSAGES_KEY
+from minitap.mobile_use.context import MobileUseContext
+from minitap.mobile_use.graph.state import State
+from minitap.mobile_use.tools.index import (
+    EXECUTOR_WRAPPERS_TOOLS,
+    VIDEO_RECORDING_WRAPPERS,
+    get_tools_from_wrappers,
+)
+from minitap.mobile_use.utils.logger import get_logger
+
+logger = get_logger(__name__)
+
+
+def convergence_node(state: State):
+    """Convergence point for parallel execution paths."""
+    return {}
+
+
+def convergence_gate(
+    state: State,
+) -> Literal["continue", "replan", "end"]:
+    """Check if all subgoals are completed at convergence point."""
+    logger.info("Starting convergence_gate")
+
+    if one_of_them_is_failure(state.subgoal_plan):
+        logger.info("One of the subgoals is in failure state, asking to replan")
+        return "replan"
+
+    if all_completed(state.subgoal_plan):
+        logger.info("All subgoals are completed, ending the goal")
+        return "end"
+
+    if not get_current_subgoal(state.subgoal_plan):
+        logger.info("No subgoal running, ending the goal")
+        return "end"
+
+    return "continue"
+
+
+def post_cortex_gate(
+    state: State,
+) -> Sequence[str]:
+    logger.info("Starting post_cortex_gate")
+    node_sequence = []
+
+    if len(state.complete_subgoals_by_ids) > 0 or not state.structured_decisions:
+        # If subgoals need to be marked as complete, add the path to the orchestrator.
+        # The 'or not state.structured_decisions' ensures we don't get stuck if Cortex does nothing.
+        node_sequence.append("review_subgoals")
+
+    if state.structured_decisions:
+        node_sequence.append("execute_decisions")
+
+    return node_sequence
+
+
+def post_executor_gate(
+    state: State,
+) -> Literal["invoke_tools", "skip"]:
+    logger.info("Starting post_executor_gate")
+    messages = state.executor_messages
+    if not messages:
+        return "skip"
+    last_message = messages[-1]
+
+    if isinstance(last_message, AIMessage):
+        tool_calls = getattr(last_message, "tool_calls", None)
+        if tool_calls and len(tool_calls) > 0:
+            logger.info("[executor] Executing " + str(len(tool_calls)) + " tool calls:")
+            for tool_call in tool_calls:
+                logger.info("-------------")
+                logger.info("[executor] - " + str(tool_call) + "\n")
+            logger.info("-------------")
+            return "invoke_tools"
+        else:
+            logger.info("[executor] ❌ No tool calls found")
+    return "skip"
+
+
+async def get_graph(ctx: MobileUseContext) -> CompiledStateGraph:
+    graph_builder = StateGraph(State)
+
+    ## Define nodes
+    graph_builder.add_node("planner", PlannerNode(ctx))
+    graph_builder.add_node("orchestrator", OrchestratorNode(ctx))
+
+    graph_builder.add_node("contextor", ContextorNode(ctx))
+
+    graph_builder.add_node("cortex", CortexNode(ctx))
+
+    graph_builder.add_node("executor", ExecutorNode(ctx))
+
+    executor_wrappers = list(EXECUTOR_WRAPPERS_TOOLS)
+    if ctx.video_recording_enabled:
+        executor_wrappers.extend(VIDEO_RECORDING_WRAPPERS)
+
+    executor_tool_node = ExecutorToolNode(
+        tools=get_tools_from_wrappers(ctx=ctx, wrappers=executor_wrappers),
+        messages_key=EXECUTOR_MESSAGES_KEY,
+        trace_id=ctx.trace_id,
+    )
+    graph_builder.add_node("executor_tools", executor_tool_node)
+
+    graph_builder.add_node("summarizer", SummarizerNode(ctx))
+
+    graph_builder.add_node(node="convergence", action=convergence_node, defer=True)
+
+    ## Linking nodes
+    graph_builder.add_edge(START, "planner")
+    graph_builder.add_edge("planner", "orchestrator")
+    graph_builder.add_edge("orchestrator", "convergence")
+    graph_builder.add_edge("contextor", "cortex")
+    graph_builder.add_conditional_edges(
+        "cortex",
+        post_cortex_gate,
+        {
+            "review_subgoals": "orchestrator",
+            "execute_decisions": "executor",
+        },
+    )
+    graph_builder.add_conditional_edges(
+        "executor",
+        post_executor_gate,
+        {"invoke_tools": "executor_tools", "skip": "summarizer"},
+    )
+    graph_builder.add_edge("executor_tools", "summarizer")
+
+    graph_builder.add_edge("summarizer", "convergence")
+
+    graph_builder.add_conditional_edges(
+        source="convergence",
+        path=convergence_gate,
+        path_map={
+            "continue": "contextor",
+            "replan": "planner",
+            "end": END,
+        },
+    )
+
+    return graph_builder.compile()