minitap-mobile-use 3.3.0__py3-none-any.whl

minitap/mobile_use/tools/utils.py

@@ -0,0 +1,336 @@
+from __future__ import annotations
+
+from typing import Literal
+
+from pydantic import BaseModel, ConfigDict
+
+from minitap.mobile_use.context import MobileUseContext
+from minitap.mobile_use.controllers.types import (
+    CoordinatesSelectorRequest,
+    PercentagesSelectorRequest,
+)
+from minitap.mobile_use.controllers.unified_controller import UnifiedMobileController
+from minitap.mobile_use.graph.state import State
+from minitap.mobile_use.tools.types import Target
+from minitap.mobile_use.utils.logger import get_logger
+from minitap.mobile_use.utils.ui_hierarchy import (
+    ElementBounds,
+    Point,
+    find_element_by_resource_id,
+    get_bounds_for_element,
+    get_element_text,
+    is_element_focused,
+)
+
+logger = get_logger(__name__)
+
+
+def find_element_by_text(
+    ui_hierarchy: list[dict], text: str, index: int | None = None
+) -> dict | None:
+    """
+    Find a UI element by its text content (adapted to both flat and rich hierarchy)
+
+    This function performs a recursive, case-insensitive partial search.
+
+    Args:
+        ui_hierarchy: List of UI element dictionaries.
+        text: The text content to search for.
+
+    Returns:
+        The complete UI element dictionary if found, None otherwise.
+    """
+
+    def search_recursive(elements: list[dict]) -> dict | None:
+        for element in elements:
+            if isinstance(element, dict):
+                src = element.get("attributes", element)
+                element_text = src.get("text", "")
+                # Guard against non-string text values (e.g., dict)
+                if not isinstance(element_text, str):
+                    element_text = ""
+                if text and text.lower() == element_text.lower():
+                    idx = index or 0
+                    if idx == 0:
+                        return element
+                    idx -= 1
+                    continue
+                if (children := element.get("children", [])) and (
+                    found := search_recursive(children)
+                ):
+                    return found
+        return None
+
+    return search_recursive(ui_hierarchy)
+
+
+async def tap_bottom_right_of_element(bounds: ElementBounds, ctx: MobileUseContext):
+    bottom_right: Point = bounds.get_relative_point(x_percent=0.99, y_percent=0.99)
+    await tap(
+        ctx=ctx,
+        selector_request=SelectorRequestWithCoordinates(
+            coordinates=CoordinatesSelectorRequest(
+                x=bottom_right.x,
+                y=bottom_right.y,
+            ),
+        ),
+    )
+
+
+async def move_cursor_to_end_if_bounds(
+    ctx: MobileUseContext,
+    state: State,
+    target: Target,
+    elt: dict | None = None,
+) -> dict | None:
+    """
+    Best-effort move of the text cursor near the end of the input by tapping the
+    bottom-right area of the focused element (if bounds are available).
+    """
+    if target.resource_id:
+        if not elt:
+            elt = find_element_by_resource_id(
+                ui_hierarchy=state.latest_ui_hierarchy or [],
+                resource_id=target.resource_id,
+                index=target.resource_id_index,
+            )
+        if not elt:
+            return None
+
+        bounds = get_bounds_for_element(elt)
+        if not bounds:
+            return elt
+
+        logger.debug("Tapping near the end of the input to move the cursor")
+        await tap_bottom_right_of_element(bounds=bounds, ctx=ctx)
+        logger.debug(f"Tapped end of input {target.resource_id}")
+        return elt
+
+    if target.bounds:
+        await tap_bottom_right_of_element(target.bounds, ctx=ctx)
+        logger.debug("Tapped end of input by coordinates")
+        return elt
+
+    if target.text:
+        text_elt = find_element_by_text(
+            state.latest_ui_hierarchy or [], target.text, index=target.text_index
+        )
+        if text_elt:
+            bounds = get_bounds_for_element(text_elt)
+            if bounds:
+                await tap_bottom_right_of_element(bounds=bounds, ctx=ctx)
+                logger.debug(f"Tapped end of input that had text'{target.text}'")
+                return text_elt
+        return None
+
+    return None
+
+
+async def focus_element_if_needed(
+    ctx: MobileUseContext, target: Target
+) -> Literal["resource_id", "coordinates", "text"] | None:
+    """
+    Ensures the element is focused, with a sanity check to prevent trusting misleading IDs.
+    """
+    controller = UnifiedMobileController(ctx)
+    rich_hierarchy = await controller.get_ui_elements()
+    elt_from_id = None
+    if target.resource_id:
+        elt_from_id = find_element_by_resource_id(
+            ui_hierarchy=rich_hierarchy,
+            resource_id=target.resource_id,
+            index=target.resource_id_index,
+            is_rich_hierarchy=False,
+        )
+
+    if elt_from_id and target.text:
+        text_from_id_elt = get_element_text(elt_from_id)
+        if not text_from_id_elt or target.text.lower() != text_from_id_elt.lower():
+            logger.warning(
+                f"ID '{target.resource_id}' and text '{target.text}' seem to be on different "
+                "elements. Ignoring the resource_id and falling back to other locators."
+            )
+            elt_from_id = None
+
+    if elt_from_id:
+        if not is_element_focused(elt_from_id):
+            tap(
+                ctx=ctx,
+                selector_request=IdSelectorRequest(id=target.resource_id),  # type: ignore
+                index=target.resource_id_index,
+            )
+            logger.debug(f"Focused (tap) on resource_id={target.resource_id}")
+            rich_hierarchy = await controller.get_ui_elements()
+            elt_from_id = find_element_by_resource_id(
+                ui_hierarchy=rich_hierarchy,
+                resource_id=target.resource_id,  # type: ignore
+                index=target.resource_id_index,
+                is_rich_hierarchy=False,
+            )
+        if elt_from_id and is_element_focused(elt_from_id):
+            logger.debug(f"Text input is focused: {target.resource_id}")
+            return "resource_id"
+        logger.warning(f"Failed to focus using resource_id='{target.resource_id}'. Fallback...")
+
+    if target.bounds:
+        relative_point = target.bounds.get_center()
+        await tap(
+            ctx=ctx,
+            selector_request=SelectorRequestWithCoordinates(
+                coordinates=CoordinatesSelectorRequest(x=relative_point.x, y=relative_point.y)
+            ),
+        )
+        logger.debug(f"Tapped on coordinates ({relative_point.x}, {relative_point.y}) to focus.")
+        return "coordinates"
+
+    if target.text:
+        text_elt = find_element_by_text(rich_hierarchy, target.text, index=target.text_index)
+        if text_elt:
+            bounds = get_bounds_for_element(text_elt)
+            if bounds:
+                relative_point = bounds.get_center()
+                await tap(
+                    ctx=ctx,
+                    selector_request=SelectorRequestWithCoordinates(
+                        coordinates=CoordinatesSelectorRequest(
+                            x=relative_point.x, y=relative_point.y
+                        )
+                    ),
+                )
+                logger.debug(f"Tapped on text element '{target.text}' to focus.")
+                return "text"
+
+    logger.error(
+        "Failed to focus element. No valid locator (resource_id, coordinates, or text) succeeded."
+    )
+    return None
+
+
+def validate_coordinates_bounds(
+    target: Target, screen_width: int, screen_height: int
+) -> str | None:
+    """
+    Validate that coordinates are within screen bounds.
+    Returns error message if invalid, None if valid.
+    """
+    if not target.bounds:
+        return None
+
+    center = target.bounds.get_center()
+    errors = []
+
+    if center.x < 0 or center.x >= screen_width:
+        errors.append(f"x={center.x} is outside screen width (0-{screen_width})")
+    if center.y < 0 or center.y >= screen_height:
+        errors.append(f"y={center.y} is outside screen height (0-{screen_height})")
+
+    return "; ".join(errors) if errors else None
+
+
+def has_valid_selectors(target: Target) -> bool:
+    """Check if target has at least one valid selector."""
+    has_coordinates = target.bounds is not None
+    has_resource_id = target.resource_id is not None and target.resource_id != ""
+    has_text = target.text is not None and target.text != ""
+    return has_coordinates or has_resource_id or has_text
+
+
+class IdSelectorRequest(BaseModel):
+    model_config = ConfigDict(extra="forbid")
+    id: str
+
+    def to_dict(self) -> dict[str, str | int]:
+        return {"id": self.id}
+
+
+class TextSelectorRequest(BaseModel):
+    model_config = ConfigDict(extra="forbid")
+    text: str
+
+    def to_dict(self) -> dict[str, str | int]:
+        return {"text": self.text}
+
+
+class SelectorRequestWithCoordinates(BaseModel):
+    model_config = ConfigDict(extra="forbid")
+    coordinates: CoordinatesSelectorRequest
+
+    def to_dict(self) -> dict[str, str | int]:
+        return {"point": self.coordinates.to_str()}
+
+
+class SelectorRequestWithPercentages(BaseModel):
+    model_config = ConfigDict(extra="forbid")
+    percentages: PercentagesSelectorRequest
+
+    def to_dict(self) -> dict[str, str | int]:
+        return {"point": self.percentages.to_str()}
+
+
+class IdWithTextSelectorRequest(BaseModel):
+    model_config = ConfigDict(extra="forbid")
+    id: str
+    text: str
+
+    def to_dict(self) -> dict[str, str | int]:
+        return {"id": self.id, "text": self.text}
+
+
+SelectorRequest = (
+    IdSelectorRequest
+    | SelectorRequestWithCoordinates
+    | SelectorRequestWithPercentages
+    | TextSelectorRequest
+    | IdWithTextSelectorRequest
+)
+
+
+def _extract_resource_id_and_text_from_selector(
+    selector: SelectorRequest,
+) -> tuple[str | None, str | None]:
+    """Extract resource_id and text from a selector."""
+    resource_id = None
+    text = None
+
+    if isinstance(selector, IdSelectorRequest):
+        resource_id = selector.id
+    elif isinstance(selector, TextSelectorRequest):
+        text = selector.text
+    elif isinstance(selector, IdWithTextSelectorRequest):
+        resource_id = selector.id
+        text = selector.text
+
+    return resource_id, text
+
+
+async def tap(
+    ctx: MobileUseContext,
+    selector_request: SelectorRequest,
+    index: int | None = None,
+):
+    """
+    Tap on a selector.
+    Index is optional and is used when you have multiple views matching the same selector.
+    ui_hierarchy is optional and used for ADB taps to find elements.
+    """
+    controller = UnifiedMobileController(ctx)
+    if isinstance(selector_request, SelectorRequestWithCoordinates):
+        result = await controller.tap_at(
+            x=selector_request.coordinates.x, y=selector_request.coordinates.y
+        )
+        return result.error if result.error else None
+    if isinstance(selector_request, SelectorRequestWithPercentages):
+        coords = selector_request.percentages.to_coords(
+            width=ctx.device.device_width,
+            height=ctx.device.device_height,
+        )
+        return await controller.tap_at(coords.x, coords.y)
+
+    # For other selectors, we need the UI hierarchy
+    resource_id, text = _extract_resource_id_and_text_from_selector(selector_request)
+
+    return await controller.tap_element(
+        resource_id=resource_id,
+        text=text,
+        index=index if index is not None else 0,
+    )

minitap/mobile_use/utils/app_launch_utils.py

@@ -0,0 +1,173 @@
+"""
+Utilities for handling app locking and initial app launch logic.
+"""
+
+import asyncio
+
+from minitap.mobile_use.context import AppLaunchResult, MobileUseContext
+from minitap.mobile_use.controllers.platform_specific_commands_controller import (
+    get_current_foreground_package,
+)
+from minitap.mobile_use.controllers.unified_controller import UnifiedMobileController
+from minitap.mobile_use.utils.logger import get_logger
+
+logger = get_logger(__name__)
+
+
+async def _poll_for_app_ready(
+    ctx: MobileUseContext,
+    app_package: str,
+    max_poll_seconds: int = 15,
+    poll_interval: float = 1.0,
+) -> tuple[bool, str | None]:
+    """
+    Poll for app to be ready after launch.
+
+    Treats mCurrentFocus=null as a loading state and keeps polling.
+    Only fails if we get a different (non-null) package or timeout.
+
+    Args:
+        ctx: Mobile use context
+        app_package: Expected package name
+        max_poll_seconds: Maximum time to poll (default: 15s)
+        poll_interval: Time between polls (default: 1s)
+
+    Returns:
+        Tuple of (success: bool, error_message: str | None)
+    """
+    polls = int(max_poll_seconds / poll_interval)
+
+    for i in range(polls):
+        current_package = get_current_foreground_package(ctx)
+
+        if current_package == app_package:
+            logger.success(f"App {app_package} is ready (took ~{i * poll_interval:.1f}s)")
+            return True, None
+
+        if current_package is None:
+            logger.debug(f"Poll {i + 1}/{polls}: App loading (mCurrentFocus=null)...")
+        else:
+            error_msg = (
+                f"Wrong app in foreground: expected '{app_package}', got '{current_package}'"
+            )
+            logger.warning(error_msg)
+            return False, error_msg
+
+        if i < polls - 1:
+            await asyncio.sleep(poll_interval)
+
+    current_package = get_current_foreground_package(ctx)
+    error_msg = (
+        f"Timeout waiting for {app_package} to load after {max_poll_seconds}s. "
+        f"Current foreground: {current_package}"
+    )
+    logger.error(error_msg)
+    return False, error_msg
+
+
+async def launch_app_with_retries(
+    ctx: MobileUseContext,
+    app_package: str,
+    max_retries: int = 3,
+    max_poll_seconds: int = 15,
+) -> tuple[bool, str | None]:
+    """
+    Launch an app with retry logic and smart polling.
+
+    Args:
+        ctx: Mobile use context
+        app_package: Package name (Android) or bundle ID (iOS) to launch
+        max_retries: Maximum number of launch attempts (default: 3)
+        max_poll_seconds: Maximum time to wait for app to load per attempt (default: 15s)
+
+    Returns:
+        Tuple of (success: bool, error_message: str | None)
+    """
+    for attempt in range(1, max_retries + 1):
+        logger.info(f"Launch attempt {attempt}/{max_retries} for app {app_package}")
+
+        controller = UnifiedMobileController(ctx)
+        launch_success = await controller.launch_app(app_package)
+        if not launch_success:
+            error_msg = f"Failed to execute launch command for {app_package}"
+            logger.error(error_msg)
+            if attempt == max_retries:
+                return False, error_msg
+            await asyncio.sleep(2)
+            continue
+
+        await asyncio.sleep(1)
+
+        success, error_msg = await _poll_for_app_ready(ctx, app_package, max_poll_seconds)
+
+        if success:
+            return True, None
+
+        if attempt < max_retries:
+            logger.warning(f"Attempt {attempt} failed: {error_msg}. Retrying...")
+            await asyncio.sleep(1)
+
+    error_msg = f"Failed to launch {app_package} after {max_retries} attempts"
+    logger.error(error_msg)
+    return False, error_msg
+
+
+async def _handle_initial_app_launch(
+    ctx: MobileUseContext,
+    locked_app_package: str,
+) -> AppLaunchResult:
+    """
+    Handle initial app launch verification and launching if needed.
+
+    If locked_app_package is set:
+    1. Check if the app is already in the foreground
+    2. If not, attempt to launch it (with retries)
+    3. Return status with success/error information
+
+    Args:
+        ctx: Mobile use context
+        locked_app_package: Package name (Android) or bundle ID (iOS) to lock to, or None
+
+    Returns:
+        AppLaunchResult with launch status and error information
+    """
+    if not locked_app_package:
+        error_msg = f"Invalid locked_app_package: '{locked_app_package}'"
+        logger.error(error_msg)
+        return AppLaunchResult(
+            locked_app_package=locked_app_package,
+            locked_app_initial_launch_success=False,
+            locked_app_initial_launch_error=error_msg,
+        )
+
+    logger.info(f"Starting initial app launch for package: {locked_app_package}")
+
+    try:
+        current_package = get_current_foreground_package(ctx)
+        logger.info(f"Current foreground app: {current_package}")
+
+        if current_package == locked_app_package:
+            logger.info(f"App {locked_app_package} is already in foreground")
+            return AppLaunchResult(
+                locked_app_package=locked_app_package,
+                locked_app_initial_launch_success=True,
+                locked_app_initial_launch_error=None,
+            )
+
+        logger.info(f"App {locked_app_package} not in foreground, attempting to launch")
+        success, error_msg = await launch_app_with_retries(ctx, locked_app_package)
+
+        return AppLaunchResult(
+            locked_app_package=locked_app_package,
+            locked_app_initial_launch_success=success,
+            locked_app_initial_launch_error=error_msg,
+        )
+
+    except Exception as e:
+        error_msg = f"Exception during initial app launch: {str(e)}"
+        logger.error(error_msg)
+        return AppLaunchResult(
+            locked_app_package=locked_app_package,
+            locked_app_initial_launch_success=False,
+            locked_app_initial_launch_error=error_msg,
+        )

minitap/mobile_use/utils/cli_helpers.py

@@ -0,0 +1,37 @@
+import sys
+
+from adbutils import AdbClient
+from rich.console import Console
+
+from minitap.mobile_use.clients.ios_client import format_device_info, get_all_ios_devices_detailed
+
+
+def display_device_status(console: Console, adb_client: AdbClient | None = None):
+    """Checks for connected devices and displays the status."""
+    console.print("\n[bold]📱 Device Status[/bold]")
+    devices = None
+    if adb_client is not None:
+        devices = adb_client.device_list()
+    if devices:
+        console.print("✅ [bold green]Android device(s) connected:[/bold green]")
+        for device in devices:
+            console.print(f"  - {device.serial}")
+    else:
+        console.print("❌ [bold red]No Android device found.[/bold red]")
+        command = "emulator -avd <avd_name>"
+        if sys.platform not in ["win32", "darwin"]:
+            command = f"./{command}"
+            console.print(
+                f"You can start an emulator using a command like: [bold]'{command}'[/bold]"
+            )
+
+    ios_devices = get_all_ios_devices_detailed()
+    if ios_devices:
+        console.print("✅ [bold green]iOS device(s) connected:[/bold green]")
+        for device in ios_devices:
+            console.print(f"  - [green]{format_device_info(device)}[/green]")
+    else:
+        console.print("❌ [bold red]No iOS device found.[/bold red]")
+        console.print(
+            "[iOS] Please make sure your emulator is running or a device is connected via USB."
+        )

minitap/mobile_use/utils/cli_selection.py

@@ -0,0 +1,143 @@
+import sys
+
+import inquirer
+from rich.console import Console
+from rich.prompt import Prompt
+
+
+def select_provider_and_model(
+    console: Console,
+    available_providers: list[str],
+    available_models: dict,
+    default_provider: str,
+    default_model: str,
+    provider: str | None = None,
+    model: str | None = None,
+) -> tuple[str, str]:
+    """
+    Interactive selection of LLM provider and model with arrow-key dropdowns when available.
+
+    Args:
+        console: Rich console for output
+        available_providers: List of available provider names
+        available_models: Dict mapping providers to their available models
+        default_provider: Default provider to use
+        default_model: Default model to use
+        provider: Pre-selected provider (optional)
+        model: Pre-selected model (optional)
+
+    Returns:
+        Tuple of (selected_provider, selected_model)
+    """
+    final_provider = provider
+    final_model = model
+
+    # Interactive provider selection
+    if not final_provider:
+        console.print("\n🤖 [bold cyan]LLM Provider Selection[/bold cyan]")
+        final_provider = _select_from_list(
+            console=console,
+            item_type="provider",
+            choices=available_providers,
+            default=default_provider,
+            message="Select LLM provider",
+        )
+
+    # Interactive model selection
+    if not final_model:
+        console.print(f"\n🎯 [bold green]Model Selection for {final_provider}[/bold green]")
+        available_model_list = (
+            available_models[final_provider]
+            if final_provider
+            else available_models[default_provider]
+        )
+
+        default_model_for_provider = (
+            default_model if default_model in available_model_list else available_model_list[0]
+        )
+
+        final_model = _select_from_list(
+            console=console,
+            item_type="model",
+            choices=available_model_list,
+            default=default_model_for_provider,
+            message=f"Select model for {final_provider}",
+        )
+
+    return final_provider, final_model
+
+
+def _select_from_list(
+    console: Console,
+    item_type: str,
+    choices: list[str],
+    default: str,
+    message: str,
+) -> str:
+    """
+    Select an item from a list using arrow keys when available, fallback to numbered selection.
+
+    Args:
+        console: Rich console for output
+        item_type: Type of item being selected (for error messages)
+        choices: List of choices to select from
+        default: Default choice
+        message: Message to display in dropdown
+
+    Returns:
+        Selected choice
+    """
+    # Try arrow-key dropdown if TTY is available, fallback to numbered selection
+    if sys.stdin.isatty():
+        try:
+            questions = [
+                inquirer.List(
+                    "selection",
+                    message=f"{message} (use arrow keys)",
+                    choices=choices,
+                    default=default,
+                ),
+            ]
+            answers = inquirer.prompt(questions)
+            return answers["selection"] if answers else default
+        except Exception:
+            # Fallback to numbered selection
+            return _numbered_selection(console, item_type, choices, default)
+    else:
+        return _numbered_selection(console, item_type, choices, default)
+
+
+def _numbered_selection(console: Console, item_type: str, choices: list[str], default: str) -> str:
+    """Fallback numbered selection when arrow keys aren't available."""
+    choices_text = "\n".join([f"  {i + 1}. {choice}" for i, choice in enumerate(choices)])
+    console.print(f"Available {item_type}s:\n{choices_text}")
+
+    default_idx = choices.index(default) + 1 if default in choices else 1
+
+    while True:
+        choice = Prompt.ask(
+            f"Select {item_type} (1-{len(choices)}) or press Enter for default",
+            default=str(default_idx),
+        )
+        try:
+            choice_idx = int(choice) - 1
+            if 0 <= choice_idx < len(choices):
+                return choices[choice_idx]
+            else:
+                console.print("[red]Invalid choice. Please try again.[/red]")
+        except ValueError:
+            console.print("[red]Please enter a number.[/red]")
+
+
+def display_llm_config(console: Console, provider: str, model: str) -> None:
+    """Display the selected LLM configuration with colors."""
+    from rich.text import Text
+
+    config_text = Text()
+    config_text.append("🤖 LLM Configuration: ", style="bold white")
+    config_text.append("Provider: ", style="white")
+    config_text.append(f"{provider}", style="bold cyan")
+    config_text.append(" | Model: ", style="white")
+    config_text.append(f"{model}", style="bold green")
+
+    console.print(config_text)