minitap-mobile-use 0.0.1.dev0__py3-none-any.whl

minitap/mobile_use/tools/mobile/stop_app.py

@@ -0,0 +1,58 @@
+from typing import Optional
+
+from langchain_core.messages import ToolMessage
+from langchain_core.tools import tool
+from langchain_core.tools.base import InjectedToolCallId
+from langgraph.prebuilt import InjectedState
+from langgraph.types import Command
+from minitap.mobile_use.context import MobileUseContext
+from minitap.mobile_use.controllers.mobile_command_controller import stop_app as stop_app_controller
+from minitap.mobile_use.graph.state import State
+from minitap.mobile_use.tools.tool_wrapper import ExecutorMetadata, ToolWrapper
+from typing_extensions import Annotated
+
+
+def get_stop_app_tool(ctx: MobileUseContext):
+    @tool
+    def stop_app(
+        tool_call_id: Annotated[str, InjectedToolCallId],
+        state: Annotated[State, InjectedState],
+        agent_thought: str,
+        executor_metadata: Optional[ExecutorMetadata],
+        package_name: Optional[str] = None,
+    ):
+        """
+        Stops current application if it is running.
+        You can also specify the package name of the app to be stopped.
+        """
+        output = stop_app_controller(ctx=ctx, package_name=package_name)
+        has_failed = output is not None
+        tool_message = ToolMessage(
+            tool_call_id=tool_call_id,
+            content=stop_app_wrapper.on_failure_fn(package_name)
+            if has_failed
+            else stop_app_wrapper.on_success_fn(package_name),
+            additional_kwargs={"error": output} if has_failed else {},
+        )
+        return Command(
+            update=stop_app_wrapper.handle_executor_state_fields(
+                ctx=ctx,
+                state=state,
+                executor_metadata=executor_metadata,
+                tool_message=tool_message,
+                is_failure=has_failed,
+                updates={
+                    "agents_thoughts": [agent_thought],
+                    "messages": [tool_message],
+                },
+            ),
+        )
+
+    return stop_app
+
+
+stop_app_wrapper = ToolWrapper(
+    tool_fn_getter=get_stop_app_tool,
+    on_success_fn=lambda package_name: f"App {package_name or 'current'} stopped successfully.",
+    on_failure_fn=lambda package_name: f"Failed to stop app {package_name or 'current'}.",
+)

minitap/mobile_use/tools/mobile/swipe.py

@@ -0,0 +1,56 @@
+from typing import Optional
+
+from langchain_core.messages import ToolMessage
+from langchain_core.tools import tool
+from langchain_core.tools.base import InjectedToolCallId
+from langgraph.prebuilt import InjectedState
+from langgraph.types import Command
+from minitap.mobile_use.context import MobileUseContext
+from minitap.mobile_use.controllers.mobile_command_controller import SwipeRequest
+from minitap.mobile_use.controllers.mobile_command_controller import swipe as swipe_controller
+from minitap.mobile_use.graph.state import State
+from minitap.mobile_use.tools.tool_wrapper import ExecutorMetadata, ToolWrapper
+from typing_extensions import Annotated
+
+
+def get_swipe_tool(ctx: MobileUseContext):
+    @tool
+    def swipe(
+        tool_call_id: Annotated[str, InjectedToolCallId],
+        state: Annotated[State, InjectedState],
+        agent_thought: str,
+        executor_metadata: Optional[ExecutorMetadata],
+        swipe_request: SwipeRequest,
+    ):
+        """
+        Swipes on the screen.
+        """
+        output = swipe_controller(ctx=ctx, swipe_request=swipe_request)
+        has_failed = output is not None
+        tool_message = ToolMessage(
+            tool_call_id=tool_call_id,
+            content=swipe_wrapper.on_failure_fn() if has_failed else swipe_wrapper.on_success_fn(),
+            additional_kwargs={"error": output} if has_failed else {},
+        )
+        return Command(
+            update=swipe_wrapper.handle_executor_state_fields(
+                ctx=ctx,
+                state=state,
+                executor_metadata=executor_metadata,
+                tool_message=tool_message,
+                is_failure=has_failed,
+                updates={
+                    "agents_thoughts": [agent_thought],
+                    "messages": [tool_message],
+                },
+            ),
+        )
+
+    return swipe
+
+
+swipe_wrapper = ToolWrapper(
+    tool_fn_getter=get_swipe_tool,
+    on_success_fn=lambda: "Swipe is successful.",
+    on_failure_fn=lambda: "Failed to swipe.",
+)

minitap/mobile_use/tools/mobile/take_screenshot.py

@@ -0,0 +1,70 @@
+from typing import Optional
+
+from langchain_core.messages import ToolMessage
+from langchain_core.tools import tool
+from langchain_core.tools.base import InjectedToolCallId
+from langgraph.prebuilt import InjectedState
+from langgraph.types import Command
+from minitap.mobile_use.context import MobileUseContext
+from minitap.mobile_use.controllers.mobile_command_controller import (
+    take_screenshot as take_screenshot_controller,
+)
+from minitap.mobile_use.graph.state import State
+from minitap.mobile_use.tools.tool_wrapper import ExecutorMetadata, ToolWrapper
+from minitap.mobile_use.utils.media import compress_base64_jpeg
+from typing_extensions import Annotated
+
+
+def get_take_screenshot_tool(ctx: MobileUseContext):
+    @tool
+    def take_screenshot(
+        tool_call_id: Annotated[str, InjectedToolCallId],
+        state: Annotated[State, InjectedState],
+        agent_thought: str,
+        executor_metadata: Optional[ExecutorMetadata],
+    ):
+        """
+        Take a screenshot of the device.
+        """
+        compressed_image_base64 = None
+        has_failed = False
+
+        try:
+            output = take_screenshot_controller(ctx=ctx)
+            compressed_image_base64 = compress_base64_jpeg(output)
+        except Exception as e:
+            output = str(e)
+            has_failed = True
+
+        tool_message = ToolMessage(
+            tool_call_id=tool_call_id,
+            content=take_screenshot_wrapper.on_failure_fn()
+            if has_failed
+            else take_screenshot_wrapper.on_success_fn(),
+            additional_kwargs={"error": output} if has_failed else {},
+        )
+        updates = {
+            "agents_thoughts": [agent_thought],
+            "messages": [tool_message],
+        }
+        if compressed_image_base64:
+            updates["latest_screenshot_base64"] = compressed_image_base64
+        return Command(
+            update=take_screenshot_wrapper.handle_executor_state_fields(
+                ctx=ctx,
+                state=state,
+                executor_metadata=executor_metadata,
+                tool_message=tool_message,
+                is_failure=has_failed,
+                updates=updates,
+            ),
+        )
+
+    return take_screenshot
+
+
+take_screenshot_wrapper = ToolWrapper(
+    tool_fn_getter=get_take_screenshot_tool,
+    on_success_fn=lambda: "Screenshot taken successfully.",
+    on_failure_fn=lambda: "Failed to take screenshot.",
+)

minitap/mobile_use/tools/mobile/tap.py

@@ -0,0 +1,66 @@
+from typing import Optional
+
+from langchain_core.messages import ToolMessage
+from langchain_core.tools import tool
+from langchain_core.tools.base import InjectedToolCallId
+from langgraph.prebuilt import InjectedState
+from langgraph.types import Command
+from minitap.mobile_use.context import MobileUseContext
+from minitap.mobile_use.controllers.mobile_command_controller import SelectorRequest
+from minitap.mobile_use.controllers.mobile_command_controller import tap as tap_controller
+from minitap.mobile_use.graph.state import State
+from minitap.mobile_use.tools.tool_wrapper import ExecutorMetadata, ToolWrapper
+from typing_extensions import Annotated
+
+
+def get_tap_tool(ctx: MobileUseContext):
+    @tool
+    def tap(
+        tool_call_id: Annotated[str, InjectedToolCallId],
+        state: Annotated[State, InjectedState],
+        agent_thought: str,
+        executor_metadata: Optional[ExecutorMetadata],
+        selector_request: SelectorRequest,
+        index: Optional[int] = None,
+    ):
+        """
+        Taps on a selector.
+        Index is optional and is used when you have multiple views matching the same selector.
+        """
+        output = tap_controller(ctx=ctx, selector_request=selector_request, index=index)
+        has_failed = output is not None
+        tool_message = ToolMessage(
+            tool_call_id=tool_call_id,
+            content=tap_wrapper.on_failure_fn(selector_request, index)
+            if has_failed
+            else tap_wrapper.on_success_fn(selector_request, index),
+            additional_kwargs={"error": output} if has_failed else {},
+        )
+        return Command(
+            update=tap_wrapper.handle_executor_state_fields(
+                ctx=ctx,
+                state=state,
+                executor_metadata=executor_metadata,
+                tool_message=tool_message,
+                is_failure=has_failed,
+                updates={
+                    "agents_thoughts": [agent_thought],
+                    "messages": [tool_message],
+                },
+            ),
+        )
+
+    return tap
+
+
+tap_wrapper = ToolWrapper(
+    tool_fn_getter=get_tap_tool,
+    on_success_fn=(
+        lambda selector_request,
+        index: f"Tap on {selector_request} {'at index {index}' if index else ''} is successful."
+    ),
+    on_failure_fn=(
+        lambda selector_request,
+        index: f"Failed to tap on {selector_request} {'at index {index}' if index else ''}."
+    ),
+)

minitap/mobile_use/tools/mobile/wait_for_animation_to_end.py

@@ -0,0 +1,68 @@
+from typing import Optional
+
+from langchain_core.messages import ToolMessage
+from langchain_core.tools import tool
+from langchain_core.tools.base import InjectedToolCallId
+from langgraph.prebuilt import InjectedState
+from langgraph.types import Command
+from minitap.mobile_use.context import MobileUseContext
+from minitap.mobile_use.controllers.mobile_command_controller import WaitTimeout
+from minitap.mobile_use.controllers.mobile_command_controller import (
+    wait_for_animation_to_end as wait_for_animation_to_end_controller,
+)
+from minitap.mobile_use.graph.state import State
+from minitap.mobile_use.tools.tool_wrapper import ExecutorMetadata, ToolWrapper
+from typing_extensions import Annotated
+
+
+def get_wait_for_animation_to_end_tool(ctx: MobileUseContext):
+    @tool
+    def wait_for_animation_to_end(
+        tool_call_id: Annotated[str, InjectedToolCallId],
+        state: Annotated[State, InjectedState],
+        agent_thought: str,
+        executor_metadata: Optional[ExecutorMetadata],
+        timeout: Optional[WaitTimeout],
+    ):
+        """
+        Waits for ongoing animations or videos to finish before continuing.
+
+        If a `timeout` (in milliseconds) is set, the command proceeds after the timeout even if
+        the animation hasn't ended.
+        The flow continues immediately once the animation is detected as complete.
+
+        Example:
+            - waitForAnimationToEnd
+            - waitForAnimationToEnd: { timeout: 5000 }
+        """
+        output = wait_for_animation_to_end_controller(ctx=ctx, timeout=timeout)
+        has_failed = output is not None
+        tool_message = ToolMessage(
+            tool_call_id=tool_call_id,
+            content=wait_for_animation_to_end_wrapper.on_failure_fn()
+            if has_failed
+            else wait_for_animation_to_end_wrapper.on_success_fn(timeout),
+            additional_kwargs={"error": output} if has_failed else {},
+        )
+        return Command(
+            update=wait_for_animation_to_end_wrapper.handle_executor_state_fields(
+                ctx=ctx,
+                state=state,
+                executor_metadata=executor_metadata,
+                tool_message=tool_message,
+                is_failure=has_failed,
+                updates={
+                    "agents_thoughts": [agent_thought],
+                    "messages": [tool_message],
+                },
+            ),
+        )
+
+    return wait_for_animation_to_end
+
+
+wait_for_animation_to_end_wrapper = ToolWrapper(
+    tool_fn_getter=get_wait_for_animation_to_end_tool,
+    on_success_fn=lambda: "Animation ended successfully.",
+    on_failure_fn=lambda: "Failed to end animation.",
+)

minitap/mobile_use/tools/tool_wrapper.py

@@ -0,0 +1,33 @@
+from typing import Callable, Optional
+
+from langchain_core.messages import ToolMessage
+from langchain_core.tools import BaseTool
+from pydantic import BaseModel
+from minitap.mobile_use.context import MobileUseContext
+from minitap.mobile_use.graph.state import State
+
+
+class ExecutorMetadata(BaseModel):
+    retrigger: bool
+
+
+class ToolWrapper(BaseModel):
+    tool_fn_getter: Callable[[MobileUseContext], BaseTool]
+    on_success_fn: Callable[..., str]
+    on_failure_fn: Callable[..., str]
+
+    def handle_executor_state_fields(
+        self,
+        ctx: MobileUseContext,
+        state: State,
+        executor_metadata: Optional[ExecutorMetadata],
+        is_failure: bool,
+        tool_message: ToolMessage,
+        updates: dict,
+    ):
+        if executor_metadata is None:
+            return state.sanitize_update(ctx=ctx, update=updates)
+        updates["executor_retrigger"] = executor_metadata.retrigger
+        updates["executor_messages"] = [tool_message]
+        updates["executor_failed"] = is_failure
+        return state.sanitize_update(ctx=ctx, update=updates)

minitap/mobile_use/utils/cli_helpers.py

@@ -0,0 +1,40 @@
+import sys
+
+from minitap.mobile_use.clients.ios_client import get_ios_devices
+from adbutils import AdbClient
+from rich.console import Console
+from typing import Optional
+
+
+def display_device_status(console: Console, adb_client: Optional[AdbClient] = None):
+    """Checks for connected devices and displays the status."""
+    console.print("\n[bold]📱 Device Status[/bold]")
+    devices = None
+    if adb_client is not None:
+        devices = adb_client.device_list()
+    if devices:
+        console.print("✅ [bold green]Android device(s) connected:[/bold green]")
+        for device in devices:
+            console.print(f"  - {device.serial}")
+    else:
+        console.print("❌ [bold red]No Android device found.[/bold red]")
+        console.print("Please make sure your emulator is running or a device is connected via USB.")
+        command = "emulator -avd <avd_name>"
+        if sys.platform not in ["win32", "darwin"]:
+            command = f"./{command}"
+        console.print(f"You can start an emulator using a command like: [bold]'{command}'[/bold]")
+        console.print("[italic]iOS detection coming soon...[/italic]")
+
+    xcrun_available, ios_devices, error_message = get_ios_devices()
+    if xcrun_available:
+        if ios_devices:
+            console.print("✅ [bold green]iOS device(s) connected:[/bold green]")
+            for device in ios_devices:
+                console.print(f"  - {device}")
+        else:
+            console.print("❌ [bold red]No iOS device found.[/bold red]")
+            console.print(
+                "[iOS] Please make sure your emulator is running or a device is connected via USB."
+            )
+        return
+    console.print(f"❌ [bold red]iOS check failed:[/bold red] {error_message}")

minitap/mobile_use/utils/cli_selection.py

@@ -0,0 +1,144 @@
+import sys
+from typing import List, Optional
+
+import inquirer
+from rich.console import Console
+from rich.prompt import Prompt
+
+
+def select_provider_and_model(
+    console: Console,
+    available_providers: List[str],
+    available_models: dict,
+    default_provider: str,
+    default_model: str,
+    provider: Optional[str] = None,
+    model: Optional[str] = None,
+) -> tuple[str, str]:
+    """
+    Interactive selection of LLM provider and model with arrow-key dropdowns when available.
+
+    Args:
+        console: Rich console for output
+        available_providers: List of available provider names
+        available_models: Dict mapping providers to their available models
+        default_provider: Default provider to use
+        default_model: Default model to use
+        provider: Pre-selected provider (optional)
+        model: Pre-selected model (optional)
+
+    Returns:
+        Tuple of (selected_provider, selected_model)
+    """
+    final_provider = provider
+    final_model = model
+
+    # Interactive provider selection
+    if not final_provider:
+        console.print("\n🤖 [bold cyan]LLM Provider Selection[/bold cyan]")
+        final_provider = _select_from_list(
+            console=console,
+            item_type="provider",
+            choices=available_providers,
+            default=default_provider,
+            message="Select LLM provider",
+        )
+
+    # Interactive model selection
+    if not final_model:
+        console.print(f"\n🎯 [bold green]Model Selection for {final_provider}[/bold green]")
+        available_model_list = (
+            available_models[final_provider]
+            if final_provider
+            else available_models[default_provider]
+        )
+
+        default_model_for_provider = (
+            default_model if default_model in available_model_list else available_model_list[0]
+        )
+
+        final_model = _select_from_list(
+            console=console,
+            item_type="model",
+            choices=available_model_list,
+            default=default_model_for_provider,
+            message=f"Select model for {final_provider}",
+        )
+
+    return final_provider, final_model
+
+
+def _select_from_list(
+    console: Console,
+    item_type: str,
+    choices: List[str],
+    default: str,
+    message: str,
+) -> str:
+    """
+    Select an item from a list using arrow keys when available, fallback to numbered selection.
+
+    Args:
+        console: Rich console for output
+        item_type: Type of item being selected (for error messages)
+        choices: List of choices to select from
+        default: Default choice
+        message: Message to display in dropdown
+
+    Returns:
+        Selected choice
+    """
+    # Try arrow-key dropdown if TTY is available, fallback to numbered selection
+    if sys.stdin.isatty():
+        try:
+            questions = [
+                inquirer.List(
+                    "selection",
+                    message=f"{message} (use arrow keys)",
+                    choices=choices,
+                    default=default,
+                ),
+            ]
+            answers = inquirer.prompt(questions)
+            return answers["selection"] if answers else default
+        except Exception:
+            # Fallback to numbered selection
+            return _numbered_selection(console, item_type, choices, default)
+    else:
+        return _numbered_selection(console, item_type, choices, default)
+
+
+def _numbered_selection(console: Console, item_type: str, choices: List[str], default: str) -> str:
+    """Fallback numbered selection when arrow keys aren't available."""
+    choices_text = "\n".join([f"  {i + 1}. {choice}" for i, choice in enumerate(choices)])
+    console.print(f"Available {item_type}s:\n{choices_text}")
+
+    default_idx = choices.index(default) + 1 if default in choices else 1
+
+    while True:
+        choice = Prompt.ask(
+            f"Select {item_type} (1-{len(choices)}) or press Enter for default",
+            default=str(default_idx),
+        )
+        try:
+            choice_idx = int(choice) - 1
+            if 0 <= choice_idx < len(choices):
+                return choices[choice_idx]
+            else:
+                console.print("[red]Invalid choice. Please try again.[/red]")
+        except ValueError:
+            console.print("[red]Please enter a number.[/red]")
+
+
+def display_llm_config(console: Console, provider: str, model: str) -> None:
+    """Display the selected LLM configuration with colors."""
+    from rich.text import Text
+
+    config_text = Text()
+    config_text.append("🤖 LLM Configuration: ", style="bold white")
+    config_text.append("Provider: ", style="white")
+    config_text.append(f"{provider}", style="bold cyan")
+    config_text.append(" | Model: ", style="white")
+    config_text.append(f"{model}", style="bold green")
+
+    console.print(config_text)

minitap/mobile_use/utils/conversations.py

@@ -0,0 +1,31 @@
+from typing import TypeGuard
+
+from langchain_core.messages import AIMessage, BaseMessage, HumanMessage, ToolMessage
+
+
+def is_ai_message(message: BaseMessage) -> TypeGuard[AIMessage]:
+    return isinstance(message, AIMessage)
+
+
+def is_human_message(message: BaseMessage) -> TypeGuard[HumanMessage]:
+    return isinstance(message, HumanMessage)
+
+
+def is_tool_message(message: BaseMessage) -> TypeGuard[ToolMessage]:
+    return isinstance(message, ToolMessage)
+
+
+def is_tool_for_name(tool_message: ToolMessage, name: str) -> bool:
+    return tool_message.name == name
+
+
+def get_screenshot_message_for_llm(screenshot_base64: str):
+    prefix = "" if screenshot_base64.startswith("data:image") else "data:image/jpeg;base64,"
+    return HumanMessage(
+        content=[
+            {
+                "type": "image_url",
+                "image_url": {"url": f"{prefix}{screenshot_base64}"},
+            }
+        ]
+    )

minitap/mobile_use/utils/decorators.py

@@ -0,0 +1,123 @@
+import asyncio
+from functools import wraps
+from typing import Any, Awaitable, Callable, Optional, TypeVar, cast, overload
+
+R = TypeVar("R")
+
+
+def wrap_with_callbacks_sync(
+    fn: Callable[..., R],
+    *,
+    before: Optional[Callable[..., None]] = None,
+    on_success: Optional[Callable[[R], None]] = None,
+    on_failure: Optional[Callable[[Exception], None]] = None,
+    suppress_exceptions: bool = False,
+) -> Callable[..., R]:
+    @wraps(fn)
+    def wrapper(*args: Any, **kwargs: Any) -> R:
+        if before:
+            before()
+        try:
+            result = fn(*args, **kwargs)
+            if on_success:
+                on_success(result)
+            return result
+        except Exception as e:
+            if on_failure:
+                on_failure(e)
+            if suppress_exceptions:
+                return None  # type: ignore
+            raise
+
+    return wrapper
+
+
+def wrap_with_callbacks_async(
+    fn: Callable[..., Awaitable[R]],
+    *,
+    before: Optional[Callable[..., None]] = None,
+    on_success: Optional[Callable[[R], None]] = None,
+    on_failure: Optional[Callable[[Exception], None]] = None,
+    suppress_exceptions: bool = False,
+) -> Callable[..., Awaitable[R]]:
+    @wraps(fn)
+    async def wrapper(*args: Any, **kwargs: Any) -> R:
+        if before:
+            before()
+        try:
+            result = await fn(*args, **kwargs)
+            if on_success:
+                on_success(result)
+            return result
+        except Exception as e:
+            if on_failure:
+                on_failure(e)
+            if suppress_exceptions:
+                return None  # type: ignore
+            raise
+
+    return wrapper
+
+
+@overload
+def wrap_with_callbacks(
+    fn: Callable[..., Awaitable[R]],
+    *,
+    before: Optional[Callable[[], None]] = ...,
+    on_success: Optional[Callable[[R], None]] = ...,
+    on_failure: Optional[Callable[[Exception], None]] = ...,
+    suppress_exceptions: bool = ...,
+) -> Callable[..., Awaitable[R]]: ...
+
+
+@overload
+def wrap_with_callbacks(
+    *,
+    before: Optional[Callable[..., None]] = ...,
+    on_success: Optional[Callable[[Any], None]] = ...,
+    on_failure: Optional[Callable[[Exception], None]] = ...,
+    suppress_exceptions: bool = ...,
+) -> Callable[[Callable[..., R]], Callable[..., R]]: ...
+
+
+@overload
+def wrap_with_callbacks(
+    fn: Callable[..., R],
+    *,
+    before: Optional[Callable[[], None]] = ...,
+    on_success: Optional[Callable[[R], None]] = ...,
+    on_failure: Optional[Callable[[Exception], None]] = ...,
+    suppress_exceptions: bool = ...,
+) -> Callable[..., R]: ...
+
+
+def wrap_with_callbacks(
+    fn: Optional[Callable[..., Any]] = None,
+    *,
+    before: Optional[Callable[[], None]] = None,
+    on_success: Optional[Callable[[Any], None]] = None,
+    on_failure: Optional[Callable[[Exception], None]] = None,
+    suppress_exceptions: bool = False,
+) -> Any:
+    def decorator(func: Callable[..., Any]) -> Any:
+        if asyncio.iscoroutinefunction(func):
+            return wrap_with_callbacks_async(
+                cast(Callable[..., Awaitable[Any]], func),
+                before=before,
+                on_success=on_success,
+                on_failure=on_failure,
+                suppress_exceptions=suppress_exceptions,
+            )
+        else:
+            return wrap_with_callbacks_sync(
+                cast(Callable[..., Any], func),
+                before=before,
+                on_success=on_success,
+                on_failure=on_failure,
+                suppress_exceptions=suppress_exceptions,
+            )
+
+    if fn is None:
+        return decorator
+    else:
+        return decorator(fn)