minitap-mobile-use 2.5.3__py3-none-any.whl → 2.7.0__py3-none-any.whl

minitap/mobile_use/controllers/types.py

@@ -0,0 +1,95 @@
+from pydantic import BaseModel, ConfigDict, Field
+
+
+class TapOutput(BaseModel):
+    """Output from tap operations."""
+
+    error: str | None = Field(default=None, description="Error message if tap failed")
+
+
+class Bounds(BaseModel):
+    """Represents the bounds of a UI element."""
+
+    x1: int
+    y1: int
+    x2: int
+    y2: int
+
+    def get_center(self) -> "CoordinatesSelectorRequest":
+        """Get the center point of the bounds."""
+        return CoordinatesSelectorRequest(
+            x=(self.x1 + self.x2) // 2,
+            y=(self.y1 + self.y2) // 2,
+        )
+
+
+class CoordinatesSelectorRequest(BaseModel):
+    model_config = ConfigDict(extra="forbid")
+    x: int
+    y: int
+
+    def to_str(self):
+        return f"{self.x}, {self.y}"
+
+
+class PercentagesSelectorRequest(BaseModel):
+    model_config = ConfigDict(extra="forbid")
+    """
+    0%,0%        # top-left corner
+    100%,100%    # bottom-right corner
+    50%,50%      # center
+    """
+
+    x_percent: int = Field(ge=0, le=100, description="X percentage (0-100)")
+    y_percent: int = Field(ge=0, le=100, description="Y percentage (0-100)")
+
+    def to_str(self):
+        return f"{self.x_percent}%, {self.y_percent}%"
+
+    def to_coords(self, width: int, height: int) -> CoordinatesSelectorRequest:
+        """Convert percentages to pixel coordinates."""
+        x = min(max(int(width * self.x_percent / 100), 0), max(0, width - 1))
+        y = min(max(int(height * self.y_percent / 100), 0), max(0, height - 1))
+        return CoordinatesSelectorRequest(x=x, y=y)
+
+
+class SwipeStartEndCoordinatesRequest(BaseModel):
+    model_config = ConfigDict(extra="forbid")
+    start: CoordinatesSelectorRequest
+    end: CoordinatesSelectorRequest
+
+    def to_dict(self):
+        return {"start": self.start.to_str(), "end": self.end.to_str()}
+
+
+class SwipeStartEndPercentagesRequest(BaseModel):
+    model_config = ConfigDict(extra="forbid")
+    start: PercentagesSelectorRequest
+    end: PercentagesSelectorRequest
+
+    def to_dict(self):
+        return {"start": self.start.to_str(), "end": self.end.to_str()}
+
+    def to_coords(self, width: int, height: int) -> SwipeStartEndCoordinatesRequest:
+        """Convert percentage-based swipe to coordinate-based swipe."""
+        return SwipeStartEndCoordinatesRequest(
+            start=self.start.to_coords(width, height),
+            end=self.end.to_coords(width, height),
+        )
+
+
+class SwipeRequest(BaseModel):
+    model_config = ConfigDict(extra="forbid")
+    swipe_mode: SwipeStartEndCoordinatesRequest | SwipeStartEndPercentagesRequest
+    duration: int | None = None  # in ms, default is 400ms
+
+    def to_dict(self):
+        res = {}
+        if isinstance(
+            self.swipe_mode,
+            SwipeStartEndCoordinatesRequest | SwipeStartEndPercentagesRequest,
+        ):
+            res |= self.swipe_mode.to_dict()
+        if self.duration:
+            res |= {"duration": self.duration}
+        return res

minitap/mobile_use/graph/graph.py

@@ -1,8 +1,7 @@
+from collections.abc import Sequence
 from typing import Literal
 
-from langchain_core.messages import (
-    AIMessage,
-)
+from langchain_core.messages import AIMessage
 from langgraph.constants import END, START
 from langgraph.graph import StateGraph
 from langgraph.graph.state import CompiledStateGraph
@@ -18,6 +17,7 @@ from minitap.mobile_use.agents.planner.utils import (
     get_current_subgoal,
     one_of_them_is_failure,
 )
+from minitap.mobile_use.agents.screen_analyzer.screen_analyzer import ScreenAnalyzerNode
 from minitap.mobile_use.agents.summarizer.summarizer import SummarizerNode
 from minitap.mobile_use.constants import EXECUTOR_MESSAGES_KEY
 from minitap.mobile_use.context import MobileUseContext
@@ -28,6 +28,22 @@ from minitap.mobile_use.utils.logger import get_logger
 logger = get_logger(__name__)
 
 
+def convergence_node(state: State):
+    """Convergence point for parallel execution paths."""
+    return {}
+
+
+def convergence_gate(
+    state: State,
+) -> Literal["continue", "end"]:
+    """Check if all subgoals are completed at convergence point."""
+    logger.info("Starting convergence_gate")
+    if all_completed(state.subgoal_plan):
+        logger.info("All subgoals are completed, ending the goal")
+        return "end"
+    return "continue"
+
+
 def post_orchestrator_gate(
     state: State,
 ) -> Literal["continue", "replan", "end"]:
@@ -50,11 +66,22 @@ def post_orchestrator_gate(
 
 def post_cortex_gate(
     state: State,
-) -> Literal["continue", "end_subgoal"]:
+) -> Sequence[str]:
     logger.info("Starting post_cortex_gate")
-    if len(state.complete_subgoals_by_ids) > 0:
-        return "end_subgoal"
-    return "continue"
+    node_sequence = []
+
+    if len(state.complete_subgoals_by_ids) > 0 or not state.structured_decisions:
+        # If subgoals need to be marked as complete, add the path to the orchestrator.
+        # The 'or not state.structured_decisions' ensures we don't get stuck if Cortex does nothing.
+        node_sequence.append("review_subgoals")
+
+    if state.structured_decisions:
+        node_sequence.append("execute_decisions")
+
+    if state.screen_analysis_prompt:
+        node_sequence.append("analyze_screen")
+
+    return node_sequence
 
 
 def post_executor_gate(
@@ -96,7 +123,11 @@ async def get_graph(ctx: MobileUseContext) -> CompiledStateGraph:
 
     graph_builder.add_node("summarizer", SummarizerNode(ctx))
 
-    # Linking nodes
+    graph_builder.add_node("screen_analyzer", ScreenAnalyzerNode(ctx))
+
+    graph_builder.add_node(node="convergence", action=convergence_node, defer=True)
+
+    ## Linking nodes
     graph_builder.add_edge(START, "planner")
     graph_builder.add_edge("planner", "orchestrator")
     graph_builder.add_conditional_edges(
@@ -113,8 +144,9 @@ async def get_graph(ctx: MobileUseContext) -> CompiledStateGraph:
         "cortex",
         post_cortex_gate,
         {
-            "continue": "executor",
-            "end_subgoal": "orchestrator",
+            "review_subgoals": "orchestrator",
+            "analyze_screen": "screen_analyzer",
+            "execute_decisions": "executor",
         },
     )
     graph_builder.add_conditional_edges(
@@ -123,6 +155,18 @@ async def get_graph(ctx: MobileUseContext) -> CompiledStateGraph:
         {"invoke_tools": "executor_tools", "skip": "summarizer"},
     )
     graph_builder.add_edge("executor_tools", "summarizer")
-    graph_builder.add_edge("summarizer", "contextor")
+
+    graph_builder.add_edge("orchestrator", "convergence")
+    graph_builder.add_edge("screen_analyzer", "convergence")
+    graph_builder.add_edge("summarizer", "convergence")
+
+    graph_builder.add_conditional_edges(
+        source="convergence",
+        path=convergence_gate,
+        path_map={
+            "continue": "contextor",
+            "end": END,
+        },
+    )
 
     return graph_builder.compile()

minitap/mobile_use/graph/state.py

@@ -1,13 +1,14 @@
+from typing import Annotated
+
 from langchain_core.messages import AIMessage, AnyMessage
 from langgraph.graph import add_messages
 from langgraph.prebuilt.chat_agent_executor import AgentStatePydantic
-from typing import Annotated
 
 from minitap.mobile_use.agents.planner.types import Subgoal
 from minitap.mobile_use.config import AgentNode
+from minitap.mobile_use.context import MobileUseContext
 from minitap.mobile_use.utils.logger import get_logger
 from minitap.mobile_use.utils.recorder import record_interaction
-from minitap.mobile_use.context import MobileUseContext
 
 logger = get_logger(__name__)
 
@@ -24,7 +25,6 @@ class State(AgentStatePydantic):
     subgoal_plan: Annotated[list[Subgoal], "The current plan, made of subgoals"]
 
     # contextor related keys
-    latest_screenshot_base64: Annotated[str | None, "Latest screenshot of the device", take_last]
     latest_ui_hierarchy: Annotated[
         list[dict] | None, "Latest UI hierarchy of the device", take_last
     ]
@@ -43,6 +43,13 @@ class State(AgentStatePydantic):
         take_last,
     ]
 
+    # screen_analyzer related keys
+    screen_analysis_prompt: Annotated[
+        str | None,
+        "Prompt for the screen_analyzer agent to analyze the screen",
+        take_last,
+    ]
+
     # executor related keys
     executor_messages: Annotated[list[AnyMessage], "Sequential Executor messages", add_messages]
     cortex_last_thought: Annotated[str | None, "Last thought of the cortex for the executor"]

minitap/mobile_use/main.py

@@ -6,6 +6,7 @@ from adbutils import AdbClient
 from langchain.callbacks.base import Callbacks
 from rich.console import Console
 from typing import Annotated
+from shutil import which
 
 from minitap.mobile_use.config import (
     initialize_llm_config,
@@ -102,10 +103,17 @@ def main(
     Run the Mobile-use agent to automate tasks on a mobile device.
     """
     console = Console()
-    adb_client = AdbClient(
-        host=settings.ADB_HOST or "localhost",
-        port=settings.ADB_PORT or 5037,
-    )
+
+    adb_client = None
+    try:
+        if which("adb"):
+            adb_client = AdbClient(
+                host=settings.ADB_HOST or "localhost",
+                port=settings.ADB_PORT or 5037,
+            )
+    except Exception:
+        pass  # ADB not available, will only support iOS devices
+
     display_device_status(console, adb_client=adb_client)
     asyncio.run(
         run_automation(

minitap/mobile_use/sdk/agent.py

@@ -53,6 +53,7 @@ from minitap.mobile_use.sdk.types.task import (
     AgentProfile,
     PlatformTaskInfo,
     PlatformTaskRequest,
+    CloudDevicePlatformTaskRequest,
     Task,
     TaskRequest,
 )
@@ -89,6 +90,8 @@ class Agent:
     _screen_api_client: ScreenApiClient
     _hw_bridge_client: DeviceHardwareClient
     _adb_client: AdbClient | None
+    _current_task: asyncio.Task | None = None
+    _task_lock: asyncio.Lock
 
     def __init__(self, *, config: AgentConfig | None = None):
         self._config = config or get_default_agent_config()
@@ -101,6 +104,7 @@ class Agent:
         self._is_default_screen_api = (
             self._config.servers.screen_api_base_url == DEFAULT_SCREEN_API_BASE_URL
         )
+        self._task_lock = asyncio.Lock()
         # Initialize platform service if API key is available in environment
         # Note: Can also be initialized later with API key from request
         if settings.MINITAP_API_KEY:
@@ -125,7 +129,7 @@ class Agent:
 
         # Get first available device ID
         if not self._config.device_id or not self._config.device_platform:
-            device_id, platform = get_first_device()
+            device_id, platform = get_first_device(logger=logger)
         else:
             device_id, platform = self._config.device_id, self._config.device_platform
 
@@ -241,6 +245,9 @@ class Agent:
                 else:
                     raise PlatformServiceUninitializedError()
                 task_info = await platform_service.create_task_run(request=request)
+                if isinstance(request, CloudDevicePlatformTaskRequest):
+                    request.task_run_id = task_info.task_run.id
+                    request.task_run_id_available_event.set()
                 self._config.agent_profiles[task_info.llm_profile.name] = task_info.llm_profile
                 request = task_info.task_request
             return await self._run_task(
@@ -336,78 +343,112 @@ class Agent:
         state = self._get_graph_state(task=task)
         graph_input = state.model_dump()
 
-        last_state: State | None = None
-        last_state_snapshot: dict | None = None
-        output = None
-        try:
-            logger.info(f"[{task_name}] Invoking graph with input: {graph_input}")
-            await task.set_status(status="running", message="Invoking graph...")
-            async for chunk in (await get_graph(context)).astream(
-                input=graph_input,
-                config={
-                    "recursion_limit": task.request.max_steps,
-                    "callbacks": self._config.graph_config_callbacks,
-                },
-                stream_mode=["messages", "custom", "updates", "values"],
-            ):
-                stream_mode, payload = chunk
-                if stream_mode == "values":
-                    last_state_snapshot = payload  # type: ignore
-                    last_state = State(**last_state_snapshot)  # type: ignore
-                    if task.request.thoughts_output_path:
-                        record_events(
-                            output_path=task.request.thoughts_output_path,
-                            events=last_state.agents_thoughts,
-                        )
-
-                if stream_mode == "updates":
-                    for _, value in payload.items():  # type: ignore node name, node output
-                        if value and "agents_thoughts" in value:
-                            new_thoughts = value["agents_thoughts"]
-                            last_item = new_thoughts[-1] if new_thoughts else None
-                            if last_item:
-                                log_agent_thought(
-                                    agent_thought=last_item,
-                                )
-
-            if not last_state:
-                err = f"[{task_name}] No result received from graph"
+        async def _execute_task_logic():
+            last_state: State | None = None
+            last_state_snapshot: dict | None = None
+            output = None
+            try:
+                logger.info(f"[{task_name}] Invoking graph with input: {graph_input}")
+                await task.set_status(status="running", message="Invoking graph...")
+                async for chunk in (await get_graph(context)).astream(
+                    input=graph_input,
+                    config={
+                        "recursion_limit": task.request.max_steps,
+                        "callbacks": self._config.graph_config_callbacks,
+                    },
+                    stream_mode=["messages", "custom", "updates", "values"],
+                ):
+                    stream_mode, payload = chunk
+                    if stream_mode == "values":
+                        last_state_snapshot = payload  # type: ignore
+                        last_state = State(**last_state_snapshot)  # type: ignore
+                        if task.request.thoughts_output_path:
+                            record_events(
+                                output_path=task.request.thoughts_output_path,
+                                events=last_state.agents_thoughts,
+                            )
+
+                    if stream_mode == "updates":
+                        for _, value in payload.items():  # type: ignore node name, node output
+                            if value and "agents_thoughts" in value:
+                                new_thoughts = value["agents_thoughts"]
+                                last_item = new_thoughts[-1] if new_thoughts else None
+                                if last_item:
+                                    log_agent_thought(
+                                        agent_thought=last_item,
+                                    )
+
+                if not last_state:
+                    err = f"[{task_name}] No result received from graph"
+                    logger.warning(err)
+                    await task.finalize(content=output, state=last_state_snapshot, error=err)
+                    return None
+
+                print_ai_response_to_stderr(graph_result=last_state)
+                output = await self._extract_output(
+                    task_name=task_name,
+                    ctx=context,
+                    request=request,
+                    output_config=output_config,
+                    state=last_state,
+                )
+                logger.info(f"✅ Automation '{task_name}' is success ✅")
+                await task.finalize(content=output, state=last_state_snapshot)
+                return output
+            except asyncio.CancelledError:
+                err = f"[{task_name}] Task cancelled"
                 logger.warning(err)
-                await task.finalize(content=output, state=last_state_snapshot, error=err)
-                return None
+                await task.finalize(
+                    content=output,
+                    state=last_state_snapshot,
+                    error=err,
+                    cancelled=True,
+                )
+                raise
+            except Exception as e:
+                err = f"[{task_name}] Error running automation: {e}"
+                logger.error(err)
+                await task.finalize(
+                    content=output,
+                    state=last_state_snapshot,
+                    error=err,
+                )
+                raise
+            finally:
+                self._finalize_tracing(task=task, context=context)
 
-            print_ai_response_to_stderr(graph_result=last_state)
-            output = await self._extract_output(
-                task_name=task_name,
-                ctx=context,
-                request=request,
-                output_config=output_config,
-                state=last_state,
-            )
-            logger.info(f"✅ Automation '{task_name}' is success ✅")
-            await task.finalize(content=output, state=last_state_snapshot)
-        except asyncio.CancelledError:
-            err = f"[{task_name}] Task cancelled"
-            logger.warning(err)
-            await task.finalize(
-                content=output,
-                state=last_state_snapshot,
-                error=err,
-                cancelled=True,
-            )
-            raise
-        except Exception as e:
-            err = f"[{task_name}] Error running automation: {e}"
-            logger.error(err)
-            await task.finalize(
-                content=output,
-                state=last_state_snapshot,
-                error=err,
-            )
-            raise
-        finally:
-            self._finalize_tracing(task=task, context=context)
-        return output
+        async with self._task_lock:
+            if self._current_task and not self._current_task.done():
+                logger.warning(
+                    "Another automation task is already running. "
+                    "Stopping it before starting the new one."
+                )
+                self.stop_current_task()
+                try:
+                    await self._current_task
+                except asyncio.CancelledError:
+                    pass
+
+            try:
+                self._current_task = asyncio.create_task(_execute_task_logic())
+                return await self._current_task
+            finally:
+                self._current_task = None
+
+    def stop_current_task(self):
+        """Requests cancellation of the currently running automation task."""
+        if self._current_task and not self._current_task.done():
+            logger.info("Requesting to stop the current automation task...")
+            was_cancelled = self._current_task.cancel()
+            if was_cancelled:
+                logger.success("Cancellation request for the current task was sent.")
+            else:
+                logger.warning(
+                    "Could not send cancellation request for the current task "
+                    "(it may already be completing)."
+                )
+        else:
+            logger.info("No active automation task to stop.")
 
     def is_healthy(self):
         """
@@ -522,11 +563,11 @@ class Agent:
             initial_goal=task.request.goal,
             subgoal_plan=[],
             latest_ui_hierarchy=None,
-            latest_screenshot_base64=None,
             focused_app_info=None,
             device_date=None,
             structured_decisions=None,
             complete_subgoals_by_ids=[],
+            screen_analysis_prompt=None,
             agents_thoughts=[],
             remaining_steps=task.request.max_steps,
             executor_messages=[],

minitap/mobile_use/sdk/examples/smart_notification_assistant.py

@@ -22,6 +22,7 @@ from datetime import datetime
 from enum import Enum
 
 from pydantic import BaseModel, Field
+
 from minitap.mobile_use.config import LLM, LLMConfig, LLMConfigUtils, LLMWithFallback
 from minitap.mobile_use.sdk import Agent
 from minitap.mobile_use.sdk.builders import Builders
@@ -62,17 +63,42 @@ def get_agent() -> Agent:
     analyzer_profile = AgentProfile(
         name="analyzer",
         llm_config=LLMConfig(
-            planner=LLM(provider="openrouter", model="meta-llama/llama-4-scout"),
-            orchestrator=LLM(provider="openrouter", model="meta-llama/llama-4-scout"),
+            planner=LLMWithFallback(
+                provider="openrouter",
+                model="meta-llama/llama-4-scout",
+                fallback=LLM(provider="openrouter", model="meta-llama/llama-4-maverick"),
+            ),
+            orchestrator=LLMWithFallback(
+                provider="openrouter",
+                model="meta-llama/llama-4-scout",
+                fallback=LLM(provider="openrouter", model="meta-llama/llama-4-maverick"),
+            ),
             cortex=LLMWithFallback(
                 provider="openai",
                 model="o4-mini",
                 fallback=LLM(provider="openai", model="gpt-5"),
             ),
-            executor=LLM(provider="openai", model="gpt-5-nano"),
+            screen_analyzer=LLMWithFallback(
+                provider="openai",
+                model="gpt-4o",
+                fallback=LLM(provider="openai", model="gpt-5-nano"),
+            ),
+            executor=LLMWithFallback(
+                provider="openai",
+                model="gpt-5-nano",
+                fallback=LLM(provider="openai", model="gpt-5-mini"),
+            ),
             utils=LLMConfigUtils(
-                outputter=LLM(provider="openai", model="gpt-5-nano"),
-                hopper=LLM(provider="openai", model="gpt-4.1"),
+                outputter=LLMWithFallback(
+                    provider="openai",
+                    model="gpt-5-nano",
+                    fallback=LLM(provider="openai", model="gpt-5-mini"),
+                ),
+                hopper=LLMWithFallback(
+                    provider="openai",
+                    model="gpt-5-nano",
+                    fallback=LLM(provider="openai", model="gpt-5-mini"),
+                ),
             ),
         ),
         # from_file="/tmp/analyzer.jsonc"  # can be loaded from file
@@ -82,17 +108,40 @@ def get_agent() -> Agent:
     action_profile = AgentProfile(
         name="note_taker",
         llm_config=LLMConfig(
-            planner=LLM(provider="openai", model="o3"),
-            orchestrator=LLM(provider="google", model="gemini-2.5-flash"),
+            planner=LLMWithFallback(
+                provider="openai", model="o3", fallback=LLM(provider="openai", model="gpt-5")
+            ),
+            orchestrator=LLMWithFallback(
+                provider="google",
+                model="gemini-2.5-flash",
+                fallback=LLM(provider="openai", model="gpt-5"),
+            ),
             cortex=LLMWithFallback(
                 provider="openai",
                 model="o4-mini",
                 fallback=LLM(provider="openai", model="gpt-5"),
             ),
-            executor=LLM(provider="openai", model="gpt-4o-mini"),
+            screen_analyzer=LLMWithFallback(
+                provider="openai",
+                model="gpt-4o",
+                fallback=LLM(provider="openai", model="gpt-5-nano"),
+            ),
+            executor=LLMWithFallback(
+                provider="openai",
+                model="gpt-4o-mini",
+                fallback=LLM(provider="openai", model="gpt-5-nano"),
+            ),
             utils=LLMConfigUtils(
-                outputter=LLM(provider="openai", model="gpt-5-nano"),
-                hopper=LLM(provider="openai", model="gpt-4.1"),
+                outputter=LLMWithFallback(
+                    provider="openai",
+                    model="gpt-5-nano",
+                    fallback=LLM(provider="openai", model="gpt-5-mini"),
+                ),
+                hopper=LLMWithFallback(
+                    provider="openai",
+                    model="gpt-5-nano",
+                    fallback=LLM(provider="openai", model="gpt-5-mini"),
+                ),
             ),
         ),
     )