minitap-mobile-use 2.2.0__py3-none-any.whl → 2.4.0__py3-none-any.whl

minitap/mobile_use/sdk/agent.py

@@ -3,23 +3,28 @@ import sys
 import tempfile
 import time
 import uuid
-from datetime import datetime
+from collections.abc import Callable, Coroutine
+from datetime import UTC, datetime
 from pathlib import Path
+from shutil import which
 from types import NoneType
-from typing import TypeVar, overload
+from typing import Any, TypeVar, overload
 
 from adbutils import AdbClient
+from dotenv import load_dotenv
 from langchain_core.messages import AIMessage
 from pydantic import BaseModel
 
 from minitap.mobile_use.agents.outputter.outputter import outputter
+from minitap.mobile_use.agents.planner.types import Subgoal
 from minitap.mobile_use.clients.device_hardware_client import DeviceHardwareClient
 from minitap.mobile_use.clients.screen_api_client import ScreenApiClient
-from minitap.mobile_use.config import OutputConfig, record_events
+from minitap.mobile_use.config import AgentNode, OutputConfig, record_events, settings
 from minitap.mobile_use.context import (
     DeviceContext,
     DevicePlatform,
     ExecutionSetup,
+    IsReplan,
     MobileUseContext,
 )
 from minitap.mobile_use.controllers.mobile_command_controller import (
@@ -31,19 +36,26 @@ from minitap.mobile_use.graph.graph import get_graph
 from minitap.mobile_use.graph.state import State
 from minitap.mobile_use.sdk.builders.agent_config_builder import get_default_agent_config
 from minitap.mobile_use.sdk.builders.task_request_builder import TaskRequestBuilder
-from minitap.mobile_use.sdk.constants import (
-    DEFAULT_HW_BRIDGE_BASE_URL,
-    DEFAULT_SCREEN_API_BASE_URL,
-)
+from minitap.mobile_use.sdk.constants import DEFAULT_HW_BRIDGE_BASE_URL, DEFAULT_SCREEN_API_BASE_URL
+from minitap.mobile_use.sdk.services.platform import PlatformService
 from minitap.mobile_use.sdk.types.agent import AgentConfig
 from minitap.mobile_use.sdk.types.exceptions import (
     AgentNotInitializedError,
     AgentProfileNotFoundError,
     AgentTaskRequestError,
     DeviceNotFoundError,
+    ExecutableNotFoundError,
+    PlatformServiceUninitializedError,
     ServerStartupError,
 )
-from minitap.mobile_use.sdk.types.task import AgentProfile, Task, TaskRequest, TaskStatus
+from minitap.mobile_use.sdk.types.platform import TaskRunPlanResponse, TaskRunStatus
+from minitap.mobile_use.sdk.types.task import (
+    AgentProfile,
+    PlatformTaskInfo,
+    PlatformTaskRequest,
+    Task,
+    TaskRequest,
+)
 from minitap.mobile_use.servers.device_hardware_bridge import BridgeStatus
 from minitap.mobile_use.servers.start_servers import (
     start_device_hardware_bridge,
@@ -63,6 +75,8 @@ logger = get_logger(__name__)
 
 TOutput = TypeVar("TOutput", bound=BaseModel | None)
 
+load_dotenv()
+
 
 class Agent:
     _config: AgentConfig
@@ -76,11 +90,23 @@ class Agent:
     _hw_bridge_client: DeviceHardwareClient
     _adb_client: AdbClient | None
 
-    def __init__(self, config: AgentConfig | None = None):
+    def __init__(self, *, config: AgentConfig | None = None):
         self._config = config or get_default_agent_config()
         self._tasks = []
         self._tmp_traces_dir = Path(tempfile.gettempdir()) / "mobile-use-traces"
         self._initialized = False
+        self._is_default_hw_bridge = (
+            self._config.servers.hw_bridge_base_url == DEFAULT_HW_BRIDGE_BASE_URL
+        )
+        self._is_default_screen_api = (
+            self._config.servers.screen_api_base_url == DEFAULT_SCREEN_API_BASE_URL
+        )
+        # Initialize platform service if API key is available in environment
+        # Note: Can also be initialized later with API key from request
+        if settings.MINITAP_API_KEY:
+            self._platform_service = PlatformService()
+        else:
+            self._platform_service = None
 
     def init(
         self,
@@ -88,6 +114,11 @@ class Agent:
         retry_count: int = 5,
         retry_wait_seconds: int = 5,
     ):
+        if not which("adb") and not which("xcrun"):
+            raise ExecutableNotFoundError("cli_tools")
+        if self._is_default_hw_bridge and not which("maestro"):
+            raise ExecutableNotFoundError("maestro")
+
         if self._initialized:
             logger.warning("Agent is already initialized. Skipping...")
             return True
@@ -183,6 +214,12 @@ class Agent:
     @overload
     async def run_task(self, *, request: TaskRequest[TOutput]) -> TOutput | None: ...
 
+    @overload
+    async def run_task(self, *, request: PlatformTaskRequest[None]) -> str | dict | None: ...
+
+    @overload
+    async def run_task(self, *, request: PlatformTaskRequest[TOutput]) -> TOutput | None: ...
+
     async def run_task(
         self,
         *,
@@ -190,10 +227,25 @@ class Agent:
         output: type[TOutput] | str | None = None,
         profile: str | AgentProfile | None = None,
         name: str | None = None,
-        request: TaskRequest[TOutput] | None = None,
+        request: TaskRequest[TOutput] | PlatformTaskRequest[TOutput] | None = None,
     ) -> str | dict | TOutput | None:
         if request is not None:
-            return await self._run_task(request)
+            task_info = None
+            platform_service = None
+            if isinstance(request, PlatformTaskRequest):
+                # Initialize platform service with API key from request if provided
+                if request.api_key:
+                    platform_service = PlatformService(api_key=request.api_key)
+                elif self._platform_service:
+                    platform_service = self._platform_service
+                else:
+                    raise PlatformServiceUninitializedError()
+                task_info = await platform_service.create_task_run(request=request)
+                self._config.agent_profiles[task_info.llm_profile.name] = task_info.llm_profile
+                request = task_info.task_request
+            return await self._run_task(
+                request=request, task_info=task_info, platform_service=platform_service
+            )
         if goal is None:
             raise AgentTaskRequestError("Goal is required")
         task_request = self.new_task(goal=goal)
@@ -208,7 +260,12 @@ class Agent:
             task_request.with_name(name=name)
         return await self._run_task(task_request.build())
 
-    async def _run_task(self, request: TaskRequest[TOutput]) -> str | dict | TOutput | None:
+    async def _run_task(
+        self,
+        request: TaskRequest[TOutput],
+        task_info: PlatformTaskInfo | None = None,
+        platform_service: PlatformService | None = None,
+    ) -> str | dict | TOutput | None:
         if not self._initialized:
             raise AgentNotInitializedError()
 
@@ -220,22 +277,48 @@ class Agent:
             agent_profile = self._config.default_profile
         logger.info(str(agent_profile))
 
+        on_status_changed = None
+        on_agent_thought = None
+        on_plan_changes = None
+        task_id = str(uuid.uuid4())
+        if task_info:
+            on_status_changed = self._get_task_status_change_callback(
+                task_info=task_info, platform_service=platform_service
+            )
+            on_agent_thought = self._get_new_agent_thought_callback(
+                task_info=task_info, platform_service=platform_service
+            )
+            on_plan_changes = self._get_plan_changes_callback(
+                task_info=task_info, platform_service=platform_service
+            )
+            task_id = task_info.task_run.id
+
         task = Task(
-            id=str(uuid.uuid4()),
+            id=task_id,
             device=self._device_context,
-            status=TaskStatus.PENDING,
+            status="pending",
             request=request,
             created_at=datetime.now(),
+            on_status_changed=on_status_changed,
         )
         self._tasks.append(task)
         task_name = task.get_name()
 
+        # Extract API key from platform service if available
+        api_key = None
+        if platform_service:
+            api_key = platform_service._api_key
+
         context = MobileUseContext(
+            trace_id=task.id,
             device=self._device_context,
             hw_bridge_client=self._hw_bridge_client,
             screen_api_client=self._screen_api_client,
             adb_client=self._adb_client,
             llm_config=agent_profile.llm_config,
+            on_agent_thought=on_agent_thought,
+            on_plan_changes=on_plan_changes,
+            minitap_api_key=api_key,
         )
 
         self._prepare_tracing(task=task, context=context)
@@ -258,7 +341,7 @@ class Agent:
         output = None
         try:
             logger.info(f"[{task_name}] Invoking graph with input: {graph_input}")
-            task.status = TaskStatus.RUNNING
+            await task.set_status(status="running", message="Invoking graph...")
             async for chunk in (await get_graph(context)).astream(
                 input=graph_input,
                 config={
@@ -290,7 +373,7 @@ class Agent:
             if not last_state:
                 err = f"[{task_name}] No result received from graph"
                 logger.warning(err)
-                task.finalize(content=output, state=last_state_snapshot, error=err)
+                await task.finalize(content=output, state=last_state_snapshot, error=err)
                 return None
 
             print_ai_response_to_stderr(graph_result=last_state)
@@ -302,16 +385,25 @@ class Agent:
                 state=last_state,
             )
             logger.info(f"✅ Automation '{task_name}' is success ✅")
-            task.finalize(content=output, state=last_state_snapshot)
+            await task.finalize(content=output, state=last_state_snapshot)
         except asyncio.CancelledError:
             err = f"[{task_name}] Task cancelled"
             logger.warning(err)
-            task.finalize(content=output, state=last_state_snapshot, error=err, cancelled=True)
+            await task.finalize(
+                content=output,
+                state=last_state_snapshot,
+                error=err,
+                cancelled=True,
+            )
             raise
         except Exception as e:
             err = f"[{task_name}] Error running automation: {e}"
             logger.error(err)
-            task.finalize(content=output, state=last_state_snapshot, error=err)
+            await task.finalize(
+                content=output,
+                state=last_state_snapshot,
+                error=err,
+            )
             raise
         finally:
             self._finalize_tracing(task=task, context=context)
@@ -342,7 +434,9 @@ class Agent:
         traces_output_path.mkdir(parents=True, exist_ok=True)
         temp_trace_path.mkdir(parents=True, exist_ok=True)
         context.execution_setup = ExecutionSetup(
-            traces_path=self._tmp_traces_dir, trace_id=task_name
+            traces_path=self._tmp_traces_dir,
+            trace_name=task_name,
+            enable_remote_tracing=task.request.enable_remote_tracing,
         )
 
     def _finalize_tracing(self, task: Task, context: MobileUseContext):
@@ -351,11 +445,11 @@ class Agent:
             return
 
         task_name = task.get_name()
-        status = "_PASS" if task.status == TaskStatus.COMPLETED else "_FAIL"
+        status = "_PASS" if task.status == "completed" else "_FAIL"
         ts = task.created_at.strftime("%Y-%m-%dT%H-%M-%S")
-        new_name = f"{exec_setup_ctx.trace_id}{status}_{ts}"
+        new_name = f"{exec_setup_ctx.trace_name}{status}_{ts}"
 
-        temp_trace_path = (self._tmp_traces_dir / exec_setup_ctx.trace_id).resolve()
+        temp_trace_path = (self._tmp_traces_dir / exec_setup_ctx.trace_name).resolve()
         traces_output_path = Path(task.request.trace_path).resolve()
 
         logger.info(f"[{task_name}] Compiling trace FROM FOLDER: " + str(temp_trace_path))
@@ -433,17 +527,11 @@ class Agent:
         self._hw_bridge_client = DeviceHardwareClient(
             base_url=self._config.servers.hw_bridge_base_url.to_url(),
         )
-        self._is_default_hw_bridge = (
-            self._config.servers.hw_bridge_base_url == DEFAULT_HW_BRIDGE_BASE_URL
-        )
         self._screen_api_client = ScreenApiClient(
             base_url=self._config.servers.screen_api_base_url.to_url(),
             retry_count=retry_count,
             retry_wait_seconds=retry_wait_seconds,
         )
-        self._is_default_screen_api = (
-            self._config.servers.screen_api_base_url == DEFAULT_SCREEN_API_BASE_URL
-        )
 
     def _run_servers(self, device_id: str, platform: DevicePlatform) -> bool:
         if self._is_default_hw_bridge:
@@ -496,7 +584,10 @@ class Agent:
 
     def _check_device_screen_api_health(self) -> bool:
         try:
+            # Required to know if the Screen API is up
             self._screen_api_client.get_with_retry("/health", timeout=5)
+            # Required to know if the Screen API actually receives screenshot from the HW Bridge API
+            self._screen_api_client.get_with_retry("/screen-info", timeout=5)
             return True
         except Exception as e:
             logger.error(f"Device Screen API health check failed: {e}")
@@ -519,6 +610,90 @@ class Agent:
             device_height=screen_data.height,
         )
 
+    def _get_task_status_change_callback(
+        self,
+        task_info: PlatformTaskInfo,
+        platform_service: PlatformService | None = None,
+    ) -> Callable[[TaskRunStatus, str | None, Any | None], Coroutine]:
+        service = platform_service or self._platform_service
+
+        async def change_status(
+            status: TaskRunStatus,
+            message: str | None = None,
+            output: Any | None = None,
+        ):
+            if not service:
+                raise PlatformServiceUninitializedError()
+            try:
+                await service.update_task_run_status(
+                    task_run_id=task_info.task_run.id,
+                    status=status,
+                    message=message,
+                    output=output,
+                )
+            except Exception as e:
+                logger.error(f"Failed to update task run status: {e}")
+
+        return change_status
+
+    def _get_plan_changes_callback(
+        self,
+        task_info: PlatformTaskInfo,
+        platform_service: PlatformService | None = None,
+    ) -> Callable[[list[Subgoal], IsReplan], Coroutine]:
+        service = platform_service or self._platform_service
+        current_plan: TaskRunPlanResponse | None = None
+
+        async def update_plan(plan: list[Subgoal], is_replan: IsReplan):
+            nonlocal current_plan
+
+            if not service:
+                raise PlatformServiceUninitializedError()
+            try:
+                if is_replan and current_plan:
+                    # End previous plan
+                    await service.upsert_task_run_plan(
+                        task_run_id=task_info.task_run.id,
+                        started_at=current_plan.started_at,
+                        plan=plan,
+                        ended_at=datetime.now(UTC),
+                        plan_id=current_plan.id,
+                    )
+                    current_plan = None
+
+                current_plan = await service.upsert_task_run_plan(
+                    task_run_id=task_info.task_run.id,
+                    started_at=current_plan.started_at if current_plan else datetime.now(UTC),
+                    plan=plan,
+                    ended_at=current_plan.ended_at if current_plan else None,
+                    plan_id=current_plan.id if current_plan else None,
+                )
+            except Exception as e:
+                logger.error(f"Failed to update plan: {e}")
+
+        return update_plan
+
+    def _get_new_agent_thought_callback(
+        self,
+        task_info: PlatformTaskInfo,
+        platform_service: PlatformService | None = None,
+    ) -> Callable[[AgentNode, str], Coroutine]:
+        service = platform_service or self._platform_service
+
+        async def add_agent_thought(agent: AgentNode, thought: str):
+            if not service:
+                raise PlatformServiceUninitializedError()
+            try:
+                await service.add_agent_thought(
+                    task_run_id=task_info.task_run.id,
+                    agent=agent,
+                    thought=thought,
+                )
+            except Exception as e:
+                logger.error(f"Failed to add agent thought: {e}")
+
+        return add_agent_thought
+
 
 def _validate_and_prepare_file(file_path: Path):
     path_obj = Path(file_path)

minitap/mobile_use/sdk/examples/README.md

@@ -3,12 +3,26 @@
 Location: `src/mobile_use/sdk/examples/`
 
 Run any example via:
+
 - `python src/mobile_use/sdk/examples/<filename>.py`
 
 ## Practical Automation Examples
 
 These examples demonstrate two different ways to use the SDK, each applying an appropriate level of complexity for the task at hand:
 
+### platform_minimal_example.py - Painless integration with the Minitap platform
+
+This script shows the simplest way to run minitap :
+
+- Visit https://platform.minitap.ai to create a task and get your API key.
+- Initialize the agent with your API key: Agent(minitap_api_key=...).
+- Ask the agent to run one of the tasks you’ve set up in the Minitap platform
+  (e.g., "like-instagram-post").
+- The task’s goal and settings live in the Minitap platform, you don’t need
+  to hardcode them here.
+- If you’ve created different profiles (LLM configurations) in the Minitap platform (like "fast-config"),
+  you can pick which one to use with the `profile` field.
+
 ### simple_photo_organizer.py - Straightforward Approach
 
 Demonstrates the simplest way to use the SDK for quick automation tasks:
@@ -32,7 +46,11 @@ Showcases more advanced SDK features while remaining practical:
 
 ## Usage Notes
 
-- **Choosing an Approach**: Use the direct approach (like in `simple_photo_organizer.py`) for simple tasks and the builder approach (like in `smart_notification_assistant.py`) when you need more customization.
+- **Choosing an Approach**:
+
+  - Use the direct approach (like `platform_minimal_example.py`) for painless setup using the Minitap platform. You can configure any task, save, run, and monitor them with a few clicks.
+  - Use the simple approach (like `simple_photo_organizer.py`) for straightforward tasks, you configure settings yourself and every LLM call happens on your device.
+  - Use the builder approach (like `smart_notification_assistant.py`) when you need more customization.
 
 - **Device Detection**: The agent detects the first available device unless you specify one with `AgentConfigBuilder.for_device(...)`.
 

minitap/mobile_use/sdk/examples/platform_minimal_example.py

@@ -0,0 +1,46 @@
+"""
+Platform Usage - Minitap SDK with API Key Example
+
+This example demonstrates how to use the mobile-use SDK via the Minitap platform:
+- Agent with minitap_api_key
+- PlatformTaskRequest with platform-provided task_id
+- All task configuration (goal, output format, etc.) managed by platform UI
+
+Platform Model:
+- API key provides authentication and agent configuration
+- task_id references pre-configured task from platform UI
+- No goal, output_format, profile selection needed in code
+- Everything bound to task_id + api_key combination
+
+Run:
+- python src/mobile_use/sdk/examples/platform_minimal_example.py
+"""
+
+import asyncio
+
+from minitap.mobile_use.sdk import Agent
+from minitap.mobile_use.sdk.types import PlatformTaskRequest
+
+
+async def main() -> None:
+    """
+    Main execution function demonstrating minitap platform usage pattern.
+
+    Visit https://platform.minitap.ai to create a task, customize your profiles,
+    and get your API key.
+    Set MINITAP_API_KEY and MINITAP_API_BASE_URL environment variables.
+    """
+    agent = Agent()
+    agent.init()
+    result = await agent.run_task(
+        request=PlatformTaskRequest(
+            task="your-task-name",
+            profile="your-profile-name",
+        )
+    )
+    print(result)
+    agent.clean()
+
+
+if __name__ == "__main__":
+    asyncio.run(main())