minitap-mobile-use 3.3.0__py3-none-any.whl

minitap/mobile_use/tools/mobile/video_recording.py

@@ -0,0 +1,177 @@
+"""
+Video recording tools for mobile devices.
+
+Provides start/stop video recording tools that delegate to platform-specific
+controllers (AndroidDeviceController, iOSDeviceController).
+"""
+
+import shutil
+from typing import Annotated
+
+from langchain_core.messages import ToolMessage
+from langchain_core.tools import tool
+from langchain_core.tools.base import BaseTool, InjectedToolCallId
+from langgraph.prebuilt import InjectedState
+from langgraph.types import Command
+
+from minitap.mobile_use.constants import EXECUTOR_MESSAGES_KEY
+from minitap.mobile_use.context import DevicePlatform, MobileUseContext
+from minitap.mobile_use.controllers.controller_factory import get_controller
+from minitap.mobile_use.graph.state import State
+from minitap.mobile_use.tools.tool_wrapper import ToolWrapper
+from minitap.mobile_use.utils.logger import get_logger
+from minitap.mobile_use.utils.video import DEFAULT_MAX_DURATION_SECONDS
+
+logger = get_logger(__name__)
+
+
+def get_start_video_recording_tool(ctx: MobileUseContext) -> BaseTool:
+    @tool
+    async def start_video_recording(
+        agent_thought: str,
+        tool_call_id: Annotated[str, InjectedToolCallId],
+        state: Annotated[State, InjectedState],
+    ):
+        """
+        Starts a background screen recording on the mobile device.
+        Recording continues until stop_video_recording is called.
+        Max duration: 3 min (Android) / 15 min (iOS). Audio is not captured.
+        """
+        platform = ctx.device.mobile_platform
+        controller = get_controller(ctx)
+
+        if platform in (DevicePlatform.ANDROID, DevicePlatform.IOS):
+            result = await controller.start_video_recording(DEFAULT_MAX_DURATION_SECONDS)
+        else:
+            from minitap.mobile_use.utils.video import VideoRecordingResult
+
+            result = VideoRecordingResult(
+                success=False,
+                message=f"Unsupported platform: {platform}",
+            )
+
+        if result.success:
+            agent_outcome = start_video_recording_wrapper.on_success_fn(result.message)
+        else:
+            agent_outcome = start_video_recording_wrapper.on_failure_fn(result.message)
+
+        tool_message = ToolMessage(
+            tool_call_id=tool_call_id,
+            content=agent_outcome,
+            status="success" if result.success else "error",
+        )
+
+        return Command(
+            update=await state.asanitize_update(
+                ctx=ctx,
+                update={
+                    "agents_thoughts": [agent_thought, agent_outcome],
+                    EXECUTOR_MESSAGES_KEY: [tool_message],
+                },
+                agent="executor",
+            ),
+        )
+
+    return start_video_recording
+
+
+def get_stop_video_recording_tool(ctx: MobileUseContext) -> BaseTool:
+    from minitap.mobile_use.agents.video_analyzer.video_analyzer import analyze_video
+
+    @tool
+    async def stop_video_recording(
+        agent_thought: str,
+        tool_call_id: Annotated[str, InjectedToolCallId],
+        state: Annotated[State, InjectedState],
+        prompt: str = "Describe what happened in the video.",
+    ):
+        """
+        Stops the current screen recording and analyzes the video content.
+        Use `prompt` to specify what to extract (e.g., "What happens after each 10s of the video?").
+        """
+        platform = ctx.device.mobile_platform
+        controller = get_controller(ctx)
+
+        if platform in (DevicePlatform.ANDROID, DevicePlatform.IOS):
+            result = await controller.stop_video_recording()
+        else:
+            from minitap.mobile_use.utils.video import VideoRecordingResult
+
+            result = VideoRecordingResult(
+                success=False,
+                message=f"Unsupported platform: {platform}",
+            )
+
+        if not result.success or result.video_path is None:
+            agent_outcome = stop_video_recording_wrapper.on_failure_fn(result.message)
+            tool_message = ToolMessage(
+                tool_call_id=tool_call_id,
+                content=agent_outcome,
+                status="error",
+            )
+            return Command(
+                update=await state.asanitize_update(
+                    ctx=ctx,
+                    update={
+                        "agents_thoughts": [agent_thought, agent_outcome],
+                        EXECUTOR_MESSAGES_KEY: [tool_message],
+                    },
+                    agent="executor",
+                ),
+            )
+
+        video_path = result.video_path
+        try:
+            analysis_result = await analyze_video(
+                ctx=ctx,
+                video_path=video_path,
+                prompt=prompt,
+            )
+            agent_outcome = stop_video_recording_wrapper.on_success_fn(analysis_result)
+            status = "success"
+        except Exception as e:
+            logger.error(f"Video analysis failed: {e}")
+            agent_outcome = stop_video_recording_wrapper.on_failure_fn(
+                f"Recording stopped but analysis failed: {e}"
+            )
+            status = "error"
+        finally:
+            try:
+                if video_path and video_path.exists():
+                    video_path.unlink()
+                    if video_path.parent.exists():
+                        shutil.rmtree(video_path.parent)
+            except Exception:
+                pass
+
+        tool_message = ToolMessage(
+            tool_call_id=tool_call_id,
+            content=agent_outcome,
+            status=status,
+        )
+
+        return Command(
+            update=await state.asanitize_update(
+                ctx=ctx,
+                update={
+                    "agents_thoughts": [agent_thought, agent_outcome],
+                    EXECUTOR_MESSAGES_KEY: [tool_message],
+                },
+                agent="executor",
+            ),
+        )
+
+    return stop_video_recording
+
+
+start_video_recording_wrapper = ToolWrapper(
+    tool_fn_getter=get_start_video_recording_tool,
+    on_success_fn=lambda message: f"Video recording started successfully. {message}",
+    on_failure_fn=lambda message: f"Failed to start video recording: {message}",
+)
+
+stop_video_recording_wrapper = ToolWrapper(
+    tool_fn_getter=get_stop_video_recording_tool,
+    on_success_fn=lambda analysis: f"Video stopped successfully. Analysis result:\n{analysis}",
+    on_failure_fn=lambda message: f"Video recording/analysis failed: {message}",
+)

minitap/mobile_use/tools/mobile/wait_for_delay.py

@@ -0,0 +1,81 @@
+import time
+from typing import Annotated
+
+from langchain_core.messages import ToolMessage
+from langchain_core.tools import tool
+from langchain_core.tools.base import InjectedToolCallId
+from langgraph.prebuilt import InjectedState
+from langgraph.types import Command
+
+from minitap.mobile_use.constants import EXECUTOR_MESSAGES_KEY
+from minitap.mobile_use.context import MobileUseContext
+from minitap.mobile_use.graph.state import State
+from minitap.mobile_use.tools.tool_wrapper import ToolWrapper
+
+MAX_DELAY_MS = 60000
+
+
+def get_wait_for_delay_tool(ctx: MobileUseContext):
+    @tool
+    async def wait_for_delay(
+        agent_thought: str,
+        time_in_ms: int,
+        tool_call_id: Annotated[str, InjectedToolCallId],
+        state: Annotated[State, InjectedState],
+    ) -> Command:
+        """
+        Wait for a delay in milliseconds.
+
+        This tool pauses execution for a specified number of milliseconds.
+        Use this when you need to introduce a controlled delay to allow the UI
+        to update after an action, regardless of whether an animation is playing.
+
+        Args:
+            time_in_ms: The number of milliseconds to wait. (capped at 60 seconds)
+
+
+        Example:
+            - wait_for_delay with time_in_ms=1000 (waits 1 second)
+            - wait_for_delay with time_in_ms=500 (waits 0.5 seconds)
+        """
+        if time_in_ms < 0:
+            time_in_ms = 1000
+        if time_in_ms > MAX_DELAY_MS:
+            time_in_ms = MAX_DELAY_MS
+        try:
+            time.sleep(time_in_ms / 1000)
+            output = None
+            has_failed = False
+        except Exception as e:
+            output = str(e)
+            has_failed = True
+        agent_outcome = (
+            wait_for_delay_wrapper.on_failure_fn()
+            if has_failed
+            else wait_for_delay_wrapper.on_success_fn(time_in_ms)
+        )
+        tool_message = ToolMessage(
+            tool_call_id=tool_call_id,
+            content=agent_outcome,
+            additional_kwargs={"error": output} if has_failed else {},
+            status="error" if has_failed else "success",
+        )
+        return Command(
+            update=await state.asanitize_update(
+                ctx=ctx,
+                update={
+                    "agents_thoughts": [agent_thought, agent_outcome],
+                    EXECUTOR_MESSAGES_KEY: [tool_message],
+                },
+                agent="executor",
+            ),
+        )
+
+    return wait_for_delay
+
+
+wait_for_delay_wrapper = ToolWrapper(
+    tool_fn_getter=get_wait_for_delay_tool,
+    on_success_fn=lambda delay: f"Successfully waited for {delay} milliseconds.",
+    on_failure_fn=lambda: "Failed to wait for delay.",
+)

minitap/mobile_use/tools/scratchpad.py

@@ -0,0 +1,147 @@
+from typing import Annotated
+
+from langchain_core.messages import ToolMessage
+from langchain_core.tools import tool
+from langchain_core.tools.base import BaseTool, InjectedToolCallId
+from langgraph.prebuilt import InjectedState
+from langgraph.types import Command
+
+from minitap.mobile_use.constants import EXECUTOR_MESSAGES_KEY
+from minitap.mobile_use.context import MobileUseContext
+from minitap.mobile_use.graph.state import State
+from minitap.mobile_use.tools.tool_wrapper import ToolWrapper
+from minitap.mobile_use.utils.logger import get_logger
+
+logger = get_logger(__name__)
+
+
+def get_save_note_tool(ctx: MobileUseContext) -> BaseTool:
+    @tool
+    async def save_note(
+        agent_thought: str,
+        key: str,
+        content: str,
+        tool_call_id: Annotated[str, InjectedToolCallId],
+        state: Annotated[State, InjectedState],
+    ):
+        """
+        Saves a text note to persistent memory with the given key.
+        If the key already exists, it will be overwritten.
+        """
+        updated_scratchpad = {**state.scratchpad, key: content}
+
+        agent_outcome = save_note_wrapper.on_success_fn(key)
+
+        tool_message = ToolMessage(
+            tool_call_id=tool_call_id,
+            content=agent_outcome,
+            status="success",
+        )
+        return Command(
+            update=await state.asanitize_update(
+                ctx=ctx,
+                update={
+                    "agents_thoughts": [agent_thought, agent_outcome],
+                    EXECUTOR_MESSAGES_KEY: [tool_message],
+                    "scratchpad": updated_scratchpad,
+                },
+                agent="executor",
+            ),
+        )
+
+    return save_note
+
+
+def get_read_note_tool(ctx: MobileUseContext) -> BaseTool:
+    @tool
+    async def read_note(
+        agent_thought: str,
+        key: str,
+        tool_call_id: Annotated[str, InjectedToolCallId],
+        state: Annotated[State, InjectedState],
+    ):
+        """
+        Reads a previously saved note from persistent memory by its key.
+        """
+        content = state.scratchpad.get(key)
+
+        if content is not None:
+            agent_outcome = read_note_wrapper.on_success_fn(key, content)
+            status = "success"
+        else:
+            agent_outcome = read_note_wrapper.on_failure_fn(key)
+            status = "error"
+
+        tool_message = ToolMessage(
+            tool_call_id=tool_call_id,
+            content=agent_outcome,
+            status=status,
+        )
+        return Command(
+            update=await state.asanitize_update(
+                ctx=ctx,
+                update={
+                    "agents_thoughts": [agent_thought, agent_outcome],
+                    EXECUTOR_MESSAGES_KEY: [tool_message],
+                },
+                agent="executor",
+            ),
+        )
+
+    return read_note
+
+
+def get_list_notes_tool(ctx: MobileUseContext) -> BaseTool:
+    @tool
+    async def list_notes(
+        agent_thought: str,
+        tool_call_id: Annotated[str, InjectedToolCallId],
+        state: Annotated[State, InjectedState],
+    ):
+        """
+        Lists all note keys currently stored in persistent memory.
+        """
+        keys = list(state.scratchpad.keys())
+
+        agent_outcome = list_notes_wrapper.on_success_fn(keys)
+
+        tool_message = ToolMessage(
+            tool_call_id=tool_call_id,
+            content=agent_outcome,
+            status="success",
+        )
+        return Command(
+            update=await state.asanitize_update(
+                ctx=ctx,
+                update={
+                    "agents_thoughts": [agent_thought, agent_outcome],
+                    EXECUTOR_MESSAGES_KEY: [tool_message],
+                },
+                agent="executor",
+            ),
+        )
+
+    return list_notes
+
+
+save_note_wrapper = ToolWrapper(
+    tool_fn_getter=get_save_note_tool,
+    on_success_fn=lambda key: (f"Successfully saved note '{key}'."),
+    on_failure_fn=lambda key: f"Failed to save note '{key}'.",
+)
+
+read_note_wrapper = ToolWrapper(
+    tool_fn_getter=get_read_note_tool,
+    on_success_fn=lambda key, content: (
+        f"Successfully read note '{key}'. '{key}' note full content: {content}"
+    ),
+    on_failure_fn=lambda key: f"Note '{key}' not found in scratchpad.",
+)
+
+list_notes_wrapper = ToolWrapper(
+    tool_fn_getter=get_list_notes_tool,
+    on_success_fn=lambda keys: (
+        f"Here are all the note keys: {keys}" if keys else "No notes saved yet."
+    ),
+    on_failure_fn=lambda: "Failed to list notes.",
+)