minitap-mobile-use 3.3.0__py3-none-any.whl

minitap/mobile_use/utils/test_ui_hierarchy.py

@@ -0,0 +1,178 @@
+from unittest.mock import patch
+
+from minitap.mobile_use.utils.ui_hierarchy import (
+    ElementBounds,
+    Point,
+    find_element_by_resource_id,
+    get_bounds_for_element,
+    get_element_text,
+    is_element_focused,
+    text_input_is_empty,
+)
+
+
+def test_text_input_is_empty():
+    assert text_input_is_empty(text=None, hint_text=None)
+    assert text_input_is_empty(text="", hint_text=None)
+    assert text_input_is_empty(text="", hint_text="")
+    assert text_input_is_empty(text="text", hint_text="text")
+
+    assert not text_input_is_empty(text="text", hint_text=None)
+    assert not text_input_is_empty(text="text", hint_text="")
+
+
+def test_find_element_by_resource_id():
+    ui_hierarchy = [
+        {"resourceId": "com.example:id/button1", "text": "Button 1", "children": []},
+        {
+            "resourceId": "com.example:id/container",
+            "children": [
+                {
+                    "resourceId": "com.example:id/nested_button",
+                    "text": "Nested Button",
+                    "children": [],
+                }
+            ],
+        },
+    ]
+
+    result = find_element_by_resource_id(ui_hierarchy, "com.example:id/button1")
+    assert result is not None
+    assert result["resourceId"] == "com.example:id/button1"
+    assert result["text"] == "Button 1"
+
+    result = find_element_by_resource_id(ui_hierarchy, "com.example:id/nested_button")
+    assert result is not None
+    assert result["resourceId"] == "com.example:id/nested_button"
+    assert result["text"] == "Nested Button"
+
+    result = find_element_by_resource_id(ui_hierarchy, "com.example:id/nonexistent")
+    assert result is None
+
+    result = find_element_by_resource_id([], "com.example:id/button1")
+    assert result is None
+
+
+def test_find_element_by_resource_id_rich_hierarchy():
+    rich_hierarchy = [
+        {"attributes": {"resource-id": "com.example:id/button1"}, "children": []},
+        {
+            "attributes": {"resource-id": "com.example:id/container"},
+            "children": [
+                {"attributes": {"resource-id": "com.example:id/nested_button"}, "children": []}
+            ],
+        },
+    ]
+
+    result = find_element_by_resource_id(
+        rich_hierarchy, "com.example:id/button1", is_rich_hierarchy=True
+    )
+    assert result is not None
+    assert result["resource-id"] == "com.example:id/button1"
+
+    result = find_element_by_resource_id(
+        rich_hierarchy, "com.example:id/nested_button", is_rich_hierarchy=True
+    )
+    assert result is not None
+    assert result["resource-id"] == "com.example:id/nested_button"
+
+    result = find_element_by_resource_id(
+        rich_hierarchy, "com.example:id/nonexistent", is_rich_hierarchy=True
+    )
+    assert result is None
+
+
+def test_is_element_focused():
+    focused_element = {"focused": "true"}
+    assert is_element_focused(focused_element)
+
+    non_focused_element = {"focused": "false"}
+    assert not is_element_focused(non_focused_element)
+
+    no_focused_element = {"text": "some text"}
+    assert not is_element_focused(no_focused_element)
+
+    none_focused_element = {"focused": None}
+    assert not is_element_focused(none_focused_element)
+
+
+def test_get_element_text():
+    element = {"text": "Button Text", "hintText": "Hint Text"}
+    assert get_element_text(element) == "Button Text"
+    assert get_element_text(element, hint_text=False) == "Button Text"
+    assert get_element_text(element, hint_text=True) == "Hint Text"
+
+    element_no_text = {"hintText": "Hint Text"}
+    assert get_element_text(element_no_text) is None
+    assert get_element_text(element_no_text, hint_text=True) == "Hint Text"
+    element_no_hint = {"text": "Button Text"}
+    assert get_element_text(element_no_hint) == "Button Text"
+    assert get_element_text(element_no_hint, hint_text=True) is None
+
+    empty_element = {}
+    assert get_element_text(empty_element) is None
+    assert get_element_text(empty_element, hint_text=True) is None
+
+
+def test_get_bounds_for_element():
+    element_with_bounds = {"bounds": {"x": 10, "y": 20, "width": 100, "height": 50}}
+    bounds = get_bounds_for_element(element_with_bounds)
+    assert bounds is not None
+    assert isinstance(bounds, ElementBounds)
+    assert bounds.x == 10
+    assert bounds.y == 20
+    assert bounds.width == 100
+    assert bounds.height == 50
+
+    element_no_bounds = {"text": "Button"}
+    bounds = get_bounds_for_element(element_no_bounds)
+    assert bounds is None
+
+    # Suppress logger output for the invalid bounds test case
+    with patch("minitap.mobile_use.utils.ui_hierarchy.logger.error"):
+        element_invalid_bounds = {
+            "bounds": {
+                "x": "invalid",  # Should be int
+                "y": 20,
+                "width": 100,
+                "height": 50,
+            }
+        }
+        bounds = get_bounds_for_element(element_invalid_bounds)
+        assert bounds is None
+
+
+def test_element_bounds():
+    bounds = ElementBounds(x=10, y=20, width=100, height=50)
+
+    center = bounds.get_center()
+    assert isinstance(center, Point)
+    assert center.x == 60
+    assert center.y == 45
+
+    center_point = bounds.get_relative_point(0.5, 0.5)
+    assert isinstance(center_point, Point)
+    assert center_point.x == 60
+    assert center_point.y == 45
+
+    top_left = bounds.get_relative_point(0.0, 0.0)
+    assert top_left.x == 10
+    assert top_left.y == 20
+
+    bottom_right = bounds.get_relative_point(1.0, 1.0)
+    assert bottom_right.x == 110
+    assert bottom_right.y == 70
+    custom_point = bounds.get_relative_point(0.95, 0.95)
+    assert custom_point.x == 105
+    assert custom_point.y == 67
+
+
+if __name__ == "__main__":
+    test_text_input_is_empty()
+    test_find_element_by_resource_id()
+    test_find_element_by_resource_id_rich_hierarchy()
+    test_is_element_focused()
+    test_get_element_text()
+    test_get_bounds_for_element()
+    test_element_bounds()
+    print("All tests passed")

minitap/mobile_use/utils/time.py

@@ -0,0 +1,6 @@
+from datetime import datetime
+
+
+def convert_timestamp_to_str(ts: float) -> str:
+    dt = datetime.fromtimestamp(ts)
+    return dt.strftime("%Y-%m-%dT%H-%M-%S")

minitap/mobile_use/utils/ui_hierarchy.py

@@ -0,0 +1,132 @@
+from pydantic import BaseModel, Field
+
+from minitap.mobile_use.utils.logger import get_logger
+
+logger = get_logger(__name__)
+
+
+def __find_element_by_ressource_id_in_rich_hierarchy(
+    hierarchy: list[dict], resource_id: str
+) -> dict | None:
+    """
+    Retrieves all the sibling elements for a given resource ID from a nested dictionary.
+
+    Args:
+      hierarchy (dict): The nested dictionary representing the UI hierarchy.
+      resource_id (str): The resource-id to find.
+
+    Returns:
+      list: A list of the sibling elements, or None if the resource_id is not found.
+    """
+    if not hierarchy:
+        return None
+
+    for child in hierarchy:
+        if child.get("attributes", {}).get("resource-id") == resource_id:
+            return child.get("attributes", {})
+
+    for child in hierarchy:
+        result = __find_element_by_ressource_id_in_rich_hierarchy(
+            child.get("children", []), resource_id
+        )
+        if result is not None:
+            return result
+
+    return None
+
+
+def text_input_is_empty(text: str | None, hint_text: str | None) -> bool:
+    return not text or text == hint_text
+
+
+def find_element_by_resource_id(
+    ui_hierarchy: list[dict],
+    resource_id: str,
+    index: int | None = None,
+    is_rich_hierarchy: bool = False,
+) -> dict | None:
+    """
+    Find a UI element by its resource-id in the UI hierarchy.
+
+    Args:
+        ui_hierarchy: List of UI element dictionaries
+        resource_id: The resource-id to search for
+            (e.g., "com.google.android.settings.intelligence:id/open_search_view_edit_text")
+
+    Returns:
+        The complete UI element dictionary if found, None otherwise
+    """
+    if is_rich_hierarchy:
+        return __find_element_by_ressource_id_in_rich_hierarchy(ui_hierarchy, resource_id)
+
+    def search_recursive(elements: list[dict]) -> dict | None:
+        for element in elements:
+            if isinstance(element, dict):
+                if element.get("resourceId") == resource_id:
+                    idx = index or 0
+                    if idx == 0:
+                        return element
+                    idx -= 1
+                    continue
+
+                children = element.get("children", [])
+                if children:
+                    result = search_recursive(children)
+                    if result:
+                        return result
+        return None
+
+    return search_recursive(ui_hierarchy)
+
+
+def is_element_focused(element: dict) -> bool:
+    return element.get("focused", None) == "true"
+
+
+def get_element_text(element: dict, hint_text: bool = False) -> str | None:
+    if hint_text:
+        return element.get("hintText", None)
+    return element.get("text", None)
+
+
+class Point(BaseModel):
+    x: int
+    y: int
+
+
+class ElementBounds(BaseModel):
+    x: int = Field(description="The x coordinate of the top-left corner of the element.")
+    y: int = Field(description="The y coordinate of the top-left corner of the element.")
+    width: int = Field(description="The width of the element.")
+    height: int = Field(description="The height of the element.")
+
+    def get_center(self) -> Point:
+        return Point(x=self.x + self.width // 2, y=self.y + self.height // 2)
+
+    def get_relative_point(self, x_percent: float, y_percent: float) -> Point:
+        """
+        Returns the coordinates of the point at x_percent of the width and y_percent
+        of the height of the element.
+
+        Ex if x_percent = 0.95 and y_percent = 0.95,
+        the point is at the bottom right of the element:
+        <------>
+        |      |
+        |     x|
+        <------>
+        """
+        return Point(
+            x=int(self.x + self.width * x_percent),
+            y=int(self.y + self.height * y_percent),
+        )
+
+
+def get_bounds_for_element(element: dict) -> ElementBounds | None:
+    bounds = element.get("bounds", None)
+    if bounds:
+        try:
+            return ElementBounds(**bounds)
+        except Exception as e:
+            logger.error(f"Failed to validate bounds: {e}")
+            return None
+    return None

minitap/mobile_use/utils/video.py

@@ -0,0 +1,281 @@
+"""
+Video recording utilities for mobile devices.
+
+Provides shared types and utilities for video recording across platforms.
+"""
+
+import asyncio
+import platform
+import shutil
+from pathlib import Path
+
+from pydantic import BaseModel, ConfigDict
+
+from minitap.mobile_use.utils.logger import get_logger
+
+logger = get_logger(__name__)
+
+DEFAULT_MAX_DURATION_SECONDS = 900  # 15 minutes
+VIDEO_READY_DELAY_SECONDS = 1
+ANDROID_DEVICE_VIDEO_PATH = "/sdcard/screen_recording.mp4"
+ANDROID_MAX_RECORDING_DURATION_SECONDS = 180  # Android screenrecord limit
+
+# Gemini API limits: 20MB for inline requests, but base64 adds ~33% overhead
+# So we target ~14MB to be safe after base64 encoding
+MAX_VIDEO_SIZE_MB = 14
+MAX_VIDEO_SIZE_BYTES = MAX_VIDEO_SIZE_MB * 1024 * 1024
+
+
+class RecordingSession(BaseModel):
+    """Tracks an active video recording session."""
+
+    model_config = ConfigDict(arbitrary_types_allowed=True)
+
+    device_id: str
+    start_time: float
+    process: asyncio.subprocess.Process | None = None
+    local_video_path: Path | None = None
+    android_device_path: str = ANDROID_DEVICE_VIDEO_PATH
+    android_video_segments: list[Path] = []
+    android_segment_index: int = 0
+    android_restart_task: asyncio.Task | None = None
+    errors: list[str] = []
+
+
+class VideoRecordingResult(BaseModel):
+    """Result of a video recording operation."""
+
+    success: bool
+    message: str
+    video_path: Path | None = None
+
+    model_config = ConfigDict(arbitrary_types_allowed=True)
+
+
+# Global session storage - keyed by device_id
+_active_recordings: dict[str, RecordingSession] = {}
+
+
+def get_active_session(device_id: str) -> RecordingSession | None:
+    """Get the active recording session for a device."""
+    return _active_recordings.get(device_id)
+
+
+def set_active_session(device_id: str, session: RecordingSession) -> None:
+    """Set the active recording session for a device."""
+    _active_recordings[device_id] = session
+
+
+def remove_active_session(device_id: str) -> RecordingSession | None:
+    """Remove and return the active recording session for a device."""
+    return _active_recordings.pop(device_id, None)
+
+
+def has_active_session(device_id: str) -> bool:
+    """Check if there's an active recording session for a device."""
+    return device_id in _active_recordings
+
+
+def is_ffmpeg_installed() -> bool:
+    """Check if ffmpeg is available in the system PATH."""
+    return shutil.which("ffmpeg") is not None
+
+
+class FFmpegNotInstalledError(Exception):
+    """Raised when ffmpeg is required but not installed."""
+
+    def __init__(self):
+        os_name = platform.system().lower()
+        if os_name == "darwin":  # macOS
+            install_instructions = "brew install ffmpeg"
+        elif os_name == "windows":
+            install_instructions = "Download from https://www.ffmpeg.org/download.html"
+        else:  # Linux and others
+            install_instructions = (
+                "Install via your package manager (e.g., apt install ffmpeg, "
+                "dnf install ffmpeg) or download from https://www.ffmpeg.org/download.html"
+            )
+
+        message = (
+            f"\n\n❌ ffmpeg is required for video recording but is not installed.\n\n"
+            f"Please install ffmpeg first:\n"
+            f"  → {install_instructions}\n\n"
+            f"After installation, restart mobile-use.\n"
+        )
+        super().__init__(message)
+
+
+def check_ffmpeg_available() -> None:
+    """
+    Check if ffmpeg is installed and raise an error if not.
+
+    Raises:
+        FFmpegNotInstalledError: If ffmpeg is not found in PATH.
+    """
+    if not is_ffmpeg_installed():
+        raise FFmpegNotInstalledError()
+
+
+async def concatenate_videos(segments: list[Path], output_path: Path) -> bool:
+    """Concatenate multiple video segments using ffmpeg."""
+    if not segments:
+        return False
+
+    if len(segments) == 1:
+        shutil.move(segments[0], output_path)
+        return True
+
+    list_file = output_path.parent / "segments.txt"
+    with open(list_file, "w") as f:
+        for segment in segments:
+            f.write(f"file '{segment}'\n")
+
+    try:
+        process = await asyncio.create_subprocess_exec(
+            "ffmpeg",
+            "-y",
+            "-f",
+            "concat",
+            "-safe",
+            "0",
+            "-i",
+            str(list_file),
+            "-c",
+            "copy",
+            str(output_path),
+            stdout=asyncio.subprocess.PIPE,
+            stderr=asyncio.subprocess.PIPE,
+        )
+        await process.wait()
+        return output_path.exists()
+    except Exception as e:
+        logger.error(f"Failed to concatenate videos: {e}")
+        return False
+    finally:
+        if list_file.exists():
+            list_file.unlink()
+
+
+def cleanup_video_segments(segments: list[Path], keep_path: Path | None = None) -> None:
+    """Clean up temporary video segments, optionally keeping one path."""
+    for segment in segments:
+        try:
+            if segment.exists() and segment != keep_path:
+                segment.unlink()
+                if segment.parent.exists() and not any(segment.parent.iterdir()):
+                    segment.parent.rmdir()
+        except Exception:
+            pass
+
+
+async def compress_video_for_api(
+    input_path: Path,
+    target_size_bytes: int = MAX_VIDEO_SIZE_BYTES,
+) -> Path:
+    """
+    Compress a video to fit within API size limits using ffmpeg.
+
+    Uses a two-pass approach:
+    1. First check if video is already small enough
+    2. If not, compress with reduced resolution and bitrate
+
+    Args:
+        input_path: Path to the input video file
+        target_size_bytes: Target maximum file size in bytes
+
+    Returns:
+        Path to the compressed video (may be same as input if no compression needed)
+    """
+    if not input_path.exists():
+        raise FileNotFoundError(f"Video file not found: {input_path}")
+
+    current_size = input_path.stat().st_size
+    logger.info(f"Video size: {current_size / 1024 / 1024:.2f} MB")
+
+    if current_size <= target_size_bytes:
+        logger.info("Video already within size limit, no compression needed")
+        return input_path
+
+    logger.info(f"Compressing video to fit within {target_size_bytes / 1024 / 1024:.1f} MB")
+
+    output_path = input_path.parent / f"compressed_{input_path.name}"
+
+    # Get video duration using ffprobe
+    duration_cmd = [
+        "ffprobe",
+        "-v",
+        "error",
+        "-show_entries",
+        "format=duration",
+        "-of",
+        "default=noprint_wrappers=1:nokey=1",
+        str(input_path),
+    ]
+
+    try:
+        proc = await asyncio.create_subprocess_exec(
+            *duration_cmd,
+            stdout=asyncio.subprocess.PIPE,
+            stderr=asyncio.subprocess.PIPE,
+        )
+        stdout, _ = await proc.communicate()
+        duration = float(stdout.decode().strip())
+    except Exception:
+        duration = 120.0  # Default estimate if probe fails
+
+    # Calculate target bitrate (bits per second)
+    # Leave some margin for container overhead
+    target_bitrate = int((target_size_bytes * 8 * 0.9) / duration)
+    # Ensure minimum quality
+    target_bitrate = max(target_bitrate, 100_000)  # At least 100kbps
+
+    logger.info(f"Target bitrate: {target_bitrate / 1000:.0f} kbps for {duration:.1f}s video")
+
+    # Compress with ffmpeg: reduce resolution to 720p max, use target bitrate
+    compress_cmd = [
+        "ffmpeg",
+        "-y",
+        "-i",
+        str(input_path),
+        "-vf",
+        "scale='min(720,iw)':'-2'",  # Max 720p width, maintain aspect
+        "-c:v",
+        "libx264",
+        "-preset",
+        "fast",
+        "-b:v",
+        str(target_bitrate),
+        "-maxrate",
+        str(int(target_bitrate * 1.5)),
+        "-bufsize",
+        str(int(target_bitrate * 2)),
+        "-c:a",
+        "aac",
+        "-b:a",
+        "64k",
+        str(output_path),
+    ]
+
+    try:
+        proc = await asyncio.create_subprocess_exec(
+            *compress_cmd,
+            stdout=asyncio.subprocess.PIPE,
+            stderr=asyncio.subprocess.PIPE,
+        )
+        _, stderr = await proc.communicate()
+
+        if proc.returncode != 0:
+            logger.error(f"ffmpeg compression failed: {stderr.decode()}")
+            return input_path  # Return original if compression fails
+
+        new_size = output_path.stat().st_size
+        logger.info(
+            f"Compressed: {current_size / 1024 / 1024:.2f} MB -> "
+            f"{new_size / 1024 / 1024:.2f} MB"
+        )
+
+        return output_path
+
+    except Exception as e:
+        logger.error(f"Video compression failed: {e}")
+        return input_path  # Return original if compression fails