minitap-mobile-use 3.3.0__py3-none-any.whl

minitap/mobile_use/utils/conversations.py

@@ -0,0 +1,31 @@
+from typing import TypeGuard
+
+from langchain_core.messages import AIMessage, BaseMessage, HumanMessage, ToolMessage
+
+
+def is_ai_message(message: BaseMessage) -> TypeGuard[AIMessage]:
+    return isinstance(message, AIMessage)
+
+
+def is_human_message(message: BaseMessage) -> TypeGuard[HumanMessage]:
+    return isinstance(message, HumanMessage)
+
+
+def is_tool_message(message: BaseMessage) -> TypeGuard[ToolMessage]:
+    return isinstance(message, ToolMessage)
+
+
+def is_tool_for_name(tool_message: ToolMessage, name: str) -> bool:
+    return tool_message.name == name
+
+
+def get_screenshot_message_for_llm(screenshot_base64: str):
+    prefix = "" if screenshot_base64.startswith("data:image") else "data:image/jpeg;base64,"
+    return HumanMessage(
+        content=[
+            {
+                "type": "image_url",
+                "image_url": {"url": f"{prefix}{screenshot_base64}"},
+            }
+        ]
+    )

minitap/mobile_use/utils/decorators.py

@@ -0,0 +1,124 @@
+import asyncio
+from functools import wraps
+from typing import Any, TypeVar, cast, overload
+from collections.abc import Awaitable, Callable
+
+R = TypeVar("R")
+
+
+def wrap_with_callbacks_sync(
+    fn: Callable[..., R],
+    *,
+    before: Callable[..., None] | None = None,
+    on_success: Callable[[R], None] | None = None,
+    on_failure: Callable[[Exception], None] | None = None,
+    suppress_exceptions: bool = False,
+) -> Callable[..., R]:
+    @wraps(fn)
+    def wrapper(*args: Any, **kwargs: Any) -> R:
+        if before:
+            before()
+        try:
+            result = fn(*args, **kwargs)
+            if on_success:
+                on_success(result)
+            return result
+        except Exception as e:
+            if on_failure:
+                on_failure(e)
+            if suppress_exceptions:
+                return None  # type: ignore
+            raise
+
+    return wrapper
+
+
+def wrap_with_callbacks_async(
+    fn: Callable[..., Awaitable[R]],
+    *,
+    before: Callable[..., None] | None = None,
+    on_success: Callable[[R], None] | None = None,
+    on_failure: Callable[[Exception], None] | None = None,
+    suppress_exceptions: bool = False,
+) -> Callable[..., Awaitable[R]]:
+    @wraps(fn)
+    async def wrapper(*args: Any, **kwargs: Any) -> R:
+        if before:
+            before()
+        try:
+            result = await fn(*args, **kwargs)
+            if on_success:
+                on_success(result)
+            return result
+        except Exception as e:
+            if on_failure:
+                on_failure(e)
+            if suppress_exceptions:
+                return None  # type: ignore
+            raise
+
+    return wrapper
+
+
+@overload
+def wrap_with_callbacks(
+    fn: Callable[..., Awaitable[R]],
+    *,
+    before: Callable[[], None] | None = ...,
+    on_success: Callable[[R], None] | None = ...,
+    on_failure: Callable[[Exception], None] | None = ...,
+    suppress_exceptions: bool = ...,
+) -> Callable[..., Awaitable[R]]: ...
+
+
+@overload
+def wrap_with_callbacks(
+    *,
+    before: Callable[..., None] | None = ...,
+    on_success: Callable[[Any], None] | None = ...,
+    on_failure: Callable[[Exception], None] | None = ...,
+    suppress_exceptions: bool = ...,
+) -> Callable[[Callable[..., R]], Callable[..., R]]: ...
+
+
+@overload
+def wrap_with_callbacks(
+    fn: Callable[..., R],
+    *,
+    before: Callable[[], None] | None = ...,
+    on_success: Callable[[R], None] | None = ...,
+    on_failure: Callable[[Exception], None] | None = ...,
+    suppress_exceptions: bool = ...,
+) -> Callable[..., R]: ...
+
+
+def wrap_with_callbacks(
+    fn: Callable[..., Any] | None = None,
+    *,
+    before: Callable[[], None] | None = None,
+    on_success: Callable[[Any], None] | None = None,
+    on_failure: Callable[[Exception], None] | None = None,
+    suppress_exceptions: bool = False,
+) -> Any:
+    def decorator(func: Callable[..., Any]) -> Any:
+        if asyncio.iscoroutinefunction(func):
+            return wrap_with_callbacks_async(
+                cast(Callable[..., Awaitable[Any]], func),
+                before=before,
+                on_success=on_success,
+                on_failure=on_failure,
+                suppress_exceptions=suppress_exceptions,
+            )
+        else:
+            return wrap_with_callbacks_sync(
+                cast(Callable[..., Any], func),
+                before=before,
+                on_success=on_success,
+                on_failure=on_failure,
+                suppress_exceptions=suppress_exceptions,
+            )
+
+    if fn is None:
+        return decorator
+    else:
+        return decorator(fn)

minitap/mobile_use/utils/errors.py

@@ -0,0 +1,6 @@
+class ControllerErrors(Exception):
+    def __init__(self, message: str):
+        self.message = message
+
+    def __str__(self):
+        return self.message

minitap/mobile_use/utils/file.py

@@ -0,0 +1,13 @@
+import json
+import re
+from typing import IO
+
+
+def strip_json_comments(text: str) -> str:
+    text = re.sub(r"//.*?$", "", text, flags=re.MULTILINE)
+    text = re.sub(r"/\*.*?\*/", "", text, flags=re.DOTALL)
+    return text
+
+
+def load_jsonc(file: IO) -> dict:
+    return json.loads(strip_json_comments(file.read()))

minitap/mobile_use/utils/logger.py

@@ -0,0 +1,183 @@
+import logging
+import sys
+from enum import Enum
+from pathlib import Path
+
+from colorama import Fore, Style, init
+
+init(autoreset=True)
+
+
+class LogLevel(Enum):
+    DEBUG = ("DEBUG", Fore.MAGENTA, "🔍")
+    INFO = ("INFO", Fore.WHITE, "ℹ")
+    SUCCESS = ("SUCCESS", Fore.GREEN, "✓")
+    WARNING = ("WARNING", Fore.YELLOW, "⚠")
+    ERROR = ("ERROR", Fore.RED, "❌")
+    CRITICAL = ("CRITICAL", Fore.RED + Style.BRIGHT, "💥")
+
+
+class MobileUseLogger:
+    def __init__(
+        self,
+        name: str,
+        log_file: str | Path | None = None,
+        console_level: str = "INFO",
+        file_level: str = "DEBUG",
+        enable_file_logging: bool = True,
+    ):
+        """
+        Initialize the MobileUse logger.
+
+        Args:
+            name: Logger name (usually __name__)
+            log_file: Path to log file (defaults to logs/{name}.log)
+            console_level: Minimum level for console output
+            file_level: Minimum level for file output
+            enable_file_logging: Whether to enable file logging
+        """
+        self.name = name
+        self.logger = logging.getLogger(name)
+        self.logger.setLevel(logging.DEBUG)
+
+        self.logger.handlers.clear()
+
+        self._setup_console_handler(console_level)
+
+        if enable_file_logging:
+            self._setup_file_handler(log_file, file_level)
+
+    def _setup_console_handler(self, level: str):
+        console_handler = logging.StreamHandler(sys.stdout)
+        console_handler.setLevel(getattr(logging, level.upper()))
+
+        console_formatter = ColoredFormatter()
+        console_handler.setFormatter(console_formatter)
+
+        self.logger.addHandler(console_handler)
+
+    def _setup_file_handler(self, log_file: str | Path | None, level: str):
+        if log_file is None:
+            log_file = Path("logs") / f"{self.name.replace('.', '_')}.log"
+
+        log_file = Path(log_file)
+        log_file.parent.mkdir(parents=True, exist_ok=True)
+
+        file_handler = logging.FileHandler(log_file, encoding="utf-8")
+        file_handler.setLevel(getattr(logging, level.upper()))
+
+        file_formatter = logging.Formatter(
+            fmt="%(asctime)s | %(name)s | %(levelname)s | %(message)s", datefmt="%Y-%m-%d %H:%M:%S"
+        )
+        file_handler.setFormatter(file_formatter)
+
+        self.logger.addHandler(file_handler)
+
+    def debug(self, message: str, **kwargs):
+        self.logger.debug(message, extra={"log_level": LogLevel.DEBUG}, **kwargs)
+
+    def info(self, message: str, **kwargs):
+        self.logger.info(message, extra={"log_level": LogLevel.INFO}, **kwargs)
+
+    def success(self, message: str, **kwargs):
+        self.logger.info(message, extra={"log_level": LogLevel.SUCCESS}, **kwargs)
+
+    def warning(self, message: str, **kwargs):
+        self.logger.warning(message, extra={"log_level": LogLevel.WARNING}, **kwargs)
+
+    def error(self, message: str, **kwargs):
+        self.logger.error(message, extra={"log_level": LogLevel.ERROR}, **kwargs)
+
+    def critical(self, message: str, **kwargs):
+        self.logger.critical(message, extra={"log_level": LogLevel.CRITICAL}, **kwargs)
+
+    def header(self, message: str, **_kwargs):
+        separator = "=" * 60
+        colored_separator = f"{Fore.CYAN}{separator}{Style.RESET_ALL}"
+        colored_message = f"{Fore.CYAN}{message}{Style.RESET_ALL}"
+
+        print(colored_separator)
+        print(colored_message)
+        print(colored_separator)
+        self.logger.info(f"\n{separator}\n{message}\n{separator}")
+
+
+class ColoredFormatter(logging.Formatter):
+    def format(self, record):
+        log_level = getattr(record, "log_level", LogLevel.INFO)
+        _level_name, color, symbol = log_level.value
+
+        colored_message = f"{color}{symbol} {record.getMessage()}{Style.RESET_ALL}"
+
+        return colored_message
+
+
+_loggers = {}
+
+
+def get_logger(
+    name: str,
+    log_file: str | Path | None = None,
+    console_level: str = "INFO",
+    file_level: str = "DEBUG",
+    enable_file_logging: bool = False,
+) -> MobileUseLogger:
+    """
+    Get or create a logger instance.
+
+    Args:
+        name: Logger name (usually __name__)
+        log_file: Path to log file (defaults to logs/{name}.log)
+        console_level: Minimum level for console output
+        file_level: Minimum level for file output
+        enable_file_logging: Whether to enable file logging
+
+    Returns:
+        MobileUseLogger instance
+    """
+    if name not in _loggers:
+        _loggers[name] = MobileUseLogger(
+            name=name,
+            log_file=log_file,
+            console_level=console_level,
+            file_level=file_level,
+            enable_file_logging=enable_file_logging,
+        )
+
+    return _loggers[name]
+
+
+def log_debug(message: str, logger_name: str = "mobile-use"):
+    get_logger(logger_name).debug(message)
+
+
+def log_info(message: str, logger_name: str = "mobile-use"):
+    get_logger(logger_name).info(message)
+
+
+def log_success(message: str, logger_name: str = "mobile-use"):
+    get_logger(logger_name).success(message)
+
+
+def log_warning(message: str, logger_name: str = "mobile-use"):
+    get_logger(logger_name).warning(message)
+
+
+def log_error(message: str, logger_name: str = "mobile-use"):
+    get_logger(logger_name).error(message)
+
+
+def log_critical(message: str, logger_name: str = "mobile-use"):
+    get_logger(logger_name).critical(message)
+
+
+def log_header(message: str, logger_name: str = "mobile-use"):
+    get_logger(logger_name).header(message)
+
+
+def get_server_logger() -> MobileUseLogger:
+    return get_logger(
+        name="mobile-use.servers",
+        console_level="INFO",
+        file_level="DEBUG",
+    )

minitap/mobile_use/utils/media.py

@@ -0,0 +1,186 @@
+import json
+import os
+import subprocess
+import tempfile
+from pathlib import Path
+
+from PIL import Image
+
+USE_FFMPEG_GIF = os.environ.get("USE_FFMPEG_GIF", "").lower() in ("1", "true", "yes")
+
+
+def quantize_and_save_gif_from_paths(
+    image_paths: list[Path],
+    output_path: Path,
+    colors: int = 128,
+    duration: int = 100,
+) -> None:
+    """
+    Create an optimized GIF from image file paths.
+
+    By default uses PIL (loads all frames into memory).
+    Set USE_FFMPEG_GIF=1 env var to use ffmpeg for memory-efficient streaming.
+
+    Args:
+        image_paths: List of paths to image files (must be sorted in desired order)
+        output_path: Path where the GIF will be saved
+        colors: Number of colors to use in quantization (lower = smaller file)
+        duration: Duration of each frame in milliseconds
+
+    Raises:
+        ValueError: If image_paths list is empty
+        RuntimeError: If ffmpeg fails (when USE_FFMPEG_GIF is enabled)
+    """
+    if not image_paths:
+        raise ValueError("image_paths must not be empty")
+
+    if USE_FFMPEG_GIF:
+        _save_gif_ffmpeg(image_paths, output_path, colors, duration)
+    else:
+        _save_gif_pillow(image_paths, output_path, colors, duration)
+
+
+def _save_gif_pillow(
+    image_paths: list[Path],
+    output_path: Path,
+    colors: int,
+    duration: int,
+) -> None:
+    """Create GIF using PIL (loads all frames into memory)."""
+
+    def frame_generator():
+        for path in image_paths:
+            with Image.open(path) as img:
+                if img.mode != "RGB":
+                    img = img.convert("RGB")
+                yield img.quantize(colors=colors, method=2)
+
+    frames = frame_generator()
+    first_frame = next(frames)
+    first_frame.save(
+        output_path,
+        save_all=True,
+        append_images=frames,
+        loop=0,
+        optimize=True,
+        duration=duration,
+    )
+
+
+def _save_gif_ffmpeg(
+    image_paths: list[Path],
+    output_path: Path,
+    colors: int,
+    duration: int,
+) -> None:
+    """Create GIF using ffmpeg (memory-efficient streaming)."""
+    fps = 1000 / duration
+
+    with tempfile.NamedTemporaryFile(mode="w", suffix=".txt", delete=False) as f:
+        for i, path in enumerate(image_paths):
+            f.write(f"file '{path.absolute()}'\n")
+            # Last file needs no duration (ffmpeg concat demuxer uses it as the final frame)
+            if i < len(image_paths) - 1:
+                f.write(f"duration {duration / 1000}\n")
+        # Repeat last file to ensure it's included (ffmpeg concat demuxer quirk)
+        f.write(f"file '{image_paths[-1].absolute()}'\n")
+        concat_file = Path(f.name)
+
+    palette_path = output_path.with_suffix(".palette.png")
+
+    try:
+        result = subprocess.run(
+            [
+                "ffmpeg",
+                "-y",
+                "-f",
+                "concat",
+                "-safe",
+                "0",
+                "-i",
+                str(concat_file),
+                "-vf",
+                f"palettegen=max_colors={min(colors, 256)}:stats_mode=diff",
+                str(palette_path),
+            ],
+            capture_output=True,
+            text=True,
+        )
+        if result.returncode != 0:
+            raise RuntimeError(f"ffmpeg palette generation failed: {result.stderr}")
+
+        result = subprocess.run(
+            [
+                "ffmpeg",
+                "-y",
+                "-f",
+                "concat",
+                "-safe",
+                "0",
+                "-i",
+                str(concat_file),
+                "-i",
+                str(palette_path),
+                "-lavfi",
+                f"fps={fps},paletteuse=dither=bayer:bayer_scale=5",
+                "-loop",
+                "0",
+                str(output_path),
+            ],
+            capture_output=True,
+            text=True,
+        )
+        if result.returncode != 0:
+            raise RuntimeError(f"ffmpeg GIF creation failed: {result.stderr}")
+
+    finally:
+        concat_file.unlink(missing_ok=True)
+        if palette_path.exists():
+            palette_path.unlink()
+
+
+def create_gif_from_trace_folder(trace_folder_path: Path):
+    image_files: list[Path] = []
+
+    for file in trace_folder_path.iterdir():
+        if file.suffix == ".jpeg":
+            image_files.append(file)
+
+    image_files.sort(key=lambda f: int(f.stem))
+
+    print(f"Found {len(image_files)} images to compile")
+
+    if not image_files:
+        return
+
+    gif_path = trace_folder_path / "trace.gif"
+    quantize_and_save_gif_from_paths(image_files, gif_path)
+    print(f"GIF created at {gif_path}")
+
+
+def remove_images_from_trace_folder(trace_folder_path: Path):
+    for file in trace_folder_path.iterdir():
+        if file.suffix == ".jpeg":
+            file.unlink()
+
+
+def create_steps_json_from_trace_folder(trace_folder_path: Path):
+    steps = []
+    for file in trace_folder_path.iterdir():
+        if file.suffix == ".json":
+            with open(file, encoding="utf-8", errors="ignore") as f:
+                json_content = f.read()
+                steps.append({"timestamp": int(file.stem), "data": json_content})
+
+    steps.sort(key=lambda f: f["timestamp"])
+
+    print("Found " + str(len(steps)) + " steps to compile")
+
+    with open(trace_folder_path / "steps.json", "w", encoding="utf-8", errors="ignore") as f:
+        f.write(json.dumps(steps))
+
+
+def remove_steps_json_from_trace_folder(trace_folder_path: Path):
+    for file in trace_folder_path.iterdir():
+        if file.suffix == ".json" and file.name != "steps.json":
+            file.unlink()

minitap/mobile_use/utils/recorder.py

@@ -0,0 +1,52 @@
+import base64
+import time
+
+from colorama import Fore, Style
+from langchain_core.messages import BaseMessage
+
+from minitap.mobile_use.context import MobileUseContext
+from minitap.mobile_use.controllers.controller_factory import create_device_controller
+from minitap.mobile_use.utils.logger import get_logger
+
+logger = get_logger(__name__)
+
+
+async def record_interaction(ctx: MobileUseContext, response: BaseMessage):
+    if not ctx.execution_setup:
+        raise ValueError("No execution setup found")
+    if not ctx.execution_setup.traces_path or not ctx.execution_setup.trace_name:
+        raise ValueError("No traces path or trace name found")
+
+    logger.info("Recording interaction")
+    controller = create_device_controller(ctx)
+    screenshot_base64 = await controller.screenshot()
+    logger.info("Screenshot taken")
+    try:
+        controller = create_device_controller(ctx)
+        compressed_screenshot_base64 = controller.get_compressed_b64_screenshot(screenshot_base64)
+    except Exception as e:
+        logger.error(f"Error compressing screenshot: {e}")
+        return "Could not record this interaction"
+    timestamp = time.time()
+    folder = ctx.execution_setup.traces_path.joinpath(ctx.execution_setup.trace_name).resolve()
+    folder.mkdir(parents=True, exist_ok=True)
+    try:
+        with open(
+            folder.joinpath(f"{int(timestamp)}.jpeg").resolve(),
+            "wb",
+        ) as f:
+            f.write(base64.b64decode(compressed_screenshot_base64))
+
+        with open(
+            folder.joinpath(f"{int(timestamp)}.json").resolve(),
+            "w",
+            encoding="utf-8",
+        ) as f:
+            f.write(response.model_dump_json())
+    except Exception as e:
+        logger.error(f"Error recording interaction: {e}")
+    return "Screenshot recorded successfully"
+
+
+def log_agent_thought(agent_thought: str):
+    logger.info(f"💭 {Fore.LIGHTMAGENTA_EX}{agent_thought}{Style.RESET_ALL}")

minitap/mobile_use/utils/requests_utils.py

@@ -0,0 +1,37 @@
+import requests
+from minitap.mobile_use.utils.logger import get_logger
+
+logger = get_logger(__name__)
+
+
+def curl_from_request(req: requests.PreparedRequest) -> str:
+    """Converts a requests.PreparedRequest object to a valid cURL command string."""
+    command = ["curl", f"-X {req.method}"]
+
+    for key, value in req.headers.items():
+        command.append(f'-H "{key}: {value}"')
+
+    if req.body:
+        body = req.body
+        if isinstance(body, bytes):
+            body = body.decode("utf-8")
+        # Escape single quotes in the body for shell safety
+        body = body.replace("'", "'\\''")
+        command.append(f"-d '{body}'")
+
+    command.append(f"'{req.url}'")
+
+    return " ".join(command)
+
+
+def logging_hook(response, *args, **kwargs):
+    """Hook to log the request as a cURL command."""
+    curl_command = curl_from_request(response.request)
+    logger.debug(f"\n--- cURL Command ---\n{curl_command}\n--------------------")
+
+
+def get_session_with_curl_logging() -> requests.Session:
+    """Returns a requests.Session with cURL logging enabled."""
+    session = requests.Session()
+    session.hooks["response"] = [logging_hook]
+    return session

minitap/mobile_use/utils/shell_utils.py

@@ -0,0 +1,20 @@
+import subprocess
+
+
+def run_shell_command_on_host(command: str) -> str:
+    """Helper to run a shell command on the host and return the output."""
+    try:
+        result = subprocess.run(
+            command,
+            shell=True,
+            check=True,
+            capture_output=True,
+            text=True,
+        )
+        return result.stdout.strip()
+    except subprocess.CalledProcessError as e:
+        # Log the error and stderr for better debugging
+        error_message = (
+            f"Command '{command}' failed with exit code {e.returncode}.\nStderr: {e.stderr.strip()}"
+        )
+        raise RuntimeError(error_message) from e