minitap-mobile-use 0.0.1.dev0__py3-none-any.whl

minitap/mobile_use/utils/errors.py

@@ -0,0 +1,6 @@
+class ControllerErrors(Exception):
+    def __init__(self, message: str):
+        self.message = message
+
+    def __str__(self):
+        return self.message

minitap/mobile_use/utils/file.py

@@ -0,0 +1,13 @@
+import json
+import re
+from typing import IO
+
+
+def strip_json_comments(text: str) -> str:
+    text = re.sub(r"//.*?$", "", text, flags=re.MULTILINE)
+    text = re.sub(r"/\*.*?\*/", "", text, flags=re.DOTALL)
+    return text
+
+
+def load_jsonc(file: IO) -> dict:
+    return json.loads(strip_json_comments(file.read()))

minitap/mobile_use/utils/logger.py

@@ -0,0 +1,184 @@
+import logging
+import sys
+from enum import Enum
+from pathlib import Path
+from typing import Optional, Union
+
+from colorama import Fore, Style, init
+
+init(autoreset=True)
+
+
+class LogLevel(Enum):
+    DEBUG = ("DEBUG", Fore.MAGENTA, "🔍")
+    INFO = ("INFO", Fore.WHITE, "ℹ")
+    SUCCESS = ("SUCCESS", Fore.GREEN, "✓")
+    WARNING = ("WARNING", Fore.YELLOW, "⚠")
+    ERROR = ("ERROR", Fore.RED, "❌")
+    CRITICAL = ("CRITICAL", Fore.RED + Style.BRIGHT, "💥")
+
+
+class MobileUseLogger:
+    def __init__(
+        self,
+        name: str,
+        log_file: Optional[Union[str, Path]] = None,
+        console_level: str = "INFO",
+        file_level: str = "DEBUG",
+        enable_file_logging: bool = True,
+    ):
+        """
+        Initialize the MobileUse logger.
+
+        Args:
+            name: Logger name (usually __name__)
+            log_file: Path to log file (defaults to logs/{name}.log)
+            console_level: Minimum level for console output
+            file_level: Minimum level for file output
+            enable_file_logging: Whether to enable file logging
+        """
+        self.name = name
+        self.logger = logging.getLogger(name)
+        self.logger.setLevel(logging.DEBUG)
+
+        self.logger.handlers.clear()
+
+        self._setup_console_handler(console_level)
+
+        if enable_file_logging:
+            self._setup_file_handler(log_file, file_level)
+
+    def _setup_console_handler(self, level: str):
+        console_handler = logging.StreamHandler(sys.stdout)
+        console_handler.setLevel(getattr(logging, level.upper()))
+
+        console_formatter = ColoredFormatter()
+        console_handler.setFormatter(console_formatter)
+
+        self.logger.addHandler(console_handler)
+
+    def _setup_file_handler(self, log_file: Optional[Union[str, Path]], level: str):
+        if log_file is None:
+            log_file = Path("logs") / f"{self.name.replace('.', '_')}.log"
+
+        log_file = Path(log_file)
+        log_file.parent.mkdir(parents=True, exist_ok=True)
+
+        file_handler = logging.FileHandler(log_file, encoding="utf-8")
+        file_handler.setLevel(getattr(logging, level.upper()))
+
+        file_formatter = logging.Formatter(
+            fmt="%(asctime)s | %(name)s | %(levelname)s | %(message)s", datefmt="%Y-%m-%d %H:%M:%S"
+        )
+        file_handler.setFormatter(file_formatter)
+
+        self.logger.addHandler(file_handler)
+
+    def debug(self, message: str, **kwargs):
+        self.logger.debug(message, extra={"log_level": LogLevel.DEBUG}, **kwargs)
+
+    def info(self, message: str, **kwargs):
+        self.logger.info(message, extra={"log_level": LogLevel.INFO}, **kwargs)
+
+    def success(self, message: str, **kwargs):
+        self.logger.info(message, extra={"log_level": LogLevel.SUCCESS}, **kwargs)
+
+    def warning(self, message: str, **kwargs):
+        self.logger.warning(message, extra={"log_level": LogLevel.WARNING}, **kwargs)
+
+    def error(self, message: str, **kwargs):
+        self.logger.error(message, extra={"log_level": LogLevel.ERROR}, **kwargs)
+
+    def critical(self, message: str, **kwargs):
+        self.logger.critical(message, extra={"log_level": LogLevel.CRITICAL}, **kwargs)
+
+    def header(self, message: str, **_kwargs):
+        separator = "=" * 60
+        colored_separator = f"{Fore.CYAN}{separator}{Style.RESET_ALL}"
+        colored_message = f"{Fore.CYAN}{message}{Style.RESET_ALL}"
+
+        print(colored_separator)
+        print(colored_message)
+        print(colored_separator)
+        self.logger.info(f"\n{separator}\n{message}\n{separator}")
+
+
+class ColoredFormatter(logging.Formatter):
+    def format(self, record):
+        log_level = getattr(record, "log_level", LogLevel.INFO)
+        _level_name, color, symbol = log_level.value
+
+        colored_message = f"{color}{symbol} {record.getMessage()}{Style.RESET_ALL}"
+
+        return colored_message
+
+
+_loggers = {}
+
+
+def get_logger(
+    name: str,
+    log_file: Optional[Union[str, Path]] = None,
+    console_level: str = "INFO",
+    file_level: str = "DEBUG",
+    enable_file_logging: bool = False,
+) -> MobileUseLogger:
+    """
+    Get or create a logger instance.
+
+    Args:
+        name: Logger name (usually __name__)
+        log_file: Path to log file (defaults to logs/{name}.log)
+        console_level: Minimum level for console output
+        file_level: Minimum level for file output
+        enable_file_logging: Whether to enable file logging
+
+    Returns:
+        MobileUseLogger instance
+    """
+    if name not in _loggers:
+        _loggers[name] = MobileUseLogger(
+            name=name,
+            log_file=log_file,
+            console_level=console_level,
+            file_level=file_level,
+            enable_file_logging=enable_file_logging,
+        )
+
+    return _loggers[name]
+
+
+def log_debug(message: str, logger_name: str = "mobile-use"):
+    get_logger(logger_name).debug(message)
+
+
+def log_info(message: str, logger_name: str = "mobile-use"):
+    get_logger(logger_name).info(message)
+
+
+def log_success(message: str, logger_name: str = "mobile-use"):
+    get_logger(logger_name).success(message)
+
+
+def log_warning(message: str, logger_name: str = "mobile-use"):
+    get_logger(logger_name).warning(message)
+
+
+def log_error(message: str, logger_name: str = "mobile-use"):
+    get_logger(logger_name).error(message)
+
+
+def log_critical(message: str, logger_name: str = "mobile-use"):
+    get_logger(logger_name).critical(message)
+
+
+def log_header(message: str, logger_name: str = "mobile-use"):
+    get_logger(logger_name).header(message)
+
+
+def get_server_logger() -> MobileUseLogger:
+    return get_logger(
+        name="mobile-use.servers",
+        console_level="INFO",
+        file_level="DEBUG",
+    )

minitap/mobile_use/utils/media.py

@@ -0,0 +1,73 @@
+import base64
+import json
+from io import BytesIO
+from pathlib import Path
+
+from PIL import Image
+
+
+def compress_base64_jpeg(base64_str: str, quality: int = 50) -> str:
+    if base64_str.startswith("data:image"):
+        base64_str = base64_str.split(",")[1]
+
+    image_data = base64.b64decode(base64_str)
+    image = Image.open(BytesIO(image_data))
+
+    compressed_io = BytesIO()
+    image.save(compressed_io, format="JPEG", quality=quality, optimize=True)
+
+    compressed_base64 = base64.b64encode(compressed_io.getvalue()).decode("utf-8")
+    return compressed_base64
+
+
+def create_gif_from_trace_folder(trace_folder_path: Path):
+    images = []
+    image_files = []
+
+    for file in trace_folder_path.iterdir():
+        if file.suffix == ".jpeg":
+            image_files.append(file)
+
+    image_files.sort(key=lambda f: int(f.stem))
+
+    print("Found " + str(len(image_files)) + " images to compile")
+
+    for file in image_files:
+        with open(file, "rb") as f:
+            image = Image.open(f).convert("RGB")
+            images.append(image)
+
+    if len(images) == 0:
+        return
+
+    gif_path = trace_folder_path / "trace.gif"
+    images[0].save(gif_path, save_all=True, append_images=images[1:], loop=0)
+    print("GIF created at " + str(gif_path))
+
+
+def remove_images_from_trace_folder(trace_folder_path: Path):
+    for file in trace_folder_path.iterdir():
+        if file.suffix == ".jpeg":
+            file.unlink()
+
+
+def create_steps_json_from_trace_folder(trace_folder_path: Path):
+    steps = []
+    for file in trace_folder_path.iterdir():
+        if file.suffix == ".json":
+            with open(file, "r", encoding="utf-8", errors="ignore") as f:
+                json_content = f.read()
+                steps.append({"timestamp": int(file.stem), "data": json_content})
+
+    steps.sort(key=lambda f: f["timestamp"])
+
+    print("Found " + str(len(steps)) + " steps to compile")
+
+    with open(trace_folder_path / "steps.json", "w", encoding="utf-8", errors="ignore") as f:
+        f.write(json.dumps(steps))
+
+
+def remove_steps_json_from_trace_folder(trace_folder_path: Path):
+    for file in trace_folder_path.iterdir():
+        if file.suffix == ".json" and file.name != "steps.json":
+            file.unlink()

minitap/mobile_use/utils/recorder.py

@@ -0,0 +1,55 @@
+import base64
+import time
+from pathlib import Path
+
+from langchain_core.messages import BaseMessage
+from minitap.mobile_use.config import record_events
+from minitap.mobile_use.context import MobileUseContext
+from minitap.mobile_use.controllers.mobile_command_controller import take_screenshot
+from minitap.mobile_use.utils.logger import get_logger
+from minitap.mobile_use.utils.media import compress_base64_jpeg
+
+logger = get_logger(__name__)
+
+
+def record_interaction(ctx: MobileUseContext, response: BaseMessage):
+    if not ctx.execution_setup:
+        raise ValueError("No execution setup found")
+
+    logger.info("Recording interaction")
+    screenshot_base64 = take_screenshot(ctx)
+    logger.info("Screenshot taken")
+    try:
+        compressed_screenshot_base64 = compress_base64_jpeg(screenshot_base64, 20)
+    except Exception as e:
+        logger.error(f"Error compressing screenshot: {e}")
+        return "Could not record this interaction"
+    timestamp = time.time()
+    folder = ctx.execution_setup.traces_path.joinpath(ctx.execution_setup.trace_id).resolve()
+    folder.mkdir(parents=True, exist_ok=True)
+    try:
+        with open(
+            folder.joinpath(f"{int(timestamp)}.jpeg").resolve(),
+            "wb",
+        ) as f:
+            f.write(base64.b64decode(compressed_screenshot_base64))
+
+        with open(
+            folder.joinpath(f"{int(timestamp)}.json").resolve(),
+            "w",
+            encoding="utf-8",
+        ) as f:
+            f.write(response.model_dump_json())
+    except Exception as e:
+        logger.error(f"Error recording interaction: {e}")
+    return "Screenshot recorded successfully"
+
+
+def log_agent_thoughts(agents_thoughts: list[str], output_path: Path | None):
+    if len(agents_thoughts) > 0:
+        last_agents_thoughts = agents_thoughts[-1]
+        previous_last_agents_thoughts = agents_thoughts[-2] if len(agents_thoughts) > 1 else None
+        if previous_last_agents_thoughts != last_agents_thoughts:
+            logger.info(f"💭 {last_agents_thoughts}")
+            if output_path:
+                record_events(output_path=output_path, events=agents_thoughts)

minitap/mobile_use/utils/requests_utils.py

@@ -0,0 +1,37 @@
+import requests
+from minitap.mobile_use.utils.logger import get_logger
+
+logger = get_logger(__name__)
+
+
+def curl_from_request(req: requests.PreparedRequest) -> str:
+    """Converts a requests.PreparedRequest object to a valid cURL command string."""
+    command = ["curl", f"-X {req.method}"]
+
+    for key, value in req.headers.items():
+        command.append(f'-H "{key}: {value}"')
+
+    if req.body:
+        body = req.body
+        if isinstance(body, bytes):
+            body = body.decode("utf-8")
+        # Escape single quotes in the body for shell safety
+        body = body.replace("'", "'\\''")
+        command.append(f"-d '{body}'")
+
+    command.append(f"'{req.url}'")
+
+    return " ".join(command)
+
+
+def logging_hook(response, *args, **kwargs):
+    """Hook to log the request as a cURL command."""
+    curl_command = curl_from_request(response.request)
+    logger.debug(f"\n--- cURL Command ---\n{curl_command}\n--------------------")
+
+
+def get_session_with_curl_logging() -> requests.Session:
+    """Returns a requests.Session with cURL logging enabled."""
+    session = requests.Session()
+    session.hooks["response"] = [logging_hook]
+    return session

minitap/mobile_use/utils/shell_utils.py

@@ -0,0 +1,20 @@
+import subprocess
+
+
+def run_shell_command_on_host(command: str) -> str:
+    """Helper to run a shell command on the host and return the output."""
+    try:
+        result = subprocess.run(
+            command,
+            shell=True,
+            check=True,
+            capture_output=True,
+            text=True,
+        )
+        return result.stdout.strip()
+    except subprocess.CalledProcessError as e:
+        # Log the error and stderr for better debugging
+        error_message = (
+            f"Command '{command}' failed with exit code {e.returncode}.\nStderr: {e.stderr.strip()}"
+        )
+        raise RuntimeError(error_message) from e

minitap/mobile_use/utils/time.py

@@ -0,0 +1,6 @@
+from datetime import datetime
+
+
+def convert_timestamp_to_str(ts: float) -> str:
+    dt = datetime.fromtimestamp(ts)
+    return dt.strftime("%Y-%m-%dT%H-%M-%S")

minitap/mobile_use/utils/ui_hierarchy.py

@@ -0,0 +1,30 @@
+from typing import Optional
+
+
+def find_element_by_resource_id(ui_hierarchy: list[dict], resource_id: str) -> Optional[dict]:
+    """
+    Find a UI element by its resource-id in the UI hierarchy.
+
+    Args:
+        ui_hierarchy: List of UI element dictionaries
+        resource_id: The resource-id to search for
+            (e.g., "com.google.android.settings.intelligence:id/open_search_view_edit_text")
+
+    Returns:
+        The complete UI element dictionary if found, None otherwise
+    """
+
+    def search_recursive(elements: list[dict]) -> Optional[dict]:
+        for element in elements:
+            if isinstance(element, dict):
+                if element.get("resourceId") == resource_id:
+                    return element
+
+                children = element.get("children", [])
+                if children:
+                    result = search_recursive(children)
+                    if result:
+                        return result
+        return None
+
+    return search_recursive(ui_hierarchy)