minitap-mcp 0.1.0__py3-none-any.whl

minitap/mcp/core/agents.py

@@ -0,0 +1,19 @@
+import os
+
+from minitap.mobile_use.sdk import Agent
+from minitap.mobile_use.sdk.builders import Builders
+
+
+def get_mobile_use_agent():
+    config = Builders.AgentConfig
+    custom_adb_socket = os.getenv("ADB_SERVER_SOCKET")
+    if custom_adb_socket:
+        parts = custom_adb_socket.split(":")
+        if len(parts) != 3:
+            raise ValueError(f"Invalid ADB server socket: {custom_adb_socket}")
+        _, host, port = parts
+        config = config.with_adb_server(host=host, port=int(port))
+    return Agent(config=config.build())
+
+
+agent = get_mobile_use_agent()

minitap/mcp/core/config.py

@@ -0,0 +1,27 @@
+"""Configuration for the MCP server."""
+
+from dotenv import load_dotenv
+from pydantic import Field, SecretStr
+from pydantic_settings import BaseSettings, SettingsConfigDict
+
+# Load environment variables from .env file
+load_dotenv(verbose=True)
+
+
+class MCPSettings(BaseSettings):
+    """Configuration class for MCP server."""
+
+    model_config = SettingsConfigDict(env_file=".env", extra="ignore")
+
+    # Minitap API configuration
+    MINITAP_API_KEY: SecretStr
+    MINITAP_API_BASE_URL: str = Field(default="https://platform.minitap.ai/api/v1")
+
+    VISION_MODEL: str = Field(default="qwen/qwen-2.5-vl-7b-instruct")
+
+    # MCP server configuration (optional, for remote access)
+    MCP_SERVER_HOST: str = Field(default="0.0.0.0")
+    MCP_SERVER_PORT: int = Field(default=8000)
+
+
+settings = MCPSettings()  # type: ignore

minitap/mcp/core/decorators.py

@@ -0,0 +1,42 @@
+"""Decorators for MCP tools."""
+
+import inspect
+from collections.abc import Callable
+from functools import wraps
+from typing import Any, TypeVar
+
+from minitap.mcp.core.device import DeviceNotFoundError
+
+F = TypeVar("F", bound=Callable[..., Any])
+
+
+def handle_tool_errors[T: Callable[..., Any]](func: T) -> T:
+    """
+    Decorator that catches all exceptions in MCP tools and returns error messages.
+
+    This prevents unhandled exceptions from causing infinite loops in the MCP server.
+    """
+
+    @wraps(func)
+    async def async_wrapper(*args: Any, **kwargs: Any) -> Any:
+        try:
+            return await func(*args, **kwargs)
+        except DeviceNotFoundError as e:
+            return f"Error: {str(e)}"
+        except Exception as e:
+            return f"Error in {func.__name__}: {type(e).__name__}: {str(e)}"
+
+    @wraps(func)
+    def sync_wrapper(*args: Any, **kwargs: Any) -> Any:
+        try:
+            return func(*args, **kwargs)
+        except DeviceNotFoundError as e:
+            return f"Error: {str(e)}"
+        except Exception as e:
+            return f"Error in {func.__name__}: {type(e).__name__}: {str(e)}"
+
+    # Check if the function is async
+    if inspect.iscoroutinefunction(func):
+        return async_wrapper  # type: ignore
+    else:
+        return sync_wrapper  # type: ignore

minitap/mcp/core/device.py

@@ -0,0 +1,242 @@
+"""Device detection and screenshot utilities for Android and iOS devices."""
+
+import base64
+import json
+import os
+import subprocess
+import tempfile
+from pathlib import Path
+from typing import Literal
+
+from adbutils import AdbClient, AdbDevice
+from pydantic import BaseModel, ConfigDict
+
+
+DevicePlatform = Literal["android", "ios"]
+
+
+class MobileDevice(BaseModel):
+    """Represents a mobile device with its platform and connection details."""
+
+    model_config = ConfigDict(arbitrary_types_allowed=True)
+
+    device_id: str
+    platform: DevicePlatform
+    adb_device: AdbDevice | None = None  # Only for Android
+
+
+class DeviceInfo(BaseModel):
+    """Serializable device information."""
+
+    device_id: str
+    platform: DevicePlatform
+    name: str | None = None
+    state: str | None = None
+
+
+class DeviceNotFoundError(Exception):
+    """Raised when no device can be found."""
+
+    pass
+
+
+def get_adb_client() -> AdbClient:
+    """Get an ADB client instance."""
+    custom_adb_socket = os.getenv("ADB_SERVER_SOCKET")
+    if not custom_adb_socket:
+        return AdbClient()
+    parts = custom_adb_socket.split(":")
+    if len(parts) != 3:
+        raise ValueError(f"Invalid ADB server socket: {custom_adb_socket}")
+    _, host, port = parts
+    return AdbClient(host=host, port=int(port))
+
+
+def list_available_devices() -> list[DeviceInfo]:
+    """
+    List all available mobile devices (Android and iOS).
+
+    Returns:
+        list[DeviceInfo]: A list of device information objects.
+    """
+    devices: list[DeviceInfo] = []
+
+    # List Android devices
+    try:
+        adb_client = get_adb_client()
+        android_devices = adb_client.device_list()
+
+        for device in android_devices:
+            if device.serial:
+                devices.append(
+                    DeviceInfo(
+                        device_id=device.serial,
+                        platform="android",
+                        name=device.serial,
+                        state="connected",
+                    )
+                )
+    except Exception:
+        # ADB not available or error listing devices
+        pass
+
+    # List iOS devices
+    try:
+        cmd = ["xcrun", "simctl", "list", "devices", "-j"]
+        result = subprocess.run(cmd, capture_output=True, text=True, check=True)
+        data = json.loads(result.stdout)
+
+        for runtime, ios_devices in data.get("devices", {}).items():
+            if "iOS" not in runtime:
+                continue
+
+            for device in ios_devices:
+                udid = device.get("udid")
+                name = device.get("name")
+                state = device.get("state")
+
+                if udid:
+                    devices.append(
+                        DeviceInfo(
+                            device_id=udid,
+                            platform="ios",
+                            name=name,
+                            state=state,
+                        )
+                    )
+    except (subprocess.CalledProcessError, FileNotFoundError, json.JSONDecodeError):
+        # xcrun not available or error listing devices
+        pass
+
+    return devices
+
+
+def find_mobile_device(device_id: str | None = None) -> MobileDevice:
+    """
+    Find a mobile device (Android via ADB or iOS via xcrun).
+
+    Args:
+        device_id: Optional device ID to target a specific device.
+                   If None, returns the first available device.
+
+    Returns:
+        MobileDevice: A reference to the device with its platform information.
+
+    Raises:
+        DeviceNotFoundError: If no device is found or the specified device_id is not found.
+    """
+    # Get all available devices
+    available_devices = list_available_devices()
+
+    if not available_devices:
+        raise DeviceNotFoundError(
+            "No mobile device found. "
+            "Make sure you have an Android device connected via ADB "
+            "or an iOS simulator running."
+        )
+
+    # Find the target device
+    target_device = None
+    if device_id:
+        # Look for specific device
+        for dev in available_devices:
+            if dev.device_id == device_id:
+                target_device = dev
+                break
+        if not target_device:
+            raise DeviceNotFoundError(
+                f"Device with ID '{device_id}' not found. "
+                "Make sure the device is connected and accessible via adb or xcrun."
+            )
+    else:
+        # Prefer connected/booted devices first
+        for dev in available_devices:
+            if dev.state in ("connected", "Booted"):
+                target_device = dev
+                break
+        # Fall back to any device if no connected/booted device found
+        if not target_device:
+            target_device = available_devices[0]
+
+    # Create MobileDevice instance with platform-specific details
+    if target_device.platform == "android":
+        # For Android, get the AdbDevice reference
+        try:
+            adb_client = get_adb_client()
+            adb_device = adb_client.device(serial=target_device.device_id)
+            return MobileDevice(
+                device_id=target_device.device_id,
+                platform="android",
+                adb_device=adb_device,
+            )
+        except Exception as e:
+            raise DeviceNotFoundError(f"Failed to connect to Android device: {e}")
+    else:
+        # For iOS, just return the device info
+        return MobileDevice(device_id=target_device.device_id, platform="ios")
+
+
+def capture_screenshot(device: MobileDevice) -> str:
+    """
+    Capture a screenshot from the given mobile device.
+
+    Args:
+        device: MobileDevice instance returned by find_mobile_device()
+
+    Returns:
+        str: Base64-encoded screenshot image (PNG format)
+
+    Raises:
+        RuntimeError: If screenshot capture fails
+    """
+    if device.platform == "android":
+        return _capture_android_screenshot(device)
+    else:
+        return _capture_ios_screenshot(device)
+
+
+def _capture_android_screenshot(device: MobileDevice) -> str:
+    """Capture screenshot from Android device using ADB."""
+    if not device.adb_device:
+        # Reconnect to device if not available
+        adb_client = get_adb_client()
+        adb_device = adb_client.device(serial=device.device_id)
+        if not adb_device:
+            raise RuntimeError(f"Android device {device.device_id} not found")
+        device.adb_device = adb_device
+
+    try:
+        # Use ADB screencap to get PNG screenshot
+        screenshot_bytes = device.adb_device.shell("screencap -p", encoding=None)
+        if isinstance(screenshot_bytes, bytes):
+            return base64.b64encode(screenshot_bytes).decode("utf-8")
+        else:
+            raise RuntimeError("Unexpected screenshot data type from ADB")
+    except Exception as e:
+        raise RuntimeError(f"Failed to capture Android screenshot: {e}")
+
+
+def _capture_ios_screenshot(device: MobileDevice) -> str:
+    """Capture screenshot from iOS simulator using xcrun."""
+    try:
+        # Create temporary file for screenshot
+        with tempfile.NamedTemporaryFile(suffix=".png", delete=False) as tmp_file:
+            tmp_path = Path(tmp_file.name)
+
+        try:
+            # Capture screenshot using xcrun simctl
+            cmd = ["xcrun", "simctl", "io", device.device_id, "screenshot", str(tmp_path)]
+            subprocess.run(cmd, capture_output=True, text=True, check=True)
+
+            # Read and encode the screenshot
+            screenshot_bytes = tmp_path.read_bytes()
+            return base64.b64encode(screenshot_bytes).decode("utf-8")
+        finally:
+            # Clean up temporary file
+            if tmp_path.exists():
+                tmp_path.unlink()
+
+    except subprocess.CalledProcessError as e:
+        raise RuntimeError(f"Failed to capture iOS screenshot: {e.stderr}")
+    except Exception as e:
+        raise RuntimeError(f"Failed to capture iOS screenshot: {e}")

minitap/mcp/core/llm.py

@@ -0,0 +1,28 @@
+from langchain_openai import ChatOpenAI
+
+from minitap.mcp.core.config import settings
+
+
+def get_minitap_llm(
+    trace_id: str,
+    remote_tracing: bool = False,
+    model: str = "google/gemini-2.5-pro",
+    temperature: float | None = None,
+    max_retries: int | None = None,
+) -> ChatOpenAI:
+    assert settings.MINITAP_API_KEY is not None
+    assert settings.MINITAP_API_BASE_URL is not None
+    if max_retries is None and model.startswith("google/"):
+        max_retries = 2
+    client = ChatOpenAI(
+        model=model,
+        temperature=temperature,
+        max_retries=max_retries,
+        api_key=settings.MINITAP_API_KEY,
+        base_url=settings.MINITAP_API_BASE_URL,
+        default_query={
+            "sessionId": trace_id,
+            "traceOnlyUsage": remote_tracing,
+        },
+    )
+    return client

minitap/mcp/core/utils.py

@@ -0,0 +1,55 @@
+import base64
+from PIL import Image
+from io import BytesIO
+
+from langchain_core.messages import HumanMessage
+
+
+def compress_base64_jpeg(base64_str: str, quality: int = 50) -> str:
+    """
+    Compress a base64-encoded image to JPEG format.
+
+    Args:
+        base64_str: Base64-encoded image string
+        quality: JPEG quality (0-100, default 50)
+
+    Returns:
+        Base64-encoded JPEG image
+    """
+    if base64_str.startswith("data:image"):
+        base64_str = base64_str.split(",")[1]
+
+    image_data = base64.b64decode(base64_str)
+    image = Image.open(BytesIO(image_data))
+
+    # Convert RGBA/LA/PA to RGB (JPEG doesn't support transparency)
+    if image.mode in ("RGBA", "LA", "PA"):
+        # Create a white background
+        background = Image.new("RGB", image.size, (255, 255, 255))
+        # Paste the image on the background using alpha channel as mask
+        if image.mode == "RGBA":
+            background.paste(image, mask=image.split()[3])  # Use alpha channel as mask
+        else:
+            background.paste(image, mask=image.split()[1])  # Use alpha for LA
+        image = background
+    elif image.mode != "RGB":
+        # Convert any other mode to RGB
+        image = image.convert("RGB")
+
+    compressed_io = BytesIO()
+    image.save(compressed_io, format="JPEG", quality=quality, optimize=True)
+
+    compressed_base64 = base64.b64encode(compressed_io.getvalue()).decode("utf-8")
+    return compressed_base64
+
+
+def get_screenshot_message_for_llm(screenshot_base64: str):
+    prefix = "" if screenshot_base64.startswith("data:image") else "data:image/jpeg;base64,"
+    return HumanMessage(
+        content=[
+            {
+                "type": "image_url",
+                "image_url": {"url": f"{prefix}{screenshot_base64}"},
+            }
+        ]
+    )

minitap/mcp/main.py

@@ -0,0 +1,109 @@
+"""MCP server for mobile-use with screen analysis capabilities."""
+
+import argparse
+import logging
+import os
+import sys
+import threading
+
+# Fix Windows console encoding for Unicode characters (emojis in logs)
+if sys.platform == "win32":
+    if hasattr(sys.stdout, "reconfigure"):
+        sys.stdout.reconfigure(encoding="utf-8")  # type: ignore[attr-defined]
+    if hasattr(sys.stderr, "reconfigure"):
+        sys.stderr.reconfigure(encoding="utf-8")  # type: ignore[attr-defined]
+    os.environ["PYTHONIOENCODING"] = "utf-8"
+
+    try:
+        import colorama
+
+        colorama.init(strip=False, convert=True, wrap=True)
+    except ImportError:
+        pass
+
+
+from fastmcp import FastMCP  # noqa: E402
+
+from minitap.mcp.core.agents import agent
+from minitap.mcp.core.config import settings  # noqa: E402
+from minitap.mcp.core.device import (
+    DeviceInfo,  # noqa: E402
+    list_available_devices,  # noqa: E402; noqa: E402
+)
+from minitap.mcp.server.middleware import MaestroCheckerMiddleware
+from minitap.mcp.server.poller import device_health_poller
+
+logger = logging.getLogger(__name__)
+
+
+mcp = FastMCP(
+    name="mobile-use-mcp",
+    instructions="""
+        This server provides analysis tools for connected
+        mobile devices (iOS or Android).
+        Call get_available_devices() to list them.
+    """,
+)
+
+from minitap.mcp.tools import (  # noqa: E402, F401
+    analyze_screen,
+    execute_mobile_command,
+    go_back,
+)
+
+
+@mcp.resource("data://devices")
+def get_available_devices() -> list[DeviceInfo]:
+    """Provides a list of connected mobile devices (iOS or Android)."""
+    return list_available_devices()
+
+
+def mcp_lifespan(**mcp_run_kwargs):
+    mcp.add_middleware(MaestroCheckerMiddleware(agent))
+
+    # Start device health poller in background
+    logger.info("Device health poller started")
+    stop_event = threading.Event()
+    poller_thread = threading.Thread(
+        target=device_health_poller,
+        args=(
+            stop_event,
+            agent,
+        ),
+    )
+    poller_thread.start()
+
+    try:
+        mcp.run(**mcp_run_kwargs)
+    except KeyboardInterrupt:
+        pass
+
+    # Stop device health poller
+    stop_event.set()
+    logger.info("Device health poller stopping...")
+    poller_thread.join()
+    logger.info("Device health poller stopped")
+
+
+def main() -> None:
+    """Main entry point for the MCP server."""
+    parser = argparse.ArgumentParser(description="Mobile Use MCP Server")
+    parser.add_argument(
+        "--server",
+        action="store_true",
+        help="Run as network server (uses MCP_SERVER_HOST and MCP_SERVER_PORT from env)",
+    )
+
+    args = parser.parse_args()
+
+    # Run MCP server with optional host/port for remote access
+    if args.server:
+        logger.info(f"Starting MCP server on {settings.MCP_SERVER_HOST}:{settings.MCP_SERVER_PORT}")
+        mcp_lifespan(
+            transport="http",
+            host=settings.MCP_SERVER_HOST,
+            port=settings.MCP_SERVER_PORT,
+        )
+    else:
+        logger.info("Starting MCP server in local mode")
+        mcp_lifespan()

minitap/mcp/server/middleware.py

@@ -0,0 +1,23 @@
+from fastmcp.exceptions import ToolError
+from fastmcp.server.middleware import Middleware, MiddlewareContext
+
+from minitap.mobile_use.sdk import Agent
+
+
+class MaestroCheckerMiddleware(Middleware):
+    def __init__(self, agent: Agent):
+        self.agent = agent
+
+    async def on_call_tool(self, context: MiddlewareContext, call_next):
+        if context.fastmcp_context:
+            try:
+                tool = await context.fastmcp_context.fastmcp.get_tool(context.message.name)
+                if "requires-maestro" in tool.tags:
+                    if not self.agent.is_healthy():
+                        raise ToolError(
+                            "Maestro not healthy.\n"
+                            "Make sure a mobile device is connected and try again."
+                        )
+            except Exception:
+                pass
+        return await call_next(context)

minitap/mcp/server/poller.py

@@ -0,0 +1,38 @@
+"""Device health monitoring poller for the MCP server."""
+
+import logging
+import time
+import threading
+
+from minitap.mcp.core.device import list_available_devices
+from minitap.mobile_use.sdk import Agent
+
+logger = logging.getLogger(__name__)
+
+
+def device_health_poller(stop_event: threading.Event, agent: Agent) -> None:
+    """
+    Background poller that monitors device availability and agent health.
+    Runs every 5 seconds to ensure a device is connected and the agent is healthy.
+
+    Args:
+        agent: The Agent instance to monitor and reinitialize if needed.
+    """
+    while not stop_event.is_set():
+        try:
+            time.sleep(5)
+
+            devices = list_available_devices()
+
+            if len(devices) > 0:
+                if not agent.is_healthy():
+                    logger.warning("Agent is not healthy. Reinitializing...")
+                    agent.clean(force=True)
+                    agent.init()
+                    logger.info("Agent reinitialized successfully")
+            else:
+                logger.info("No mobile device found, retrying in 5 seconds...")
+
+        except Exception as e:
+            logger.error(f"Error in device health poller: {e}")
+    agent.clean(force=True)

minitap/mcp/tools/analyze_screen.py

@@ -0,0 +1,58 @@
+from pathlib import Path
+from jinja2 import Template
+from langchain_core.messages import BaseMessage, HumanMessage, SystemMessage
+from uuid import uuid4
+
+from pydantic import Field
+
+from minitap.mcp.core.config import settings
+from minitap.mcp.core.decorators import handle_tool_errors
+from minitap.mcp.core.device import capture_screenshot, find_mobile_device
+from minitap.mcp.core.llm import get_minitap_llm
+from minitap.mcp.core.utils import compress_base64_jpeg, get_screenshot_message_for_llm
+from minitap.mcp.main import mcp
+
+
+@mcp.tool(
+    name="analyze_screen",
+    description="""
+    Analyze what is shown on the mobile device screen.
+    This tool takes a screenshot file path and uses a vision-capable LLM
+    to analyze and describe what's on the screen. Useful for understanding
+    UI elements, extracting text, or identifying specific features.
+    """,
+)
+@handle_tool_errors
+async def analyze_screen(
+    prompt: str = Field(
+        description="Prompt for the analysis.",
+    ),
+    device_id: str | None = Field(
+        default=None,
+        description="ID of the device screen to analyze. "
+        "If not provided, the first available device is taken.",
+    ),
+) -> str | list | dict:
+    system_message = Template(
+        Path(__file__).parent.joinpath("screen_analyzer.md").read_text(encoding="utf-8")
+    ).render()
+
+    # Find the device and capture screenshot
+    device = find_mobile_device(device_id=device_id)
+    screenshot_base64 = capture_screenshot(device)
+    compressed_image_base64 = compress_base64_jpeg(screenshot_base64)
+
+    messages: list[BaseMessage] = [
+        SystemMessage(content=system_message),
+        get_screenshot_message_for_llm(compressed_image_base64),
+        HumanMessage(content=prompt),
+    ]
+
+    llm = get_minitap_llm(
+        trace_id=str(uuid4()),
+        remote_tracing=True,
+        model=settings.VISION_MODEL,
+        temperature=1,
+    )
+    response = await llm.ainvoke(messages)
+    return response.content

minitap/mcp/tools/execute_mobile_command.py

@@ -0,0 +1,64 @@
+"""Tool for running manual tasks on a connected mobile device."""
+
+from collections.abc import Mapping
+from typing import Any
+
+from minitap.mobile_use.sdk.types import ManualTaskConfig
+from minitap.mobile_use.sdk.types.task import PlatformTaskRequest
+from pydantic import Field
+
+from minitap.mcp.core.agents import agent
+from minitap.mcp.core.decorators import handle_tool_errors
+from minitap.mcp.main import mcp
+
+
+def _serialize_result(result: Any) -> Any:
+    """Convert SDK responses to serializable data for MCP."""
+    if hasattr(result, "model_dump"):
+        return result.model_dump()
+    if hasattr(result, "dict"):
+        return result.dict()
+    if isinstance(result, Mapping):
+        return dict(result)
+    return result
+
+
+@mcp.tool(
+    name="execute_mobile_command",
+    tags={"requires-maestro"},
+    description="""
+    Execute a natural language command on a mobile device using the Minitap SDK.
+    This tool allows you to control your Android or iOS device using natural language.
+    Examples:
+    - "Open the settings app and tell me the battery level"
+    - "Find the first 3 unread emails in Gmail"
+    - "Take a screenshot and save it"
+    
+    The tool uses the Minitap platform with API key authentication.
+    Set MINITAP_API_KEY and MINITAP_API_BASE_URL environment variables.
+    Visit https://platform.minitap.ai to get your API key.
+    """,
+)
+@handle_tool_errors
+async def execute_mobile_command(
+    goal: str = Field(description="High-level goal describing the action to perform."),
+    output_description: str | None = Field(
+        default=None,
+        description="Optional description of the expected output format. "
+        "For example: 'A JSON array with sender and subject for each email' "
+        "or 'The battery percentage as a number'.",
+    ),
+    profile: str = Field(
+        default="default",
+        description="Name of the profile to use for this task. Defaults to 'default'.",
+    ),
+) -> str | dict[str, Any]:
+    """Run a manual task on a mobile device via the Minitap platform."""
+    try:
+        request = PlatformTaskRequest(
+            task=ManualTaskConfig(goal=goal, output_description=output_description),
+        )
+        result = await agent.run_task(request=request)
+        return _serialize_result(result)
+    finally:
+        agent.clean()

minitap/mcp/tools/go_back.py

@@ -0,0 +1,42 @@
+import requests
+
+from minitap.mcp.core.decorators import handle_tool_errors
+from minitap.mcp.main import mcp
+
+
+@mcp.tool(
+    name="go_back",
+    tags={"requires-maestro"},
+    description="""
+    Sends a 'back' command to the mobile device automation server.
+    """,
+)
+@handle_tool_errors
+async def go_back() -> str:
+    """Send a back command to the mobile device."""
+    try:
+        response = requests.post(
+            "http://localhost:9999/api/run-command",
+            headers={
+                "User-Agent": "python-requests/2.32.4",
+                "Accept-Encoding": "gzip, deflate, zstd",
+                "Accept": "*/*",
+                "Connection": "keep-alive",
+                "Content-Type": "application/json",
+            },
+            json={"yaml": "back\n"},
+            timeout=30,
+        )
+
+        if response.status_code == 200:
+            return f"Successfully sent back command. Response: {response.text}"
+        else:
+            return (
+                f"Failed to send back command. "
+                f"Status code: {response.status_code}, Response: {response.text}"
+            )
+
+    except requests.exceptions.RequestException as e:
+        return f"Error sending back command: {str(e)}"
+    except Exception as e:
+        return f"Unexpected error: {str(e)}"

minitap/mcp/tools/screen_analyzer.md

@@ -0,0 +1,17 @@
+You are given:
+
+1. A screenshot of a mobile device.
+2. A prompt describing what information to extract.
+
+Your task:
+
+- Look at the screenshot and **answer the prompt directly and completely**.
+- Provide a **detailed, structured description** of the relevant content (text, layout, icons, menus, timestamps, notifications, etc.).
+- If the prompt asks for specific data, extract it exactly as shown.
+- If the screenshot contains structured information (e.g., receipt, chat, settings), present it clearly using lists or tables.
+- Do not guess — if something is unclear or missing, state that explicitly.
+
+**Output format:**
+
+1. **Direct answer** to the prompt.
+2. **Detailed breakdown** of the screenshot content supporting the answer.

minitap_mcp-0.1.0.dist-info/METADATA

@@ -0,0 +1,348 @@
+Metadata-Version: 2.3
+Name: minitap-mcp
+Version: 0.1.0
+Summary: MCP server for mobile-use
+Author: Pierre-Louis Favreau, Jean-Pierre Lo, Clément Guiguet
+Requires-Dist: fastmcp>=2.12.4
+Requires-Dist: python-dotenv>=1.1.1
+Requires-Dist: pydantic>=2.12.0
+Requires-Dist: pydantic-settings>=2.10.1
+Requires-Dist: minitap-mobile-use>=2.5.3
+Requires-Dist: jinja2>=3.1.6
+Requires-Dist: langchain-core>=0.3.75
+Requires-Dist: ruff==0.5.3 ; extra == 'dev'
+Requires-Dist: pytest==8.4.1 ; extra == 'dev'
+Requires-Dist: pytest-cov==5.0.0 ; extra == 'dev'
+Requires-Python: >=3.12
+Project-URL: Homepage, https://minitap.ai/
+Project-URL: Source, https://github.com/minitap-ai/mobile-use
+Provides-Extra: dev
+Description-Content-Type: text/markdown
+
+# Mobile-Use MCP Server
+
+A Model Context Protocol (MCP) server that provides AI-powered mobile device screen analysis. Automatically detects connected Android (via ADB) and iOS devices (via xcrun), captures screenshots, and analyzes them using vision language models.
+
+## Features
+
+- **🔍 Device Discovery**: Automatically finds connected Android devices (ADB) and iOS simulators (xcrun)
+- **📱 Screen Analysis**: Capture and analyze device screenshots using vision-capable LLMs
+- **🤖 Natural Language Control**: Execute commands on your device using natural language via the mobile-use SDK
+- **🚀 Easy Integration**: Built with FastMCP for seamless MCP protocol implementation
+- **⚙️ Flexible Configuration**: Uses Minitap API with support for various vision models
+
+## Installation
+
+### Prerequisites
+
+- **Python 3.12+**
+- **uv** (recommended) or pip
+- **For Android**: ADB installed and accessible
+- **For iOS**: Xcode Command Line Tools (macOS only)
+- **Minitap API Key** - Get one at [platform.minitap.ai](https://platform.minitap.ai)
+
+### Setup
+
+1. **Clone and navigate to the project:**
+
+```bash
+cd minitap-mcp
+```
+
+2. **Install dependencies:**
+
+```bash
+# Create a virtual environment
+uv venv
+source .venv/bin/activate
+
+# Install dependencies
+uv sync
+```
+
+3. **Configure for MCP usage:**
+
+The MCP server is configured via environment variables passed from your MCP client (e.g., Windsurf). 
+
+Required environment variable:
+- `MINITAP_API_KEY`: Your Minitap API key
+
+Optional environment variables:
+- `MINITAP_API_BASE_URL`: API base URL (default: `https://platform.minitap.ai/api/v1`)
+- `VISION_MODEL`: Vision model to use (default: `baidu/ernie-4.5-vl-28b-a3b`)
+- `ADB_SERVER_SOCKET`: Custom ADB server socket (format: `tcp:host:port`)
+
+## Available Resources & Tools
+
+### Resource: `data://devices`
+
+Lists all connected mobile devices (Android and iOS).
+
+**Returns:** Array of device information objects with:
+- `device_id`: Device serial (Android) or UDID (iOS)
+- `platform`: `"android"` or `"ios"`
+- `name`: Device name
+- `state`: Device state (`"connected"` or `"Booted"`)
+
+### Tool: `analyze_screen`
+
+Captures a screenshot from a mobile device and analyzes it using a vision language model.
+
+**Parameters:**
+- `prompt` (required): Analysis prompt describing what information to extract
+- `device_id` (optional): Specific device ID to target. If not provided, uses the first available device.
+
+**Returns:** AI-generated analysis of the screenshot based on the prompt.
+
+**Example:**
+```
+Prompt: "What app is currently open? List all visible UI elements."
+```
+
+The tool will:
+1. Find the specified device (or first available)
+2. Capture a screenshot
+3. Analyze it with the vision model
+4. Return the analysis
+
+### Tool: `execute_mobile_command`
+
+Execute natural language commands on your mobile device using the mobile-use SDK. This tool allows you to control your Android or iOS device with simple instructions.
+
+**Parameters:**
+- `goal` (required): Natural language command to execute on the device
+- `output_description` (optional): Description of the expected output format (e.g., "A JSON list of objects with sender and subject keys")
+- `profile` (optional): Name of the profile to use for this task. Defaults to 'default'
+
+**Returns:** Execution result with status, output, and any extracted data.
+
+**Examples:**
+```python
+# Simple command
+goal: "Go to settings and tell me my current battery level"
+
+# Data extraction with structured output
+goal: "Open Gmail, find first 3 unread emails, and list their sender and subject line"
+output_description: "A JSON list of objects, each with 'sender' and 'subject' keys"
+
+# App navigation
+goal: "Open Twitter and scroll to the latest tweet"
+```
+
+The tool will:
+1. Find the specified device (or first available)
+2. Execute the command using the mobile-use AI agent
+3. Return the result or extracted data
+
+## Usage
+
+### Running the MCP Server
+
+#### Local Mode (Default)
+
+The MCP server is typically started by your MCP client (e.g., Windsurf). For manual testing:
+
+```bash
+minitap-mcp
+```
+
+#### Network Server Mode
+
+You can run the MCP server as a network server for remote access:
+
+```bash
+# Run as network server (uses MCP_SERVER_HOST and MCP_SERVER_PORT from env)
+minitap-mcp --server
+```
+
+The server will bind to the host and port specified in your environment variables:
+- `MCP_SERVER_HOST` (default: `0.0.0.0`)
+- `MCP_SERVER_PORT` (default: `8000`)
+
+Configure these in your `.env` file or via environment variables to customize the binding address.
+
+Inside Windsurf, you can configure the MCP server by adding the following to your `~/.codeium/windsurf/mcp_settings.json` file:
+
+```json
+{
+  "mcpServers": {
+    "minitap-mcp": {
+      "serverUrl": "http://localhost:8000/mcp"
+    }
+  }
+}
+```
+
+N.B. You may need to change the port based on what you've configured in your `.env` file.
+
+## Development
+
+### Quick Testing
+
+Test device detection and screenshot capture (no API key required):
+
+```bash
+python tests/test_devices.py
+```
+
+Test the complete MCP flow with LLM analysis (requires API key):
+
+```bash
+cp .env.example .env
+# Edit .env and add your MINITAP_API_KEY
+python tests/test_mcp.py
+```
+
+### Code Quality
+
+**Format code:**
+```bash
+ruff format .
+```
+
+**Lint:**
+```bash
+ruff check --fix
+```
+
+## Project Structure
+
+```
+minitap/mcp/
+├── __init__.py
+├── main.py                    # FastMCP server entry point
+├── core/
+│   ├── __init__.py
+│   ├── config.py              # Pydantic settings configuration
+│   ├── decorators.py          # Error handling decorators
+│   ├── device.py              # Device discovery & screenshot capture
+│   ├── llm.py                 # LLM client initialization
+│   └── utils.py               # Utility functions (image compression, etc.)
+└── tools/
+    ├── __init__.py
+    ├── analyze_screen.py      # Screen analysis tool
+    ├── execute_mobile_command.py  # Mobile-use SDK integration tool
+    └── screen_analyzer.md     # System prompt for analysis
+
+tests/
+├── test_devices.py            # Device detection tests
+└── test_mcp.py                # Full MCP integration tests
+```
+
+## Creating New Tools
+
+When adding new MCP tools, use the `@handle_tool_errors` decorator to prevent unhandled exceptions from causing infinite loops:
+
+```python
+from minitap.mcp.core.decorators import handle_tool_errors
+from minitap.mcp.main import mcp
+
+@mcp.tool(name="my_tool", description="...")
+@handle_tool_errors  # Add this decorator
+async def my_tool(param: str) -> str:
+    # Your tool logic here
+    # Any exception will be caught and returned as an error message
+    return "result"
+```
+
+The decorator automatically:
+- Catches all exceptions (including `DeviceNotFoundError`)
+- Returns user-friendly error messages
+- Prevents the MCP server from hanging or looping infinitely
+- Works with both sync and async functions
+
+## Integration with Windsurf
+
+To use this MCP server in Windsurf, add it to your MCP settings:
+
+**Location:** `~/.codeium/windsurf/mcp_settings.json`
+
+**Configuration:**
+
+```json
+{
+  "mcpServers": {
+    "minitap-mcp": {
+      "command": "uv",
+      "args": ["-c", "cd /path/to/minitap-mcp && source .venv/bin/activate && uv sync && minitap-mcp"],
+      "env": {
+        "MINITAP_API_KEY": "your_minitap_api_key_here",
+        "MINITAP_API_BASE_URL": "https://platform.minitap.ai/api/v1",
+        "VISION_MODEL": "baidu/ernie-4.5-vl-28b-a3b" // optional
+      }
+    }
+  }
+}
+```
+
+**After configuration:**
+1. Restart Windsurf
+2. The `analyze_screen` and `execute_mobile_command` tools will be available in Cascade
+3. The `data://devices` resource will list connected devices
+
+### Available Vision Models
+
+The Minitap API supports various vision models:
+- `qwen/qwen-2.5-vl-7b-instruct` (default)
+- `baidu/ernie-4.5-vl-28b-a3b`
+- `openai/gpt-4o`
+- And more - check the Minitap platform for the full list
+
+## Device Requirements
+
+### Android Devices
+
+**Requirements:**
+- ADB installed and in PATH
+- USB debugging enabled on the device
+- Device connected via USB or network ADB
+
+**Verify connection:**
+```bash
+adb devices
+```
+
+**Custom ADB Server:**
+If using a custom ADB server (e.g., Docker, WSL), set the socket:
+```bash
+export ADB_SERVER_SOCKET="tcp:localhost:5037"
+```
+
+N.B. You may need to reboot your IDE
+
+### iOS Devices
+
+**Requirements:**
+- macOS with Xcode Command Line Tools
+- iOS Simulator running
+
+**Verify simulators:**
+```bash
+xcrun simctl list devices booted
+```
+
+**Start a simulator:**
+```bash
+open -a Simulator
+```
+
+## Troubleshooting
+
+### No devices found
+
+1. **Android:** Run `adb devices` to verify device connection
+2. **iOS:** Run `xcrun simctl list devices booted` to check running simulators
+3. Ensure USB debugging is enabled (Android)
+4. Try restarting ADB: `adb kill-server && adb start-server`
+
+### Screenshot capture fails
+
+1. Ensure device screen is unlocked
+2. For Android, verify screencap permission
+3. For iOS, ensure simulator is fully booted
+
+### Tool not detected in Windsurf
+
+1. Verify the import in `main.py` includes the tools module
+2. Check that `tools/__init__.py` exists
+3. Restart Windsurf after configuration changes

minitap_mcp-0.1.0.dist-info/RECORD

@@ -0,0 +1,18 @@
+minitap/mcp/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+minitap/mcp/core/agents.py,sha256=aRqHA1r8ejWOGlTbku_TXv6WV8Vucp5LP4puhEit7tc,573
+minitap/mcp/core/config.py,sha256=kaIlb_4AQ3DNwtcLDZF528cgrtj5LGcCFNdHZfEwrUo,820
+minitap/mcp/core/decorators.py,sha256=iekv181o_rkv0upacFWkmPqxsZRTzuLFyOZ0sIDtQnQ,1317
+minitap/mcp/core/device.py,sha256=sEO3Z-8F325hDOObdH1YBhZE60f17FmIclt5UlhY_nU,7875
+minitap/mcp/core/llm.py,sha256=z_pYZkZcAchsiWPh4W79frQPANsfYyFPUe8DJo8lZO0,822
+minitap/mcp/core/utils.py,sha256=3uExpRoh7affIieZx3TLlZTmZCcoxWfx1YpPbwhjiJY,1791
+minitap/mcp/main.py,sha256=4ytnVMd7yzk7I9MnFsPB8U9b9bMyhVf6Yq623pYccj0,3060
+minitap/mcp/server/middleware.py,sha256=fbry_IiHmwUxVjsWgOU2goybcS1kLRXFZZ89KPH1d8E,880
+minitap/mcp/server/poller.py,sha256=C2h5Ir3nY5gZ6qTDOHBw_Tb8PfAY54A-we2HrwjNLvg,1222
+minitap/mcp/tools/analyze_screen.py,sha256=fjcjf3tTZDlxzmiQFHFNgw38bxPz4eisw57zuxshN2A,1984
+minitap/mcp/tools/execute_mobile_command.py,sha256=fpmr0LnV7DDEiHwDDrDVZ-SaoVUmReZWH6sRBahPWes,2320
+minitap/mcp/tools/go_back.py,sha256=lEmADkDkXu8JGm-sY7zL7M6GlBy-lD7Iffv4yzwoQfo,1301
+minitap/mcp/tools/screen_analyzer.md,sha256=TTO80JQWusbA9cKAZn-9cqhgVHm6F_qJh5w152hG3YM,734
+minitap_mcp-0.1.0.dist-info/WHEEL,sha256=X16MKk8bp2DRsAuyteHJ-9qOjzmnY0x1aj0P1ftqqWA,78
+minitap_mcp-0.1.0.dist-info/entry_points.txt,sha256=rYVoXm7tSQCqQTtHx4Lovgn1YsjwtEEHfddKrfEVHuY,55
+minitap_mcp-0.1.0.dist-info/METADATA,sha256=DB6OoQCvyqHD05Xks_splNrQzhKj32MO8mvf2KaWnmw,9920
+minitap_mcp-0.1.0.dist-info/RECORD,,

minitap_mcp-0.1.0.dist-info/WHEEL

@@ -0,0 +1,4 @@
+Wheel-Version: 1.0
+Generator: uv 0.9.2
+Root-Is-Purelib: true
+Tag: py3-none-any

minitap_mcp-0.1.0.dist-info/entry_points.txt

@@ -0,0 +1,3 @@
+[console_scripts]
+minitap-mcp = minitap.mcp.main:main
+