minitap-mcp 0.1.1__py3-none-any.whl → 0.4.0__py3-none-any.whl

minitap/mcp/core/agents/compare_screenshots.md

@@ -0,0 +1,62 @@
+You will be given _two screenshots_.
+
+1. "Expected screenshot" — this is the design from Figma.
+2. "Implemented screenshot" — this is the actual phone screen that has been built.
+
+Your task is to **compare the two screenshots** in detail, and generate a structured report that includes:
+
+- A comprehensive list of **all visible differences** between the expected design and the implemented screen.
+- For each difference, provide:
+  - A clear **description** of what changed (for example: "The 'Submit' button label changed from 'Submit' to 'Send'", "The icon moved 8px to the right", "The background colour of header changed from #FFFFFF to #F6F6F6", etc.).
+  - The **type of change** (e.g., text change, color change, position/movement, size change, added element, removed element, style change).
+  - The **location** of the change (for example: "bottom-centre of screen", "top header area", "to the right of search bar"). If possible, approximate coordinates or bounding box (e.g., "approx. 240×180 px at screen width 1080").
+  - The **impact on implementation** (i.e., reasoning about what this means: "The implemented version uses a different text label – so behaviour may differ", "The icon moved and may overlap another element", etc.).
+  - A **recommendation** if relevant (e.g., "Should revert to #FFFFFF to match design", "Check alignment of icon relative to search bar", etc.).
+
+**Important**:
+
+- Assume the screenshots are aligned (same resolution and scale); if not aligned mention that as a difference.
+- Focus on _visible UI differences_ (layout, text, style, iconography) – you do _not_ need to inspect source code, only what is visually rendered.
+- Do _not_ produce generic comments like "looks like a difference" – aim for _precise, actionable descriptions_.
+- **IGNORE dynamic/personal content** that naturally differs between mockups and real implementations:
+  - User profile information (names, usernames, email addresses, profile pictures)
+  - Time-based information (current time, dates, timestamps, "2 hours ago", etc.)
+  - Dynamic data (notification counts, unread badges, live statistics)
+  - Sample/placeholder content that varies (e.g., "John Doe" vs "Jane Smith")
+  - System status information (battery level, signal strength, network indicators)
+  - Only flag these as differences if the _structure, layout, or styling_ of these elements differs, not the content itself.
+- Output in a structured format, for example:
+
+```
+
+1. Location: [top header – full width]
+   Change: Background colour changed from #FFFFFF → #F6F6F6
+   Type: Colour change
+   Impact: The header will appear darker than design; text contrast may be lower.
+   Recommendation: Update header background to #FFFFFF as in design.
+
+```
+
+- At the end produce a summary with ONLY:
+  - Total number of differences found
+  - Overall "match score" out of 100 (your estimation of how closely the implementation matches the design)
+  - Do NOT include any recap, overview, or macro-level summary of changes - all details are already captured in the differences list above.
+
+### Input:
+
+- Screenshot A: Expected (Figma)
+- Screenshot B: Implemented (Phone)
+  Provide both screenshots and then the prompt.
+
+### Output:
+
+Structured list of differences + summary.
+
+Please use the following to start the analysis.
+**Input:**
+First screen is the Figma screenshot (what is expected)
+Second screen is what is expected (taken from the phone, after the implementation)
+
+You will have this data in the next messages sent by the user.
+
+Go ahead and generate your report.

minitap/mcp/core/agents/compare_screenshots.py

@@ -0,0 +1,65 @@
+import asyncio
+from pathlib import Path
+from uuid import uuid4
+
+from jinja2 import Template
+from langchain_core.messages import BaseMessage, HumanMessage, SystemMessage
+from pydantic import BaseModel
+
+from minitap.mcp.core.device import capture_screenshot, find_mobile_device
+from minitap.mcp.core.llm import get_minitap_llm
+from minitap.mcp.core.utils import get_screenshot_message_for_llm
+
+
+class CompareScreenshotsOutput(BaseModel):
+    comparison_text: str
+    expected_screenshot_base64: str
+    current_screenshot_base64: str
+
+
+async def compare_screenshots(
+    expected_screenshot_base64: str,
+) -> CompareScreenshotsOutput:
+    """
+    Compare screenshots and return the comparison text along with both screenshots.
+
+    Returns:
+        CompareScreenshotsOutput
+    """
+    system_message = Template(
+        Path(__file__).parent.joinpath("compare_screenshots.md").read_text(encoding="utf-8")
+    ).render()
+
+    device = find_mobile_device()
+    current_screenshot = capture_screenshot(device)
+
+    messages: list[BaseMessage] = [
+        SystemMessage(content=system_message),
+        HumanMessage(content="Here is the Figma screenshot (what needs to be matched):"),
+        get_screenshot_message_for_llm(expected_screenshot_base64),
+        HumanMessage(content="Here is the screenshot of the mobile device:"),
+        get_screenshot_message_for_llm(current_screenshot),
+    ]
+
+    llm = get_minitap_llm(
+        trace_id=str(uuid4()),
+        remote_tracing=True,
+        model="google/gemini-2.5-pro",
+        temperature=1,
+    )
+    response = await llm.ainvoke(messages)
+    return CompareScreenshotsOutput(
+        comparison_text=str(response.content),
+        expected_screenshot_base64=expected_screenshot_base64,
+        current_screenshot_base64=current_screenshot,
+    )
+
+
+async def main():
+    expected_screenshot_base64 = "Base64 encoded screenshot to compare with."
+    result = await compare_screenshots(expected_screenshot_base64)
+    print(result.model_dump_json(indent=2))
+
+
+if __name__ == "__main__":
+    asyncio.run(main())

minitap/mcp/core/agents/extract_figma_assets.md

@@ -0,0 +1,64 @@
+You are an expert at parsing React/TypeScript code to extract asset URLs and generate clean, documented code implementations.
+
+Your task is to:
+
+1. Extract all asset URLs from the provided code snippet
+2. Generate a clean `code_implementation` output that includes the React code with embedded comments referencing implementation and node guidelines
+
+**Instructions:**
+
+## Part 1: Extract Asset URLs
+
+1. Look for all constant declarations that contain URLs pointing to assets (images, SVGs, etc.)
+2. These constants typically follow patterns like:
+
+   - `const imgVariableName = "http://localhost:3845/assets/[hash].[extension]";`
+   - The variable names usually start with `img` followed by a descriptive name in camelCase
+
+3. For each asset URL found, extract:
+   - The **variable name** (e.g., `imgSignal`, `imgBatteryThreeQuarters`)
+   - The **full URL** (e.g., `http://localhost:3845/assets/685c5ac58caa29556e29737cf8f8c9605d9c8571.svg`)
+   - The **file extension** from the URL (e.g., `svg`, `png`, `jpg`)
+
+## Part 2: Generate Code Implementation
+
+The `code_implementation` field should contain:
+
+1. The React/TypeScript code with **LOCAL asset imports** instead of HTTP URLs:
+
+   - Convert `const imgSignal = "http://localhost:3845/assets/[hash].svg";`
+   - To `import imgSignal from './assets/imgSignal.svg';` (or appropriate relative path)
+   - Use the **exact same variable names** as in the original const declarations
+   - **CRITICAL**: Preserve the variable naming convention
+
+2. Preserve all `data-node-id` attributes and other metadata in the code
+
+## Part 3: Return Format
+
+Return a JSON object with two fields:
+
+- `assets`: Array of extracted asset objects
+- `code_implementation`: String containing the React code with embedded guideline comments
+
+```json
+{
+  "assets": [
+    {
+      "variable_name": "imgSignal",
+      "url": "http://localhost:3845/assets/685c5ac58caa29556e29737cf8f8c9605d9c8571.svg",
+      "extension": "svg"
+    },
+    ...
+  ],
+  "code_implementation": "import ... function ..."
+}
+```
+
+**Important:**
+
+- Only extract asset URLs
+- Preserve the exact variable names as they appear in the code
+- DO NOT MISS any assets
+- If no assets are found, return an empty array for `assets`
+- Return ONLY the JSON object with both `assets` and `code_implementation` fields
+- Do NOT include the const declarations of the assets in the code_implementation output - convert them to imports.

minitap/mcp/core/agents/extract_figma_assets.py

@@ -0,0 +1,65 @@
+"""Agent to extract Figma asset URLs from design context code."""
+
+from pathlib import Path
+from uuid import uuid4
+
+from jinja2 import Template
+from langchain_core.messages import BaseMessage, HumanMessage, SystemMessage
+from pydantic import BaseModel, Field
+
+from minitap.mcp.core.llm import get_minitap_llm
+
+
+class FigmaAsset(BaseModel):
+    """Represents a single Figma asset."""
+
+    variable_name: str = Field(description="The variable name from the code (e.g., imgSignal)")
+    url: str = Field(description="The full URL to the asset")
+    extension: str = Field(description="The file extension (e.g., svg, png, jpg)")
+
+
+class ExtractedAssets(BaseModel):
+    """Container for all extracted Figma assets."""
+
+    assets: list[FigmaAsset] = Field(
+        default_factory=list,
+        description="List of all extracted assets from the Figma design context",
+    )
+    code_implementation: str = Field(
+        description=(
+            "The React/TypeScript code\n"
+            "with the local url declarations turned into const declarations"
+        )
+    )
+
+
+async def extract_figma_assets(design_context_code: str) -> ExtractedAssets:
+    """Extract asset URLs from Figma design context code.
+
+    Args:
+        design_context_code: The React/TypeScript code from get_design_context
+
+    Returns:
+        List of dictionaries containing variable_name, url, and extension
+    """
+    system_message = Template(
+        Path(__file__).parent.joinpath("extract_figma_assets.md").read_text(encoding="utf-8")
+    ).render()
+
+    messages: list[BaseMessage] = [
+        SystemMessage(content=system_message),
+        HumanMessage(
+            content=f"Here is the code to analyze:\n\n```typescript\n{design_context_code}\n```"
+        ),
+    ]
+
+    llm = get_minitap_llm(
+        trace_id=str(uuid4()),
+        remote_tracing=True,
+        model="google/gemini-2.5-pro",
+        temperature=0,
+    ).with_structured_output(ExtractedAssets)
+
+    result: ExtractedAssets = await llm.ainvoke(messages)  # type: ignore
+
+    return result

minitap/mcp/core/config.py

@@ -14,11 +14,14 @@ class MCPSettings(BaseSettings):
     model_config = SettingsConfigDict(env_file=".env", extra="ignore")
 
     # Minitap API configuration
-    MINITAP_API_KEY: SecretStr
+    MINITAP_API_KEY: SecretStr | None = Field(default=None)
     MINITAP_API_BASE_URL: str = Field(default="https://platform.minitap.ai/api/v1")
 
     VISION_MODEL: str = Field(default="qwen/qwen-2.5-vl-7b-instruct")
 
+    # Figma MCP server configuration
+    FIGMA_MCP_SERVER_URL: str = Field(default="http://127.0.0.1:3845/mcp")
+
     # MCP server configuration (optional, for remote access)
     MCP_SERVER_HOST: str = Field(default="0.0.0.0")
     MCP_SERVER_PORT: int = Field(default=8000)

minitap/mcp/core/models.py

@@ -0,0 +1,59 @@
+"""Core models for the MCP server."""
+
+from enum import Enum
+
+from pydantic import BaseModel, Field
+
+
+class FigmaAsset(BaseModel):
+    """Represents a single Figma asset."""
+
+    variable_name: str = Field(description="The variable name from the code (e.g., imgSignal)")
+    url: str = Field(description="The full URL to the asset")
+    extension: str = Field(description="The file extension (e.g., svg, png, jpg)")
+
+
+class FigmaDesignContextOutput(BaseModel):
+    """Output from Figma design context containing code and guidelines."""
+
+    code_implementation: str = Field(description="The React/TypeScript code implementation")
+    code_implementation_guidelines: str | None = Field(
+        default=None, description="Guidelines for implementing the code"
+    )
+    nodes_guidelines: str | None = Field(
+        default=None, description="Guidelines specific to the nodes"
+    )
+
+
+class DownloadStatus(str, Enum):
+    """Status of asset download operation."""
+
+    SUCCESS = "success"
+    FAILED = "failed"
+
+
+class AssetDownloadResult(BaseModel):
+    """Result of downloading a single asset."""
+
+    filename: str = Field(description="The filename of the asset")
+    status: DownloadStatus = Field(description="The download status")
+    error: str | None = Field(default=None, description="Error message if download failed")
+
+
+class AssetDownloadSummary(BaseModel):
+    """Summary of all asset download operations."""
+
+    successful: list[AssetDownloadResult] = Field(
+        default_factory=list, description="List of successfully downloaded assets"
+    )
+    failed: list[AssetDownloadResult] = Field(
+        default_factory=list, description="List of failed asset downloads"
+    )
+
+    def success_count(self) -> int:
+        """Return the number of successful downloads."""
+        return len(self.successful)
+
+    def failure_count(self) -> int:
+        """Return the number of failed downloads."""
+        return len(self.failed)

minitap/mcp/core/sdk_agent.py

@@ -0,0 +1,27 @@
+import os
+
+from minitap.mobile_use.sdk import Agent
+from minitap.mobile_use.sdk.builders import Builders
+
+# Lazy-initialized singleton agent
+_agent: Agent | None = None
+
+
+def get_mobile_use_agent() -> Agent:
+    """Get or create the mobile-use agent singleton.
+
+    This function lazily initializes the agent on first call, ensuring
+    that CLI arguments are parsed before agent creation.
+    """
+    global _agent
+    if _agent is None:
+        config = Builders.AgentConfig
+        custom_adb_socket = os.getenv("ADB_SERVER_SOCKET")
+        if custom_adb_socket:
+            parts = custom_adb_socket.split(":")
+            if len(parts) != 3:
+                raise ValueError(f"Invalid ADB server socket: {custom_adb_socket}")
+            _, host, port = parts
+            config = config.with_adb_server(host=host, port=int(port))
+        _agent = Agent(config=config.build())
+    return _agent

minitap/mcp/main.py

@@ -23,18 +23,56 @@ if sys.platform == "win32":
 
 
 from fastmcp import FastMCP  # noqa: E402
+from minitap.mobile_use.config import settings as sdk_settings
 
-from minitap.mcp.core.agents import agent
 from minitap.mcp.core.config import settings  # noqa: E402
-from minitap.mcp.core.device import (
-    DeviceInfo,  # noqa: E402
-    list_available_devices,  # noqa: E402; noqa: E402
-)
+from minitap.mcp.core.device import DeviceInfo  # noqa: E402
+from minitap.mcp.core.device import list_available_devices
 from minitap.mcp.server.middleware import MaestroCheckerMiddleware
 from minitap.mcp.server.poller import device_health_poller
 
-logger = logging.getLogger(__name__)
 
+def main() -> None:
+    """Main entry point for the MCP server."""
+
+    parser = argparse.ArgumentParser(description="Mobile Use MCP Server")
+    parser.add_argument("--api-key", type=str, required=False, default=None)
+    parser.add_argument("--llm-profile", type=str, required=False, default=None)
+    parser.add_argument(
+        "--server",
+        action="store_true",
+        help="Run as network server (uses MCP_SERVER_HOST and MCP_SERVER_PORT from env)",
+    )
+
+    args = parser.parse_args()
+
+    if args.api_key:
+        os.environ["MINITAP_API_KEY"] = args.api_key
+        settings.__init__()
+        sdk_settings.__init__()
+
+    if args.llm_profile:
+        os.environ["MINITAP_LLM_PROFILE_NAME"] = args.llm_profile
+        settings.__init__()
+        sdk_settings.__init__()
+
+    if not settings.MINITAP_API_KEY:
+        raise ValueError("Minitap API key is required to run the MCP")
+
+    # Run MCP server with optional host/port for remote access
+    if args.server:
+        logger.info(f"Starting MCP server on {settings.MCP_SERVER_HOST}:{settings.MCP_SERVER_PORT}")
+        mcp_lifespan(
+            transport="http",
+            host=settings.MCP_SERVER_HOST,
+            port=settings.MCP_SERVER_PORT,
+        )
+    else:
+        logger.info("Starting MCP server in local mode")
+        mcp_lifespan()
+
+
+logger = logging.getLogger(__name__)
 
 mcp = FastMCP(
     name="mobile-use-mcp",
@@ -44,12 +82,10 @@ mcp = FastMCP(
         Call get_available_devices() to list them.
     """,
 )
-
-from minitap.mcp.tools import (  # noqa: E402, F401
-    analyze_screen,
-    execute_mobile_command,
-    go_back,
-)
+from minitap.mcp.tools import analyze_screen  # noqa: E402, F401
+from minitap.mcp.tools import compare_screenshot_with_figma  # noqa: E402, F401
+from minitap.mcp.tools import execute_mobile_command  # noqa: E402, F401
+from minitap.mcp.tools import save_figma_assets  # noqa: E402, F401
 
 
 @mcp.resource("data://devices")
@@ -59,6 +95,9 @@ def get_available_devices() -> list[DeviceInfo]:
 
 
 def mcp_lifespan(**mcp_run_kwargs):
+    from minitap.mcp.core.sdk_agent import get_mobile_use_agent  # noqa: E402
+
+    agent = get_mobile_use_agent()
     mcp.add_middleware(MaestroCheckerMiddleware(agent))
 
     # Start device health poller in background
@@ -70,40 +109,25 @@ def mcp_lifespan(**mcp_run_kwargs):
             stop_event,
             agent,
         ),
+        daemon=True,
     )
     poller_thread.start()
 
     try:
         mcp.run(**mcp_run_kwargs)
     except KeyboardInterrupt:
-        pass
-
-    # Stop device health poller
-    stop_event.set()
-    logger.info("Device health poller stopping...")
-    poller_thread.join()
-    logger.info("Device health poller stopped")
-
-
-def main() -> None:
-    """Main entry point for the MCP server."""
-    parser = argparse.ArgumentParser(description="Mobile Use MCP Server")
-    parser.add_argument(
-        "--server",
-        action="store_true",
-        help="Run as network server (uses MCP_SERVER_HOST and MCP_SERVER_PORT from env)",
-    )
-
-    args = parser.parse_args()
-
-    # Run MCP server with optional host/port for remote access
-    if args.server:
-        logger.info(f"Starting MCP server on {settings.MCP_SERVER_HOST}:{settings.MCP_SERVER_PORT}")
-        mcp_lifespan(
-            transport="http",
-            host=settings.MCP_SERVER_HOST,
-            port=settings.MCP_SERVER_PORT,
-        )
-    else:
-        logger.info("Starting MCP server in local mode")
-        mcp_lifespan()
+        logger.info("Keyboard interrupt received, shutting down...")
+    except Exception as e:
+        logger.error(f"Error running MCP server: {e}")
+    finally:
+        # Stop device health poller
+        logger.info("Stopping device health poller...")
+        stop_event.set()
+
+        # Give the poller thread a reasonable time to stop gracefully
+        poller_thread.join(timeout=10.0)
+
+        if poller_thread.is_alive():
+            logger.warning("Device health poller thread did not stop gracefully")
+        else:
+            logger.info("Device health poller stopped successfully")

minitap/mcp/server/poller.py

@@ -1,38 +1,78 @@
 """Device health monitoring poller for the MCP server."""
 
+import asyncio
 import logging
-import time
 import threading
 
-from minitap.mcp.core.device import list_available_devices
 from minitap.mobile_use.sdk import Agent
 
+from minitap.mcp.core.device import list_available_devices
+
 logger = logging.getLogger(__name__)
 
 
-def device_health_poller(stop_event: threading.Event, agent: Agent) -> None:
+async def _async_device_health_poller(stop_event: threading.Event, agent: Agent) -> None:
     """
-    Background poller that monitors device availability and agent health.
-    Runs every 5 seconds to ensure a device is connected and the agent is healthy.
+    Async implementation of device health poller.
 
     Args:
+        stop_event: Threading event to signal when to stop polling.
         agent: The Agent instance to monitor and reinitialize if needed.
     """
     while not stop_event.is_set():
         try:
-            time.sleep(5)
+            # Sleep in smaller chunks to be more responsive to stop signal
+            for _ in range(50):  # 50 * 0.1 = 5 seconds total
+                if stop_event.is_set():
+                    break
+                await asyncio.sleep(0.1)
+
+            if stop_event.is_set():
+                break
 
             devices = list_available_devices()
 
             if len(devices) > 0:
                 if not agent.is_healthy():
                     logger.warning("Agent is not healthy. Reinitializing...")
-                    agent.clean(force=True)
-                    agent.init()
+                    await agent.clean(force=True)
+                    await agent.init()
                     logger.info("Agent reinitialized successfully")
             else:
                 logger.info("No mobile device found, retrying in 5 seconds...")
 
         except Exception as e:
             logger.error(f"Error in device health poller: {e}")
-    agent.clean(force=True)
+
+    try:
+        await agent.clean(force=True)
+        logger.info("Agent cleaned up successfully")
+    except Exception as e:
+        logger.error(f"Error cleaning up agent: {e}")
+
+
+def device_health_poller(stop_event: threading.Event, agent: Agent) -> None:
+    """
+    Background poller that monitors device availability and agent health.
+    Runs every 5 seconds to ensure a device is connected and the agent is healthy.
+
+    This is a sync wrapper that runs the async poller in a new event loop.
+
+    Args:
+        stop_event: Threading event to signal when to stop polling.
+        agent: The Agent instance to monitor and reinitialize if needed.
+    """
+    loop = None
+    try:
+        loop = asyncio.new_event_loop()
+        asyncio.set_event_loop(loop)
+
+        loop.run_until_complete(_async_device_health_poller(stop_event, agent))
+    except Exception as e:
+        logger.error(f"Error in device health poller thread: {e}")
+    finally:
+        if loop is not None:
+            try:
+                loop.close()
+            except Exception:
+                pass