minitap-mcp 0.5.3__py3-none-any.whl → 0.6.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- minitap/mcp/core/agents/compare_screenshots/agent.py +12 -2
- minitap/mcp/core/cloud_apk.py +109 -0
- minitap/mcp/core/config.py +7 -0
- minitap/mcp/core/sdk_agent.py +8 -0
- minitap/mcp/main.py +162 -36
- minitap/mcp/server/cloud_mobile.py +397 -0
- minitap/mcp/server/middleware.py +11 -13
- minitap/mcp/server/poller.py +6 -6
- minitap/mcp/tools/analyze_screen.py +14 -3
- minitap/mcp/tools/compare_screenshot_with_figma.py +1 -1
- minitap/mcp/tools/execute_mobile_command.py +43 -2
- {minitap_mcp-0.5.3.dist-info → minitap_mcp-0.6.0.dist-info}/METADATA +57 -4
- {minitap_mcp-0.5.3.dist-info → minitap_mcp-0.6.0.dist-info}/RECORD +15 -13
- {minitap_mcp-0.5.3.dist-info → minitap_mcp-0.6.0.dist-info}/WHEEL +1 -1
- {minitap_mcp-0.5.3.dist-info → minitap_mcp-0.6.0.dist-info}/entry_points.txt +0 -0
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import asyncio
|
|
2
|
+
import base64
|
|
2
3
|
from pathlib import Path
|
|
3
4
|
from uuid import uuid4
|
|
4
5
|
|
|
@@ -9,6 +10,7 @@ from pydantic import BaseModel
|
|
|
9
10
|
from minitap.mcp.core.device import capture_screenshot, find_mobile_device
|
|
10
11
|
from minitap.mcp.core.llm import get_minitap_llm
|
|
11
12
|
from minitap.mcp.core.utils.images import get_screenshot_message_for_llm
|
|
13
|
+
from minitap.mcp.server.cloud_mobile import get_cloud_mobile_id, get_cloud_screenshot
|
|
12
14
|
|
|
13
15
|
|
|
14
16
|
class CompareScreenshotsOutput(BaseModel):
|
|
@@ -23,6 +25,8 @@ async def compare_screenshots(
|
|
|
23
25
|
"""
|
|
24
26
|
Compare screenshots and return the comparison text along with both screenshots.
|
|
25
27
|
|
|
28
|
+
Supports both local devices (Android/iOS) and cloud devices.
|
|
29
|
+
|
|
26
30
|
Returns:
|
|
27
31
|
CompareScreenshotsOutput
|
|
28
32
|
"""
|
|
@@ -30,8 +34,14 @@ async def compare_screenshots(
|
|
|
30
34
|
Path(__file__).parent.joinpath("prompt.md").read_text(encoding="utf-8")
|
|
31
35
|
).render()
|
|
32
36
|
|
|
33
|
-
|
|
34
|
-
|
|
37
|
+
cloud_mobile_id = get_cloud_mobile_id()
|
|
38
|
+
|
|
39
|
+
if cloud_mobile_id:
|
|
40
|
+
screenshot_bytes = await get_cloud_screenshot(cloud_mobile_id)
|
|
41
|
+
current_screenshot = base64.b64encode(screenshot_bytes).decode("utf-8")
|
|
42
|
+
else:
|
|
43
|
+
device = find_mobile_device()
|
|
44
|
+
current_screenshot = capture_screenshot(device)
|
|
35
45
|
|
|
36
46
|
messages: list[BaseMessage] = [
|
|
37
47
|
SystemMessage(content=system_message),
|
|
@@ -0,0 +1,109 @@
|
|
|
1
|
+
"""Cloud APK deployment utilities for uploading and installing APKs on cloud mobiles."""
|
|
2
|
+
|
|
3
|
+
import uuid
|
|
4
|
+
from pathlib import Path
|
|
5
|
+
|
|
6
|
+
import httpx
|
|
7
|
+
|
|
8
|
+
from minitap.mcp.core.config import settings
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
async def upload_apk_to_cloud_mobile(apk_path: str) -> str:
|
|
12
|
+
"""
|
|
13
|
+
Upload an APK file via Platform storage API to user storage bucket.
|
|
14
|
+
|
|
15
|
+
Args:
|
|
16
|
+
apk_path: Path to the APK file
|
|
17
|
+
|
|
18
|
+
Returns:
|
|
19
|
+
Filename to use with install-apk endpoint
|
|
20
|
+
|
|
21
|
+
Raises:
|
|
22
|
+
FileNotFoundError: If APK file doesn't exist
|
|
23
|
+
httpx.HTTPError: If upload fails
|
|
24
|
+
ValueError: If MINITAP_API_KEY or MINITAP_API_BASE_URL is not configured
|
|
25
|
+
"""
|
|
26
|
+
if not settings.MINITAP_API_KEY:
|
|
27
|
+
raise ValueError("MINITAP_API_KEY is not configured")
|
|
28
|
+
if not settings.MINITAP_API_BASE_URL:
|
|
29
|
+
raise ValueError("MINITAP_API_BASE_URL is not configured")
|
|
30
|
+
|
|
31
|
+
apk_file = Path(apk_path)
|
|
32
|
+
if not apk_file.exists():
|
|
33
|
+
raise FileNotFoundError(f"APK file not found: {apk_path}")
|
|
34
|
+
|
|
35
|
+
filename = f"app_{uuid.uuid4().hex[:6]}.apk"
|
|
36
|
+
api_key = settings.MINITAP_API_KEY.get_secret_value()
|
|
37
|
+
api_base_url = settings.MINITAP_API_BASE_URL.rstrip("/")
|
|
38
|
+
|
|
39
|
+
async with httpx.AsyncClient(timeout=300.0) as client:
|
|
40
|
+
# Step 1: Get signed upload URL from storage API
|
|
41
|
+
response = await client.get(
|
|
42
|
+
f"{api_base_url}/storage/signed-upload",
|
|
43
|
+
headers={"Authorization": f"Bearer {api_key}"},
|
|
44
|
+
params={"filenames": filename},
|
|
45
|
+
)
|
|
46
|
+
response.raise_for_status()
|
|
47
|
+
upload_data = response.json()
|
|
48
|
+
|
|
49
|
+
# Extract the signed URL for our file
|
|
50
|
+
signed_urls = upload_data.get("signed_urls", {})
|
|
51
|
+
if filename not in signed_urls:
|
|
52
|
+
raise ValueError(f"No signed URL returned for {filename}")
|
|
53
|
+
|
|
54
|
+
signed_url = signed_urls[filename]
|
|
55
|
+
|
|
56
|
+
# Step 2: Upload APK to signed URL
|
|
57
|
+
with open(apk_file, "rb") as f:
|
|
58
|
+
upload_response = await client.put(
|
|
59
|
+
signed_url,
|
|
60
|
+
content=f.read(),
|
|
61
|
+
headers={"Content-Type": "application/vnd.android.package-archive"},
|
|
62
|
+
)
|
|
63
|
+
upload_response.raise_for_status()
|
|
64
|
+
|
|
65
|
+
# Step 3: Return filename for install-apk call
|
|
66
|
+
return filename
|
|
67
|
+
|
|
68
|
+
|
|
69
|
+
async def install_apk_on_cloud_mobile(filename: str) -> None:
|
|
70
|
+
"""
|
|
71
|
+
Install an APK on a cloud mobile device via mobile-manager API.
|
|
72
|
+
|
|
73
|
+
Args:
|
|
74
|
+
filename: Filename returned from upload_apk_to_cloud_mobile
|
|
75
|
+
|
|
76
|
+
Raises:
|
|
77
|
+
httpx.HTTPError: If installation fails
|
|
78
|
+
ValueError: If required config settings are not configured
|
|
79
|
+
"""
|
|
80
|
+
if not settings.MINITAP_API_KEY:
|
|
81
|
+
raise ValueError("MINITAP_API_KEY is not configured")
|
|
82
|
+
if not settings.MINITAP_DAAS_API:
|
|
83
|
+
raise ValueError("MINITAP_DAAS_API is not configured")
|
|
84
|
+
if not settings.CLOUD_MOBILE_NAME:
|
|
85
|
+
raise ValueError("CLOUD_MOBILE_NAME is not configured")
|
|
86
|
+
|
|
87
|
+
api_key = settings.MINITAP_API_KEY.get_secret_value()
|
|
88
|
+
base_url = settings.MINITAP_DAAS_API
|
|
89
|
+
cloud_mobile_name = settings.CLOUD_MOBILE_NAME
|
|
90
|
+
|
|
91
|
+
async with httpx.AsyncClient(timeout=120.0) as client:
|
|
92
|
+
cloud_mobile_response = await client.get(
|
|
93
|
+
f"{base_url}/virtual-mobiles/{cloud_mobile_name}",
|
|
94
|
+
headers={"Authorization": f"Bearer {api_key}"},
|
|
95
|
+
)
|
|
96
|
+
cloud_mobile_response.raise_for_status()
|
|
97
|
+
response_data = cloud_mobile_response.json()
|
|
98
|
+
cloud_mobile_uuid = response_data.get("id")
|
|
99
|
+
if not cloud_mobile_uuid:
|
|
100
|
+
raise ValueError(f"Cloud mobile '{cloud_mobile_name}' response missing 'id' field")
|
|
101
|
+
response = await client.post(
|
|
102
|
+
f"{base_url}/virtual-mobiles/{cloud_mobile_uuid}/install-apk",
|
|
103
|
+
headers={
|
|
104
|
+
"Authorization": f"Bearer {api_key}",
|
|
105
|
+
"Content-Type": "application/json",
|
|
106
|
+
},
|
|
107
|
+
json={"filename": filename},
|
|
108
|
+
)
|
|
109
|
+
response.raise_for_status()
|
minitap/mcp/core/config.py
CHANGED
|
@@ -16,6 +16,7 @@ class MCPSettings(BaseSettings):
|
|
|
16
16
|
# Minitap API configuration
|
|
17
17
|
MINITAP_API_KEY: SecretStr | None = Field(default=None)
|
|
18
18
|
MINITAP_API_BASE_URL: str = Field(default="https://platform.minitap.ai/api/v1")
|
|
19
|
+
MINITAP_DAAS_API: str = Field(default="https://platform.minitap.ai/api/daas")
|
|
19
20
|
OPEN_ROUTER_API_KEY: SecretStr | None = Field(default=None)
|
|
20
21
|
|
|
21
22
|
VISION_MODEL: str = Field(default="qwen/qwen-2.5-vl-7b-instruct")
|
|
@@ -27,5 +28,11 @@ class MCPSettings(BaseSettings):
|
|
|
27
28
|
MCP_SERVER_HOST: str = Field(default="0.0.0.0")
|
|
28
29
|
MCP_SERVER_PORT: int = Field(default=8000)
|
|
29
30
|
|
|
31
|
+
# Cloud Mobile configuration
|
|
32
|
+
# When set, the MCP server runs in cloud mode connecting to a Minitap cloud mobile
|
|
33
|
+
# instead of requiring a local device. Value can be a device name.
|
|
34
|
+
# Create cloud mobiles at https://platform.minitap.ai/cloud-mobiles
|
|
35
|
+
CLOUD_MOBILE_NAME: str | None = Field(default=None)
|
|
36
|
+
|
|
30
37
|
|
|
31
38
|
settings = MCPSettings() # type: ignore
|
minitap/mcp/core/sdk_agent.py
CHANGED
|
@@ -3,6 +3,8 @@ import os
|
|
|
3
3
|
from minitap.mobile_use.sdk import Agent
|
|
4
4
|
from minitap.mobile_use.sdk.builders import Builders
|
|
5
5
|
|
|
6
|
+
from minitap.mcp.core.config import settings
|
|
7
|
+
|
|
6
8
|
# Lazy-initialized singleton agent
|
|
7
9
|
_agent: Agent | None = None
|
|
8
10
|
|
|
@@ -23,5 +25,11 @@ def get_mobile_use_agent() -> Agent:
|
|
|
23
25
|
raise ValueError(f"Invalid ADB server socket: {custom_adb_socket}")
|
|
24
26
|
_, host, port = parts
|
|
25
27
|
config = config.with_adb_server(host=host, port=int(port))
|
|
28
|
+
|
|
29
|
+
# Add cloud mobile configuration if set
|
|
30
|
+
if settings.CLOUD_MOBILE_NAME:
|
|
31
|
+
config = config.for_cloud_mobile(cloud_mobile_id_or_ref=settings.CLOUD_MOBILE_NAME)
|
|
32
|
+
|
|
26
33
|
_agent = Agent(config=config.build())
|
|
34
|
+
|
|
27
35
|
return _agent
|
minitap/mcp/main.py
CHANGED
|
@@ -4,6 +4,9 @@ import argparse
|
|
|
4
4
|
import os
|
|
5
5
|
import sys
|
|
6
6
|
import threading
|
|
7
|
+
from collections.abc import AsyncIterator
|
|
8
|
+
from contextlib import asynccontextmanager
|
|
9
|
+
from dataclasses import dataclass
|
|
7
10
|
|
|
8
11
|
# Fix Windows console encoding for Unicode characters (emojis in logs)
|
|
9
12
|
if sys.platform == "win32":
|
|
@@ -25,13 +28,16 @@ from fastmcp import FastMCP # noqa: E402
|
|
|
25
28
|
from minitap.mobile_use.config import settings as sdk_settings
|
|
26
29
|
|
|
27
30
|
from minitap.mcp.core.config import settings # noqa: E402
|
|
28
|
-
from minitap.mcp.core.device import
|
|
29
|
-
|
|
31
|
+
from minitap.mcp.core.device import (
|
|
32
|
+
DeviceInfo, # noqa: E402
|
|
33
|
+
list_available_devices,
|
|
34
|
+
)
|
|
30
35
|
from minitap.mcp.core.logging_config import (
|
|
31
36
|
configure_logging, # noqa: E402
|
|
32
37
|
get_logger,
|
|
33
38
|
)
|
|
34
|
-
from minitap.mcp.server.
|
|
39
|
+
from minitap.mcp.server.cloud_mobile import CloudMobileService
|
|
40
|
+
from minitap.mcp.server.middleware import LocalDeviceHealthMiddleware
|
|
35
41
|
from minitap.mcp.server.poller import device_health_poller
|
|
36
42
|
|
|
37
43
|
configure_logging(log_level=os.getenv("LOG_LEVEL", "INFO"))
|
|
@@ -41,7 +47,20 @@ def main() -> None:
|
|
|
41
47
|
"""Main entry point for the MCP server."""
|
|
42
48
|
|
|
43
49
|
parser = argparse.ArgumentParser(description="Mobile Use MCP Server")
|
|
44
|
-
parser.add_argument(
|
|
50
|
+
parser.add_argument(
|
|
51
|
+
"--api-key",
|
|
52
|
+
type=str,
|
|
53
|
+
required=False,
|
|
54
|
+
default=None,
|
|
55
|
+
help="Minitap API key for authentication",
|
|
56
|
+
)
|
|
57
|
+
parser.add_argument(
|
|
58
|
+
"--cloud-mobile-name",
|
|
59
|
+
type=str,
|
|
60
|
+
required=False,
|
|
61
|
+
default=None,
|
|
62
|
+
help="Name of the cloud mobile device to connect to (enables cloud mode)",
|
|
63
|
+
)
|
|
45
64
|
parser.add_argument("--llm-profile", type=str, required=False, default=None)
|
|
46
65
|
parser.add_argument(
|
|
47
66
|
"--server",
|
|
@@ -63,6 +82,11 @@ def main() -> None:
|
|
|
63
82
|
settings.__init__()
|
|
64
83
|
sdk_settings.__init__()
|
|
65
84
|
|
|
85
|
+
if args.cloud_mobile_name:
|
|
86
|
+
os.environ["CLOUD_MOBILE_NAME"] = args.cloud_mobile_name
|
|
87
|
+
settings.__init__()
|
|
88
|
+
sdk_settings.__init__()
|
|
89
|
+
|
|
66
90
|
if args.llm_profile:
|
|
67
91
|
os.environ["MINITAP_LLM_PROFILE_NAME"] = args.llm_profile
|
|
68
92
|
settings.__init__()
|
|
@@ -79,18 +103,143 @@ def main() -> None:
|
|
|
79
103
|
# Run MCP server with optional host/port for remote access
|
|
80
104
|
if args.server:
|
|
81
105
|
logger.info(f"Starting MCP server on {settings.MCP_SERVER_HOST}:{settings.MCP_SERVER_PORT}")
|
|
82
|
-
|
|
106
|
+
run_mcp_server(
|
|
83
107
|
transport="http",
|
|
84
108
|
host=settings.MCP_SERVER_HOST,
|
|
85
109
|
port=settings.MCP_SERVER_PORT,
|
|
86
110
|
)
|
|
87
111
|
else:
|
|
88
112
|
logger.info("Starting MCP server in local mode")
|
|
89
|
-
|
|
113
|
+
run_mcp_server()
|
|
90
114
|
|
|
91
115
|
|
|
92
116
|
logger = get_logger(__name__)
|
|
93
117
|
|
|
118
|
+
|
|
119
|
+
@dataclass
|
|
120
|
+
class MCPLifespanContext:
|
|
121
|
+
"""Context for MCP server lifespan.
|
|
122
|
+
|
|
123
|
+
Stores references to services that need cleanup on shutdown.
|
|
124
|
+
"""
|
|
125
|
+
|
|
126
|
+
cloud_mobile_service: CloudMobileService | None = None
|
|
127
|
+
local_poller_stop_event: threading.Event | None = None
|
|
128
|
+
local_poller_thread: threading.Thread | None = None
|
|
129
|
+
|
|
130
|
+
|
|
131
|
+
@asynccontextmanager
|
|
132
|
+
async def mcp_lifespan(server: FastMCP) -> AsyncIterator[MCPLifespanContext]:
|
|
133
|
+
"""Lifespan context manager for MCP server.
|
|
134
|
+
|
|
135
|
+
Handles startup/shutdown for both cloud and local modes.
|
|
136
|
+
|
|
137
|
+
Cloud mode (CLOUD_MOBILE_NAME set):
|
|
138
|
+
- Connects to cloud mobile on startup
|
|
139
|
+
- Maintains keep-alive polling
|
|
140
|
+
- Disconnects on shutdown (critical for billing!)
|
|
141
|
+
|
|
142
|
+
Local mode (CLOUD_MOBILE_NAME not set):
|
|
143
|
+
- Starts device health poller
|
|
144
|
+
- Monitors local device connection
|
|
145
|
+
|
|
146
|
+
Args:
|
|
147
|
+
server: The FastMCP server instance.
|
|
148
|
+
|
|
149
|
+
Yields:
|
|
150
|
+
MCPLifespanContext with references to running services.
|
|
151
|
+
|
|
152
|
+
Raises:
|
|
153
|
+
RuntimeError: If cloud mobile connection fails
|
|
154
|
+
(crashes MCP to prevent false "connected" state).
|
|
155
|
+
"""
|
|
156
|
+
from minitap.mcp.core.sdk_agent import get_mobile_use_agent # noqa: E402
|
|
157
|
+
|
|
158
|
+
context = MCPLifespanContext()
|
|
159
|
+
api_key = settings.MINITAP_API_KEY.get_secret_value() if settings.MINITAP_API_KEY else None
|
|
160
|
+
|
|
161
|
+
# Check if running in cloud mode
|
|
162
|
+
if settings.CLOUD_MOBILE_NAME:
|
|
163
|
+
# ==================== CLOUD MODE ====================
|
|
164
|
+
logger.info(f"Starting MCP in CLOUD mode with mobile: {settings.CLOUD_MOBILE_NAME}")
|
|
165
|
+
|
|
166
|
+
if not api_key:
|
|
167
|
+
logger.error("MINITAP_API_KEY is required")
|
|
168
|
+
raise RuntimeError(
|
|
169
|
+
"MINITAP_API_KEY is required when CLOUD_MOBILE_NAME is set. "
|
|
170
|
+
"Please set your API key via --api-key or MINITAP_API_KEY environment variable."
|
|
171
|
+
)
|
|
172
|
+
|
|
173
|
+
# Create and connect cloud mobile service
|
|
174
|
+
cloud_service = CloudMobileService(
|
|
175
|
+
cloud_mobile_name=settings.CLOUD_MOBILE_NAME,
|
|
176
|
+
api_key=api_key,
|
|
177
|
+
)
|
|
178
|
+
|
|
179
|
+
try:
|
|
180
|
+
await cloud_service.connect()
|
|
181
|
+
except Exception as e:
|
|
182
|
+
# CRITICAL: If cloud mobile not found, crash the MCP!
|
|
183
|
+
# This prevents the IDE from showing the server as "connected"
|
|
184
|
+
# when it actually can't do anything useful.
|
|
185
|
+
logger.error(f"Failed to connect to cloud mobile: {e}")
|
|
186
|
+
raise RuntimeError(
|
|
187
|
+
f"Cloud mobile connection failed. The MCP server cannot start.\n{e}"
|
|
188
|
+
) from e
|
|
189
|
+
|
|
190
|
+
context.cloud_mobile_service = cloud_service
|
|
191
|
+
logger.info("Cloud mobile connected, MCP server ready")
|
|
192
|
+
|
|
193
|
+
else:
|
|
194
|
+
# ==================== LOCAL MODE ====================
|
|
195
|
+
logger.info("Starting MCP in LOCAL mode (no CLOUD_MOBILE_NAME set)")
|
|
196
|
+
|
|
197
|
+
agent = get_mobile_use_agent()
|
|
198
|
+
server.add_middleware(LocalDeviceHealthMiddleware(agent))
|
|
199
|
+
|
|
200
|
+
# Start device health poller in background
|
|
201
|
+
stop_event = threading.Event()
|
|
202
|
+
poller_thread = threading.Thread(
|
|
203
|
+
target=device_health_poller,
|
|
204
|
+
args=(stop_event, agent),
|
|
205
|
+
daemon=True,
|
|
206
|
+
)
|
|
207
|
+
poller_thread.start()
|
|
208
|
+
|
|
209
|
+
context.local_poller_stop_event = stop_event
|
|
210
|
+
context.local_poller_thread = poller_thread
|
|
211
|
+
logger.info("Device health poller started")
|
|
212
|
+
|
|
213
|
+
try:
|
|
214
|
+
yield context
|
|
215
|
+
finally:
|
|
216
|
+
# ==================== SHUTDOWN ====================
|
|
217
|
+
logger.info("MCP server shutting down, cleaning up resources...")
|
|
218
|
+
|
|
219
|
+
if context.cloud_mobile_service:
|
|
220
|
+
# CRITICAL: Stop cloud mobile connection to stop billing!
|
|
221
|
+
logger.info("Disconnecting cloud mobile (stopping billing)...")
|
|
222
|
+
try:
|
|
223
|
+
await context.cloud_mobile_service.disconnect()
|
|
224
|
+
logger.info("Cloud mobile disconnected successfully")
|
|
225
|
+
except Exception as e:
|
|
226
|
+
logger.error(f"Error disconnecting cloud mobile: {e}")
|
|
227
|
+
|
|
228
|
+
if context.local_poller_stop_event and context.local_poller_thread:
|
|
229
|
+
# Stop local device health poller
|
|
230
|
+
logger.info("Stopping device health poller...")
|
|
231
|
+
context.local_poller_stop_event.set()
|
|
232
|
+
context.local_poller_thread.join(timeout=10.0)
|
|
233
|
+
|
|
234
|
+
if context.local_poller_thread.is_alive():
|
|
235
|
+
logger.warning("Device health poller thread did not stop gracefully")
|
|
236
|
+
else:
|
|
237
|
+
logger.info("Device health poller stopped successfully")
|
|
238
|
+
|
|
239
|
+
logger.info("MCP server shutdown complete")
|
|
240
|
+
|
|
241
|
+
|
|
242
|
+
# Create MCP server with lifespan handler
|
|
94
243
|
mcp = FastMCP(
|
|
95
244
|
name="mobile-use-mcp",
|
|
96
245
|
instructions="""
|
|
@@ -98,7 +247,9 @@ mcp = FastMCP(
|
|
|
98
247
|
mobile devices (iOS or Android).
|
|
99
248
|
Call get_available_devices() to list them.
|
|
100
249
|
""",
|
|
250
|
+
lifespan=mcp_lifespan,
|
|
101
251
|
)
|
|
252
|
+
|
|
102
253
|
from minitap.mcp.tools import analyze_screen # noqa: E402, F401
|
|
103
254
|
from minitap.mcp.tools import compare_screenshot_with_figma # noqa: E402, F401
|
|
104
255
|
from minitap.mcp.tools import execute_mobile_command # noqa: E402, F401
|
|
@@ -111,40 +262,15 @@ def get_available_devices() -> list[DeviceInfo]:
|
|
|
111
262
|
return list_available_devices()
|
|
112
263
|
|
|
113
264
|
|
|
114
|
-
def
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
agent = get_mobile_use_agent()
|
|
118
|
-
mcp.add_middleware(MaestroCheckerMiddleware(agent))
|
|
119
|
-
|
|
120
|
-
# Start device health poller in background
|
|
121
|
-
logger.info("Device health poller started")
|
|
122
|
-
stop_event = threading.Event()
|
|
123
|
-
poller_thread = threading.Thread(
|
|
124
|
-
target=device_health_poller,
|
|
125
|
-
args=(
|
|
126
|
-
stop_event,
|
|
127
|
-
agent,
|
|
128
|
-
),
|
|
129
|
-
daemon=True,
|
|
130
|
-
)
|
|
131
|
-
poller_thread.start()
|
|
265
|
+
def run_mcp_server(**mcp_run_kwargs):
|
|
266
|
+
"""Run the MCP server with proper exception handling.
|
|
132
267
|
|
|
268
|
+
This wraps mcp.run() with exception handling for clean shutdown.
|
|
269
|
+
"""
|
|
133
270
|
try:
|
|
134
271
|
mcp.run(**mcp_run_kwargs)
|
|
135
272
|
except KeyboardInterrupt:
|
|
136
273
|
logger.info("Keyboard interrupt received, shutting down...")
|
|
137
274
|
except Exception as e:
|
|
138
275
|
logger.error(f"Error running MCP server: {e}")
|
|
139
|
-
|
|
140
|
-
# Stop device health poller
|
|
141
|
-
logger.info("Stopping device health poller...")
|
|
142
|
-
stop_event.set()
|
|
143
|
-
|
|
144
|
-
# Give the poller thread a reasonable time to stop gracefully
|
|
145
|
-
poller_thread.join(timeout=10.0)
|
|
146
|
-
|
|
147
|
-
if poller_thread.is_alive():
|
|
148
|
-
logger.warning("Device health poller thread did not stop gracefully")
|
|
149
|
-
else:
|
|
150
|
-
logger.info("Device health poller stopped successfully")
|
|
276
|
+
raise
|
|
@@ -0,0 +1,397 @@
|
|
|
1
|
+
"""Cloud mobile service for managing cloud-hosted mobile devices.
|
|
2
|
+
|
|
3
|
+
This module handles:
|
|
4
|
+
- Connecting to cloud mobiles on the Minitap platform via HTTP API
|
|
5
|
+
- Keep-alive polling to maintain the connection (prevents idle shutdown)
|
|
6
|
+
- Proper cleanup when the MCP server stops
|
|
7
|
+
|
|
8
|
+
API Endpoints used:
|
|
9
|
+
- GET /api/daas/virtual-mobiles/{id} - Fetch device info by ID or reference name
|
|
10
|
+
- POST /api/daas/virtual-mobiles/{id}/keep-alive - Keep device alive (prevents billing timeout)
|
|
11
|
+
"""
|
|
12
|
+
|
|
13
|
+
import asyncio
|
|
14
|
+
import logging
|
|
15
|
+
from contextvars import ContextVar
|
|
16
|
+
from dataclasses import dataclass
|
|
17
|
+
from typing import Any
|
|
18
|
+
|
|
19
|
+
import aiohttp
|
|
20
|
+
|
|
21
|
+
from minitap.mcp.core.config import settings
|
|
22
|
+
|
|
23
|
+
logger = logging.getLogger(__name__)
|
|
24
|
+
|
|
25
|
+
# Context variable to store cloud mobile ID accessible from any MCP tool
|
|
26
|
+
# This is server-wide (not request-scoped) as it persists for the MCP lifecycle
|
|
27
|
+
_cloud_mobile_id: ContextVar[str | None] = ContextVar("cloud_mobile_id", default=None)
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
def get_cloud_mobile_id() -> str | None:
|
|
31
|
+
"""Get the current cloud mobile ID (UUID) from context.
|
|
32
|
+
|
|
33
|
+
Returns:
|
|
34
|
+
The cloud mobile UUID if running in cloud mode, None otherwise.
|
|
35
|
+
"""
|
|
36
|
+
return _cloud_mobile_id.get()
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
def set_cloud_mobile_id(mobile_id: str | None) -> None:
|
|
40
|
+
"""Set the cloud mobile ID in context.
|
|
41
|
+
|
|
42
|
+
Args:
|
|
43
|
+
mobile_id: The cloud mobile UUID or None to clear.
|
|
44
|
+
"""
|
|
45
|
+
_cloud_mobile_id.set(mobile_id)
|
|
46
|
+
|
|
47
|
+
|
|
48
|
+
@dataclass
|
|
49
|
+
class VirtualMobileInfo:
|
|
50
|
+
"""Information about a virtual mobile from the API."""
|
|
51
|
+
|
|
52
|
+
id: str # UUID
|
|
53
|
+
reference_name: str | None
|
|
54
|
+
state: str # Ready, Starting, Stopping, Stopped, Error
|
|
55
|
+
uptime_seconds: int
|
|
56
|
+
cost_micro_dollars: int
|
|
57
|
+
|
|
58
|
+
@classmethod
|
|
59
|
+
def from_api_response(cls, data: dict[str, Any]) -> "VirtualMobileInfo":
|
|
60
|
+
"""Create from API response."""
|
|
61
|
+
return cls(
|
|
62
|
+
id=data["id"],
|
|
63
|
+
reference_name=data.get("referenceName"),
|
|
64
|
+
state=data.get("state", {}).get("current", "Unknown"),
|
|
65
|
+
uptime_seconds=data.get("uptimeSeconds", 0),
|
|
66
|
+
cost_micro_dollars=data.get("costMicroDollars", 0),
|
|
67
|
+
)
|
|
68
|
+
|
|
69
|
+
|
|
70
|
+
class CloudMobileService:
|
|
71
|
+
"""Service for managing cloud mobile connections via HTTP API.
|
|
72
|
+
|
|
73
|
+
This service handles:
|
|
74
|
+
1. Fetching cloud mobile info by name/UUID
|
|
75
|
+
2. Sending keep-alive pings to prevent idle shutdown
|
|
76
|
+
3. Proper cleanup on MCP server shutdown
|
|
77
|
+
|
|
78
|
+
The keep-alive is CRITICAL for billing - the platform will shut down
|
|
79
|
+
idle VMs after a timeout. The MCP server must send keep-alives while
|
|
80
|
+
waiting for user commands.
|
|
81
|
+
"""
|
|
82
|
+
|
|
83
|
+
KEEP_ALIVE_INTERVAL_SECONDS = 30
|
|
84
|
+
API_TIMEOUT_SECONDS = 30
|
|
85
|
+
|
|
86
|
+
def __init__(self, cloud_mobile_name: str, api_key: str):
|
|
87
|
+
"""Initialize the cloud mobile service.
|
|
88
|
+
|
|
89
|
+
Args:
|
|
90
|
+
cloud_mobile_name: The reference name or UUID of the cloud mobile.
|
|
91
|
+
api_key: The Minitap API key for authentication.
|
|
92
|
+
"""
|
|
93
|
+
self.cloud_mobile_name = cloud_mobile_name
|
|
94
|
+
self.api_key = api_key
|
|
95
|
+
self._base_url = settings.MINITAP_DAAS_API.rstrip("/")
|
|
96
|
+
self._mobile_id: str | None = None # UUID, resolved from name
|
|
97
|
+
self._mobile_info: VirtualMobileInfo | None = None
|
|
98
|
+
self._keep_alive_task: asyncio.Task | None = None
|
|
99
|
+
self._stop_event = asyncio.Event()
|
|
100
|
+
self._session: aiohttp.ClientSession | None = None
|
|
101
|
+
|
|
102
|
+
def _get_headers(self) -> dict[str, str]:
|
|
103
|
+
"""Get HTTP headers for API requests."""
|
|
104
|
+
return {
|
|
105
|
+
"Authorization": f"Bearer {self.api_key}",
|
|
106
|
+
"Content-Type": "application/json",
|
|
107
|
+
}
|
|
108
|
+
|
|
109
|
+
async def _ensure_session(self) -> aiohttp.ClientSession:
|
|
110
|
+
"""Get or create the aiohttp session."""
|
|
111
|
+
if self._session is None or self._session.closed:
|
|
112
|
+
timeout = aiohttp.ClientTimeout(total=self.API_TIMEOUT_SECONDS)
|
|
113
|
+
self._session = aiohttp.ClientSession(timeout=timeout)
|
|
114
|
+
return self._session
|
|
115
|
+
|
|
116
|
+
async def _close_session(self) -> None:
|
|
117
|
+
"""Close the aiohttp session."""
|
|
118
|
+
if self._session and not self._session.closed:
|
|
119
|
+
await self._session.close()
|
|
120
|
+
self._session = None
|
|
121
|
+
|
|
122
|
+
async def _fetch_virtual_mobile(self) -> VirtualMobileInfo:
|
|
123
|
+
"""Fetch virtual mobile info from API.
|
|
124
|
+
|
|
125
|
+
GET /api/daas/virtual-mobiles/{id}
|
|
126
|
+
|
|
127
|
+
Args:
|
|
128
|
+
id can be UUID or reference name.
|
|
129
|
+
|
|
130
|
+
Returns:
|
|
131
|
+
VirtualMobileInfo with device details.
|
|
132
|
+
|
|
133
|
+
Raises:
|
|
134
|
+
RuntimeError: If device not found or API error.
|
|
135
|
+
"""
|
|
136
|
+
session = await self._ensure_session()
|
|
137
|
+
url = f"{self._base_url}/virtual-mobiles/{self.cloud_mobile_name}"
|
|
138
|
+
|
|
139
|
+
logger.debug(f"Fetching virtual mobile: {url}")
|
|
140
|
+
|
|
141
|
+
async with session.get(url, headers=self._get_headers()) as response:
|
|
142
|
+
if response.status == 404:
|
|
143
|
+
raise RuntimeError(
|
|
144
|
+
f"Cloud mobile '{self.cloud_mobile_name}' not found. "
|
|
145
|
+
"Please verify the name/UUID exists in your Minitap Platform account."
|
|
146
|
+
)
|
|
147
|
+
if response.status == 401:
|
|
148
|
+
raise RuntimeError(
|
|
149
|
+
"Authentication failed. Please verify your MINITAP_API_KEY is valid."
|
|
150
|
+
)
|
|
151
|
+
if response.status == 403:
|
|
152
|
+
raise RuntimeError(
|
|
153
|
+
f"Access denied to cloud mobile '{self.cloud_mobile_name}'. "
|
|
154
|
+
"Please verify your API key has access to this device."
|
|
155
|
+
)
|
|
156
|
+
if response.status != 200:
|
|
157
|
+
error_text = await response.text()
|
|
158
|
+
raise RuntimeError(
|
|
159
|
+
f"Failed to fetch cloud mobile: HTTP {response.status} - {error_text}"
|
|
160
|
+
)
|
|
161
|
+
|
|
162
|
+
data = await response.json()
|
|
163
|
+
return VirtualMobileInfo.from_api_response(data)
|
|
164
|
+
|
|
165
|
+
async def _send_keep_alive(self) -> bool:
|
|
166
|
+
"""Send keep-alive ping to prevent idle shutdown.
|
|
167
|
+
|
|
168
|
+
POST /api/daas/virtual-mobiles/{id}/keep-alive
|
|
169
|
+
|
|
170
|
+
Returns:
|
|
171
|
+
True if successful, False otherwise.
|
|
172
|
+
"""
|
|
173
|
+
if not self._mobile_id:
|
|
174
|
+
logger.warning("Cannot send keep-alive: no mobile ID")
|
|
175
|
+
return False
|
|
176
|
+
|
|
177
|
+
session = await self._ensure_session()
|
|
178
|
+
url = f"{self._base_url}/virtual-mobiles/{self._mobile_id}/keep-alive"
|
|
179
|
+
|
|
180
|
+
try:
|
|
181
|
+
async with session.post(url, headers=self._get_headers()) as response:
|
|
182
|
+
if response.status == 204:
|
|
183
|
+
logger.debug(f"Keep-alive sent successfully for {self._mobile_id}")
|
|
184
|
+
return True
|
|
185
|
+
else:
|
|
186
|
+
error_text = await response.text()
|
|
187
|
+
logger.warning(f"Keep-alive failed: HTTP {response.status} - {error_text}")
|
|
188
|
+
return False
|
|
189
|
+
except Exception as e:
|
|
190
|
+
logger.error(f"Keep-alive request failed: {e}")
|
|
191
|
+
return False
|
|
192
|
+
|
|
193
|
+
async def connect(self) -> None:
|
|
194
|
+
"""Connect to the cloud mobile and start keep-alive polling.
|
|
195
|
+
|
|
196
|
+
1. Fetches device info to verify it exists
|
|
197
|
+
2. Stores the UUID for keep-alive calls
|
|
198
|
+
3. Starts background keep-alive polling
|
|
199
|
+
|
|
200
|
+
Raises:
|
|
201
|
+
RuntimeError: If no cloud mobile is found with the given name.
|
|
202
|
+
"""
|
|
203
|
+
logger.info(f"Connecting to cloud mobile: {self.cloud_mobile_name}")
|
|
204
|
+
|
|
205
|
+
try:
|
|
206
|
+
# Fetch device info to verify it exists and get UUID
|
|
207
|
+
self._mobile_info = await self._fetch_virtual_mobile()
|
|
208
|
+
self._mobile_id = self._mobile_info.id
|
|
209
|
+
|
|
210
|
+
logger.info(
|
|
211
|
+
f"Connected to cloud mobile: {self.cloud_mobile_name} "
|
|
212
|
+
f"(id={self._mobile_id}, state={self._mobile_info.state})"
|
|
213
|
+
)
|
|
214
|
+
|
|
215
|
+
# Check if device is in a usable state
|
|
216
|
+
if self._mobile_info.state not in ("Ready", "Starting"):
|
|
217
|
+
logger.warning(
|
|
218
|
+
f"Cloud mobile state is '{self._mobile_info.state}'. "
|
|
219
|
+
"Device may not be ready for use."
|
|
220
|
+
)
|
|
221
|
+
|
|
222
|
+
# Store the mobile ID in context for access from MCP tools
|
|
223
|
+
set_cloud_mobile_id(self._mobile_id)
|
|
224
|
+
|
|
225
|
+
# Send initial keep-alive
|
|
226
|
+
await self._send_keep_alive()
|
|
227
|
+
|
|
228
|
+
# Start keep-alive polling
|
|
229
|
+
self._stop_event.clear()
|
|
230
|
+
self._keep_alive_task = asyncio.create_task(
|
|
231
|
+
self._keep_alive_loop(),
|
|
232
|
+
name=f"cloud_mobile_keep_alive_{self._mobile_id}",
|
|
233
|
+
)
|
|
234
|
+
|
|
235
|
+
except Exception as e:
|
|
236
|
+
logger.error(f"Failed to connect to cloud mobile '{self.cloud_mobile_name}': {e}")
|
|
237
|
+
await self._close_session()
|
|
238
|
+
raise RuntimeError(
|
|
239
|
+
f"Failed to connect to cloud mobile '{self.cloud_mobile_name}'. "
|
|
240
|
+
"Please verify:\n"
|
|
241
|
+
" 1. The cloud mobile exists in your Minitap Platform account\n"
|
|
242
|
+
" 2. The CLOUD_MOBILE_NAME matches exactly (case-sensitive)\n"
|
|
243
|
+
" 3. Your MINITAP_API_KEY has access to this cloud mobile\n"
|
|
244
|
+
f"Original error: {e}"
|
|
245
|
+
) from e
|
|
246
|
+
|
|
247
|
+
async def _keep_alive_loop(self) -> None:
|
|
248
|
+
"""Background task that sends periodic keep-alive pings.
|
|
249
|
+
|
|
250
|
+
This maintains the connection to the cloud mobile and prevents
|
|
251
|
+
idle shutdown (which would stop billing but also lose the session).
|
|
252
|
+
"""
|
|
253
|
+
logger.info(
|
|
254
|
+
f"Starting cloud mobile keep-alive polling "
|
|
255
|
+
f"(interval={self.KEEP_ALIVE_INTERVAL_SECONDS}s)"
|
|
256
|
+
)
|
|
257
|
+
|
|
258
|
+
consecutive_failures = 0
|
|
259
|
+
max_failures = 3
|
|
260
|
+
|
|
261
|
+
while not self._stop_event.is_set():
|
|
262
|
+
try:
|
|
263
|
+
# Wait for the interval, but check stop_event frequently
|
|
264
|
+
for _ in range(self.KEEP_ALIVE_INTERVAL_SECONDS * 10):
|
|
265
|
+
if self._stop_event.is_set():
|
|
266
|
+
break
|
|
267
|
+
await asyncio.sleep(0.1)
|
|
268
|
+
|
|
269
|
+
if self._stop_event.is_set():
|
|
270
|
+
break
|
|
271
|
+
|
|
272
|
+
# Send keep-alive ping
|
|
273
|
+
success = await self._send_keep_alive()
|
|
274
|
+
|
|
275
|
+
if success:
|
|
276
|
+
consecutive_failures = 0
|
|
277
|
+
else:
|
|
278
|
+
consecutive_failures += 1
|
|
279
|
+
if consecutive_failures >= max_failures:
|
|
280
|
+
logger.error(
|
|
281
|
+
f"Keep-alive failed {max_failures} times consecutively. "
|
|
282
|
+
"Cloud mobile may have been shut down."
|
|
283
|
+
)
|
|
284
|
+
# Don't stop the loop - keep trying in case it recovers
|
|
285
|
+
|
|
286
|
+
except asyncio.CancelledError:
|
|
287
|
+
logger.info("Cloud mobile keep-alive task cancelled")
|
|
288
|
+
break
|
|
289
|
+
except Exception as e:
|
|
290
|
+
logger.error(f"Error in cloud mobile keep-alive: {e}")
|
|
291
|
+
consecutive_failures += 1
|
|
292
|
+
# Don't break on errors, try to continue
|
|
293
|
+
await asyncio.sleep(5)
|
|
294
|
+
|
|
295
|
+
logger.info("Cloud mobile keep-alive polling stopped")
|
|
296
|
+
|
|
297
|
+
async def disconnect(self) -> None:
|
|
298
|
+
"""Disconnect from the cloud mobile and stop keep-alive polling.
|
|
299
|
+
|
|
300
|
+
This MUST be called when the MCP server shuts down to:
|
|
301
|
+
1. Stop keep-alive polling (allows VM to idle-shutdown if not used)
|
|
302
|
+
2. Clean up HTTP session
|
|
303
|
+
|
|
304
|
+
Note: We intentionally do NOT call a "stop" endpoint here.
|
|
305
|
+
Stopping keep-alive will let the VM idle-shutdown naturally
|
|
306
|
+
after its configured timeout, which is the expected behavior.
|
|
307
|
+
"""
|
|
308
|
+
logger.info(f"Disconnecting from cloud mobile: {self.cloud_mobile_name}")
|
|
309
|
+
|
|
310
|
+
# Signal the keep-alive loop to stop
|
|
311
|
+
self._stop_event.set()
|
|
312
|
+
|
|
313
|
+
# Cancel and wait for keep-alive task
|
|
314
|
+
if self._keep_alive_task and not self._keep_alive_task.done():
|
|
315
|
+
self._keep_alive_task.cancel()
|
|
316
|
+
try:
|
|
317
|
+
await asyncio.wait_for(self._keep_alive_task, timeout=5.0)
|
|
318
|
+
except TimeoutError:
|
|
319
|
+
logger.warning("Keep-alive task did not stop in time")
|
|
320
|
+
except asyncio.CancelledError:
|
|
321
|
+
pass
|
|
322
|
+
|
|
323
|
+
# Close HTTP session
|
|
324
|
+
await self._close_session()
|
|
325
|
+
|
|
326
|
+
# Clear the context
|
|
327
|
+
set_cloud_mobile_id(None)
|
|
328
|
+
self._mobile_id = None
|
|
329
|
+
self._mobile_info = None
|
|
330
|
+
|
|
331
|
+
logger.info("Cloud mobile disconnected (keep-alive stopped)")
|
|
332
|
+
|
|
333
|
+
@property
|
|
334
|
+
def mobile_id(self) -> str | None:
|
|
335
|
+
"""Get the cloud mobile UUID."""
|
|
336
|
+
return self._mobile_id
|
|
337
|
+
|
|
338
|
+
@property
|
|
339
|
+
def mobile_info(self) -> VirtualMobileInfo | None:
|
|
340
|
+
"""Get the cloud mobile info."""
|
|
341
|
+
return self._mobile_info
|
|
342
|
+
|
|
343
|
+
@property
|
|
344
|
+
def is_connected(self) -> bool:
|
|
345
|
+
"""Check if connected to a cloud mobile."""
|
|
346
|
+
return self._mobile_id is not None
|
|
347
|
+
|
|
348
|
+
|
|
349
|
+
async def get_cloud_screenshot(mobile_id: str | None = None) -> bytes:
|
|
350
|
+
"""Get a screenshot from a cloud mobile device.
|
|
351
|
+
|
|
352
|
+
GET /api/daas/virtual-mobiles/{id}/screenshot
|
|
353
|
+
|
|
354
|
+
Args:
|
|
355
|
+
mobile_id: The cloud mobile UUID. If None, uses the current context.
|
|
356
|
+
|
|
357
|
+
Returns:
|
|
358
|
+
Screenshot image bytes (PNG format).
|
|
359
|
+
|
|
360
|
+
Raises:
|
|
361
|
+
RuntimeError: If no cloud mobile is connected or screenshot fails.
|
|
362
|
+
"""
|
|
363
|
+
target_id = mobile_id or get_cloud_mobile_id()
|
|
364
|
+
|
|
365
|
+
if not target_id:
|
|
366
|
+
raise RuntimeError(
|
|
367
|
+
"No cloud mobile connected. "
|
|
368
|
+
"Either provide a mobile_id or ensure CLOUD_MOBILE_NAME is set."
|
|
369
|
+
)
|
|
370
|
+
|
|
371
|
+
api_key = settings.MINITAP_API_KEY.get_secret_value() if settings.MINITAP_API_KEY else None
|
|
372
|
+
if not api_key:
|
|
373
|
+
raise RuntimeError("MINITAP_API_KEY is required for cloud screenshot.")
|
|
374
|
+
|
|
375
|
+
base_url = settings.MINITAP_DAAS_API.rstrip("/")
|
|
376
|
+
url = f"{base_url}/virtual-mobiles/{target_id}/screenshot"
|
|
377
|
+
|
|
378
|
+
headers = {
|
|
379
|
+
"Authorization": f"Bearer {api_key}",
|
|
380
|
+
}
|
|
381
|
+
|
|
382
|
+
timeout = aiohttp.ClientTimeout(total=30)
|
|
383
|
+
async with aiohttp.ClientSession(timeout=timeout) as session:
|
|
384
|
+
async with session.get(url, headers=headers) as response:
|
|
385
|
+
if response.status == 404:
|
|
386
|
+
raise RuntimeError(f"Cloud mobile '{target_id}' not found.")
|
|
387
|
+
if response.status == 401:
|
|
388
|
+
raise RuntimeError("Authentication failed for cloud screenshot.")
|
|
389
|
+
if response.status == 403:
|
|
390
|
+
raise RuntimeError(f"Access denied to cloud mobile '{target_id}'.")
|
|
391
|
+
if response.status != 200:
|
|
392
|
+
error_text = await response.text()
|
|
393
|
+
raise RuntimeError(
|
|
394
|
+
f"Failed to get cloud screenshot: HTTP {response.status} - {error_text}"
|
|
395
|
+
)
|
|
396
|
+
|
|
397
|
+
return await response.read()
|
minitap/mcp/server/middleware.py
CHANGED
|
@@ -1,23 +1,21 @@
|
|
|
1
1
|
from fastmcp.exceptions import ToolError
|
|
2
2
|
from fastmcp.server.middleware import Middleware, MiddlewareContext
|
|
3
|
-
|
|
4
3
|
from minitap.mobile_use.sdk import Agent
|
|
5
4
|
|
|
6
5
|
|
|
7
|
-
class
|
|
6
|
+
class LocalDeviceHealthMiddleware(Middleware):
|
|
7
|
+
"""Middleware that checks local device health before tool calls.
|
|
8
|
+
|
|
9
|
+
Only used in local mode (when CLOUD_MOBILE_NAME is not set).
|
|
10
|
+
For cloud mode, device health is managed by the cloud service.
|
|
11
|
+
"""
|
|
12
|
+
|
|
8
13
|
def __init__(self, agent: Agent):
|
|
9
14
|
self.agent = agent
|
|
10
15
|
|
|
11
16
|
async def on_call_tool(self, context: MiddlewareContext, call_next):
|
|
12
|
-
if
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
if not self.agent.is_healthy():
|
|
17
|
-
raise ToolError(
|
|
18
|
-
"Maestro not healthy.\n"
|
|
19
|
-
"Make sure a mobile device is connected and try again."
|
|
20
|
-
)
|
|
21
|
-
except Exception:
|
|
22
|
-
pass
|
|
17
|
+
if not self.agent._initialized:
|
|
18
|
+
raise ToolError(
|
|
19
|
+
"Agent not initialized.\nMake sure a mobile device is connected and try again."
|
|
20
|
+
)
|
|
23
21
|
return await call_next(context)
|
minitap/mcp/server/poller.py
CHANGED
|
@@ -33,11 +33,10 @@ async def _async_device_health_poller(stop_event: threading.Event, agent: Agent)
|
|
|
33
33
|
devices = list_available_devices()
|
|
34
34
|
|
|
35
35
|
if len(devices) > 0:
|
|
36
|
-
if not agent.
|
|
37
|
-
logger.warning("Agent is not
|
|
38
|
-
await agent.clean(force=True)
|
|
36
|
+
if not agent._initialized:
|
|
37
|
+
logger.warning("Agent is not initialized. Initializing...")
|
|
39
38
|
await agent.init()
|
|
40
|
-
logger.info("Agent
|
|
39
|
+
logger.info("Agent initialized successfully")
|
|
41
40
|
else:
|
|
42
41
|
logger.info("No mobile device found, retrying in 5 seconds...")
|
|
43
42
|
|
|
@@ -45,8 +44,9 @@ async def _async_device_health_poller(stop_event: threading.Event, agent: Agent)
|
|
|
45
44
|
logger.error(f"Error in device health poller: {e}")
|
|
46
45
|
|
|
47
46
|
try:
|
|
48
|
-
|
|
49
|
-
|
|
47
|
+
if agent._initialized:
|
|
48
|
+
await agent.clean(force=True)
|
|
49
|
+
logger.info("Agent cleaned up successfully")
|
|
50
50
|
except Exception as e:
|
|
51
51
|
logger.error(f"Error cleaning up agent: {e}")
|
|
52
52
|
|
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import base64
|
|
1
2
|
from pathlib import Path
|
|
2
3
|
from uuid import uuid4
|
|
3
4
|
|
|
@@ -11,6 +12,7 @@ from minitap.mcp.core.device import capture_screenshot, find_mobile_device
|
|
|
11
12
|
from minitap.mcp.core.llm import get_minitap_llm
|
|
12
13
|
from minitap.mcp.core.utils.images import compress_base64_jpeg, get_screenshot_message_for_llm
|
|
13
14
|
from minitap.mcp.main import mcp
|
|
15
|
+
from minitap.mcp.server.cloud_mobile import get_cloud_mobile_id, get_cloud_screenshot
|
|
14
16
|
|
|
15
17
|
|
|
16
18
|
@mcp.tool(
|
|
@@ -37,9 +39,18 @@ async def analyze_screen(
|
|
|
37
39
|
Path(__file__).parent.joinpath("screen_analyzer.md").read_text(encoding="utf-8")
|
|
38
40
|
).render()
|
|
39
41
|
|
|
40
|
-
#
|
|
41
|
-
|
|
42
|
-
|
|
42
|
+
# Check if running in cloud mode
|
|
43
|
+
cloud_mobile_id = get_cloud_mobile_id()
|
|
44
|
+
|
|
45
|
+
if cloud_mobile_id:
|
|
46
|
+
# Cloud mode: fetch screenshot from DaaS API
|
|
47
|
+
screenshot_bytes = await get_cloud_screenshot(cloud_mobile_id)
|
|
48
|
+
screenshot_base64 = base64.b64encode(screenshot_bytes).decode("utf-8")
|
|
49
|
+
else:
|
|
50
|
+
# Local mode: capture from local device
|
|
51
|
+
device = find_mobile_device(device_id=device_id)
|
|
52
|
+
screenshot_base64 = capture_screenshot(device)
|
|
53
|
+
|
|
43
54
|
compressed_image_base64 = compress_base64_jpeg(screenshot_base64)
|
|
44
55
|
|
|
45
56
|
messages: list[BaseMessage] = [
|
|
@@ -23,7 +23,7 @@ from minitap.mcp.main import mcp
|
|
|
23
23
|
Compare a screenshot of the current state with a Figma design.
|
|
24
24
|
|
|
25
25
|
This tool:
|
|
26
|
-
1. Captures a screenshot of the current state
|
|
26
|
+
1. Captures a screenshot of the current state (supports both local and cloud devices)
|
|
27
27
|
2. Compares the live device screenshot with the Figma design
|
|
28
28
|
3. Returns a detailed comparison report with both screenshots for visual context
|
|
29
29
|
""",
|
|
@@ -8,6 +8,8 @@ from minitap.mobile_use.sdk.types import ManualTaskConfig
|
|
|
8
8
|
from minitap.mobile_use.sdk.types.task import PlatformTaskRequest
|
|
9
9
|
from pydantic import Field
|
|
10
10
|
|
|
11
|
+
from minitap.mcp.core.cloud_apk import install_apk_on_cloud_mobile, upload_apk_to_cloud_mobile
|
|
12
|
+
from minitap.mcp.core.config import settings
|
|
11
13
|
from minitap.mcp.core.decorators import handle_tool_errors
|
|
12
14
|
from minitap.mcp.core.sdk_agent import get_mobile_use_agent
|
|
13
15
|
from minitap.mcp.main import mcp
|
|
@@ -26,14 +28,30 @@ def _serialize_result(result: Any) -> Any:
|
|
|
26
28
|
|
|
27
29
|
@mcp.tool(
|
|
28
30
|
name="execute_mobile_command",
|
|
29
|
-
tags={"requires-maestro"},
|
|
30
31
|
description="""
|
|
31
32
|
Execute a natural language command on a mobile device using the Minitap SDK.
|
|
32
33
|
This tool allows you to control your Android or iOS device using natural language.
|
|
34
|
+
|
|
33
35
|
Examples:
|
|
34
36
|
- "Open the settings app and tell me the battery level"
|
|
35
37
|
- "Find the first 3 unread emails in Gmail"
|
|
36
38
|
- "Take a screenshot and save it"
|
|
39
|
+
|
|
40
|
+
APK Deployment (Cloud Mobile Only):
|
|
41
|
+
When CLOUD_MOBILE_NAME is set, you can deploy and test APKs on cloud mobiles:
|
|
42
|
+
- Set apk_path to the path of your locally built APK
|
|
43
|
+
- The APK will be uploaded to cloud storage and installed on the device
|
|
44
|
+
- Requires MINITAP_API_KEY environment variable
|
|
45
|
+
- Must provide locked_app_package when using apk_path
|
|
46
|
+
|
|
47
|
+
Example with APK deployment:
|
|
48
|
+
execute_mobile_command(
|
|
49
|
+
apk_path="/path/to/app-debug.apk",
|
|
50
|
+
locked_app_package="com.example.myapp",
|
|
51
|
+
goal="Test the login flow with valid credentials"
|
|
52
|
+
)
|
|
53
|
+
|
|
54
|
+
Note: If apk path is set and no cloud mobile name -> it will raise a tool error
|
|
37
55
|
""",
|
|
38
56
|
)
|
|
39
57
|
@handle_tool_errors
|
|
@@ -49,18 +67,41 @@ async def execute_mobile_command(
|
|
|
49
67
|
default=None,
|
|
50
68
|
description="Optional package name of the app to lock the device to. "
|
|
51
69
|
"Will launch the app if not already running, and keep it in foreground "
|
|
52
|
-
"until the task is completed.",
|
|
70
|
+
"until the task is completed. REQUIRED when using apk_path.",
|
|
71
|
+
),
|
|
72
|
+
apk_path: str | None = Field(
|
|
73
|
+
default=None,
|
|
74
|
+
description="Path to local APK file to deploy to cloud mobile. "
|
|
75
|
+
"Only works when CLOUD_MOBILE_NAME is set. "
|
|
76
|
+
"The APK will be uploaded to cloud storage and installed before task execution. "
|
|
77
|
+
"Requires MINITAP_API_KEY to be configured. ",
|
|
53
78
|
),
|
|
54
79
|
) -> str | dict[str, Any]:
|
|
55
80
|
"""Run a manual task on a mobile device via the Minitap platform."""
|
|
56
81
|
try:
|
|
82
|
+
if apk_path:
|
|
83
|
+
if not settings.CLOUD_MOBILE_NAME:
|
|
84
|
+
raise ToolError(
|
|
85
|
+
"apk_path parameter requires CLOUD_MOBILE_NAME to be set. "
|
|
86
|
+
"APK deployment is only supported in cloud mobile mode."
|
|
87
|
+
)
|
|
88
|
+
|
|
89
|
+
# Step 1: Upload APK via Platform storage API
|
|
90
|
+
filename = await upload_apk_to_cloud_mobile(apk_path=apk_path)
|
|
91
|
+
|
|
92
|
+
# Step 2: Install APK on cloud mobile
|
|
93
|
+
await install_apk_on_cloud_mobile(filename=filename)
|
|
94
|
+
|
|
57
95
|
request = PlatformTaskRequest(
|
|
58
96
|
task=ManualTaskConfig(
|
|
59
97
|
goal=goal,
|
|
60
98
|
output_description=output_description,
|
|
61
99
|
),
|
|
100
|
+
execution_origin="mcp",
|
|
62
101
|
)
|
|
63
102
|
agent = get_mobile_use_agent()
|
|
103
|
+
if not agent._initialized:
|
|
104
|
+
await agent.init()
|
|
64
105
|
result = await agent.run_task(
|
|
65
106
|
request=request,
|
|
66
107
|
locked_app_package=locked_app_package,
|
|
@@ -1,17 +1,18 @@
|
|
|
1
1
|
Metadata-Version: 2.3
|
|
2
2
|
Name: minitap-mcp
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.6.0
|
|
4
4
|
Summary: Model Context Protocol server for controlling Android & iOS devices with natural language
|
|
5
5
|
Author: Pierre-Louis Favreau, Jean-Pierre Lo, Clément Guiguet
|
|
6
6
|
Requires-Dist: fastmcp>=2.12.4
|
|
7
7
|
Requires-Dist: python-dotenv>=1.1.1
|
|
8
8
|
Requires-Dist: pydantic>=2.12.0
|
|
9
9
|
Requires-Dist: pydantic-settings>=2.10.1
|
|
10
|
-
Requires-Dist: minitap-mobile-use>=
|
|
10
|
+
Requires-Dist: minitap-mobile-use>=3.0.0
|
|
11
11
|
Requires-Dist: jinja2>=3.1.6
|
|
12
12
|
Requires-Dist: langchain-core>=0.3.75
|
|
13
13
|
Requires-Dist: pillow>=11.1.0
|
|
14
14
|
Requires-Dist: structlog>=24.4.0
|
|
15
|
+
Requires-Dist: aiohttp>=3.9.0
|
|
15
16
|
Requires-Dist: ruff==0.5.3 ; extra == 'dev'
|
|
16
17
|
Requires-Dist: pytest==8.4.1 ; extra == 'dev'
|
|
17
18
|
Requires-Dist: pytest-cov==5.0.0 ; extra == 'dev'
|
|
@@ -40,11 +41,19 @@ Before running the MCP server, ensure you have the required mobile automation to
|
|
|
40
41
|
- **For Android devices:**
|
|
41
42
|
|
|
42
43
|
- [ADB (Android Debug Bridge)](https://developer.android.com/tools/adb) - For device communication
|
|
43
|
-
- [Maestro](https://maestro.mobile.dev/) - For mobile automation
|
|
44
44
|
|
|
45
45
|
- **For iOS devices (macOS only):**
|
|
46
46
|
- Xcode Command Line Tools with `xcrun`
|
|
47
|
-
- [
|
|
47
|
+
- **[fb-idb](https://fbidb.io/docs/installation/)**: Facebook's iOS Development Bridge for device automation.
|
|
48
|
+
|
|
49
|
+
```bash
|
|
50
|
+
# Install via Homebrew (macOS)
|
|
51
|
+
brew tap facebook/fb
|
|
52
|
+
brew install idb-companion
|
|
53
|
+
```
|
|
54
|
+
|
|
55
|
+
> [!NOTE]
|
|
56
|
+
> `idb_companion` is required to communicate with iOS simulators. Make sure it's in your PATH after installation.
|
|
48
57
|
|
|
49
58
|
For detailed setup instructions, see the [mobile-use repository](https://github.com/minitap-ai/mobile-use).
|
|
50
59
|
|
|
@@ -244,6 +253,50 @@ The tool will:
|
|
|
244
253
|
3. Compare both screenshots using vision AI
|
|
245
254
|
4. Return a detailed analysis highlighting differences
|
|
246
255
|
|
|
256
|
+
## Cloud Mobile Mode
|
|
257
|
+
|
|
258
|
+
Run the MCP server with cloud-hosted mobile devices instead of requiring a local device. This enables:
|
|
259
|
+
|
|
260
|
+
- **Zero local setup**: No ADB or physical device required
|
|
261
|
+
- **Remote development**: Control cloud mobiles from anywhere
|
|
262
|
+
- **Scalable automation**: Access multiple cloud devices
|
|
263
|
+
|
|
264
|
+
### Setting Up Cloud Mobile Mode
|
|
265
|
+
|
|
266
|
+
1. **Create a Cloud Mobile** on [Minitap Platform](https://platform.minitap.ai/cloud-mobiles):
|
|
267
|
+
- Click **Create New Device**
|
|
268
|
+
- Choose platform (currently Android v11 / API level 30)
|
|
269
|
+
- Set a **Reference Name** (e.g., `my-dev-device`)
|
|
270
|
+
|
|
271
|
+
2. **Configure the environment variable**:
|
|
272
|
+
|
|
273
|
+
```bash
|
|
274
|
+
# Using reference name (recommended)
|
|
275
|
+
export CLOUD_MOBILE_NAME="my-dev-device"
|
|
276
|
+
```
|
|
277
|
+
|
|
278
|
+
3. **Start the server** (no local device needed):
|
|
279
|
+
|
|
280
|
+
```bash
|
|
281
|
+
minitap-mcp --server --api-key your_minitap_api_key
|
|
282
|
+
```
|
|
283
|
+
|
|
284
|
+
The server will:
|
|
285
|
+
- Connect to your cloud mobile on startup
|
|
286
|
+
- Maintain a keep-alive connection while running
|
|
287
|
+
- Automatically disconnect when the server stops
|
|
288
|
+
|
|
289
|
+
> ⚠️ **Important**: Cloud mobiles are billed while connected. The MCP server automatically stops the connection when you close your IDE or stop the server. Make sure to properly shut down the server to avoid unexpected charges.
|
|
290
|
+
|
|
291
|
+
### Cloud vs Local Mode
|
|
292
|
+
|
|
293
|
+
| Feature | Local Mode | Cloud Mode |
|
|
294
|
+
|---------|------------|------------|
|
|
295
|
+
| Device requirement | Physical/emulator | None |
|
|
296
|
+
| Setup complexity | ADB setup required | Low (env var only) |
|
|
297
|
+
| `CLOUD_MOBILE_NAME` | Not set | Set to device name/UUID |
|
|
298
|
+
| Billing | None | Per-minute usage |
|
|
299
|
+
|
|
247
300
|
## Advanced Configuration
|
|
248
301
|
|
|
249
302
|
### Custom ADB Server
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
minitap/mcp/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
2
|
-
minitap/mcp/core/agents/compare_screenshots/agent.py,sha256=
|
|
2
|
+
minitap/mcp/core/agents/compare_screenshots/agent.py,sha256=XRYyrv9QA4eb1b44hPjAqz6MKYM6y3THosxP21xzBY4,2458
|
|
3
3
|
minitap/mcp/core/agents/compare_screenshots/eval/prompts/prompt_1.md,sha256=qAyqOroSJROgrvlbsLCtiwFyBKuIMCQ-720A5cwgwPY,3563
|
|
4
4
|
minitap/mcp/core/agents/compare_screenshots/eval/scenario_1_add_cartoon_img_and_move_button/actual.png,sha256=woKE-aTTdb-9ArqfnV-xKKyit1Fu_hklavVwAaMQ14E,99455
|
|
5
5
|
minitap/mcp/core/agents/compare_screenshots/eval/scenario_1_add_cartoon_img_and_move_button/figma.png,sha256=ghxi1P-ofnmMv5_ASm0Rzo5ll_C8-E6ojQUCHRR33TA,102098
|
|
@@ -7,24 +7,26 @@ minitap/mcp/core/agents/compare_screenshots/eval/scenario_1_add_cartoon_img_and_
|
|
|
7
7
|
minitap/mcp/core/agents/compare_screenshots/eval/scenario_1_add_cartoon_img_and_move_button/prompt_1/model_params.json,sha256=p93kbUGxLYwc4NAEzPTBDA2XZ1ucsa7yevXZRe6V4Mc,37
|
|
8
8
|
minitap/mcp/core/agents/compare_screenshots/eval/scenario_1_add_cartoon_img_and_move_button/prompt_1/output.md,sha256=Q0_5w9x5WAMqx6-I7KgjlS-tt7HnWKIunP43J7F0W_o,3703
|
|
9
9
|
minitap/mcp/core/agents/compare_screenshots/prompt.md,sha256=qAyqOroSJROgrvlbsLCtiwFyBKuIMCQ-720A5cwgwPY,3563
|
|
10
|
-
minitap/mcp/core/
|
|
10
|
+
minitap/mcp/core/cloud_apk.py,sha256=Xeg1jrycm50UC72J4qVZ2NS9qs20oXKhvBcmBRs4rEA,3896
|
|
11
|
+
minitap/mcp/core/config.py,sha256=v_neP4lqIx7sigiAPxUALZg5QBrIx-N4mJ848YRfG84,1428
|
|
11
12
|
minitap/mcp/core/decorators.py,sha256=kMx_mlaa-2U1AgCoYkgPoLOa-iOoKUF1OjcNV7x59Ds,2940
|
|
12
13
|
minitap/mcp/core/device.py,sha256=sEO3Z-8F325hDOObdH1YBhZE60f17FmIclt5UlhY_nU,7875
|
|
13
14
|
minitap/mcp/core/llm.py,sha256=tI5m5rFDLeMkXE5WExnzYSzHU3nTIEiSC9nAsPzVMaU,1144
|
|
14
15
|
minitap/mcp/core/logging_config.py,sha256=OJlArPJxflbhckerFsRHVTzy3jwsLsNSPN0LVpkmpNM,1861
|
|
15
16
|
minitap/mcp/core/models.py,sha256=egLScxPAMo4u5cqY33UKba7z7DsdgqfPW409UAqW1Jg,1942
|
|
16
|
-
minitap/mcp/core/sdk_agent.py,sha256
|
|
17
|
+
minitap/mcp/core/sdk_agent.py,sha256=5YEr7aDjoiwbRQkZBK3jDa08c5QhPwLxahzlYrEB_KE,1132
|
|
17
18
|
minitap/mcp/core/utils/figma.py,sha256=L5aAHm59mrRYaqrwMJSM24SSdZPu2yVg-wsHTF3L8vk,2310
|
|
18
19
|
minitap/mcp/core/utils/images.py,sha256=3uExpRoh7affIieZx3TLlZTmZCcoxWfx1YpPbwhjiJY,1791
|
|
19
|
-
minitap/mcp/main.py,sha256=
|
|
20
|
-
minitap/mcp/server/
|
|
21
|
-
minitap/mcp/server/
|
|
22
|
-
minitap/mcp/
|
|
23
|
-
minitap/mcp/tools/
|
|
24
|
-
minitap/mcp/tools/
|
|
20
|
+
minitap/mcp/main.py,sha256=6UjO26otULFbTmoI_WziknaRyBTXSazH9sqXRP-wns8,9250
|
|
21
|
+
minitap/mcp/server/cloud_mobile.py,sha256=kntcdMkc89QoXyLc-f5bzWFm6UBj52NqrBNHopAv-ag,14573
|
|
22
|
+
minitap/mcp/server/middleware.py,sha256=SjPc4pcfPuG0TnaDH7a19DS_HRFPl3bkbovdOLzy_IU,768
|
|
23
|
+
minitap/mcp/server/poller.py,sha256=JsdW6nvj4r3tsn8AaTwXD4H9dVAAau4BhJXHXHit9nA,2528
|
|
24
|
+
minitap/mcp/tools/analyze_screen.py,sha256=_7tj7AJvopotvItGIN55OoztJXmVu3j2ZsBmYq28pVw,2422
|
|
25
|
+
minitap/mcp/tools/compare_screenshot_with_figma.py,sha256=gMaYItE_bg_EwY2-Ux9RQd-Cnz6s2RCtYh1Q5n7HciU,4621
|
|
26
|
+
minitap/mcp/tools/execute_mobile_command.py,sha256=e49Y14KLTJ63ZQvLUrn-CvC5TQrylCXsUDEuJOTE5j8,4271
|
|
25
27
|
minitap/mcp/tools/save_figma_assets.py,sha256=V1gnQsJ1tciOxiK08aaqQxOEerJkKzxU8r4hJmkXHtA,9945
|
|
26
28
|
minitap/mcp/tools/screen_analyzer.md,sha256=TTO80JQWusbA9cKAZn-9cqhgVHm6F_qJh5w152hG3YM,734
|
|
27
|
-
minitap_mcp-0.
|
|
28
|
-
minitap_mcp-0.
|
|
29
|
-
minitap_mcp-0.
|
|
30
|
-
minitap_mcp-0.
|
|
29
|
+
minitap_mcp-0.6.0.dist-info/WHEEL,sha256=93kfTGt3a0Dykt_T-gsjtyS5_p8F_d6CE1NwmBOirzo,79
|
|
30
|
+
minitap_mcp-0.6.0.dist-info/entry_points.txt,sha256=rYVoXm7tSQCqQTtHx4Lovgn1YsjwtEEHfddKrfEVHuY,55
|
|
31
|
+
minitap_mcp-0.6.0.dist-info/METADATA,sha256=pzp6FGqqncEUQ9YNyheBLGonB2Pe8DDE2LKJRwJ-HE4,10589
|
|
32
|
+
minitap_mcp-0.6.0.dist-info/RECORD,,
|
|
File without changes
|