minitap-mcp 0.9.0__tar.gz → 0.9.2__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {minitap_mcp-0.9.0 → minitap_mcp-0.9.2}/PKG-INFO +2 -2
- {minitap_mcp-0.9.0 → minitap_mcp-0.9.2}/minitap/mcp/core/agents/compare_screenshots/agent.py +3 -11
- {minitap_mcp-0.9.0 → minitap_mcp-0.9.2}/minitap/mcp/core/config.py +5 -5
- {minitap_mcp-0.9.0 → minitap_mcp-0.9.2}/minitap/mcp/core/logging_config.py +3 -1
- {minitap_mcp-0.9.0 → minitap_mcp-0.9.2}/minitap/mcp/core/sdk_agent.py +0 -6
- {minitap_mcp-0.9.0 → minitap_mcp-0.9.2}/minitap/mcp/main.py +15 -68
- {minitap_mcp-0.9.0 → minitap_mcp-0.9.2}/minitap/mcp/server/middleware.py +6 -2
- minitap_mcp-0.9.2/minitap/mcp/tools/execute_mobile_command.py +260 -0
- {minitap_mcp-0.9.0 → minitap_mcp-0.9.2}/minitap/mcp/tools/take_screenshot.py +2 -18
- {minitap_mcp-0.9.0 → minitap_mcp-0.9.2}/minitap/mcp/tools/upload_screenshot.py +8 -21
- {minitap_mcp-0.9.0 → minitap_mcp-0.9.2}/pyproject.toml +2 -2
- minitap_mcp-0.9.0/minitap/mcp/core/cloud_apk.py +0 -117
- minitap_mcp-0.9.0/minitap/mcp/server/cloud_mobile.py +0 -492
- minitap_mcp-0.9.0/minitap/mcp/tools/execute_mobile_command.py +0 -182
- {minitap_mcp-0.9.0 → minitap_mcp-0.9.2}/PYPI_README.md +0 -0
- {minitap_mcp-0.9.0 → minitap_mcp-0.9.2}/minitap/mcp/__init__.py +0 -0
- {minitap_mcp-0.9.0 → minitap_mcp-0.9.2}/minitap/mcp/core/agents/compare_screenshots/eval/prompts/prompt_1.md +0 -0
- {minitap_mcp-0.9.0 → minitap_mcp-0.9.2}/minitap/mcp/core/agents/compare_screenshots/eval/scenario_1_add_cartoon_img_and_move_button/actual.png +0 -0
- {minitap_mcp-0.9.0 → minitap_mcp-0.9.2}/minitap/mcp/core/agents/compare_screenshots/eval/scenario_1_add_cartoon_img_and_move_button/figma.png +0 -0
- {minitap_mcp-0.9.0 → minitap_mcp-0.9.2}/minitap/mcp/core/agents/compare_screenshots/eval/scenario_1_add_cartoon_img_and_move_button/human_feedback.txt +0 -0
- {minitap_mcp-0.9.0 → minitap_mcp-0.9.2}/minitap/mcp/core/agents/compare_screenshots/eval/scenario_1_add_cartoon_img_and_move_button/prompt_1/model_params.json +0 -0
- {minitap_mcp-0.9.0 → minitap_mcp-0.9.2}/minitap/mcp/core/agents/compare_screenshots/eval/scenario_1_add_cartoon_img_and_move_button/prompt_1/output.md +0 -0
- {minitap_mcp-0.9.0 → minitap_mcp-0.9.2}/minitap/mcp/core/agents/compare_screenshots/prompt.md +0 -0
- {minitap_mcp-0.9.0 → minitap_mcp-0.9.2}/minitap/mcp/core/decorators.py +0 -0
- {minitap_mcp-0.9.0 → minitap_mcp-0.9.2}/minitap/mcp/core/device.py +0 -0
- {minitap_mcp-0.9.0 → minitap_mcp-0.9.2}/minitap/mcp/core/llm.py +0 -0
- {minitap_mcp-0.9.0 → minitap_mcp-0.9.2}/minitap/mcp/core/models.py +0 -0
- {minitap_mcp-0.9.0 → minitap_mcp-0.9.2}/minitap/mcp/core/storage.py +0 -0
- {minitap_mcp-0.9.0 → minitap_mcp-0.9.2}/minitap/mcp/core/task_runs.py +0 -0
- {minitap_mcp-0.9.0 → minitap_mcp-0.9.2}/minitap/mcp/core/utils/figma.py +0 -0
- {minitap_mcp-0.9.0 → minitap_mcp-0.9.2}/minitap/mcp/core/utils/images.py +0 -0
- {minitap_mcp-0.9.0 → minitap_mcp-0.9.2}/minitap/mcp/server/poller.py +0 -0
- {minitap_mcp-0.9.0 → minitap_mcp-0.9.2}/minitap/mcp/server/remote_proxy.py +0 -0
- {minitap_mcp-0.9.0 → minitap_mcp-0.9.2}/minitap/mcp/tools/read_swift_logs.py +0 -0
- {minitap_mcp-0.9.0 → minitap_mcp-0.9.2}/minitap/mcp/tools/screen_analyzer.md +0 -0
|
@@ -1,13 +1,13 @@
|
|
|
1
1
|
Metadata-Version: 2.3
|
|
2
2
|
Name: minitap-mcp
|
|
3
|
-
Version: 0.9.
|
|
3
|
+
Version: 0.9.2
|
|
4
4
|
Summary: Model Context Protocol server for controlling Android & iOS devices with natural language
|
|
5
5
|
Author: Pierre-Louis Favreau, Jean-Pierre Lo, Clément Guiguet
|
|
6
6
|
Requires-Dist: fastmcp>=2.12.4
|
|
7
7
|
Requires-Dist: python-dotenv>=1.1.1
|
|
8
8
|
Requires-Dist: pydantic>=2.12.0
|
|
9
9
|
Requires-Dist: pydantic-settings>=2.10.1
|
|
10
|
-
Requires-Dist: minitap-mobile-use>=3.
|
|
10
|
+
Requires-Dist: minitap-mobile-use>=3.5.0
|
|
11
11
|
Requires-Dist: jinja2>=3.1.6
|
|
12
12
|
Requires-Dist: langchain-core>=0.3.75
|
|
13
13
|
Requires-Dist: pillow>=11.1.0
|
{minitap_mcp-0.9.0 → minitap_mcp-0.9.2}/minitap/mcp/core/agents/compare_screenshots/agent.py
RENAMED
|
@@ -1,5 +1,4 @@
|
|
|
1
1
|
import asyncio
|
|
2
|
-
import base64
|
|
3
2
|
from pathlib import Path
|
|
4
3
|
from uuid import uuid4
|
|
5
4
|
|
|
@@ -10,7 +9,6 @@ from pydantic import BaseModel
|
|
|
10
9
|
from minitap.mcp.core.device import capture_screenshot, find_mobile_device
|
|
11
10
|
from minitap.mcp.core.llm import get_minitap_llm
|
|
12
11
|
from minitap.mcp.core.utils.images import get_screenshot_message_for_llm
|
|
13
|
-
from minitap.mcp.server.cloud_mobile import get_cloud_mobile_id, get_cloud_screenshot
|
|
14
12
|
|
|
15
13
|
|
|
16
14
|
class CompareScreenshotsOutput(BaseModel):
|
|
@@ -25,7 +23,7 @@ async def compare_screenshots(
|
|
|
25
23
|
"""
|
|
26
24
|
Compare screenshots and return the comparison text along with both screenshots.
|
|
27
25
|
|
|
28
|
-
|
|
26
|
+
Captures screenshot from local device and compares with expected screenshot.
|
|
29
27
|
|
|
30
28
|
Returns:
|
|
31
29
|
CompareScreenshotsOutput
|
|
@@ -34,14 +32,8 @@ async def compare_screenshots(
|
|
|
34
32
|
Path(__file__).parent.joinpath("prompt.md").read_text(encoding="utf-8")
|
|
35
33
|
).render()
|
|
36
34
|
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
if cloud_mobile_id:
|
|
40
|
-
screenshot_bytes = await get_cloud_screenshot(cloud_mobile_id)
|
|
41
|
-
current_screenshot = base64.b64encode(screenshot_bytes).decode("utf-8")
|
|
42
|
-
else:
|
|
43
|
-
device = find_mobile_device()
|
|
44
|
-
current_screenshot = capture_screenshot(device)
|
|
35
|
+
device = find_mobile_device()
|
|
36
|
+
current_screenshot = capture_screenshot(device)
|
|
45
37
|
|
|
46
38
|
messages: list[BaseMessage] = [
|
|
47
39
|
SystemMessage(content=system_message),
|
|
@@ -62,11 +62,11 @@ class MCPSettings(BaseSettings):
|
|
|
62
62
|
MCP_SERVER_HOST: str = Field(default="0.0.0.0")
|
|
63
63
|
MCP_SERVER_PORT: int = Field(default=8000)
|
|
64
64
|
|
|
65
|
-
#
|
|
66
|
-
# When
|
|
67
|
-
#
|
|
68
|
-
#
|
|
69
|
-
|
|
65
|
+
# Local device configuration
|
|
66
|
+
# When False, disables the local device health poller.
|
|
67
|
+
# Useful when running the MCP server without a local device attached
|
|
68
|
+
# (e.g., only using cloud_platform parameter in execute_mobile_command).
|
|
69
|
+
ENABLE_LOCAL_DEVICE: bool = Field(default=True)
|
|
70
70
|
|
|
71
71
|
# Trajectory GIF download configuration
|
|
72
72
|
# When set, downloads the trajectory GIF after task execution to the specified folder.
|
|
@@ -13,9 +13,11 @@ def configure_logging(log_level: str = "INFO") -> None:
|
|
|
13
13
|
log_level: The logging level (DEBUG, INFO, WARNING, ERROR, CRITICAL)
|
|
14
14
|
"""
|
|
15
15
|
# Configure standard library logging
|
|
16
|
+
# CRITICAL: Use stderr to avoid polluting stdout with non-JSONRPC data
|
|
17
|
+
# MCP servers communicate via stdio - stdout MUST only contain JSONRPC messages
|
|
16
18
|
logging.basicConfig(
|
|
17
19
|
format="%(message)s",
|
|
18
|
-
stream=sys.
|
|
20
|
+
stream=sys.stderr,
|
|
19
21
|
level=getattr(logging, log_level.upper()),
|
|
20
22
|
)
|
|
21
23
|
|
|
@@ -3,8 +3,6 @@ import os
|
|
|
3
3
|
from minitap.mobile_use.sdk import Agent
|
|
4
4
|
from minitap.mobile_use.sdk.builders import Builders
|
|
5
5
|
|
|
6
|
-
from minitap.mcp.core.config import settings
|
|
7
|
-
|
|
8
6
|
# Lazy-initialized singleton agent
|
|
9
7
|
_agent: Agent | None = None
|
|
10
8
|
|
|
@@ -26,10 +24,6 @@ def get_mobile_use_agent() -> Agent:
|
|
|
26
24
|
_, host, port = parts
|
|
27
25
|
config = config.with_adb_server(host=host, port=int(port))
|
|
28
26
|
|
|
29
|
-
# Add cloud mobile configuration if set
|
|
30
|
-
if settings.CLOUD_MOBILE_NAME:
|
|
31
|
-
config = config.for_cloud_mobile(cloud_mobile_id_or_ref=settings.CLOUD_MOBILE_NAME)
|
|
32
|
-
|
|
33
27
|
_agent = Agent(config=config.build())
|
|
34
28
|
|
|
35
29
|
return _agent
|
|
@@ -35,7 +35,6 @@ from minitap.mcp.core.logging_config import (
|
|
|
35
35
|
configure_logging, # noqa: E402
|
|
36
36
|
get_logger,
|
|
37
37
|
)
|
|
38
|
-
from minitap.mcp.server.cloud_mobile import CloudMobileService
|
|
39
38
|
from minitap.mcp.server.middleware import LocalDeviceHealthMiddleware
|
|
40
39
|
from minitap.mcp.server.poller import device_health_poller
|
|
41
40
|
from minitap.mobile_use.config import settings as sdk_settings
|
|
@@ -54,13 +53,7 @@ def main() -> None:
|
|
|
54
53
|
default=None,
|
|
55
54
|
help="Minitap API key for authentication",
|
|
56
55
|
)
|
|
57
|
-
|
|
58
|
-
"--cloud-mobile-name",
|
|
59
|
-
type=str,
|
|
60
|
-
required=False,
|
|
61
|
-
default=None,
|
|
62
|
-
help="Name of the cloud mobile device to connect to (enables cloud mode)",
|
|
63
|
-
)
|
|
56
|
+
|
|
64
57
|
parser.add_argument("--llm-profile", type=str, required=False, default=None)
|
|
65
58
|
parser.add_argument(
|
|
66
59
|
"--server",
|
|
@@ -82,11 +75,6 @@ def main() -> None:
|
|
|
82
75
|
settings.__init__()
|
|
83
76
|
sdk_settings.__init__()
|
|
84
77
|
|
|
85
|
-
if args.cloud_mobile_name:
|
|
86
|
-
os.environ["CLOUD_MOBILE_NAME"] = args.cloud_mobile_name
|
|
87
|
-
settings.__init__()
|
|
88
|
-
sdk_settings.__init__()
|
|
89
|
-
|
|
90
78
|
if args.llm_profile:
|
|
91
79
|
os.environ["MINITAP_LLM_PROFILE_NAME"] = args.llm_profile
|
|
92
80
|
settings.__init__()
|
|
@@ -123,7 +111,6 @@ class MCPLifespanContext:
|
|
|
123
111
|
Stores references to services that need cleanup on shutdown.
|
|
124
112
|
"""
|
|
125
113
|
|
|
126
|
-
cloud_mobile_service: CloudMobileService | None = None
|
|
127
114
|
local_poller_stop_event: threading.Event | None = None
|
|
128
115
|
local_poller_thread: threading.Thread | None = None
|
|
129
116
|
remote_mcp_proxy: FastMCP | None = None
|
|
@@ -133,67 +120,30 @@ class MCPLifespanContext:
|
|
|
133
120
|
async def mcp_lifespan(server: FastMCP) -> AsyncIterator[MCPLifespanContext]:
|
|
134
121
|
"""Lifespan context manager for MCP server.
|
|
135
122
|
|
|
136
|
-
Handles startup/shutdown for
|
|
123
|
+
Handles startup/shutdown for local and cloud-only modes.
|
|
137
124
|
|
|
138
|
-
|
|
139
|
-
- Connects to cloud mobile on startup
|
|
140
|
-
- Maintains keep-alive polling
|
|
141
|
-
- Disconnects on shutdown (critical for billing!)
|
|
142
|
-
|
|
143
|
-
Local mode (CLOUD_MOBILE_NAME not set):
|
|
125
|
+
Local mode (ENABLE_LOCAL_DEVICE=True, default):
|
|
144
126
|
- Starts device health poller
|
|
145
127
|
- Monitors local device connection
|
|
146
128
|
|
|
129
|
+
Cloud-only mode (ENABLE_LOCAL_DEVICE=False):
|
|
130
|
+
- No local device polling
|
|
131
|
+
- Use cloud_platform parameter in execute_mobile_command to run tasks
|
|
132
|
+
|
|
147
133
|
Args:
|
|
148
134
|
server: The FastMCP server instance.
|
|
149
135
|
|
|
150
136
|
Yields:
|
|
151
137
|
MCPLifespanContext with references to running services.
|
|
152
|
-
|
|
153
|
-
Raises:
|
|
154
|
-
RuntimeError: If cloud mobile connection fails
|
|
155
|
-
(crashes MCP to prevent false "connected" state).
|
|
156
138
|
"""
|
|
157
139
|
from minitap.mcp.core.sdk_agent import get_mobile_use_agent # noqa: E402
|
|
158
140
|
|
|
159
141
|
context = MCPLifespanContext()
|
|
160
142
|
api_key = settings.MINITAP_API_KEY.get_secret_value() if settings.MINITAP_API_KEY else None
|
|
161
143
|
|
|
162
|
-
|
|
163
|
-
if settings.CLOUD_MOBILE_NAME:
|
|
164
|
-
# ==================== CLOUD MODE ====================
|
|
165
|
-
logger.info(f"Starting MCP in CLOUD mode with mobile: {settings.CLOUD_MOBILE_NAME}")
|
|
166
|
-
|
|
167
|
-
if not api_key:
|
|
168
|
-
logger.error("MINITAP_API_KEY is required")
|
|
169
|
-
raise RuntimeError(
|
|
170
|
-
"MINITAP_API_KEY is required when CLOUD_MOBILE_NAME is set. "
|
|
171
|
-
"Please set your API key via --api-key or MINITAP_API_KEY environment variable."
|
|
172
|
-
)
|
|
173
|
-
|
|
174
|
-
# Create and connect cloud mobile service
|
|
175
|
-
cloud_service = CloudMobileService(
|
|
176
|
-
cloud_mobile_name=settings.CLOUD_MOBILE_NAME,
|
|
177
|
-
api_key=api_key,
|
|
178
|
-
)
|
|
179
|
-
|
|
180
|
-
try:
|
|
181
|
-
await cloud_service.connect()
|
|
182
|
-
except Exception as e:
|
|
183
|
-
# CRITICAL: If cloud mobile not found, crash the MCP!
|
|
184
|
-
# This prevents the IDE from showing the server as "connected"
|
|
185
|
-
# when it actually can't do anything useful.
|
|
186
|
-
logger.error(f"Failed to connect to cloud mobile: {e}")
|
|
187
|
-
raise RuntimeError(
|
|
188
|
-
f"Cloud mobile connection failed. The MCP server cannot start.\n{e}"
|
|
189
|
-
) from e
|
|
190
|
-
|
|
191
|
-
context.cloud_mobile_service = cloud_service
|
|
192
|
-
logger.info("Cloud mobile connected, MCP server ready")
|
|
193
|
-
|
|
194
|
-
else:
|
|
144
|
+
if settings.ENABLE_LOCAL_DEVICE:
|
|
195
145
|
# ==================== LOCAL MODE ====================
|
|
196
|
-
logger.info("Starting MCP in LOCAL mode
|
|
146
|
+
logger.info("Starting MCP in LOCAL mode with device polling")
|
|
197
147
|
|
|
198
148
|
agent = get_mobile_use_agent()
|
|
199
149
|
server.add_middleware(LocalDeviceHealthMiddleware(agent))
|
|
@@ -210,6 +160,12 @@ async def mcp_lifespan(server: FastMCP) -> AsyncIterator[MCPLifespanContext]:
|
|
|
210
160
|
context.local_poller_stop_event = stop_event
|
|
211
161
|
context.local_poller_thread = poller_thread
|
|
212
162
|
logger.info("Device health poller started")
|
|
163
|
+
else:
|
|
164
|
+
# ==================== CLOUD-ONLY MODE ====================
|
|
165
|
+
logger.info(
|
|
166
|
+
"Starting MCP without local device polling (ENABLE_LOCAL_DEVICE=False). "
|
|
167
|
+
"Use cloud_platform parameter in execute_mobile_command to run tasks."
|
|
168
|
+
)
|
|
213
169
|
|
|
214
170
|
# ==================== REMOTE MCP PROXY ====================
|
|
215
171
|
# Mount remote MCP proxy if configured (works in both cloud and local modes)
|
|
@@ -257,15 +213,6 @@ async def mcp_lifespan(server: FastMCP) -> AsyncIterator[MCPLifespanContext]:
|
|
|
257
213
|
# ==================== SHUTDOWN ====================
|
|
258
214
|
logger.info("MCP server shutting down, cleaning up resources...")
|
|
259
215
|
|
|
260
|
-
if context.cloud_mobile_service:
|
|
261
|
-
# CRITICAL: Stop cloud mobile connection to stop billing!
|
|
262
|
-
logger.info("Disconnecting cloud mobile (stopping billing)...")
|
|
263
|
-
try:
|
|
264
|
-
await context.cloud_mobile_service.disconnect()
|
|
265
|
-
logger.info("Cloud mobile disconnected successfully")
|
|
266
|
-
except Exception as e:
|
|
267
|
-
logger.error(f"Error disconnecting cloud mobile: {e}")
|
|
268
|
-
|
|
269
216
|
if context.local_poller_stop_event and context.local_poller_thread:
|
|
270
217
|
# Stop local device health poller
|
|
271
218
|
logger.info("Stopping device health poller...")
|
|
@@ -6,14 +6,18 @@ from minitap.mobile_use.sdk import Agent
|
|
|
6
6
|
class LocalDeviceHealthMiddleware(Middleware):
|
|
7
7
|
"""Middleware that checks local device health before tool calls.
|
|
8
8
|
|
|
9
|
-
Only used in local mode (when
|
|
10
|
-
For cloud mode, device health is managed by the cloud service.
|
|
9
|
+
Only used in local mode (when ENABLE_LOCAL_DEVICE is True).
|
|
11
10
|
"""
|
|
12
11
|
|
|
13
12
|
def __init__(self, agent: Agent):
|
|
14
13
|
self.agent = agent
|
|
15
14
|
|
|
16
15
|
async def on_call_tool(self, context: MiddlewareContext, call_next):
|
|
16
|
+
# Skip agent check if platform is set (Limrun mode creates its own agent)
|
|
17
|
+
tool_args = getattr(context.message, "arguments", None) or {}
|
|
18
|
+
if tool_args.get("cloud_platform"):
|
|
19
|
+
return await call_next(context)
|
|
20
|
+
|
|
17
21
|
if not self.agent._initialized:
|
|
18
22
|
raise ToolError(
|
|
19
23
|
"Agent not initialized.\nMake sure a mobile device is connected and try again."
|
|
@@ -0,0 +1,260 @@
|
|
|
1
|
+
"""Tool for running manual tasks on a connected mobile device."""
|
|
2
|
+
|
|
3
|
+
from collections.abc import Mapping
|
|
4
|
+
from pathlib import Path
|
|
5
|
+
from typing import Any
|
|
6
|
+
|
|
7
|
+
from fastmcp.exceptions import ToolError
|
|
8
|
+
from fastmcp.tools.tool import ToolResult
|
|
9
|
+
from mcp.types import TextContent
|
|
10
|
+
from minitap.mobile_use.sdk import Agent
|
|
11
|
+
from minitap.mobile_use.sdk.builders import Builders
|
|
12
|
+
from minitap.mobile_use.sdk.types import LimrunPlatform, ManualTaskConfig
|
|
13
|
+
from minitap.mobile_use.sdk.types.task import PlatformTaskRequest
|
|
14
|
+
from pydantic import Field
|
|
15
|
+
|
|
16
|
+
from minitap.mcp.core.config import settings
|
|
17
|
+
from minitap.mcp.core.decorators import handle_tool_errors
|
|
18
|
+
from minitap.mcp.core.logging_config import get_logger
|
|
19
|
+
from minitap.mcp.core.sdk_agent import get_mobile_use_agent
|
|
20
|
+
from minitap.mcp.core.storage import StorageDownloadError, download_trajectory_gif
|
|
21
|
+
from minitap.mcp.core.task_runs import TaskRunsError, get_latest_task_run_id
|
|
22
|
+
from minitap.mcp.main import mcp
|
|
23
|
+
|
|
24
|
+
logger = get_logger(__name__)
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
def _serialize_result(result: Any) -> Any:
|
|
28
|
+
"""Convert SDK responses to serializable data for MCP."""
|
|
29
|
+
if hasattr(result, "model_dump"):
|
|
30
|
+
return result.model_dump()
|
|
31
|
+
if hasattr(result, "dict"):
|
|
32
|
+
return result.dict()
|
|
33
|
+
if isinstance(result, Mapping):
|
|
34
|
+
return dict(result)
|
|
35
|
+
return result
|
|
36
|
+
|
|
37
|
+
|
|
38
|
+
@mcp.tool(
|
|
39
|
+
name="execute_mobile_command",
|
|
40
|
+
description="""
|
|
41
|
+
Execute a natural language command on a mobile device using the Minitap SDK.
|
|
42
|
+
This tool allows you to control Android or iOS devices using natural language.
|
|
43
|
+
|
|
44
|
+
Set cloud_platform to run on a cloud device (recommended for most use cases).
|
|
45
|
+
|
|
46
|
+
Examples:
|
|
47
|
+
- execute_mobile_command(cloud_platform="android", goal="Open Settings and check battery level")
|
|
48
|
+
- execute_mobile_command(cloud_platform="ios", goal="Open Safari and search for weather")
|
|
49
|
+
|
|
50
|
+
App Deployment:
|
|
51
|
+
You can deploy and test apps by providing the path to a locally built app:
|
|
52
|
+
- Set app_path to the path of your .apk file (Android) or .app folder (iOS)
|
|
53
|
+
- The app will be automatically uploaded and installed on the device
|
|
54
|
+
- Must provide locked_app_package (package name or bundle ID) when using app_path
|
|
55
|
+
|
|
56
|
+
Examples with app deployment:
|
|
57
|
+
- execute_mobile_command(cloud_platform="android", goal="Test login flow",
|
|
58
|
+
app_path="/path/to/app-debug.apk", locked_app_package="com.example.myapp")
|
|
59
|
+
- execute_mobile_command(cloud_platform="ios", goal="Verify onboarding screens",
|
|
60
|
+
app_path="/path/to/MyApp.app", locked_app_package="com.example.myapp")
|
|
61
|
+
""",
|
|
62
|
+
)
|
|
63
|
+
@handle_tool_errors
|
|
64
|
+
async def execute_mobile_command(
|
|
65
|
+
goal: str = Field(description="High-level goal describing the action to perform."),
|
|
66
|
+
cloud_platform: LimrunPlatform | None = Field(
|
|
67
|
+
default=None,
|
|
68
|
+
description="Cloud platform to run on: 'android' or 'ios'. "
|
|
69
|
+
"When set, a cloud device is automatically provisioned for this task "
|
|
70
|
+
"and cleaned up afterwards.",
|
|
71
|
+
),
|
|
72
|
+
output_description: str | None = Field(
|
|
73
|
+
default=None,
|
|
74
|
+
description="Optional description of the expected output format. "
|
|
75
|
+
"For example: 'A JSON array with sender and subject for each email' "
|
|
76
|
+
"or 'The battery percentage as a number'.",
|
|
77
|
+
),
|
|
78
|
+
locked_app_package: str | None = Field(
|
|
79
|
+
default=None,
|
|
80
|
+
description="Optional package name of the app to lock the device to. "
|
|
81
|
+
"Will launch the app if not already running, and keep it in foreground "
|
|
82
|
+
"until the task is completed. REQUIRED when using app_path.",
|
|
83
|
+
),
|
|
84
|
+
app_path: str | None = Field(
|
|
85
|
+
default=None,
|
|
86
|
+
description="Path to local .apk file (Android) or .app folder (iOS) to deploy. "
|
|
87
|
+
"The app will be automatically uploaded and installed on the device. "
|
|
88
|
+
"Must provide locked_app_package when using app_path.",
|
|
89
|
+
),
|
|
90
|
+
) -> str | dict[str, Any] | ToolResult:
|
|
91
|
+
"""Run a manual task on a mobile device via the Minitap platform."""
|
|
92
|
+
try:
|
|
93
|
+
# Cloud platform mode: provision a cloud device on-demand
|
|
94
|
+
if cloud_platform:
|
|
95
|
+
return await _execute_with_cloud_platform(
|
|
96
|
+
cloud_platform=cloud_platform,
|
|
97
|
+
goal=goal,
|
|
98
|
+
output_description=output_description,
|
|
99
|
+
locked_app_package=locked_app_package,
|
|
100
|
+
app_path=app_path,
|
|
101
|
+
)
|
|
102
|
+
|
|
103
|
+
# Local device mode
|
|
104
|
+
if app_path:
|
|
105
|
+
raise ToolError(
|
|
106
|
+
"app_path parameter requires cloud_platform to be set. "
|
|
107
|
+
"App deployment is only supported in cloud platform mode."
|
|
108
|
+
)
|
|
109
|
+
|
|
110
|
+
request = PlatformTaskRequest(
|
|
111
|
+
task=ManualTaskConfig(
|
|
112
|
+
goal=goal,
|
|
113
|
+
output_description=output_description,
|
|
114
|
+
),
|
|
115
|
+
execution_origin="mcp",
|
|
116
|
+
)
|
|
117
|
+
agent = get_mobile_use_agent()
|
|
118
|
+
if not agent._initialized:
|
|
119
|
+
await agent.init()
|
|
120
|
+
result = await agent.run_task(
|
|
121
|
+
request=request,
|
|
122
|
+
locked_app_package=locked_app_package,
|
|
123
|
+
)
|
|
124
|
+
|
|
125
|
+
trajectory_gif_path: Path | None = None
|
|
126
|
+
if settings.TRAJECTORY_GIF_DOWNLOAD_FOLDER:
|
|
127
|
+
trajectory_gif_path = await _download_trajectory_gif_if_available()
|
|
128
|
+
|
|
129
|
+
serialized_result = _serialize_result(result)
|
|
130
|
+
|
|
131
|
+
# If trajectory was saved, return a ToolResult with multiple content items
|
|
132
|
+
if trajectory_gif_path:
|
|
133
|
+
import json
|
|
134
|
+
|
|
135
|
+
result_text = (
|
|
136
|
+
json.dumps(serialized_result, indent=2)
|
|
137
|
+
if isinstance(serialized_result, dict)
|
|
138
|
+
else str(serialized_result)
|
|
139
|
+
)
|
|
140
|
+
return ToolResult(
|
|
141
|
+
content=[
|
|
142
|
+
TextContent(type="text", text=result_text),
|
|
143
|
+
TextContent(type="text", text=f"Trajectory saved to {trajectory_gif_path}"),
|
|
144
|
+
],
|
|
145
|
+
)
|
|
146
|
+
|
|
147
|
+
return serialized_result
|
|
148
|
+
except Exception as e:
|
|
149
|
+
raise ToolError(str(e))
|
|
150
|
+
|
|
151
|
+
|
|
152
|
+
async def _execute_with_cloud_platform(
|
|
153
|
+
cloud_platform: LimrunPlatform,
|
|
154
|
+
goal: str,
|
|
155
|
+
output_description: str | None,
|
|
156
|
+
locked_app_package: str | None,
|
|
157
|
+
app_path: str | None = None,
|
|
158
|
+
) -> str | dict[str, Any] | ToolResult:
|
|
159
|
+
"""Execute a task using a cloud device.
|
|
160
|
+
|
|
161
|
+
Provisions a cloud device, runs the task, and cleans up afterwards.
|
|
162
|
+
"""
|
|
163
|
+
api_key = settings.MINITAP_API_KEY.get_secret_value() if settings.MINITAP_API_KEY else None
|
|
164
|
+
if not api_key:
|
|
165
|
+
raise ToolError(
|
|
166
|
+
"MINITAP_API_KEY is required for cloud platform mode. "
|
|
167
|
+
"Please set your API key via --api-key or MINITAP_API_KEY environment variable."
|
|
168
|
+
)
|
|
169
|
+
|
|
170
|
+
logger.info(f"Provisioning {cloud_platform.value} cloud device for task execution...")
|
|
171
|
+
|
|
172
|
+
# Create a dedicated agent for this cloud task
|
|
173
|
+
config = Builders.AgentConfig.for_limrun(platform=cloud_platform, api_key=api_key)
|
|
174
|
+
agent = Agent(config=config.build())
|
|
175
|
+
|
|
176
|
+
try:
|
|
177
|
+
await agent.init(api_key=api_key)
|
|
178
|
+
logger.info(f"{cloud_platform.value} cloud device ready, executing task...")
|
|
179
|
+
|
|
180
|
+
request = PlatformTaskRequest(
|
|
181
|
+
task=ManualTaskConfig(
|
|
182
|
+
goal=goal,
|
|
183
|
+
output_description=output_description,
|
|
184
|
+
),
|
|
185
|
+
execution_origin="mcp",
|
|
186
|
+
)
|
|
187
|
+
|
|
188
|
+
# Pass app_path to run_task - SDK handles upload and installation
|
|
189
|
+
result = await agent.run_task(
|
|
190
|
+
request=request,
|
|
191
|
+
locked_app_package=locked_app_package,
|
|
192
|
+
app_path=app_path,
|
|
193
|
+
)
|
|
194
|
+
|
|
195
|
+
trajectory_gif_path: Path | None = None
|
|
196
|
+
if settings.TRAJECTORY_GIF_DOWNLOAD_FOLDER:
|
|
197
|
+
trajectory_gif_path = await _download_trajectory_gif_if_available()
|
|
198
|
+
|
|
199
|
+
serialized_result = _serialize_result(result)
|
|
200
|
+
|
|
201
|
+
if trajectory_gif_path:
|
|
202
|
+
import json
|
|
203
|
+
|
|
204
|
+
result_text = (
|
|
205
|
+
json.dumps(serialized_result, indent=2)
|
|
206
|
+
if isinstance(serialized_result, dict)
|
|
207
|
+
else str(serialized_result)
|
|
208
|
+
)
|
|
209
|
+
return ToolResult(
|
|
210
|
+
content=[
|
|
211
|
+
TextContent(type="text", text=result_text),
|
|
212
|
+
TextContent(type="text", text=f"Trajectory saved to {trajectory_gif_path}"),
|
|
213
|
+
],
|
|
214
|
+
)
|
|
215
|
+
|
|
216
|
+
return serialized_result
|
|
217
|
+
finally:
|
|
218
|
+
# Always clean up the cloud device
|
|
219
|
+
logger.info(f"Cleaning up {cloud_platform.value} cloud device...")
|
|
220
|
+
try:
|
|
221
|
+
await agent.clean()
|
|
222
|
+
logger.info(f"{cloud_platform.value} cloud device cleaned up successfully")
|
|
223
|
+
except Exception as e:
|
|
224
|
+
logger.error(f"Error cleaning up cloud device: {e}")
|
|
225
|
+
|
|
226
|
+
|
|
227
|
+
async def _download_trajectory_gif_if_available() -> Path | None:
|
|
228
|
+
"""Download the trajectory GIF if available and folder is configured.
|
|
229
|
+
|
|
230
|
+
Fetches the latest task run ID from the API and downloads the GIF.
|
|
231
|
+
|
|
232
|
+
Returns:
|
|
233
|
+
The path to the downloaded GIF file, or None if download failed or not configured.
|
|
234
|
+
"""
|
|
235
|
+
download_folder = settings.TRAJECTORY_GIF_DOWNLOAD_FOLDER
|
|
236
|
+
if not download_folder:
|
|
237
|
+
logger.warning("TRAJECTORY_GIF_DOWNLOAD_FOLDER not configured, skipping GIF download")
|
|
238
|
+
return None
|
|
239
|
+
|
|
240
|
+
task_run_id = None
|
|
241
|
+
try:
|
|
242
|
+
task_run_id = await get_latest_task_run_id()
|
|
243
|
+
|
|
244
|
+
gif_path = await download_trajectory_gif(
|
|
245
|
+
task_run_id=task_run_id,
|
|
246
|
+
download_path=download_folder,
|
|
247
|
+
)
|
|
248
|
+
logger.info(
|
|
249
|
+
"Trajectory GIF downloaded",
|
|
250
|
+
task_run_id=task_run_id,
|
|
251
|
+
path=str(gif_path),
|
|
252
|
+
)
|
|
253
|
+
return gif_path
|
|
254
|
+
except (StorageDownloadError, TaskRunsError) as e:
|
|
255
|
+
logger.warning(
|
|
256
|
+
"Failed to download trajectory GIF",
|
|
257
|
+
task_run_id=task_run_id,
|
|
258
|
+
error=str(e),
|
|
259
|
+
)
|
|
260
|
+
return None
|
|
@@ -1,18 +1,11 @@
|
|
|
1
1
|
"""Simple screenshot capture tool - returns raw base64 image without LLM analysis."""
|
|
2
2
|
|
|
3
|
-
import base64
|
|
4
|
-
|
|
5
3
|
from mcp.types import ImageContent
|
|
6
4
|
from pydantic import Field
|
|
7
5
|
|
|
8
6
|
from minitap.mcp.core.decorators import handle_tool_errors
|
|
9
7
|
from minitap.mcp.core.device import capture_screenshot, find_mobile_device
|
|
10
8
|
from minitap.mcp.main import mcp
|
|
11
|
-
from minitap.mcp.server.cloud_mobile import (
|
|
12
|
-
check_cloud_mobile_status,
|
|
13
|
-
get_cloud_mobile_id,
|
|
14
|
-
get_cloud_screenshot,
|
|
15
|
-
)
|
|
16
9
|
|
|
17
10
|
|
|
18
11
|
@mcp.tool(
|
|
@@ -32,17 +25,8 @@ async def take_screenshot(
|
|
|
32
25
|
),
|
|
33
26
|
) -> list[ImageContent]:
|
|
34
27
|
"""Capture screenshot and return as base64 image content."""
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
if cloud_mobile_id:
|
|
38
|
-
# Cloud mode: use cloud screenshot API
|
|
39
|
-
await check_cloud_mobile_status(cloud_mobile_id)
|
|
40
|
-
screenshot_bytes = await get_cloud_screenshot(cloud_mobile_id)
|
|
41
|
-
screenshot_base64 = base64.b64encode(screenshot_bytes).decode("utf-8")
|
|
42
|
-
else:
|
|
43
|
-
# Local mode: capture from local device
|
|
44
|
-
device = find_mobile_device(device_id=device_id)
|
|
45
|
-
screenshot_base64 = capture_screenshot(device)
|
|
28
|
+
device = find_mobile_device(device_id=device_id)
|
|
29
|
+
screenshot_base64 = capture_screenshot(device)
|
|
46
30
|
|
|
47
31
|
return [
|
|
48
32
|
ImageContent(
|
|
@@ -5,8 +5,6 @@ to remote storage, returning a filename that can be used with other tools
|
|
|
5
5
|
like figma_compare_screenshot.
|
|
6
6
|
"""
|
|
7
7
|
|
|
8
|
-
import base64
|
|
9
|
-
|
|
10
8
|
from fastmcp.exceptions import ToolError
|
|
11
9
|
from fastmcp.tools.tool import ToolResult
|
|
12
10
|
|
|
@@ -15,7 +13,6 @@ from minitap.mcp.core.device import capture_screenshot, find_mobile_device
|
|
|
15
13
|
from minitap.mcp.core.logging_config import get_logger
|
|
16
14
|
from minitap.mcp.core.storage import StorageUploadError, upload_screenshot_to_storage
|
|
17
15
|
from minitap.mcp.main import mcp
|
|
18
|
-
from minitap.mcp.server.cloud_mobile import get_cloud_mobile_id, get_cloud_screenshot
|
|
19
16
|
|
|
20
17
|
logger = get_logger(__name__)
|
|
21
18
|
|
|
@@ -26,7 +23,7 @@ logger = get_logger(__name__)
|
|
|
26
23
|
Capture a screenshot from the connected device and upload it to storage.
|
|
27
24
|
|
|
28
25
|
This tool:
|
|
29
|
-
1. Captures a screenshot from the connected
|
|
26
|
+
1. Captures a screenshot from the connected local device
|
|
30
27
|
2. Uploads the screenshot to remote storage
|
|
31
28
|
3. Returns a filename that can be used with other tools
|
|
32
29
|
|
|
@@ -43,23 +40,13 @@ async def upload_screenshot() -> ToolResult:
|
|
|
43
40
|
"""Capture and upload a device screenshot, return the filename."""
|
|
44
41
|
logger.info("Capturing and uploading device screenshot")
|
|
45
42
|
|
|
46
|
-
# Step 1: Capture screenshot from device
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
screenshot_base64 = base64.b64encode(screenshot_bytes).decode("utf-8")
|
|
54
|
-
except Exception as e:
|
|
55
|
-
raise ToolError(f"Failed to capture cloud device screenshot: {e}") from e
|
|
56
|
-
else:
|
|
57
|
-
logger.debug("Capturing screenshot from local device")
|
|
58
|
-
try:
|
|
59
|
-
device = find_mobile_device()
|
|
60
|
-
screenshot_base64 = capture_screenshot(device)
|
|
61
|
-
except Exception as e:
|
|
62
|
-
raise ToolError(f"Failed to capture local device screenshot: {e}") from e
|
|
43
|
+
# Step 1: Capture screenshot from local device
|
|
44
|
+
logger.debug("Capturing screenshot from local device")
|
|
45
|
+
try:
|
|
46
|
+
device = find_mobile_device()
|
|
47
|
+
screenshot_base64 = capture_screenshot(device)
|
|
48
|
+
except Exception as e:
|
|
49
|
+
raise ToolError(f"Failed to capture local device screenshot: {e}") from e
|
|
63
50
|
|
|
64
51
|
logger.info("Screenshot captured from device")
|
|
65
52
|
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
[project]
|
|
2
2
|
name = "minitap-mcp"
|
|
3
|
-
version = "0.9.
|
|
3
|
+
version = "0.9.2"
|
|
4
4
|
description = "Model Context Protocol server for controlling Android & iOS devices with natural language"
|
|
5
5
|
readme = "PYPI_README.md"
|
|
6
6
|
|
|
@@ -17,7 +17,7 @@ dependencies = [
|
|
|
17
17
|
"python-dotenv>=1.1.1",
|
|
18
18
|
"pydantic>=2.12.0",
|
|
19
19
|
"pydantic-settings>=2.10.1",
|
|
20
|
-
"minitap-mobile-use>=3.
|
|
20
|
+
"minitap-mobile-use>=3.5.0",
|
|
21
21
|
"jinja2>=3.1.6",
|
|
22
22
|
"langchain-core>=0.3.75",
|
|
23
23
|
"pillow>=11.1.0",
|