react-native-ai-debugger 1.5.2 → 1.6.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/build/__tests__/unit/toolDescriptions.test.d.ts +2 -0
- package/build/__tests__/unit/toolDescriptions.test.d.ts.map +1 -0
- package/build/__tests__/unit/toolDescriptions.test.js +145 -0
- package/build/__tests__/unit/toolDescriptions.test.js.map +1 -0
- package/build/core/guides.d.ts +7 -0
- package/build/core/guides.d.ts.map +1 -1
- package/build/core/guides.js +35 -12
- package/build/core/guides.js.map +1 -1
- package/build/core/nativeOnlyHints.js +2 -2
- package/build/core/nativeOnlyHints.js.map +1 -1
- package/build/index.d.ts +4 -1
- package/build/index.d.ts.map +1 -1
- package/build/index.js +1377 -1509
- package/build/index.js.map +1 -1
- package/package.json +1 -1
package/build/index.js
CHANGED
|
@@ -5,13 +5,13 @@ import { StreamableHTTPServerTransport } from "@modelcontextprotocol/sdk/server/
|
|
|
5
5
|
import { createServer as createHttpServer } from "node:http";
|
|
6
6
|
import { existsSync, unlinkSync } from "fs";
|
|
7
7
|
import { z } from "zod";
|
|
8
|
-
import { getGuideOverview, getGuideByTopic, getAvailableTopics } from "./core/guides.js";
|
|
8
|
+
import { getGuideOverview, getGuideByTopic, getAvailableTopics, DECISION_TREE } from "./core/guides.js";
|
|
9
9
|
import { getLicenseStatus, getDashboardUrl, getUsageInfo } from "./core/license.js";
|
|
10
10
|
import { API_BASE_URL } from "./core/config.js";
|
|
11
11
|
import { getPostHogClient, identifyIfDevMode, shutdownPostHog } from "./core/posthog.js";
|
|
12
12
|
import { getInstallationId, getServerVersion, getPackageName, isDevMode, TELEMETRY_JSONL_PATH, categorizeError } from "./core/telemetry.js";
|
|
13
13
|
import { isSDKInstalled, querySDKNetwork, getSDKNetworkEntry, getSDKNetworkStats, clearSDKNetwork, querySDKConsole, getSDKConsoleStats, clearSDKConsole } from "./core/sdkBridge.js";
|
|
14
|
-
import { tap
|
|
14
|
+
import { tap } from "./pro/tap.js";
|
|
15
15
|
import { getActivateLicenseConfig, handleActivateLicense, getDeleteAccountConfig, handleDeleteAccount, } from "./tools/accountTools.js";
|
|
16
16
|
import { logBuffers, networkBuffers, getLogBuffer, getNetworkBuffer, getTotalLogCount, getConnectedAppByDevice, getConnectedAppBySimulatorUdid, getConnectedAppByAndroidDeviceId, LogBuffer, NetworkBuffer, bundleErrorBuffer, imageBuffer, connectedApps, getActiveOrBootedSimulatorUdid, scanMetroPorts, fetchDevices, filterDebuggableDevices, connectToDevice, getConnectedApps, executeInApp, listDebugGlobals, inspectGlobal, reloadApp,
|
|
17
17
|
// React Component Inspection
|
|
@@ -32,16 +32,10 @@ recognizeText, inferIOSDevicePixelRatio,
|
|
|
32
32
|
listAndroidDevices, androidScreenshot, androidInstallApp, androidLaunchApp, androidListPackages,
|
|
33
33
|
// Android UI Input (Phase 2)
|
|
34
34
|
ANDROID_KEY_EVENTS, androidTap, androidLongPress, androidSwipe, androidInputText, androidKeyEvent, androidGetScreenSize, androidGetDensity, androidGetStatusBarHeight,
|
|
35
|
-
// Android Accessibility (UI Hierarchy)
|
|
36
|
-
androidDescribeAll, androidDescribePoint,
|
|
37
|
-
// Android Element Finding (no screenshots)
|
|
38
|
-
androidFindElement, androidWaitForElement,
|
|
39
35
|
// iOS
|
|
40
36
|
listIOSSimulators, iosScreenshot, iosInstallApp, iosLaunchApp, iosOpenUrl, iosTerminateApp, iosBootSimulator,
|
|
41
37
|
// iOS UI driver tools
|
|
42
|
-
isUiDriverAvailable, iosTap,
|
|
43
|
-
// iOS Element Finding (no screenshots)
|
|
44
|
-
iosFindElement, iosWaitForElement, getDevicePixelRatio, getIOSSafeAreaTop,
|
|
38
|
+
isUiDriverAvailable, iosTap, iosButton, iosDescribeAll, IOS_BUTTON_TYPES, getDevicePixelRatio, getIOSSafeAreaTop,
|
|
45
39
|
// Telemetry
|
|
46
40
|
initTelemetry, trackToolInvocation, getTargetPlatform,
|
|
47
41
|
// Format utilities (TONL)
|
|
@@ -91,7 +85,13 @@ const server = new McpServer({
|
|
|
91
85
|
name: "react-native-ai-devtools",
|
|
92
86
|
version: "1.0.0"
|
|
93
87
|
}, {
|
|
94
|
-
instructions:
|
|
88
|
+
instructions: [
|
|
89
|
+
"React Native debugging MCP server.",
|
|
90
|
+
"",
|
|
91
|
+
DECISION_TREE,
|
|
92
|
+
"",
|
|
93
|
+
"Call get_usage_guide with no arguments for the same decision tree plus a summary of every guide."
|
|
94
|
+
].join("\n")
|
|
95
95
|
});
|
|
96
96
|
// ============================================================================
|
|
97
97
|
// Telemetry Wrapper
|
|
@@ -178,9 +178,28 @@ const NON_METRO_TOOLS = new Set([
|
|
|
178
178
|
"android_install_app",
|
|
179
179
|
"send_feedback"
|
|
180
180
|
]);
|
|
181
|
-
//
|
|
181
|
+
// Banner helpers for platform-specific tool descriptions. Appended after the
|
|
182
|
+
// verbatim first sentence of every ios_*/android_* tool to steer agents toward
|
|
183
|
+
// cross-platform primaries (tap, get_screen_layout, etc) unless native-only
|
|
184
|
+
// behavior is required. See src/core/nativeOnlyHints.ts for the complementary
|
|
185
|
+
// runtime hint shown when Metro is absent.
|
|
186
|
+
const platformFallbackBanner = (prefer) => `\n[PLATFORM FALLBACK — prefer ${prefer} unless you specifically need native-only behavior]`;
|
|
187
|
+
const platformUniqueBanner = (useCase) => `\n[PLATFORM-SPECIFIC — no cross-platform equivalent; use when ${useCase}]`;
|
|
188
|
+
const primaryInteractionBanner = () => `\n[PRIMARY INTERACTION TOOL — works on iOS and Android; prefer over ios_*/android_* siblings]`;
|
|
189
|
+
// Registry for dev meta-tool — stores handlers and configs for dynamic dispatch.
|
|
190
|
+
// Also exported so unit tests can enumerate every registered tool without booting
|
|
191
|
+
// the server. Populated by registerToolWithTelemetry AND by the server.registerTool
|
|
192
|
+
// interceptor installed below, so it captures every registration site.
|
|
182
193
|
/* eslint-disable @typescript-eslint/no-explicit-any */
|
|
183
|
-
const toolRegistry = new Map();
|
|
194
|
+
export const toolRegistry = new Map();
|
|
195
|
+
// Interceptor: capture every direct server.registerTool call into toolRegistry so
|
|
196
|
+
// tests and the dev meta-tool see the full surface (including platform-native tools
|
|
197
|
+
// and the dev meta-tool itself that bypass registerToolWithTelemetry).
|
|
198
|
+
const _originalRegisterTool = server.registerTool.bind(server);
|
|
199
|
+
server.registerTool = (name, config, handler) => {
|
|
200
|
+
toolRegistry.set(name, { config, handler });
|
|
201
|
+
return _originalRegisterTool(name, config, handler);
|
|
202
|
+
};
|
|
184
203
|
function registerToolWithTelemetry(toolName, config, handler, emptyResultDetector) {
|
|
185
204
|
toolRegistry.set(toolName, { config, handler });
|
|
186
205
|
server.registerTool(toolName, config, async (args) => {
|
|
@@ -360,7 +379,13 @@ registerToolWithTelemetry("get_usage_guide", {
|
|
|
360
379
|
});
|
|
361
380
|
// Tool: Scan for Metro servers
|
|
362
381
|
registerToolWithTelemetry("scan_metro", {
|
|
363
|
-
description: "Scan for running Metro bundler servers and automatically connect to any found React Native apps. This is typically the FIRST tool to call when starting a debugging session - it establishes the connection needed for other tools like get_logs, list_debug_globals, execute_in_app, and reload_app
|
|
382
|
+
description: "Scan for running Metro bundler servers and automatically connect to any found React Native apps. This is typically the FIRST tool to call when starting a debugging session - it establishes the connection needed for other tools like get_logs, list_debug_globals, execute_in_app, and reload_app.\n" +
|
|
383
|
+
"PURPOSE: Discover Metro on common ports (8081, 8082, 19000-19002) and auto-connect all React Native debugger targets it advertises.\n" +
|
|
384
|
+
"WHEN TO USE: At the start of any session, or after the user restarts Metro / boots a new simulator.\n" +
|
|
385
|
+
"WORKFLOW: scan_metro -> get_apps -> get_logs / ios_screenshot / tap.\n" +
|
|
386
|
+
"GOOD: scan_metro()\n" +
|
|
387
|
+
"BAD: scan_metro() called repeatedly in a loop — use ensure_connection to re-verify an existing connection.\n" +
|
|
388
|
+
"SEE ALSO: call get_usage_guide(topic=\"setup\") for the full session-setup playbook.",
|
|
364
389
|
inputSchema: {
|
|
365
390
|
startPort: z.coerce.number().optional().default(8081).describe("Start port for scanning (default: 8081)"),
|
|
366
391
|
endPort: z.coerce.number().optional().default(19002).describe("End port for scanning (default: 19002)")
|
|
@@ -482,7 +507,7 @@ registerToolWithTelemetry("scan_metro", {
|
|
|
482
507
|
const uiDriverOk = await isUiDriverAvailable();
|
|
483
508
|
if (!uiDriverOk) {
|
|
484
509
|
results.push("");
|
|
485
|
-
results.push("⚠️ WARNING: iOS UI driver is NOT installed.
|
|
510
|
+
results.push("⚠️ WARNING: iOS UI driver is NOT installed. The `tap` tool and other UI-interaction tools will fail.");
|
|
486
511
|
results.push(" Install the recommended driver: brew install cameroncooke/axe/axe");
|
|
487
512
|
results.push(" Then set IOS_DRIVER=axe in your MCP server environment.");
|
|
488
513
|
results.push(" Alternative: brew install idb-companion (used by default)");
|
|
@@ -497,224 +522,15 @@ registerToolWithTelemetry("scan_metro", {
|
|
|
497
522
|
]
|
|
498
523
|
};
|
|
499
524
|
});
|
|
500
|
-
// Tool: Get connected apps
|
|
501
|
-
registerToolWithTelemetry("get_apps", {
|
|
502
|
-
description: "List currently connected React Native apps and their connection status. If no apps are connected, run scan_metro first to establish a connection.",
|
|
503
|
-
inputSchema: {}
|
|
504
|
-
}, async () => {
|
|
505
|
-
const apps = getConnectedApps();
|
|
506
|
-
if (apps.length === 0) {
|
|
507
|
-
return {
|
|
508
|
-
content: [
|
|
509
|
-
{
|
|
510
|
-
type: "text",
|
|
511
|
-
text: "No connected devices. Run scan_metro to discover and connect to Metro servers."
|
|
512
|
-
}
|
|
513
|
-
]
|
|
514
|
-
};
|
|
515
|
-
}
|
|
516
|
-
const deviceLines = apps
|
|
517
|
-
.filter(({ isConnected }) => isConnected)
|
|
518
|
-
.map(({ app }, i) => {
|
|
519
|
-
const name = app.deviceInfo.deviceName || app.deviceInfo.title;
|
|
520
|
-
const appId = app.deviceInfo.appId || app.deviceInfo.title.split(" (")[0] || "unknown";
|
|
521
|
-
const lines = [` ${i + 1}. ${name} — ${appId} (${app.platform}, port ${app.port})`];
|
|
522
|
-
if (app.appDetection) {
|
|
523
|
-
const d = app.appDetection;
|
|
524
|
-
const version = d.reactNativeVersion !== "unknown" ? `RN ${d.reactNativeVersion}` : "RN unknown";
|
|
525
|
-
const expo = d.expoSdkVersion ? `, Expo SDK ${d.expoSdkVersion}` : "";
|
|
526
|
-
lines.push(` Environment: ${version}, ${d.architecture} arch, ${d.jsEngine}, ${app.platform} ${d.osVersion}${expo}`);
|
|
527
|
-
}
|
|
528
|
-
return lines.join("\n");
|
|
529
|
-
});
|
|
530
|
-
const text = [
|
|
531
|
-
`Connected devices:`,
|
|
532
|
-
...deviceLines,
|
|
533
|
-
``,
|
|
534
|
-
`Use device="${apps[0].app.deviceInfo.deviceName}" to target a specific device.`,
|
|
535
|
-
``,
|
|
536
|
-
`Total logs in buffer: ${getTotalLogCount()}`
|
|
537
|
-
].join("\n");
|
|
538
|
-
return {
|
|
539
|
-
content: [{ type: "text", text }]
|
|
540
|
-
};
|
|
541
|
-
});
|
|
542
|
-
// Tool: Get connection status (detailed health and gap tracking)
|
|
543
|
-
registerToolWithTelemetry("get_connection_status", {
|
|
544
|
-
description: "Get detailed connection health status including uptime, recent disconnects/reconnects, and connection gaps that may indicate missing data.",
|
|
545
|
-
inputSchema: {}
|
|
546
|
-
}, async () => {
|
|
547
|
-
const connections = getConnectedApps();
|
|
548
|
-
const states = getAllConnectionStates();
|
|
549
|
-
const metadata = getAllConnectionMetadata();
|
|
550
|
-
const lines = [];
|
|
551
|
-
lines.push("=== Connection Status ===\n");
|
|
552
|
-
if (connections.length === 0 && states.size === 0) {
|
|
553
|
-
lines.push("No connections established. Run scan_metro to connect.");
|
|
554
|
-
return {
|
|
555
|
-
content: [{ type: "text", text: lines.join("\n") }]
|
|
556
|
-
};
|
|
557
|
-
}
|
|
558
|
-
// Show active connections
|
|
559
|
-
for (const { key, app, isConnected } of connections) {
|
|
560
|
-
const state = states.get(key);
|
|
561
|
-
const contextHealth = getContextHealth(key);
|
|
562
|
-
lines.push(`--- ${app.deviceInfo.title} (Port ${app.port}) ---`);
|
|
563
|
-
lines.push(` Status: ${isConnected ? "CONNECTED" : "DISCONNECTED"}`);
|
|
564
|
-
if (state) {
|
|
565
|
-
if (state.lastConnectedTime) {
|
|
566
|
-
const uptime = Date.now() - state.lastConnectedTime.getTime();
|
|
567
|
-
lines.push(` Connected since: ${state.lastConnectedTime.toLocaleTimeString()}`);
|
|
568
|
-
lines.push(` Uptime: ${formatDuration(uptime)}`);
|
|
569
|
-
}
|
|
570
|
-
if (state.status === "reconnecting") {
|
|
571
|
-
lines.push(` Reconnecting: Attempt ${state.reconnectionAttempts}`);
|
|
572
|
-
}
|
|
573
|
-
// Show recent gaps (last 5 minutes)
|
|
574
|
-
if (state.connectionGaps.length > 0) {
|
|
575
|
-
const recentGaps = state.connectionGaps.filter((g) => Date.now() - g.disconnectedAt.getTime() < 300000);
|
|
576
|
-
if (recentGaps.length > 0) {
|
|
577
|
-
lines.push(` Recent gaps: ${recentGaps.length}`);
|
|
578
|
-
for (const gap of recentGaps.slice(-3)) {
|
|
579
|
-
const duration = gap.durationMs ? formatDuration(gap.durationMs) : "ongoing";
|
|
580
|
-
lines.push(` - ${gap.disconnectedAt.toLocaleTimeString()} (${duration}): ${gap.reason}`);
|
|
581
|
-
}
|
|
582
|
-
}
|
|
583
|
-
}
|
|
584
|
-
}
|
|
585
|
-
// Show context health
|
|
586
|
-
if (contextHealth) {
|
|
587
|
-
lines.push(` Context Health:`);
|
|
588
|
-
lines.push(` Context ID: ${contextHealth.contextId ?? "unknown"}`);
|
|
589
|
-
lines.push(` Status: ${contextHealth.isStale ? "STALE" : "HEALTHY"}`);
|
|
590
|
-
if (contextHealth.lastHealthCheck) {
|
|
591
|
-
const healthResult = contextHealth.lastHealthCheckSuccess ? "PASS" : "FAIL";
|
|
592
|
-
lines.push(` Last Check: ${contextHealth.lastHealthCheck.toLocaleTimeString()} (${healthResult})`);
|
|
593
|
-
}
|
|
594
|
-
if (contextHealth.lastContextCreated) {
|
|
595
|
-
lines.push(` Context Created: ${contextHealth.lastContextCreated.toLocaleTimeString()}`);
|
|
596
|
-
}
|
|
597
|
-
if (contextHealth.lastContextDestroyed) {
|
|
598
|
-
lines.push(` Context Destroyed: ${contextHealth.lastContextDestroyed.toLocaleTimeString()}`);
|
|
599
|
-
}
|
|
600
|
-
}
|
|
601
|
-
lines.push("");
|
|
602
|
-
}
|
|
603
|
-
// Show disconnected/reconnecting states without active connections
|
|
604
|
-
for (const [key, state] of states.entries()) {
|
|
605
|
-
if (!connections.find((c) => c.key === key)) {
|
|
606
|
-
const meta = metadata.get(key);
|
|
607
|
-
lines.push(`--- ${meta?.deviceInfo.title || key} (Disconnected) ---`);
|
|
608
|
-
lines.push(` Status: ${state.status.toUpperCase()}`);
|
|
609
|
-
if (state.lastDisconnectTime) {
|
|
610
|
-
lines.push(` Disconnected at: ${state.lastDisconnectTime.toLocaleTimeString()}`);
|
|
611
|
-
}
|
|
612
|
-
if (state.reconnectionAttempts > 0) {
|
|
613
|
-
lines.push(` Reconnection attempts: ${state.reconnectionAttempts}`);
|
|
614
|
-
}
|
|
615
|
-
lines.push("");
|
|
616
|
-
}
|
|
617
|
-
}
|
|
618
|
-
return {
|
|
619
|
-
content: [{ type: "text", text: lines.join("\n") }]
|
|
620
|
-
};
|
|
621
|
-
});
|
|
622
|
-
// Tool: License status
|
|
623
|
-
registerToolWithTelemetry("get_license_status", {
|
|
624
|
-
description: "Get your installation ID and license status. Shows your unique Installation ID (needed to activate Pro in the dashboard), current license tier, and cache validity.",
|
|
625
|
-
inputSchema: {},
|
|
626
|
-
}, async () => {
|
|
627
|
-
const status = getLicenseStatus();
|
|
628
|
-
const lines = [];
|
|
629
|
-
lines.push(`Installation ID: ${status.installationId}`);
|
|
630
|
-
lines.push(`License: ${status.tier.charAt(0).toUpperCase() + status.tier.slice(1)}`);
|
|
631
|
-
if (status.plan) {
|
|
632
|
-
lines.push(`Plan expires: ${status.plan.expiresAt}`);
|
|
633
|
-
}
|
|
634
|
-
lines.push(`Cache valid until: ${status.cacheExpiresAt}`);
|
|
635
|
-
// Usage limits
|
|
636
|
-
const usage = getUsageInfo();
|
|
637
|
-
if (usage) {
|
|
638
|
-
lines.push("");
|
|
639
|
-
lines.push("--- Usage ---");
|
|
640
|
-
if (usage.creditsRemaining !== null) {
|
|
641
|
-
lines.push(`Credits remaining: ${usage.creditsRemaining.toLocaleString()}`);
|
|
642
|
-
lines.push(`Used this month: ${usage.used.toLocaleString()}`);
|
|
643
|
-
}
|
|
644
|
-
else {
|
|
645
|
-
lines.push(`Monthly usage: ${usage.used} / ${usage.limit}`);
|
|
646
|
-
}
|
|
647
|
-
lines.push(`Month: ${usage.monthKey}`);
|
|
648
|
-
if (usage.promotionalPeriod) {
|
|
649
|
-
const endsAt = usage.promotionalPeriodEndsAt
|
|
650
|
-
? new Date(usage.promotionalPeriodEndsAt).toLocaleDateString("en-GB", { day: "2-digit", month: "2-digit", year: "numeric" })
|
|
651
|
-
: "unknown";
|
|
652
|
-
lines.push(`Status: Active (promotional period — no limits until ${endsAt})`);
|
|
653
|
-
}
|
|
654
|
-
else {
|
|
655
|
-
lines.push(`Status: ${usage.canUse ? "Active" : "Limit reached"}`);
|
|
656
|
-
}
|
|
657
|
-
}
|
|
658
|
-
if (status.tier === "free") {
|
|
659
|
-
const dashboardUrl = getDashboardUrl();
|
|
660
|
-
lines.push("");
|
|
661
|
-
if (dashboardUrl) {
|
|
662
|
-
lines.push(`Link your account: ${dashboardUrl}/link?id=${status.installationId}`);
|
|
663
|
-
}
|
|
664
|
-
}
|
|
665
|
-
return {
|
|
666
|
-
content: [{ type: "text", text: lines.join("\n") }],
|
|
667
|
-
};
|
|
668
|
-
});
|
|
669
|
-
// Tool: Send feedback / bug report / feature request
|
|
670
|
-
registerToolWithTelemetry("send_feedback", {
|
|
671
|
-
description: "Create a feedback report (bug, feature request, or general feedback) for the react-native-ai-devtools team. " +
|
|
672
|
-
"Auto-collects environment info. Returns a pre-filled GitHub issue URL and formatted issue body. " +
|
|
673
|
-
"Ask the user to open the URL and paste the body to submit.",
|
|
674
|
-
inputSchema: {
|
|
675
|
-
type: z.enum(["feedback", "feature_request", "bug"]).describe('Type of feedback: "feedback", "feature_request", or "bug"'),
|
|
676
|
-
title: z.string().describe("Short summary (becomes the GitHub issue title)"),
|
|
677
|
-
description: z.string().describe("Detailed description of the feedback, feature request, or bug"),
|
|
678
|
-
workflow_context: z.string().optional().describe("What the user was trying to accomplish when they decided to give feedback")
|
|
679
|
-
}
|
|
680
|
-
}, async ({ type, title, description, workflow_context }) => {
|
|
681
|
-
// Collect environment info
|
|
682
|
-
const serverVersion = getServerVersion();
|
|
683
|
-
const platform = process.platform;
|
|
684
|
-
const deviceType = getTargetPlatform();
|
|
685
|
-
const licenseStatus = getLicenseStatus();
|
|
686
|
-
const env = {
|
|
687
|
-
serverVersion,
|
|
688
|
-
platform,
|
|
689
|
-
deviceType,
|
|
690
|
-
licenseTier: licenseStatus.tier
|
|
691
|
-
};
|
|
692
|
-
const input = { type, title, description, workflowContext: workflow_context };
|
|
693
|
-
const issueBody = formatIssueBody(input, env);
|
|
694
|
-
const githubUrl = buildGitHubUrl(title, type);
|
|
695
|
-
const output = [
|
|
696
|
-
"## Feedback Report Ready",
|
|
697
|
-
"",
|
|
698
|
-
`**GitHub Issue URL:** ${githubUrl}`,
|
|
699
|
-
"",
|
|
700
|
-
"**Issue body to paste:**",
|
|
701
|
-
"",
|
|
702
|
-
"```markdown",
|
|
703
|
-
issueBody,
|
|
704
|
-
"```",
|
|
705
|
-
"",
|
|
706
|
-
"Please ask the user to:",
|
|
707
|
-
"1. Open the GitHub URL above",
|
|
708
|
-
"2. Paste the issue body into the description field",
|
|
709
|
-
"3. Review and submit the issue"
|
|
710
|
-
].join("\n");
|
|
711
|
-
return {
|
|
712
|
-
content: [{ type: "text", text: output }]
|
|
713
|
-
};
|
|
714
|
-
});
|
|
715
525
|
// Tool: Ensure connection health
|
|
716
526
|
registerToolWithTelemetry("ensure_connection", {
|
|
717
|
-
description: "Verify or establish a healthy connection to a React Native app. Use before running commands if connection may be stale, or after navigation/reload. This tool runs a health check and will auto-reconnect if needed
|
|
527
|
+
description: "Verify or establish a healthy connection to a React Native app. Use before running commands if connection may be stale, or after navigation/reload. This tool runs a health check and will auto-reconnect if needed.\n" +
|
|
528
|
+
"PURPOSE: Health-check the existing CDP connection and transparently reconnect if it has gone stale, without rescanning all Metro ports.\n" +
|
|
529
|
+
"WHEN TO USE: After a suspected disconnect (silent gaps, reload_app, app crash) or before long-running flows where a mid-flow drop would be costly. Cheaper than scan_metro when you already connected once this session.\n" +
|
|
530
|
+
"WORKFLOW: scan_metro (once) -> ensure_connection(healthCheck=true) -> resume tool calls. Use forceRefresh=true if the first probe still looks dead.\n" +
|
|
531
|
+
"GOOD: ensure_connection({ healthCheck: true })\n" +
|
|
532
|
+
"BAD: ensure_connection() before scan_metro has ever run — call scan_metro first.\n" +
|
|
533
|
+
"SEE ALSO: call get_usage_guide(topic=\"setup\") for the full session-setup playbook.",
|
|
718
534
|
inputSchema: {
|
|
719
535
|
port: z.coerce.number().optional().describe("Metro port (default: auto-detect)"),
|
|
720
536
|
healthCheck: z
|
|
@@ -763,48 +579,280 @@ registerToolWithTelemetry("ensure_connection", {
|
|
|
763
579
|
content: [{ type: "text", text: lines.join("\n") }]
|
|
764
580
|
};
|
|
765
581
|
});
|
|
766
|
-
// Tool: Get
|
|
767
|
-
registerToolWithTelemetry("
|
|
768
|
-
description: "
|
|
769
|
-
|
|
770
|
-
|
|
771
|
-
|
|
772
|
-
|
|
773
|
-
|
|
774
|
-
|
|
775
|
-
|
|
776
|
-
|
|
777
|
-
|
|
778
|
-
|
|
779
|
-
|
|
780
|
-
|
|
781
|
-
|
|
782
|
-
|
|
783
|
-
|
|
784
|
-
|
|
785
|
-
|
|
786
|
-
|
|
787
|
-
|
|
788
|
-
|
|
789
|
-
|
|
790
|
-
|
|
791
|
-
|
|
792
|
-
|
|
793
|
-
|
|
794
|
-
|
|
795
|
-
|
|
796
|
-
|
|
797
|
-
.
|
|
798
|
-
.
|
|
799
|
-
.
|
|
800
|
-
|
|
801
|
-
|
|
802
|
-
}
|
|
803
|
-
|
|
804
|
-
|
|
805
|
-
|
|
806
|
-
|
|
807
|
-
|
|
582
|
+
// Tool: Get connected apps
|
|
583
|
+
registerToolWithTelemetry("get_apps", {
|
|
584
|
+
description: "List currently connected React Native apps and their connection status. If no apps are connected, run scan_metro first to establish a connection.\n" +
|
|
585
|
+
"PURPOSE: Enumerate active debug targets with device names, platforms, ports, and detected RN/Expo versions so you can target the right one.\n" +
|
|
586
|
+
"WHEN TO USE: After scan_metro to confirm what connected, or before passing a device=\"...\" filter to another tool.\n" +
|
|
587
|
+
"WORKFLOW: scan_metro -> get_apps -> get_logs / ios_screenshot / tap (with device=\"...\" if multiple).\n" +
|
|
588
|
+
"LIMITATIONS: Only lists devices the MCP has successfully connected to — stale targets don't appear here, use get_connection_status for health details.\n" +
|
|
589
|
+
"GOOD: get_apps()\n" +
|
|
590
|
+
"BAD: Calling get_apps in a tight loop — the list doesn't change without a scan_metro or disconnect_metro.\n" +
|
|
591
|
+
"SEE ALSO: call get_usage_guide(topic=\"setup\") for the full session-setup playbook.",
|
|
592
|
+
inputSchema: {}
|
|
593
|
+
}, async () => {
|
|
594
|
+
const apps = getConnectedApps();
|
|
595
|
+
if (apps.length === 0) {
|
|
596
|
+
return {
|
|
597
|
+
content: [
|
|
598
|
+
{
|
|
599
|
+
type: "text",
|
|
600
|
+
text: "No connected devices. Run scan_metro to discover and connect to Metro servers."
|
|
601
|
+
}
|
|
602
|
+
]
|
|
603
|
+
};
|
|
604
|
+
}
|
|
605
|
+
const deviceLines = apps
|
|
606
|
+
.filter(({ isConnected }) => isConnected)
|
|
607
|
+
.map(({ app }, i) => {
|
|
608
|
+
const name = app.deviceInfo.deviceName || app.deviceInfo.title;
|
|
609
|
+
const appId = app.deviceInfo.appId || app.deviceInfo.title.split(" (")[0] || "unknown";
|
|
610
|
+
const lines = [` ${i + 1}. ${name} — ${appId} (${app.platform}, port ${app.port})`];
|
|
611
|
+
if (app.appDetection) {
|
|
612
|
+
const d = app.appDetection;
|
|
613
|
+
const version = d.reactNativeVersion !== "unknown" ? `RN ${d.reactNativeVersion}` : "RN unknown";
|
|
614
|
+
const expo = d.expoSdkVersion ? `, Expo SDK ${d.expoSdkVersion}` : "";
|
|
615
|
+
lines.push(` Environment: ${version}, ${d.architecture} arch, ${d.jsEngine}, ${app.platform} ${d.osVersion}${expo}`);
|
|
616
|
+
}
|
|
617
|
+
return lines.join("\n");
|
|
618
|
+
});
|
|
619
|
+
const text = [
|
|
620
|
+
`Connected devices:`,
|
|
621
|
+
...deviceLines,
|
|
622
|
+
``,
|
|
623
|
+
`Use device="${apps[0].app.deviceInfo.deviceName}" to target a specific device.`,
|
|
624
|
+
``,
|
|
625
|
+
`Total logs in buffer: ${getTotalLogCount()}`
|
|
626
|
+
].join("\n");
|
|
627
|
+
return {
|
|
628
|
+
content: [{ type: "text", text }]
|
|
629
|
+
};
|
|
630
|
+
});
|
|
631
|
+
// Tool: Unified tap — tries fiber, accessibility, OCR, coordinate strategies
|
|
632
|
+
registerToolWithTelemetry("tap", {
|
|
633
|
+
description: "Tap a UI element. Automatically tries multiple strategies: fiber tree (React), accessibility tree (native), and OCR (visual)." +
|
|
634
|
+
primaryInteractionBanner() + "\n" +
|
|
635
|
+
"PURPOSE: Single unified tap entry point — resolves text/testID/component/coordinates into a real touch event on the correct device.\n" +
|
|
636
|
+
"WHEN TO USE: Any time you need to press a button, focus an input, open a menu, or verify a handler fires. Prefer testID, then text, then component, then (x,y) from a screenshot's pressables list.\n" +
|
|
637
|
+
"WORKFLOW: ios_screenshot or android_screenshot -> tap(testID=\"...\") | tap(text=\"...\") | tap(x, y) -> screenshot again to verify. Use burst=true when meaningful=false but visual feedback looks transient.\n" +
|
|
638
|
+
"LIMITATIONS: iOS needs AXe (brew install cameroncooke/axe/axe) or IDB for accessibility/coordinate taps. Non-ASCII text skips fiber (Hermes); prefer testID. When iOS AND Android are connected, pass platform explicitly.\n" +
|
|
639
|
+
"GOOD: tap({ testID: \"login-btn\" }); tap({ text: \"Submit\" }); tap({ x: 300, y: 600 }); tap({ x: 300, y: 600, native: true, platform: \"android\" })\n" +
|
|
640
|
+
"BAD: tap({ text: \"\" }) or tap({ x: 0, y: 0 }) — missing a target. tap({ text: \"Submit\" }) without first screenshotting an ambiguous screen.\n" +
|
|
641
|
+
"SEE ALSO: call get_usage_guide(topic=\"interact\") for the full device-interaction playbook.",
|
|
642
|
+
inputSchema: {
|
|
643
|
+
text: z
|
|
644
|
+
.string()
|
|
645
|
+
.optional()
|
|
646
|
+
.describe("Visible text to match (case-insensitive substring). ASCII only for fiber strategy; OCR handles non-ASCII."),
|
|
647
|
+
testID: z
|
|
648
|
+
.string()
|
|
649
|
+
.optional()
|
|
650
|
+
.describe("Exact match on the element's testID prop."),
|
|
651
|
+
component: z
|
|
652
|
+
.string()
|
|
653
|
+
.optional()
|
|
654
|
+
.describe("Component name match (case-insensitive substring, e.g. 'Button', 'MenuItem')."),
|
|
655
|
+
index: z.coerce
|
|
656
|
+
.number()
|
|
657
|
+
.optional()
|
|
658
|
+
.describe("Zero-based index when multiple elements match (default: 0)."),
|
|
659
|
+
x: z.coerce
|
|
660
|
+
.number()
|
|
661
|
+
.optional()
|
|
662
|
+
.describe("X coordinate in pixels (from screenshot). Must provide both x and y."),
|
|
663
|
+
y: z.coerce
|
|
664
|
+
.number()
|
|
665
|
+
.optional()
|
|
666
|
+
.describe("Y coordinate in pixels (from screenshot). Must provide both x and y."),
|
|
667
|
+
strategy: z
|
|
668
|
+
.enum(["auto", "fiber", "accessibility", "ocr", "coordinate"])
|
|
669
|
+
.optional()
|
|
670
|
+
.default("auto")
|
|
671
|
+
.describe('"auto" (default) tries fiber -> accessibility -> OCR. Set explicitly to skip strategies you know will fail.'),
|
|
672
|
+
maxTraversalDepth: z.coerce
|
|
673
|
+
.number()
|
|
674
|
+
.optional()
|
|
675
|
+
.describe("Max parent levels to traverse when searching by component name (default: 15). " +
|
|
676
|
+
"Increase if your component is deeply wrapped (e.g. inside multiple HOCs/animation wrappers)."),
|
|
677
|
+
native: z
|
|
678
|
+
.boolean()
|
|
679
|
+
.optional()
|
|
680
|
+
.default(false)
|
|
681
|
+
.describe("When true, tap coordinates directly via ADB/simctl without requiring a React Native connection. " +
|
|
682
|
+
"Useful for interacting with native UI, system dialogs, or non-RN apps. Requires x/y coordinates."),
|
|
683
|
+
platform: z
|
|
684
|
+
.enum(["ios", "android"])
|
|
685
|
+
.optional()
|
|
686
|
+
.describe("Target platform. Required when both iOS and Android devices are connected. Auto-detected if only one platform is available."),
|
|
687
|
+
screenshot: z
|
|
688
|
+
.boolean()
|
|
689
|
+
.optional()
|
|
690
|
+
.default(true)
|
|
691
|
+
.describe("Capture and return a post-tap screenshot. Enabled by default. Set to false for fastest execution (no screenshots)."),
|
|
692
|
+
verify: z
|
|
693
|
+
.boolean()
|
|
694
|
+
.optional()
|
|
695
|
+
.describe("Run before/after screenshot diff to detect if the tap had a meaningful visual effect. " +
|
|
696
|
+
"Default: true for coordinate/accessibility/ocr strategies, false for fiber. Requires screenshot=true."),
|
|
697
|
+
burst: z
|
|
698
|
+
.boolean()
|
|
699
|
+
.optional()
|
|
700
|
+
.default(false)
|
|
701
|
+
.describe("Enable burst screenshot capture for enhanced verification. " +
|
|
702
|
+
"Captures 4 rapid screenshots (~150ms intervals) after the tap to detect transient visual feedback " +
|
|
703
|
+
"(press animations, highlights, ripples) that may settle before a standard after-screenshot. " +
|
|
704
|
+
"Results are stored in the image buffer (use get_images to inspect individual frames). " +
|
|
705
|
+
"Default: false."),
|
|
706
|
+
},
|
|
707
|
+
}, async (args) => {
|
|
708
|
+
const result = await tap({
|
|
709
|
+
text: args.text,
|
|
710
|
+
testID: args.testID,
|
|
711
|
+
component: args.component,
|
|
712
|
+
index: args.index,
|
|
713
|
+
x: args.x,
|
|
714
|
+
y: args.y,
|
|
715
|
+
strategy: args.strategy,
|
|
716
|
+
maxTraversalDepth: args.maxTraversalDepth,
|
|
717
|
+
native: args.native,
|
|
718
|
+
platform: args.platform,
|
|
719
|
+
screenshot: args.screenshot,
|
|
720
|
+
verify: args.verify,
|
|
721
|
+
burst: args.burst,
|
|
722
|
+
});
|
|
723
|
+
const { screenshot: screenshotData, ...resultWithoutScreenshot } = result;
|
|
724
|
+
const text = JSON.stringify(resultWithoutScreenshot, null, 2);
|
|
725
|
+
// Pack strategy mode + attempted strategies into errorContext for telemetry
|
|
726
|
+
// e.g. "s=ocr|fiber:no_pressable|ocr:no_match" or "s=auto|fiber:no_pressable|accessibility:not_found|ocr:no_match"
|
|
727
|
+
const stratPrefix = args.strategy && args.strategy !== "auto" ? `s=${args.strategy}|` : "";
|
|
728
|
+
const errorContext = result.attempted?.length
|
|
729
|
+
? stratPrefix + result.attempted.map(a => `${a.strategy}:${a.reason.slice(0, 40)}`).join("|")
|
|
730
|
+
: undefined;
|
|
731
|
+
const content = [
|
|
732
|
+
{ type: "text", text },
|
|
733
|
+
];
|
|
734
|
+
if (screenshotData) {
|
|
735
|
+
content.push({
|
|
736
|
+
type: "image",
|
|
737
|
+
data: screenshotData.image,
|
|
738
|
+
mimeType: "image/jpeg",
|
|
739
|
+
});
|
|
740
|
+
}
|
|
741
|
+
return {
|
|
742
|
+
content,
|
|
743
|
+
isError: !result.success && !result.ambiguous,
|
|
744
|
+
_errorMessage: !result.success && !result.ambiguous
|
|
745
|
+
? `${JSON.stringify(result.query)}|${result.error || ""}`
|
|
746
|
+
: undefined,
|
|
747
|
+
_errorContext: errorContext,
|
|
748
|
+
_meaningful: result.verification?.meaningful,
|
|
749
|
+
_changeRate: result.verification?.changeRate,
|
|
750
|
+
_tapStrategy: result.method,
|
|
751
|
+
_iosDriver: result.platform === "ios" ? (process.env.IOS_DRIVER?.toLowerCase() || "idb") : undefined,
|
|
752
|
+
};
|
|
753
|
+
});
|
|
754
|
+
// Tool: Get full screen layout (all components with layout styles)
|
|
755
|
+
registerToolWithTelemetry("get_screen_layout", {
|
|
756
|
+
description: "Get a screen map showing visible components as an indented tree with actual screen positions. Uses measureInWindow for real coordinates and filters out off-screen components. Returns meaningful component names with text content and frame data (x,y width x height). Coordinates are in **points** (iOS) or **dp** (Android) — NOT screenshot pixels. Use tap(text=...) or tap(testID=...) to interact with discovered components. Use extended=true to include layout styles (padding, margin, flex, backgroundColor, etc.)." +
|
|
757
|
+
primaryInteractionBanner() + "\n" +
|
|
758
|
+
"PURPOSE: Quickest textual map of what is actually on screen right now — component names, positions, and text — so you can plan taps and inspections without guessing.\n" +
|
|
759
|
+
"WHEN TO USE: First step whenever the user asks \"what's on screen\", \"why is X covering Y\", or before tapping a visually ambiguous element.\n" +
|
|
760
|
+
"WORKFLOW: get_screen_layout -> find_components(pattern=\"...\") or inspect_component(componentName=\"...\") -> tap(testID=...) -> get_screen_layout again to confirm.\n" +
|
|
761
|
+
"LIMITATIONS: Coordinates are points/dp, not screenshot pixels — pass them to tap() which handles conversion, do not multiply by devicePixelRatio yourself.\n" +
|
|
762
|
+
"GOOD: get_screen_layout({ extended: true })\n" +
|
|
763
|
+
"BAD: get_screen_layout({ summary: true }) when you actually need to pick a specific element — summary hides the tree.\n" +
|
|
764
|
+
"SEE ALSO: call get_usage_guide(topic=\"layout\") for the full layout-check playbook.",
|
|
765
|
+
inputSchema: {
|
|
766
|
+
extended: z
|
|
767
|
+
.boolean()
|
|
768
|
+
.optional()
|
|
769
|
+
.default(false)
|
|
770
|
+
.describe("Include layout styles (padding, margin, flex, backgroundColor, borderRadius, etc.) for each component. Default: false for compact output."),
|
|
771
|
+
summary: z
|
|
772
|
+
.boolean()
|
|
773
|
+
.optional()
|
|
774
|
+
.default(false)
|
|
775
|
+
.describe("Return only component counts by name instead of full tree (default: false)"),
|
|
776
|
+
device: z.string().optional().describe("Target device name (substring match). Omit for default device. Run get_apps to see connected devices.")
|
|
777
|
+
}
|
|
778
|
+
}, async ({ extended, summary, device }) => {
|
|
779
|
+
if (!hasMetro()) {
|
|
780
|
+
const hint = await metroMissingHintIfAbsent("get_screen_layout");
|
|
781
|
+
return {
|
|
782
|
+
content: [{ type: "text", text: `Screen Layout unavailable.${hint}` }],
|
|
783
|
+
isError: true
|
|
784
|
+
};
|
|
785
|
+
}
|
|
786
|
+
const result = await getScreenLayout({ extended, summary, device });
|
|
787
|
+
if (!result.success) {
|
|
788
|
+
return {
|
|
789
|
+
content: [
|
|
790
|
+
{
|
|
791
|
+
type: "text",
|
|
792
|
+
text: `Error: ${result.error}`
|
|
793
|
+
}
|
|
794
|
+
],
|
|
795
|
+
isError: true
|
|
796
|
+
};
|
|
797
|
+
}
|
|
798
|
+
return {
|
|
799
|
+
content: [
|
|
800
|
+
{
|
|
801
|
+
type: "text",
|
|
802
|
+
text: `Screen Layout:\n\n${result.result}`
|
|
803
|
+
}
|
|
804
|
+
]
|
|
805
|
+
};
|
|
806
|
+
});
|
|
807
|
+
// Tool: Get console logs
|
|
808
|
+
registerToolWithTelemetry("get_logs", {
|
|
809
|
+
description: "Retrieve console logs from connected React Native app. Tip: Use summary=true first for a quick overview (counts by level + last 5 messages), then fetch specific logs as needed.\n" +
|
|
810
|
+
"PURPOSE: Pull captured console output (log/warn/error/info/debug) from the in-memory buffer for the connected app.\n" +
|
|
811
|
+
"WHEN TO USE: Start of any log-driven investigation, verifying a code change picked up via Fast Refresh, or confirming a reported error actually fires.\n" +
|
|
812
|
+
"WORKFLOW: scan_metro -> get_logs(summary=true) -> narrow with search_logs(text=\"...\") or get_logs(level=\"error\") -> clear_logs between reproductions.\n" +
|
|
813
|
+
"LIMITATIONS: Circular buffer (~500 entries). Only captures logs emitted after the app connected; pre-connect logs are lost.\n" +
|
|
814
|
+
"GOOD: get_logs({ summary: true }) then get_logs({ level: \"error\", maxLogs: 20 })\n" +
|
|
815
|
+
"BAD: get_logs({ maxLogs: 500, verbose: true }) as a first call — floods context; start with summary=true.\n" +
|
|
816
|
+
"SEE ALSO: call get_usage_guide(topic=\"logs\") for the full console-debugging playbook.",
|
|
817
|
+
inputSchema: {
|
|
818
|
+
maxLogs: z.coerce
|
|
819
|
+
.number()
|
|
820
|
+
.optional()
|
|
821
|
+
.default(50)
|
|
822
|
+
.describe("Maximum number of logs to return (default: 50)"),
|
|
823
|
+
level: z
|
|
824
|
+
.enum(["all", "log", "warn", "error", "info", "debug"])
|
|
825
|
+
.optional()
|
|
826
|
+
.default("all")
|
|
827
|
+
.describe("Filter by log level (default: all)"),
|
|
828
|
+
startFromText: z.string().optional().describe("Start from the first log line containing this text"),
|
|
829
|
+
maxMessageLength: z.coerce
|
|
830
|
+
.number()
|
|
831
|
+
.optional()
|
|
832
|
+
.default(500)
|
|
833
|
+
.describe("Max characters per message (default: 500, set to 0 for unlimited). Tip: Use lower values for overview, higher when debugging specific data structures."),
|
|
834
|
+
verbose: z
|
|
835
|
+
.boolean()
|
|
836
|
+
.optional()
|
|
837
|
+
.default(false)
|
|
838
|
+
.describe("Disable all truncation and return full messages. Tip: Use with lower maxLogs (e.g., 10) to avoid token overload when inspecting large objects."),
|
|
839
|
+
format: z
|
|
840
|
+
.enum(["text", "tonl"])
|
|
841
|
+
.optional()
|
|
842
|
+
.default("tonl")
|
|
843
|
+
.describe("Output format: 'text' or 'tonl' (default, compact token-optimized format, ~30-50% smaller)"),
|
|
844
|
+
summary: z
|
|
845
|
+
.boolean()
|
|
846
|
+
.optional()
|
|
847
|
+
.default(false)
|
|
848
|
+
.describe("Return summary statistics instead of full logs (count by level + last 5 messages). Use for quick overview."),
|
|
849
|
+
device: z.string().optional().describe("Target device name (substring match). Omit for all devices. Run get_apps to see connected devices.")
|
|
850
|
+
}
|
|
851
|
+
}, async ({ maxLogs, level, startFromText, maxMessageLength, verbose, format, summary, device }) => {
|
|
852
|
+
// Check if SDK is installed — prefer SDK data for richer logs
|
|
853
|
+
const sdkAvailable = await isSDKInstalled();
|
|
854
|
+
if (sdkAvailable) {
|
|
855
|
+
if (summary) {
|
|
808
856
|
const sdkStats = await getSDKConsoleStats();
|
|
809
857
|
if (sdkStats.success) {
|
|
810
858
|
const s = sdkStats.data;
|
|
@@ -987,7 +1035,14 @@ registerToolWithTelemetry("get_logs", {
|
|
|
987
1035
|
() => getTotalLogCount() === 0);
|
|
988
1036
|
// Tool: Search logs
|
|
989
1037
|
registerToolWithTelemetry("search_logs", {
|
|
990
|
-
description: "Search console logs for text (case-insensitive)"
|
|
1038
|
+
description: "Search console logs for text (case-insensitive).\n" +
|
|
1039
|
+
"PURPOSE: Find log lines matching a substring across the connected app's console buffer.\n" +
|
|
1040
|
+
"WHEN TO USE: User reports a known error/warning, or wants to trace a specific event (e.g., \"redux\", \"auth failed\"). For unfocused exploration, prefer get_logs.\n" +
|
|
1041
|
+
"WORKFLOW: scan_metro -> search_logs(text=\"...\") -> if empty, get_logs to verify buffer populated.\n" +
|
|
1042
|
+
"LIMITATIONS: Only matches text captured AFTER the app connected; won't find pre-connect logs.\n" +
|
|
1043
|
+
"GOOD: search_logs({ text: \"TypeError\" })\n" +
|
|
1044
|
+
"BAD: search_logs({ text: \"\" }) (use get_logs for a raw dump)\n" +
|
|
1045
|
+
"SEE ALSO: call get_usage_guide(topic=\"logs\") for the full console-debugging playbook.",
|
|
991
1046
|
inputSchema: {
|
|
992
1047
|
text: z.string().describe("Text to search for in log messages"),
|
|
993
1048
|
maxResults: z.coerce
|
|
@@ -1075,15 +1130,584 @@ registerToolWithTelemetry("search_logs", {
|
|
|
1075
1130
|
]
|
|
1076
1131
|
};
|
|
1077
1132
|
});
|
|
1078
|
-
// Tool:
|
|
1079
|
-
registerToolWithTelemetry("
|
|
1080
|
-
description: "
|
|
1133
|
+
// Tool: iOS screenshot
|
|
1134
|
+
registerToolWithTelemetry("ios_screenshot", {
|
|
1135
|
+
description: "Take a screenshot from an iOS simulator. Returns the image plus two lists: (1) pressable elements — the tappable components on screen with ready-to-use pixel tap coordinates, testIDs, and labels; (2) screen layout — all visible React components for context. Prefer tap(text=\"...\") when text is exact and unique; otherwise use tap(x, y) with coordinates from the pressables list — this is the most reliable way to tap icons or visually-identified elements. Use component names from the layout for inspect_component/find_components.\n" +
|
|
1136
|
+
"PURPOSE: Snapshot what the user sees on iOS AND get tap-ready pressables + a structured component map in one call.\n" +
|
|
1137
|
+
"WHEN TO USE: Any visual verification, before/after comparison, or as the starting point for tapping UI by coordinates.\n" +
|
|
1138
|
+
"WORKFLOW: ios_screenshot -> pick element from pressables -> tap(x, y) or tap(testID=...) -> ios_screenshot to verify.\n" +
|
|
1139
|
+
"LIMITATIONS: Requires a booted iOS simulator (simctl). For physical devices or system dialogs without RN, combine with tap(..., native=true).\n" +
|
|
1140
|
+
"GOOD: ios_screenshot()\n" +
|
|
1141
|
+
"BAD: ios_screenshot({ udid: \"guess\" }) with a made-up UDID — run list_ios_simulators first.\n" +
|
|
1142
|
+
"SEE ALSO: call get_usage_guide(topic=\"interact\") for the full device-interaction playbook.",
|
|
1081
1143
|
inputSchema: {
|
|
1082
|
-
|
|
1144
|
+
outputPath: z
|
|
1145
|
+
.string()
|
|
1146
|
+
.optional()
|
|
1147
|
+
.describe("Optional path to save the screenshot. If not provided, saves to temp directory."),
|
|
1148
|
+
udid: z
|
|
1149
|
+
.string()
|
|
1150
|
+
.optional()
|
|
1151
|
+
.describe("Optional simulator UDID (from list_ios_simulators). Uses booted simulator if not specified.")
|
|
1083
1152
|
}
|
|
1084
|
-
}, async ({
|
|
1085
|
-
|
|
1086
|
-
|
|
1153
|
+
}, async ({ outputPath, udid }) => {
|
|
1154
|
+
const result = await iosScreenshot(outputPath, udid);
|
|
1155
|
+
if (!result.success) {
|
|
1156
|
+
return {
|
|
1157
|
+
content: [
|
|
1158
|
+
{
|
|
1159
|
+
type: "text",
|
|
1160
|
+
text: `Error: ${result.error}`
|
|
1161
|
+
}
|
|
1162
|
+
],
|
|
1163
|
+
isError: true
|
|
1164
|
+
};
|
|
1165
|
+
}
|
|
1166
|
+
// Include image data if available
|
|
1167
|
+
if (result.data) {
|
|
1168
|
+
// Build info text with coordinate guidance for iOS
|
|
1169
|
+
const pixelWidth = result.originalWidth || 0;
|
|
1170
|
+
const pixelHeight = result.originalHeight || 0;
|
|
1171
|
+
// Resolve the RN app running on THIS simulator so enrichment pulls
|
|
1172
|
+
// fiber/layout data from the right app (not the first-connected one,
|
|
1173
|
+
// which may belong to a different simulator).
|
|
1174
|
+
const resolvedUdid = udid || (await getActiveOrBootedSimulatorUdid());
|
|
1175
|
+
const targetApp = resolvedUdid ? getConnectedAppBySimulatorUdid(resolvedUdid) : null;
|
|
1176
|
+
const targetDeviceName = targetApp?.deviceInfo.deviceName;
|
|
1177
|
+
// Store screenshot metadata on the matching app (not an arbitrary one)
|
|
1178
|
+
if (targetApp) {
|
|
1179
|
+
targetApp.lastScreenshot = {
|
|
1180
|
+
originalWidth: pixelWidth,
|
|
1181
|
+
originalHeight: pixelHeight,
|
|
1182
|
+
scaleFactor: result.scaleFactor || 1,
|
|
1183
|
+
};
|
|
1184
|
+
}
|
|
1185
|
+
// Try to get actual screen dimensions and safe area from accessibility tree
|
|
1186
|
+
let pointWidth = 0;
|
|
1187
|
+
let pointHeight = 0;
|
|
1188
|
+
let scaleFactor = 3; // Default to 3x for modern iPhones
|
|
1189
|
+
let safeAreaTop = 59; // Default safe area offset
|
|
1190
|
+
try {
|
|
1191
|
+
const describeResult = await iosDescribeAll(udid);
|
|
1192
|
+
if (describeResult.success && describeResult.elements && describeResult.elements.length > 0) {
|
|
1193
|
+
// First element is typically the Application with full screen frame
|
|
1194
|
+
const rootElement = describeResult.elements[0];
|
|
1195
|
+
// Try parsed frame first, then parse AXFrame string
|
|
1196
|
+
if (rootElement.frame) {
|
|
1197
|
+
pointWidth = Math.round(rootElement.frame.width);
|
|
1198
|
+
pointHeight = Math.round(rootElement.frame.height);
|
|
1199
|
+
// The frame.y of the root element indicates where content starts (after status bar)
|
|
1200
|
+
if (rootElement.frame.y > 0) {
|
|
1201
|
+
safeAreaTop = Math.round(rootElement.frame.y);
|
|
1202
|
+
}
|
|
1203
|
+
}
|
|
1204
|
+
else if (rootElement.AXFrame) {
|
|
1205
|
+
// Parse format: "{{x, y}, {width, height}}"
|
|
1206
|
+
const match = rootElement.AXFrame.match(/\{\{([\d.]+),\s*([\d.]+)\},\s*\{([\d.]+),\s*([\d.]+)\}\}/);
|
|
1207
|
+
if (match) {
|
|
1208
|
+
const frameY = parseFloat(match[2]);
|
|
1209
|
+
pointWidth = Math.round(parseFloat(match[3]));
|
|
1210
|
+
pointHeight = Math.round(parseFloat(match[4]));
|
|
1211
|
+
if (frameY > 0) {
|
|
1212
|
+
safeAreaTop = Math.round(frameY);
|
|
1213
|
+
}
|
|
1214
|
+
}
|
|
1215
|
+
}
|
|
1216
|
+
// Calculate actual scale factor
|
|
1217
|
+
if (pointWidth > 0) {
|
|
1218
|
+
scaleFactor = Math.round(pixelWidth / pointWidth);
|
|
1219
|
+
}
|
|
1220
|
+
}
|
|
1221
|
+
}
|
|
1222
|
+
catch {
|
|
1223
|
+
// Fallback: use 3x scale for modern devices
|
|
1224
|
+
}
|
|
1225
|
+
// Fallback if we couldn't get dimensions
|
|
1226
|
+
if (pointWidth === 0) {
|
|
1227
|
+
pointWidth = Math.round(pixelWidth / scaleFactor);
|
|
1228
|
+
pointHeight = Math.round(pixelHeight / scaleFactor);
|
|
1229
|
+
}
|
|
1230
|
+
const safeAreaOffsetPixels = safeAreaTop * scaleFactor;
|
|
1231
|
+
// The Screen Layout tree was previously appended here but produced huge noisy
|
|
1232
|
+
// output (nested Svg/G/Path duplicates). Agents should use get_screen_layout
|
|
1233
|
+
// explicitly when they need the tree. The Pressable elements block below is
|
|
1234
|
+
// the signal most consumers actually want.
|
|
1235
|
+
let pressablesText = null;
|
|
1236
|
+
const canEnrich = !!targetApp;
|
|
1237
|
+
// Enrich with pressable elements (filtered list of tappable components, converted to screenshot pixel coords)
|
|
1238
|
+
try {
|
|
1239
|
+
if (!canEnrich)
|
|
1240
|
+
throw new Error("skip");
|
|
1241
|
+
const pressables = await getPressableElements({ device: targetDeviceName });
|
|
1242
|
+
if (pressables.success && pressables.parsedElements && pressables.parsedElements.length > 0) {
|
|
1243
|
+
const screenshotScale = result.scaleFactor || 1;
|
|
1244
|
+
pressablesText = pressables.parsedElements.map((el) => {
|
|
1245
|
+
// See enrichScreenshotWithLayout: shift y when fiber reports the element
|
|
1246
|
+
// inside the safe-area band (a react-native-screens modal artifact).
|
|
1247
|
+
let centerYPoints = el.center.y;
|
|
1248
|
+
if (safeAreaTop > 0 && centerYPoints < safeAreaTop) {
|
|
1249
|
+
centerYPoints += safeAreaTop;
|
|
1250
|
+
}
|
|
1251
|
+
const px = Math.round((el.center.x * scaleFactor) / screenshotScale);
|
|
1252
|
+
const py = Math.round((centerYPoints * scaleFactor) / screenshotScale);
|
|
1253
|
+
const label = el.accessibilityLabel || el.text || el.testID || (el.intent ? `${el.intent} icon` : el.component);
|
|
1254
|
+
const idPart = el.testID ? ` testID="${el.testID}"` : "";
|
|
1255
|
+
const kindPart = el.isInput ? " [input]" : "";
|
|
1256
|
+
const wrapPart = el.isWrapper ? " [wrapper — skip]" : "";
|
|
1257
|
+
const nearPart = !el.text && !el.accessibilityLabel && el.nearbyText ? ` near "${el.nearbyText}"` : "";
|
|
1258
|
+
return ` (${px}, ${py}) ${el.component}: "${label}"${nearPart}${idPart}${kindPart}${wrapPart}`;
|
|
1259
|
+
}).join("\n");
|
|
1260
|
+
}
|
|
1261
|
+
}
|
|
1262
|
+
catch {
|
|
1263
|
+
// Non-fatal: screenshot works without pressables enrichment
|
|
1264
|
+
}
|
|
1265
|
+
let infoText = `Screenshot captured (${pixelWidth}x${pixelHeight} pixels)`;
|
|
1266
|
+
infoText += `\n📱 iOS screen: ${pointWidth}x${pointHeight} points (${scaleFactor}x scale)`;
|
|
1267
|
+
infoText += `\n📐 tap() handles pixel-to-point conversion automatically — pass pixel coords from this image directly`;
|
|
1268
|
+
infoText += `\n⚠️ Status bar + safe area: ${safeAreaTop} points (${safeAreaOffsetPixels} pixels) from top`;
|
|
1269
|
+
if (result.scaleFactor && result.scaleFactor > 1) {
|
|
1270
|
+
infoText += `\n🖼️ Image was scaled down to fit API limits (scale: ${result.scaleFactor.toFixed(3)})`;
|
|
1271
|
+
}
|
|
1272
|
+
if (pressablesText) {
|
|
1273
|
+
infoText += `\n\n🎯 Pressable elements (ready-to-tap, coordinates in screenshot pixels):`;
|
|
1274
|
+
infoText += `\n${pressablesText}`;
|
|
1275
|
+
infoText += `\n\n💡 Next steps:`;
|
|
1276
|
+
infoText += `\n • tap(text="Button Label") — when text is exact and unique`;
|
|
1277
|
+
infoText += `\n • tap(testID="id") or tap(component="Name") — when you know the identifier`;
|
|
1278
|
+
infoText += `\n • tap(x=<px>, y=<px>) — use coordinates from the pressable elements list above (reliable for icons and ambiguous elements)`;
|
|
1279
|
+
infoText += `\n • get_screen_layout — full component tree when you need more than pressables`;
|
|
1280
|
+
}
|
|
1281
|
+
else {
|
|
1282
|
+
if (!canEnrich && connectedApps.size > 0) {
|
|
1283
|
+
infoText += `\n\nℹ️ Pressable enrichment skipped: no RN app is connected to simulator ${resolvedUdid}.`;
|
|
1284
|
+
infoText += ` ${connectedApps.size} other app(s) are connected on different device(s) — their fiber data was intentionally not used to avoid mismatched output.`;
|
|
1285
|
+
}
|
|
1286
|
+
infoText += `\n\n💡 Next steps:`;
|
|
1287
|
+
infoText += `\n • tap(text="Button Label") — tap element by visible text`;
|
|
1288
|
+
infoText += `\n • tap(x=<px>, y=<px>) — tap at coordinates from this screenshot`;
|
|
1289
|
+
infoText += `\n • get_screen_layout — get full UI tree with real on-screen positions`;
|
|
1290
|
+
}
|
|
1291
|
+
// Check for LogBox overlay — only on the matching RN app; skip otherwise
|
|
1292
|
+
// to avoid surfacing warnings from a different simulator's app.
|
|
1293
|
+
if (canEnrich) {
|
|
1294
|
+
try {
|
|
1295
|
+
const logBoxState = await detectLogBox(targetDeviceName);
|
|
1296
|
+
if (logBoxState && logBoxState.total > 0) {
|
|
1297
|
+
infoText += formatLogBoxWarning(logBoxState);
|
|
1298
|
+
}
|
|
1299
|
+
}
|
|
1300
|
+
catch {
|
|
1301
|
+
// Non-fatal: LogBox detection failure should not break screenshot
|
|
1302
|
+
}
|
|
1303
|
+
}
|
|
1304
|
+
imageBuffer.add({
|
|
1305
|
+
id: `ios-${Date.now()}-${Math.random().toString(36).slice(2, 8)}`,
|
|
1306
|
+
image: result.data,
|
|
1307
|
+
timestamp: Date.now(),
|
|
1308
|
+
source: "ios_screenshot",
|
|
1309
|
+
metadata: {
|
|
1310
|
+
width: result.originalWidth || 0,
|
|
1311
|
+
height: result.originalHeight || 0,
|
|
1312
|
+
scaleFactor: result.scaleFactor || 1,
|
|
1313
|
+
platform: "ios",
|
|
1314
|
+
},
|
|
1315
|
+
});
|
|
1316
|
+
return {
|
|
1317
|
+
content: [
|
|
1318
|
+
{
|
|
1319
|
+
type: "text",
|
|
1320
|
+
text: infoText
|
|
1321
|
+
},
|
|
1322
|
+
{
|
|
1323
|
+
type: "image",
|
|
1324
|
+
data: result.data.toString("base64"),
|
|
1325
|
+
mimeType: "image/jpeg"
|
|
1326
|
+
}
|
|
1327
|
+
]
|
|
1328
|
+
};
|
|
1329
|
+
}
|
|
1330
|
+
return {
|
|
1331
|
+
content: [
|
|
1332
|
+
{
|
|
1333
|
+
type: "text",
|
|
1334
|
+
text: `Screenshot saved to: ${result.result}`
|
|
1335
|
+
}
|
|
1336
|
+
]
|
|
1337
|
+
};
|
|
1338
|
+
});
|
|
1339
|
+
// Tool: Android screenshot
|
|
1340
|
+
registerToolWithTelemetry("android_screenshot", {
|
|
1341
|
+
description: "Take a screenshot from an Android device/emulator. Returns the image plus two lists: (1) pressable elements — the tappable components on screen with ready-to-use pixel tap coordinates, testIDs, and labels; (2) screen layout — all visible React components for context. Prefer tap(text=\"...\") when text is exact and unique; otherwise use tap(x, y) with coordinates from the pressables list — this is the most reliable way to tap icons or visually-identified elements. Use component names from the layout for inspect_component/find_components.\n" +
|
|
1342
|
+
"PURPOSE: Snapshot what the user sees on Android AND get tap-ready pressables + a structured component map in one call.\n" +
|
|
1343
|
+
"WHEN TO USE: Any visual verification, before/after comparison, or as the starting point for tapping UI by coordinates on Android.\n" +
|
|
1344
|
+
"WORKFLOW: android_screenshot -> pick element from pressables -> tap(x, y) or tap(testID=...) -> android_screenshot to verify.\n" +
|
|
1345
|
+
"LIMITATIONS: Requires adb in PATH and a running device/emulator. For non-RN surfaces (system dialogs, permission prompts), combine with tap(..., native=true).\n" +
|
|
1346
|
+
"GOOD: android_screenshot()\n" +
|
|
1347
|
+
"BAD: android_screenshot({ deviceId: \"guess\" }) with a made-up serial — run list_android_devices first.\n" +
|
|
1348
|
+
"SEE ALSO: call get_usage_guide(topic=\"interact\") for the full device-interaction playbook.",
|
|
1349
|
+
inputSchema: {
|
|
1350
|
+
outputPath: z
|
|
1351
|
+
.string()
|
|
1352
|
+
.optional()
|
|
1353
|
+
.describe("Optional path to save the screenshot. If not provided, saves to temp directory."),
|
|
1354
|
+
deviceId: z
|
|
1355
|
+
.string()
|
|
1356
|
+
.optional()
|
|
1357
|
+
.describe("Optional device ID (from list_android_devices). Uses first available device if not specified.")
|
|
1358
|
+
}
|
|
1359
|
+
}, async ({ outputPath, deviceId }) => {
|
|
1360
|
+
const result = await androidScreenshot(outputPath, deviceId);
|
|
1361
|
+
if (!result.success) {
|
|
1362
|
+
return {
|
|
1363
|
+
content: [
|
|
1364
|
+
{
|
|
1365
|
+
type: "text",
|
|
1366
|
+
text: `Error: ${result.error}`
|
|
1367
|
+
}
|
|
1368
|
+
],
|
|
1369
|
+
isError: true
|
|
1370
|
+
};
|
|
1371
|
+
}
|
|
1372
|
+
// Include image data if available
|
|
1373
|
+
if (result.data) {
|
|
1374
|
+
// Build info text with coordinate conversion guidance
|
|
1375
|
+
const pixelWidth = result.originalWidth || 0;
|
|
1376
|
+
const pixelHeight = result.originalHeight || 0;
|
|
1377
|
+
// Resolve the RN app running on THIS Android device so enrichment
|
|
1378
|
+
// pulls data from the right app (not whichever app is "first").
|
|
1379
|
+
const targetApp = getConnectedAppByAndroidDeviceId(deviceId);
|
|
1380
|
+
const targetDeviceName = targetApp?.deviceInfo.deviceName;
|
|
1381
|
+
// Store screenshot metadata on the matching app (not an arbitrary one)
|
|
1382
|
+
if (targetApp) {
|
|
1383
|
+
targetApp.lastScreenshot = {
|
|
1384
|
+
originalWidth: pixelWidth,
|
|
1385
|
+
originalHeight: pixelHeight,
|
|
1386
|
+
scaleFactor: result.scaleFactor || 1,
|
|
1387
|
+
};
|
|
1388
|
+
}
|
|
1389
|
+
let infoText = `Screenshot captured (${pixelWidth}x${pixelHeight} pixels)`;
|
|
1390
|
+
// Get status bar height for coordinate guidance
|
|
1391
|
+
let statusBarPixels = 63; // Default fallback
|
|
1392
|
+
let statusBarDp = 24;
|
|
1393
|
+
let densityDpi = 440; // Common default
|
|
1394
|
+
try {
|
|
1395
|
+
const statusBarResult = await androidGetStatusBarHeight(deviceId);
|
|
1396
|
+
if (statusBarResult.success && statusBarResult.heightPixels) {
|
|
1397
|
+
statusBarPixels = statusBarResult.heightPixels;
|
|
1398
|
+
statusBarDp = statusBarResult.heightDp || 24;
|
|
1399
|
+
}
|
|
1400
|
+
const densityResult = await androidGetDensity(deviceId);
|
|
1401
|
+
if (densityResult.success && densityResult.density) {
|
|
1402
|
+
densityDpi = densityResult.density;
|
|
1403
|
+
}
|
|
1404
|
+
}
|
|
1405
|
+
catch {
|
|
1406
|
+
// Use defaults
|
|
1407
|
+
}
|
|
1408
|
+
// Enrich with screen layout data (component names + tap coordinates)
|
|
1409
|
+
// Android: density ratio = densityDpi / 160 (160dpi = 1dp = 1px baseline)
|
|
1410
|
+
const androidPixelRatio = densityDpi / 160;
|
|
1411
|
+
// Skip enrichment when no RN app is connected to this Android device —
|
|
1412
|
+
// otherwise we would pull fiber data from a different device's app.
|
|
1413
|
+
const canEnrich = !!targetApp;
|
|
1414
|
+
let pressablesText = null;
|
|
1415
|
+
// Screen Layout tree previously appended here was dropped — it was noisy
|
|
1416
|
+
// (nested Svg/G/Path duplicates). Use get_screen_layout when the tree is needed.
|
|
1417
|
+
// Enrich with pressable elements (filtered list of tappable components, converted to screenshot pixel coords)
|
|
1418
|
+
try {
|
|
1419
|
+
if (!canEnrich)
|
|
1420
|
+
throw new Error("skip");
|
|
1421
|
+
const pressables = await getPressableElements({ device: targetDeviceName });
|
|
1422
|
+
if (pressables.success && pressables.parsedElements && pressables.parsedElements.length > 0) {
|
|
1423
|
+
const screenshotScale = result.scaleFactor || 1;
|
|
1424
|
+
pressablesText = pressables.parsedElements.map((el) => {
|
|
1425
|
+
const px = Math.round((el.center.x * androidPixelRatio) / screenshotScale);
|
|
1426
|
+
const py = Math.round((el.center.y * androidPixelRatio) / screenshotScale);
|
|
1427
|
+
const label = el.accessibilityLabel || el.text || el.testID || (el.intent ? `${el.intent} icon` : el.component);
|
|
1428
|
+
const idPart = el.testID ? ` testID="${el.testID}"` : "";
|
|
1429
|
+
const kindPart = el.isInput ? " [input]" : "";
|
|
1430
|
+
const wrapPart = el.isWrapper ? " [wrapper — skip]" : "";
|
|
1431
|
+
const nearPart = !el.text && !el.accessibilityLabel && el.nearbyText ? ` near "${el.nearbyText}"` : "";
|
|
1432
|
+
return ` (${px}, ${py}) ${el.component}: "${label}"${nearPart}${idPart}${kindPart}${wrapPart}`;
|
|
1433
|
+
}).join("\n");
|
|
1434
|
+
}
|
|
1435
|
+
}
|
|
1436
|
+
catch {
|
|
1437
|
+
// Non-fatal: screenshot works without pressables enrichment
|
|
1438
|
+
}
|
|
1439
|
+
infoText += `\n📱 Android uses PIXELS for all coordinates`;
|
|
1440
|
+
if (result.scaleFactor && result.scaleFactor > 1) {
|
|
1441
|
+
infoText += `\n🖼️ Image was scaled down to fit API limits (scale: ${result.scaleFactor.toFixed(3)})`;
|
|
1442
|
+
infoText += `\n📐 tap() handles coordinate conversion automatically — pass pixel coords from this image directly`;
|
|
1443
|
+
}
|
|
1444
|
+
else {
|
|
1445
|
+
infoText += `\n📐 Screenshot coords = tap coords (no conversion needed)`;
|
|
1446
|
+
}
|
|
1447
|
+
infoText += `\n⚠️ Status bar: ${statusBarPixels}px (${statusBarDp}dp) from top - app content starts below this`;
|
|
1448
|
+
infoText += `\n📊 Display density: ${densityDpi}dpi`;
|
|
1449
|
+
if (pressablesText) {
|
|
1450
|
+
infoText += `\n\n🎯 Pressable elements (ready-to-tap, coordinates in screenshot pixels):`;
|
|
1451
|
+
infoText += `\n${pressablesText}`;
|
|
1452
|
+
infoText += `\n\n💡 Next steps:`;
|
|
1453
|
+
infoText += `\n • tap(text="Button Label") — when text is exact and unique`;
|
|
1454
|
+
infoText += `\n • tap(testID="id") or tap(component="Name") — when you know the identifier`;
|
|
1455
|
+
infoText += `\n • tap(x=<px>, y=<px>) — use coordinates from the pressable elements list above (reliable for icons and ambiguous elements)`;
|
|
1456
|
+
infoText += `\n • get_screen_layout — full component tree when you need more than pressables`;
|
|
1457
|
+
}
|
|
1458
|
+
else {
|
|
1459
|
+
if (!canEnrich && connectedApps.size > 0) {
|
|
1460
|
+
infoText += `\n\nℹ️ Pressable enrichment skipped: no RN app is connected to device ${deviceId ?? "(default)"}.`;
|
|
1461
|
+
infoText += ` ${connectedApps.size} other app(s) are connected on different device(s) — their fiber data was intentionally not used to avoid mismatched output.`;
|
|
1462
|
+
}
|
|
1463
|
+
infoText += `\n\n💡 Next steps:`;
|
|
1464
|
+
infoText += `\n • tap(text="Button Label") — tap element by visible text`;
|
|
1465
|
+
infoText += `\n • tap(x=<px>, y=<px>) — tap at coordinates from this screenshot`;
|
|
1466
|
+
infoText += `\n • get_screen_layout — get full UI tree with real on-screen positions`;
|
|
1467
|
+
}
|
|
1468
|
+
// Check for LogBox overlay — only on the matching RN app; skip otherwise
|
|
1469
|
+
// to avoid surfacing warnings from a different device's app.
|
|
1470
|
+
if (canEnrich) {
|
|
1471
|
+
try {
|
|
1472
|
+
const logBoxState = await detectLogBox(targetDeviceName);
|
|
1473
|
+
if (logBoxState && logBoxState.total > 0) {
|
|
1474
|
+
infoText += formatLogBoxWarning(logBoxState);
|
|
1475
|
+
}
|
|
1476
|
+
}
|
|
1477
|
+
catch {
|
|
1478
|
+
// Non-fatal: LogBox detection failure should not break screenshot
|
|
1479
|
+
}
|
|
1480
|
+
}
|
|
1481
|
+
imageBuffer.add({
|
|
1482
|
+
id: `android-${Date.now()}-${Math.random().toString(36).slice(2, 8)}`,
|
|
1483
|
+
image: result.data,
|
|
1484
|
+
timestamp: Date.now(),
|
|
1485
|
+
source: "android_screenshot",
|
|
1486
|
+
metadata: {
|
|
1487
|
+
width: result.originalWidth || 0,
|
|
1488
|
+
height: result.originalHeight || 0,
|
|
1489
|
+
scaleFactor: result.scaleFactor || 1,
|
|
1490
|
+
platform: "android",
|
|
1491
|
+
},
|
|
1492
|
+
});
|
|
1493
|
+
return {
|
|
1494
|
+
content: [
|
|
1495
|
+
{
|
|
1496
|
+
type: "text",
|
|
1497
|
+
text: infoText
|
|
1498
|
+
},
|
|
1499
|
+
{
|
|
1500
|
+
type: "image",
|
|
1501
|
+
data: result.data.toString("base64"),
|
|
1502
|
+
mimeType: "image/jpeg"
|
|
1503
|
+
}
|
|
1504
|
+
]
|
|
1505
|
+
};
|
|
1506
|
+
}
|
|
1507
|
+
return {
|
|
1508
|
+
content: [
|
|
1509
|
+
{
|
|
1510
|
+
type: "text",
|
|
1511
|
+
text: `Screenshot saved to: ${result.result}`
|
|
1512
|
+
}
|
|
1513
|
+
]
|
|
1514
|
+
};
|
|
1515
|
+
});
|
|
1516
|
+
// Tool: Get connection status (detailed health and gap tracking)
|
|
1517
|
+
registerToolWithTelemetry("get_connection_status", {
|
|
1518
|
+
description: "Get detailed connection health status including uptime, recent disconnects/reconnects, and connection gaps that may indicate missing data.\n" +
|
|
1519
|
+
"PURPOSE: Diagnose flaky CDP sessions — quantify uptime, count reconnects, and expose gaps where logs/network data could be missing.\n" +
|
|
1520
|
+
"WHEN TO USE: When logs look suspiciously empty, tools complain about disconnect/reconnect, or the app was suspended and resumed.\n" +
|
|
1521
|
+
"WORKFLOW: get_connection_status -> if unhealthy: disconnect_metro -> scan_metro to rebuild a clean session.\n" +
|
|
1522
|
+
"LIMITATIONS: Reports only MCP-side view; doesn't know why Metro dropped the socket (simulator sleep, app backgrounded, etc).\n" +
|
|
1523
|
+
"GOOD: get_connection_status() after noticing stale data.\n" +
|
|
1524
|
+
"BAD: Polling get_connection_status as a heartbeat — use ensure_connection(healthCheck=true) for a live probe.\n" +
|
|
1525
|
+
"SEE ALSO: call get_usage_guide(topic=\"setup\") for the full session-setup playbook.",
|
|
1526
|
+
inputSchema: {}
|
|
1527
|
+
}, async () => {
|
|
1528
|
+
const connections = getConnectedApps();
|
|
1529
|
+
const states = getAllConnectionStates();
|
|
1530
|
+
const metadata = getAllConnectionMetadata();
|
|
1531
|
+
const lines = [];
|
|
1532
|
+
lines.push("=== Connection Status ===\n");
|
|
1533
|
+
if (connections.length === 0 && states.size === 0) {
|
|
1534
|
+
lines.push("No connections established. Run scan_metro to connect.");
|
|
1535
|
+
return {
|
|
1536
|
+
content: [{ type: "text", text: lines.join("\n") }]
|
|
1537
|
+
};
|
|
1538
|
+
}
|
|
1539
|
+
// Show active connections
|
|
1540
|
+
for (const { key, app, isConnected } of connections) {
|
|
1541
|
+
const state = states.get(key);
|
|
1542
|
+
const contextHealth = getContextHealth(key);
|
|
1543
|
+
lines.push(`--- ${app.deviceInfo.title} (Port ${app.port}) ---`);
|
|
1544
|
+
lines.push(` Status: ${isConnected ? "CONNECTED" : "DISCONNECTED"}`);
|
|
1545
|
+
if (state) {
|
|
1546
|
+
if (state.lastConnectedTime) {
|
|
1547
|
+
const uptime = Date.now() - state.lastConnectedTime.getTime();
|
|
1548
|
+
lines.push(` Connected since: ${state.lastConnectedTime.toLocaleTimeString()}`);
|
|
1549
|
+
lines.push(` Uptime: ${formatDuration(uptime)}`);
|
|
1550
|
+
}
|
|
1551
|
+
if (state.status === "reconnecting") {
|
|
1552
|
+
lines.push(` Reconnecting: Attempt ${state.reconnectionAttempts}`);
|
|
1553
|
+
}
|
|
1554
|
+
// Show recent gaps (last 5 minutes)
|
|
1555
|
+
if (state.connectionGaps.length > 0) {
|
|
1556
|
+
const recentGaps = state.connectionGaps.filter((g) => Date.now() - g.disconnectedAt.getTime() < 300000);
|
|
1557
|
+
if (recentGaps.length > 0) {
|
|
1558
|
+
lines.push(` Recent gaps: ${recentGaps.length}`);
|
|
1559
|
+
for (const gap of recentGaps.slice(-3)) {
|
|
1560
|
+
const duration = gap.durationMs ? formatDuration(gap.durationMs) : "ongoing";
|
|
1561
|
+
lines.push(` - ${gap.disconnectedAt.toLocaleTimeString()} (${duration}): ${gap.reason}`);
|
|
1562
|
+
}
|
|
1563
|
+
}
|
|
1564
|
+
}
|
|
1565
|
+
}
|
|
1566
|
+
// Show context health
|
|
1567
|
+
if (contextHealth) {
|
|
1568
|
+
lines.push(` Context Health:`);
|
|
1569
|
+
lines.push(` Context ID: ${contextHealth.contextId ?? "unknown"}`);
|
|
1570
|
+
lines.push(` Status: ${contextHealth.isStale ? "STALE" : "HEALTHY"}`);
|
|
1571
|
+
if (contextHealth.lastHealthCheck) {
|
|
1572
|
+
const healthResult = contextHealth.lastHealthCheckSuccess ? "PASS" : "FAIL";
|
|
1573
|
+
lines.push(` Last Check: ${contextHealth.lastHealthCheck.toLocaleTimeString()} (${healthResult})`);
|
|
1574
|
+
}
|
|
1575
|
+
if (contextHealth.lastContextCreated) {
|
|
1576
|
+
lines.push(` Context Created: ${contextHealth.lastContextCreated.toLocaleTimeString()}`);
|
|
1577
|
+
}
|
|
1578
|
+
if (contextHealth.lastContextDestroyed) {
|
|
1579
|
+
lines.push(` Context Destroyed: ${contextHealth.lastContextDestroyed.toLocaleTimeString()}`);
|
|
1580
|
+
}
|
|
1581
|
+
}
|
|
1582
|
+
lines.push("");
|
|
1583
|
+
}
|
|
1584
|
+
// Show disconnected/reconnecting states without active connections
|
|
1585
|
+
for (const [key, state] of states.entries()) {
|
|
1586
|
+
if (!connections.find((c) => c.key === key)) {
|
|
1587
|
+
const meta = metadata.get(key);
|
|
1588
|
+
lines.push(`--- ${meta?.deviceInfo.title || key} (Disconnected) ---`);
|
|
1589
|
+
lines.push(` Status: ${state.status.toUpperCase()}`);
|
|
1590
|
+
if (state.lastDisconnectTime) {
|
|
1591
|
+
lines.push(` Disconnected at: ${state.lastDisconnectTime.toLocaleTimeString()}`);
|
|
1592
|
+
}
|
|
1593
|
+
if (state.reconnectionAttempts > 0) {
|
|
1594
|
+
lines.push(` Reconnection attempts: ${state.reconnectionAttempts}`);
|
|
1595
|
+
}
|
|
1596
|
+
lines.push("");
|
|
1597
|
+
}
|
|
1598
|
+
}
|
|
1599
|
+
return {
|
|
1600
|
+
content: [{ type: "text", text: lines.join("\n") }]
|
|
1601
|
+
};
|
|
1602
|
+
});
|
|
1603
|
+
// Tool: License status
|
|
1604
|
+
registerToolWithTelemetry("get_license_status", {
|
|
1605
|
+
description: "Get your installation ID and license status. Shows your unique Installation ID (needed to activate Pro in the dashboard), current license tier, and cache validity.",
|
|
1606
|
+
inputSchema: {},
|
|
1607
|
+
}, async () => {
|
|
1608
|
+
const status = getLicenseStatus();
|
|
1609
|
+
const lines = [];
|
|
1610
|
+
lines.push(`Installation ID: ${status.installationId}`);
|
|
1611
|
+
lines.push(`License: ${status.tier.charAt(0).toUpperCase() + status.tier.slice(1)}`);
|
|
1612
|
+
if (status.plan) {
|
|
1613
|
+
lines.push(`Plan expires: ${status.plan.expiresAt}`);
|
|
1614
|
+
}
|
|
1615
|
+
lines.push(`Cache valid until: ${status.cacheExpiresAt}`);
|
|
1616
|
+
// Usage limits
|
|
1617
|
+
const usage = getUsageInfo();
|
|
1618
|
+
if (usage) {
|
|
1619
|
+
lines.push("");
|
|
1620
|
+
lines.push("--- Usage ---");
|
|
1621
|
+
if (usage.creditsRemaining !== null) {
|
|
1622
|
+
lines.push(`Credits remaining: ${usage.creditsRemaining.toLocaleString()}`);
|
|
1623
|
+
lines.push(`Used this month: ${usage.used.toLocaleString()}`);
|
|
1624
|
+
}
|
|
1625
|
+
else {
|
|
1626
|
+
lines.push(`Monthly usage: ${usage.used} / ${usage.limit}`);
|
|
1627
|
+
}
|
|
1628
|
+
lines.push(`Month: ${usage.monthKey}`);
|
|
1629
|
+
if (usage.promotionalPeriod) {
|
|
1630
|
+
const endsAt = usage.promotionalPeriodEndsAt
|
|
1631
|
+
? new Date(usage.promotionalPeriodEndsAt).toLocaleDateString("en-GB", { day: "2-digit", month: "2-digit", year: "numeric" })
|
|
1632
|
+
: "unknown";
|
|
1633
|
+
lines.push(`Status: Active (promotional period — no limits until ${endsAt})`);
|
|
1634
|
+
}
|
|
1635
|
+
else {
|
|
1636
|
+
lines.push(`Status: ${usage.canUse ? "Active" : "Limit reached"}`);
|
|
1637
|
+
}
|
|
1638
|
+
}
|
|
1639
|
+
if (status.tier === "free") {
|
|
1640
|
+
const dashboardUrl = getDashboardUrl();
|
|
1641
|
+
lines.push("");
|
|
1642
|
+
if (dashboardUrl) {
|
|
1643
|
+
lines.push(`Link your account: ${dashboardUrl}/link?id=${status.installationId}`);
|
|
1644
|
+
}
|
|
1645
|
+
}
|
|
1646
|
+
return {
|
|
1647
|
+
content: [{ type: "text", text: lines.join("\n") }],
|
|
1648
|
+
};
|
|
1649
|
+
});
|
|
1650
|
+
// Tool: Send feedback / bug report / feature request
|
|
1651
|
+
registerToolWithTelemetry("send_feedback", {
|
|
1652
|
+
description: "Create a feedback report (bug, feature request, or general feedback) for the react-native-ai-devtools team. " +
|
|
1653
|
+
"Auto-collects environment info. Returns a pre-filled GitHub issue URL and formatted issue body. " +
|
|
1654
|
+
"Ask the user to open the URL and paste the body to submit.",
|
|
1655
|
+
inputSchema: {
|
|
1656
|
+
type: z.enum(["feedback", "feature_request", "bug"]).describe('Type of feedback: "feedback", "feature_request", or "bug"'),
|
|
1657
|
+
title: z.string().describe("Short summary (becomes the GitHub issue title)"),
|
|
1658
|
+
description: z.string().describe("Detailed description of the feedback, feature request, or bug"),
|
|
1659
|
+
workflow_context: z.string().optional().describe("What the user was trying to accomplish when they decided to give feedback")
|
|
1660
|
+
}
|
|
1661
|
+
}, async ({ type, title, description, workflow_context }) => {
|
|
1662
|
+
// Collect environment info
|
|
1663
|
+
const serverVersion = getServerVersion();
|
|
1664
|
+
const platform = process.platform;
|
|
1665
|
+
const deviceType = getTargetPlatform();
|
|
1666
|
+
const licenseStatus = getLicenseStatus();
|
|
1667
|
+
const env = {
|
|
1668
|
+
serverVersion,
|
|
1669
|
+
platform,
|
|
1670
|
+
deviceType,
|
|
1671
|
+
licenseTier: licenseStatus.tier
|
|
1672
|
+
};
|
|
1673
|
+
const input = { type, title, description, workflowContext: workflow_context };
|
|
1674
|
+
const issueBody = formatIssueBody(input, env);
|
|
1675
|
+
const githubUrl = buildGitHubUrl(title, type);
|
|
1676
|
+
const output = [
|
|
1677
|
+
"## Feedback Report Ready",
|
|
1678
|
+
"",
|
|
1679
|
+
`**GitHub Issue URL:** ${githubUrl}`,
|
|
1680
|
+
"",
|
|
1681
|
+
"**Issue body to paste:**",
|
|
1682
|
+
"",
|
|
1683
|
+
"```markdown",
|
|
1684
|
+
issueBody,
|
|
1685
|
+
"```",
|
|
1686
|
+
"",
|
|
1687
|
+
"Please ask the user to:",
|
|
1688
|
+
"1. Open the GitHub URL above",
|
|
1689
|
+
"2. Paste the issue body into the description field",
|
|
1690
|
+
"3. Review and submit the issue"
|
|
1691
|
+
].join("\n");
|
|
1692
|
+
return {
|
|
1693
|
+
content: [{ type: "text", text: output }]
|
|
1694
|
+
};
|
|
1695
|
+
});
|
|
1696
|
+
// Tool: Clear logs
|
|
1697
|
+
registerToolWithTelemetry("clear_logs", {
|
|
1698
|
+
description: "Clear the log buffer.\n" +
|
|
1699
|
+
"PURPOSE: Empty the in-memory console buffer (and the SDK buffer if installed) so the next get_logs / search_logs only sees fresh entries.\n" +
|
|
1700
|
+
"WHEN TO USE: Before reproducing a bug so the resulting logs are isolated; between test iterations to avoid noise from earlier runs.\n" +
|
|
1701
|
+
"WORKFLOW: clear_logs -> reproduce the issue (tap / navigate / reload_app) -> get_logs or search_logs.\n" +
|
|
1702
|
+
"GOOD: clear_logs() right before tap(text=\"Submit\")\n" +
|
|
1703
|
+
"BAD: clear_logs() AFTER the repro — you just deleted the evidence.\n" +
|
|
1704
|
+
"SEE ALSO: call get_usage_guide(topic=\"logs\") for the full console-debugging playbook.",
|
|
1705
|
+
inputSchema: {
|
|
1706
|
+
device: z.string().optional().describe("Target device name (substring match). Omit to clear all devices. Run get_apps to see connected devices.")
|
|
1707
|
+
}
|
|
1708
|
+
}, async ({ device }) => {
|
|
1709
|
+
if (device) {
|
|
1710
|
+
const app = getConnectedAppByDevice(device);
|
|
1087
1711
|
if (!app)
|
|
1088
1712
|
throw new Error(`No connected device matches "${device}"`);
|
|
1089
1713
|
const deviceName = app.deviceInfo.deviceName || app.deviceInfo.title || "unknown";
|
|
@@ -1107,7 +1731,11 @@ registerToolWithTelemetry("clear_logs", {
|
|
|
1107
1731
|
});
|
|
1108
1732
|
// Tool: Connect to specific Metro port
|
|
1109
1733
|
registerToolWithTelemetry("connect_metro", {
|
|
1110
|
-
description: "Connect to a Metro server on a specific port
|
|
1734
|
+
description: "Connect to a Metro server on a specific port.\n" +
|
|
1735
|
+
"[DEPRECATED IN PRACTICE — prefer `scan_metro` which auto-discovers all common ports and connects every Bridgeless target at once. Use `connect_metro` only when you need to target a specific non-default port]\n" +
|
|
1736
|
+
"PURPOSE: Establish a CDP WebSocket connection to a single Metro server on a known, non-default port.\n" +
|
|
1737
|
+
"WHEN TO USE: Only when the Metro bundler is running on a port outside the common 8081/8082/19000-19002 range that `scan_metro` already covers — otherwise always prefer `scan_metro`.\n" +
|
|
1738
|
+
"SEE ALSO: call scan_metro for auto-discovery; call get_apps afterwards to confirm the device attached.",
|
|
1111
1739
|
inputSchema: {
|
|
1112
1740
|
port: z.coerce.number().default(8081).describe("Metro server port (default: 8081)")
|
|
1113
1741
|
}
|
|
@@ -1164,7 +1792,14 @@ registerToolWithTelemetry("connect_metro", {
|
|
|
1164
1792
|
});
|
|
1165
1793
|
// Tool: Disconnect from Metro
|
|
1166
1794
|
registerToolWithTelemetry("disconnect_metro", {
|
|
1167
|
-
description: "Disconnect from Metro servers and stop auto-reconnection. Without device param: disconnects ALL devices. With device param: disconnects only the matching device. Use this to remove stale connections or free the CDP slot for the built-in debugger. Log and network buffers are preserved. Reconnect later with scan_metro
|
|
1795
|
+
description: "Disconnect from Metro servers and stop auto-reconnection. Without device param: disconnects ALL devices. With device param: disconnects only the matching device. Use this to remove stale connections or free the CDP slot for the built-in debugger. Log and network buffers are preserved. Reconnect later with scan_metro.\n" +
|
|
1796
|
+
"PURPOSE: Cleanly release CDP slots so another debugger (Flipper, React DevTools, Chrome) can attach, or nuke a stale connection the MCP keeps reviving.\n" +
|
|
1797
|
+
"WHEN TO USE: Before launching a native debugger, when connections keep flapping, or after a simulator/device restart left zombie targets.\n" +
|
|
1798
|
+
"WORKFLOW: disconnect_metro -> attach other debugger / restart app -> scan_metro to reconnect.\n" +
|
|
1799
|
+
"LIMITATIONS: Suppresses auto-reconnect until scan_metro or connect_metro is called again; buffers persist but won't receive new events.\n" +
|
|
1800
|
+
"GOOD: disconnect_metro(); disconnect_metro({ device: \"iPhone\" })\n" +
|
|
1801
|
+
"BAD: Using disconnect_metro to clear logs — use clear_logs instead; disconnect breaks capture.\n" +
|
|
1802
|
+
"SEE ALSO: call get_usage_guide(topic=\"setup\") for the full session-setup playbook.",
|
|
1168
1803
|
inputSchema: {
|
|
1169
1804
|
device: z.string().optional().describe("Target device name (substring match) to disconnect. Omit to disconnect all devices. Run get_apps to see connected devices.")
|
|
1170
1805
|
}
|
|
@@ -1253,7 +1888,8 @@ registerToolWithTelemetry("execute_in_app", {
|
|
|
1253
1888
|
"- NO emoji or non-ASCII characters in string literals — causes parse errors\n" +
|
|
1254
1889
|
"- Keep expressions simple and synchronous when possible\n\n" +
|
|
1255
1890
|
"GOOD examples: `__DEV__`, `__APOLLO_CLIENT__.cache.extract()`, `__EXPO_ROUTER__.navigate('/settings')`\n" +
|
|
1256
|
-
"BAD examples: `async () => { await fetch(...) }`, `require('react-native')`, `console.log('\\u{1F600}')
|
|
1891
|
+
"BAD examples: `async () => { await fetch(...) }`, `require('react-native')`, `console.log('\\u{1F600}')`\n" +
|
|
1892
|
+
"SEE ALSO: call get_usage_guide(topic=\"state\") for the full app-state playbook.",
|
|
1257
1893
|
inputSchema: {
|
|
1258
1894
|
expression: z
|
|
1259
1895
|
.string()
|
|
@@ -1321,7 +1957,14 @@ registerToolWithTelemetry("execute_in_app", {
|
|
|
1321
1957
|
});
|
|
1322
1958
|
// Tool: List debug globals available in the app
|
|
1323
1959
|
registerToolWithTelemetry("list_debug_globals", {
|
|
1324
|
-
description: "List globally available debugging objects in the connected React Native app (Apollo Client, Redux store, React DevTools, etc.). Use this to discover what state management and debugging tools are available
|
|
1960
|
+
description: "List globally available debugging objects in the connected React Native app (Apollo Client, Redux store, React DevTools, etc.). Use this to discover what state management and debugging tools are available.\n" +
|
|
1961
|
+
"PURPOSE: Enumerate the app's `globalThis.*` surface so you know which stores, clients, and debug hooks you can drill into.\n" +
|
|
1962
|
+
"WHEN TO USE: Start of a state-debugging session, or when you don't know whether the app exposes a Redux/Apollo/Zustand handle.\n" +
|
|
1963
|
+
"WORKFLOW: list_debug_globals -> inspect_global(objectName=\"...\") -> execute_in_app for reads/mutations.\n" +
|
|
1964
|
+
"LIMITATIONS: Only sees variables explicitly assigned to a global (e.g., `globalThis.store = store`). Module-scoped state is invisible — expose it first.\n" +
|
|
1965
|
+
"GOOD: list_debug_globals()\n" +
|
|
1966
|
+
"BAD: Calling before scan_metro — needs a live connection.\n" +
|
|
1967
|
+
"SEE ALSO: call get_usage_guide(topic=\"state\") for the full app-state playbook.",
|
|
1325
1968
|
inputSchema: {
|
|
1326
1969
|
device: z.string().optional().describe("Target device name (substring match). Omit for default device. Run get_apps to see connected devices.")
|
|
1327
1970
|
}
|
|
@@ -1349,7 +1992,14 @@ registerToolWithTelemetry("list_debug_globals", {
|
|
|
1349
1992
|
});
|
|
1350
1993
|
// Tool: Inspect a global object to see its properties and types
|
|
1351
1994
|
registerToolWithTelemetry("inspect_global", {
|
|
1352
|
-
description: "Inspect a global object to see its properties, types, and whether they are callable functions. Use this BEFORE calling methods on unfamiliar objects to avoid errors
|
|
1995
|
+
description: "Inspect a global object to see its properties, types, and whether they are callable functions. Use this BEFORE calling methods on unfamiliar objects to avoid errors.\n" +
|
|
1996
|
+
"PURPOSE: Surface the shape of a global (Apollo client, Redux store, Expo Router, etc.) — keys, types, and which members are callable — without executing arbitrary code.\n" +
|
|
1997
|
+
"WHEN TO USE: After list_debug_globals identifies a promising global and before you try execute_in_app on it.\n" +
|
|
1998
|
+
"WORKFLOW: list_debug_globals -> inspect_global(objectName=\"__APOLLO_CLIENT__\") -> execute_in_app(\"__APOLLO_CLIENT__.cache.extract()\").\n" +
|
|
1999
|
+
"LIMITATIONS: Only reads one level deep; nested objects show as `[object Object]` — re-inspect the child path with execute_in_app. Throws ReferenceError for globals that don't exist.\n" +
|
|
2000
|
+
"GOOD: inspect_global({ objectName: \"__APOLLO_CLIENT__\" })\n" +
|
|
2001
|
+
"BAD: inspect_global({ objectName: \"store.state\" }) — path expressions aren't supported; use execute_in_app for property chains.\n" +
|
|
2002
|
+
"SEE ALSO: call get_usage_guide(topic=\"state\") for the full app-state playbook.",
|
|
1353
2003
|
inputSchema: {
|
|
1354
2004
|
objectName: z
|
|
1355
2005
|
.string()
|
|
@@ -1389,7 +2039,14 @@ registerToolWithTelemetry("inspect_global", {
|
|
|
1389
2039
|
// ============================================================================
|
|
1390
2040
|
// Tool: Get the React component tree
|
|
1391
2041
|
registerToolWithTelemetry("get_component_tree", {
|
|
1392
|
-
description: "Get the full React component tree from the running app. Shows the complete fiber hierarchy including providers, navigation wrappers, and internal components. For a screen overview with positions and text, use get_screen_layout instead. Use structureOnly=true for compact names-only output
|
|
2042
|
+
description: "Get the full React component tree from the running app. Shows the complete fiber hierarchy including providers, navigation wrappers, and internal components. For a screen overview with positions and text, use get_screen_layout instead. Use structureOnly=true for compact names-only output.\n" +
|
|
2043
|
+
"PURPOSE: Expose the entire fiber tree — including providers, navigators, and off-screen subtrees — when get_screen_layout's visible-only view isn't enough.\n" +
|
|
2044
|
+
"WHEN TO USE: Debugging context propagation, navigation wrappers, hidden modals, or when you need to understand the full React architecture.\n" +
|
|
2045
|
+
"WORKFLOW: get_component_tree(structureOnly=true) for overview -> find_components for targeted lookup -> inspect_component for props/state.\n" +
|
|
2046
|
+
"LIMITATIONS: Full trees can be large; always start with structureOnly=true. Ignores non-React native views. Minified builds return display names that may be opaque.\n" +
|
|
2047
|
+
"GOOD: get_component_tree({ structureOnly: true })\n" +
|
|
2048
|
+
"BAD: get_component_tree({ includeProps: true, includeStyles: true }) on a large app — likely hits response-size limits. Prefer inspect_component for specific nodes.\n" +
|
|
2049
|
+
"SEE ALSO: call get_usage_guide(topic=\"inspect\") for the full component-inspect playbook.",
|
|
1393
2050
|
inputSchema: {
|
|
1394
2051
|
structureOnly: z
|
|
1395
2052
|
.boolean()
|
|
@@ -1452,54 +2109,17 @@ registerToolWithTelemetry("get_component_tree", {
|
|
|
1452
2109
|
]
|
|
1453
2110
|
};
|
|
1454
2111
|
});
|
|
1455
|
-
// Tool: Get full screen layout (all components with layout styles)
|
|
1456
|
-
registerToolWithTelemetry("get_screen_layout", {
|
|
1457
|
-
description: "Get a screen map showing visible components as an indented tree with actual screen positions. Uses measureInWindow for real coordinates and filters out off-screen components. Returns meaningful component names with text content and frame data (x,y width x height). Coordinates are in **points** (iOS) or **dp** (Android) — NOT screenshot pixels. Use tap(text=...) or tap(testID=...) to interact with discovered components. Use extended=true to include layout styles (padding, margin, flex, backgroundColor, etc.).",
|
|
1458
|
-
inputSchema: {
|
|
1459
|
-
extended: z
|
|
1460
|
-
.boolean()
|
|
1461
|
-
.optional()
|
|
1462
|
-
.default(false)
|
|
1463
|
-
.describe("Include layout styles (padding, margin, flex, backgroundColor, borderRadius, etc.) for each component. Default: false for compact output."),
|
|
1464
|
-
summary: z
|
|
1465
|
-
.boolean()
|
|
1466
|
-
.optional()
|
|
1467
|
-
.default(false)
|
|
1468
|
-
.describe("Return only component counts by name instead of full tree (default: false)"),
|
|
1469
|
-
device: z.string().optional().describe("Target device name (substring match). Omit for default device. Run get_apps to see connected devices.")
|
|
1470
|
-
}
|
|
1471
|
-
}, async ({ extended, summary, device }) => {
|
|
1472
|
-
if (!hasMetro()) {
|
|
1473
|
-
const hint = await metroMissingHintIfAbsent("get_screen_layout");
|
|
1474
|
-
return {
|
|
1475
|
-
content: [{ type: "text", text: `Screen Layout unavailable.${hint}` }],
|
|
1476
|
-
isError: true
|
|
1477
|
-
};
|
|
1478
|
-
}
|
|
1479
|
-
const result = await getScreenLayout({ extended, summary, device });
|
|
1480
|
-
if (!result.success) {
|
|
1481
|
-
return {
|
|
1482
|
-
content: [
|
|
1483
|
-
{
|
|
1484
|
-
type: "text",
|
|
1485
|
-
text: `Error: ${result.error}`
|
|
1486
|
-
}
|
|
1487
|
-
],
|
|
1488
|
-
isError: true
|
|
1489
|
-
};
|
|
1490
|
-
}
|
|
1491
|
-
return {
|
|
1492
|
-
content: [
|
|
1493
|
-
{
|
|
1494
|
-
type: "text",
|
|
1495
|
-
text: `Screen Layout:\n\n${result.result}`
|
|
1496
|
-
}
|
|
1497
|
-
]
|
|
1498
|
-
};
|
|
1499
|
-
});
|
|
1500
2112
|
// Tool: Get all pressable elements on screen
|
|
1501
2113
|
registerToolWithTelemetry("get_pressable_elements", {
|
|
1502
|
-
description: "Find all pressable (onPress) and input (TextInput) elements currently visible on screen. Returns component names, ready-to-tap center coordinates in SCREENSHOT PIXELS (same space as ios_screenshot/android_screenshot — pass directly to tap(x, y)), text labels, testID, accessibilityLabel, and a spatial nearbyText hint for icon-only buttons. Each element includes hasLabel (true if it contains text) and isInput (true for TextInput fields)
|
|
2114
|
+
description: "Find all pressable (onPress) and input (TextInput) elements currently visible on screen. Returns component names, ready-to-tap center coordinates in SCREENSHOT PIXELS (same space as ios_screenshot/android_screenshot — pass directly to tap(x, y)), text labels, testID, accessibilityLabel, and a spatial nearbyText hint for icon-only buttons. Each element includes hasLabel (true if it contains text) and isInput (true for TextInput fields).\n" +
|
|
2115
|
+
"HELPER — call before `tap` when you need to enumerate candidate elements before committing to a target; not a replacement for tap itself.\n" +
|
|
2116
|
+
"PURPOSE: Produce a ready-to-tap inventory of every interactive element on screen with screenshot-pixel coordinates that tap(x, y) accepts directly.\n" +
|
|
2117
|
+
"WHEN TO USE: Before tapping icon-only buttons, when text-based tap keeps failing, or to enumerate what the user can actually interact with.\n" +
|
|
2118
|
+
"WORKFLOW: ios_screenshot / android_screenshot -> get_pressable_elements -> tap(testID=\"...\") or tap(x, y) using the center coordinates.\n" +
|
|
2119
|
+
"LIMITATIONS: Visible-only (off-screen pressables are excluded). Requires a live React connection. Coordinates are in screenshot pixels — the tap tool converts to points internally.\n" +
|
|
2120
|
+
"GOOD: get_pressable_elements()\n" +
|
|
2121
|
+
"BAD: Calling when tap(testID=\"...\") already works — testID matching is faster and more stable.\n" +
|
|
2122
|
+
"SEE ALSO: call get_usage_guide(topic=\"interact\") for the full interaction playbook.",
|
|
1503
2123
|
inputSchema: {
|
|
1504
2124
|
device: z.string().optional().describe("Target device name (substring match). Omit for default device. Run get_apps to see connected devices.")
|
|
1505
2125
|
}
|
|
@@ -1642,7 +2262,15 @@ function formatPressablesInPixels(elements, opts) {
|
|
|
1642
2262
|
}
|
|
1643
2263
|
// Tool: Inspect a specific component by name
|
|
1644
2264
|
registerToolWithTelemetry("inspect_component", {
|
|
1645
|
-
description: "Inspect a specific React component by name. **DRILL-DOWN TOOL**: Use after get_screen_layout or find_components to identify which component to inspect. Returns props, style, state (hooks), and optionally children tree. Use childrenDepth to control how deep nested children go."
|
|
2265
|
+
description: "Inspect a specific React component by name. **DRILL-DOWN TOOL**: Use after get_screen_layout or find_components to identify which component to inspect. Returns props, style, state (hooks), and optionally children tree. Use childrenDepth to control how deep nested children go." +
|
|
2266
|
+
primaryInteractionBanner() + "\n" +
|
|
2267
|
+
"PURPOSE: Reveal a mounted component's live props, hook state, and (optionally) child subtree so you can reason about why it renders the way it does.\n" +
|
|
2268
|
+
"WHEN TO USE: User asks \"why is this button disabled\", \"what props does X receive\", or you need to confirm state changed after a tap.\n" +
|
|
2269
|
+
"WORKFLOW: get_screen_layout or find_components -> inspect_component(componentName=\"Foo\") -> tap or execute_in_app to change state -> inspect_component again.\n" +
|
|
2270
|
+
"LIMITATIONS: Requires the component to be currently mounted in the fiber tree. Name matching is exact; use find_components for fuzzy/regex lookup.\n" +
|
|
2271
|
+
"GOOD: inspect_component({ componentName: \"SneakerCard\", index: 0 })\n" +
|
|
2272
|
+
"BAD: inspect_component({ componentName: \"Card\" }) when many Card instances exist — pass index or narrow via find_components.\n" +
|
|
2273
|
+
"SEE ALSO: call get_usage_guide(topic=\"inspect\") for the full component-inspect playbook.",
|
|
1646
2274
|
inputSchema: {
|
|
1647
2275
|
componentName: z
|
|
1648
2276
|
.string()
|
|
@@ -1703,7 +2331,15 @@ registerToolWithTelemetry("inspect_component", {
|
|
|
1703
2331
|
});
|
|
1704
2332
|
// Tool: Find components matching a pattern
|
|
1705
2333
|
registerToolWithTelemetry("find_components", {
|
|
1706
|
-
description: "Find components matching a name pattern. **TARGETED SEARCH**: Use after get_screen_layout or get_component_tree(structureOnly=true) to find specific components by pattern. Use includeLayout=true to get padding/margin/flex styles."
|
|
2334
|
+
description: "Find components matching a name pattern. **TARGETED SEARCH**: Use after get_screen_layout or get_component_tree(structureOnly=true) to find specific components by pattern. Use includeLayout=true to get padding/margin/flex styles." +
|
|
2335
|
+
primaryInteractionBanner() + "\n" +
|
|
2336
|
+
"PURPOSE: Fast regex search over the entire fiber tree — including off-screen and wrapper components — to locate every instance of a component by name.\n" +
|
|
2337
|
+
"WHEN TO USE: You know roughly what the component is called (e.g., \"Button\", \"Screen$\") but not where it lives, or you need counts/paths before drilling in with inspect_component.\n" +
|
|
2338
|
+
"WORKFLOW: get_screen_layout (orient) -> find_components(pattern=\"...\") -> inspect_component(componentName=\"...\", index=N).\n" +
|
|
2339
|
+
"LIMITATIONS: Matches the React display name only; minified builds may return opaque names. Large result sets — use maxResults or a tighter pattern.\n" +
|
|
2340
|
+
"GOOD: find_components({ pattern: \"Button\" }); find_components({ pattern: \"Screen$\" })\n" +
|
|
2341
|
+
"BAD: find_components({ pattern: \".*\" }) — floods the response; narrow the regex.\n" +
|
|
2342
|
+
"SEE ALSO: call get_usage_guide(topic=\"inspect\") for the full component-inspect playbook.",
|
|
1707
2343
|
inputSchema: {
|
|
1708
2344
|
pattern: z
|
|
1709
2345
|
.string()
|
|
@@ -1732,153 +2368,29 @@ registerToolWithTelemetry("find_components", {
|
|
|
1732
2368
|
if (!result.success) {
|
|
1733
2369
|
return {
|
|
1734
2370
|
content: [
|
|
1735
|
-
{
|
|
1736
|
-
type: "text",
|
|
1737
|
-
text: `Error: ${result.error}`
|
|
1738
|
-
}
|
|
1739
|
-
],
|
|
1740
|
-
isError: true
|
|
1741
|
-
};
|
|
1742
|
-
}
|
|
1743
|
-
return {
|
|
1744
|
-
content: [
|
|
1745
|
-
{
|
|
1746
|
-
type: "text",
|
|
1747
|
-
text: `Find Components (pattern: "${pattern}"):\n\n${result.result}`
|
|
1748
|
-
}
|
|
1749
|
-
]
|
|
1750
|
-
};
|
|
1751
|
-
});
|
|
1752
|
-
// Tool: Unified tap — tries fiber, accessibility, OCR, coordinate strategies
|
|
1753
|
-
registerToolWithTelemetry("tap", {
|
|
1754
|
-
description: "Tap a UI element. Automatically tries multiple strategies: fiber tree (React), accessibility tree (native), and OCR (visual). " +
|
|
1755
|
-
"Auto-detects platform (iOS/Android). For coordinates, pass raw pixel values from screenshot — tap handles all coordinate conversion internally (iOS pixel-to-point, scale factor adjustments).\n\n" +
|
|
1756
|
-
"PREREQUISITE: For iOS, an iOS UI driver must be installed — AXe (recommended: brew install cameroncooke/axe/axe) or IDB (brew install idb-companion). Without it, accessibility and coordinate strategies will fail.\n\n" +
|
|
1757
|
-
"IMPORTANT: When both iOS and Android devices are connected, specify platform explicitly to avoid tapping on the wrong device.\n\n" +
|
|
1758
|
-
"Examples:\n" +
|
|
1759
|
-
"- tap(text=\"Submit\") — finds and taps element with matching text\n" +
|
|
1760
|
-
"- tap(testID=\"login-btn\") — finds by testID\n" +
|
|
1761
|
-
"- tap(component=\"HamburgerIcon\") — finds by React component name\n" +
|
|
1762
|
-
"- tap(x=300, y=600) — taps at pixel coordinates; use coordinates from the pressable elements list returned by ios_screenshot/android_screenshot for reliable icon and ambiguous-element taps\n" +
|
|
1763
|
-
"- tap(text=\"Menu\", strategy=\"ocr\") — forces OCR strategy only\n" +
|
|
1764
|
-
"- tap(x=300, y=600, native=true, platform=\"android\") — taps directly via ADB/simctl without React Native connection\n\n" +
|
|
1765
|
-
"Returns a post-tap screenshot by default (set screenshot=false to disable). " +
|
|
1766
|
-
"For coordinate/accessibility/OCR taps, automatically verifies the tap had a visual effect via before/after screenshot diff (set verify=false to disable).",
|
|
1767
|
-
inputSchema: {
|
|
1768
|
-
text: z
|
|
1769
|
-
.string()
|
|
1770
|
-
.optional()
|
|
1771
|
-
.describe("Visible text to match (case-insensitive substring). ASCII only for fiber strategy; OCR handles non-ASCII."),
|
|
1772
|
-
testID: z
|
|
1773
|
-
.string()
|
|
1774
|
-
.optional()
|
|
1775
|
-
.describe("Exact match on the element's testID prop."),
|
|
1776
|
-
component: z
|
|
1777
|
-
.string()
|
|
1778
|
-
.optional()
|
|
1779
|
-
.describe("Component name match (case-insensitive substring, e.g. 'Button', 'MenuItem')."),
|
|
1780
|
-
index: z.coerce
|
|
1781
|
-
.number()
|
|
1782
|
-
.optional()
|
|
1783
|
-
.describe("Zero-based index when multiple elements match (default: 0)."),
|
|
1784
|
-
x: z.coerce
|
|
1785
|
-
.number()
|
|
1786
|
-
.optional()
|
|
1787
|
-
.describe("X coordinate in pixels (from screenshot). Must provide both x and y."),
|
|
1788
|
-
y: z.coerce
|
|
1789
|
-
.number()
|
|
1790
|
-
.optional()
|
|
1791
|
-
.describe("Y coordinate in pixels (from screenshot). Must provide both x and y."),
|
|
1792
|
-
strategy: z
|
|
1793
|
-
.enum(["auto", "fiber", "accessibility", "ocr", "coordinate"])
|
|
1794
|
-
.optional()
|
|
1795
|
-
.default("auto")
|
|
1796
|
-
.describe('"auto" (default) tries fiber -> accessibility -> OCR. Set explicitly to skip strategies you know will fail.'),
|
|
1797
|
-
maxTraversalDepth: z.coerce
|
|
1798
|
-
.number()
|
|
1799
|
-
.optional()
|
|
1800
|
-
.describe("Max parent levels to traverse when searching by component name (default: 15). " +
|
|
1801
|
-
"Increase if your component is deeply wrapped (e.g. inside multiple HOCs/animation wrappers)."),
|
|
1802
|
-
native: z
|
|
1803
|
-
.boolean()
|
|
1804
|
-
.optional()
|
|
1805
|
-
.default(false)
|
|
1806
|
-
.describe("When true, tap coordinates directly via ADB/simctl without requiring a React Native connection. " +
|
|
1807
|
-
"Useful for interacting with native UI, system dialogs, or non-RN apps. Requires x/y coordinates."),
|
|
1808
|
-
platform: z
|
|
1809
|
-
.enum(["ios", "android"])
|
|
1810
|
-
.optional()
|
|
1811
|
-
.describe("Target platform. Required when both iOS and Android devices are connected. Auto-detected if only one platform is available."),
|
|
1812
|
-
screenshot: z
|
|
1813
|
-
.boolean()
|
|
1814
|
-
.optional()
|
|
1815
|
-
.default(true)
|
|
1816
|
-
.describe("Capture and return a post-tap screenshot. Enabled by default. Set to false for fastest execution (no screenshots)."),
|
|
1817
|
-
verify: z
|
|
1818
|
-
.boolean()
|
|
1819
|
-
.optional()
|
|
1820
|
-
.describe("Run before/after screenshot diff to detect if the tap had a meaningful visual effect. " +
|
|
1821
|
-
"Default: true for coordinate/accessibility/ocr strategies, false for fiber. Requires screenshot=true."),
|
|
1822
|
-
burst: z
|
|
1823
|
-
.boolean()
|
|
1824
|
-
.optional()
|
|
1825
|
-
.default(false)
|
|
1826
|
-
.describe("Enable burst screenshot capture for enhanced verification. " +
|
|
1827
|
-
"Captures 4 rapid screenshots (~150ms intervals) after the tap to detect transient visual feedback " +
|
|
1828
|
-
"(press animations, highlights, ripples) that may settle before a standard after-screenshot. " +
|
|
1829
|
-
"Results are stored in the image buffer (use get_images to inspect individual frames). " +
|
|
1830
|
-
"Default: false."),
|
|
1831
|
-
},
|
|
1832
|
-
}, async (args) => {
|
|
1833
|
-
const result = await tap({
|
|
1834
|
-
text: args.text,
|
|
1835
|
-
testID: args.testID,
|
|
1836
|
-
component: args.component,
|
|
1837
|
-
index: args.index,
|
|
1838
|
-
x: args.x,
|
|
1839
|
-
y: args.y,
|
|
1840
|
-
strategy: args.strategy,
|
|
1841
|
-
maxTraversalDepth: args.maxTraversalDepth,
|
|
1842
|
-
native: args.native,
|
|
1843
|
-
platform: args.platform,
|
|
1844
|
-
screenshot: args.screenshot,
|
|
1845
|
-
verify: args.verify,
|
|
1846
|
-
burst: args.burst,
|
|
1847
|
-
});
|
|
1848
|
-
const { screenshot: screenshotData, ...resultWithoutScreenshot } = result;
|
|
1849
|
-
const text = JSON.stringify(resultWithoutScreenshot, null, 2);
|
|
1850
|
-
// Pack strategy mode + attempted strategies into errorContext for telemetry
|
|
1851
|
-
// e.g. "s=ocr|fiber:no_pressable|ocr:no_match" or "s=auto|fiber:no_pressable|accessibility:not_found|ocr:no_match"
|
|
1852
|
-
const stratPrefix = args.strategy && args.strategy !== "auto" ? `s=${args.strategy}|` : "";
|
|
1853
|
-
const errorContext = result.attempted?.length
|
|
1854
|
-
? stratPrefix + result.attempted.map(a => `${a.strategy}:${a.reason.slice(0, 40)}`).join("|")
|
|
1855
|
-
: undefined;
|
|
1856
|
-
const content = [
|
|
1857
|
-
{ type: "text", text },
|
|
1858
|
-
];
|
|
1859
|
-
if (screenshotData) {
|
|
1860
|
-
content.push({
|
|
1861
|
-
type: "image",
|
|
1862
|
-
data: screenshotData.image,
|
|
1863
|
-
mimeType: "image/jpeg",
|
|
1864
|
-
});
|
|
2371
|
+
{
|
|
2372
|
+
type: "text",
|
|
2373
|
+
text: `Error: ${result.error}`
|
|
2374
|
+
}
|
|
2375
|
+
],
|
|
2376
|
+
isError: true
|
|
2377
|
+
};
|
|
1865
2378
|
}
|
|
1866
2379
|
return {
|
|
1867
|
-
content
|
|
1868
|
-
|
|
1869
|
-
|
|
1870
|
-
|
|
1871
|
-
|
|
1872
|
-
|
|
1873
|
-
_meaningful: result.verification?.meaningful,
|
|
1874
|
-
_changeRate: result.verification?.changeRate,
|
|
1875
|
-
_tapStrategy: result.method,
|
|
1876
|
-
_iosDriver: result.platform === "ios" ? (process.env.IOS_DRIVER?.toLowerCase() || "idb") : undefined,
|
|
2380
|
+
content: [
|
|
2381
|
+
{
|
|
2382
|
+
type: "text",
|
|
2383
|
+
text: `Find Components (pattern: "${pattern}"):\n\n${result.result}`
|
|
2384
|
+
}
|
|
2385
|
+
]
|
|
1877
2386
|
};
|
|
1878
2387
|
});
|
|
1879
2388
|
// Tool: Toggle Element Inspector programmatically
|
|
1880
2389
|
registerToolWithTelemetry("toggle_element_inspector", {
|
|
1881
|
-
description: "Toggle React Native's Element Inspector overlay on/off. Rarely needed directly — get_inspector_selection auto-enables the inspector when called with coordinates. Use this only when you need manual control over the overlay visibility
|
|
2390
|
+
description: "Toggle React Native's Element Inspector overlay on/off. Rarely needed directly — get_inspector_selection auto-enables the inspector when called with coordinates. Use this only when you need manual control over the overlay visibility.\n" +
|
|
2391
|
+
"PURPOSE: Manual control over the on-device inspector overlay — mostly useful for screenshots that should show the inspector UI or for disabling it after a session.\n" +
|
|
2392
|
+
"WHEN TO USE: Only when get_inspector_selection's auto-enable isn't appropriate (e.g., leaving the overlay on for a user screenshot).\n" +
|
|
2393
|
+
"SEE ALSO: call get_usage_guide(topic=\"inspect\") for the full component-inspect playbook.",
|
|
1882
2394
|
inputSchema: {
|
|
1883
2395
|
device: z.string().optional().describe("Target device name (substring match). Omit for default device. Run get_apps to see connected devices.")
|
|
1884
2396
|
}
|
|
@@ -1930,7 +2442,14 @@ registerToolWithTelemetry("toggle_element_inspector", {
|
|
|
1930
2442
|
});
|
|
1931
2443
|
// Tool: Get currently selected element from Element Inspector
|
|
1932
2444
|
registerToolWithTelemetry("get_inspector_selection", {
|
|
1933
|
-
description: "Identify the React component at a screen location by reading RN's Element Inspector. Returns a clean component hierarchy with file paths — ideal for finding the real component name (e.g. HomeScreen > SneakerCard > PulseActionButton). If x/y provided: auto-enables inspector, taps at coordinates, returns hierarchy. If no coordinates: returns current selection
|
|
2445
|
+
description: "Identify the React component at a screen location by reading RN's Element Inspector. Returns a clean component hierarchy with file paths — ideal for finding the real component name (e.g. HomeScreen > SneakerCard > PulseActionButton). If x/y provided: auto-enables inspector, taps at coordinates, returns hierarchy. If no coordinates: returns current selection.\n" +
|
|
2446
|
+
"PURPOSE: Map a screen coordinate to a real component name AND source file path, so you know which file to edit.\n" +
|
|
2447
|
+
"WHEN TO USE: You have a screenshot and want to know \"what component renders this pixel?\" — especially for file-path grounding before making code changes.\n" +
|
|
2448
|
+
"WORKFLOW: ios_screenshot / android_screenshot / ocr_screenshot -> get_inspector_selection(x, y) -> edit the returned source file.\n" +
|
|
2449
|
+
"LIMITATIONS: Requires RN dev mode (inspector only exists in __DEV__). x/y are in points/dp, not screenshot pixels — tap-style coordinates are fine since the tool taps internally. Flaky on Bridgeless if the inspector fails to initialize.\n" +
|
|
2450
|
+
"GOOD: get_inspector_selection({ x: 180, y: 420 })\n" +
|
|
2451
|
+
"BAD: Using it for layout measurements — prefer inspect_at_point which returns the exact frame.\n" +
|
|
2452
|
+
"SEE ALSO: call get_usage_guide(topic=\"inspect\") for the full component-inspect playbook.",
|
|
1934
2453
|
inputSchema: {
|
|
1935
2454
|
x: z
|
|
1936
2455
|
.number()
|
|
@@ -2033,7 +2552,14 @@ registerToolWithTelemetry("get_inspector_selection", {
|
|
|
2033
2552
|
});
|
|
2034
2553
|
// Tool: Inspect component at coordinates (like Element Inspector)
|
|
2035
2554
|
registerToolWithTelemetry("inspect_at_point", {
|
|
2036
|
-
description: "Inspect the React component at specific (x, y) coordinates for layout debugging. Returns component props, measured frame (position/size in dp), and component path. Works on both Paper and Fabric. Coordinates are in dp (density-independent pixels). To convert from screenshot pixels: divide by the device pixel ratio (e.g. 540px / 2.625 = 205dp). Best for: checking layout bounds, reading component props/styles, pixel-perfect debugging. For identifying component names: prefer get_inspector_selection which returns a cleaner hierarchy with file paths
|
|
2555
|
+
description: "Inspect the React component at specific (x, y) coordinates for layout debugging. Returns component props, measured frame (position/size in dp), and component path. Works on both Paper and Fabric. Coordinates are in dp (density-independent pixels). To convert from screenshot pixels: divide by the device pixel ratio (e.g. 540px / 2.625 = 205dp). Best for: checking layout bounds, reading component props/styles, pixel-perfect debugging. For identifying component names: prefer get_inspector_selection which returns a cleaner hierarchy with file paths.\n" +
|
|
2556
|
+
"PURPOSE: Pixel-precise layout diagnosis — returns the exact measured frame, props, and style of whatever renders at a point.\n" +
|
|
2557
|
+
"WHEN TO USE: Checking why a button is clipped, why an element's hit area is wrong, or reading the actual computed size/position of a component.\n" +
|
|
2558
|
+
"WORKFLOW: ios_screenshot -> find suspect pixel -> convert to dp (pixel / pixelRatio) -> inspect_at_point(x, y).\n" +
|
|
2559
|
+
"LIMITATIONS: Coordinates MUST be in dp, not screenshot pixels — this differs from tap() which accepts either. Wrong unit = wrong component. Requires live React connection.\n" +
|
|
2560
|
+
"GOOD: inspect_at_point({ x: 205, y: 360 }) // dp after dividing\n" +
|
|
2561
|
+
"BAD: inspect_at_point({ x: 540, y: 960 }) // raw screenshot pixels — picks the wrong node.\n" +
|
|
2562
|
+
"SEE ALSO: call get_usage_guide(topic=\"inspect\") for the full component-inspect playbook.",
|
|
2037
2563
|
inputSchema: {
|
|
2038
2564
|
x: z
|
|
2039
2565
|
.number()
|
|
@@ -2099,7 +2625,14 @@ registerToolWithTelemetry("inspect_at_point", {
|
|
|
2099
2625
|
});
|
|
2100
2626
|
// Tool: Get network requests
|
|
2101
2627
|
registerToolWithTelemetry("get_network_requests", {
|
|
2102
|
-
description: "Retrieve captured network requests from connected React Native app. Shows URL, method, status, and timing. Note: On Bridgeless targets (Expo SDK 52+) without the SDK, capture may miss early startup requests. Install react-native-ai-devtools-sdk for full capture with headers and response bodies. Tip: Use summary=true first for stats overview
|
|
2628
|
+
description: "Retrieve captured network requests from connected React Native app. Shows URL, method, status, and timing. Note: On Bridgeless targets (Expo SDK 52+) without the SDK, capture may miss early startup requests. Install react-native-ai-devtools-sdk for full capture with headers and response bodies. Tip: Use summary=true first for stats overview.\n" +
|
|
2629
|
+
"PURPOSE: Inspect HTTP traffic the app made since connection — URLs, methods, status codes, and timings — to debug API, auth, and caching issues.\n" +
|
|
2630
|
+
"WHEN TO USE: User reports a failed login/load, slow screen, or wrong data. Confirm a request fired, check its status, and pivot to get_request_details for headers/body.\n" +
|
|
2631
|
+
"WORKFLOW: scan_metro -> reproduce action -> get_network_requests({ summary: true }) -> get_network_requests({ status: 500 }) or search_network -> get_request_details(id).\n" +
|
|
2632
|
+
"LIMITATIONS: Bridgeless targets without the SDK may miss pre-connect requests and response bodies — install react-native-ai-devtools-sdk for full fidelity.\n" +
|
|
2633
|
+
"GOOD: get_network_requests({ summary: true }) then get_network_requests({ urlPattern: \"/login\", status: 401 })\n" +
|
|
2634
|
+
"BAD: get_network_requests({ maxRequests: 500 }) as the first call — start with summary=true.\n" +
|
|
2635
|
+
"SEE ALSO: call get_usage_guide(topic=\"network\") for the full network-inspect playbook.",
|
|
2103
2636
|
inputSchema: {
|
|
2104
2637
|
maxRequests: z
|
|
2105
2638
|
.number()
|
|
@@ -2253,7 +2786,14 @@ registerToolWithTelemetry("get_network_requests", {
|
|
|
2253
2786
|
total += b.size; return total === 0; });
|
|
2254
2787
|
// Tool: Search network requests
|
|
2255
2788
|
registerToolWithTelemetry("search_network", {
|
|
2256
|
-
description: "Search network requests by URL pattern (case-insensitive)"
|
|
2789
|
+
description: "Search network requests by URL pattern (case-insensitive).\n" +
|
|
2790
|
+
"PURPOSE: Filter the network buffer to requests whose URL matches a substring — fast way to find a specific endpoint in a noisy app.\n" +
|
|
2791
|
+
"WHEN TO USE: You know part of the URL (e.g., \"/graphql\", \"users\", a domain) and want matching requests across all devices.\n" +
|
|
2792
|
+
"WORKFLOW: search_network(urlPattern=\"/api/\") -> get_request_details(requestId=\"...\") for full headers/body.\n" +
|
|
2793
|
+
"LIMITATIONS: Matches URL only; for method/status/body filtering use get_network_requests. Bodies are only present when the SDK is installed.\n" +
|
|
2794
|
+
"GOOD: search_network({ urlPattern: \"/graphql\" })\n" +
|
|
2795
|
+
"BAD: search_network({ urlPattern: \"\" }) — empty pattern matches everything; use get_network_requests instead.\n" +
|
|
2796
|
+
"SEE ALSO: call get_usage_guide(topic=\"network\") for the full network-inspect playbook.",
|
|
2257
2797
|
inputSchema: {
|
|
2258
2798
|
urlPattern: z.string().describe("URL pattern to search for"),
|
|
2259
2799
|
maxResults: z.number().optional().default(50).describe("Maximum number of results to return (default: 50)"),
|
|
@@ -2323,7 +2863,14 @@ registerToolWithTelemetry("search_network", {
|
|
|
2323
2863
|
});
|
|
2324
2864
|
// Tool: Get request details
|
|
2325
2865
|
registerToolWithTelemetry("get_request_details", {
|
|
2326
|
-
description: "Get full details of a specific network request including headers, body, and timing. With the SDK installed, includes full request/response bodies. Without SDK, bodies are not available on most targets. Use get_network_requests first to find the request ID
|
|
2866
|
+
description: "Get full details of a specific network request including headers, body, and timing. With the SDK installed, includes full request/response bodies. Without SDK, bodies are not available on most targets. Use get_network_requests first to find the request ID.\n" +
|
|
2867
|
+
"PURPOSE: Drill into a single network entry — full request/response headers, body, status, and timing breakdown.\n" +
|
|
2868
|
+
"WHEN TO USE: After get_network_requests or search_network returns a suspect ID and you need the payload to diagnose.\n" +
|
|
2869
|
+
"WORKFLOW: get_network_requests / search_network -> copy id -> get_request_details(requestId).\n" +
|
|
2870
|
+
"LIMITATIONS: Bodies require the react-native-ai-devtools-sdk in the app; on CDP-only targets response bodies are missing. Large bodies are truncated — raise maxBodyLength.\n" +
|
|
2871
|
+
"GOOD: get_request_details({ requestId: \"42\", maxBodyLength: 4000 })\n" +
|
|
2872
|
+
"BAD: Guessing requestIds — always get them from get_network_requests / search_network first.\n" +
|
|
2873
|
+
"SEE ALSO: call get_usage_guide(topic=\"network\") for the full network-inspect playbook.",
|
|
2327
2874
|
inputSchema: {
|
|
2328
2875
|
requestId: z.string().describe("The request ID to get details for"),
|
|
2329
2876
|
maxBodyLength: z.coerce
|
|
@@ -2412,7 +2959,14 @@ registerToolWithTelemetry("get_request_details", {
|
|
|
2412
2959
|
});
|
|
2413
2960
|
// Tool: Get network stats
|
|
2414
2961
|
registerToolWithTelemetry("get_network_stats", {
|
|
2415
|
-
description: "Get statistics about captured network requests: counts by method, status code, and domain
|
|
2962
|
+
description: "Get statistics about captured network requests: counts by method, status code, and domain.\n" +
|
|
2963
|
+
"PURPOSE: High-level view of traffic shape — totals, error counts, average duration, and top domains — without scanning every request.\n" +
|
|
2964
|
+
"WHEN TO USE: To quickly answer \"is there a spike in 5xx?\", \"which domain is chattiest?\", or to sanity-check capture is working.\n" +
|
|
2965
|
+
"WORKFLOW: get_network_stats -> if errors > 0: get_network_requests(status=\"500\") -> get_request_details.\n" +
|
|
2966
|
+
"LIMITATIONS: Reflects only the buffered window (last 200 entries per device). Resets when clear_network is called.\n" +
|
|
2967
|
+
"GOOD: get_network_stats()\n" +
|
|
2968
|
+
"BAD: Using it to find a specific request — use search_network or get_network_requests.\n" +
|
|
2969
|
+
"SEE ALSO: call get_usage_guide(topic=\"network\") for the full network-inspect playbook.",
|
|
2416
2970
|
inputSchema: {
|
|
2417
2971
|
device: z.string().optional().describe("Target device name (substring match). Omit for all devices. Run get_apps to see connected devices.")
|
|
2418
2972
|
}
|
|
@@ -2475,7 +3029,14 @@ registerToolWithTelemetry("get_network_stats", {
|
|
|
2475
3029
|
total += b.size; return total === 0; });
|
|
2476
3030
|
// Tool: Clear network requests
|
|
2477
3031
|
registerToolWithTelemetry("clear_network", {
|
|
2478
|
-
description: "Clear the network request buffer"
|
|
3032
|
+
description: "Clear the network request buffer.\n" +
|
|
3033
|
+
"PURPOSE: Reset the captured request list to isolate new traffic from a specific user action.\n" +
|
|
3034
|
+
"WHEN TO USE: Right before reproducing a bug so the buffer contains only the relevant requests.\n" +
|
|
3035
|
+
"WORKFLOW: clear_network -> trigger action (tap, execute_in_app) -> get_network_requests / search_network.\n" +
|
|
3036
|
+
"LIMITATIONS: Irreversible — cleared requests cannot be recovered. Also clears the SDK's in-app buffer when SDK is present.\n" +
|
|
3037
|
+
"GOOD: clear_network() before a reproduction.\n" +
|
|
3038
|
+
"BAD: Using clear_network as a workaround for stale connections — use scan_metro / ensure_connection instead.\n" +
|
|
3039
|
+
"SEE ALSO: call get_usage_guide(topic=\"network\") for the full network-inspect playbook.",
|
|
2479
3040
|
inputSchema: {
|
|
2480
3041
|
device: z.string().optional().describe("Target device name (substring match). Omit to clear all devices. Run get_apps to see connected devices.")
|
|
2481
3042
|
}
|
|
@@ -2512,7 +3073,14 @@ registerToolWithTelemetry("clear_network", {
|
|
|
2512
3073
|
});
|
|
2513
3074
|
// Tool: Get images from shared image buffer
|
|
2514
3075
|
registerToolWithTelemetry("get_images", {
|
|
2515
|
-
description: "Access the shared image buffer containing screenshots from all tools (ios_screenshot, android_screenshot, ocr_screenshot, tap verification). Returns metadata only by default — use id or groupId+frameIndex to retrieve actual image data. Tap burst verification stores frame groups here when burst=true is used
|
|
3076
|
+
description: "Access the shared image buffer containing screenshots from all tools (ios_screenshot, android_screenshot, ocr_screenshot, tap verification). Returns metadata only by default — use id or groupId+frameIndex to retrieve actual image data. Tap burst verification stores frame groups here when burst=true is used.\n" +
|
|
3077
|
+
"PURPOSE: Retrieve prior screenshots — especially tap burst frames — without re-taking them, for visual diffing or reviewing transient UI states.\n" +
|
|
3078
|
+
"WHEN TO USE: After tap(burst=true) reports transientChangeDetected, or to compare before/after frames without another screenshot round-trip.\n" +
|
|
3079
|
+
"WORKFLOW: tap(burst=true) -> note verification.burstGroupId -> get_images(groupId, frameIndex=N) to inspect individual frames.\n" +
|
|
3080
|
+
"LIMITATIONS: Circular buffer (50 entries) — old images are evicted. Metadata is cheap; fetching image data is not — request specific ids, not bulk.\n" +
|
|
3081
|
+
"GOOD: get_images({ list: true }); get_images({ groupId: \"burst-abc\", frameIndex: 2 })\n" +
|
|
3082
|
+
"BAD: get_images() with no filter when buffer is full — floods context. Use list:true or last:N first.\n" +
|
|
3083
|
+
"SEE ALSO: call get_usage_guide(topic=\"interact\") for the full interaction playbook.",
|
|
2516
3084
|
inputSchema: {
|
|
2517
3085
|
list: z.boolean().optional().describe("List all entries and groups (metadata only, no image data)"),
|
|
2518
3086
|
id: z.string().optional().describe("Retrieve a specific image by ID (returns image data)"),
|
|
@@ -2574,7 +3142,9 @@ registerToolWithTelemetry("get_images", {
|
|
|
2574
3142
|
});
|
|
2575
3143
|
// Tool: Reload the app
|
|
2576
3144
|
registerToolWithTelemetry("reload_app", {
|
|
2577
|
-
description: "Reload the React Native app (triggers JavaScript bundle reload like pressing 'r' in Metro)
|
|
3145
|
+
description: "Reload the React Native app (triggers JavaScript bundle reload like pressing 'r' in Metro).\n" +
|
|
3146
|
+
"WHEN TO USE: Only after native code / config changes or a broken app state — NOT after JS/TS edits (Fast Refresh handles those in 1-2s).\n" +
|
|
3147
|
+
"Will auto-connect to Metro if no connection exists. Note: After reload, the app may take a few seconds to fully restart and become responsive — wait before running other tools. IMPORTANT: React Native has Fast Refresh enabled by default - code changes are automatically applied without needing reload. Only use when: (1) logs/behavior don't reflect code changes after a few seconds, (2) app is in broken/error state, or (3) need to reset app state completely (navigation stack, context, etc.).",
|
|
2578
3148
|
inputSchema: {
|
|
2579
3149
|
device: z.string().optional().describe("Target device name (substring match). Omit for default device. Run get_apps to see connected devices.")
|
|
2580
3150
|
}
|
|
@@ -2607,7 +3177,14 @@ registerToolWithTelemetry("logbox", {
|
|
|
2607
3177
|
'"push" displays a message in the LogBox error banner (visible to the developer watching the device). ' +
|
|
2608
3178
|
'"ignore" adds patterns to suppress future LogBox entries for this session. ' +
|
|
2609
3179
|
'"detect" reads current LogBox state without modifying it. ' +
|
|
2610
|
-
"Only works in __DEV__ mode — LogBox does not exist in production builds
|
|
3180
|
+
"Only works in __DEV__ mode — LogBox does not exist in production builds.\n" +
|
|
3181
|
+
"PURPOSE: Control RN's on-device red/yellow overlay — clear it when it blocks UI, suppress noisy repeats, or push a message back to the developer.\n" +
|
|
3182
|
+
"WHEN TO USE: Screenshot/tap reports LogBox is obstructing the screen, an error banner prevents interaction, or you want to surface info to the dev watching the simulator.\n" +
|
|
3183
|
+
"WORKFLOW: logbox(action=\"detect\") -> if present: logbox(action=\"dismiss\") to read + clear -> continue UI work. Use action=\"ignore\" with patterns to stop repeat noise.\n" +
|
|
3184
|
+
"LIMITATIONS: Dev-only — no effect in production builds. \"push\" at level=\"warning\" won't show a banner unless LogBox is already open.\n" +
|
|
3185
|
+
"GOOD: logbox({ action: \"dismiss\" }); logbox({ action: \"ignore\", patterns: [\"[APOLLO]\"] })\n" +
|
|
3186
|
+
"BAD: Spamming logbox(action=\"push\") for every tool step — annoys the developer.\n" +
|
|
3187
|
+
"SEE ALSO: call get_usage_guide(topic=\"logs\") for the full logs playbook.",
|
|
2611
3188
|
inputSchema: {
|
|
2612
3189
|
action: z.enum(["dismiss", "push", "ignore", "detect"]).describe('Action to perform: "dismiss", "push", "ignore", or "detect"'),
|
|
2613
3190
|
message: z.string().optional().describe('Message to push into LogBox (required when action="push")'),
|
|
@@ -2782,7 +3359,14 @@ registerToolWithTelemetry("logbox", {
|
|
|
2782
3359
|
// ============================================================================
|
|
2783
3360
|
// Tool: Get bundle status
|
|
2784
3361
|
registerToolWithTelemetry("get_bundle_status", {
|
|
2785
|
-
description: "Get the current Metro bundler status including build state and any recent bundling errors. Use this to check if there are compilation/bundling errors that prevent the app from loading
|
|
3362
|
+
description: "Get the current Metro bundler status including build state and any recent bundling errors. Use this to check if there are compilation/bundling errors that prevent the app from loading.\n" +
|
|
3363
|
+
"PURPOSE: Snapshot Metro's current build state (idle / transforming / error) together with any captured errors — a fast \"is the bundler healthy?\" check.\n" +
|
|
3364
|
+
"WHEN TO USE: Before diving into runtime debugging — rules out compile-time failures that would make get_logs and tap pointless.\n" +
|
|
3365
|
+
"WORKFLOW: get_bundle_status -> if errors present: get_bundle_errors for detail -> fix -> clear_bundle_errors.\n" +
|
|
3366
|
+
"LIMITATIONS: Relies on Metro's WebSocket event stream; if Metro isn't running or the connection dropped, status may be stale.\n" +
|
|
3367
|
+
"GOOD: get_bundle_status() at the start of a debug session.\n" +
|
|
3368
|
+
"BAD: Polling every second — Metro events are push-based; just call once and act on the result.\n" +
|
|
3369
|
+
"SEE ALSO: call get_usage_guide(topic=\"bundle\") for the full bundle-check playbook.",
|
|
2786
3370
|
inputSchema: {}
|
|
2787
3371
|
}, async () => {
|
|
2788
3372
|
// Get port from first connected app if available
|
|
@@ -2800,7 +3384,14 @@ registerToolWithTelemetry("get_bundle_status", {
|
|
|
2800
3384
|
});
|
|
2801
3385
|
// Tool: Get bundle errors
|
|
2802
3386
|
registerToolWithTelemetry("get_bundle_errors", {
|
|
2803
|
-
description: "Retrieve captured Metro bundling/compilation errors. These are errors that occur during the bundle build process (import resolution, syntax errors, transform errors) that prevent the app from loading. If no errors are captured but Metro is running without connected apps, automatically falls back to screenshot+OCR to capture the error from the device screen
|
|
3387
|
+
description: "Retrieve captured Metro bundling/compilation errors. These are errors that occur during the bundle build process (import resolution, syntax errors, transform errors) that prevent the app from loading. If no errors are captured but Metro is running without connected apps, automatically falls back to screenshot+OCR to capture the error from the device screen.\n" +
|
|
3388
|
+
"PURPOSE: Surface Metro's build-time failures (not runtime JS errors) that keep the app from booting or hot-reloading.\n" +
|
|
3389
|
+
"WHEN TO USE: App shows the red error screen, refuses to connect, or Fast Refresh stops working after an edit. Also use when get_logs is silent but the app is clearly broken.\n" +
|
|
3390
|
+
"WORKFLOW: get_bundle_status -> get_bundle_errors -> fix source -> clear_bundle_errors -> reload_app.\n" +
|
|
3391
|
+
"LIMITATIONS: Captures errors Metro emits via its WebSocket; the screenshot+OCR fallback requires a booted simulator and the platform param.\n" +
|
|
3392
|
+
"GOOD: get_bundle_errors({ platform: \"ios\" })\n" +
|
|
3393
|
+
"BAD: Using get_bundle_errors to look for runtime TypeErrors — those live in get_logs, not the bundler.\n" +
|
|
3394
|
+
"SEE ALSO: call get_usage_guide(topic=\"bundle\") for the full bundle-check playbook.",
|
|
2804
3395
|
inputSchema: {
|
|
2805
3396
|
maxErrors: z.number().optional().default(10).describe("Maximum number of errors to return (default: 10)"),
|
|
2806
3397
|
platform: z
|
|
@@ -2857,283 +3448,122 @@ registerToolWithTelemetry("get_bundle_errors", {
|
|
|
2857
3448
|
try {
|
|
2858
3449
|
let screenshotResult;
|
|
2859
3450
|
if (platform === "android") {
|
|
2860
|
-
screenshotResult = await androidScreenshot(undefined, deviceId);
|
|
2861
|
-
}
|
|
2862
|
-
else {
|
|
2863
|
-
screenshotResult = await iosScreenshot(undefined, deviceId);
|
|
2864
|
-
}
|
|
2865
|
-
if (!screenshotResult.success || !screenshotResult.data) {
|
|
2866
|
-
return {
|
|
2867
|
-
content: [
|
|
2868
|
-
{
|
|
2869
|
-
type: "text",
|
|
2870
|
-
text: `Bundle Errors (0 captured):\n\nNo bundle errors captured via CDP.\n\nMetro is running on port(s) ${metroState.metroPorts.join(", ")} but no apps are connected (possible bundle error).\n\nScreenshot fallback failed: ${screenshotResult.error || "No image data"}`
|
|
2871
|
-
}
|
|
2872
|
-
]
|
|
2873
|
-
};
|
|
2874
|
-
}
|
|
2875
|
-
// Calculate device pixel ratio for iOS
|
|
2876
|
-
const devicePixelRatio = platform === "ios" && screenshotResult.originalWidth && screenshotResult.originalHeight
|
|
2877
|
-
? inferIOSDevicePixelRatio(screenshotResult.originalWidth, screenshotResult.originalHeight)
|
|
2878
|
-
: 1;
|
|
2879
|
-
// Run OCR on the screenshot
|
|
2880
|
-
const ocrResult = await recognizeText(screenshotResult.data, {
|
|
2881
|
-
scaleFactor: screenshotResult.scaleFactor || 1,
|
|
2882
|
-
platform,
|
|
2883
|
-
devicePixelRatio
|
|
2884
|
-
});
|
|
2885
|
-
if (!ocrResult.success || !ocrResult.fullText) {
|
|
2886
|
-
return {
|
|
2887
|
-
content: [
|
|
2888
|
-
{
|
|
2889
|
-
type: "text",
|
|
2890
|
-
text: `Bundle Errors (0 captured):\n\nNo bundle errors captured via CDP.\n\nMetro is running on port(s) ${metroState.metroPorts.join(", ")} but no apps are connected.\n\nScreenshot captured but OCR found no text. The screen may not show an error message.`
|
|
2891
|
-
}
|
|
2892
|
-
]
|
|
2893
|
-
};
|
|
2894
|
-
}
|
|
2895
|
-
// Parse the OCR text for error information
|
|
2896
|
-
const parsedError = parseErrorScreenText(ocrResult.fullText);
|
|
2897
|
-
if (!parsedError.found || !parsedError.error) {
|
|
2898
|
-
return {
|
|
2899
|
-
content: [
|
|
2900
|
-
{
|
|
2901
|
-
type: "text",
|
|
2902
|
-
text: `Bundle Errors (0 captured):\n\nNo bundle errors captured via CDP.\n\nMetro is running on port(s) ${metroState.metroPorts.join(", ")} but no apps are connected.\n\nScreenshot OCR text:\n${ocrResult.fullText.substring(0, 1000)}${ocrResult.fullText.length > 1000 ? "..." : ""}\n\n(No error pattern detected in text)`
|
|
2903
|
-
}
|
|
2904
|
-
]
|
|
2905
|
-
};
|
|
2906
|
-
}
|
|
2907
|
-
// Add the parsed error to the buffer for future reference
|
|
2908
|
-
bundleErrorBuffer.add(parsedError.error);
|
|
2909
|
-
return {
|
|
2910
|
-
content: [
|
|
2911
|
-
{
|
|
2912
|
-
type: "text",
|
|
2913
|
-
text: `Bundle Errors (1 captured via screenshot fallback):\n\n${formatParsedError(parsedError)}`
|
|
2914
|
-
}
|
|
2915
|
-
]
|
|
2916
|
-
};
|
|
2917
|
-
}
|
|
2918
|
-
catch (fallbackError) {
|
|
2919
|
-
return {
|
|
2920
|
-
content: [
|
|
2921
|
-
{
|
|
2922
|
-
type: "text",
|
|
2923
|
-
text: `Bundle Errors (0 captured):\n\nNo bundle errors captured via CDP.\n\nMetro is running on port(s) ${metroState.metroPorts.join(", ")} but no apps are connected.\n\nScreenshot fallback error: ${fallbackError instanceof Error ? fallbackError.message : String(fallbackError)}`
|
|
2924
|
-
}
|
|
2925
|
-
]
|
|
2926
|
-
};
|
|
2927
|
-
}
|
|
2928
|
-
});
|
|
2929
|
-
// Tool: Clear bundle errors
|
|
2930
|
-
registerToolWithTelemetry("clear_bundle_errors", {
|
|
2931
|
-
description: "Clear the bundle error buffer",
|
|
2932
|
-
inputSchema: {}
|
|
2933
|
-
}, async () => {
|
|
2934
|
-
const count = bundleErrorBuffer.clear();
|
|
2935
|
-
return {
|
|
2936
|
-
content: [
|
|
2937
|
-
{
|
|
2938
|
-
type: "text",
|
|
2939
|
-
text: `Cleared ${count} bundle errors from buffer.`
|
|
2940
|
-
}
|
|
2941
|
-
]
|
|
2942
|
-
};
|
|
2943
|
-
});
|
|
2944
|
-
// ============================================================================
|
|
2945
|
-
// Android Tools
|
|
2946
|
-
// ============================================================================
|
|
2947
|
-
// Tool: List Android devices
|
|
2948
|
-
registerToolWithTelemetry("list_android_devices", {
|
|
2949
|
-
description: "List connected Android devices and emulators via ADB",
|
|
2950
|
-
inputSchema: {}
|
|
2951
|
-
}, async () => {
|
|
2952
|
-
const result = await listAndroidDevices();
|
|
2953
|
-
return {
|
|
2954
|
-
content: [
|
|
2955
|
-
{
|
|
2956
|
-
type: "text",
|
|
2957
|
-
text: result.success ? result.result : `Error: ${result.error}`
|
|
2958
|
-
}
|
|
2959
|
-
],
|
|
2960
|
-
isError: !result.success
|
|
2961
|
-
};
|
|
2962
|
-
});
|
|
2963
|
-
// Tool: Android screenshot
|
|
2964
|
-
registerToolWithTelemetry("android_screenshot", {
|
|
2965
|
-
description: "Take a screenshot from an Android device/emulator. Returns the image plus two lists: (1) pressable elements — the tappable components on screen with ready-to-use pixel tap coordinates, testIDs, and labels; (2) screen layout — all visible React components for context. Prefer tap(text=\"...\") when text is exact and unique; otherwise use tap(x, y) with coordinates from the pressables list — this is the most reliable way to tap icons or visually-identified elements. Use component names from the layout for inspect_component/find_components.",
|
|
2966
|
-
inputSchema: {
|
|
2967
|
-
outputPath: z
|
|
2968
|
-
.string()
|
|
2969
|
-
.optional()
|
|
2970
|
-
.describe("Optional path to save the screenshot. If not provided, saves to temp directory."),
|
|
2971
|
-
deviceId: z
|
|
2972
|
-
.string()
|
|
2973
|
-
.optional()
|
|
2974
|
-
.describe("Optional device ID (from list_android_devices). Uses first available device if not specified.")
|
|
2975
|
-
}
|
|
2976
|
-
}, async ({ outputPath, deviceId }) => {
|
|
2977
|
-
const result = await androidScreenshot(outputPath, deviceId);
|
|
2978
|
-
if (!result.success) {
|
|
2979
|
-
return {
|
|
2980
|
-
content: [
|
|
2981
|
-
{
|
|
2982
|
-
type: "text",
|
|
2983
|
-
text: `Error: ${result.error}`
|
|
2984
|
-
}
|
|
2985
|
-
],
|
|
2986
|
-
isError: true
|
|
2987
|
-
};
|
|
2988
|
-
}
|
|
2989
|
-
// Include image data if available
|
|
2990
|
-
if (result.data) {
|
|
2991
|
-
// Build info text with coordinate conversion guidance
|
|
2992
|
-
const pixelWidth = result.originalWidth || 0;
|
|
2993
|
-
const pixelHeight = result.originalHeight || 0;
|
|
2994
|
-
// Resolve the RN app running on THIS Android device so enrichment
|
|
2995
|
-
// pulls data from the right app (not whichever app is "first").
|
|
2996
|
-
const targetApp = getConnectedAppByAndroidDeviceId(deviceId);
|
|
2997
|
-
const targetDeviceName = targetApp?.deviceInfo.deviceName;
|
|
2998
|
-
// Store screenshot metadata on the matching app (not an arbitrary one)
|
|
2999
|
-
if (targetApp) {
|
|
3000
|
-
targetApp.lastScreenshot = {
|
|
3001
|
-
originalWidth: pixelWidth,
|
|
3002
|
-
originalHeight: pixelHeight,
|
|
3003
|
-
scaleFactor: result.scaleFactor || 1,
|
|
3004
|
-
};
|
|
3005
|
-
}
|
|
3006
|
-
let infoText = `Screenshot captured (${pixelWidth}x${pixelHeight} pixels)`;
|
|
3007
|
-
// Get status bar height for coordinate guidance
|
|
3008
|
-
let statusBarPixels = 63; // Default fallback
|
|
3009
|
-
let statusBarDp = 24;
|
|
3010
|
-
let densityDpi = 440; // Common default
|
|
3011
|
-
try {
|
|
3012
|
-
const statusBarResult = await androidGetStatusBarHeight(deviceId);
|
|
3013
|
-
if (statusBarResult.success && statusBarResult.heightPixels) {
|
|
3014
|
-
statusBarPixels = statusBarResult.heightPixels;
|
|
3015
|
-
statusBarDp = statusBarResult.heightDp || 24;
|
|
3016
|
-
}
|
|
3017
|
-
const densityResult = await androidGetDensity(deviceId);
|
|
3018
|
-
if (densityResult.success && densityResult.density) {
|
|
3019
|
-
densityDpi = densityResult.density;
|
|
3020
|
-
}
|
|
3021
|
-
}
|
|
3022
|
-
catch {
|
|
3023
|
-
// Use defaults
|
|
3024
|
-
}
|
|
3025
|
-
// Enrich with screen layout data (component names + tap coordinates)
|
|
3026
|
-
// Android: density ratio = densityDpi / 160 (160dpi = 1dp = 1px baseline)
|
|
3027
|
-
const androidPixelRatio = densityDpi / 160;
|
|
3028
|
-
// Skip enrichment when no RN app is connected to this Android device —
|
|
3029
|
-
// otherwise we would pull fiber data from a different device's app.
|
|
3030
|
-
const canEnrich = !!targetApp;
|
|
3031
|
-
let pressablesText = null;
|
|
3032
|
-
// Screen Layout tree previously appended here was dropped — it was noisy
|
|
3033
|
-
// (nested Svg/G/Path duplicates). Use get_screen_layout when the tree is needed.
|
|
3034
|
-
// Enrich with pressable elements (filtered list of tappable components, converted to screenshot pixel coords)
|
|
3035
|
-
try {
|
|
3036
|
-
if (!canEnrich)
|
|
3037
|
-
throw new Error("skip");
|
|
3038
|
-
const pressables = await getPressableElements({ device: targetDeviceName });
|
|
3039
|
-
if (pressables.success && pressables.parsedElements && pressables.parsedElements.length > 0) {
|
|
3040
|
-
const screenshotScale = result.scaleFactor || 1;
|
|
3041
|
-
pressablesText = pressables.parsedElements.map((el) => {
|
|
3042
|
-
const px = Math.round((el.center.x * androidPixelRatio) / screenshotScale);
|
|
3043
|
-
const py = Math.round((el.center.y * androidPixelRatio) / screenshotScale);
|
|
3044
|
-
const label = el.accessibilityLabel || el.text || el.testID || (el.intent ? `${el.intent} icon` : el.component);
|
|
3045
|
-
const idPart = el.testID ? ` testID="${el.testID}"` : "";
|
|
3046
|
-
const kindPart = el.isInput ? " [input]" : "";
|
|
3047
|
-
const wrapPart = el.isWrapper ? " [wrapper — skip]" : "";
|
|
3048
|
-
const nearPart = !el.text && !el.accessibilityLabel && el.nearbyText ? ` near "${el.nearbyText}"` : "";
|
|
3049
|
-
return ` (${px}, ${py}) ${el.component}: "${label}"${nearPart}${idPart}${kindPart}${wrapPart}`;
|
|
3050
|
-
}).join("\n");
|
|
3051
|
-
}
|
|
3052
|
-
}
|
|
3053
|
-
catch {
|
|
3054
|
-
// Non-fatal: screenshot works without pressables enrichment
|
|
3055
|
-
}
|
|
3056
|
-
infoText += `\n📱 Android uses PIXELS for all coordinates`;
|
|
3057
|
-
if (result.scaleFactor && result.scaleFactor > 1) {
|
|
3058
|
-
infoText += `\n🖼️ Image was scaled down to fit API limits (scale: ${result.scaleFactor.toFixed(3)})`;
|
|
3059
|
-
infoText += `\n📐 tap() handles coordinate conversion automatically — pass pixel coords from this image directly`;
|
|
3060
|
-
}
|
|
3061
|
-
else {
|
|
3062
|
-
infoText += `\n📐 Screenshot coords = tap coords (no conversion needed)`;
|
|
3063
|
-
}
|
|
3064
|
-
infoText += `\n⚠️ Status bar: ${statusBarPixels}px (${statusBarDp}dp) from top - app content starts below this`;
|
|
3065
|
-
infoText += `\n📊 Display density: ${densityDpi}dpi`;
|
|
3066
|
-
if (pressablesText) {
|
|
3067
|
-
infoText += `\n\n🎯 Pressable elements (ready-to-tap, coordinates in screenshot pixels):`;
|
|
3068
|
-
infoText += `\n${pressablesText}`;
|
|
3069
|
-
infoText += `\n\n💡 Next steps:`;
|
|
3070
|
-
infoText += `\n • tap(text="Button Label") — when text is exact and unique`;
|
|
3071
|
-
infoText += `\n • tap(testID="id") or tap(component="Name") — when you know the identifier`;
|
|
3072
|
-
infoText += `\n • tap(x=<px>, y=<px>) — use coordinates from the pressable elements list above (reliable for icons and ambiguous elements)`;
|
|
3073
|
-
infoText += `\n • get_screen_layout — full component tree when you need more than pressables`;
|
|
3074
|
-
}
|
|
3075
|
-
else {
|
|
3076
|
-
if (!canEnrich && connectedApps.size > 0) {
|
|
3077
|
-
infoText += `\n\nℹ️ Pressable enrichment skipped: no RN app is connected to device ${deviceId ?? "(default)"}.`;
|
|
3078
|
-
infoText += ` ${connectedApps.size} other app(s) are connected on different device(s) — their fiber data was intentionally not used to avoid mismatched output.`;
|
|
3079
|
-
}
|
|
3080
|
-
infoText += `\n\n💡 Next steps:`;
|
|
3081
|
-
infoText += `\n • tap(text="Button Label") — tap element by visible text`;
|
|
3082
|
-
infoText += `\n • tap(x=<px>, y=<px>) — tap at coordinates from this screenshot`;
|
|
3083
|
-
infoText += `\n • android_describe_all — get full UI tree with exact tap coordinates`;
|
|
3084
|
-
infoText += `\n • android_find_element(text="...") — find element coordinates without tapping`;
|
|
3085
|
-
}
|
|
3086
|
-
// Check for LogBox overlay — only on the matching RN app; skip otherwise
|
|
3087
|
-
// to avoid surfacing warnings from a different device's app.
|
|
3088
|
-
if (canEnrich) {
|
|
3089
|
-
try {
|
|
3090
|
-
const logBoxState = await detectLogBox(targetDeviceName);
|
|
3091
|
-
if (logBoxState && logBoxState.total > 0) {
|
|
3092
|
-
infoText += formatLogBoxWarning(logBoxState);
|
|
3093
|
-
}
|
|
3094
|
-
}
|
|
3095
|
-
catch {
|
|
3096
|
-
// Non-fatal: LogBox detection failure should not break screenshot
|
|
3097
|
-
}
|
|
3451
|
+
screenshotResult = await androidScreenshot(undefined, deviceId);
|
|
3098
3452
|
}
|
|
3099
|
-
|
|
3100
|
-
|
|
3101
|
-
|
|
3102
|
-
|
|
3103
|
-
|
|
3104
|
-
|
|
3105
|
-
|
|
3106
|
-
|
|
3107
|
-
|
|
3108
|
-
|
|
3109
|
-
|
|
3453
|
+
else {
|
|
3454
|
+
screenshotResult = await iosScreenshot(undefined, deviceId);
|
|
3455
|
+
}
|
|
3456
|
+
if (!screenshotResult.success || !screenshotResult.data) {
|
|
3457
|
+
return {
|
|
3458
|
+
content: [
|
|
3459
|
+
{
|
|
3460
|
+
type: "text",
|
|
3461
|
+
text: `Bundle Errors (0 captured):\n\nNo bundle errors captured via CDP.\n\nMetro is running on port(s) ${metroState.metroPorts.join(", ")} but no apps are connected (possible bundle error).\n\nScreenshot fallback failed: ${screenshotResult.error || "No image data"}`
|
|
3462
|
+
}
|
|
3463
|
+
]
|
|
3464
|
+
};
|
|
3465
|
+
}
|
|
3466
|
+
// Calculate device pixel ratio for iOS
|
|
3467
|
+
const devicePixelRatio = platform === "ios" && screenshotResult.originalWidth && screenshotResult.originalHeight
|
|
3468
|
+
? inferIOSDevicePixelRatio(screenshotResult.originalWidth, screenshotResult.originalHeight)
|
|
3469
|
+
: 1;
|
|
3470
|
+
// Run OCR on the screenshot
|
|
3471
|
+
const ocrResult = await recognizeText(screenshotResult.data, {
|
|
3472
|
+
scaleFactor: screenshotResult.scaleFactor || 1,
|
|
3473
|
+
platform,
|
|
3474
|
+
devicePixelRatio
|
|
3110
3475
|
});
|
|
3476
|
+
if (!ocrResult.success || !ocrResult.fullText) {
|
|
3477
|
+
return {
|
|
3478
|
+
content: [
|
|
3479
|
+
{
|
|
3480
|
+
type: "text",
|
|
3481
|
+
text: `Bundle Errors (0 captured):\n\nNo bundle errors captured via CDP.\n\nMetro is running on port(s) ${metroState.metroPorts.join(", ")} but no apps are connected.\n\nScreenshot captured but OCR found no text. The screen may not show an error message.`
|
|
3482
|
+
}
|
|
3483
|
+
]
|
|
3484
|
+
};
|
|
3485
|
+
}
|
|
3486
|
+
// Parse the OCR text for error information
|
|
3487
|
+
const parsedError = parseErrorScreenText(ocrResult.fullText);
|
|
3488
|
+
if (!parsedError.found || !parsedError.error) {
|
|
3489
|
+
return {
|
|
3490
|
+
content: [
|
|
3491
|
+
{
|
|
3492
|
+
type: "text",
|
|
3493
|
+
text: `Bundle Errors (0 captured):\n\nNo bundle errors captured via CDP.\n\nMetro is running on port(s) ${metroState.metroPorts.join(", ")} but no apps are connected.\n\nScreenshot OCR text:\n${ocrResult.fullText.substring(0, 1000)}${ocrResult.fullText.length > 1000 ? "..." : ""}\n\n(No error pattern detected in text)`
|
|
3494
|
+
}
|
|
3495
|
+
]
|
|
3496
|
+
};
|
|
3497
|
+
}
|
|
3498
|
+
// Add the parsed error to the buffer for future reference
|
|
3499
|
+
bundleErrorBuffer.add(parsedError.error);
|
|
3111
3500
|
return {
|
|
3112
3501
|
content: [
|
|
3113
3502
|
{
|
|
3114
3503
|
type: "text",
|
|
3115
|
-
text:
|
|
3116
|
-
}
|
|
3504
|
+
text: `Bundle Errors (1 captured via screenshot fallback):\n\n${formatParsedError(parsedError)}`
|
|
3505
|
+
}
|
|
3506
|
+
]
|
|
3507
|
+
};
|
|
3508
|
+
}
|
|
3509
|
+
catch (fallbackError) {
|
|
3510
|
+
return {
|
|
3511
|
+
content: [
|
|
3117
3512
|
{
|
|
3118
|
-
type: "
|
|
3119
|
-
|
|
3120
|
-
mimeType: "image/jpeg"
|
|
3513
|
+
type: "text",
|
|
3514
|
+
text: `Bundle Errors (0 captured):\n\nNo bundle errors captured via CDP.\n\nMetro is running on port(s) ${metroState.metroPorts.join(", ")} but no apps are connected.\n\nScreenshot fallback error: ${fallbackError instanceof Error ? fallbackError.message : String(fallbackError)}`
|
|
3121
3515
|
}
|
|
3122
3516
|
]
|
|
3123
3517
|
};
|
|
3124
3518
|
}
|
|
3519
|
+
});
|
|
3520
|
+
// Tool: Clear bundle errors
|
|
3521
|
+
registerToolWithTelemetry("clear_bundle_errors", {
|
|
3522
|
+
description: "Clear the bundle error buffer.\n" +
|
|
3523
|
+
"PURPOSE: Reset captured Metro build errors after a fix so get_bundle_status/get_bundle_errors reflect the current state.\n" +
|
|
3524
|
+
"WHEN TO USE: After fixing a bundling error and before re-triggering Metro to confirm the error is gone.\n" +
|
|
3525
|
+
"SEE ALSO: call get_usage_guide(topic=\"bundle\") for the full bundle-check playbook.",
|
|
3526
|
+
inputSchema: {}
|
|
3527
|
+
}, async () => {
|
|
3528
|
+
const count = bundleErrorBuffer.clear();
|
|
3125
3529
|
return {
|
|
3126
3530
|
content: [
|
|
3127
3531
|
{
|
|
3128
3532
|
type: "text",
|
|
3129
|
-
text: `
|
|
3533
|
+
text: `Cleared ${count} bundle errors from buffer.`
|
|
3130
3534
|
}
|
|
3131
3535
|
]
|
|
3132
3536
|
};
|
|
3133
3537
|
});
|
|
3538
|
+
// ============================================================================
|
|
3539
|
+
// Android Tools
|
|
3540
|
+
// ============================================================================
|
|
3541
|
+
// Tool: List Android devices
|
|
3542
|
+
registerToolWithTelemetry("list_android_devices", {
|
|
3543
|
+
description: "List connected Android devices and emulators via ADB.\n" +
|
|
3544
|
+
"PURPOSE: Discover which physical devices and emulators are visible to adb so you can pick a target UDID/serial.\n" +
|
|
3545
|
+
"WHEN TO USE: Before android_install_app / android_launch_app, or when a tool reports \"no device\" and you need to confirm visibility.\n" +
|
|
3546
|
+
"SEE ALSO: call get_usage_guide(topic=\"setup\") for the full session-setup playbook.",
|
|
3547
|
+
inputSchema: {}
|
|
3548
|
+
}, async () => {
|
|
3549
|
+
const result = await listAndroidDevices();
|
|
3550
|
+
return {
|
|
3551
|
+
content: [
|
|
3552
|
+
{
|
|
3553
|
+
type: "text",
|
|
3554
|
+
text: result.success ? result.result : `Error: ${result.error}`
|
|
3555
|
+
}
|
|
3556
|
+
],
|
|
3557
|
+
isError: !result.success
|
|
3558
|
+
};
|
|
3559
|
+
});
|
|
3134
3560
|
// Tool: Android install app
|
|
3135
3561
|
registerToolWithTelemetry("android_install_app", {
|
|
3136
|
-
description: "Install an APK on an Android device/emulator"
|
|
3562
|
+
description: "Install an APK on an Android device/emulator" +
|
|
3563
|
+
platformUniqueBanner("installing an Android APK") +
|
|
3564
|
+
"\nPURPOSE: Push a built APK to a connected Android device or emulator via adb." +
|
|
3565
|
+
"\nWHEN TO USE: After producing a fresh build, when switching app variants, or when preparing a clean test run." +
|
|
3566
|
+
"\nSEE ALSO: call get_usage_guide(topic=\"setup\") for the full session-setup playbook.",
|
|
3137
3567
|
inputSchema: {
|
|
3138
3568
|
apkPath: z.string().describe("Path to the APK file to install"),
|
|
3139
3569
|
deviceId: z
|
|
@@ -3165,7 +3595,11 @@ registerToolWithTelemetry("android_install_app", {
|
|
|
3165
3595
|
});
|
|
3166
3596
|
// Tool: Android launch app
|
|
3167
3597
|
registerToolWithTelemetry("android_launch_app", {
|
|
3168
|
-
description: "Launch an app on an Android device/emulator by package name"
|
|
3598
|
+
description: "Launch an app on an Android device/emulator by package name" +
|
|
3599
|
+
platformUniqueBanner("launching an Android app by package name") +
|
|
3600
|
+
"\nPURPOSE: Start an installed Android app by its package (and optional activity) so the next tool calls hit a running process." +
|
|
3601
|
+
"\nWHEN TO USE: After android_install_app, after a force-stop, or when the app isn't foregrounded before interaction." +
|
|
3602
|
+
"\nSEE ALSO: call get_usage_guide(topic=\"setup\") for the full session-setup playbook.",
|
|
3169
3603
|
inputSchema: {
|
|
3170
3604
|
packageName: z.string().describe("Package name of the app (e.g., com.example.myapp)"),
|
|
3171
3605
|
activityName: z
|
|
@@ -3191,573 +3625,225 @@ registerToolWithTelemetry("android_launch_app", {
|
|
|
3191
3625
|
});
|
|
3192
3626
|
// Tool: Android list packages
|
|
3193
3627
|
registerToolWithTelemetry("android_list_packages", {
|
|
3194
|
-
description: "List installed packages on an Android device/emulator"
|
|
3195
|
-
|
|
3196
|
-
|
|
3197
|
-
|
|
3198
|
-
|
|
3199
|
-
.describe("Optional device ID. Uses first available device if not specified."),
|
|
3200
|
-
filter: z.string().optional().describe("Optional filter to search packages by name (case-insensitive)")
|
|
3201
|
-
}
|
|
3202
|
-
}, async ({ deviceId, filter }) => {
|
|
3203
|
-
const result = await androidListPackages(deviceId, filter);
|
|
3204
|
-
return {
|
|
3205
|
-
content: [
|
|
3206
|
-
{
|
|
3207
|
-
type: "text",
|
|
3208
|
-
text: result.success ? result.result : `Error: ${result.error}`
|
|
3209
|
-
}
|
|
3210
|
-
],
|
|
3211
|
-
isError: !result.success
|
|
3212
|
-
};
|
|
3213
|
-
});
|
|
3214
|
-
// ============================================================================
|
|
3215
|
-
// Android UI Input Tools (Phase 2)
|
|
3216
|
-
// ============================================================================
|
|
3217
|
-
// Tool: Android long press
|
|
3218
|
-
registerToolWithTelemetry("android_long_press", {
|
|
3219
|
-
description: "Long press at specific coordinates on an Android device/emulator screen",
|
|
3220
|
-
inputSchema: {
|
|
3221
|
-
x: z.coerce.number().describe("X coordinate in pixels"),
|
|
3222
|
-
y: z.coerce.number().describe("Y coordinate in pixels"),
|
|
3223
|
-
durationMs: z.number().optional().default(1000).describe("Press duration in milliseconds (default: 1000)"),
|
|
3224
|
-
deviceId: z
|
|
3225
|
-
.string()
|
|
3226
|
-
.optional()
|
|
3227
|
-
.describe("Optional device ID. Uses first available device if not specified.")
|
|
3228
|
-
}
|
|
3229
|
-
}, async ({ x, y, durationMs, deviceId }) => {
|
|
3230
|
-
const result = await androidLongPress(x, y, durationMs, deviceId);
|
|
3231
|
-
return {
|
|
3232
|
-
content: [
|
|
3233
|
-
{
|
|
3234
|
-
type: "text",
|
|
3235
|
-
text: result.success ? result.result : `Error: ${result.error}`
|
|
3236
|
-
}
|
|
3237
|
-
],
|
|
3238
|
-
isError: !result.success
|
|
3239
|
-
};
|
|
3240
|
-
});
|
|
3241
|
-
// Tool: Android swipe
|
|
3242
|
-
registerToolWithTelemetry("android_swipe", {
|
|
3243
|
-
description: "Swipe from one point to another on an Android device/emulator screen",
|
|
3244
|
-
inputSchema: {
|
|
3245
|
-
startX: z.coerce.number().describe("Starting X coordinate in pixels"),
|
|
3246
|
-
startY: z.coerce.number().describe("Starting Y coordinate in pixels"),
|
|
3247
|
-
endX: z.coerce.number().describe("Ending X coordinate in pixels"),
|
|
3248
|
-
endY: z.coerce.number().describe("Ending Y coordinate in pixels"),
|
|
3249
|
-
durationMs: z.number().optional().default(300).describe("Swipe duration in milliseconds (default: 300)"),
|
|
3250
|
-
deviceId: z
|
|
3251
|
-
.string()
|
|
3252
|
-
.optional()
|
|
3253
|
-
.describe("Optional device ID. Uses first available device if not specified.")
|
|
3254
|
-
}
|
|
3255
|
-
}, async ({ startX, startY, endX, endY, durationMs, deviceId }) => {
|
|
3256
|
-
const result = await androidSwipe(startX, startY, endX, endY, durationMs, deviceId);
|
|
3257
|
-
return {
|
|
3258
|
-
content: [
|
|
3259
|
-
{
|
|
3260
|
-
type: "text",
|
|
3261
|
-
text: result.success ? result.result : `Error: ${result.error}`
|
|
3262
|
-
}
|
|
3263
|
-
],
|
|
3264
|
-
isError: !result.success
|
|
3265
|
-
};
|
|
3266
|
-
});
|
|
3267
|
-
// Tool: Android input text
|
|
3268
|
-
registerToolWithTelemetry("android_input_text", {
|
|
3269
|
-
description: "Type text on an Android device/emulator. The text will be input at the current focus point (tap an input field first).",
|
|
3270
|
-
inputSchema: {
|
|
3271
|
-
text: z.string().describe("Text to type"),
|
|
3272
|
-
deviceId: z
|
|
3273
|
-
.string()
|
|
3274
|
-
.optional()
|
|
3275
|
-
.describe("Optional device ID. Uses first available device if not specified.")
|
|
3276
|
-
}
|
|
3277
|
-
}, async ({ text, deviceId }) => {
|
|
3278
|
-
const result = await androidInputText(text, deviceId);
|
|
3279
|
-
return {
|
|
3280
|
-
content: [
|
|
3281
|
-
{
|
|
3282
|
-
type: "text",
|
|
3283
|
-
text: result.success ? result.result : `Error: ${result.error}`
|
|
3284
|
-
}
|
|
3285
|
-
],
|
|
3286
|
-
isError: !result.success
|
|
3287
|
-
};
|
|
3288
|
-
});
|
|
3289
|
-
// Tool: Android key event
|
|
3290
|
-
registerToolWithTelemetry("android_key_event", {
|
|
3291
|
-
description: `Send a key event to an Android device/emulator. Common keys: ${Object.keys(ANDROID_KEY_EVENTS).join(", ")}`,
|
|
3292
|
-
inputSchema: {
|
|
3293
|
-
key: z.string().describe(`Key name (${Object.keys(ANDROID_KEY_EVENTS).join(", ")}) or numeric keycode`),
|
|
3294
|
-
deviceId: z
|
|
3295
|
-
.string()
|
|
3296
|
-
.optional()
|
|
3297
|
-
.describe("Optional device ID. Uses first available device if not specified.")
|
|
3298
|
-
}
|
|
3299
|
-
}, async ({ key, deviceId }) => {
|
|
3300
|
-
// Try to parse as number first, otherwise treat as key name
|
|
3301
|
-
const keyCode = /^\d+$/.test(key) ? parseInt(key, 10) : key.toUpperCase();
|
|
3302
|
-
const result = await androidKeyEvent(keyCode, deviceId);
|
|
3303
|
-
return {
|
|
3304
|
-
content: [
|
|
3305
|
-
{
|
|
3306
|
-
type: "text",
|
|
3307
|
-
text: result.success ? result.result : `Error: ${result.error}`
|
|
3308
|
-
}
|
|
3309
|
-
],
|
|
3310
|
-
isError: !result.success
|
|
3311
|
-
};
|
|
3312
|
-
});
|
|
3313
|
-
// Tool: Android get screen size
|
|
3314
|
-
registerToolWithTelemetry("android_get_screen_size", {
|
|
3315
|
-
description: "Get the screen size (resolution) of an Android device/emulator",
|
|
3316
|
-
inputSchema: {
|
|
3317
|
-
deviceId: z
|
|
3318
|
-
.string()
|
|
3319
|
-
.optional()
|
|
3320
|
-
.describe("Optional device ID. Uses first available device if not specified.")
|
|
3321
|
-
}
|
|
3322
|
-
}, async ({ deviceId }) => {
|
|
3323
|
-
const result = await androidGetScreenSize(deviceId);
|
|
3324
|
-
if (!result.success) {
|
|
3325
|
-
return {
|
|
3326
|
-
content: [
|
|
3327
|
-
{
|
|
3328
|
-
type: "text",
|
|
3329
|
-
text: `Error: ${result.error}`
|
|
3330
|
-
}
|
|
3331
|
-
],
|
|
3332
|
-
isError: true
|
|
3333
|
-
};
|
|
3334
|
-
}
|
|
3335
|
-
return {
|
|
3336
|
-
content: [
|
|
3337
|
-
{
|
|
3338
|
-
type: "text",
|
|
3339
|
-
text: `Screen size: ${result.width}x${result.height} pixels`
|
|
3340
|
-
}
|
|
3341
|
-
]
|
|
3342
|
-
};
|
|
3343
|
-
});
|
|
3344
|
-
// ============================================================================
|
|
3345
|
-
// Android Accessibility Tools (UI Hierarchy)
|
|
3346
|
-
// ============================================================================
|
|
3347
|
-
// Tool: Android describe all (UI hierarchy)
|
|
3348
|
-
server.registerTool("android_describe_all", {
|
|
3349
|
-
description: "Get the full UI accessibility tree from the Android device using uiautomator. Returns a hierarchical view of all UI elements with their text, content-description, resource-id, bounds, and tap coordinates.",
|
|
3628
|
+
description: "List installed packages on an Android device/emulator" +
|
|
3629
|
+
platformUniqueBanner("listing installed Android packages") +
|
|
3630
|
+
"\nPURPOSE: Enumerate package names visible to adb so you can confirm installation or pick the right target for android_launch_app." +
|
|
3631
|
+
"\nWHEN TO USE: Before android_launch_app when you don't know the exact package name, or to verify an install succeeded." +
|
|
3632
|
+
"\nSEE ALSO: call get_usage_guide(topic=\"setup\") for the full session-setup playbook.",
|
|
3350
3633
|
inputSchema: {
|
|
3351
3634
|
deviceId: z
|
|
3352
3635
|
.string()
|
|
3353
3636
|
.optional()
|
|
3354
|
-
.describe("Optional device ID. Uses first available device if not specified.")
|
|
3355
|
-
|
|
3356
|
-
}, async ({ deviceId }) => {
|
|
3357
|
-
const result = await androidDescribeAll(deviceId);
|
|
3358
|
-
let text = result.success ? result.formatted : `Error: ${result.error}`;
|
|
3359
|
-
// Check for LogBox overlay
|
|
3360
|
-
if (result.success) {
|
|
3361
|
-
try {
|
|
3362
|
-
const logBoxState = await detectLogBox();
|
|
3363
|
-
if (logBoxState && logBoxState.total > 0) {
|
|
3364
|
-
text += formatLogBoxWarning(logBoxState);
|
|
3365
|
-
}
|
|
3366
|
-
}
|
|
3367
|
-
catch {
|
|
3368
|
-
// Non-fatal: LogBox detection failure should not break describe_all
|
|
3369
|
-
}
|
|
3637
|
+
.describe("Optional device ID. Uses first available device if not specified."),
|
|
3638
|
+
filter: z.string().optional().describe("Optional filter to search packages by name (case-insensitive)")
|
|
3370
3639
|
}
|
|
3640
|
+
}, async ({ deviceId, filter }) => {
|
|
3641
|
+
const result = await androidListPackages(deviceId, filter);
|
|
3371
3642
|
return {
|
|
3372
3643
|
content: [
|
|
3373
3644
|
{
|
|
3374
3645
|
type: "text",
|
|
3375
|
-
text
|
|
3646
|
+
text: result.success ? result.result : `Error: ${result.error}`
|
|
3376
3647
|
}
|
|
3377
3648
|
],
|
|
3378
3649
|
isError: !result.success
|
|
3379
3650
|
};
|
|
3380
3651
|
});
|
|
3381
|
-
//
|
|
3382
|
-
|
|
3383
|
-
|
|
3652
|
+
// ============================================================================
|
|
3653
|
+
// Android UI Input Tools (Phase 2)
|
|
3654
|
+
// ============================================================================
|
|
3655
|
+
// Tool: Android long press
|
|
3656
|
+
registerToolWithTelemetry("android_long_press", {
|
|
3657
|
+
description: "Long press at specific coordinates on an Android device/emulator screen" +
|
|
3658
|
+
platformFallbackBanner("`tap` for short taps; keep android_long_press for long-press gestures specifically") +
|
|
3659
|
+
"\nPURPOSE: Emit a sustained touch at raw pixel coordinates to trigger long-press handlers (context menus, drag starts, multi-select)." +
|
|
3660
|
+
"\nWHEN TO USE: Only when a long-press gesture is required — regular taps should go through `tap`." +
|
|
3661
|
+
"\nSEE ALSO: call get_usage_guide(topic=\"interact\") for the full UI-interaction playbook.",
|
|
3384
3662
|
inputSchema: {
|
|
3385
3663
|
x: z.coerce.number().describe("X coordinate in pixels"),
|
|
3386
3664
|
y: z.coerce.number().describe("Y coordinate in pixels"),
|
|
3665
|
+
durationMs: z.number().optional().default(1000).describe("Press duration in milliseconds (default: 1000)"),
|
|
3387
3666
|
deviceId: z
|
|
3388
3667
|
.string()
|
|
3389
3668
|
.optional()
|
|
3390
3669
|
.describe("Optional device ID. Uses first available device if not specified.")
|
|
3391
3670
|
}
|
|
3392
|
-
}, async ({ x, y, deviceId }) => {
|
|
3393
|
-
const result = await
|
|
3671
|
+
}, async ({ x, y, durationMs, deviceId }) => {
|
|
3672
|
+
const result = await androidLongPress(x, y, durationMs, deviceId);
|
|
3394
3673
|
return {
|
|
3395
3674
|
content: [
|
|
3396
3675
|
{
|
|
3397
3676
|
type: "text",
|
|
3398
|
-
text: result.success ? result.
|
|
3677
|
+
text: result.success ? result.result : `Error: ${result.error}`
|
|
3399
3678
|
}
|
|
3400
3679
|
],
|
|
3401
3680
|
isError: !result.success
|
|
3402
3681
|
};
|
|
3403
3682
|
});
|
|
3404
|
-
// Tool: Android
|
|
3405
|
-
|
|
3406
|
-
description: "
|
|
3407
|
-
|
|
3408
|
-
|
|
3409
|
-
|
|
3410
|
-
|
|
3411
|
-
contentDescContains: z.string().optional().describe("Partial content-description match (case-insensitive)"),
|
|
3412
|
-
resourceId: z
|
|
3413
|
-
.string()
|
|
3414
|
-
.optional()
|
|
3415
|
-
.describe("Resource ID match (e.g., 'com.app:id/button' or just 'button')"),
|
|
3416
|
-
index: z
|
|
3417
|
-
.number()
|
|
3418
|
-
.optional()
|
|
3419
|
-
.describe("If multiple elements match, select the nth one (0-indexed, default: 0)"),
|
|
3420
|
-
deviceId: z
|
|
3421
|
-
.string()
|
|
3422
|
-
.optional()
|
|
3423
|
-
.describe("Optional device ID. Uses first available device if not specified.")
|
|
3424
|
-
}
|
|
3425
|
-
}, async ({ text, textContains, contentDesc, contentDescContains, resourceId, index, deviceId }) => {
|
|
3426
|
-
const result = await androidFindElement({ text, textContains, contentDesc, contentDescContains, resourceId, index }, deviceId);
|
|
3427
|
-
if (!result.success) {
|
|
3428
|
-
return {
|
|
3429
|
-
content: [{ type: "text", text: `Error: ${result.error}` }],
|
|
3430
|
-
isError: true
|
|
3431
|
-
};
|
|
3432
|
-
}
|
|
3433
|
-
if (!result.found) {
|
|
3434
|
-
return {
|
|
3435
|
-
content: [
|
|
3436
|
-
{
|
|
3437
|
-
type: "text",
|
|
3438
|
-
text: result.error || "Element not found"
|
|
3439
|
-
}
|
|
3440
|
-
]
|
|
3441
|
-
};
|
|
3442
|
-
}
|
|
3443
|
-
const el = result.element;
|
|
3444
|
-
const info = [
|
|
3445
|
-
`Found element (${result.matchCount} match${result.matchCount > 1 ? "es" : ""})`,
|
|
3446
|
-
` Text: "${el.text}"`,
|
|
3447
|
-
` Content-desc: "${el.contentDesc}"`,
|
|
3448
|
-
` Resource ID: "${el.resourceId}"`,
|
|
3449
|
-
` Class: ${el.className}`,
|
|
3450
|
-
` Bounds: [${el.bounds.left},${el.bounds.top}][${el.bounds.right},${el.bounds.bottom}]`,
|
|
3451
|
-
` Center (tap coords): (${el.center.x}, ${el.center.y})`,
|
|
3452
|
-
` Clickable: ${el.clickable}, Enabled: ${el.enabled}`
|
|
3453
|
-
].join("\n");
|
|
3454
|
-
return {
|
|
3455
|
-
content: [{ type: "text", text: info }]
|
|
3456
|
-
};
|
|
3457
|
-
});
|
|
3458
|
-
// Tool: Android wait for element
|
|
3459
|
-
server.registerTool("android_wait_for_element", {
|
|
3460
|
-
description: "Wait for a UI element to appear on Android screen. Polls the accessibility tree until the element is found or timeout is reached. Use this FIRST after navigation to ensure screen is ready, then use find_element + tap.",
|
|
3683
|
+
// Tool: Android swipe
|
|
3684
|
+
registerToolWithTelemetry("android_swipe", {
|
|
3685
|
+
description: "Swipe from one point to another on an Android device/emulator screen" +
|
|
3686
|
+
platformFallbackBanner("`tap` for targeted interactions; keep android_swipe for raw-coordinate gestures") +
|
|
3687
|
+
"\nPURPOSE: Perform a raw-coordinate swipe gesture for scrolling, paging, dismissing sheets, or drawer opens on Android." +
|
|
3688
|
+
"\nWHEN TO USE: When you need a gesture rather than a tap — scroll lists, swipe carousels, or pull-to-refresh." +
|
|
3689
|
+
"\nSEE ALSO: call get_usage_guide(topic=\"interact\") for the full UI-interaction playbook.",
|
|
3461
3690
|
inputSchema: {
|
|
3462
|
-
|
|
3463
|
-
|
|
3464
|
-
|
|
3465
|
-
|
|
3466
|
-
|
|
3467
|
-
.string()
|
|
3468
|
-
.optional()
|
|
3469
|
-
.describe("Resource ID match (e.g., 'com.app:id/button' or just 'button')"),
|
|
3470
|
-
index: z
|
|
3471
|
-
.number()
|
|
3472
|
-
.optional()
|
|
3473
|
-
.describe("If multiple elements match, select the nth one (0-indexed, default: 0)"),
|
|
3474
|
-
timeoutMs: z
|
|
3475
|
-
.number()
|
|
3476
|
-
.optional()
|
|
3477
|
-
.default(10000)
|
|
3478
|
-
.describe("Maximum time to wait in milliseconds (default: 10000)"),
|
|
3479
|
-
pollIntervalMs: z
|
|
3480
|
-
.number()
|
|
3481
|
-
.optional()
|
|
3482
|
-
.default(500)
|
|
3483
|
-
.describe("Time between polls in milliseconds (default: 500)"),
|
|
3691
|
+
startX: z.coerce.number().describe("Starting X coordinate in pixels"),
|
|
3692
|
+
startY: z.coerce.number().describe("Starting Y coordinate in pixels"),
|
|
3693
|
+
endX: z.coerce.number().describe("Ending X coordinate in pixels"),
|
|
3694
|
+
endY: z.coerce.number().describe("Ending Y coordinate in pixels"),
|
|
3695
|
+
durationMs: z.number().optional().default(300).describe("Swipe duration in milliseconds (default: 300)"),
|
|
3484
3696
|
deviceId: z
|
|
3485
3697
|
.string()
|
|
3486
3698
|
.optional()
|
|
3487
3699
|
.describe("Optional device ID. Uses first available device if not specified.")
|
|
3488
3700
|
}
|
|
3489
|
-
}, async ({
|
|
3490
|
-
const result = await
|
|
3491
|
-
text,
|
|
3492
|
-
textContains,
|
|
3493
|
-
contentDesc,
|
|
3494
|
-
contentDescContains,
|
|
3495
|
-
resourceId,
|
|
3496
|
-
index,
|
|
3497
|
-
timeoutMs,
|
|
3498
|
-
pollIntervalMs
|
|
3499
|
-
}, deviceId);
|
|
3500
|
-
if (!result.success) {
|
|
3501
|
-
return {
|
|
3502
|
-
content: [{ type: "text", text: `Error: ${result.error}` }],
|
|
3503
|
-
isError: true
|
|
3504
|
-
};
|
|
3505
|
-
}
|
|
3506
|
-
if (!result.found) {
|
|
3507
|
-
return {
|
|
3508
|
-
content: [
|
|
3509
|
-
{
|
|
3510
|
-
type: "text",
|
|
3511
|
-
text: result.timedOut
|
|
3512
|
-
? `Timed out after ${result.elapsedMs}ms - element not found`
|
|
3513
|
-
: result.error || "Element not found"
|
|
3514
|
-
}
|
|
3515
|
-
]
|
|
3516
|
-
};
|
|
3517
|
-
}
|
|
3518
|
-
const el = result.element;
|
|
3519
|
-
const info = [
|
|
3520
|
-
`Element found after ${result.elapsedMs}ms`,
|
|
3521
|
-
` Text: "${el.text}"`,
|
|
3522
|
-
` Content-desc: "${el.contentDesc}"`,
|
|
3523
|
-
` Resource ID: "${el.resourceId}"`,
|
|
3524
|
-
` Center (tap coords): (${el.center.x}, ${el.center.y})`,
|
|
3525
|
-
` Clickable: ${el.clickable}, Enabled: ${el.enabled}`
|
|
3526
|
-
].join("\n");
|
|
3527
|
-
return {
|
|
3528
|
-
content: [{ type: "text", text: info }]
|
|
3529
|
-
};
|
|
3530
|
-
});
|
|
3531
|
-
// ============================================================================
|
|
3532
|
-
// iOS Simulator Tools
|
|
3533
|
-
// ============================================================================
|
|
3534
|
-
// Tool: List iOS simulators
|
|
3535
|
-
registerToolWithTelemetry("list_ios_simulators", {
|
|
3536
|
-
description: "List available iOS simulators",
|
|
3537
|
-
inputSchema: {
|
|
3538
|
-
onlyBooted: z
|
|
3539
|
-
.boolean()
|
|
3540
|
-
.optional()
|
|
3541
|
-
.default(false)
|
|
3542
|
-
.describe("Only show currently running simulators (default: false)")
|
|
3543
|
-
}
|
|
3544
|
-
}, async ({ onlyBooted }) => {
|
|
3545
|
-
const result = await listIOSSimulators(onlyBooted);
|
|
3701
|
+
}, async ({ startX, startY, endX, endY, durationMs, deviceId }) => {
|
|
3702
|
+
const result = await androidSwipe(startX, startY, endX, endY, durationMs, deviceId);
|
|
3546
3703
|
return {
|
|
3547
3704
|
content: [
|
|
3548
3705
|
{
|
|
3549
3706
|
type: "text",
|
|
3550
3707
|
text: result.success ? result.result : `Error: ${result.error}`
|
|
3551
3708
|
}
|
|
3552
|
-
],
|
|
3553
|
-
isError: !result.success
|
|
3554
|
-
};
|
|
3555
|
-
});
|
|
3556
|
-
// Tool:
|
|
3557
|
-
registerToolWithTelemetry("
|
|
3558
|
-
description: "
|
|
3559
|
-
|
|
3560
|
-
|
|
3561
|
-
|
|
3562
|
-
|
|
3563
|
-
|
|
3564
|
-
|
|
3565
|
-
|
|
3566
|
-
|
|
3567
|
-
.
|
|
3568
|
-
|
|
3569
|
-
|
|
3570
|
-
|
|
3571
|
-
|
|
3572
|
-
|
|
3573
|
-
|
|
3574
|
-
|
|
3575
|
-
|
|
3576
|
-
|
|
3577
|
-
}
|
|
3578
|
-
],
|
|
3579
|
-
isError: true
|
|
3580
|
-
};
|
|
3581
|
-
}
|
|
3582
|
-
// Include image data if available
|
|
3583
|
-
if (result.data) {
|
|
3584
|
-
// Build info text with coordinate guidance for iOS
|
|
3585
|
-
const pixelWidth = result.originalWidth || 0;
|
|
3586
|
-
const pixelHeight = result.originalHeight || 0;
|
|
3587
|
-
// Resolve the RN app running on THIS simulator so enrichment pulls
|
|
3588
|
-
// fiber/layout data from the right app (not the first-connected one,
|
|
3589
|
-
// which may belong to a different simulator).
|
|
3590
|
-
const resolvedUdid = udid || (await getActiveOrBootedSimulatorUdid());
|
|
3591
|
-
const targetApp = resolvedUdid ? getConnectedAppBySimulatorUdid(resolvedUdid) : null;
|
|
3592
|
-
const targetDeviceName = targetApp?.deviceInfo.deviceName;
|
|
3593
|
-
// Store screenshot metadata on the matching app (not an arbitrary one)
|
|
3594
|
-
if (targetApp) {
|
|
3595
|
-
targetApp.lastScreenshot = {
|
|
3596
|
-
originalWidth: pixelWidth,
|
|
3597
|
-
originalHeight: pixelHeight,
|
|
3598
|
-
scaleFactor: result.scaleFactor || 1,
|
|
3599
|
-
};
|
|
3600
|
-
}
|
|
3601
|
-
// Try to get actual screen dimensions and safe area from accessibility tree
|
|
3602
|
-
let pointWidth = 0;
|
|
3603
|
-
let pointHeight = 0;
|
|
3604
|
-
let scaleFactor = 3; // Default to 3x for modern iPhones
|
|
3605
|
-
let safeAreaTop = 59; // Default safe area offset
|
|
3606
|
-
try {
|
|
3607
|
-
const describeResult = await iosDescribeAll(udid);
|
|
3608
|
-
if (describeResult.success && describeResult.elements && describeResult.elements.length > 0) {
|
|
3609
|
-
// First element is typically the Application with full screen frame
|
|
3610
|
-
const rootElement = describeResult.elements[0];
|
|
3611
|
-
// Try parsed frame first, then parse AXFrame string
|
|
3612
|
-
if (rootElement.frame) {
|
|
3613
|
-
pointWidth = Math.round(rootElement.frame.width);
|
|
3614
|
-
pointHeight = Math.round(rootElement.frame.height);
|
|
3615
|
-
// The frame.y of the root element indicates where content starts (after status bar)
|
|
3616
|
-
if (rootElement.frame.y > 0) {
|
|
3617
|
-
safeAreaTop = Math.round(rootElement.frame.y);
|
|
3618
|
-
}
|
|
3619
|
-
}
|
|
3620
|
-
else if (rootElement.AXFrame) {
|
|
3621
|
-
// Parse format: "{{x, y}, {width, height}}"
|
|
3622
|
-
const match = rootElement.AXFrame.match(/\{\{([\d.]+),\s*([\d.]+)\},\s*\{([\d.]+),\s*([\d.]+)\}\}/);
|
|
3623
|
-
if (match) {
|
|
3624
|
-
const frameY = parseFloat(match[2]);
|
|
3625
|
-
pointWidth = Math.round(parseFloat(match[3]));
|
|
3626
|
-
pointHeight = Math.round(parseFloat(match[4]));
|
|
3627
|
-
if (frameY > 0) {
|
|
3628
|
-
safeAreaTop = Math.round(frameY);
|
|
3629
|
-
}
|
|
3630
|
-
}
|
|
3631
|
-
}
|
|
3632
|
-
// Calculate actual scale factor
|
|
3633
|
-
if (pointWidth > 0) {
|
|
3634
|
-
scaleFactor = Math.round(pixelWidth / pointWidth);
|
|
3635
|
-
}
|
|
3636
|
-
}
|
|
3637
|
-
}
|
|
3638
|
-
catch {
|
|
3639
|
-
// Fallback: use 3x scale for modern devices
|
|
3640
|
-
}
|
|
3641
|
-
// Fallback if we couldn't get dimensions
|
|
3642
|
-
if (pointWidth === 0) {
|
|
3643
|
-
pointWidth = Math.round(pixelWidth / scaleFactor);
|
|
3644
|
-
pointHeight = Math.round(pixelHeight / scaleFactor);
|
|
3645
|
-
}
|
|
3646
|
-
const safeAreaOffsetPixels = safeAreaTop * scaleFactor;
|
|
3647
|
-
// The Screen Layout tree was previously appended here but produced huge noisy
|
|
3648
|
-
// output (nested Svg/G/Path duplicates). Agents should use get_screen_layout
|
|
3649
|
-
// explicitly when they need the tree. The Pressable elements block below is
|
|
3650
|
-
// the signal most consumers actually want.
|
|
3651
|
-
let pressablesText = null;
|
|
3652
|
-
const canEnrich = !!targetApp;
|
|
3653
|
-
// Enrich with pressable elements (filtered list of tappable components, converted to screenshot pixel coords)
|
|
3654
|
-
try {
|
|
3655
|
-
if (!canEnrich)
|
|
3656
|
-
throw new Error("skip");
|
|
3657
|
-
const pressables = await getPressableElements({ device: targetDeviceName });
|
|
3658
|
-
if (pressables.success && pressables.parsedElements && pressables.parsedElements.length > 0) {
|
|
3659
|
-
const screenshotScale = result.scaleFactor || 1;
|
|
3660
|
-
pressablesText = pressables.parsedElements.map((el) => {
|
|
3661
|
-
// See enrichScreenshotWithLayout: shift y when fiber reports the element
|
|
3662
|
-
// inside the safe-area band (a react-native-screens modal artifact).
|
|
3663
|
-
let centerYPoints = el.center.y;
|
|
3664
|
-
if (safeAreaTop > 0 && centerYPoints < safeAreaTop) {
|
|
3665
|
-
centerYPoints += safeAreaTop;
|
|
3666
|
-
}
|
|
3667
|
-
const px = Math.round((el.center.x * scaleFactor) / screenshotScale);
|
|
3668
|
-
const py = Math.round((centerYPoints * scaleFactor) / screenshotScale);
|
|
3669
|
-
const label = el.accessibilityLabel || el.text || el.testID || (el.intent ? `${el.intent} icon` : el.component);
|
|
3670
|
-
const idPart = el.testID ? ` testID="${el.testID}"` : "";
|
|
3671
|
-
const kindPart = el.isInput ? " [input]" : "";
|
|
3672
|
-
const wrapPart = el.isWrapper ? " [wrapper — skip]" : "";
|
|
3673
|
-
const nearPart = !el.text && !el.accessibilityLabel && el.nearbyText ? ` near "${el.nearbyText}"` : "";
|
|
3674
|
-
return ` (${px}, ${py}) ${el.component}: "${label}"${nearPart}${idPart}${kindPart}${wrapPart}`;
|
|
3675
|
-
}).join("\n");
|
|
3676
|
-
}
|
|
3677
|
-
}
|
|
3678
|
-
catch {
|
|
3679
|
-
// Non-fatal: screenshot works without pressables enrichment
|
|
3680
|
-
}
|
|
3681
|
-
let infoText = `Screenshot captured (${pixelWidth}x${pixelHeight} pixels)`;
|
|
3682
|
-
infoText += `\n📱 iOS screen: ${pointWidth}x${pointHeight} points (${scaleFactor}x scale)`;
|
|
3683
|
-
infoText += `\n📐 tap() handles pixel-to-point conversion automatically — pass pixel coords from this image directly`;
|
|
3684
|
-
infoText += `\n⚠️ Status bar + safe area: ${safeAreaTop} points (${safeAreaOffsetPixels} pixels) from top`;
|
|
3685
|
-
if (result.scaleFactor && result.scaleFactor > 1) {
|
|
3686
|
-
infoText += `\n🖼️ Image was scaled down to fit API limits (scale: ${result.scaleFactor.toFixed(3)})`;
|
|
3687
|
-
}
|
|
3688
|
-
if (pressablesText) {
|
|
3689
|
-
infoText += `\n\n🎯 Pressable elements (ready-to-tap, coordinates in screenshot pixels):`;
|
|
3690
|
-
infoText += `\n${pressablesText}`;
|
|
3691
|
-
infoText += `\n\n💡 Next steps:`;
|
|
3692
|
-
infoText += `\n • tap(text="Button Label") — when text is exact and unique`;
|
|
3693
|
-
infoText += `\n • tap(testID="id") or tap(component="Name") — when you know the identifier`;
|
|
3694
|
-
infoText += `\n • tap(x=<px>, y=<px>) — use coordinates from the pressable elements list above (reliable for icons and ambiguous elements)`;
|
|
3695
|
-
infoText += `\n • get_screen_layout — full component tree when you need more than pressables`;
|
|
3696
|
-
}
|
|
3697
|
-
else {
|
|
3698
|
-
if (!canEnrich && connectedApps.size > 0) {
|
|
3699
|
-
infoText += `\n\nℹ️ Pressable enrichment skipped: no RN app is connected to simulator ${resolvedUdid}.`;
|
|
3700
|
-
infoText += ` ${connectedApps.size} other app(s) are connected on different device(s) — their fiber data was intentionally not used to avoid mismatched output.`;
|
|
3701
|
-
}
|
|
3702
|
-
infoText += `\n\n💡 Next steps:`;
|
|
3703
|
-
infoText += `\n • tap(text="Button Label") — tap element by visible text`;
|
|
3704
|
-
infoText += `\n • tap(x=<px>, y=<px>) — tap at coordinates from this screenshot`;
|
|
3705
|
-
infoText += `\n • ios_describe_all — get full UI tree with exact tap coordinates`;
|
|
3706
|
-
infoText += `\n • ios_find_element(label="...") — find element coordinates without tapping`;
|
|
3707
|
-
}
|
|
3708
|
-
// Check for LogBox overlay — only on the matching RN app; skip otherwise
|
|
3709
|
-
// to avoid surfacing warnings from a different simulator's app.
|
|
3710
|
-
if (canEnrich) {
|
|
3711
|
-
try {
|
|
3712
|
-
const logBoxState = await detectLogBox(targetDeviceName);
|
|
3713
|
-
if (logBoxState && logBoxState.total > 0) {
|
|
3714
|
-
infoText += formatLogBoxWarning(logBoxState);
|
|
3715
|
-
}
|
|
3709
|
+
],
|
|
3710
|
+
isError: !result.success
|
|
3711
|
+
};
|
|
3712
|
+
});
|
|
3713
|
+
// Tool: Android input text
|
|
3714
|
+
registerToolWithTelemetry("android_input_text", {
|
|
3715
|
+
description: "Type text on an Android device/emulator." +
|
|
3716
|
+
platformFallbackBanner("`tap(text=...)` — it auto-focuses TextInput via the fiber tree") +
|
|
3717
|
+
" The text will be input at the current focus point (tap an input field first)." +
|
|
3718
|
+
"\nPURPOSE: Send keystrokes to whichever input currently has focus on Android — the tool does NOT focus a field itself." +
|
|
3719
|
+
"\nWHEN TO USE: Only after an input is already focused, or when `tap(text=...)` on the input didn't take focus for some reason." +
|
|
3720
|
+
"\nSEE ALSO: call get_usage_guide(topic=\"interact\") for the full UI-interaction playbook.",
|
|
3721
|
+
inputSchema: {
|
|
3722
|
+
text: z.string().describe("Text to type"),
|
|
3723
|
+
deviceId: z
|
|
3724
|
+
.string()
|
|
3725
|
+
.optional()
|
|
3726
|
+
.describe("Optional device ID. Uses first available device if not specified.")
|
|
3727
|
+
}
|
|
3728
|
+
}, async ({ text, deviceId }) => {
|
|
3729
|
+
const result = await androidInputText(text, deviceId);
|
|
3730
|
+
return {
|
|
3731
|
+
content: [
|
|
3732
|
+
{
|
|
3733
|
+
type: "text",
|
|
3734
|
+
text: result.success ? result.result : `Error: ${result.error}`
|
|
3716
3735
|
}
|
|
3717
|
-
|
|
3718
|
-
|
|
3736
|
+
],
|
|
3737
|
+
isError: !result.success
|
|
3738
|
+
};
|
|
3739
|
+
});
|
|
3740
|
+
// Tool: Android key event
|
|
3741
|
+
registerToolWithTelemetry("android_key_event", {
|
|
3742
|
+
description: "Send a key event to an Android device/emulator." +
|
|
3743
|
+
platformUniqueBanner("sending Android key events (BACK, HOME, MENU, etc.)") +
|
|
3744
|
+
` Common keys: ${Object.keys(ANDROID_KEY_EVENTS).join(", ")}` +
|
|
3745
|
+
"\nPURPOSE: Dispatch Android system keys (BACK, HOME, MENU, ENTER, DEL, etc.) that aren't reachable via on-screen tap." +
|
|
3746
|
+
"\nWHEN TO USE: Navigate back from a screen, submit a form with ENTER, dismiss the keyboard, or press hardware-style keys during a flow." +
|
|
3747
|
+
"\nSEE ALSO: call get_usage_guide(topic=\"interact\") for the full UI-interaction playbook.",
|
|
3748
|
+
inputSchema: {
|
|
3749
|
+
key: z.string().describe(`Key name (${Object.keys(ANDROID_KEY_EVENTS).join(", ")}) or numeric keycode`),
|
|
3750
|
+
deviceId: z
|
|
3751
|
+
.string()
|
|
3752
|
+
.optional()
|
|
3753
|
+
.describe("Optional device ID. Uses first available device if not specified.")
|
|
3754
|
+
}
|
|
3755
|
+
}, async ({ key, deviceId }) => {
|
|
3756
|
+
// Try to parse as number first, otherwise treat as key name
|
|
3757
|
+
const keyCode = /^\d+$/.test(key) ? parseInt(key, 10) : key.toUpperCase();
|
|
3758
|
+
const result = await androidKeyEvent(keyCode, deviceId);
|
|
3759
|
+
return {
|
|
3760
|
+
content: [
|
|
3761
|
+
{
|
|
3762
|
+
type: "text",
|
|
3763
|
+
text: result.success ? result.result : `Error: ${result.error}`
|
|
3719
3764
|
}
|
|
3720
|
-
|
|
3721
|
-
|
|
3722
|
-
|
|
3723
|
-
|
|
3724
|
-
|
|
3725
|
-
|
|
3726
|
-
|
|
3727
|
-
|
|
3728
|
-
|
|
3729
|
-
|
|
3730
|
-
|
|
3731
|
-
|
|
3732
|
-
|
|
3765
|
+
],
|
|
3766
|
+
isError: !result.success
|
|
3767
|
+
};
|
|
3768
|
+
});
|
|
3769
|
+
// Tool: Android get screen size
|
|
3770
|
+
registerToolWithTelemetry("android_get_screen_size", {
|
|
3771
|
+
description: "Get the screen size (resolution) of an Android device/emulator" +
|
|
3772
|
+
platformUniqueBanner("reading Android device pixel resolution") +
|
|
3773
|
+
"\nPURPOSE: Return the device's pixel width and height so you can compute safe tap/swipe coordinates." +
|
|
3774
|
+
"\nWHEN TO USE: Before scripting raw-coordinate gestures on an unfamiliar device, or when normalizing coordinates across devices." +
|
|
3775
|
+
"\nSEE ALSO: call get_usage_guide(topic=\"interact\") for the full UI-interaction playbook.",
|
|
3776
|
+
inputSchema: {
|
|
3777
|
+
deviceId: z
|
|
3778
|
+
.string()
|
|
3779
|
+
.optional()
|
|
3780
|
+
.describe("Optional device ID. Uses first available device if not specified.")
|
|
3781
|
+
}
|
|
3782
|
+
}, async ({ deviceId }) => {
|
|
3783
|
+
const result = await androidGetScreenSize(deviceId);
|
|
3784
|
+
if (!result.success) {
|
|
3733
3785
|
return {
|
|
3734
3786
|
content: [
|
|
3735
3787
|
{
|
|
3736
3788
|
type: "text",
|
|
3737
|
-
text:
|
|
3738
|
-
},
|
|
3739
|
-
{
|
|
3740
|
-
type: "image",
|
|
3741
|
-
data: result.data.toString("base64"),
|
|
3742
|
-
mimeType: "image/jpeg"
|
|
3789
|
+
text: `Error: ${result.error}`
|
|
3743
3790
|
}
|
|
3744
|
-
]
|
|
3791
|
+
],
|
|
3792
|
+
isError: true
|
|
3745
3793
|
};
|
|
3746
3794
|
}
|
|
3747
3795
|
return {
|
|
3748
3796
|
content: [
|
|
3749
3797
|
{
|
|
3750
3798
|
type: "text",
|
|
3751
|
-
text: `
|
|
3799
|
+
text: `Screen size: ${result.width}x${result.height} pixels`
|
|
3752
3800
|
}
|
|
3753
3801
|
]
|
|
3754
3802
|
};
|
|
3755
3803
|
});
|
|
3804
|
+
// ============================================================================
|
|
3805
|
+
// Android Accessibility Tools (UI Hierarchy)
|
|
3806
|
+
// ============================================================================
|
|
3807
|
+
// ============================================================================
|
|
3808
|
+
// iOS Simulator Tools
|
|
3809
|
+
// ============================================================================
|
|
3810
|
+
// Tool: List iOS simulators
|
|
3811
|
+
registerToolWithTelemetry("list_ios_simulators", {
|
|
3812
|
+
description: "List available iOS simulators.\n" +
|
|
3813
|
+
"PURPOSE: Enumerate installed iOS simulators with their UDIDs and boot state so you can boot or install into the right one.\n" +
|
|
3814
|
+
"WHEN TO USE: Before ios_boot_simulator / ios_install_app, or when you need a UDID for a specific device name.\n" +
|
|
3815
|
+
"SEE ALSO: call get_usage_guide(topic=\"setup\") for the full session-setup playbook.",
|
|
3816
|
+
inputSchema: {
|
|
3817
|
+
onlyBooted: z
|
|
3818
|
+
.boolean()
|
|
3819
|
+
.optional()
|
|
3820
|
+
.default(false)
|
|
3821
|
+
.describe("Only show currently running simulators (default: false)")
|
|
3822
|
+
}
|
|
3823
|
+
}, async ({ onlyBooted }) => {
|
|
3824
|
+
const result = await listIOSSimulators(onlyBooted);
|
|
3825
|
+
return {
|
|
3826
|
+
content: [
|
|
3827
|
+
{
|
|
3828
|
+
type: "text",
|
|
3829
|
+
text: result.success ? result.result : `Error: ${result.error}`
|
|
3830
|
+
}
|
|
3831
|
+
],
|
|
3832
|
+
isError: !result.success
|
|
3833
|
+
};
|
|
3834
|
+
});
|
|
3756
3835
|
// Tool: OCR Screenshot - Extract text with coordinates from screenshot
|
|
3757
3836
|
registerToolWithTelemetry("ocr_screenshot", {
|
|
3758
3837
|
description: "RECOMMENDED: Use this tool FIRST when you need to find and tap UI elements. Takes a screenshot and extracts all visible text with tap-ready coordinates using OCR. " +
|
|
3759
3838
|
"ADVANTAGES over accessibility trees: (1) Works on ANY visible text regardless of accessibility labels, (2) Returns ready-to-use tapX/tapY coordinates - no conversion needed, (3) Faster than parsing accessibility hierarchies, (4) Works consistently across iOS and Android. " +
|
|
3760
|
-
"USE THIS FOR: Finding buttons, labels, menu items, tab bars, or any text you need to tap. Simply find the text in the results and use its tapX/tapY with the tap command
|
|
3839
|
+
"USE THIS FOR: Finding buttons, labels, menu items, tab bars, or any text you need to tap. Simply find the text in the results and use its tapX/tapY with the tap command.\n" +
|
|
3840
|
+
"PURPOSE: Visually locate text on screen and return coordinates safe to pass straight into tap.\n" +
|
|
3841
|
+
"WHEN TO USE: Non-RN surfaces, third-party WebViews, accessibility-poor screens, or when fiber/testID strategies have failed.\n" +
|
|
3842
|
+
"WORKFLOW: ocr_screenshot(platform=\"ios\") -> scan results for the label -> tap(x=tapX, y=tapY) -> ios_screenshot to verify.\n" +
|
|
3843
|
+
"LIMITATIONS: OCR accuracy degrades on very small or stylized text; icons with no label won't appear — use tap(component=...) instead.\n" +
|
|
3844
|
+
"GOOD: ocr_screenshot({ platform: \"ios\" })\n" +
|
|
3845
|
+
"BAD: ocr_screenshot used just to view the screen — plain ios_screenshot / android_screenshot is cheaper when you don't need OCR text.\n" +
|
|
3846
|
+
"SEE ALSO: call get_usage_guide(topic=\"interact\") for the full device-interaction playbook.",
|
|
3761
3847
|
inputSchema: {
|
|
3762
3848
|
platform: z.enum(["ios", "android"]).describe("Platform to capture screenshot from"),
|
|
3763
3849
|
deviceId: z
|
|
@@ -3874,7 +3960,11 @@ registerToolWithTelemetry("ocr_screenshot", {
|
|
|
3874
3960
|
});
|
|
3875
3961
|
// Tool: iOS install app
|
|
3876
3962
|
registerToolWithTelemetry("ios_install_app", {
|
|
3877
|
-
description: "Install an app bundle (.app) on an iOS simulator"
|
|
3963
|
+
description: "Install an app bundle (.app) on an iOS simulator" +
|
|
3964
|
+
platformUniqueBanner("installing an iOS .app/.ipa bundle") +
|
|
3965
|
+
"\nPURPOSE: Deploy a built .app bundle onto a booted iOS simulator via simctl." +
|
|
3966
|
+
"\nWHEN TO USE: After producing a fresh simulator build, when switching app variants, or when preparing a clean test run." +
|
|
3967
|
+
"\nSEE ALSO: call get_usage_guide(topic=\"setup\") for the full session-setup playbook.",
|
|
3878
3968
|
inputSchema: {
|
|
3879
3969
|
appPath: z.string().describe("Path to the .app bundle to install"),
|
|
3880
3970
|
udid: z.string().optional().describe("Optional simulator UDID. Uses booted simulator if not specified.")
|
|
@@ -3893,7 +3983,11 @@ registerToolWithTelemetry("ios_install_app", {
|
|
|
3893
3983
|
});
|
|
3894
3984
|
// Tool: iOS launch app
|
|
3895
3985
|
registerToolWithTelemetry("ios_launch_app", {
|
|
3896
|
-
description: "Launch an app on an iOS simulator by bundle ID"
|
|
3986
|
+
description: "Launch an app on an iOS simulator by bundle ID" +
|
|
3987
|
+
platformUniqueBanner("launching an iOS app by bundle ID") +
|
|
3988
|
+
"\nPURPOSE: Start an installed iOS app by its bundle ID so the next tool calls hit a running process." +
|
|
3989
|
+
"\nWHEN TO USE: After ios_install_app, after ios_terminate_app, or when the app isn't foregrounded before interaction." +
|
|
3990
|
+
"\nSEE ALSO: call get_usage_guide(topic=\"setup\") for the full session-setup playbook.",
|
|
3897
3991
|
inputSchema: {
|
|
3898
3992
|
bundleId: z.string().describe("Bundle ID of the app (e.g., com.example.myapp)"),
|
|
3899
3993
|
udid: z.string().optional().describe("Optional simulator UDID. Uses booted simulator if not specified.")
|
|
@@ -3912,7 +4006,14 @@ registerToolWithTelemetry("ios_launch_app", {
|
|
|
3912
4006
|
});
|
|
3913
4007
|
// Tool: iOS open URL
|
|
3914
4008
|
registerToolWithTelemetry("ios_open_url", {
|
|
3915
|
-
description: "Open a URL in the iOS simulator (opens in default handler or Safari)"
|
|
4009
|
+
description: "Open a URL in the iOS simulator (opens in default handler or Safari).\n" +
|
|
4010
|
+
"PURPOSE: Drive an iOS simulator into a deep link or universal link entry point so you can exercise routing from an external entry.\n" +
|
|
4011
|
+
"WHEN TO USE: Testing deep-link handlers, universal link routing, OAuth/SSO callback URLs, or any flow that enters the app via a URL.\n" +
|
|
4012
|
+
"WORKFLOW: ios_boot_simulator -> ios_launch_app (or have the app running) -> ios_open_url -> ios_screenshot / get_screen_layout to verify the target screen rendered.\n" +
|
|
4013
|
+
"GOOD: ios_open_url(url=\"myapp://product/42\") to land directly on a product screen.\n" +
|
|
4014
|
+
"BAD: ios_open_url(url=\"...\") used as a substitute for in-app navigation when the user would normally tap — prefer `tap` for normal interaction flows.\n" +
|
|
4015
|
+
"SEE ALSO: call get_usage_guide(topic=\"interact\") for the full UI-interaction playbook." +
|
|
4016
|
+
platformUniqueBanner("testing iOS deep links or universal links"),
|
|
3916
4017
|
inputSchema: {
|
|
3917
4018
|
url: z.string().describe("URL to open (e.g., https://example.com or myapp://path)"),
|
|
3918
4019
|
udid: z.string().optional().describe("Optional simulator UDID. Uses booted simulator if not specified.")
|
|
@@ -3931,7 +4032,11 @@ registerToolWithTelemetry("ios_open_url", {
|
|
|
3931
4032
|
});
|
|
3932
4033
|
// Tool: iOS terminate app
|
|
3933
4034
|
registerToolWithTelemetry("ios_terminate_app", {
|
|
3934
|
-
description: "Terminate a running app on an iOS simulator"
|
|
4035
|
+
description: "Terminate a running app on an iOS simulator" +
|
|
4036
|
+
platformUniqueBanner("force-terminating an iOS app") +
|
|
4037
|
+
"\nPURPOSE: Force-kill an iOS app process so the next launch starts from a cold state." +
|
|
4038
|
+
"\nWHEN TO USE: To reset app state fully (beyond what reload_app does), or before reinstalling a new build." +
|
|
4039
|
+
"\nSEE ALSO: call get_usage_guide(topic=\"setup\") for the full session-setup playbook.",
|
|
3935
4040
|
inputSchema: {
|
|
3936
4041
|
bundleId: z.string().describe("Bundle ID of the app to terminate"),
|
|
3937
4042
|
udid: z.string().optional().describe("Optional simulator UDID. Uses booted simulator if not specified.")
|
|
@@ -3950,7 +4055,12 @@ registerToolWithTelemetry("ios_terminate_app", {
|
|
|
3950
4055
|
});
|
|
3951
4056
|
// Tool: iOS boot simulator
|
|
3952
4057
|
registerToolWithTelemetry("ios_boot_simulator", {
|
|
3953
|
-
description: "Boot an iOS simulator by UDID
|
|
4058
|
+
description: "Boot an iOS simulator by UDID.\n" +
|
|
4059
|
+
"PURPOSE: Bring a specific simulator online so you can install/launch an app in it.\n" +
|
|
4060
|
+
"WHEN TO USE: At session start when no simulator is running, or after switching between device models.\n" +
|
|
4061
|
+
"SEE ALSO: call get_usage_guide(topic=\"setup\") for the full session-setup playbook." +
|
|
4062
|
+
platformUniqueBanner("booting an iOS simulator") +
|
|
4063
|
+
" Use list_ios_simulators to find available simulators.",
|
|
3954
4064
|
inputSchema: {
|
|
3955
4065
|
udid: z.string().describe("UDID of the simulator to boot (from list_ios_simulators)")
|
|
3956
4066
|
}
|
|
@@ -3971,57 +4081,14 @@ registerToolWithTelemetry("ios_boot_simulator", {
|
|
|
3971
4081
|
// Recommended: AXe — brew install cameroncooke/axe/axe (set IOS_DRIVER=axe)
|
|
3972
4082
|
// Alternative: IDB — brew install idb-companion (default)
|
|
3973
4083
|
// ============================================================================
|
|
3974
|
-
// Tool: iOS swipe
|
|
3975
|
-
server.registerTool("ios_swipe", {
|
|
3976
|
-
description: "Swipe gesture on an iOS simulator screen. Pass screenshot pixel coordinates directly — the same values returned by ios_screenshot pressable elements and screen layout. Coordinates are auto-converted to native iOS points internally. Requires an iOS UI driver: AXe (recommended: brew install cameroncooke/axe/axe) or IDB (brew install idb-companion).",
|
|
3977
|
-
inputSchema: {
|
|
3978
|
-
startX: z.coerce.number().describe("Starting X coordinate in screenshot pixels (from ios_screenshot output)"),
|
|
3979
|
-
startY: z.coerce.number().describe("Starting Y coordinate in screenshot pixels (from ios_screenshot output)"),
|
|
3980
|
-
endX: z.coerce.number().describe("Ending X coordinate in screenshot pixels (from ios_screenshot output)"),
|
|
3981
|
-
endY: z.coerce.number().describe("Ending Y coordinate in screenshot pixels (from ios_screenshot output)"),
|
|
3982
|
-
duration: z.coerce.number().optional().describe("Optional swipe duration in seconds"),
|
|
3983
|
-
delta: z.coerce.number().optional().describe("Optional delta between touch events (step size)"),
|
|
3984
|
-
udid: z.string().optional().describe("Optional simulator UDID. Uses booted simulator if not specified.")
|
|
3985
|
-
}
|
|
3986
|
-
}, async ({ startX, startY, endX, endY, duration, delta, udid }) => {
|
|
3987
|
-
const dpr = await getDevicePixelRatio(udid);
|
|
3988
|
-
const firstApp = connectedApps.values().next().value;
|
|
3989
|
-
const scaleFactor = firstApp?.lastScreenshot?.scaleFactor ?? 1;
|
|
3990
|
-
const start = convertScreenshotToTapCoords(startX, startY, "ios", dpr, scaleFactor);
|
|
3991
|
-
const end = convertScreenshotToTapCoords(endX, endY, "ios", dpr, scaleFactor);
|
|
3992
|
-
const result = await iosSwipe(start.x, start.y, end.x, end.y, { duration, delta, udid });
|
|
3993
|
-
return {
|
|
3994
|
-
content: [
|
|
3995
|
-
{
|
|
3996
|
-
type: "text",
|
|
3997
|
-
text: result.success ? result.result : `Error: ${result.error}`
|
|
3998
|
-
}
|
|
3999
|
-
],
|
|
4000
|
-
isError: !result.success
|
|
4001
|
-
};
|
|
4002
|
-
});
|
|
4003
|
-
// Tool: iOS input text
|
|
4004
|
-
server.registerTool("ios_input_text", {
|
|
4005
|
-
description: "Type text into the active input field on an iOS simulator. Requires an iOS UI driver: AXe (recommended: brew install cameroncooke/axe/axe) or IDB (brew install idb-companion).",
|
|
4006
|
-
inputSchema: {
|
|
4007
|
-
text: z.string().describe("Text to type into the active input field"),
|
|
4008
|
-
udid: z.string().optional().describe("Optional simulator UDID. Uses booted simulator if not specified.")
|
|
4009
|
-
}
|
|
4010
|
-
}, async ({ text, udid }) => {
|
|
4011
|
-
const result = await iosInputText(text, udid);
|
|
4012
|
-
return {
|
|
4013
|
-
content: [
|
|
4014
|
-
{
|
|
4015
|
-
type: "text",
|
|
4016
|
-
text: result.success ? result.result : `Error: ${result.error}`
|
|
4017
|
-
}
|
|
4018
|
-
],
|
|
4019
|
-
isError: !result.success
|
|
4020
|
-
};
|
|
4021
|
-
});
|
|
4022
4084
|
// Tool: iOS button
|
|
4023
4085
|
server.registerTool("ios_button", {
|
|
4024
|
-
description: "Press a hardware button on an iOS simulator.
|
|
4086
|
+
description: "Press a hardware button on an iOS simulator." +
|
|
4087
|
+
platformUniqueBanner("pressing iOS hardware buttons (HOME, LOCK, SIRI, APPLE_PAY)") +
|
|
4088
|
+
" Requires an iOS UI driver: AXe (recommended: brew install cameroncooke/axe/axe) or IDB (brew install idb-companion)." +
|
|
4089
|
+
"\nPURPOSE: Trigger iOS hardware buttons (HOME, LOCK, SIDE_BUTTON, SIRI, APPLE_PAY) that aren't reachable via on-screen tap." +
|
|
4090
|
+
"\nWHEN TO USE: Send the app to background (HOME), lock the simulator (LOCK), or exercise Siri/Apple Pay flows." +
|
|
4091
|
+
"\nSEE ALSO: call get_usage_guide(topic=\"interact\") for the full UI-interaction playbook.",
|
|
4025
4092
|
inputSchema: {
|
|
4026
4093
|
button: z
|
|
4027
4094
|
.enum(IOS_BUTTON_TYPES)
|
|
@@ -4041,210 +4108,6 @@ server.registerTool("ios_button", {
|
|
|
4041
4108
|
isError: !result.success
|
|
4042
4109
|
};
|
|
4043
4110
|
});
|
|
4044
|
-
// Tool: iOS key event
|
|
4045
|
-
server.registerTool("ios_key_event", {
|
|
4046
|
-
description: "Send a key event to an iOS simulator by keycode. Requires an iOS UI driver: AXe (recommended: brew install cameroncooke/axe/axe) or IDB (brew install idb-companion).",
|
|
4047
|
-
inputSchema: {
|
|
4048
|
-
keycode: z.coerce.number().describe("iOS keycode to send"),
|
|
4049
|
-
duration: z.coerce.number().optional().describe("Optional key press duration in seconds"),
|
|
4050
|
-
udid: z.string().optional().describe("Optional simulator UDID. Uses booted simulator if not specified.")
|
|
4051
|
-
}
|
|
4052
|
-
}, async ({ keycode, duration, udid }) => {
|
|
4053
|
-
const result = await iosKeyEvent(keycode, { duration, udid });
|
|
4054
|
-
return {
|
|
4055
|
-
content: [
|
|
4056
|
-
{
|
|
4057
|
-
type: "text",
|
|
4058
|
-
text: result.success ? result.result : `Error: ${result.error}`
|
|
4059
|
-
}
|
|
4060
|
-
],
|
|
4061
|
-
isError: !result.success
|
|
4062
|
-
};
|
|
4063
|
-
});
|
|
4064
|
-
// Tool: iOS key sequence
|
|
4065
|
-
server.registerTool("ios_key_sequence", {
|
|
4066
|
-
description: "Send a sequence of key events to an iOS simulator. Requires an iOS UI driver: AXe (recommended: brew install cameroncooke/axe/axe) or IDB (brew install idb-companion).",
|
|
4067
|
-
inputSchema: {
|
|
4068
|
-
keycodes: z.array(z.coerce.number()).describe("Array of iOS keycodes to send in sequence"),
|
|
4069
|
-
udid: z.string().optional().describe("Optional simulator UDID. Uses booted simulator if not specified.")
|
|
4070
|
-
}
|
|
4071
|
-
}, async ({ keycodes, udid }) => {
|
|
4072
|
-
const result = await iosKeySequence(keycodes, udid);
|
|
4073
|
-
return {
|
|
4074
|
-
content: [
|
|
4075
|
-
{
|
|
4076
|
-
type: "text",
|
|
4077
|
-
text: result.success ? result.result : `Error: ${result.error}`
|
|
4078
|
-
}
|
|
4079
|
-
],
|
|
4080
|
-
isError: !result.success
|
|
4081
|
-
};
|
|
4082
|
-
});
|
|
4083
|
-
// Tool: iOS describe all (accessibility tree)
|
|
4084
|
-
server.registerTool("ios_describe_all", {
|
|
4085
|
-
description: "Get accessibility information for the entire iOS simulator screen. Returns a nested tree of UI elements with labels, values, and frames. Requires an iOS UI driver: AXe (recommended: brew install cameroncooke/axe/axe) or IDB (brew install idb-companion).",
|
|
4086
|
-
inputSchema: {
|
|
4087
|
-
udid: z.string().optional().describe("Optional simulator UDID. Uses booted simulator if not specified.")
|
|
4088
|
-
}
|
|
4089
|
-
}, async ({ udid }) => {
|
|
4090
|
-
const result = await iosDescribeAll(udid);
|
|
4091
|
-
let text = result.success ? result.result : `Error: ${result.error}`;
|
|
4092
|
-
// Check for LogBox overlay
|
|
4093
|
-
if (result.success) {
|
|
4094
|
-
try {
|
|
4095
|
-
const logBoxState = await detectLogBox();
|
|
4096
|
-
if (logBoxState && logBoxState.total > 0) {
|
|
4097
|
-
text += formatLogBoxWarning(logBoxState);
|
|
4098
|
-
}
|
|
4099
|
-
}
|
|
4100
|
-
catch {
|
|
4101
|
-
// Non-fatal: LogBox detection failure should not break describe_all
|
|
4102
|
-
}
|
|
4103
|
-
}
|
|
4104
|
-
return {
|
|
4105
|
-
content: [
|
|
4106
|
-
{
|
|
4107
|
-
type: "text",
|
|
4108
|
-
text
|
|
4109
|
-
}
|
|
4110
|
-
],
|
|
4111
|
-
isError: !result.success
|
|
4112
|
-
};
|
|
4113
|
-
});
|
|
4114
|
-
// Tool: iOS describe point
|
|
4115
|
-
server.registerTool("ios_describe_point", {
|
|
4116
|
-
description: "Get accessibility information for the UI element at a specific point on the iOS simulator screen. Requires an iOS UI driver: AXe (recommended: brew install cameroncooke/axe/axe) or IDB (brew install idb-companion).",
|
|
4117
|
-
inputSchema: {
|
|
4118
|
-
x: z.coerce.number().describe("X coordinate in pixels"),
|
|
4119
|
-
y: z.coerce.number().describe("Y coordinate in pixels"),
|
|
4120
|
-
udid: z.string().optional().describe("Optional simulator UDID. Uses booted simulator if not specified.")
|
|
4121
|
-
}
|
|
4122
|
-
}, async ({ x, y, udid }) => {
|
|
4123
|
-
const result = await iosDescribePoint(x, y, udid);
|
|
4124
|
-
return {
|
|
4125
|
-
content: [
|
|
4126
|
-
{
|
|
4127
|
-
type: "text",
|
|
4128
|
-
text: result.success ? result.result : `Error: ${result.error}`
|
|
4129
|
-
}
|
|
4130
|
-
],
|
|
4131
|
-
isError: !result.success
|
|
4132
|
-
};
|
|
4133
|
-
});
|
|
4134
|
-
// Tool: iOS find element (no screenshot needed)
|
|
4135
|
-
server.registerTool("ios_find_element", {
|
|
4136
|
-
description: "Find a UI element on iOS simulator by accessibility label or value. Returns element details including tap coordinates. Requires an iOS UI driver: AXe (recommended: brew install cameroncooke/axe/axe) or IDB (brew install idb-companion). Workflow: 1) wait_for_element, 2) find_element, 3) tap with returned coordinates. Prefer this over screenshots for button taps.",
|
|
4137
|
-
inputSchema: {
|
|
4138
|
-
label: z.string().optional().describe("Exact accessibility label match"),
|
|
4139
|
-
labelContains: z.string().optional().describe("Partial label match (case-insensitive)"),
|
|
4140
|
-
value: z.string().optional().describe("Exact accessibility value match"),
|
|
4141
|
-
valueContains: z.string().optional().describe("Partial value match (case-insensitive)"),
|
|
4142
|
-
type: z.string().optional().describe("Element type to match (e.g., 'Button', 'TextField')"),
|
|
4143
|
-
index: z
|
|
4144
|
-
.number()
|
|
4145
|
-
.optional()
|
|
4146
|
-
.describe("If multiple elements match, select the nth one (0-indexed, default: 0)"),
|
|
4147
|
-
udid: z.string().optional().describe("Optional simulator UDID. Uses booted simulator if not specified.")
|
|
4148
|
-
}
|
|
4149
|
-
}, async ({ label, labelContains, value, valueContains, type, index, udid }) => {
|
|
4150
|
-
const result = await iosFindElement({ label, labelContains, value, valueContains, type, index }, udid);
|
|
4151
|
-
if (!result.success) {
|
|
4152
|
-
return {
|
|
4153
|
-
content: [{ type: "text", text: `Error: ${result.error}` }],
|
|
4154
|
-
isError: true
|
|
4155
|
-
};
|
|
4156
|
-
}
|
|
4157
|
-
if (!result.found) {
|
|
4158
|
-
return {
|
|
4159
|
-
content: [
|
|
4160
|
-
{
|
|
4161
|
-
type: "text",
|
|
4162
|
-
text: result.error || "Element not found"
|
|
4163
|
-
}
|
|
4164
|
-
]
|
|
4165
|
-
};
|
|
4166
|
-
}
|
|
4167
|
-
const el = result.element;
|
|
4168
|
-
const info = [
|
|
4169
|
-
`Found element (${result.matchCount} match${result.matchCount > 1 ? "es" : ""})`,
|
|
4170
|
-
` Label: "${el.label}"`,
|
|
4171
|
-
` Value: "${el.value}"`,
|
|
4172
|
-
` Type: ${el.type}`,
|
|
4173
|
-
` Frame: {x: ${el.frame.x}, y: ${el.frame.y}, w: ${el.frame.width}, h: ${el.frame.height}}`,
|
|
4174
|
-
` Center (tap coords): (${el.center.x}, ${el.center.y}) — iOS points. Use tap(text="...") or tap(testID="...") for reliable targeting.`,
|
|
4175
|
-
` Enabled: ${el.enabled}`
|
|
4176
|
-
].join("\n");
|
|
4177
|
-
return {
|
|
4178
|
-
content: [{ type: "text", text: info }]
|
|
4179
|
-
};
|
|
4180
|
-
});
|
|
4181
|
-
// Tool: iOS wait for element
|
|
4182
|
-
server.registerTool("ios_wait_for_element", {
|
|
4183
|
-
description: "Wait for a UI element to appear on iOS simulator. Polls until found or timeout. Requires an iOS UI driver: AXe (recommended: brew install cameroncooke/axe/axe) or IDB (brew install idb-companion). Use this FIRST after navigation to ensure screen is ready, then use find_element + tap.",
|
|
4184
|
-
inputSchema: {
|
|
4185
|
-
label: z.string().optional().describe("Exact accessibility label match"),
|
|
4186
|
-
labelContains: z.string().optional().describe("Partial label match (case-insensitive)"),
|
|
4187
|
-
value: z.string().optional().describe("Exact accessibility value match"),
|
|
4188
|
-
valueContains: z.string().optional().describe("Partial value match (case-insensitive)"),
|
|
4189
|
-
type: z.string().optional().describe("Element type to match (e.g., 'Button', 'TextField')"),
|
|
4190
|
-
index: z
|
|
4191
|
-
.number()
|
|
4192
|
-
.optional()
|
|
4193
|
-
.describe("If multiple elements match, select the nth one (0-indexed, default: 0)"),
|
|
4194
|
-
timeoutMs: z
|
|
4195
|
-
.number()
|
|
4196
|
-
.optional()
|
|
4197
|
-
.default(10000)
|
|
4198
|
-
.describe("Maximum time to wait in milliseconds (default: 10000)"),
|
|
4199
|
-
pollIntervalMs: z
|
|
4200
|
-
.number()
|
|
4201
|
-
.optional()
|
|
4202
|
-
.default(500)
|
|
4203
|
-
.describe("Time between polls in milliseconds (default: 500)"),
|
|
4204
|
-
udid: z.string().optional().describe("Optional simulator UDID. Uses booted simulator if not specified.")
|
|
4205
|
-
}
|
|
4206
|
-
}, async ({ label, labelContains, value, valueContains, type, index, timeoutMs, pollIntervalMs, udid }) => {
|
|
4207
|
-
const result = await iosWaitForElement({
|
|
4208
|
-
label,
|
|
4209
|
-
labelContains,
|
|
4210
|
-
value,
|
|
4211
|
-
valueContains,
|
|
4212
|
-
type,
|
|
4213
|
-
index,
|
|
4214
|
-
timeoutMs,
|
|
4215
|
-
pollIntervalMs
|
|
4216
|
-
}, udid);
|
|
4217
|
-
if (!result.success) {
|
|
4218
|
-
return {
|
|
4219
|
-
content: [{ type: "text", text: `Error: ${result.error}` }],
|
|
4220
|
-
isError: true
|
|
4221
|
-
};
|
|
4222
|
-
}
|
|
4223
|
-
if (!result.found) {
|
|
4224
|
-
return {
|
|
4225
|
-
content: [
|
|
4226
|
-
{
|
|
4227
|
-
type: "text",
|
|
4228
|
-
text: result.timedOut
|
|
4229
|
-
? `Timed out after ${result.elapsedMs}ms - element not found`
|
|
4230
|
-
: result.error || "Element not found"
|
|
4231
|
-
}
|
|
4232
|
-
]
|
|
4233
|
-
};
|
|
4234
|
-
}
|
|
4235
|
-
const el = result.element;
|
|
4236
|
-
const info = [
|
|
4237
|
-
`Element found after ${result.elapsedMs}ms`,
|
|
4238
|
-
` Label: "${el.label}"`,
|
|
4239
|
-
` Value: "${el.value}"`,
|
|
4240
|
-
` Type: ${el.type}`,
|
|
4241
|
-
` Center (tap coords): (${el.center.x}, ${el.center.y}) — iOS points. Use tap(text="...") or tap(testID="...") for reliable targeting.`,
|
|
4242
|
-
` Enabled: ${el.enabled}`
|
|
4243
|
-
].join("\n");
|
|
4244
|
-
return {
|
|
4245
|
-
content: [{ type: "text", text: info }]
|
|
4246
|
-
};
|
|
4247
|
-
});
|
|
4248
4111
|
// Tool: Activate Pro license
|
|
4249
4112
|
registerToolWithTelemetry("activate_license", getActivateLicenseConfig(), handleActivateLicense);
|
|
4250
4113
|
// Tool: Delete account
|
|
@@ -4345,10 +4208,15 @@ async function main() {
|
|
|
4345
4208
|
console.error("[rn-ai-debugger] Server started on stdio");
|
|
4346
4209
|
}
|
|
4347
4210
|
}
|
|
4348
|
-
|
|
4349
|
-
|
|
4350
|
-
|
|
4351
|
-
|
|
4211
|
+
// Skip boot when loaded by unit tests — tests import this module purely to
|
|
4212
|
+
// enumerate `toolRegistry`. Jest sets JEST_WORKER_ID; RN_AI_DEVTOOLS_TEST_MODE
|
|
4213
|
+
// is a manual escape hatch. Production + dev:mcp leave both unset and boot.
|
|
4214
|
+
if (!process.env.RN_AI_DEVTOOLS_TEST_MODE && !process.env.JEST_WORKER_ID) {
|
|
4215
|
+
main().catch((error) => {
|
|
4216
|
+
console.error("[rn-ai-debugger] Fatal error:", error);
|
|
4217
|
+
process.exit(1);
|
|
4218
|
+
});
|
|
4219
|
+
}
|
|
4352
4220
|
// Graceful shutdown: close CDP connections so the slot is freed for other sessions
|
|
4353
4221
|
function gracefulShutdown() {
|
|
4354
4222
|
suppressReconnection();
|