react-native-ai-debugger 1.4.21 ā 1.4.34
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +95 -82
- package/build/__tests__/unit/fiberMatch.test.d.ts +2 -0
- package/build/__tests__/unit/fiberMatch.test.d.ts.map +1 -0
- package/build/__tests__/unit/fiberMatch.test.js +114 -0
- package/build/__tests__/unit/fiberMatch.test.js.map +1 -0
- package/build/__tests__/unit/tap.test.js +113 -1
- package/build/__tests__/unit/tap.test.js.map +1 -1
- package/build/core/android.d.ts.map +1 -1
- package/build/core/android.js +54 -98
- package/build/core/android.js.map +1 -1
- package/build/core/appDetection.d.ts.map +1 -1
- package/build/core/appDetection.js +29 -5
- package/build/core/appDetection.js.map +1 -1
- package/build/core/executor.d.ts +25 -0
- package/build/core/executor.d.ts.map +1 -1
- package/build/core/executor.js +1108 -385
- package/build/core/executor.js.map +1 -1
- package/build/core/feedback.d.ts +18 -0
- package/build/core/feedback.d.ts.map +1 -0
- package/build/core/feedback.js +73 -0
- package/build/core/feedback.js.map +1 -0
- package/build/core/fiberMatch.d.ts +25 -0
- package/build/core/fiberMatch.d.ts.map +1 -0
- package/build/core/fiberMatch.js +35 -0
- package/build/core/fiberMatch.js.map +1 -0
- package/build/core/guides.d.ts.map +1 -1
- package/build/core/guides.js +39 -0
- package/build/core/guides.js.map +1 -1
- package/build/core/index.d.ts +3 -2
- package/build/core/index.d.ts.map +1 -1
- package/build/core/index.js +4 -2
- package/build/core/index.js.map +1 -1
- package/build/core/ios.d.ts.map +1 -1
- package/build/core/ios.js +10 -5
- package/build/core/ios.js.map +1 -1
- package/build/core/license.d.ts.map +1 -1
- package/build/core/license.js +26 -35
- package/build/core/license.js.map +1 -1
- package/build/core/logbox.d.ts +6 -0
- package/build/core/logbox.d.ts.map +1 -1
- package/build/core/logbox.js +24 -0
- package/build/core/logbox.js.map +1 -1
- package/build/core/telemetry.d.ts +3 -0
- package/build/core/telemetry.d.ts.map +1 -1
- package/build/core/telemetry.js +18 -4
- package/build/core/telemetry.js.map +1 -1
- package/build/index.js +221 -54
- package/build/index.js.map +1 -1
- package/build/pro/tap.d.ts +13 -0
- package/build/pro/tap.d.ts.map +1 -1
- package/build/pro/tap.js +47 -14
- package/build/pro/tap.js.map +1 -1
- package/build/tools/accountTools.d.ts.map +1 -1
- package/build/tools/accountTools.js +3 -1
- package/build/tools/accountTools.js.map +1 -1
- package/package.json +1 -2
- package/build/core/httpServer.d.ts +0 -14
- package/build/core/httpServer.d.ts.map +0 -1
- package/build/core/httpServer.js.map +0 -1
- package/build/core/httpServerProcess.d.ts +0 -25
- package/build/core/httpServerProcess.d.ts.map +0 -1
- package/build/core/httpServerProcess.js.map +0 -1
- package/build/core/tap.d.ts +0 -91
- package/build/core/tap.d.ts.map +0 -1
- package/build/core/tap.js +0 -542
- package/build/core/tap.js.map +0 -1
- package/build/httpServerStandalone.d.ts +0 -7
- package/build/httpServerStandalone.d.ts.map +0 -1
- package/build/httpServerStandalone.js.map +0 -1
package/build/index.js
CHANGED
|
@@ -9,13 +9,13 @@ import { getGuideOverview, getGuideByTopic, getAvailableTopics } from "./core/gu
|
|
|
9
9
|
import { getLicenseStatus, getDashboardUrl, getUsageInfo } from "./core/license.js";
|
|
10
10
|
import { API_BASE_URL } from "./core/config.js";
|
|
11
11
|
import { getPostHogClient, identifyIfDevMode, shutdownPostHog } from "./core/posthog.js";
|
|
12
|
-
import { getInstallationId,
|
|
12
|
+
import { getInstallationId, getServerVersion, getPackageName, isDevMode, TELEMETRY_JSONL_PATH, categorizeError } from "./core/telemetry.js";
|
|
13
13
|
import { isSDKInstalled, querySDKNetwork, getSDKNetworkEntry, getSDKNetworkStats, clearSDKNetwork, querySDKConsole, getSDKConsoleStats, clearSDKConsole } from "./core/sdkBridge.js";
|
|
14
14
|
import { tap } from "./pro/tap.js";
|
|
15
15
|
import { getActivateLicenseConfig, handleActivateLicense, getDeleteAccountConfig, handleDeleteAccount, } from "./tools/accountTools.js";
|
|
16
16
|
import { logBuffers, networkBuffers, getLogBuffer, getNetworkBuffer, getTotalLogCount, getConnectedAppByDevice, LogBuffer, NetworkBuffer, bundleErrorBuffer, imageBuffer, connectedApps, scanMetroPorts, fetchDevices, filterDebuggableDevices, connectToDevice, getConnectedApps, executeInApp, listDebugGlobals, inspectGlobal, reloadApp,
|
|
17
17
|
// React Component Inspection
|
|
18
|
-
getComponentTree, getScreenLayout, inspectComponent, findComponents, inspectAtPoint, toggleElementInspector, isInspectorActive, getInspectorSelection, getFirstConnectedApp, getLogs, searchLogs, getLogSummary, getNetworkRequests, searchNetworkRequests, getNetworkStats, formatRequestDetails,
|
|
18
|
+
getComponentTree, getScreenLayout, getPressableElements, inspectComponent, findComponents, inspectAtPoint, toggleElementInspector, isInspectorActive, getInspectorSelection, getFirstConnectedApp, getLogs, searchLogs, getLogSummary, getNetworkRequests, searchNetworkRequests, getNetworkStats, formatRequestDetails,
|
|
19
19
|
// Connection state
|
|
20
20
|
getAllConnectionStates, getAllConnectionMetadata, getRecentGaps, formatDuration, cancelAllReconnectionTimers, cancelReconnectionTimer, clearAllConnectionState, clearAllCDPMessageTimes, suppressReconnection, suppressReconnectionForKey, clearReconnectionSuppression, purgeStaleConnectionsForPorts,
|
|
21
21
|
// Context health tracking
|
|
@@ -38,8 +38,8 @@ androidDescribeAll, androidDescribePoint,
|
|
|
38
38
|
androidFindElement, androidWaitForElement,
|
|
39
39
|
// iOS
|
|
40
40
|
listIOSSimulators, iosScreenshot, iosInstallApp, iosLaunchApp, iosOpenUrl, iosTerminateApp, iosBootSimulator,
|
|
41
|
-
// iOS
|
|
42
|
-
iosTap, iosSwipe, iosInputText, iosButton, iosKeyEvent, iosKeySequence, iosDescribeAll, iosDescribePoint, IOS_BUTTON_TYPES,
|
|
41
|
+
// iOS UI driver tools
|
|
42
|
+
isUiDriverAvailable, iosTap, iosSwipe, iosInputText, iosButton, iosKeyEvent, iosKeySequence, iosDescribeAll, iosDescribePoint, IOS_BUTTON_TYPES,
|
|
43
43
|
// iOS Element Finding (no screenshots)
|
|
44
44
|
iosFindElement, iosWaitForElement,
|
|
45
45
|
// Telemetry
|
|
@@ -47,25 +47,7 @@ initTelemetry, trackToolInvocation, getTargetPlatform,
|
|
|
47
47
|
// Format utilities (TONL)
|
|
48
48
|
formatLogsAsTonl, formatNetworkAsTonl,
|
|
49
49
|
// LogBox detection & control
|
|
50
|
-
detectLogBox, formatLogBoxWarning, dismissLogBox, formatDismissedEntries, pushLogBox, addLogBoxIgnorePatterns, verifyLogPipeline } from "./core/index.js";
|
|
51
|
-
// Legacy package migration: notify once per session via LogBox
|
|
52
|
-
const LEGACY_PACKAGE_NAME = "react-native-ai-debugger";
|
|
53
|
-
let legacyMigrationNotified = false;
|
|
54
|
-
async function notifyLegacyPackageMigration() {
|
|
55
|
-
if (legacyMigrationNotified)
|
|
56
|
-
return;
|
|
57
|
-
if (getPackageName() !== LEGACY_PACKAGE_NAME)
|
|
58
|
-
return;
|
|
59
|
-
legacyMigrationNotified = true;
|
|
60
|
-
try {
|
|
61
|
-
await pushLogBox("You are using the deprecated 'react-native-ai-debugger' package. " +
|
|
62
|
-
"Please update your MCP settings to use 'react-native-ai-devtools' instead. " +
|
|
63
|
-
"See: https://www.npmjs.com/package/react-native-ai-devtools", "warning", true);
|
|
64
|
-
}
|
|
65
|
-
catch {
|
|
66
|
-
// Best-effort ā don't break the connection flow
|
|
67
|
-
}
|
|
68
|
-
}
|
|
50
|
+
detectLogBox, formatLogBoxWarning, dismissLogBox, formatDismissedEntries, pushLogBox, addLogBoxIgnorePatterns, verifyLogPipeline, formatIssueBody, buildGitHubUrl, shouldShowFeedbackHint, markFeedbackHintShown, } from "./core/index.js";
|
|
69
51
|
// Helper: resolve log buffer for a device (or create a merged buffer from all devices)
|
|
70
52
|
function resolveLogBuffer(device) {
|
|
71
53
|
if (device) {
|
|
@@ -173,6 +155,27 @@ function estimateImageTokens(base64Data) {
|
|
|
173
155
|
return Math.ceil(base64Data.length / 4); // fallback
|
|
174
156
|
}
|
|
175
157
|
}
|
|
158
|
+
// Tools that do NOT require an active Metro connection ā excluded from feedback hint trigger
|
|
159
|
+
const NON_METRO_TOOLS = new Set([
|
|
160
|
+
"scan_metro",
|
|
161
|
+
"connect_metro",
|
|
162
|
+
"disconnect_metro",
|
|
163
|
+
"ensure_connection",
|
|
164
|
+
"get_connection_status",
|
|
165
|
+
"get_license_status",
|
|
166
|
+
"activate_license",
|
|
167
|
+
"delete_account",
|
|
168
|
+
"get_usage_guide",
|
|
169
|
+
"get_apps",
|
|
170
|
+
"list_ios_simulators",
|
|
171
|
+
"list_android_devices",
|
|
172
|
+
"ios_boot_simulator",
|
|
173
|
+
"ios_launch_app",
|
|
174
|
+
"android_launch_app",
|
|
175
|
+
"ios_install_app",
|
|
176
|
+
"android_install_app",
|
|
177
|
+
"send_feedback"
|
|
178
|
+
]);
|
|
176
179
|
// Registry for dev meta-tool ā stores handlers and configs for dynamic dispatch
|
|
177
180
|
/* eslint-disable @typescript-eslint/no-explicit-any */
|
|
178
181
|
const toolRegistry = new Map();
|
|
@@ -260,12 +263,20 @@ function registerToolWithTelemetry(toolName, config, handler, emptyResultDetecto
|
|
|
260
263
|
responsePreview = textParts.join("\n").substring(0, 2000);
|
|
261
264
|
}
|
|
262
265
|
}
|
|
266
|
+
// First-install feedback hint ā fires once on first successful Metro-connected tool
|
|
267
|
+
if (!NON_METRO_TOOLS.has(toolName) && shouldShowFeedbackHint()) {
|
|
268
|
+
markFeedbackHintShown();
|
|
269
|
+
// Fire-and-forget ā don't block the tool response
|
|
270
|
+
pushLogBox("Congratulations on your first tool call! If you encounter any issues or have ideas for improvement, ask your AI assistant to call send_feedback. Your feedback helps me make this product better for everyone. Best regards, rn-ai-devtools developer.", "warning", true, "logbox").catch(() => {
|
|
271
|
+
// Non-fatal ā hint delivery failure should not affect tool execution
|
|
272
|
+
});
|
|
273
|
+
}
|
|
263
274
|
return result;
|
|
264
275
|
}
|
|
265
276
|
catch (error) {
|
|
266
277
|
success = false;
|
|
267
278
|
errorMessage = error instanceof Error ? error.message : String(error);
|
|
268
|
-
getPostHogClient()?.captureException(error, getInstallationId(), { tool: toolName, server_version: getServerVersion() });
|
|
279
|
+
getPostHogClient()?.captureException(error, getInstallationId(), { tool: toolName, server_version: getServerVersion(), package_name: getPackageName() });
|
|
269
280
|
throw error;
|
|
270
281
|
}
|
|
271
282
|
finally {
|
|
@@ -278,8 +289,16 @@ function registerToolWithTelemetry(toolName, config, handler, emptyResultDetecto
|
|
|
278
289
|
success,
|
|
279
290
|
duration,
|
|
280
291
|
server_version: getServerVersion(),
|
|
292
|
+
package_name: getPackageName(),
|
|
281
293
|
...(errorMessage && { error: errorMessage.substring(0, 200) }),
|
|
294
|
+
...(errorMessage && { error_category: categorizeError(errorMessage) }),
|
|
282
295
|
...(getTargetPlatform() && { platform: getTargetPlatform() }),
|
|
296
|
+
...(tapStrategy && { tap_strategy: tapStrategy }),
|
|
297
|
+
...(meaningful !== undefined && { meaningful }),
|
|
298
|
+
...(changeRate !== undefined && { change_rate: changeRate }),
|
|
299
|
+
...(iosDriver && { ios_driver: iosDriver }),
|
|
300
|
+
...(emptyResult !== undefined && { empty_result: emptyResult }),
|
|
301
|
+
...(emptyReason && { empty_reason: emptyReason }),
|
|
283
302
|
},
|
|
284
303
|
});
|
|
285
304
|
}
|
|
@@ -424,6 +443,21 @@ registerToolWithTelemetry("scan_metro", {
|
|
|
424
443
|
// Build events connection is optional
|
|
425
444
|
}
|
|
426
445
|
}
|
|
446
|
+
// Proactive check: warn if iOS UI driver is missing
|
|
447
|
+
const hasIosDevice = Array.from(allDeviceNames.values()).some(({ device }) => {
|
|
448
|
+
const title = (device.title || "").toLowerCase();
|
|
449
|
+
return title.includes("iphone") || title.includes("ipad") || title.includes("ios");
|
|
450
|
+
});
|
|
451
|
+
if (hasIosDevice) {
|
|
452
|
+
const uiDriverOk = await isUiDriverAvailable();
|
|
453
|
+
if (!uiDriverOk) {
|
|
454
|
+
results.push("");
|
|
455
|
+
results.push("ā ļø WARNING: iOS UI driver is NOT installed. Tools like tap, ios_swipe, ios_input_text, ios_describe_all, and ios_find_element will fail.");
|
|
456
|
+
results.push(" Install the recommended driver: brew install cameroncooke/axe/axe");
|
|
457
|
+
results.push(" Then set IOS_DRIVER=axe in your MCP server environment.");
|
|
458
|
+
results.push(" Alternative: brew install idb-companion (used by default)");
|
|
459
|
+
}
|
|
460
|
+
}
|
|
427
461
|
return {
|
|
428
462
|
content: [
|
|
429
463
|
{
|
|
@@ -454,7 +488,14 @@ registerToolWithTelemetry("get_apps", {
|
|
|
454
488
|
.map(({ app }, i) => {
|
|
455
489
|
const name = app.deviceInfo.deviceName || app.deviceInfo.title;
|
|
456
490
|
const appId = app.deviceInfo.appId || app.deviceInfo.title.split(" (")[0] || "unknown";
|
|
457
|
-
|
|
491
|
+
const lines = [` ${i + 1}. ${name} ā ${appId} (${app.platform}, port ${app.port})`];
|
|
492
|
+
if (app.appDetection) {
|
|
493
|
+
const d = app.appDetection;
|
|
494
|
+
const version = d.reactNativeVersion !== "unknown" ? `RN ${d.reactNativeVersion}` : "RN unknown";
|
|
495
|
+
const expo = d.expoSdkVersion ? `, Expo SDK ${d.expoSdkVersion}` : "";
|
|
496
|
+
lines.push(` Environment: ${version}, ${d.architecture} arch, ${d.jsEngine}, ${app.platform} ${d.osVersion}${expo}`);
|
|
497
|
+
}
|
|
498
|
+
return lines.join("\n");
|
|
458
499
|
});
|
|
459
500
|
const text = [
|
|
460
501
|
`Connected devices:`,
|
|
@@ -595,6 +636,52 @@ registerToolWithTelemetry("get_license_status", {
|
|
|
595
636
|
content: [{ type: "text", text: lines.join("\n") }],
|
|
596
637
|
};
|
|
597
638
|
});
|
|
639
|
+
// Tool: Send feedback / bug report / feature request
|
|
640
|
+
registerToolWithTelemetry("send_feedback", {
|
|
641
|
+
description: "Create a feedback report (bug, feature request, or general feedback) for the react-native-ai-devtools team. " +
|
|
642
|
+
"Auto-collects environment info. Returns a pre-filled GitHub issue URL and formatted issue body. " +
|
|
643
|
+
"Ask the user to open the URL and paste the body to submit.",
|
|
644
|
+
inputSchema: {
|
|
645
|
+
type: z.enum(["feedback", "feature_request", "bug"]).describe('Type of feedback: "feedback", "feature_request", or "bug"'),
|
|
646
|
+
title: z.string().describe("Short summary (becomes the GitHub issue title)"),
|
|
647
|
+
description: z.string().describe("Detailed description of the feedback, feature request, or bug"),
|
|
648
|
+
workflow_context: z.string().optional().describe("What the user was trying to accomplish when they decided to give feedback")
|
|
649
|
+
}
|
|
650
|
+
}, async ({ type, title, description, workflow_context }) => {
|
|
651
|
+
// Collect environment info
|
|
652
|
+
const serverVersion = getServerVersion();
|
|
653
|
+
const platform = process.platform;
|
|
654
|
+
const deviceType = getTargetPlatform();
|
|
655
|
+
const licenseStatus = getLicenseStatus();
|
|
656
|
+
const env = {
|
|
657
|
+
serverVersion,
|
|
658
|
+
platform,
|
|
659
|
+
deviceType,
|
|
660
|
+
licenseTier: licenseStatus.tier
|
|
661
|
+
};
|
|
662
|
+
const input = { type, title, description, workflowContext: workflow_context };
|
|
663
|
+
const issueBody = formatIssueBody(input, env);
|
|
664
|
+
const githubUrl = buildGitHubUrl(title, type);
|
|
665
|
+
const output = [
|
|
666
|
+
"## Feedback Report Ready",
|
|
667
|
+
"",
|
|
668
|
+
`**GitHub Issue URL:** ${githubUrl}`,
|
|
669
|
+
"",
|
|
670
|
+
"**Issue body to paste:**",
|
|
671
|
+
"",
|
|
672
|
+
"```markdown",
|
|
673
|
+
issueBody,
|
|
674
|
+
"```",
|
|
675
|
+
"",
|
|
676
|
+
"Please ask the user to:",
|
|
677
|
+
"1. Open the GitHub URL above",
|
|
678
|
+
"2. Paste the issue body into the description field",
|
|
679
|
+
"3. Review and submit the issue"
|
|
680
|
+
].join("\n");
|
|
681
|
+
return {
|
|
682
|
+
content: [{ type: "text", text: output }]
|
|
683
|
+
};
|
|
684
|
+
});
|
|
598
685
|
// Tool: Ensure connection health
|
|
599
686
|
registerToolWithTelemetry("ensure_connection", {
|
|
600
687
|
description: "Verify or establish a healthy connection to a React Native app. Use before running commands if connection may be stale, or after navigation/reload. This tool runs a health check and will auto-reconnect if needed.",
|
|
@@ -624,8 +711,6 @@ registerToolWithTelemetry("ensure_connection", {
|
|
|
624
711
|
isError: true
|
|
625
712
|
};
|
|
626
713
|
}
|
|
627
|
-
// Notify legacy package users (fire-and-forget, once per session)
|
|
628
|
-
notifyLegacyPackageMigration();
|
|
629
714
|
const lines = [];
|
|
630
715
|
lines.push("=== Connection Ensured ===\n");
|
|
631
716
|
if (result.connectionInfo) {
|
|
@@ -1372,6 +1457,34 @@ registerToolWithTelemetry("get_screen_layout", {
|
|
|
1372
1457
|
]
|
|
1373
1458
|
};
|
|
1374
1459
|
});
|
|
1460
|
+
// Tool: Get all pressable elements on screen
|
|
1461
|
+
registerToolWithTelemetry("get_pressable_elements", {
|
|
1462
|
+
description: "Find all pressable (onPress) and input (TextInput) elements currently visible on screen. Returns component names, tap-ready center coordinates (in points/dp), text labels, testID, and accessibilityLabel. Useful when you need to tap an icon or button but can't identify it from a screenshot alone. Each element includes hasLabel (true if it contains text) and isInput (true for TextInput fields).",
|
|
1463
|
+
inputSchema: {
|
|
1464
|
+
device: z.string().optional().describe("Target device name (substring match). Omit for default device. Run get_apps to see connected devices.")
|
|
1465
|
+
}
|
|
1466
|
+
}, async ({ device }) => {
|
|
1467
|
+
const result = await getPressableElements({ device });
|
|
1468
|
+
if (!result.success) {
|
|
1469
|
+
return {
|
|
1470
|
+
content: [
|
|
1471
|
+
{
|
|
1472
|
+
type: "text",
|
|
1473
|
+
text: `Error: ${result.error}`
|
|
1474
|
+
}
|
|
1475
|
+
],
|
|
1476
|
+
isError: true
|
|
1477
|
+
};
|
|
1478
|
+
}
|
|
1479
|
+
return {
|
|
1480
|
+
content: [
|
|
1481
|
+
{
|
|
1482
|
+
type: "text",
|
|
1483
|
+
text: result.result || "No pressable elements found."
|
|
1484
|
+
}
|
|
1485
|
+
]
|
|
1486
|
+
};
|
|
1487
|
+
});
|
|
1375
1488
|
// Tool: Inspect a specific component by name
|
|
1376
1489
|
registerToolWithTelemetry("inspect_component", {
|
|
1377
1490
|
description: "Inspect a specific React component by name. **DRILL-DOWN TOOL**: Use after get_screen_layout or find_components to identify which component to inspect. Returns props, style, state (hooks), and optionally children tree. Use childrenDepth to control how deep nested children go.",
|
|
@@ -1485,12 +1598,13 @@ registerToolWithTelemetry("find_components", {
|
|
|
1485
1598
|
registerToolWithTelemetry("tap", {
|
|
1486
1599
|
description: "Tap a UI element. Automatically tries multiple strategies: fiber tree (React), accessibility tree (native), and OCR (visual). " +
|
|
1487
1600
|
"Auto-detects platform (iOS/Android). For coordinates, pass raw pixel values from screenshot ā tap handles all coordinate conversion internally (iOS pixel-to-point, scale factor adjustments).\n\n" +
|
|
1601
|
+
"PREREQUISITE: For iOS, an iOS UI driver must be installed ā AXe (recommended: brew install cameroncooke/axe/axe) or IDB (brew install idb-companion). Without it, accessibility and coordinate strategies will fail.\n\n" +
|
|
1488
1602
|
"IMPORTANT: When both iOS and Android devices are connected, specify platform explicitly to avoid tapping on the wrong device.\n\n" +
|
|
1489
1603
|
"Examples:\n" +
|
|
1490
1604
|
"- tap(text=\"Submit\") ā finds and taps element with matching text\n" +
|
|
1491
1605
|
"- tap(testID=\"login-btn\") ā finds by testID\n" +
|
|
1492
1606
|
"- tap(component=\"HamburgerIcon\") ā finds by React component name\n" +
|
|
1493
|
-
"- tap(x=300, y=600) ā taps at pixel coordinates
|
|
1607
|
+
"- tap(x=300, y=600) ā taps at pixel coordinates; use coordinates from the pressable elements list returned by ios_screenshot/android_screenshot for reliable icon and ambiguous-element taps\n" +
|
|
1494
1608
|
"- tap(text=\"Menu\", strategy=\"ocr\") ā forces OCR strategy only\n" +
|
|
1495
1609
|
"- tap(x=300, y=600, native=true, platform=\"android\") ā taps directly via ADB/simctl without React Native connection\n\n" +
|
|
1496
1610
|
"Returns a post-tap screenshot by default (set screenshot=false to disable). " +
|
|
@@ -2668,7 +2782,7 @@ registerToolWithTelemetry("list_android_devices", {
|
|
|
2668
2782
|
});
|
|
2669
2783
|
// Tool: Android screenshot
|
|
2670
2784
|
registerToolWithTelemetry("android_screenshot", {
|
|
2671
|
-
description: "Take a screenshot from an Android device/emulator. Returns the image
|
|
2785
|
+
description: "Take a screenshot from an Android device/emulator. Returns the image plus two lists: (1) pressable elements ā the tappable components on screen with ready-to-use pixel tap coordinates, testIDs, and labels; (2) screen layout ā all visible React components for context. Prefer tap(text=\"...\") when text is exact and unique; otherwise use tap(x, y) with coordinates from the pressables list ā this is the most reliable way to tap icons or visually-identified elements. Use component names from the layout for inspect_component/find_components.",
|
|
2672
2786
|
inputSchema: {
|
|
2673
2787
|
outputPath: z
|
|
2674
2788
|
.string()
|
|
@@ -2735,6 +2849,26 @@ registerToolWithTelemetry("android_screenshot", {
|
|
|
2735
2849
|
catch {
|
|
2736
2850
|
// Non-fatal: screenshot works without layout enrichment
|
|
2737
2851
|
}
|
|
2852
|
+
// Enrich with pressable elements (filtered list of tappable components, converted to screenshot pixel coords)
|
|
2853
|
+
let pressablesText = null;
|
|
2854
|
+
try {
|
|
2855
|
+
const pressables = await getPressableElements({});
|
|
2856
|
+
if (pressables.success && pressables.parsedElements && pressables.parsedElements.length > 0) {
|
|
2857
|
+
const screenshotScale = result.scaleFactor || 1;
|
|
2858
|
+
pressablesText = pressables.parsedElements.map((el) => {
|
|
2859
|
+
const px = Math.round((el.center.x * androidPixelRatio) / screenshotScale);
|
|
2860
|
+
const py = Math.round((el.center.y * androidPixelRatio) / screenshotScale);
|
|
2861
|
+
const label = el.accessibilityLabel || el.text || el.testID || el.component;
|
|
2862
|
+
const idPart = el.testID ? ` testID="${el.testID}"` : "";
|
|
2863
|
+
const kindPart = el.isInput ? " [input]" : "";
|
|
2864
|
+
const wrapPart = el.isWrapper ? " [wrapper ā skip]" : "";
|
|
2865
|
+
return ` (${px}, ${py}) ${el.component}: "${label}"${idPart}${kindPart}${wrapPart}`;
|
|
2866
|
+
}).join("\n");
|
|
2867
|
+
}
|
|
2868
|
+
}
|
|
2869
|
+
catch {
|
|
2870
|
+
// Non-fatal: screenshot works without pressables enrichment
|
|
2871
|
+
}
|
|
2738
2872
|
infoText += `\nš± Android uses PIXELS for all coordinates`;
|
|
2739
2873
|
if (result.scaleFactor && result.scaleFactor > 1) {
|
|
2740
2874
|
infoText += `\nš¼ļø Image was scaled down to fit API limits (scale: ${result.scaleFactor.toFixed(3)})`;
|
|
@@ -2745,14 +2879,20 @@ registerToolWithTelemetry("android_screenshot", {
|
|
|
2745
2879
|
}
|
|
2746
2880
|
infoText += `\nā ļø Status bar: ${statusBarPixels}px (${statusBarDp}dp) from top - app content starts below this`;
|
|
2747
2881
|
infoText += `\nš Display density: ${densityDpi}dpi`;
|
|
2882
|
+
if (pressablesText) {
|
|
2883
|
+
infoText += `\n\nšÆ Pressable elements (ready-to-tap, coordinates in screenshot pixels):`;
|
|
2884
|
+
infoText += `\n${pressablesText}`;
|
|
2885
|
+
}
|
|
2748
2886
|
if (layoutText) {
|
|
2749
|
-
infoText += `\n\nš Screen Layout (components
|
|
2887
|
+
infoText += `\n\nš Screen Layout (all components, tap coordinates in pixels):`;
|
|
2750
2888
|
infoText += `\n${layoutText}`;
|
|
2889
|
+
}
|
|
2890
|
+
if (pressablesText || layoutText) {
|
|
2751
2891
|
infoText += `\n\nš” Next steps:`;
|
|
2752
|
-
infoText += `\n ⢠tap(text="Button Label") ā
|
|
2753
|
-
infoText += `\n ⢠tap(testID="id") or tap(component="Name") ā
|
|
2754
|
-
infoText += `\n ⢠tap(x=<px>, y=<px>) ā use
|
|
2755
|
-
infoText += `\n ⢠inspect_component("ComponentName") ā inspect a component from the layout
|
|
2892
|
+
infoText += `\n ⢠tap(text="Button Label") ā when text is exact and unique`;
|
|
2893
|
+
infoText += `\n ⢠tap(testID="id") or tap(component="Name") ā when you know the identifier`;
|
|
2894
|
+
infoText += `\n ⢠tap(x=<px>, y=<px>) ā use coordinates from the pressable elements list above (reliable for icons and ambiguous elements)`;
|
|
2895
|
+
infoText += `\n ⢠inspect_component("ComponentName") ā inspect a component from the layout`;
|
|
2756
2896
|
}
|
|
2757
2897
|
else {
|
|
2758
2898
|
infoText += `\n\nš” Next steps:`;
|
|
@@ -3230,7 +3370,7 @@ registerToolWithTelemetry("list_ios_simulators", {
|
|
|
3230
3370
|
});
|
|
3231
3371
|
// Tool: iOS screenshot
|
|
3232
3372
|
registerToolWithTelemetry("ios_screenshot", {
|
|
3233
|
-
description: "Take a screenshot from an iOS simulator. Returns the image
|
|
3373
|
+
description: "Take a screenshot from an iOS simulator. Returns the image plus two lists: (1) pressable elements ā the tappable components on screen with ready-to-use pixel tap coordinates, testIDs, and labels; (2) screen layout ā all visible React components for context. Prefer tap(text=\"...\") when text is exact and unique; otherwise use tap(x, y) with coordinates from the pressables list ā this is the most reliable way to tap icons or visually-identified elements. Use component names from the layout for inspect_component/find_components.",
|
|
3234
3374
|
inputSchema: {
|
|
3235
3375
|
outputPath: z
|
|
3236
3376
|
.string()
|
|
@@ -3322,6 +3462,26 @@ registerToolWithTelemetry("ios_screenshot", {
|
|
|
3322
3462
|
catch {
|
|
3323
3463
|
// Non-fatal: screenshot works without layout enrichment
|
|
3324
3464
|
}
|
|
3465
|
+
// Enrich with pressable elements (filtered list of tappable components, converted to screenshot pixel coords)
|
|
3466
|
+
let pressablesText = null;
|
|
3467
|
+
try {
|
|
3468
|
+
const pressables = await getPressableElements({});
|
|
3469
|
+
if (pressables.success && pressables.parsedElements && pressables.parsedElements.length > 0) {
|
|
3470
|
+
const screenshotScale = result.scaleFactor || 1;
|
|
3471
|
+
pressablesText = pressables.parsedElements.map((el) => {
|
|
3472
|
+
const px = Math.round((el.center.x * scaleFactor) / screenshotScale);
|
|
3473
|
+
const py = Math.round((el.center.y * scaleFactor) / screenshotScale);
|
|
3474
|
+
const label = el.accessibilityLabel || el.text || el.testID || el.component;
|
|
3475
|
+
const idPart = el.testID ? ` testID="${el.testID}"` : "";
|
|
3476
|
+
const kindPart = el.isInput ? " [input]" : "";
|
|
3477
|
+
const wrapPart = el.isWrapper ? " [wrapper ā skip]" : "";
|
|
3478
|
+
return ` (${px}, ${py}) ${el.component}: "${label}"${idPart}${kindPart}${wrapPart}`;
|
|
3479
|
+
}).join("\n");
|
|
3480
|
+
}
|
|
3481
|
+
}
|
|
3482
|
+
catch {
|
|
3483
|
+
// Non-fatal: screenshot works without pressables enrichment
|
|
3484
|
+
}
|
|
3325
3485
|
let infoText = `Screenshot captured (${pixelWidth}x${pixelHeight} pixels)`;
|
|
3326
3486
|
infoText += `\nš± iOS screen: ${pointWidth}x${pointHeight} points (${scaleFactor}x scale)`;
|
|
3327
3487
|
infoText += `\nš tap() handles pixel-to-point conversion automatically ā pass pixel coords from this image directly`;
|
|
@@ -3329,14 +3489,20 @@ registerToolWithTelemetry("ios_screenshot", {
|
|
|
3329
3489
|
if (result.scaleFactor && result.scaleFactor > 1) {
|
|
3330
3490
|
infoText += `\nš¼ļø Image was scaled down to fit API limits (scale: ${result.scaleFactor.toFixed(3)})`;
|
|
3331
3491
|
}
|
|
3492
|
+
if (pressablesText) {
|
|
3493
|
+
infoText += `\n\nšÆ Pressable elements (ready-to-tap, coordinates in screenshot pixels):`;
|
|
3494
|
+
infoText += `\n${pressablesText}`;
|
|
3495
|
+
}
|
|
3332
3496
|
if (layoutText) {
|
|
3333
|
-
infoText += `\n\nš Screen Layout (components
|
|
3497
|
+
infoText += `\n\nš Screen Layout (all components, tap coordinates in pixels):`;
|
|
3334
3498
|
infoText += `\n${layoutText}`;
|
|
3499
|
+
}
|
|
3500
|
+
if (pressablesText || layoutText) {
|
|
3335
3501
|
infoText += `\n\nš” Next steps:`;
|
|
3336
|
-
infoText += `\n ⢠tap(text="Button Label") ā
|
|
3337
|
-
infoText += `\n ⢠tap(testID="id") or tap(component="Name") ā
|
|
3338
|
-
infoText += `\n ⢠tap(x=<px>, y=<px>) ā use
|
|
3339
|
-
infoText += `\n ⢠inspect_component("ComponentName") ā inspect a component from the layout
|
|
3502
|
+
infoText += `\n ⢠tap(text="Button Label") ā when text is exact and unique`;
|
|
3503
|
+
infoText += `\n ⢠tap(testID="id") or tap(component="Name") ā when you know the identifier`;
|
|
3504
|
+
infoText += `\n ⢠tap(x=<px>, y=<px>) ā use coordinates from the pressable elements list above (reliable for icons and ambiguous elements)`;
|
|
3505
|
+
infoText += `\n ⢠inspect_component("ComponentName") ā inspect a component from the layout`;
|
|
3340
3506
|
}
|
|
3341
3507
|
else {
|
|
3342
3508
|
infoText += `\n\nš” Next steps:`;
|
|
@@ -3604,12 +3770,13 @@ registerToolWithTelemetry("ios_boot_simulator", {
|
|
|
3604
3770
|
};
|
|
3605
3771
|
});
|
|
3606
3772
|
// ============================================================================
|
|
3607
|
-
// iOS
|
|
3608
|
-
//
|
|
3773
|
+
// iOS UI Interaction Tools (require an iOS UI driver)
|
|
3774
|
+
// Recommended: AXe ā brew install cameroncooke/axe/axe (set IOS_DRIVER=axe)
|
|
3775
|
+
// Alternative: IDB ā brew install idb-companion (default)
|
|
3609
3776
|
// ============================================================================
|
|
3610
3777
|
// Tool: iOS swipe
|
|
3611
3778
|
server.registerTool("ios_swipe", {
|
|
3612
|
-
description: "Swipe gesture on an iOS simulator screen. Requires
|
|
3779
|
+
description: "Swipe gesture on an iOS simulator screen. Requires an iOS UI driver: AXe (recommended: brew install cameroncooke/axe/axe) or IDB (brew install idb-companion).",
|
|
3613
3780
|
inputSchema: {
|
|
3614
3781
|
startX: z.coerce.number().describe("Starting X coordinate in pixels"),
|
|
3615
3782
|
startY: z.coerce.number().describe("Starting Y coordinate in pixels"),
|
|
@@ -3633,7 +3800,7 @@ server.registerTool("ios_swipe", {
|
|
|
3633
3800
|
});
|
|
3634
3801
|
// Tool: iOS input text
|
|
3635
3802
|
server.registerTool("ios_input_text", {
|
|
3636
|
-
description: "Type text into the active input field on an iOS simulator. Requires
|
|
3803
|
+
description: "Type text into the active input field on an iOS simulator. Requires an iOS UI driver: AXe (recommended: brew install cameroncooke/axe/axe) or IDB (brew install idb-companion).",
|
|
3637
3804
|
inputSchema: {
|
|
3638
3805
|
text: z.string().describe("Text to type into the active input field"),
|
|
3639
3806
|
udid: z.string().optional().describe("Optional simulator UDID. Uses booted simulator if not specified.")
|
|
@@ -3652,7 +3819,7 @@ server.registerTool("ios_input_text", {
|
|
|
3652
3819
|
});
|
|
3653
3820
|
// Tool: iOS button
|
|
3654
3821
|
server.registerTool("ios_button", {
|
|
3655
|
-
description: "Press a hardware button on an iOS simulator. Requires
|
|
3822
|
+
description: "Press a hardware button on an iOS simulator. Requires an iOS UI driver: AXe (recommended: brew install cameroncooke/axe/axe) or IDB (brew install idb-companion).",
|
|
3656
3823
|
inputSchema: {
|
|
3657
3824
|
button: z
|
|
3658
3825
|
.enum(IOS_BUTTON_TYPES)
|
|
@@ -3674,7 +3841,7 @@ server.registerTool("ios_button", {
|
|
|
3674
3841
|
});
|
|
3675
3842
|
// Tool: iOS key event
|
|
3676
3843
|
server.registerTool("ios_key_event", {
|
|
3677
|
-
description: "Send a key event to an iOS simulator by keycode. Requires
|
|
3844
|
+
description: "Send a key event to an iOS simulator by keycode. Requires an iOS UI driver: AXe (recommended: brew install cameroncooke/axe/axe) or IDB (brew install idb-companion).",
|
|
3678
3845
|
inputSchema: {
|
|
3679
3846
|
keycode: z.coerce.number().describe("iOS keycode to send"),
|
|
3680
3847
|
duration: z.coerce.number().optional().describe("Optional key press duration in seconds"),
|
|
@@ -3694,7 +3861,7 @@ server.registerTool("ios_key_event", {
|
|
|
3694
3861
|
});
|
|
3695
3862
|
// Tool: iOS key sequence
|
|
3696
3863
|
server.registerTool("ios_key_sequence", {
|
|
3697
|
-
description: "Send a sequence of key events to an iOS simulator. Requires
|
|
3864
|
+
description: "Send a sequence of key events to an iOS simulator. Requires an iOS UI driver: AXe (recommended: brew install cameroncooke/axe/axe) or IDB (brew install idb-companion).",
|
|
3698
3865
|
inputSchema: {
|
|
3699
3866
|
keycodes: z.array(z.coerce.number()).describe("Array of iOS keycodes to send in sequence"),
|
|
3700
3867
|
udid: z.string().optional().describe("Optional simulator UDID. Uses booted simulator if not specified.")
|
|
@@ -3713,7 +3880,7 @@ server.registerTool("ios_key_sequence", {
|
|
|
3713
3880
|
});
|
|
3714
3881
|
// Tool: iOS describe all (accessibility tree)
|
|
3715
3882
|
server.registerTool("ios_describe_all", {
|
|
3716
|
-
description: "Get accessibility information for the entire iOS simulator screen. Returns a nested tree of UI elements with labels, values, and frames. Requires
|
|
3883
|
+
description: "Get accessibility information for the entire iOS simulator screen. Returns a nested tree of UI elements with labels, values, and frames. Requires an iOS UI driver: AXe (recommended: brew install cameroncooke/axe/axe) or IDB (brew install idb-companion).",
|
|
3717
3884
|
inputSchema: {
|
|
3718
3885
|
udid: z.string().optional().describe("Optional simulator UDID. Uses booted simulator if not specified.")
|
|
3719
3886
|
}
|
|
@@ -3744,7 +3911,7 @@ server.registerTool("ios_describe_all", {
|
|
|
3744
3911
|
});
|
|
3745
3912
|
// Tool: iOS describe point
|
|
3746
3913
|
server.registerTool("ios_describe_point", {
|
|
3747
|
-
description: "Get accessibility information for the UI element at a specific point on the iOS simulator screen. Requires
|
|
3914
|
+
description: "Get accessibility information for the UI element at a specific point on the iOS simulator screen. Requires an iOS UI driver: AXe (recommended: brew install cameroncooke/axe/axe) or IDB (brew install idb-companion).",
|
|
3748
3915
|
inputSchema: {
|
|
3749
3916
|
x: z.coerce.number().describe("X coordinate in pixels"),
|
|
3750
3917
|
y: z.coerce.number().describe("Y coordinate in pixels"),
|
|
@@ -3764,7 +3931,7 @@ server.registerTool("ios_describe_point", {
|
|
|
3764
3931
|
});
|
|
3765
3932
|
// Tool: iOS find element (no screenshot needed)
|
|
3766
3933
|
server.registerTool("ios_find_element", {
|
|
3767
|
-
description: "Find a UI element on iOS simulator by accessibility label or value. Returns element details including tap coordinates. Requires IDB (brew install idb-companion). Workflow: 1) wait_for_element, 2) find_element, 3) tap with returned coordinates. Prefer this over screenshots for button taps.",
|
|
3934
|
+
description: "Find a UI element on iOS simulator by accessibility label or value. Returns element details including tap coordinates. Requires an iOS UI driver: AXe (recommended: brew install cameroncooke/axe/axe) or IDB (brew install idb-companion). Workflow: 1) wait_for_element, 2) find_element, 3) tap with returned coordinates. Prefer this over screenshots for button taps.",
|
|
3768
3935
|
inputSchema: {
|
|
3769
3936
|
label: z.string().optional().describe("Exact accessibility label match"),
|
|
3770
3937
|
labelContains: z.string().optional().describe("Partial label match (case-insensitive)"),
|
|
@@ -3811,7 +3978,7 @@ server.registerTool("ios_find_element", {
|
|
|
3811
3978
|
});
|
|
3812
3979
|
// Tool: iOS wait for element
|
|
3813
3980
|
server.registerTool("ios_wait_for_element", {
|
|
3814
|
-
description: "Wait for a UI element to appear on iOS simulator. Polls until found or timeout. Requires IDB (brew install idb-companion). Use this FIRST after navigation to ensure screen is ready, then use find_element + tap.",
|
|
3981
|
+
description: "Wait for a UI element to appear on iOS simulator. Polls until found or timeout. Requires an iOS UI driver: AXe (recommended: brew install cameroncooke/axe/axe) or IDB (brew install idb-companion). Use this FIRST after navigation to ensure screen is ready, then use find_element + tap.",
|
|
3815
3982
|
inputSchema: {
|
|
3816
3983
|
label: z.string().optional().describe("Exact accessibility label match"),
|
|
3817
3984
|
labelContains: z.string().optional().describe("Partial label match (case-insensitive)"),
|
|
@@ -3966,18 +4133,18 @@ async function main() {
|
|
|
3966
4133
|
res.end("Not found");
|
|
3967
4134
|
});
|
|
3968
4135
|
httpServer.listen(httpPort, () => {
|
|
3969
|
-
console.error(`[rn-ai-
|
|
4136
|
+
console.error(`[rn-ai-debugger] MCP HTTP server listening on http://localhost:${httpPort}/mcp`);
|
|
3970
4137
|
});
|
|
3971
4138
|
}
|
|
3972
4139
|
else {
|
|
3973
4140
|
// Stdio transport mode ā default for production
|
|
3974
4141
|
const transport = new StdioServerTransport();
|
|
3975
4142
|
await server.connect(transport);
|
|
3976
|
-
console.error("[rn-ai-
|
|
4143
|
+
console.error("[rn-ai-debugger] Server started on stdio");
|
|
3977
4144
|
}
|
|
3978
4145
|
}
|
|
3979
4146
|
main().catch((error) => {
|
|
3980
|
-
console.error("[rn-ai-
|
|
4147
|
+
console.error("[rn-ai-debugger] Fatal error:", error);
|
|
3981
4148
|
process.exit(1);
|
|
3982
4149
|
});
|
|
3983
4150
|
// Graceful shutdown: close CDP connections so the slot is freed for other sessions
|