react-native-ai-debugger 1.4.21 ā 1.4.33
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +92 -80
- package/build/__tests__/unit/fiberMatch.test.d.ts +2 -0
- package/build/__tests__/unit/fiberMatch.test.d.ts.map +1 -0
- package/build/__tests__/unit/fiberMatch.test.js +114 -0
- package/build/__tests__/unit/fiberMatch.test.js.map +1 -0
- package/build/__tests__/unit/tap.test.js +113 -1
- package/build/__tests__/unit/tap.test.js.map +1 -1
- package/build/core/android.d.ts.map +1 -1
- package/build/core/android.js +54 -98
- package/build/core/android.js.map +1 -1
- package/build/core/appDetection.d.ts.map +1 -1
- package/build/core/appDetection.js +29 -5
- package/build/core/appDetection.js.map +1 -1
- package/build/core/executor.d.ts +25 -0
- package/build/core/executor.d.ts.map +1 -1
- package/build/core/executor.js +1108 -385
- package/build/core/executor.js.map +1 -1
- package/build/core/feedback.d.ts +18 -0
- package/build/core/feedback.d.ts.map +1 -0
- package/build/core/feedback.js +73 -0
- package/build/core/feedback.js.map +1 -0
- package/build/core/fiberMatch.d.ts +25 -0
- package/build/core/fiberMatch.d.ts.map +1 -0
- package/build/core/fiberMatch.js +35 -0
- package/build/core/fiberMatch.js.map +1 -0
- package/build/core/guides.d.ts.map +1 -1
- package/build/core/guides.js +39 -0
- package/build/core/guides.js.map +1 -1
- package/build/core/index.d.ts +3 -2
- package/build/core/index.d.ts.map +1 -1
- package/build/core/index.js +4 -2
- package/build/core/index.js.map +1 -1
- package/build/core/ios.d.ts.map +1 -1
- package/build/core/ios.js +10 -5
- package/build/core/ios.js.map +1 -1
- package/build/core/license.d.ts.map +1 -1
- package/build/core/license.js +16 -38
- package/build/core/license.js.map +1 -1
- package/build/core/logbox.d.ts +6 -0
- package/build/core/logbox.d.ts.map +1 -1
- package/build/core/logbox.js +24 -0
- package/build/core/logbox.js.map +1 -1
- package/build/core/telemetry.d.ts +3 -1
- package/build/core/telemetry.d.ts.map +1 -1
- package/build/core/telemetry.js +16 -15
- package/build/core/telemetry.js.map +1 -1
- package/build/index.js +219 -53
- package/build/index.js.map +1 -1
- package/build/pro/tap.d.ts +13 -0
- package/build/pro/tap.d.ts.map +1 -1
- package/build/pro/tap.js +47 -14
- package/build/pro/tap.js.map +1 -1
- package/package.json +1 -2
- package/build/core/httpServer.d.ts +0 -14
- package/build/core/httpServer.d.ts.map +0 -1
- package/build/core/httpServer.js.map +0 -1
- package/build/core/httpServerProcess.d.ts +0 -25
- package/build/core/httpServerProcess.d.ts.map +0 -1
- package/build/core/httpServerProcess.js.map +0 -1
- package/build/core/tap.d.ts +0 -91
- package/build/core/tap.d.ts.map +0 -1
- package/build/core/tap.js +0 -542
- package/build/core/tap.js.map +0 -1
- package/build/httpServerStandalone.d.ts +0 -7
- package/build/httpServerStandalone.d.ts.map +0 -1
- package/build/httpServerStandalone.js.map +0 -1
package/build/index.js
CHANGED
|
@@ -9,13 +9,13 @@ import { getGuideOverview, getGuideByTopic, getAvailableTopics } from "./core/gu
|
|
|
9
9
|
import { getLicenseStatus, getDashboardUrl, getUsageInfo } from "./core/license.js";
|
|
10
10
|
import { API_BASE_URL } from "./core/config.js";
|
|
11
11
|
import { getPostHogClient, identifyIfDevMode, shutdownPostHog } from "./core/posthog.js";
|
|
12
|
-
import { getInstallationId,
|
|
12
|
+
import { getInstallationId, getServerVersion, isDevMode, TELEMETRY_JSONL_PATH, categorizeError } from "./core/telemetry.js";
|
|
13
13
|
import { isSDKInstalled, querySDKNetwork, getSDKNetworkEntry, getSDKNetworkStats, clearSDKNetwork, querySDKConsole, getSDKConsoleStats, clearSDKConsole } from "./core/sdkBridge.js";
|
|
14
14
|
import { tap } from "./pro/tap.js";
|
|
15
15
|
import { getActivateLicenseConfig, handleActivateLicense, getDeleteAccountConfig, handleDeleteAccount, } from "./tools/accountTools.js";
|
|
16
16
|
import { logBuffers, networkBuffers, getLogBuffer, getNetworkBuffer, getTotalLogCount, getConnectedAppByDevice, LogBuffer, NetworkBuffer, bundleErrorBuffer, imageBuffer, connectedApps, scanMetroPorts, fetchDevices, filterDebuggableDevices, connectToDevice, getConnectedApps, executeInApp, listDebugGlobals, inspectGlobal, reloadApp,
|
|
17
17
|
// React Component Inspection
|
|
18
|
-
getComponentTree, getScreenLayout, inspectComponent, findComponents, inspectAtPoint, toggleElementInspector, isInspectorActive, getInspectorSelection, getFirstConnectedApp, getLogs, searchLogs, getLogSummary, getNetworkRequests, searchNetworkRequests, getNetworkStats, formatRequestDetails,
|
|
18
|
+
getComponentTree, getScreenLayout, getPressableElements, inspectComponent, findComponents, inspectAtPoint, toggleElementInspector, isInspectorActive, getInspectorSelection, getFirstConnectedApp, getLogs, searchLogs, getLogSummary, getNetworkRequests, searchNetworkRequests, getNetworkStats, formatRequestDetails,
|
|
19
19
|
// Connection state
|
|
20
20
|
getAllConnectionStates, getAllConnectionMetadata, getRecentGaps, formatDuration, cancelAllReconnectionTimers, cancelReconnectionTimer, clearAllConnectionState, clearAllCDPMessageTimes, suppressReconnection, suppressReconnectionForKey, clearReconnectionSuppression, purgeStaleConnectionsForPorts,
|
|
21
21
|
// Context health tracking
|
|
@@ -38,8 +38,8 @@ androidDescribeAll, androidDescribePoint,
|
|
|
38
38
|
androidFindElement, androidWaitForElement,
|
|
39
39
|
// iOS
|
|
40
40
|
listIOSSimulators, iosScreenshot, iosInstallApp, iosLaunchApp, iosOpenUrl, iosTerminateApp, iosBootSimulator,
|
|
41
|
-
// iOS
|
|
42
|
-
iosTap, iosSwipe, iosInputText, iosButton, iosKeyEvent, iosKeySequence, iosDescribeAll, iosDescribePoint, IOS_BUTTON_TYPES,
|
|
41
|
+
// iOS UI driver tools
|
|
42
|
+
isUiDriverAvailable, iosTap, iosSwipe, iosInputText, iosButton, iosKeyEvent, iosKeySequence, iosDescribeAll, iosDescribePoint, IOS_BUTTON_TYPES,
|
|
43
43
|
// iOS Element Finding (no screenshots)
|
|
44
44
|
iosFindElement, iosWaitForElement,
|
|
45
45
|
// Telemetry
|
|
@@ -47,25 +47,7 @@ initTelemetry, trackToolInvocation, getTargetPlatform,
|
|
|
47
47
|
// Format utilities (TONL)
|
|
48
48
|
formatLogsAsTonl, formatNetworkAsTonl,
|
|
49
49
|
// LogBox detection & control
|
|
50
|
-
detectLogBox, formatLogBoxWarning, dismissLogBox, formatDismissedEntries, pushLogBox, addLogBoxIgnorePatterns, verifyLogPipeline } from "./core/index.js";
|
|
51
|
-
// Legacy package migration: notify once per session via LogBox
|
|
52
|
-
const LEGACY_PACKAGE_NAME = "react-native-ai-debugger";
|
|
53
|
-
let legacyMigrationNotified = false;
|
|
54
|
-
async function notifyLegacyPackageMigration() {
|
|
55
|
-
if (legacyMigrationNotified)
|
|
56
|
-
return;
|
|
57
|
-
if (getPackageName() !== LEGACY_PACKAGE_NAME)
|
|
58
|
-
return;
|
|
59
|
-
legacyMigrationNotified = true;
|
|
60
|
-
try {
|
|
61
|
-
await pushLogBox("You are using the deprecated 'react-native-ai-debugger' package. " +
|
|
62
|
-
"Please update your MCP settings to use 'react-native-ai-devtools' instead. " +
|
|
63
|
-
"See: https://www.npmjs.com/package/react-native-ai-devtools", "warning", true);
|
|
64
|
-
}
|
|
65
|
-
catch {
|
|
66
|
-
// Best-effort ā don't break the connection flow
|
|
67
|
-
}
|
|
68
|
-
}
|
|
50
|
+
detectLogBox, formatLogBoxWarning, dismissLogBox, formatDismissedEntries, pushLogBox, addLogBoxIgnorePatterns, verifyLogPipeline, formatIssueBody, buildGitHubUrl, shouldShowFeedbackHint, markFeedbackHintShown, } from "./core/index.js";
|
|
69
51
|
// Helper: resolve log buffer for a device (or create a merged buffer from all devices)
|
|
70
52
|
function resolveLogBuffer(device) {
|
|
71
53
|
if (device) {
|
|
@@ -173,6 +155,27 @@ function estimateImageTokens(base64Data) {
|
|
|
173
155
|
return Math.ceil(base64Data.length / 4); // fallback
|
|
174
156
|
}
|
|
175
157
|
}
|
|
158
|
+
// Tools that do NOT require an active Metro connection ā excluded from feedback hint trigger
|
|
159
|
+
const NON_METRO_TOOLS = new Set([
|
|
160
|
+
"scan_metro",
|
|
161
|
+
"connect_metro",
|
|
162
|
+
"disconnect_metro",
|
|
163
|
+
"ensure_connection",
|
|
164
|
+
"get_connection_status",
|
|
165
|
+
"get_license_status",
|
|
166
|
+
"activate_license",
|
|
167
|
+
"delete_account",
|
|
168
|
+
"get_usage_guide",
|
|
169
|
+
"get_apps",
|
|
170
|
+
"list_ios_simulators",
|
|
171
|
+
"list_android_devices",
|
|
172
|
+
"ios_boot_simulator",
|
|
173
|
+
"ios_launch_app",
|
|
174
|
+
"android_launch_app",
|
|
175
|
+
"ios_install_app",
|
|
176
|
+
"android_install_app",
|
|
177
|
+
"send_feedback"
|
|
178
|
+
]);
|
|
176
179
|
// Registry for dev meta-tool ā stores handlers and configs for dynamic dispatch
|
|
177
180
|
/* eslint-disable @typescript-eslint/no-explicit-any */
|
|
178
181
|
const toolRegistry = new Map();
|
|
@@ -260,6 +263,14 @@ function registerToolWithTelemetry(toolName, config, handler, emptyResultDetecto
|
|
|
260
263
|
responsePreview = textParts.join("\n").substring(0, 2000);
|
|
261
264
|
}
|
|
262
265
|
}
|
|
266
|
+
// First-install feedback hint ā fires once on first successful Metro-connected tool
|
|
267
|
+
if (!NON_METRO_TOOLS.has(toolName) && shouldShowFeedbackHint()) {
|
|
268
|
+
markFeedbackHintShown();
|
|
269
|
+
// Fire-and-forget ā don't block the tool response
|
|
270
|
+
pushLogBox("Congratulations on your first tool call! If you encounter any issues or have ideas for improvement, ask your AI assistant to call send_feedback. Your feedback helps me make this product better for everyone. Best regards, rn-ai-devtools developer.", "warning", true, "logbox").catch(() => {
|
|
271
|
+
// Non-fatal ā hint delivery failure should not affect tool execution
|
|
272
|
+
});
|
|
273
|
+
}
|
|
263
274
|
return result;
|
|
264
275
|
}
|
|
265
276
|
catch (error) {
|
|
@@ -279,7 +290,14 @@ function registerToolWithTelemetry(toolName, config, handler, emptyResultDetecto
|
|
|
279
290
|
duration,
|
|
280
291
|
server_version: getServerVersion(),
|
|
281
292
|
...(errorMessage && { error: errorMessage.substring(0, 200) }),
|
|
293
|
+
...(errorMessage && { error_category: categorizeError(errorMessage) }),
|
|
282
294
|
...(getTargetPlatform() && { platform: getTargetPlatform() }),
|
|
295
|
+
...(tapStrategy && { tap_strategy: tapStrategy }),
|
|
296
|
+
...(meaningful !== undefined && { meaningful }),
|
|
297
|
+
...(changeRate !== undefined && { change_rate: changeRate }),
|
|
298
|
+
...(iosDriver && { ios_driver: iosDriver }),
|
|
299
|
+
...(emptyResult !== undefined && { empty_result: emptyResult }),
|
|
300
|
+
...(emptyReason && { empty_reason: emptyReason }),
|
|
283
301
|
},
|
|
284
302
|
});
|
|
285
303
|
}
|
|
@@ -424,6 +442,21 @@ registerToolWithTelemetry("scan_metro", {
|
|
|
424
442
|
// Build events connection is optional
|
|
425
443
|
}
|
|
426
444
|
}
|
|
445
|
+
// Proactive check: warn if iOS UI driver is missing
|
|
446
|
+
const hasIosDevice = Array.from(allDeviceNames.values()).some(({ device }) => {
|
|
447
|
+
const title = (device.title || "").toLowerCase();
|
|
448
|
+
return title.includes("iphone") || title.includes("ipad") || title.includes("ios");
|
|
449
|
+
});
|
|
450
|
+
if (hasIosDevice) {
|
|
451
|
+
const uiDriverOk = await isUiDriverAvailable();
|
|
452
|
+
if (!uiDriverOk) {
|
|
453
|
+
results.push("");
|
|
454
|
+
results.push("ā ļø WARNING: iOS UI driver is NOT installed. Tools like tap, ios_swipe, ios_input_text, ios_describe_all, and ios_find_element will fail.");
|
|
455
|
+
results.push(" Install the recommended driver: brew install cameroncooke/axe/axe");
|
|
456
|
+
results.push(" Then set IOS_DRIVER=axe in your MCP server environment.");
|
|
457
|
+
results.push(" Alternative: brew install idb-companion (used by default)");
|
|
458
|
+
}
|
|
459
|
+
}
|
|
427
460
|
return {
|
|
428
461
|
content: [
|
|
429
462
|
{
|
|
@@ -454,7 +487,14 @@ registerToolWithTelemetry("get_apps", {
|
|
|
454
487
|
.map(({ app }, i) => {
|
|
455
488
|
const name = app.deviceInfo.deviceName || app.deviceInfo.title;
|
|
456
489
|
const appId = app.deviceInfo.appId || app.deviceInfo.title.split(" (")[0] || "unknown";
|
|
457
|
-
|
|
490
|
+
const lines = [` ${i + 1}. ${name} ā ${appId} (${app.platform}, port ${app.port})`];
|
|
491
|
+
if (app.appDetection) {
|
|
492
|
+
const d = app.appDetection;
|
|
493
|
+
const version = d.reactNativeVersion !== "unknown" ? `RN ${d.reactNativeVersion}` : "RN unknown";
|
|
494
|
+
const expo = d.expoSdkVersion ? `, Expo SDK ${d.expoSdkVersion}` : "";
|
|
495
|
+
lines.push(` Environment: ${version}, ${d.architecture} arch, ${d.jsEngine}, ${app.platform} ${d.osVersion}${expo}`);
|
|
496
|
+
}
|
|
497
|
+
return lines.join("\n");
|
|
458
498
|
});
|
|
459
499
|
const text = [
|
|
460
500
|
`Connected devices:`,
|
|
@@ -595,6 +635,52 @@ registerToolWithTelemetry("get_license_status", {
|
|
|
595
635
|
content: [{ type: "text", text: lines.join("\n") }],
|
|
596
636
|
};
|
|
597
637
|
});
|
|
638
|
+
// Tool: Send feedback / bug report / feature request
|
|
639
|
+
registerToolWithTelemetry("send_feedback", {
|
|
640
|
+
description: "Create a feedback report (bug, feature request, or general feedback) for the react-native-ai-devtools team. " +
|
|
641
|
+
"Auto-collects environment info. Returns a pre-filled GitHub issue URL and formatted issue body. " +
|
|
642
|
+
"Ask the user to open the URL and paste the body to submit.",
|
|
643
|
+
inputSchema: {
|
|
644
|
+
type: z.enum(["feedback", "feature_request", "bug"]).describe('Type of feedback: "feedback", "feature_request", or "bug"'),
|
|
645
|
+
title: z.string().describe("Short summary (becomes the GitHub issue title)"),
|
|
646
|
+
description: z.string().describe("Detailed description of the feedback, feature request, or bug"),
|
|
647
|
+
workflow_context: z.string().optional().describe("What the user was trying to accomplish when they decided to give feedback")
|
|
648
|
+
}
|
|
649
|
+
}, async ({ type, title, description, workflow_context }) => {
|
|
650
|
+
// Collect environment info
|
|
651
|
+
const serverVersion = getServerVersion();
|
|
652
|
+
const platform = process.platform;
|
|
653
|
+
const deviceType = getTargetPlatform();
|
|
654
|
+
const licenseStatus = getLicenseStatus();
|
|
655
|
+
const env = {
|
|
656
|
+
serverVersion,
|
|
657
|
+
platform,
|
|
658
|
+
deviceType,
|
|
659
|
+
licenseTier: licenseStatus.tier
|
|
660
|
+
};
|
|
661
|
+
const input = { type, title, description, workflowContext: workflow_context };
|
|
662
|
+
const issueBody = formatIssueBody(input, env);
|
|
663
|
+
const githubUrl = buildGitHubUrl(title, type);
|
|
664
|
+
const output = [
|
|
665
|
+
"## Feedback Report Ready",
|
|
666
|
+
"",
|
|
667
|
+
`**GitHub Issue URL:** ${githubUrl}`,
|
|
668
|
+
"",
|
|
669
|
+
"**Issue body to paste:**",
|
|
670
|
+
"",
|
|
671
|
+
"```markdown",
|
|
672
|
+
issueBody,
|
|
673
|
+
"```",
|
|
674
|
+
"",
|
|
675
|
+
"Please ask the user to:",
|
|
676
|
+
"1. Open the GitHub URL above",
|
|
677
|
+
"2. Paste the issue body into the description field",
|
|
678
|
+
"3. Review and submit the issue"
|
|
679
|
+
].join("\n");
|
|
680
|
+
return {
|
|
681
|
+
content: [{ type: "text", text: output }]
|
|
682
|
+
};
|
|
683
|
+
});
|
|
598
684
|
// Tool: Ensure connection health
|
|
599
685
|
registerToolWithTelemetry("ensure_connection", {
|
|
600
686
|
description: "Verify or establish a healthy connection to a React Native app. Use before running commands if connection may be stale, or after navigation/reload. This tool runs a health check and will auto-reconnect if needed.",
|
|
@@ -624,8 +710,6 @@ registerToolWithTelemetry("ensure_connection", {
|
|
|
624
710
|
isError: true
|
|
625
711
|
};
|
|
626
712
|
}
|
|
627
|
-
// Notify legacy package users (fire-and-forget, once per session)
|
|
628
|
-
notifyLegacyPackageMigration();
|
|
629
713
|
const lines = [];
|
|
630
714
|
lines.push("=== Connection Ensured ===\n");
|
|
631
715
|
if (result.connectionInfo) {
|
|
@@ -1372,6 +1456,34 @@ registerToolWithTelemetry("get_screen_layout", {
|
|
|
1372
1456
|
]
|
|
1373
1457
|
};
|
|
1374
1458
|
});
|
|
1459
|
+
// Tool: Get all pressable elements on screen
|
|
1460
|
+
registerToolWithTelemetry("get_pressable_elements", {
|
|
1461
|
+
description: "Find all pressable (onPress) and input (TextInput) elements currently visible on screen. Returns component names, tap-ready center coordinates (in points/dp), text labels, testID, and accessibilityLabel. Useful when you need to tap an icon or button but can't identify it from a screenshot alone. Each element includes hasLabel (true if it contains text) and isInput (true for TextInput fields).",
|
|
1462
|
+
inputSchema: {
|
|
1463
|
+
device: z.string().optional().describe("Target device name (substring match). Omit for default device. Run get_apps to see connected devices.")
|
|
1464
|
+
}
|
|
1465
|
+
}, async ({ device }) => {
|
|
1466
|
+
const result = await getPressableElements({ device });
|
|
1467
|
+
if (!result.success) {
|
|
1468
|
+
return {
|
|
1469
|
+
content: [
|
|
1470
|
+
{
|
|
1471
|
+
type: "text",
|
|
1472
|
+
text: `Error: ${result.error}`
|
|
1473
|
+
}
|
|
1474
|
+
],
|
|
1475
|
+
isError: true
|
|
1476
|
+
};
|
|
1477
|
+
}
|
|
1478
|
+
return {
|
|
1479
|
+
content: [
|
|
1480
|
+
{
|
|
1481
|
+
type: "text",
|
|
1482
|
+
text: result.result || "No pressable elements found."
|
|
1483
|
+
}
|
|
1484
|
+
]
|
|
1485
|
+
};
|
|
1486
|
+
});
|
|
1375
1487
|
// Tool: Inspect a specific component by name
|
|
1376
1488
|
registerToolWithTelemetry("inspect_component", {
|
|
1377
1489
|
description: "Inspect a specific React component by name. **DRILL-DOWN TOOL**: Use after get_screen_layout or find_components to identify which component to inspect. Returns props, style, state (hooks), and optionally children tree. Use childrenDepth to control how deep nested children go.",
|
|
@@ -1485,12 +1597,13 @@ registerToolWithTelemetry("find_components", {
|
|
|
1485
1597
|
registerToolWithTelemetry("tap", {
|
|
1486
1598
|
description: "Tap a UI element. Automatically tries multiple strategies: fiber tree (React), accessibility tree (native), and OCR (visual). " +
|
|
1487
1599
|
"Auto-detects platform (iOS/Android). For coordinates, pass raw pixel values from screenshot ā tap handles all coordinate conversion internally (iOS pixel-to-point, scale factor adjustments).\n\n" +
|
|
1600
|
+
"PREREQUISITE: For iOS, an iOS UI driver must be installed ā AXe (recommended: brew install cameroncooke/axe/axe) or IDB (brew install idb-companion). Without it, accessibility and coordinate strategies will fail.\n\n" +
|
|
1488
1601
|
"IMPORTANT: When both iOS and Android devices are connected, specify platform explicitly to avoid tapping on the wrong device.\n\n" +
|
|
1489
1602
|
"Examples:\n" +
|
|
1490
1603
|
"- tap(text=\"Submit\") ā finds and taps element with matching text\n" +
|
|
1491
1604
|
"- tap(testID=\"login-btn\") ā finds by testID\n" +
|
|
1492
1605
|
"- tap(component=\"HamburgerIcon\") ā finds by React component name\n" +
|
|
1493
|
-
"- tap(x=300, y=600) ā taps at pixel coordinates
|
|
1606
|
+
"- tap(x=300, y=600) ā taps at pixel coordinates; use coordinates from the pressable elements list returned by ios_screenshot/android_screenshot for reliable icon and ambiguous-element taps\n" +
|
|
1494
1607
|
"- tap(text=\"Menu\", strategy=\"ocr\") ā forces OCR strategy only\n" +
|
|
1495
1608
|
"- tap(x=300, y=600, native=true, platform=\"android\") ā taps directly via ADB/simctl without React Native connection\n\n" +
|
|
1496
1609
|
"Returns a post-tap screenshot by default (set screenshot=false to disable). " +
|
|
@@ -2668,7 +2781,7 @@ registerToolWithTelemetry("list_android_devices", {
|
|
|
2668
2781
|
});
|
|
2669
2782
|
// Tool: Android screenshot
|
|
2670
2783
|
registerToolWithTelemetry("android_screenshot", {
|
|
2671
|
-
description: "Take a screenshot from an Android device/emulator. Returns the image
|
|
2784
|
+
description: "Take a screenshot from an Android device/emulator. Returns the image plus two lists: (1) pressable elements ā the tappable components on screen with ready-to-use pixel tap coordinates, testIDs, and labels; (2) screen layout ā all visible React components for context. Prefer tap(text=\"...\") when text is exact and unique; otherwise use tap(x, y) with coordinates from the pressables list ā this is the most reliable way to tap icons or visually-identified elements. Use component names from the layout for inspect_component/find_components.",
|
|
2672
2785
|
inputSchema: {
|
|
2673
2786
|
outputPath: z
|
|
2674
2787
|
.string()
|
|
@@ -2735,6 +2848,26 @@ registerToolWithTelemetry("android_screenshot", {
|
|
|
2735
2848
|
catch {
|
|
2736
2849
|
// Non-fatal: screenshot works without layout enrichment
|
|
2737
2850
|
}
|
|
2851
|
+
// Enrich with pressable elements (filtered list of tappable components, converted to screenshot pixel coords)
|
|
2852
|
+
let pressablesText = null;
|
|
2853
|
+
try {
|
|
2854
|
+
const pressables = await getPressableElements({});
|
|
2855
|
+
if (pressables.success && pressables.parsedElements && pressables.parsedElements.length > 0) {
|
|
2856
|
+
const screenshotScale = result.scaleFactor || 1;
|
|
2857
|
+
pressablesText = pressables.parsedElements.map((el) => {
|
|
2858
|
+
const px = Math.round((el.center.x * androidPixelRatio) / screenshotScale);
|
|
2859
|
+
const py = Math.round((el.center.y * androidPixelRatio) / screenshotScale);
|
|
2860
|
+
const label = el.accessibilityLabel || el.text || el.testID || el.component;
|
|
2861
|
+
const idPart = el.testID ? ` testID="${el.testID}"` : "";
|
|
2862
|
+
const kindPart = el.isInput ? " [input]" : "";
|
|
2863
|
+
const wrapPart = el.isWrapper ? " [wrapper ā skip]" : "";
|
|
2864
|
+
return ` (${px}, ${py}) ${el.component}: "${label}"${idPart}${kindPart}${wrapPart}`;
|
|
2865
|
+
}).join("\n");
|
|
2866
|
+
}
|
|
2867
|
+
}
|
|
2868
|
+
catch {
|
|
2869
|
+
// Non-fatal: screenshot works without pressables enrichment
|
|
2870
|
+
}
|
|
2738
2871
|
infoText += `\nš± Android uses PIXELS for all coordinates`;
|
|
2739
2872
|
if (result.scaleFactor && result.scaleFactor > 1) {
|
|
2740
2873
|
infoText += `\nš¼ļø Image was scaled down to fit API limits (scale: ${result.scaleFactor.toFixed(3)})`;
|
|
@@ -2745,14 +2878,20 @@ registerToolWithTelemetry("android_screenshot", {
|
|
|
2745
2878
|
}
|
|
2746
2879
|
infoText += `\nā ļø Status bar: ${statusBarPixels}px (${statusBarDp}dp) from top - app content starts below this`;
|
|
2747
2880
|
infoText += `\nš Display density: ${densityDpi}dpi`;
|
|
2881
|
+
if (pressablesText) {
|
|
2882
|
+
infoText += `\n\nšÆ Pressable elements (ready-to-tap, coordinates in screenshot pixels):`;
|
|
2883
|
+
infoText += `\n${pressablesText}`;
|
|
2884
|
+
}
|
|
2748
2885
|
if (layoutText) {
|
|
2749
|
-
infoText += `\n\nš Screen Layout (components
|
|
2886
|
+
infoText += `\n\nš Screen Layout (all components, tap coordinates in pixels):`;
|
|
2750
2887
|
infoText += `\n${layoutText}`;
|
|
2888
|
+
}
|
|
2889
|
+
if (pressablesText || layoutText) {
|
|
2751
2890
|
infoText += `\n\nš” Next steps:`;
|
|
2752
|
-
infoText += `\n ⢠tap(text="Button Label") ā
|
|
2753
|
-
infoText += `\n ⢠tap(testID="id") or tap(component="Name") ā
|
|
2754
|
-
infoText += `\n ⢠tap(x=<px>, y=<px>) ā use
|
|
2755
|
-
infoText += `\n ⢠inspect_component("ComponentName") ā inspect a component from the layout
|
|
2891
|
+
infoText += `\n ⢠tap(text="Button Label") ā when text is exact and unique`;
|
|
2892
|
+
infoText += `\n ⢠tap(testID="id") or tap(component="Name") ā when you know the identifier`;
|
|
2893
|
+
infoText += `\n ⢠tap(x=<px>, y=<px>) ā use coordinates from the pressable elements list above (reliable for icons and ambiguous elements)`;
|
|
2894
|
+
infoText += `\n ⢠inspect_component("ComponentName") ā inspect a component from the layout`;
|
|
2756
2895
|
}
|
|
2757
2896
|
else {
|
|
2758
2897
|
infoText += `\n\nš” Next steps:`;
|
|
@@ -3230,7 +3369,7 @@ registerToolWithTelemetry("list_ios_simulators", {
|
|
|
3230
3369
|
});
|
|
3231
3370
|
// Tool: iOS screenshot
|
|
3232
3371
|
registerToolWithTelemetry("ios_screenshot", {
|
|
3233
|
-
description: "Take a screenshot from an iOS simulator. Returns the image
|
|
3372
|
+
description: "Take a screenshot from an iOS simulator. Returns the image plus two lists: (1) pressable elements ā the tappable components on screen with ready-to-use pixel tap coordinates, testIDs, and labels; (2) screen layout ā all visible React components for context. Prefer tap(text=\"...\") when text is exact and unique; otherwise use tap(x, y) with coordinates from the pressables list ā this is the most reliable way to tap icons or visually-identified elements. Use component names from the layout for inspect_component/find_components.",
|
|
3234
3373
|
inputSchema: {
|
|
3235
3374
|
outputPath: z
|
|
3236
3375
|
.string()
|
|
@@ -3322,6 +3461,26 @@ registerToolWithTelemetry("ios_screenshot", {
|
|
|
3322
3461
|
catch {
|
|
3323
3462
|
// Non-fatal: screenshot works without layout enrichment
|
|
3324
3463
|
}
|
|
3464
|
+
// Enrich with pressable elements (filtered list of tappable components, converted to screenshot pixel coords)
|
|
3465
|
+
let pressablesText = null;
|
|
3466
|
+
try {
|
|
3467
|
+
const pressables = await getPressableElements({});
|
|
3468
|
+
if (pressables.success && pressables.parsedElements && pressables.parsedElements.length > 0) {
|
|
3469
|
+
const screenshotScale = result.scaleFactor || 1;
|
|
3470
|
+
pressablesText = pressables.parsedElements.map((el) => {
|
|
3471
|
+
const px = Math.round((el.center.x * scaleFactor) / screenshotScale);
|
|
3472
|
+
const py = Math.round((el.center.y * scaleFactor) / screenshotScale);
|
|
3473
|
+
const label = el.accessibilityLabel || el.text || el.testID || el.component;
|
|
3474
|
+
const idPart = el.testID ? ` testID="${el.testID}"` : "";
|
|
3475
|
+
const kindPart = el.isInput ? " [input]" : "";
|
|
3476
|
+
const wrapPart = el.isWrapper ? " [wrapper ā skip]" : "";
|
|
3477
|
+
return ` (${px}, ${py}) ${el.component}: "${label}"${idPart}${kindPart}${wrapPart}`;
|
|
3478
|
+
}).join("\n");
|
|
3479
|
+
}
|
|
3480
|
+
}
|
|
3481
|
+
catch {
|
|
3482
|
+
// Non-fatal: screenshot works without pressables enrichment
|
|
3483
|
+
}
|
|
3325
3484
|
let infoText = `Screenshot captured (${pixelWidth}x${pixelHeight} pixels)`;
|
|
3326
3485
|
infoText += `\nš± iOS screen: ${pointWidth}x${pointHeight} points (${scaleFactor}x scale)`;
|
|
3327
3486
|
infoText += `\nš tap() handles pixel-to-point conversion automatically ā pass pixel coords from this image directly`;
|
|
@@ -3329,14 +3488,20 @@ registerToolWithTelemetry("ios_screenshot", {
|
|
|
3329
3488
|
if (result.scaleFactor && result.scaleFactor > 1) {
|
|
3330
3489
|
infoText += `\nš¼ļø Image was scaled down to fit API limits (scale: ${result.scaleFactor.toFixed(3)})`;
|
|
3331
3490
|
}
|
|
3491
|
+
if (pressablesText) {
|
|
3492
|
+
infoText += `\n\nšÆ Pressable elements (ready-to-tap, coordinates in screenshot pixels):`;
|
|
3493
|
+
infoText += `\n${pressablesText}`;
|
|
3494
|
+
}
|
|
3332
3495
|
if (layoutText) {
|
|
3333
|
-
infoText += `\n\nš Screen Layout (components
|
|
3496
|
+
infoText += `\n\nš Screen Layout (all components, tap coordinates in pixels):`;
|
|
3334
3497
|
infoText += `\n${layoutText}`;
|
|
3498
|
+
}
|
|
3499
|
+
if (pressablesText || layoutText) {
|
|
3335
3500
|
infoText += `\n\nš” Next steps:`;
|
|
3336
|
-
infoText += `\n ⢠tap(text="Button Label") ā
|
|
3337
|
-
infoText += `\n ⢠tap(testID="id") or tap(component="Name") ā
|
|
3338
|
-
infoText += `\n ⢠tap(x=<px>, y=<px>) ā use
|
|
3339
|
-
infoText += `\n ⢠inspect_component("ComponentName") ā inspect a component from the layout
|
|
3501
|
+
infoText += `\n ⢠tap(text="Button Label") ā when text is exact and unique`;
|
|
3502
|
+
infoText += `\n ⢠tap(testID="id") or tap(component="Name") ā when you know the identifier`;
|
|
3503
|
+
infoText += `\n ⢠tap(x=<px>, y=<px>) ā use coordinates from the pressable elements list above (reliable for icons and ambiguous elements)`;
|
|
3504
|
+
infoText += `\n ⢠inspect_component("ComponentName") ā inspect a component from the layout`;
|
|
3340
3505
|
}
|
|
3341
3506
|
else {
|
|
3342
3507
|
infoText += `\n\nš” Next steps:`;
|
|
@@ -3604,12 +3769,13 @@ registerToolWithTelemetry("ios_boot_simulator", {
|
|
|
3604
3769
|
};
|
|
3605
3770
|
});
|
|
3606
3771
|
// ============================================================================
|
|
3607
|
-
// iOS
|
|
3608
|
-
//
|
|
3772
|
+
// iOS UI Interaction Tools (require an iOS UI driver)
|
|
3773
|
+
// Recommended: AXe ā brew install cameroncooke/axe/axe (set IOS_DRIVER=axe)
|
|
3774
|
+
// Alternative: IDB ā brew install idb-companion (default)
|
|
3609
3775
|
// ============================================================================
|
|
3610
3776
|
// Tool: iOS swipe
|
|
3611
3777
|
server.registerTool("ios_swipe", {
|
|
3612
|
-
description: "Swipe gesture on an iOS simulator screen. Requires
|
|
3778
|
+
description: "Swipe gesture on an iOS simulator screen. Requires an iOS UI driver: AXe (recommended: brew install cameroncooke/axe/axe) or IDB (brew install idb-companion).",
|
|
3613
3779
|
inputSchema: {
|
|
3614
3780
|
startX: z.coerce.number().describe("Starting X coordinate in pixels"),
|
|
3615
3781
|
startY: z.coerce.number().describe("Starting Y coordinate in pixels"),
|
|
@@ -3633,7 +3799,7 @@ server.registerTool("ios_swipe", {
|
|
|
3633
3799
|
});
|
|
3634
3800
|
// Tool: iOS input text
|
|
3635
3801
|
server.registerTool("ios_input_text", {
|
|
3636
|
-
description: "Type text into the active input field on an iOS simulator. Requires
|
|
3802
|
+
description: "Type text into the active input field on an iOS simulator. Requires an iOS UI driver: AXe (recommended: brew install cameroncooke/axe/axe) or IDB (brew install idb-companion).",
|
|
3637
3803
|
inputSchema: {
|
|
3638
3804
|
text: z.string().describe("Text to type into the active input field"),
|
|
3639
3805
|
udid: z.string().optional().describe("Optional simulator UDID. Uses booted simulator if not specified.")
|
|
@@ -3652,7 +3818,7 @@ server.registerTool("ios_input_text", {
|
|
|
3652
3818
|
});
|
|
3653
3819
|
// Tool: iOS button
|
|
3654
3820
|
server.registerTool("ios_button", {
|
|
3655
|
-
description: "Press a hardware button on an iOS simulator. Requires
|
|
3821
|
+
description: "Press a hardware button on an iOS simulator. Requires an iOS UI driver: AXe (recommended: brew install cameroncooke/axe/axe) or IDB (brew install idb-companion).",
|
|
3656
3822
|
inputSchema: {
|
|
3657
3823
|
button: z
|
|
3658
3824
|
.enum(IOS_BUTTON_TYPES)
|
|
@@ -3674,7 +3840,7 @@ server.registerTool("ios_button", {
|
|
|
3674
3840
|
});
|
|
3675
3841
|
// Tool: iOS key event
|
|
3676
3842
|
server.registerTool("ios_key_event", {
|
|
3677
|
-
description: "Send a key event to an iOS simulator by keycode. Requires
|
|
3843
|
+
description: "Send a key event to an iOS simulator by keycode. Requires an iOS UI driver: AXe (recommended: brew install cameroncooke/axe/axe) or IDB (brew install idb-companion).",
|
|
3678
3844
|
inputSchema: {
|
|
3679
3845
|
keycode: z.coerce.number().describe("iOS keycode to send"),
|
|
3680
3846
|
duration: z.coerce.number().optional().describe("Optional key press duration in seconds"),
|
|
@@ -3694,7 +3860,7 @@ server.registerTool("ios_key_event", {
|
|
|
3694
3860
|
});
|
|
3695
3861
|
// Tool: iOS key sequence
|
|
3696
3862
|
server.registerTool("ios_key_sequence", {
|
|
3697
|
-
description: "Send a sequence of key events to an iOS simulator. Requires
|
|
3863
|
+
description: "Send a sequence of key events to an iOS simulator. Requires an iOS UI driver: AXe (recommended: brew install cameroncooke/axe/axe) or IDB (brew install idb-companion).",
|
|
3698
3864
|
inputSchema: {
|
|
3699
3865
|
keycodes: z.array(z.coerce.number()).describe("Array of iOS keycodes to send in sequence"),
|
|
3700
3866
|
udid: z.string().optional().describe("Optional simulator UDID. Uses booted simulator if not specified.")
|
|
@@ -3713,7 +3879,7 @@ server.registerTool("ios_key_sequence", {
|
|
|
3713
3879
|
});
|
|
3714
3880
|
// Tool: iOS describe all (accessibility tree)
|
|
3715
3881
|
server.registerTool("ios_describe_all", {
|
|
3716
|
-
description: "Get accessibility information for the entire iOS simulator screen. Returns a nested tree of UI elements with labels, values, and frames. Requires
|
|
3882
|
+
description: "Get accessibility information for the entire iOS simulator screen. Returns a nested tree of UI elements with labels, values, and frames. Requires an iOS UI driver: AXe (recommended: brew install cameroncooke/axe/axe) or IDB (brew install idb-companion).",
|
|
3717
3883
|
inputSchema: {
|
|
3718
3884
|
udid: z.string().optional().describe("Optional simulator UDID. Uses booted simulator if not specified.")
|
|
3719
3885
|
}
|
|
@@ -3744,7 +3910,7 @@ server.registerTool("ios_describe_all", {
|
|
|
3744
3910
|
});
|
|
3745
3911
|
// Tool: iOS describe point
|
|
3746
3912
|
server.registerTool("ios_describe_point", {
|
|
3747
|
-
description: "Get accessibility information for the UI element at a specific point on the iOS simulator screen. Requires
|
|
3913
|
+
description: "Get accessibility information for the UI element at a specific point on the iOS simulator screen. Requires an iOS UI driver: AXe (recommended: brew install cameroncooke/axe/axe) or IDB (brew install idb-companion).",
|
|
3748
3914
|
inputSchema: {
|
|
3749
3915
|
x: z.coerce.number().describe("X coordinate in pixels"),
|
|
3750
3916
|
y: z.coerce.number().describe("Y coordinate in pixels"),
|
|
@@ -3764,7 +3930,7 @@ server.registerTool("ios_describe_point", {
|
|
|
3764
3930
|
});
|
|
3765
3931
|
// Tool: iOS find element (no screenshot needed)
|
|
3766
3932
|
server.registerTool("ios_find_element", {
|
|
3767
|
-
description: "Find a UI element on iOS simulator by accessibility label or value. Returns element details including tap coordinates. Requires IDB (brew install idb-companion). Workflow: 1) wait_for_element, 2) find_element, 3) tap with returned coordinates. Prefer this over screenshots for button taps.",
|
|
3933
|
+
description: "Find a UI element on iOS simulator by accessibility label or value. Returns element details including tap coordinates. Requires an iOS UI driver: AXe (recommended: brew install cameroncooke/axe/axe) or IDB (brew install idb-companion). Workflow: 1) wait_for_element, 2) find_element, 3) tap with returned coordinates. Prefer this over screenshots for button taps.",
|
|
3768
3934
|
inputSchema: {
|
|
3769
3935
|
label: z.string().optional().describe("Exact accessibility label match"),
|
|
3770
3936
|
labelContains: z.string().optional().describe("Partial label match (case-insensitive)"),
|
|
@@ -3811,7 +3977,7 @@ server.registerTool("ios_find_element", {
|
|
|
3811
3977
|
});
|
|
3812
3978
|
// Tool: iOS wait for element
|
|
3813
3979
|
server.registerTool("ios_wait_for_element", {
|
|
3814
|
-
description: "Wait for a UI element to appear on iOS simulator. Polls until found or timeout. Requires IDB (brew install idb-companion). Use this FIRST after navigation to ensure screen is ready, then use find_element + tap.",
|
|
3980
|
+
description: "Wait for a UI element to appear on iOS simulator. Polls until found or timeout. Requires an iOS UI driver: AXe (recommended: brew install cameroncooke/axe/axe) or IDB (brew install idb-companion). Use this FIRST after navigation to ensure screen is ready, then use find_element + tap.",
|
|
3815
3981
|
inputSchema: {
|
|
3816
3982
|
label: z.string().optional().describe("Exact accessibility label match"),
|
|
3817
3983
|
labelContains: z.string().optional().describe("Partial label match (case-insensitive)"),
|
|
@@ -3966,18 +4132,18 @@ async function main() {
|
|
|
3966
4132
|
res.end("Not found");
|
|
3967
4133
|
});
|
|
3968
4134
|
httpServer.listen(httpPort, () => {
|
|
3969
|
-
console.error(`[rn-ai-
|
|
4135
|
+
console.error(`[rn-ai-debugger] MCP HTTP server listening on http://localhost:${httpPort}/mcp`);
|
|
3970
4136
|
});
|
|
3971
4137
|
}
|
|
3972
4138
|
else {
|
|
3973
4139
|
// Stdio transport mode ā default for production
|
|
3974
4140
|
const transport = new StdioServerTransport();
|
|
3975
4141
|
await server.connect(transport);
|
|
3976
|
-
console.error("[rn-ai-
|
|
4142
|
+
console.error("[rn-ai-debugger] Server started on stdio");
|
|
3977
4143
|
}
|
|
3978
4144
|
}
|
|
3979
4145
|
main().catch((error) => {
|
|
3980
|
-
console.error("[rn-ai-
|
|
4146
|
+
console.error("[rn-ai-debugger] Fatal error:", error);
|
|
3981
4147
|
process.exit(1);
|
|
3982
4148
|
});
|
|
3983
4149
|
// Graceful shutdown: close CDP connections so the slot is freed for other sessions
|