react-native-ai-devtools 1.4.19 → 1.4.21

package/build/index.js

@@ -27,7 +27,7 @@ connectMetroBuildEvents, disconnectMetroBuildEvents, getBundleErrors, getBundleS
 // Error screen parsing (OCR fallback)
 parseErrorScreenText, formatParsedError, 
 // OCR
-recognizeText, inferIOSDevicePixelRatio, 
+recognizeText, inferIOSDevicePixelRatio, enrichScreenshotWithLayout, 
 // Android
 listAndroidDevices, androidScreenshot, androidInstallApp, androidLaunchApp, androidListPackages, 
 // Android UI Input (Phase 2)
@@ -1470,7 +1470,7 @@ registerToolWithTelemetry("tap", {
         "- tap(text=\"Submit\") — finds and taps element with matching text\n" +
         "- tap(testID=\"login-btn\") — finds by testID\n" +
         "- tap(component=\"HamburgerIcon\") — finds by React component name\n" +
-        "- tap(x=300, y=600) — taps at pixel coordinates from screenshot (conversion handled internally)\n" +
+        "- tap(x=300, y=600) — taps at pixel coordinates (LAST RESORT — only use if text/testID/component strategies all fail)\n" +
         "- tap(text=\"Menu\", strategy=\"ocr\") — forces OCR strategy only\n" +
         "- tap(x=300, y=600, native=true, platform=\"android\") — taps directly via ADB/simctl without React Native connection\n\n" +
         "Returns a post-tap screenshot by default (set screenshot=false to disable). " +
@@ -2648,7 +2648,7 @@ registerToolWithTelemetry("list_android_devices", {
 });
 // Tool: Android screenshot
 registerToolWithTelemetry("android_screenshot", {
-    description: "Take a screenshot from an Android device/emulator. Returns the image data that can be displayed.",
+    description: "Take a screenshot from an Android device/emulator. Returns the image and a screen layout overlay showing visible React components with names, text content, hierarchy, and tap-ready pixel coordinates. Use component names for inspect_component/find_components. Use tap(text=...) for interaction — only fall back to tap(x,y) coordinates from the layout if all tap strategies fail.",
     inputSchema: {
         outputPath: z
             .string()
@@ -2705,6 +2705,16 @@ registerToolWithTelemetry("android_screenshot", {
         catch {
             // Use defaults
         }
+        // Enrich with screen layout data (component names + tap coordinates)
+        // Android: density ratio = densityDpi / 160 (160dpi = 1dp = 1px baseline)
+        const androidPixelRatio = densityDpi / 160;
+        let layoutText = null;
+        try {
+            layoutText = await enrichScreenshotWithLayout(androidPixelRatio, result.scaleFactor || 1);
+        }
+        catch {
+            // Non-fatal: screenshot works without layout enrichment
+        }
         infoText += `\n📱 Android uses PIXELS for all coordinates`;
         if (result.scaleFactor && result.scaleFactor > 1) {
             infoText += `\n🖼️ Image was scaled down to fit API limits (scale: ${result.scaleFactor.toFixed(3)})`;
@@ -2715,11 +2725,22 @@ registerToolWithTelemetry("android_screenshot", {
         }
         infoText += `\n⚠️ Status bar: ${statusBarPixels}px (${statusBarDp}dp) from top - app content starts below this`;
         infoText += `\n📊 Display density: ${densityDpi}dpi`;
-        infoText += `\n\n💡 Next steps:`;
-        infoText += `\n  • tap(text="Button Label") — tap element by visible text`;
-        infoText += `\n  • tap(x=<px>, y=<px>) — tap at coordinates from this screenshot`;
-        infoText += `\n  • android_describe_all — get full UI tree with exact tap coordinates`;
-        infoText += `\n  • android_find_element(text="...") — find element coordinates without tapping`;
+        if (layoutText) {
+            infoText += `\n\n📋 Screen Layout (components with tap coordinates in pixels):`;
+            infoText += `\n${layoutText}`;
+            infoText += `\n\n💡 Next steps:`;
+            infoText += `\n  • tap(text="Button Label") — tap element by visible text (preferred)`;
+            infoText += `\n  • tap(testID="id") or tap(component="Name") — tap by testID or component name`;
+            infoText += `\n  • tap(x=<px>, y=<px>) — use tap coordinates from layout above ONLY if text/testID/component tap fails`;
+            infoText += `\n  • inspect_component("ComponentName") — inspect a component from the layout above`;
+        }
+        else {
+            infoText += `\n\n💡 Next steps:`;
+            infoText += `\n  • tap(text="Button Label") — tap element by visible text`;
+            infoText += `\n  • tap(x=<px>, y=<px>) — tap at coordinates from this screenshot`;
+            infoText += `\n  • android_describe_all — get full UI tree with exact tap coordinates`;
+            infoText += `\n  • android_find_element(text="...") — find element coordinates without tapping`;
+        }
         // Check for LogBox overlay (uses default CDP device — native deviceId cannot be mapped to CDP device name)
         try {
             const logBoxState = await detectLogBox();
@@ -3189,7 +3210,7 @@ registerToolWithTelemetry("list_ios_simulators", {
 });
 // Tool: iOS screenshot
 registerToolWithTelemetry("ios_screenshot", {
-    description: "Take a screenshot from an iOS simulator. Returns the image data that can be displayed.",
+    description: "Take a screenshot from an iOS simulator. Returns the image and a screen layout overlay showing visible React components with names, text content, hierarchy, and tap-ready pixel coordinates. Use component names for inspect_component/find_components. Use tap(text=...) for interaction — only fall back to tap(x,y) coordinates from the layout if all tap strategies fail.",
     inputSchema: {
         outputPath: z
             .string()
@@ -3273,6 +3294,14 @@ registerToolWithTelemetry("ios_screenshot", {
             pointHeight = Math.round(pixelHeight / scaleFactor);
         }
         const safeAreaOffsetPixels = safeAreaTop * scaleFactor;
+        // Enrich with screen layout data (component names + tap coordinates)
+        let layoutText = null;
+        try {
+            layoutText = await enrichScreenshotWithLayout(scaleFactor, result.scaleFactor || 1);
+        }
+        catch {
+            // Non-fatal: screenshot works without layout enrichment
+        }
         let infoText = `Screenshot captured (${pixelWidth}x${pixelHeight} pixels)`;
         infoText += `\n📱 iOS screen: ${pointWidth}x${pointHeight} points (${scaleFactor}x scale)`;
         infoText += `\n📐 tap() handles pixel-to-point conversion automatically — pass pixel coords from this image directly`;
@@ -3280,11 +3309,22 @@ registerToolWithTelemetry("ios_screenshot", {
         if (result.scaleFactor && result.scaleFactor > 1) {
             infoText += `\n🖼️ Image was scaled down to fit API limits (scale: ${result.scaleFactor.toFixed(3)})`;
         }
-        infoText += `\n\n💡 Next steps:`;
-        infoText += `\n  • tap(text="Button Label") — tap element by visible text`;
-        infoText += `\n  • tap(x=<px>, y=<px>) — tap at coordinates from this screenshot`;
-        infoText += `\n  • ios_describe_all — get full UI tree with exact tap coordinates`;
-        infoText += `\n  • ios_find_element(label="...") — find element coordinates without tapping`;
+        if (layoutText) {
+            infoText += `\n\n📋 Screen Layout (components with tap coordinates in pixels):`;
+            infoText += `\n${layoutText}`;
+            infoText += `\n\n💡 Next steps:`;
+            infoText += `\n  • tap(text="Button Label") — tap element by visible text (preferred)`;
+            infoText += `\n  • tap(testID="id") or tap(component="Name") — tap by testID or component name`;
+            infoText += `\n  • tap(x=<px>, y=<px>) — use tap coordinates from layout above ONLY if text/testID/component tap fails`;
+            infoText += `\n  • inspect_component("ComponentName") — inspect a component from the layout above`;
+        }
+        else {
+            infoText += `\n\n💡 Next steps:`;
+            infoText += `\n  • tap(text="Button Label") — tap element by visible text`;
+            infoText += `\n  • tap(x=<px>, y=<px>) — tap at coordinates from this screenshot`;
+            infoText += `\n  • ios_describe_all — get full UI tree with exact tap coordinates`;
+            infoText += `\n  • ios_find_element(label="...") — find element coordinates without tapping`;
+        }
         // Check for LogBox overlay (uses default CDP device — native UDID cannot be mapped to CDP device name)
         try {
             const logBoxState = await detectLogBox();