react-native-ai-debugger 1.6.16 → 1.8.0

package/build/index.js

@@ -9,6 +9,8 @@ import { getGuideOverview, getGuideByTopic, getAvailableTopics, DECISION_TREE }
 import { getLicenseStatus, getDashboardUrl, getUsageInfo, getPricingInfo, formatPlanPrice } from "./core/license.js";
 import { API_BASE_URL } from "./core/config.js";
 import { getPostHogClient, identifyIfDevMode, shutdownPostHog } from "./core/posthog.js";
+import { UserInputError } from "./core/errors.js";
+import { clearFocusedInput, dismissKeyboard, inputTextWithReplace } from "./core/focusedInputTools.js";
 import { getInstallationId, getServerVersion, getPackageName, isDevMode, TELEMETRY_JSONL_PATH, categorizeError } from "./core/telemetry.js";
 import { isSDKInstalled, querySDKNetwork, getSDKNetworkEntry, getSDKNetworkStats, clearSDKNetwork, querySDKConsole, getSDKConsoleStats, clearSDKConsole } from "./core/sdkBridge.js";
 import { reduxDispatch, reduxGetState } from "./core/redux.js";
@@ -36,7 +38,7 @@ ANDROID_KEY_EVENTS, androidLongPress, androidSwipe, androidInputText, androidKey
 // iOS
 listIOSSimulators, iosScreenshot, iosInstallApp, iosLaunchApp, iosOpenUrl, iosTerminateApp, iosBootSimulator, 
 // iOS UI driver tools
-isUiDriverAvailable, iosButton, iosDescribeAll, IOS_BUTTON_TYPES, getDevicePixelRatio, getIOSSafeAreaTop, 
+isUiDriverAvailable, iosButton, iosInputText, iosDescribeAll, detectIOSSystemOverlay, formatIOSSystemOverlayWarning, IOS_BUTTON_TYPES, getDevicePixelRatio, getIOSSafeAreaTop, 
 // Telemetry
 initTelemetry, trackToolInvocation, getTargetPlatform, 
 // Format utilities (TONL)
@@ -50,7 +52,7 @@ function resolveLogBuffer(device) {
     if (device) {
         const app = getConnectedAppByDevice(device);
         if (!app)
-            throw new Error(`No connected device matches "${device}"`);
+            throw new UserInputError(`No connected device matches "${device}"`);
         const deviceName = app.deviceInfo.deviceName || app.deviceInfo.title || "unknown";
         return getLogBuffer(deviceName);
     }
@@ -68,7 +70,7 @@ function resolveNetworkBuffer(device) {
     if (device) {
         const app = getConnectedAppByDevice(device);
         if (!app)
-            throw new Error(`No connected device matches "${device}"`);
+            throw new UserInputError(`No connected device matches "${device}"`);
         const deviceName = app.deviceInfo.deviceName || app.deviceInfo.title || "unknown";
         return getNetworkBuffer(deviceName);
     }
@@ -331,7 +333,14 @@ function registerToolWithTelemetry(toolName, config, handler, emptyResultDetecto
         catch (error) {
             success = false;
             errorMessage = error instanceof Error ? error.message : String(error);
-            getPostHogClient()?.captureException(error, getInstallationId(), { tool: toolName, server_version: getServerVersion(), package_name: getPackageName() });
+            // H2 (Step 9): UserInputError marks agent-input mistakes (unknown
+            // device, missing predicate, ambiguous match). They flow through
+            // telemetry's trackToolInvocation in the finally block; we just
+            // skip the dedicated error-tracking pipe so the dashboard surfaces
+            // real product bugs rather than validation noise.
+            if (!(error instanceof UserInputError)) {
+                getPostHogClient()?.captureException(error, getInstallationId(), { tool: toolName, server_version: getServerVersion(), package_name: getPackageName() });
+            }
             throw error;
         }
         finally {
@@ -727,12 +736,14 @@ registerToolWithTelemetry("tap", {
             .boolean()
             .optional()
             .default(true)
-            .describe("Capture and return a post-tap screenshot. Enabled by default. Set to false for fastest execution (no screenshots)."),
+            .describe("Return post-tap image bytes in the response. Default true. Set to false to drop the PNG bytes — verification still runs (set verify=false to skip that too). Combine with verify=true to get the meaningful/changeRate signal without paying the ~1MB-per-tap bandwidth cost."),
         verify: z
             .boolean()
             .optional()
             .describe("Run before/after screenshot diff to detect if the tap had a meaningful visual effect. " +
-            "Default: true for coordinate/accessibility/ocr strategies, false for fiber. Requires screenshot=true."),
+            "Default: true for coordinate/accessibility/ocr strategies, false for fiber. " +
+            "Independent of `screenshot` — verify can run with screenshot=false (the diff is computed internally; image bytes are dropped). " +
+            "When skipped, the response contains `verification: { skipped: true, skippedReason }` so callers can tell apart \"ran clean\" from \"never ran\"."),
         burst: z
             .boolean()
             .optional()
@@ -1292,12 +1303,15 @@ registerToolWithTelemetry("ios_screenshot", {
         // explicitly when they need the tree. The Pressable elements block below is
         // the signal most consumers actually want.
         let pressablesText = null;
-        const canEnrich = !!targetApp;
-        // Enrich with pressable elements (filtered list of tappable components, converted to screenshot pixel coords)
+        // Enrich with pressable elements. With targetApp present we use the
+        // fiber path (most accurate). Without targetApp, getPressableElements
+        // falls back to iOS accessibility tree (E1) when we pass platform.
         try {
-            if (!canEnrich)
-                throw new Error("skip");
-            const pressables = await getPressableElements({ device: targetDeviceName });
+            const pressables = await getPressableElements({
+                device: targetDeviceName,
+                platform: "ios",
+                udid: resolvedUdid ?? undefined
+            });
             if (pressables.success && pressables.parsedElements && pressables.parsedElements.length > 0) {
                 const screenshotScale = result.scaleFactor || 1;
                 pressablesText = pressables.parsedElements.map((el) => {
@@ -1338,7 +1352,7 @@ registerToolWithTelemetry("ios_screenshot", {
             infoText += `\n  • get_screen_layout — full component tree when you need more than pressables`;
         }
         else {
-            if (!canEnrich && connectedApps.size > 0) {
+            if (!targetApp && connectedApps.size > 0) {
                 infoText += `\n\nℹ️ Pressable enrichment skipped: no RN app is connected to simulator ${resolvedUdid}.`;
                 infoText += ` ${connectedApps.size} other app(s) are connected on different device(s) — their fiber data was intentionally not used to avoid mismatched output.`;
             }
@@ -1349,7 +1363,7 @@ registerToolWithTelemetry("ios_screenshot", {
         }
         // Check for LogBox overlay — only on the matching RN app; skip otherwise
         // to avoid surfacing warnings from a different simulator's app.
-        if (canEnrich) {
+        if (targetApp) {
             try {
                 const logBoxState = await detectLogBox(targetDeviceName);
                 if (logBoxState && logBoxState.total > 0) {
@@ -1360,6 +1374,20 @@ registerToolWithTelemetry("ios_screenshot", {
                 // Non-fatal: LogBox detection failure should not break screenshot
             }
         }
+        // I1 (2026-05-16): detect native iOS system overlays (auth sheets, alerts,
+        // permission dialogs) that sit on top of the RN app. The pressables list
+        // reflects the RN screen underneath; without this warning the agent will
+        // happily tap inert RN buttons and loop. Runs whether or not there's a
+        // matching RN app — the overlay belongs to the simulator, not the app.
+        try {
+            const overlay = await detectIOSSystemOverlay(resolvedUdid ?? undefined);
+            if (overlay) {
+                infoText += formatIOSSystemOverlayWarning(overlay);
+            }
+        }
+        catch {
+            // Non-fatal: overlay detection failure should not break screenshot
+        }
         imageBuffer.add({
             id: `ios-${Date.now()}-${Math.random().toString(36).slice(2, 8)}`,
             image: result.data,
@@ -1464,25 +1492,30 @@ registerToolWithTelemetry("android_screenshot", {
         catch {
             // Use defaults
         }
-        // Enrich with screen layout data (component names + tap coordinates)
-        // Android: density ratio = densityDpi / 160 (160dpi = 1dp = 1px baseline)
-        const androidPixelRatio = densityDpi / 160;
-        // Skip enrichment when no RN app is connected to this Android device —
-        // otherwise we would pull fiber data from a different device's app.
-        const canEnrich = !!targetApp;
+        // Enrich with screen layout data (component names + tap coordinates).
+        // On Bridgeless/Fabric Android (the only target architecture we support;
+        // legacy arch is <5% of users and not a priority), both code paths that
+        // feed this enrichment return DEVICE PIXELS:
+        //   - fiber path: React's measureInWindow on Fabric returns native pixels.
+        //   - a11y fallback: uiautomator's bounds are already device pixels.
+        // The earlier formula multiplied by densityDpi/160 on the (incorrect)
+        // assumption that fiber returned DP — inflating every coordinate by
+        // ~2.6× on a 420dpi emulator and producing numbers like (1170, 4054)
+        // for a button visually sitting near (445, 1370) in the JPEG. Drop the
+        // density factor; only the scaleFactor downscale is needed.
         let pressablesText = null;
         // Screen Layout tree previously appended here was dropped — it was noisy
         // (nested Svg/G/Path duplicates). Use get_screen_layout when the tree is needed.
-        // Enrich with pressable elements (filtered list of tappable components, converted to screenshot pixel coords)
+        // Enrich with pressable elements. With targetApp present we use the
+        // fiber path (most accurate). Without targetApp, getPressableElements
+        // falls back to the Android accessibility (uiautomator) tree (E1).
         try {
-            if (!canEnrich)
-                throw new Error("skip");
-            const pressables = await getPressableElements({ device: targetDeviceName });
+            const pressables = await getPressableElements({ device: targetDeviceName, platform: "android" });
             if (pressables.success && pressables.parsedElements && pressables.parsedElements.length > 0) {
                 const screenshotScale = result.scaleFactor || 1;
                 pressablesText = pressables.parsedElements.map((el) => {
-                    const px = Math.round((el.center.x * androidPixelRatio) / screenshotScale);
-                    const py = Math.round((el.center.y * androidPixelRatio) / screenshotScale);
+                    const px = Math.round(el.center.x / screenshotScale);
+                    const py = Math.round(el.center.y / screenshotScale);
                     const label = el.accessibilityLabel || el.text || el.testID || (el.intent ? `${el.intent} icon` : el.component);
                     const idPart = el.testID ? ` testID="${el.testID}"` : "";
                     const kindPart = el.isInput ? " [input]" : "";
@@ -1515,7 +1548,7 @@ registerToolWithTelemetry("android_screenshot", {
             infoText += `\n  • get_screen_layout — full component tree when you need more than pressables`;
         }
         else {
-            if (!canEnrich && connectedApps.size > 0) {
+            if (!targetApp && connectedApps.size > 0) {
                 infoText += `\n\nℹ️ Pressable enrichment skipped: no RN app is connected to device ${deviceId ?? "(default)"}.`;
                 infoText += ` ${connectedApps.size} other app(s) are connected on different device(s) — their fiber data was intentionally not used to avoid mismatched output.`;
             }
@@ -1526,7 +1559,7 @@ registerToolWithTelemetry("android_screenshot", {
         }
         // Check for LogBox overlay — only on the matching RN app; skip otherwise
         // to avoid surfacing warnings from a different device's app.
-        if (canEnrich) {
+        if (targetApp) {
             try {
                 const logBoxState = await detectLogBox(targetDeviceName);
                 if (logBoxState && logBoxState.total > 0) {
@@ -1768,7 +1801,7 @@ registerToolWithTelemetry("clear_logs", {
     if (device) {
         const app = getConnectedAppByDevice(device);
         if (!app)
-            throw new Error(`No connected device matches "${device}"`);
+            throw new UserInputError(`No connected device matches "${device}"`);
         const deviceName = app.deviceInfo.deviceName || app.deviceInfo.title || "unknown";
         const count = getLogBuffer(deviceName).clear();
         return { content: [{ type: "text", text: `Cleared ${count} log entries from ${deviceName}.` }] };
@@ -3190,7 +3223,7 @@ registerToolWithTelemetry("clear_network", {
     if (device) {
         const app = getConnectedAppByDevice(device);
         if (!app)
-            throw new Error(`No connected device matches "${device}"`);
+            throw new UserInputError(`No connected device matches "${device}"`);
         const deviceName = app.deviceInfo.deviceName || app.deviceInfo.title || "unknown";
         totalCleared = getNetworkBuffer(deviceName).clear();
     }
@@ -3863,16 +3896,25 @@ registerToolWithTelemetry("android_input_text", {
         " The text will be input at the current focus point (tap an input field first)." +
         "\nPURPOSE: Send keystrokes to whichever input currently has focus on Android — the tool does NOT focus a field itself." +
         "\nWHEN TO USE: Only after an input is already focused, or when `tap(text=...)` on the input didn't take focus for some reason." +
+        "\nREPLACE MODE: pass replace:true to clear the focused field first (via React onChangeText so controlled state stays consistent), then type the new value. Use for pre-filled fields where appending would corrupt the value." +
         "\nSEE ALSO: call get_usage_guide(topic=\"interact\") for the full UI-interaction playbook.",
     inputSchema: {
         text: z.string().describe("Text to type"),
+        replace: z
+            .boolean()
+            .optional()
+            .describe("If true, clear the focused TextInput via React onChangeText before typing. Use to set a pre-filled field to an exact value without concatenation. Requires Bridgeless/Fabric."),
+        device: z
+            .string()
+            .optional()
+            .describe("Optional RN device name (substring match) — needed by replace:true when multiple RN apps are connected, to disambiguate which device's focused input to clear. Single-device sessions can omit."),
         deviceId: z
             .string()
             .optional()
             .describe("Optional device ID. Uses first available device if not specified.")
     }
-}, async ({ text, deviceId }) => {
-    const result = await androidInputText(text, deviceId);
+}, async ({ text, replace, device, deviceId }) => {
+    const result = await inputTextWithReplace(text, replace === true, (t) => androidInputText(t, deviceId), () => clearFocusedInput(device));
     return {
         content: [
             {
@@ -4254,6 +4296,91 @@ server.registerTool("ios_button", {
         isError: !result.success
     };
 });
+// Tool: Clear focused text input
+registerToolWithTelemetry("clear_focused_input", {
+    description: "Clear the contents of the currently focused TextInput, updating React state correctly so controlled components (Formik, react-hook-form, useState) stay consistent." +
+        "\nPURPOSE: Reset whatever TextInput has focus to empty, with the React state owner notified via onChangeText. Use BEFORE typing a replacement value into a pre-filled field." +
+        "\nWHEN TO USE: After tap(testID=...) focuses an input that already has text. Pair with ios_input_text/android_input_text (or use their replace:true flag for one-shot)." +
+        "\nLIMITATIONS: Requires Bridgeless/Fabric (RN new architecture). Returns 'no focused TextInput' if nothing is focused — does not silently no-op." +
+        "\nSEE ALSO: dismiss_keyboard, ios_input_text({replace:true}), android_input_text({replace:true}). call get_usage_guide(topic=\"interact\") for the full UI-interaction playbook.",
+    inputSchema: {
+        device: z
+            .string()
+            .optional()
+            .describe("Optional device name (substring match). Uses default device if not specified.")
+    }
+}, async ({ device }) => {
+    const result = await clearFocusedInput(device);
+    return {
+        content: [
+            {
+                type: "text",
+                text: result.success ? `Cleared focused input (via ${result.via}).` : `Error: ${result.error}`
+            }
+        ],
+        isError: !result.success
+    };
+});
+// Tool: Dismiss keyboard
+registerToolWithTelemetry("dismiss_keyboard", {
+    description: "Blur the currently focused TextInput, dismissing the on-screen keyboard." +
+        "\nPURPOSE: Close the keyboard when it's blocking content beneath the input, or move focus off an input before a tap that would otherwise be intercepted." +
+        "\nWHEN TO USE: After typing into a field and before tapping a button that is hidden by the keyboard. Or to verify a 'tap outside dismisses' UX is wired up." +
+        "\nLIMITATIONS: Requires Bridgeless/Fabric (RN new architecture). Returns 'no focused TextInput' if nothing is focused.",
+    inputSchema: {
+        device: z
+            .string()
+            .optional()
+            .describe("Optional device name (substring match). Uses default device if not specified.")
+    }
+}, async ({ device }) => {
+    const result = await dismissKeyboard(device);
+    return {
+        content: [
+            {
+                type: "text",
+                text: result.success
+                    ? `Dismissed keyboard (nativeTag ${result.nativeTag}).`
+                    : `Error: ${result.error}`
+            }
+        ],
+        isError: !result.success
+    };
+});
+// Tool: iOS input text
+registerToolWithTelemetry("ios_input_text", {
+    description: "Type text on an iOS simulator." +
+        platformFallbackBanner("`tap(text=...)` — it auto-focuses TextInput via the fiber tree") +
+        " The text is typed into whichever field currently has focus (tap an input first). Mirrors `android_input_text` so cross-platform agents can use `<platform>_input_text` without branching on the iOS driver shell-out." +
+        "\nPURPOSE: Send keystrokes to the focused field on an iOS simulator via the active UI driver (AXe — preferred — or IDB)." +
+        "\nWHEN TO USE: Only after an input is already focused, or when `tap(testID=...)` on the input didn't take focus for some reason. Use the testID-first flow whenever possible — it's faster and survives UI repositioning." +
+        "\nREPLACE MODE: pass replace:true to clear the focused field first (via React onChangeText so controlled state stays consistent), then type the new value. Use for pre-filled fields where appending would corrupt the value." +
+        "\nLIMITATIONS: AXe types via the US-keyboard HID — non-ASCII characters (Cyrillic, CJK, Arabic) may not transmit correctly. If the active driver is AXe and the text contains non-ASCII chars, prefer pasting via the simulator pasteboard or setting IOS_DRIVER=idb." +
+        "\nSEE ALSO: call get_usage_guide(topic=\"interact\") for the full UI-interaction playbook.",
+    inputSchema: {
+        text: z.string().describe("Text to type into the currently focused field."),
+        replace: z
+            .boolean()
+            .optional()
+            .describe("If true, clear the focused TextInput via React onChangeText before typing. Use to set a pre-filled field to an exact value without concatenation. Requires Bridgeless/Fabric."),
+        device: z
+            .string()
+            .optional()
+            .describe("Optional RN device name (substring match) — needed by replace:true when multiple RN apps are connected, to disambiguate which device's focused input to clear. Single-device sessions can omit."),
+        udid: z.string().optional().describe("Optional simulator UDID (from list_ios_simulators). Uses booted simulator if not specified.")
+    }
+}, async ({ text, replace, device, udid }) => {
+    const result = await inputTextWithReplace(text, replace === true, (t) => iosInputText(t, udid), () => clearFocusedInput(device));
+    return {
+        content: [
+            {
+                type: "text",
+                text: result.success ? result.result : `Error: ${result.error}`
+            }
+        ],
+        isError: !result.success
+    };
+});
 // Tool: Activate Pro license
 registerToolWithTelemetry("activate_license", getActivateLicenseConfig(), handleActivateLicense);
 // Tool: Delete account