react-native-ai-debugger 1.9.0 → 1.9.1

package/build/pro/tap.js

@@ -1009,20 +1009,24 @@ async function tryCoordinateStrategy(pixelX, pixelY, platform, lastScreenshot, u
     }
 }
 const SETTLE_DELAY_MS = 800;
-const TAP_TIMEOUT_MS = 20000;
+const TAP_TIMEOUT_MS = 25000;
 const MIN_STRATEGY_BUDGET_MS = 500;
 // Per-strategy budget. OCR cap on Android is bumped via maxStrategyMs() because
 // the ADB screencap+pull leg has ~2s variance on real devices; iOS stays at 5s
 // where xcrun simctl screenshot is consistent.
+// Heavy strategies (fiber on deep trees with multi-depth retries, axe accessibility
+// dumps on dense iOS screens) need more headroom — previous caps produced spurious
+// timeouts that the agent read as "element missing" when the strategy simply didn't
+// get to finish. The overall TAP_TIMEOUT_MS budget still bounds the worst case.
 const MAX_STRATEGY_MS = {
-    fiber: 5000,
-    accessibility: 3000,
-    ocr: 5000,
+    fiber: 8000,
+    accessibility: 6000,
+    ocr: 6000,
     coordinate: 3000
 };
 function maxStrategyMs(strategy, platform) {
     if (strategy === "ocr" && platform === "android")
-        return 7000;
+        return 9000;
     return MAX_STRATEGY_MS[strategy] ?? 5000;
 }
 // Matches only the outer withTimeout wrapper message for a tap strategy.
@@ -1030,7 +1034,7 @@ function maxStrategyMs(strategy, platform) {
 // must NOT be classified as a tap-level timeout.
 const STRATEGY_TIMEOUT_RE = /^(fiber|accessibility|ocr|coordinate) timed out after \d+ms$/;
 export function isTapTimeout(attempted) {
-    return attempted.some((a) => STRATEGY_TIMEOUT_RE.test(a.reason) || a.reason.startsWith("Skipped —"));
+    return attempted.some((a) => a.outcome === "timeout" || a.outcome === "skipped" || STRATEGY_TIMEOUT_RE.test(a.reason) || a.reason.startsWith("Skipped —"));
 }
 function withTimeout(promise, ms, label) {
     return new Promise((resolve, reject) => {
@@ -1181,14 +1185,16 @@ async function verifyAndCapture(platform, shouldVerify, shouldScreenshot, before
     const afterWithMarker = markerPx
         ? await drawTapMarker(after.buffer, markerPx.x, markerPx.y)
         : after.buffer;
-    const screenshot = shouldScreenshot
-        ? {
-            image: screenshotToBase64(afterWithMarker),
-            width: after.width,
-            height: after.height,
-            scaleFactor: after.scaleFactor
-        }
-        : undefined;
+    // Construct the screenshot eagerly. Caller decides whether to drop it
+    // based on `shouldScreenshot` AND the verification outcome — when a tap
+    // is unmeaningful we surface the image regardless so the agent can
+    // diagnose without an extra round-trip.
+    const screenshot = {
+        image: screenshotToBase64(afterWithMarker),
+        width: after.width,
+        height: after.height,
+        scaleFactor: after.scaleFactor
+    };
     let verification;
     if (!shouldVerify) {
         verification = {
@@ -1252,7 +1258,7 @@ async function verifyAndCapture(platform, shouldVerify, shouldScreenshot, before
         groupId: verifyGroupId,
         metadata: { phase: "after", changeRate: verification?.changeRate }
     });
-    return { screenshot, verification, afterBuffer: after.buffer, afterWithMarkerBuffer: afterWithMarker };
+    return { screenshot, verification, afterWithMarkerBuffer: afterWithMarker };
 }
 // --- Burst Capture ---
 const BURST_FRAME_COUNT = 4;
@@ -1348,7 +1354,6 @@ async function burstCaptureAndVerify(platform, beforeBuffer, udid, beforeScaleFa
     return {
         screenshot,
         verification,
-        afterBuffer: frames[lastFrameIdx],
         afterWithMarkerBuffer: markedFrames[lastFrameIdx]
     };
 }
@@ -1478,7 +1483,6 @@ async function captureTapArtifact(ctx) {
             chosenElement: null,
             screenshots: {
                 before: ctx.screenshotBuffer ?? null,
-                after: ctx.afterBuffer ?? null,
                 afterWithMarker: ctx.afterWithMarker ?? null
             },
             deviceMeta: {
@@ -1813,14 +1817,16 @@ export async function tap(options) {
         if (strat === "fiber" && !hasMetro) {
             attempted.push({
                 strategy: "fiber",
-                reason: "Skipped — no Metro connection (required for fiber)"
+                reason: "Skipped — no Metro connection (required for fiber)",
+                outcome: "skipped"
             });
             return false;
         }
         if (uiDriverMissing && UI_DRIVER_REQUIRED_STRATEGIES.includes(strat)) {
             attempted.push({
                 strategy: strat,
-                reason: "Skipped — iOS UI driver is not installed (required for iOS tap/accessibility/OCR)"
+                reason: "Skipped — iOS UI driver is not installed (required for iOS tap/accessibility/OCR)",
+                outcome: "skipped"
             });
             return false;
         }
@@ -1845,10 +1851,12 @@ export async function tap(options) {
     // capture cost is paid either way; bandwidth cost is what `screenshot` toggles.
     const shouldScreenshot = options.screenshot !== false;
     const canVerify = options.verify !== false;
-    // Capture "before" screenshot for verification
+    // Capture "before" screenshot. Always attempted (independent of canVerify)
+    // so failure artifacts always carry a before.png — the diagnostic value of
+    // the before frame doesn't depend on whether we run the post-tap diff.
     let beforeBuffer = null;
     let beforeScaleFactor;
-    if (canVerify) {
+    {
         const before = await captureScreenshot(platform, targetUdid);
         beforeBuffer = before?.buffer || null;
         beforeScaleFactor = before?.scaleFactor;
@@ -1876,7 +1884,8 @@ export async function tap(options) {
         if (remaining < MIN_STRATEGY_BUDGET_MS) {
             attempted.push({
                 strategy: strat,
-                reason: `Skipped — only ${remaining}ms remaining (budget ${TAP_TIMEOUT_MS}ms)`
+                reason: `Skipped — only ${remaining}ms remaining (budget ${TAP_TIMEOUT_MS}ms)`,
+                outcome: "skipped"
             });
             continue;
         }
@@ -1949,7 +1958,13 @@ export async function tap(options) {
                 // Fall through past the catch block to the success handler.
             }
             else {
-                attempted.push({ strategy: strat, reason });
+                // Classify the throw: a strategy-level timeout means "result UNKNOWN"
+                // (the strategy didn't finish), distinct from "ran clean and found
+                // nothing". Agents must not infer absence from timeouts. The
+                // `outcome` field carries that distinction; `reason` stays the raw
+                // message so existing log-matchers / classifiers keep working.
+                const outcome = STRATEGY_TIMEOUT_RE.test(reason) ? "timeout" : "error";
+                attempted.push({ strategy: strat, reason, outcome });
                 continue;
             }
         }
@@ -1960,7 +1975,6 @@ export async function tap(options) {
                 ocrPrewarmCtrl?.abort();
             let screenshot;
             let verification;
-            let afterBuffer;
             let afterWithMarkerBuffer;
             let dprForMarker;
             if (platform === "ios") {
@@ -1981,15 +1995,22 @@ export async function tap(options) {
                 devicePixelRatio: dprForMarker
             });
             if (options.burst && canVerify && beforeBuffer) {
-                ({ screenshot, verification, afterBuffer, afterWithMarkerBuffer } = await burstCaptureAndVerify(platform, beforeBuffer, targetUdid, beforeScaleFactor, strategyMarker));
+                ({ screenshot, verification, afterWithMarkerBuffer } = await burstCaptureAndVerify(platform, beforeBuffer, targetUdid, beforeScaleFactor, strategyMarker));
                 // Burst always produces a screenshot for diff visualization. Drop the
-                // bytes when the caller didn't ask for them — frames are still in
-                // imageBuffer (accessible via get_images(groupId=verification.burstGroupId)).
-                if (!shouldScreenshot)
+                // bytes when the caller didn't ask for them AND the tap was meaningful
+                // — keeping the screenshot on unmeaningful taps saves the agent a
+                // round-trip when it needs to diagnose. Frames remain in imageBuffer
+                // either way (accessible via get_images(groupId=verification.burstGroupId)).
+                if (!shouldScreenshot && verification?.meaningful !== false)
                     screenshot = undefined;
             }
             else {
-                ({ screenshot, verification, afterBuffer, afterWithMarkerBuffer } = await verifyAndCapture(platform, canVerify, shouldScreenshot, beforeBuffer, targetUdid, beforeScaleFactor, strategyMarker));
+                ({ screenshot, verification, afterWithMarkerBuffer } = await verifyAndCapture(platform, canVerify, 
+                /* shouldScreenshot */ true, beforeBuffer, targetUdid, beforeScaleFactor, strategyMarker));
+                // Mirror the burst-path gate: drop the screenshot only when the
+                // caller didn't ask for it AND the tap was meaningful.
+                if (!shouldScreenshot && verification?.meaningful !== false)
+                    screenshot = undefined;
             }
             if (screenshot && app) {
                 app.lastScreenshot = {
@@ -2014,7 +2035,9 @@ export async function tap(options) {
             });
             // Capture an artifact for "successful but unmeaningful" taps so we can
             // diagnose taps that landed wrong or hit non-responsive elements.
-            if (verification && !verification.skipped && (verification.changeRate ?? 1) < 0.001) {
+            // Uses the same `meaningful` flag the agent sees so dashboard outcomes
+            // stay consistent with what the caller observed.
+            if (verification && !verification.skipped && verification.meaningful === false) {
                 const unmeaningfulSignals = await captureTapArtifact({
                     query,
                     outcome: "unmeaningful",
@@ -2024,7 +2047,6 @@ export async function tap(options) {
                     deviceName,
                     screenshotMeta: screenshot ? { width: screenshot.width, height: screenshot.height } : undefined,
                     screenshotBuffer: beforeBuffer,
-                    afterBuffer: afterBuffer ?? null,
                     afterWithMarker: afterWithMarkerBuffer ?? null,
                     chosenTapPoint: strategyMarker ? { x: strategyMarker.x, y: strategyMarker.y } : null,
                     verification,
@@ -2034,7 +2056,19 @@ export async function tap(options) {
             }
             return successResult;
         }
-        attempted.push({ strategy: strat, reason: result.reason });
+        // Classify the strategy's non-success result. The reason string carries
+        // the diagnostic from the strategy; outcome lets agents branch on category
+        // without parsing prose. "invisible" / "ambiguous" are pinned by substring
+        // matches the fiber strategy emits via pressables.ts.
+        const reasonStr = result.reason || "";
+        let attemptOutcome = "not-found";
+        if (result.ambiguous)
+            attemptOutcome = "ambiguous";
+        else if (reasonStr.indexOf("but none are visible") !== -1)
+            attemptOutcome = "invisible";
+        else if (STRATEGY_TIMEOUT_RE.test(reasonStr))
+            attemptOutcome = "timeout";
+        attempted.push({ strategy: strat, reason: reasonStr, outcome: attemptOutcome });
         // If fiber found an element with measured coordinates, do a native tap directly
         if (strat === "fiber" && result.convertedTo && result.pressed) {
             try {
@@ -2092,8 +2126,11 @@ export async function tap(options) {
                     ({ screenshot, verification } = await burstCaptureAndVerify(platform, beforeBuffer, targetUdid, beforeScaleFactor, fiberMarker));
                 }
                 else {
-                    ({ screenshot, verification } = await verifyAndCapture(platform, canVerify, shouldScreenshot, beforeBuffer, targetUdid, beforeScaleFactor, fiberMarker));
+                    ({ screenshot, verification } = await verifyAndCapture(platform, canVerify, 
+                    /* shouldScreenshot */ true, beforeBuffer, targetUdid, beforeScaleFactor, fiberMarker));
                 }
+                if (!shouldScreenshot && verification?.meaningful !== false)
+                    screenshot = undefined;
                 if (screenshot && app) {
                     app.lastScreenshot = {
                         originalWidth: screenshot.width,
@@ -2151,7 +2188,20 @@ export async function tap(options) {
     // itself ran out of time.
     const hitTimeout = isTapTimeout(attempted);
     const elapsed = TAP_TIMEOUT_MS - remainingMs();
-    const suggestion = buildSuggestion(query, strategies, platform);
+    // If fiber located the element but visibility filtered it out, prepend a
+    // scroll-or-dismiss hint so the agent doesn't waste a turn assuming the
+    // testID/component is missing.
+    const fiberSawInvisible = attempted.some(a => a.strategy === "fiber" && a.outcome === "invisible");
+    const allTimedOut = attempted.length > 0 && attempted.every(a => a.outcome === "timeout" || a.outcome === "skipped");
+    let suggestion = buildSuggestion(query, strategies, platform);
+    if (fiberSawInvisible) {
+        suggestion = `Element exists in the React tree but is not on screen. Try scrolling it into view (${platform === "ios" ? "ios_swipe / arrow keys" : "android_swipe"}), dismiss any overlay covering it, or wait for layout to settle. ` + suggestion;
+    }
+    else if (allTimedOut) {
+        // No strategy got to a definitive answer — don't let the agent conclude
+        // the element is missing. Steer toward retry or a different strategy.
+        suggestion = `All strategies timed out — the element's presence is UNKNOWN. Retry the tap (transient slowness is common on dense screens), or try a different strategy explicitly (e.g. strategy='fiber' if accessibility timed out). ` + suggestion;
+    }
     const { screenshot: failScreenshot } = shouldScreenshot
         ? await verifyAndCapture(platform, false, true, null, targetUdid)
         : { screenshot: undefined };
@@ -2162,10 +2212,22 @@ export async function tap(options) {
             scaleFactor: failScreenshot.scaleFactor
         };
     }
+    // Pick the right error framing: a definitive "not found" is different from
+    // "all strategies timed out — we don't know whether it exists".
+    let errorOverride;
+    if (hitTimeout) {
+        errorOverride = `Tap timed out after ${elapsed}ms (budget ${TAP_TIMEOUT_MS}ms)`;
+    }
+    else if (allTimedOut) {
+        errorOverride = `All tap strategies timed out before completing — element presence is UNKNOWN`;
+    }
+    else if (fiberSawInvisible) {
+        errorOverride = `Element matches the query but is not visible on screen`;
+    }
     const failureResult = formatTapFailure({
         query,
         attempted,
-        error: hitTimeout ? `Tap timed out after ${elapsed}ms (budget ${TAP_TIMEOUT_MS}ms)` : undefined,
+        error: errorOverride,
         suggestion,
         device: deviceName,
         screenshot: failScreenshot