react-native-ai-debugger 1.9.1 → 1.10.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -1
- package/build/__tests__/integration/tap-strategies.test.js +1 -1
- package/build/__tests__/integration/tap-strategies.test.js.map +1 -1
- package/build/__tests__/unit/androidEmulatorAvds.test.d.ts +2 -0
- package/build/__tests__/unit/androidEmulatorAvds.test.d.ts.map +1 -0
- package/build/__tests__/unit/androidEmulatorAvds.test.js +98 -0
- package/build/__tests__/unit/androidEmulatorAvds.test.js.map +1 -0
- package/build/__tests__/unit/deviceDiscovery.test.d.ts +2 -0
- package/build/__tests__/unit/deviceDiscovery.test.d.ts.map +1 -0
- package/build/__tests__/unit/deviceDiscovery.test.js +119 -0
- package/build/__tests__/unit/deviceDiscovery.test.js.map +1 -0
- package/build/__tests__/unit/deviceResolver.test.d.ts +2 -0
- package/build/__tests__/unit/deviceResolver.test.d.ts.map +1 -0
- package/build/__tests__/unit/deviceResolver.test.js +226 -0
- package/build/__tests__/unit/deviceResolver.test.js.map +1 -0
- package/build/__tests__/unit/tap.test.js +13 -30
- package/build/__tests__/unit/tap.test.js.map +1 -1
- package/build/__tests__/unit/verifyAction.test.d.ts +2 -0
- package/build/__tests__/unit/verifyAction.test.d.ts.map +1 -0
- package/build/__tests__/unit/verifyAction.test.js +26 -0
- package/build/__tests__/unit/verifyAction.test.js.map +1 -0
- package/build/core/android.d.ts +20 -0
- package/build/core/android.d.ts.map +1 -1
- package/build/core/android.js +98 -0
- package/build/core/android.js.map +1 -1
- package/build/core/connection.d.ts.map +1 -1
- package/build/core/connection.js +20 -9
- package/build/core/connection.js.map +1 -1
- package/build/core/deviceDiscovery.d.ts +52 -0
- package/build/core/deviceDiscovery.d.ts.map +1 -0
- package/build/core/deviceDiscovery.js +74 -0
- package/build/core/deviceDiscovery.js.map +1 -0
- package/build/core/deviceResolver.d.ts +44 -0
- package/build/core/deviceResolver.d.ts.map +1 -0
- package/build/core/deviceResolver.js +191 -0
- package/build/core/deviceResolver.js.map +1 -0
- package/build/core/guides.d.ts.map +1 -1
- package/build/core/guides.js +3 -4
- package/build/core/guides.js.map +1 -1
- package/build/core/types.d.ts +1 -0
- package/build/core/types.d.ts.map +1 -1
- package/build/pro/tap.d.ts +5 -10
- package/build/pro/tap.d.ts.map +1 -1
- package/build/pro/tap.js +98 -395
- package/build/pro/tap.js.map +1 -1
- package/build/pro/verifyAction.d.ts +49 -0
- package/build/pro/verifyAction.d.ts.map +1 -0
- package/build/pro/verifyAction.js +255 -0
- package/build/pro/verifyAction.js.map +1 -0
- package/build/tools/connectionTools.d.ts.map +1 -1
- package/build/tools/connectionTools.js +27 -2
- package/build/tools/connectionTools.js.map +1 -1
- package/build/tools/deviceTools.d.ts.map +1 -1
- package/build/tools/deviceTools.js +104 -2
- package/build/tools/deviceTools.js.map +1 -1
- package/build/tools/interactionTools.d.ts.map +1 -1
- package/build/tools/interactionTools.js +113 -151
- package/build/tools/interactionTools.js.map +1 -1
- package/package.json +1 -1
package/build/pro/tap.js
CHANGED
|
@@ -1,29 +1,34 @@
|
|
|
1
|
-
import { connectedApps
|
|
1
|
+
import { connectedApps } from "../core/state.js";
|
|
2
2
|
import { executeInApp } from "../core/executor.js";
|
|
3
3
|
import { pressElement } from "../core/executor.js";
|
|
4
|
-
import { iosTap, iosFindElement, iosScreenshot,
|
|
5
|
-
import { androidTap, androidFindElement
|
|
4
|
+
import { iosTap, iosFindElement, iosScreenshot, isUiDriverAvailable, getUiDriverInstallHint, getIOSSafeAreaTop } from "../core/ios.js";
|
|
5
|
+
import { androidTap, androidFindElement } from "../core/android.js";
|
|
6
6
|
import { compareScreenshots } from "./screenshot-diff.js";
|
|
7
7
|
import { scanMetroPorts, fetchDevices, selectMainDevice } from "../core/metro.js";
|
|
8
8
|
import { connectToDevice, clearReconnectionSuppression, getConnectedAppByDevice } from "../core/connection.js";
|
|
9
|
+
import { resolveDeviceTarget, formatResolverError } from "../core/deviceResolver.js";
|
|
9
10
|
import { notifyDriverMissing } from "../core/logbox.js";
|
|
10
11
|
import { captureFailureArtifact } from "../core/failureArtifact.js";
|
|
12
|
+
import { captureScreenshot, verifyAndCapture, burstCaptureAndVerify, } from "./verifyAction.js";
|
|
11
13
|
export function buildVerificationExplanation(v) {
|
|
12
14
|
const pct = (rate) => (rate * 100).toFixed(1) + "%";
|
|
15
|
+
const action = v.action ?? "tap";
|
|
16
|
+
const Action = action[0].toUpperCase() + action.slice(1);
|
|
17
|
+
const target = action === "swipe" ? "scroll surface" : "element";
|
|
13
18
|
if (v.meaningful && !v.transientChangeDetected) {
|
|
14
|
-
return
|
|
19
|
+
return `${Action} caused a visible UI change (${pct(v.changeRate)} pixel diff). The screen updated as expected.`;
|
|
15
20
|
}
|
|
16
21
|
if (v.meaningful && v.transientChangeDetected) {
|
|
17
22
|
return (`No persistent change, but transient visual feedback detected ` +
|
|
18
23
|
`(frame ${v.peakFrame} peak ${pct(v.peakChangeRate || 0)} diff). ` +
|
|
19
|
-
|
|
24
|
+
`${Action} triggered a momentary animation that settled back to original state.`);
|
|
20
25
|
}
|
|
21
26
|
if (v.transientChangeDetected === false) {
|
|
22
27
|
return (`No visual change detected — neither persistent nor transient across burst frames. ` +
|
|
23
|
-
`The
|
|
28
|
+
`The ${target} may not respond visually or the ${action} may have missed.`);
|
|
24
29
|
}
|
|
25
30
|
return (`No visual change detected between before and after screenshots. ` +
|
|
26
|
-
`The
|
|
31
|
+
`The ${target} may not respond visually or the ${action} may have missed.`);
|
|
27
32
|
}
|
|
28
33
|
const BURST_CHANGE_THRESHOLD = 0.005;
|
|
29
34
|
export async function analyzeBurstFrames(frames, options) {
|
|
@@ -1008,7 +1013,6 @@ async function tryCoordinateStrategy(pixelX, pixelY, platform, lastScreenshot, u
|
|
|
1008
1013
|
};
|
|
1009
1014
|
}
|
|
1010
1015
|
}
|
|
1011
|
-
const SETTLE_DELAY_MS = 800;
|
|
1012
1016
|
const TAP_TIMEOUT_MS = 25000;
|
|
1013
1017
|
const MIN_STRATEGY_BUDGET_MS = 500;
|
|
1014
1018
|
// Per-strategy budget. OCR cap on Android is bumped via maxStrategyMs() because
|
|
@@ -1064,54 +1068,6 @@ function withCancelableTimeout(make, ms, label) {
|
|
|
1064
1068
|
throw err;
|
|
1065
1069
|
});
|
|
1066
1070
|
}
|
|
1067
|
-
async function captureScreenshot(platform, udid) {
|
|
1068
|
-
try {
|
|
1069
|
-
const result = platform === "ios" ? await iosScreenshot(undefined, udid) : await androidScreenshot();
|
|
1070
|
-
if (!result.success || !result.data)
|
|
1071
|
-
return null;
|
|
1072
|
-
return {
|
|
1073
|
-
buffer: result.data,
|
|
1074
|
-
width: result.originalWidth || 0,
|
|
1075
|
-
height: result.originalHeight || 0,
|
|
1076
|
-
scaleFactor: result.scaleFactor || 1
|
|
1077
|
-
};
|
|
1078
|
-
}
|
|
1079
|
-
catch {
|
|
1080
|
-
return null;
|
|
1081
|
-
}
|
|
1082
|
-
}
|
|
1083
|
-
function screenshotToBase64(buffer) {
|
|
1084
|
-
return buffer.toString("base64");
|
|
1085
|
-
}
|
|
1086
|
-
/**
|
|
1087
|
-
* Composite a red crosshair marker onto a screenshot at the given pixel coordinates.
|
|
1088
|
-
* Uses sharp + SVG so we don't need to inject into the app. Coordinates are in the
|
|
1089
|
-
* screenshot's own pixel space (i.e. the space the returned image uses).
|
|
1090
|
-
*/
|
|
1091
|
-
async function drawTapMarker(input, x, y) {
|
|
1092
|
-
try {
|
|
1093
|
-
const sharp = (await import("sharp")).default;
|
|
1094
|
-
const size = 72;
|
|
1095
|
-
const half = size / 2;
|
|
1096
|
-
const svg = `<svg width="${size}" height="${size}" xmlns="http://www.w3.org/2000/svg">
|
|
1097
|
-
<circle cx="${half}" cy="${half}" r="${half - 8}" fill="none" stroke="white" stroke-width="6" opacity="0.85"/>
|
|
1098
|
-
<circle cx="${half}" cy="${half}" r="${half - 8}" fill="none" stroke="#FF2D55" stroke-width="3" opacity="1"/>
|
|
1099
|
-
<line x1="${half}" y1="8" x2="${half}" y2="${size - 8}" stroke="white" stroke-width="6" opacity="0.85"/>
|
|
1100
|
-
<line x1="8" y1="${half}" x2="${size - 8}" y2="${half}" stroke="white" stroke-width="6" opacity="0.85"/>
|
|
1101
|
-
<line x1="${half}" y1="8" x2="${half}" y2="${size - 8}" stroke="#FF2D55" stroke-width="2.5"/>
|
|
1102
|
-
<line x1="8" y1="${half}" x2="${size - 8}" y2="${half}" stroke="#FF2D55" stroke-width="2.5"/>
|
|
1103
|
-
<circle cx="${half}" cy="${half}" r="3" fill="#FF2D55"/>
|
|
1104
|
-
</svg>`;
|
|
1105
|
-
const left = Math.round(x - half);
|
|
1106
|
-
const top = Math.round(y - half);
|
|
1107
|
-
return await sharp(input)
|
|
1108
|
-
.composite([{ input: Buffer.from(svg), left, top }])
|
|
1109
|
-
.toBuffer();
|
|
1110
|
-
}
|
|
1111
|
-
catch {
|
|
1112
|
-
return input;
|
|
1113
|
-
}
|
|
1114
|
-
}
|
|
1115
1071
|
/**
|
|
1116
1072
|
* Compute marker coordinates in screenshot-pixel space (what the returned PNG uses).
|
|
1117
1073
|
* Unit rules by strategy, assuming screenshotScale = downscale factor:
|
|
@@ -1154,209 +1110,6 @@ function computeMarkerPx(args) {
|
|
|
1154
1110
|
y: Math.round(convertedTo.y / scale)
|
|
1155
1111
|
};
|
|
1156
1112
|
}
|
|
1157
|
-
async function verifyAndCapture(platform, shouldVerify, shouldScreenshot, beforeBuffer, udid, beforeScaleFactor, markerPx) {
|
|
1158
|
-
// Decoupled 2026-05-16 (I5): screenshot and verify are independent knobs.
|
|
1159
|
-
// - shouldVerify=true: capture after-screenshot internally, diff against beforeBuffer, return verification.
|
|
1160
|
-
// - shouldScreenshot=true: also return the (marker-overlaid) image bytes in the response.
|
|
1161
|
-
// - both false: fastest path, return only a `skipped` verification stub so the agent can tell apart "ran clean" from "skipped intentionally".
|
|
1162
|
-
if (!shouldVerify && !shouldScreenshot) {
|
|
1163
|
-
return {
|
|
1164
|
-
verification: {
|
|
1165
|
-
skipped: true,
|
|
1166
|
-
skippedReason: "screenshot=false, verify=false",
|
|
1167
|
-
explanation: "Verification skipped (screenshot=false and verify=false)."
|
|
1168
|
-
}
|
|
1169
|
-
};
|
|
1170
|
-
}
|
|
1171
|
-
await new Promise((resolve) => setTimeout(resolve, SETTLE_DELAY_MS));
|
|
1172
|
-
const after = await captureScreenshot(platform, udid);
|
|
1173
|
-
if (!after) {
|
|
1174
|
-
if (shouldVerify) {
|
|
1175
|
-
return {
|
|
1176
|
-
verification: {
|
|
1177
|
-
skipped: true,
|
|
1178
|
-
skippedReason: "after-screenshot capture failed",
|
|
1179
|
-
explanation: "Verification skipped — could not capture post-tap screenshot."
|
|
1180
|
-
}
|
|
1181
|
-
};
|
|
1182
|
-
}
|
|
1183
|
-
return {};
|
|
1184
|
-
}
|
|
1185
|
-
const afterWithMarker = markerPx
|
|
1186
|
-
? await drawTapMarker(after.buffer, markerPx.x, markerPx.y)
|
|
1187
|
-
: after.buffer;
|
|
1188
|
-
// Construct the screenshot eagerly. Caller decides whether to drop it
|
|
1189
|
-
// based on `shouldScreenshot` AND the verification outcome — when a tap
|
|
1190
|
-
// is unmeaningful we surface the image regardless so the agent can
|
|
1191
|
-
// diagnose without an extra round-trip.
|
|
1192
|
-
const screenshot = {
|
|
1193
|
-
image: screenshotToBase64(afterWithMarker),
|
|
1194
|
-
width: after.width,
|
|
1195
|
-
height: after.height,
|
|
1196
|
-
scaleFactor: after.scaleFactor
|
|
1197
|
-
};
|
|
1198
|
-
let verification;
|
|
1199
|
-
if (!shouldVerify) {
|
|
1200
|
-
verification = {
|
|
1201
|
-
skipped: true,
|
|
1202
|
-
skippedReason: "verify=false",
|
|
1203
|
-
explanation: "Verification skipped (verify=false)."
|
|
1204
|
-
};
|
|
1205
|
-
}
|
|
1206
|
-
else if (!beforeBuffer) {
|
|
1207
|
-
verification = {
|
|
1208
|
-
skipped: true,
|
|
1209
|
-
skippedReason: "before-screenshot unavailable",
|
|
1210
|
-
explanation: "Verification skipped — could not capture pre-tap screenshot."
|
|
1211
|
-
};
|
|
1212
|
-
}
|
|
1213
|
-
else {
|
|
1214
|
-
try {
|
|
1215
|
-
const rawStatusBar = platform === "ios" ? 177 : 142; // pixels in original screenshot space
|
|
1216
|
-
const scale = beforeScaleFactor || after.scaleFactor || 1;
|
|
1217
|
-
const statusBarHeight = Math.round(rawStatusBar / scale);
|
|
1218
|
-
const diff = await compareScreenshots(beforeBuffer, after.buffer, {
|
|
1219
|
-
statusBarHeight
|
|
1220
|
-
});
|
|
1221
|
-
verification = {
|
|
1222
|
-
meaningful: diff.changed,
|
|
1223
|
-
changeRate: diff.changeRate,
|
|
1224
|
-
changedPixels: diff.changedPixels,
|
|
1225
|
-
totalPixels: diff.totalPixels,
|
|
1226
|
-
explanation: buildVerificationExplanation({
|
|
1227
|
-
meaningful: diff.changed,
|
|
1228
|
-
changeRate: diff.changeRate,
|
|
1229
|
-
changedPixels: diff.changedPixels,
|
|
1230
|
-
totalPixels: diff.totalPixels
|
|
1231
|
-
})
|
|
1232
|
-
};
|
|
1233
|
-
}
|
|
1234
|
-
catch (err) {
|
|
1235
|
-
verification = {
|
|
1236
|
-
skipped: true,
|
|
1237
|
-
skippedReason: `diff failed: ${err instanceof Error ? err.message : String(err)}`,
|
|
1238
|
-
explanation: "Verification skipped — pixel diff threw."
|
|
1239
|
-
};
|
|
1240
|
-
}
|
|
1241
|
-
}
|
|
1242
|
-
const verifyGroupId = `verify-${Date.now()}-${Math.random().toString(36).slice(2, 8)}`;
|
|
1243
|
-
if (beforeBuffer) {
|
|
1244
|
-
imageBuffer.add({
|
|
1245
|
-
id: `${verifyGroupId}-before`,
|
|
1246
|
-
image: beforeBuffer,
|
|
1247
|
-
timestamp: Date.now(),
|
|
1248
|
-
source: "tap-verify",
|
|
1249
|
-
groupId: verifyGroupId,
|
|
1250
|
-
metadata: { phase: "before" }
|
|
1251
|
-
});
|
|
1252
|
-
}
|
|
1253
|
-
imageBuffer.add({
|
|
1254
|
-
id: `${verifyGroupId}-after`,
|
|
1255
|
-
image: afterWithMarker,
|
|
1256
|
-
timestamp: Date.now(),
|
|
1257
|
-
source: "tap-verify",
|
|
1258
|
-
groupId: verifyGroupId,
|
|
1259
|
-
metadata: { phase: "after", changeRate: verification?.changeRate }
|
|
1260
|
-
});
|
|
1261
|
-
return { screenshot, verification, afterWithMarkerBuffer: afterWithMarker };
|
|
1262
|
-
}
|
|
1263
|
-
// --- Burst Capture ---
|
|
1264
|
-
const BURST_FRAME_COUNT = 4;
|
|
1265
|
-
const BURST_FRAME_INTERVAL_MS = 150;
|
|
1266
|
-
async function burstCaptureAndVerify(platform, beforeBuffer, udid, beforeScaleFactor, markerPx) {
|
|
1267
|
-
if (!beforeBuffer)
|
|
1268
|
-
return {};
|
|
1269
|
-
const frames = [beforeBuffer];
|
|
1270
|
-
let capturedScaleFactor = beforeScaleFactor || 1;
|
|
1271
|
-
for (let i = 0; i < BURST_FRAME_COUNT; i++) {
|
|
1272
|
-
await new Promise((resolve) => setTimeout(resolve, BURST_FRAME_INTERVAL_MS));
|
|
1273
|
-
const capture = await captureScreenshot(platform, udid);
|
|
1274
|
-
if (capture) {
|
|
1275
|
-
frames.push(capture.buffer);
|
|
1276
|
-
if (i === 0)
|
|
1277
|
-
capturedScaleFactor = capture.scaleFactor || capturedScaleFactor;
|
|
1278
|
-
}
|
|
1279
|
-
}
|
|
1280
|
-
if (frames.length < 2)
|
|
1281
|
-
return {};
|
|
1282
|
-
const rawStatusBar = platform === "ios" ? 177 : 142; // pixels in original screenshot space
|
|
1283
|
-
const statusBarHeight = Math.round(rawStatusBar / capturedScaleFactor);
|
|
1284
|
-
// Run the diff on clean frames (no marker) so the cross doesn't register as change.
|
|
1285
|
-
const analysis = await analyzeBurstFrames(frames, { statusBarHeight });
|
|
1286
|
-
// Apply marker to post-tap frames only (frames[0] is the before state).
|
|
1287
|
-
const markedFrames = [];
|
|
1288
|
-
for (let i = 0; i < frames.length; i++) {
|
|
1289
|
-
if (markerPx && i > 0) {
|
|
1290
|
-
markedFrames.push(await drawTapMarker(frames[i], markerPx.x, markerPx.y));
|
|
1291
|
-
}
|
|
1292
|
-
else {
|
|
1293
|
-
markedFrames.push(frames[i]);
|
|
1294
|
-
}
|
|
1295
|
-
}
|
|
1296
|
-
const groupId = `burst-${Date.now()}-${Math.random().toString(36).slice(2, 8)}`;
|
|
1297
|
-
for (let i = 0; i < markedFrames.length; i++) {
|
|
1298
|
-
imageBuffer.add({
|
|
1299
|
-
id: `${groupId}-f${i}`,
|
|
1300
|
-
image: markedFrames[i],
|
|
1301
|
-
timestamp: Date.now(),
|
|
1302
|
-
source: "tap-burst",
|
|
1303
|
-
groupId,
|
|
1304
|
-
metadata: {
|
|
1305
|
-
frameIndex: i,
|
|
1306
|
-
isBefore: i === 0,
|
|
1307
|
-
changeRate: i === 0 ? 0 : analysis.framesWithChange.includes(i) ? analysis.peakChangeRate : 0
|
|
1308
|
-
}
|
|
1309
|
-
});
|
|
1310
|
-
}
|
|
1311
|
-
imageBuffer.addGroup({
|
|
1312
|
-
groupId,
|
|
1313
|
-
intent: "tap-verification",
|
|
1314
|
-
source: "tap-burst",
|
|
1315
|
-
timestamp: Date.now(),
|
|
1316
|
-
frameCount: frames.length,
|
|
1317
|
-
summary: {
|
|
1318
|
-
peakChangeRate: analysis.peakChangeRate,
|
|
1319
|
-
peakFrame: analysis.peakFrame,
|
|
1320
|
-
framesWithChange: analysis.framesWithChange,
|
|
1321
|
-
transientChangeDetected: analysis.transientChangeDetected,
|
|
1322
|
-
persistentChangeRate: analysis.persistentChangeRate
|
|
1323
|
-
}
|
|
1324
|
-
});
|
|
1325
|
-
// Get dimensions from the last frame using sharp
|
|
1326
|
-
const sharp = (await import("sharp")).default;
|
|
1327
|
-
const meta = await sharp(markedFrames[markedFrames.length - 1]).metadata();
|
|
1328
|
-
const screenshot = {
|
|
1329
|
-
image: screenshotToBase64(markedFrames[markedFrames.length - 1]),
|
|
1330
|
-
width: meta.width || 0,
|
|
1331
|
-
height: meta.height || 0,
|
|
1332
|
-
scaleFactor: 1
|
|
1333
|
-
};
|
|
1334
|
-
const verification = {
|
|
1335
|
-
meaningful: analysis.meaningful,
|
|
1336
|
-
changeRate: analysis.persistentChangeRate,
|
|
1337
|
-
changedPixels: 0,
|
|
1338
|
-
totalPixels: 0,
|
|
1339
|
-
transientChangeDetected: analysis.transientChangeDetected,
|
|
1340
|
-
peakChangeRate: analysis.peakChangeRate,
|
|
1341
|
-
peakFrame: analysis.peakFrame,
|
|
1342
|
-
burstGroupId: groupId,
|
|
1343
|
-
explanation: buildVerificationExplanation({
|
|
1344
|
-
meaningful: analysis.meaningful,
|
|
1345
|
-
changeRate: analysis.persistentChangeRate,
|
|
1346
|
-
changedPixels: 0,
|
|
1347
|
-
totalPixels: 0,
|
|
1348
|
-
transientChangeDetected: analysis.transientChangeDetected,
|
|
1349
|
-
peakChangeRate: analysis.peakChangeRate,
|
|
1350
|
-
peakFrame: analysis.peakFrame
|
|
1351
|
-
})
|
|
1352
|
-
};
|
|
1353
|
-
const lastFrameIdx = frames.length - 1;
|
|
1354
|
-
return {
|
|
1355
|
-
screenshot,
|
|
1356
|
-
verification,
|
|
1357
|
-
afterWithMarkerBuffer: markedFrames[lastFrameIdx]
|
|
1358
|
-
};
|
|
1359
|
-
}
|
|
1360
1113
|
// D2 (Step 6): in-memory ring buffer of recent coord-strategy artifact keys
|
|
1361
1114
|
// `${round(x/20)}:${round(y/20)}`. Entries expire after 60s. The first
|
|
1362
1115
|
// failed coord tap at a grid cell uploads its artifact; retries within the
|
|
@@ -1554,67 +1307,16 @@ export async function tap(options) {
|
|
|
1554
1307
|
}
|
|
1555
1308
|
// Native mode: bypass React Native connection, tap directly via ADB/simctl
|
|
1556
1309
|
if (options.native && hasCoordinates) {
|
|
1557
|
-
|
|
1558
|
-
|
|
1559
|
-
// surface a clear error rather than silently overriding.
|
|
1560
|
-
if (options.udid && platform === "android") {
|
|
1310
|
+
const nativeResolved = await resolveDeviceTarget(options.device);
|
|
1311
|
+
if (!nativeResolved.ok) {
|
|
1561
1312
|
return {
|
|
1562
1313
|
success: false,
|
|
1563
1314
|
query,
|
|
1564
|
-
error:
|
|
1315
|
+
error: formatResolverError(nativeResolved.error)
|
|
1565
1316
|
};
|
|
1566
1317
|
}
|
|
1567
|
-
|
|
1568
|
-
|
|
1569
|
-
if (options.udid) {
|
|
1570
|
-
platform = "ios";
|
|
1571
|
-
nativeUdid = options.udid;
|
|
1572
|
-
}
|
|
1573
|
-
else if (!platform) {
|
|
1574
|
-
// If `device` is provided, try resolving it as an iOS simulator name first —
|
|
1575
|
-
// otherwise the auto-detect would fall through to getActiveOrBootedSimulatorUdid()
|
|
1576
|
-
// and ignore the caller's targeting hint when multiple simulators are booted.
|
|
1577
|
-
const deviceMatchedUdid = options.device
|
|
1578
|
-
? await findSimulatorByName(options.device).catch(() => null)
|
|
1579
|
-
: null;
|
|
1580
|
-
if (deviceMatchedUdid) {
|
|
1581
|
-
platform = "ios";
|
|
1582
|
-
nativeUdid = deviceMatchedUdid;
|
|
1583
|
-
}
|
|
1584
|
-
else {
|
|
1585
|
-
const [androidDevice, iosSimulator] = await Promise.all([
|
|
1586
|
-
getDefaultAndroidDevice().catch(() => null),
|
|
1587
|
-
getActiveOrBootedSimulatorUdid().catch(() => null)
|
|
1588
|
-
]);
|
|
1589
|
-
if (androidDevice && iosSimulator) {
|
|
1590
|
-
return {
|
|
1591
|
-
success: false,
|
|
1592
|
-
query,
|
|
1593
|
-
error: 'Multiple platforms detected (both Android and iOS). Specify platform: "android" or platform: "ios" to target the correct device.'
|
|
1594
|
-
};
|
|
1595
|
-
}
|
|
1596
|
-
if (androidDevice) {
|
|
1597
|
-
platform = "android";
|
|
1598
|
-
}
|
|
1599
|
-
else if (iosSimulator) {
|
|
1600
|
-
platform = "ios";
|
|
1601
|
-
nativeUdid = iosSimulator;
|
|
1602
|
-
}
|
|
1603
|
-
else {
|
|
1604
|
-
return {
|
|
1605
|
-
success: false,
|
|
1606
|
-
query,
|
|
1607
|
-
error: "No Android device or iOS simulator found. Connect a device or start a simulator."
|
|
1608
|
-
};
|
|
1609
|
-
}
|
|
1610
|
-
}
|
|
1611
|
-
}
|
|
1612
|
-
else if (platform === "ios") {
|
|
1613
|
-
nativeUdid =
|
|
1614
|
-
(options.device ? await findSimulatorByName(options.device) : null) ??
|
|
1615
|
-
(await getActiveOrBootedSimulatorUdid()) ??
|
|
1616
|
-
undefined;
|
|
1617
|
-
}
|
|
1318
|
+
const platform = nativeResolved.target.platform;
|
|
1319
|
+
const nativeUdid = nativeResolved.target.iosUdid;
|
|
1618
1320
|
const nativeShouldScreenshot = options.screenshot !== false;
|
|
1619
1321
|
// Decoupled (I5, 2026-05-16): verify runs even when image bytes aren't returned.
|
|
1620
1322
|
const nativeShouldVerify = options.verify !== false;
|
|
@@ -1672,12 +1374,26 @@ export async function tap(options) {
|
|
|
1672
1374
|
screenshotScale: nativeScreenshotMeta?.scaleFactor || 1
|
|
1673
1375
|
});
|
|
1674
1376
|
if (options.burst && nativeShouldVerify && nativeBeforeBuffer) {
|
|
1675
|
-
({ screenshot, verification } = await burstCaptureAndVerify(
|
|
1377
|
+
({ screenshot, verification } = await burstCaptureAndVerify({
|
|
1378
|
+
platform,
|
|
1379
|
+
beforeBuffer: nativeBeforeBuffer,
|
|
1380
|
+
udid: nativeUdid,
|
|
1381
|
+
beforeScaleFactor: nativeScreenshotMeta?.scaleFactor,
|
|
1382
|
+
markerPx: nativeMarker
|
|
1383
|
+
}));
|
|
1676
1384
|
if (!nativeShouldScreenshot)
|
|
1677
1385
|
screenshot = undefined;
|
|
1678
1386
|
}
|
|
1679
1387
|
else {
|
|
1680
|
-
({ screenshot, verification } = await verifyAndCapture(
|
|
1388
|
+
({ screenshot, verification } = await verifyAndCapture({
|
|
1389
|
+
platform,
|
|
1390
|
+
shouldVerify: nativeShouldVerify,
|
|
1391
|
+
shouldScreenshot: nativeShouldScreenshot,
|
|
1392
|
+
beforeBuffer: nativeBeforeBuffer,
|
|
1393
|
+
udid: nativeUdid,
|
|
1394
|
+
beforeScaleFactor: nativeScreenshotMeta?.scaleFactor,
|
|
1395
|
+
markerPx: nativeMarker
|
|
1396
|
+
}));
|
|
1681
1397
|
}
|
|
1682
1398
|
return formatTapSuccess({
|
|
1683
1399
|
method: "native-coordinate",
|
|
@@ -1695,46 +1411,48 @@ export async function tap(options) {
|
|
|
1695
1411
|
suggestion: `Take a screenshot (${platform === "ios" ? "ios_screenshot" : "android_screenshot"}) to verify coordinates.`
|
|
1696
1412
|
});
|
|
1697
1413
|
}
|
|
1698
|
-
//
|
|
1699
|
-
|
|
1414
|
+
// Resolve device target via the unified resolver. Replaces the prior chain
|
|
1415
|
+
// of (udid → connectedApp → findSimulatorByName) lookups and the standalone
|
|
1416
|
+
// "no platform → probe OS for devices" branch. Returns a structured error
|
|
1417
|
+
// when ambiguous; surface that to the caller so they can disambiguate.
|
|
1418
|
+
const resolved = await resolveDeviceTarget(options.device);
|
|
1419
|
+
if (!resolved.ok) {
|
|
1700
1420
|
return {
|
|
1701
1421
|
success: false,
|
|
1702
1422
|
query,
|
|
1703
|
-
error:
|
|
1423
|
+
error: formatResolverError(resolved.error)
|
|
1704
1424
|
};
|
|
1705
1425
|
}
|
|
1706
|
-
|
|
1426
|
+
const platform = resolved.target.platform;
|
|
1427
|
+
let targetUdid = resolved.target.iosUdid;
|
|
1428
|
+
// Pick the connected app to bias strategy selection. Prefer the registry
|
|
1429
|
+
// entry whose identifier matches the resolved target; fall back to a
|
|
1430
|
+
// device-name match (handy when registry entries lack identifiers yet);
|
|
1431
|
+
// last resort is the first connected app on the matching platform.
|
|
1707
1432
|
const allApps = Array.from(connectedApps.values());
|
|
1708
1433
|
let hasMetro = allApps.length > 0;
|
|
1709
|
-
// Pick the connected app to bias platform/strategy selection.
|
|
1710
|
-
// Precedence:
|
|
1711
|
-
// 1. udid → match the iOS app whose simulatorUdid equals udid (if any)
|
|
1712
|
-
// 2. device → substring match against connected app's deviceName/title
|
|
1713
|
-
// 3. platform → first app on that platform
|
|
1714
|
-
// 4. first connected app
|
|
1715
1434
|
let app;
|
|
1716
|
-
if (
|
|
1717
|
-
app = allApps.find((a) => a.platform === "ios" && a.simulatorUdid ===
|
|
1435
|
+
if (resolved.target.iosUdid) {
|
|
1436
|
+
app = allApps.find((a) => a.platform === "ios" && a.simulatorUdid === resolved.target.iosUdid);
|
|
1437
|
+
}
|
|
1438
|
+
else if (resolved.target.androidSerial) {
|
|
1439
|
+
app = allApps.find((a) => a.platform === "android" && a.adbSerial === resolved.target.androidSerial);
|
|
1718
1440
|
}
|
|
1719
1441
|
if (!app && options.device) {
|
|
1720
1442
|
try {
|
|
1721
1443
|
app = getConnectedAppByDevice(options.device) ?? undefined;
|
|
1722
1444
|
}
|
|
1723
|
-
catch
|
|
1724
|
-
// getConnectedAppByDevice throws on ambiguous matches
|
|
1725
|
-
|
|
1726
|
-
|
|
1727
|
-
|
|
1728
|
-
error: err instanceof Error ? err.message : String(err)
|
|
1729
|
-
};
|
|
1445
|
+
catch {
|
|
1446
|
+
// getConnectedAppByDevice throws on ambiguous matches; the resolver
|
|
1447
|
+
// would have already returned that error, so reaching here means
|
|
1448
|
+
// the resolved target's identifier just doesn't appear in the
|
|
1449
|
+
// registry yet — fall through to the platform-first pick below.
|
|
1730
1450
|
}
|
|
1731
1451
|
}
|
|
1732
1452
|
if (!app) {
|
|
1733
|
-
app =
|
|
1734
|
-
? (allApps.find((a) => a.platform === options.platform) ?? allApps[0])
|
|
1735
|
-
: allApps[0];
|
|
1453
|
+
app = allApps.find((a) => a.platform === platform) ?? allApps[0];
|
|
1736
1454
|
}
|
|
1737
|
-
// Try to auto-connect to Metro (for fiber strategy), but don't fail if it doesn't work
|
|
1455
|
+
// Try to auto-connect to Metro (for fiber strategy), but don't fail if it doesn't work.
|
|
1738
1456
|
if (!hasMetro) {
|
|
1739
1457
|
try {
|
|
1740
1458
|
await withTimeout((async () => {
|
|
@@ -1751,56 +1469,15 @@ export async function tap(options) {
|
|
|
1751
1469
|
})(), Math.min(remainingMs(), 3000), "auto-connect");
|
|
1752
1470
|
const apps = Array.from(connectedApps.values());
|
|
1753
1471
|
hasMetro = apps.length > 0;
|
|
1754
|
-
app = apps[0];
|
|
1472
|
+
app = apps.find((a) => a.platform === platform) ?? apps[0];
|
|
1473
|
+
if (platform === "ios" && !targetUdid) {
|
|
1474
|
+
targetUdid = app?.simulatorUdid ?? targetUdid;
|
|
1475
|
+
}
|
|
1755
1476
|
}
|
|
1756
1477
|
catch {
|
|
1757
1478
|
// Auto-connect failed — Metro-dependent strategies will be skipped
|
|
1758
1479
|
}
|
|
1759
1480
|
}
|
|
1760
|
-
// Detect platform — from Metro connection or from available devices
|
|
1761
|
-
let platform = options.platform;
|
|
1762
|
-
if (!platform) {
|
|
1763
|
-
if (app) {
|
|
1764
|
-
platform = app.platform;
|
|
1765
|
-
}
|
|
1766
|
-
else {
|
|
1767
|
-
// No Metro — detect from available devices
|
|
1768
|
-
const [androidDevice, iosSimulator] = await Promise.all([
|
|
1769
|
-
getDefaultAndroidDevice().catch(() => null),
|
|
1770
|
-
getActiveOrBootedSimulatorUdid().catch(() => null)
|
|
1771
|
-
]);
|
|
1772
|
-
if (androidDevice && iosSimulator) {
|
|
1773
|
-
return {
|
|
1774
|
-
success: false,
|
|
1775
|
-
query,
|
|
1776
|
-
error: 'Both iOS and Android devices detected but no Metro connection. Specify platform="ios" or platform="android".'
|
|
1777
|
-
};
|
|
1778
|
-
}
|
|
1779
|
-
if (androidDevice)
|
|
1780
|
-
platform = "android";
|
|
1781
|
-
else if (iosSimulator)
|
|
1782
|
-
platform = "ios";
|
|
1783
|
-
}
|
|
1784
|
-
}
|
|
1785
|
-
if (!platform) {
|
|
1786
|
-
return {
|
|
1787
|
-
success: false,
|
|
1788
|
-
query,
|
|
1789
|
-
error: "No devices found. Boot a simulator or connect an Android device."
|
|
1790
|
-
};
|
|
1791
|
-
}
|
|
1792
|
-
// Resolve target iOS simulator UDID.
|
|
1793
|
-
// Precedence: explicit udid → connected app's simulatorUdid → findSimulatorByName(device or app deviceName)
|
|
1794
|
-
// This ensures taps go to the correct device when multiple simulators are booted.
|
|
1795
|
-
let targetUdid;
|
|
1796
|
-
if (platform === "ios") {
|
|
1797
|
-
targetUdid =
|
|
1798
|
-
options.udid ??
|
|
1799
|
-
app?.simulatorUdid ??
|
|
1800
|
-
(options.device ? await findSimulatorByName(options.device) : null) ??
|
|
1801
|
-
(app?.deviceInfo?.deviceName ? await findSimulatorByName(app.deviceInfo.deviceName) : null) ??
|
|
1802
|
-
undefined;
|
|
1803
|
-
}
|
|
1804
1481
|
const deviceName = app?.deviceInfo?.deviceName;
|
|
1805
1482
|
// Determine strategies
|
|
1806
1483
|
const strategies = getAvailableStrategies(query, strategy);
|
|
@@ -1995,7 +1672,13 @@ export async function tap(options) {
|
|
|
1995
1672
|
devicePixelRatio: dprForMarker
|
|
1996
1673
|
});
|
|
1997
1674
|
if (options.burst && canVerify && beforeBuffer) {
|
|
1998
|
-
({ screenshot, verification, afterWithMarkerBuffer } = await burstCaptureAndVerify(
|
|
1675
|
+
({ screenshot, verification, afterWithMarkerBuffer } = await burstCaptureAndVerify({
|
|
1676
|
+
platform,
|
|
1677
|
+
beforeBuffer,
|
|
1678
|
+
udid: targetUdid,
|
|
1679
|
+
beforeScaleFactor,
|
|
1680
|
+
markerPx: strategyMarker
|
|
1681
|
+
}));
|
|
1999
1682
|
// Burst always produces a screenshot for diff visualization. Drop the
|
|
2000
1683
|
// bytes when the caller didn't ask for them AND the tap was meaningful
|
|
2001
1684
|
// — keeping the screenshot on unmeaningful taps saves the agent a
|
|
@@ -2005,8 +1688,15 @@ export async function tap(options) {
|
|
|
2005
1688
|
screenshot = undefined;
|
|
2006
1689
|
}
|
|
2007
1690
|
else {
|
|
2008
|
-
({ screenshot, verification, afterWithMarkerBuffer } = await verifyAndCapture(
|
|
2009
|
-
|
|
1691
|
+
({ screenshot, verification, afterWithMarkerBuffer } = await verifyAndCapture({
|
|
1692
|
+
platform,
|
|
1693
|
+
shouldVerify: canVerify,
|
|
1694
|
+
shouldScreenshot: true,
|
|
1695
|
+
beforeBuffer,
|
|
1696
|
+
udid: targetUdid,
|
|
1697
|
+
beforeScaleFactor,
|
|
1698
|
+
markerPx: strategyMarker
|
|
1699
|
+
}));
|
|
2010
1700
|
// Mirror the burst-path gate: drop the screenshot only when the
|
|
2011
1701
|
// caller didn't ask for it AND the tap was meaningful.
|
|
2012
1702
|
if (!shouldScreenshot && verification?.meaningful !== false)
|
|
@@ -2123,11 +1813,24 @@ export async function tap(options) {
|
|
|
2123
1813
|
androidDensityScale: fnDensity
|
|
2124
1814
|
});
|
|
2125
1815
|
if (options.burst && canVerify && beforeBuffer) {
|
|
2126
|
-
({ screenshot, verification } = await burstCaptureAndVerify(
|
|
1816
|
+
({ screenshot, verification } = await burstCaptureAndVerify({
|
|
1817
|
+
platform,
|
|
1818
|
+
beforeBuffer,
|
|
1819
|
+
udid: targetUdid,
|
|
1820
|
+
beforeScaleFactor,
|
|
1821
|
+
markerPx: fiberMarker
|
|
1822
|
+
}));
|
|
2127
1823
|
}
|
|
2128
1824
|
else {
|
|
2129
|
-
({ screenshot, verification } = await verifyAndCapture(
|
|
2130
|
-
|
|
1825
|
+
({ screenshot, verification } = await verifyAndCapture({
|
|
1826
|
+
platform,
|
|
1827
|
+
shouldVerify: canVerify,
|
|
1828
|
+
shouldScreenshot: true,
|
|
1829
|
+
beforeBuffer,
|
|
1830
|
+
udid: targetUdid,
|
|
1831
|
+
beforeScaleFactor,
|
|
1832
|
+
markerPx: fiberMarker
|
|
1833
|
+
}));
|
|
2131
1834
|
}
|
|
2132
1835
|
if (!shouldScreenshot && verification?.meaningful !== false)
|
|
2133
1836
|
screenshot = undefined;
|
|
@@ -2161,7 +1864,7 @@ export async function tap(options) {
|
|
|
2161
1864
|
// Do NOT fall through to other strategies (they can't resolve ambiguity).
|
|
2162
1865
|
if (result.matches && result.ambiguous) {
|
|
2163
1866
|
const { screenshot: matchScreenshot } = shouldScreenshot
|
|
2164
|
-
? await verifyAndCapture(platform, false, true, null, targetUdid)
|
|
1867
|
+
? await verifyAndCapture({ platform, shouldVerify: false, shouldScreenshot: true, beforeBuffer: null, udid: targetUdid })
|
|
2165
1868
|
: { screenshot: undefined };
|
|
2166
1869
|
if (matchScreenshot && app) {
|
|
2167
1870
|
app.lastScreenshot = {
|
|
@@ -2195,7 +1898,7 @@ export async function tap(options) {
|
|
|
2195
1898
|
const allTimedOut = attempted.length > 0 && attempted.every(a => a.outcome === "timeout" || a.outcome === "skipped");
|
|
2196
1899
|
let suggestion = buildSuggestion(query, strategies, platform);
|
|
2197
1900
|
if (fiberSawInvisible) {
|
|
2198
|
-
suggestion = `Element exists in the React tree but is not on screen. Try scrolling it into view (
|
|
1901
|
+
suggestion = `Element exists in the React tree but is not on screen. Try scrolling it into view with swipe(), dismiss any overlay covering it, or wait for layout to settle. ` + suggestion;
|
|
2199
1902
|
}
|
|
2200
1903
|
else if (allTimedOut) {
|
|
2201
1904
|
// No strategy got to a definitive answer — don't let the agent conclude
|
|
@@ -2203,7 +1906,7 @@ export async function tap(options) {
|
|
|
2203
1906
|
suggestion = `All strategies timed out — the element's presence is UNKNOWN. Retry the tap (transient slowness is common on dense screens), or try a different strategy explicitly (e.g. strategy='fiber' if accessibility timed out). ` + suggestion;
|
|
2204
1907
|
}
|
|
2205
1908
|
const { screenshot: failScreenshot } = shouldScreenshot
|
|
2206
|
-
? await verifyAndCapture(platform, false, true, null, targetUdid)
|
|
1909
|
+
? await verifyAndCapture({ platform, shouldVerify: false, shouldScreenshot: true, beforeBuffer: null, udid: targetUdid })
|
|
2207
1910
|
: { screenshot: undefined };
|
|
2208
1911
|
if (failScreenshot && app) {
|
|
2209
1912
|
app.lastScreenshot = {
|
|
@@ -2279,7 +1982,7 @@ function buildSuggestion(query, triedStrategies, platform) {
|
|
|
2279
1982
|
// the error) keeps the existing message stable for tooling that parses it.
|
|
2280
1983
|
if (query.testID && looksLikeVirtualizedListItem(query.testID)) {
|
|
2281
1984
|
suggestions.push("testID looks like a virtualized list item — scroll it on-screen first " +
|
|
2282
|
-
`(${platform === "ios" ? "ios_button arrow keys
|
|
1985
|
+
`(swipe()${platform === "ios" ? " or ios_button arrow keys" : ""}) before tapping`);
|
|
2283
1986
|
}
|
|
2284
1987
|
suggestions.push(`Take a screenshot (${platform === "ios" ? "ios_screenshot" : "android_screenshot"}) ` +
|
|
2285
1988
|
"to verify the element is visible, then use x/y coordinates");
|