react-native-ai-debugger 1.9.1 → 1.10.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (59) hide show
  1. package/README.md +1 -1
  2. package/build/__tests__/integration/tap-strategies.test.js +1 -1
  3. package/build/__tests__/integration/tap-strategies.test.js.map +1 -1
  4. package/build/__tests__/unit/androidEmulatorAvds.test.d.ts +2 -0
  5. package/build/__tests__/unit/androidEmulatorAvds.test.d.ts.map +1 -0
  6. package/build/__tests__/unit/androidEmulatorAvds.test.js +98 -0
  7. package/build/__tests__/unit/androidEmulatorAvds.test.js.map +1 -0
  8. package/build/__tests__/unit/deviceDiscovery.test.d.ts +2 -0
  9. package/build/__tests__/unit/deviceDiscovery.test.d.ts.map +1 -0
  10. package/build/__tests__/unit/deviceDiscovery.test.js +119 -0
  11. package/build/__tests__/unit/deviceDiscovery.test.js.map +1 -0
  12. package/build/__tests__/unit/deviceResolver.test.d.ts +2 -0
  13. package/build/__tests__/unit/deviceResolver.test.d.ts.map +1 -0
  14. package/build/__tests__/unit/deviceResolver.test.js +226 -0
  15. package/build/__tests__/unit/deviceResolver.test.js.map +1 -0
  16. package/build/__tests__/unit/tap.test.js +13 -30
  17. package/build/__tests__/unit/tap.test.js.map +1 -1
  18. package/build/__tests__/unit/verifyAction.test.d.ts +2 -0
  19. package/build/__tests__/unit/verifyAction.test.d.ts.map +1 -0
  20. package/build/__tests__/unit/verifyAction.test.js +26 -0
  21. package/build/__tests__/unit/verifyAction.test.js.map +1 -0
  22. package/build/core/android.d.ts +20 -0
  23. package/build/core/android.d.ts.map +1 -1
  24. package/build/core/android.js +98 -0
  25. package/build/core/android.js.map +1 -1
  26. package/build/core/connection.d.ts.map +1 -1
  27. package/build/core/connection.js +20 -9
  28. package/build/core/connection.js.map +1 -1
  29. package/build/core/deviceDiscovery.d.ts +52 -0
  30. package/build/core/deviceDiscovery.d.ts.map +1 -0
  31. package/build/core/deviceDiscovery.js +74 -0
  32. package/build/core/deviceDiscovery.js.map +1 -0
  33. package/build/core/deviceResolver.d.ts +44 -0
  34. package/build/core/deviceResolver.d.ts.map +1 -0
  35. package/build/core/deviceResolver.js +191 -0
  36. package/build/core/deviceResolver.js.map +1 -0
  37. package/build/core/guides.d.ts.map +1 -1
  38. package/build/core/guides.js +3 -4
  39. package/build/core/guides.js.map +1 -1
  40. package/build/core/types.d.ts +1 -0
  41. package/build/core/types.d.ts.map +1 -1
  42. package/build/pro/tap.d.ts +5 -10
  43. package/build/pro/tap.d.ts.map +1 -1
  44. package/build/pro/tap.js +98 -395
  45. package/build/pro/tap.js.map +1 -1
  46. package/build/pro/verifyAction.d.ts +49 -0
  47. package/build/pro/verifyAction.d.ts.map +1 -0
  48. package/build/pro/verifyAction.js +255 -0
  49. package/build/pro/verifyAction.js.map +1 -0
  50. package/build/tools/connectionTools.d.ts.map +1 -1
  51. package/build/tools/connectionTools.js +27 -2
  52. package/build/tools/connectionTools.js.map +1 -1
  53. package/build/tools/deviceTools.d.ts.map +1 -1
  54. package/build/tools/deviceTools.js +104 -2
  55. package/build/tools/deviceTools.js.map +1 -1
  56. package/build/tools/interactionTools.d.ts.map +1 -1
  57. package/build/tools/interactionTools.js +113 -151
  58. package/build/tools/interactionTools.js.map +1 -1
  59. package/package.json +1 -1
package/build/pro/tap.js CHANGED
@@ -1,29 +1,34 @@
1
- import { connectedApps, imageBuffer } from "../core/state.js";
1
+ import { connectedApps } from "../core/state.js";
2
2
  import { executeInApp } from "../core/executor.js";
3
3
  import { pressElement } from "../core/executor.js";
4
- import { iosTap, iosFindElement, iosScreenshot, getActiveOrBootedSimulatorUdid, findSimulatorByName, isUiDriverAvailable, getUiDriverInstallHint, getIOSSafeAreaTop } from "../core/ios.js";
5
- import { androidTap, androidFindElement, getDefaultAndroidDevice, androidScreenshot } from "../core/android.js";
4
+ import { iosTap, iosFindElement, iosScreenshot, isUiDriverAvailable, getUiDriverInstallHint, getIOSSafeAreaTop } from "../core/ios.js";
5
+ import { androidTap, androidFindElement } from "../core/android.js";
6
6
  import { compareScreenshots } from "./screenshot-diff.js";
7
7
  import { scanMetroPorts, fetchDevices, selectMainDevice } from "../core/metro.js";
8
8
  import { connectToDevice, clearReconnectionSuppression, getConnectedAppByDevice } from "../core/connection.js";
9
+ import { resolveDeviceTarget, formatResolverError } from "../core/deviceResolver.js";
9
10
  import { notifyDriverMissing } from "../core/logbox.js";
10
11
  import { captureFailureArtifact } from "../core/failureArtifact.js";
12
+ import { captureScreenshot, verifyAndCapture, burstCaptureAndVerify, } from "./verifyAction.js";
11
13
  export function buildVerificationExplanation(v) {
12
14
  const pct = (rate) => (rate * 100).toFixed(1) + "%";
15
+ const action = v.action ?? "tap";
16
+ const Action = action[0].toUpperCase() + action.slice(1);
17
+ const target = action === "swipe" ? "scroll surface" : "element";
13
18
  if (v.meaningful && !v.transientChangeDetected) {
14
- return `Tap caused a visible UI change (${pct(v.changeRate)} pixel diff). The screen updated as expected.`;
19
+ return `${Action} caused a visible UI change (${pct(v.changeRate)} pixel diff). The screen updated as expected.`;
15
20
  }
16
21
  if (v.meaningful && v.transientChangeDetected) {
17
22
  return (`No persistent change, but transient visual feedback detected ` +
18
23
  `(frame ${v.peakFrame} peak ${pct(v.peakChangeRate || 0)} diff). ` +
19
- `Tap triggered a press animation that settled back to original state.`);
24
+ `${Action} triggered a momentary animation that settled back to original state.`);
20
25
  }
21
26
  if (v.transientChangeDetected === false) {
22
27
  return (`No visual change detected — neither persistent nor transient across burst frames. ` +
23
- `The element may not respond visually or the tap may have missed.`);
28
+ `The ${target} may not respond visually or the ${action} may have missed.`);
24
29
  }
25
30
  return (`No visual change detected between before and after screenshots. ` +
26
- `The element may not respond visually or the tap may have missed.`);
31
+ `The ${target} may not respond visually or the ${action} may have missed.`);
27
32
  }
28
33
  const BURST_CHANGE_THRESHOLD = 0.005;
29
34
  export async function analyzeBurstFrames(frames, options) {
@@ -1008,7 +1013,6 @@ async function tryCoordinateStrategy(pixelX, pixelY, platform, lastScreenshot, u
1008
1013
  };
1009
1014
  }
1010
1015
  }
1011
- const SETTLE_DELAY_MS = 800;
1012
1016
  const TAP_TIMEOUT_MS = 25000;
1013
1017
  const MIN_STRATEGY_BUDGET_MS = 500;
1014
1018
  // Per-strategy budget. OCR cap on Android is bumped via maxStrategyMs() because
@@ -1064,54 +1068,6 @@ function withCancelableTimeout(make, ms, label) {
1064
1068
  throw err;
1065
1069
  });
1066
1070
  }
1067
- async function captureScreenshot(platform, udid) {
1068
- try {
1069
- const result = platform === "ios" ? await iosScreenshot(undefined, udid) : await androidScreenshot();
1070
- if (!result.success || !result.data)
1071
- return null;
1072
- return {
1073
- buffer: result.data,
1074
- width: result.originalWidth || 0,
1075
- height: result.originalHeight || 0,
1076
- scaleFactor: result.scaleFactor || 1
1077
- };
1078
- }
1079
- catch {
1080
- return null;
1081
- }
1082
- }
1083
- function screenshotToBase64(buffer) {
1084
- return buffer.toString("base64");
1085
- }
1086
- /**
1087
- * Composite a red crosshair marker onto a screenshot at the given pixel coordinates.
1088
- * Uses sharp + SVG so we don't need to inject into the app. Coordinates are in the
1089
- * screenshot's own pixel space (i.e. the space the returned image uses).
1090
- */
1091
- async function drawTapMarker(input, x, y) {
1092
- try {
1093
- const sharp = (await import("sharp")).default;
1094
- const size = 72;
1095
- const half = size / 2;
1096
- const svg = `<svg width="${size}" height="${size}" xmlns="http://www.w3.org/2000/svg">
1097
- <circle cx="${half}" cy="${half}" r="${half - 8}" fill="none" stroke="white" stroke-width="6" opacity="0.85"/>
1098
- <circle cx="${half}" cy="${half}" r="${half - 8}" fill="none" stroke="#FF2D55" stroke-width="3" opacity="1"/>
1099
- <line x1="${half}" y1="8" x2="${half}" y2="${size - 8}" stroke="white" stroke-width="6" opacity="0.85"/>
1100
- <line x1="8" y1="${half}" x2="${size - 8}" y2="${half}" stroke="white" stroke-width="6" opacity="0.85"/>
1101
- <line x1="${half}" y1="8" x2="${half}" y2="${size - 8}" stroke="#FF2D55" stroke-width="2.5"/>
1102
- <line x1="8" y1="${half}" x2="${size - 8}" y2="${half}" stroke="#FF2D55" stroke-width="2.5"/>
1103
- <circle cx="${half}" cy="${half}" r="3" fill="#FF2D55"/>
1104
- </svg>`;
1105
- const left = Math.round(x - half);
1106
- const top = Math.round(y - half);
1107
- return await sharp(input)
1108
- .composite([{ input: Buffer.from(svg), left, top }])
1109
- .toBuffer();
1110
- }
1111
- catch {
1112
- return input;
1113
- }
1114
- }
1115
1071
  /**
1116
1072
  * Compute marker coordinates in screenshot-pixel space (what the returned PNG uses).
1117
1073
  * Unit rules by strategy, assuming screenshotScale = downscale factor:
@@ -1154,209 +1110,6 @@ function computeMarkerPx(args) {
1154
1110
  y: Math.round(convertedTo.y / scale)
1155
1111
  };
1156
1112
  }
1157
- async function verifyAndCapture(platform, shouldVerify, shouldScreenshot, beforeBuffer, udid, beforeScaleFactor, markerPx) {
1158
- // Decoupled 2026-05-16 (I5): screenshot and verify are independent knobs.
1159
- // - shouldVerify=true: capture after-screenshot internally, diff against beforeBuffer, return verification.
1160
- // - shouldScreenshot=true: also return the (marker-overlaid) image bytes in the response.
1161
- // - both false: fastest path, return only a `skipped` verification stub so the agent can tell apart "ran clean" from "skipped intentionally".
1162
- if (!shouldVerify && !shouldScreenshot) {
1163
- return {
1164
- verification: {
1165
- skipped: true,
1166
- skippedReason: "screenshot=false, verify=false",
1167
- explanation: "Verification skipped (screenshot=false and verify=false)."
1168
- }
1169
- };
1170
- }
1171
- await new Promise((resolve) => setTimeout(resolve, SETTLE_DELAY_MS));
1172
- const after = await captureScreenshot(platform, udid);
1173
- if (!after) {
1174
- if (shouldVerify) {
1175
- return {
1176
- verification: {
1177
- skipped: true,
1178
- skippedReason: "after-screenshot capture failed",
1179
- explanation: "Verification skipped — could not capture post-tap screenshot."
1180
- }
1181
- };
1182
- }
1183
- return {};
1184
- }
1185
- const afterWithMarker = markerPx
1186
- ? await drawTapMarker(after.buffer, markerPx.x, markerPx.y)
1187
- : after.buffer;
1188
- // Construct the screenshot eagerly. Caller decides whether to drop it
1189
- // based on `shouldScreenshot` AND the verification outcome — when a tap
1190
- // is unmeaningful we surface the image regardless so the agent can
1191
- // diagnose without an extra round-trip.
1192
- const screenshot = {
1193
- image: screenshotToBase64(afterWithMarker),
1194
- width: after.width,
1195
- height: after.height,
1196
- scaleFactor: after.scaleFactor
1197
- };
1198
- let verification;
1199
- if (!shouldVerify) {
1200
- verification = {
1201
- skipped: true,
1202
- skippedReason: "verify=false",
1203
- explanation: "Verification skipped (verify=false)."
1204
- };
1205
- }
1206
- else if (!beforeBuffer) {
1207
- verification = {
1208
- skipped: true,
1209
- skippedReason: "before-screenshot unavailable",
1210
- explanation: "Verification skipped — could not capture pre-tap screenshot."
1211
- };
1212
- }
1213
- else {
1214
- try {
1215
- const rawStatusBar = platform === "ios" ? 177 : 142; // pixels in original screenshot space
1216
- const scale = beforeScaleFactor || after.scaleFactor || 1;
1217
- const statusBarHeight = Math.round(rawStatusBar / scale);
1218
- const diff = await compareScreenshots(beforeBuffer, after.buffer, {
1219
- statusBarHeight
1220
- });
1221
- verification = {
1222
- meaningful: diff.changed,
1223
- changeRate: diff.changeRate,
1224
- changedPixels: diff.changedPixels,
1225
- totalPixels: diff.totalPixels,
1226
- explanation: buildVerificationExplanation({
1227
- meaningful: diff.changed,
1228
- changeRate: diff.changeRate,
1229
- changedPixels: diff.changedPixels,
1230
- totalPixels: diff.totalPixels
1231
- })
1232
- };
1233
- }
1234
- catch (err) {
1235
- verification = {
1236
- skipped: true,
1237
- skippedReason: `diff failed: ${err instanceof Error ? err.message : String(err)}`,
1238
- explanation: "Verification skipped — pixel diff threw."
1239
- };
1240
- }
1241
- }
1242
- const verifyGroupId = `verify-${Date.now()}-${Math.random().toString(36).slice(2, 8)}`;
1243
- if (beforeBuffer) {
1244
- imageBuffer.add({
1245
- id: `${verifyGroupId}-before`,
1246
- image: beforeBuffer,
1247
- timestamp: Date.now(),
1248
- source: "tap-verify",
1249
- groupId: verifyGroupId,
1250
- metadata: { phase: "before" }
1251
- });
1252
- }
1253
- imageBuffer.add({
1254
- id: `${verifyGroupId}-after`,
1255
- image: afterWithMarker,
1256
- timestamp: Date.now(),
1257
- source: "tap-verify",
1258
- groupId: verifyGroupId,
1259
- metadata: { phase: "after", changeRate: verification?.changeRate }
1260
- });
1261
- return { screenshot, verification, afterWithMarkerBuffer: afterWithMarker };
1262
- }
1263
- // --- Burst Capture ---
1264
- const BURST_FRAME_COUNT = 4;
1265
- const BURST_FRAME_INTERVAL_MS = 150;
1266
- async function burstCaptureAndVerify(platform, beforeBuffer, udid, beforeScaleFactor, markerPx) {
1267
- if (!beforeBuffer)
1268
- return {};
1269
- const frames = [beforeBuffer];
1270
- let capturedScaleFactor = beforeScaleFactor || 1;
1271
- for (let i = 0; i < BURST_FRAME_COUNT; i++) {
1272
- await new Promise((resolve) => setTimeout(resolve, BURST_FRAME_INTERVAL_MS));
1273
- const capture = await captureScreenshot(platform, udid);
1274
- if (capture) {
1275
- frames.push(capture.buffer);
1276
- if (i === 0)
1277
- capturedScaleFactor = capture.scaleFactor || capturedScaleFactor;
1278
- }
1279
- }
1280
- if (frames.length < 2)
1281
- return {};
1282
- const rawStatusBar = platform === "ios" ? 177 : 142; // pixels in original screenshot space
1283
- const statusBarHeight = Math.round(rawStatusBar / capturedScaleFactor);
1284
- // Run the diff on clean frames (no marker) so the cross doesn't register as change.
1285
- const analysis = await analyzeBurstFrames(frames, { statusBarHeight });
1286
- // Apply marker to post-tap frames only (frames[0] is the before state).
1287
- const markedFrames = [];
1288
- for (let i = 0; i < frames.length; i++) {
1289
- if (markerPx && i > 0) {
1290
- markedFrames.push(await drawTapMarker(frames[i], markerPx.x, markerPx.y));
1291
- }
1292
- else {
1293
- markedFrames.push(frames[i]);
1294
- }
1295
- }
1296
- const groupId = `burst-${Date.now()}-${Math.random().toString(36).slice(2, 8)}`;
1297
- for (let i = 0; i < markedFrames.length; i++) {
1298
- imageBuffer.add({
1299
- id: `${groupId}-f${i}`,
1300
- image: markedFrames[i],
1301
- timestamp: Date.now(),
1302
- source: "tap-burst",
1303
- groupId,
1304
- metadata: {
1305
- frameIndex: i,
1306
- isBefore: i === 0,
1307
- changeRate: i === 0 ? 0 : analysis.framesWithChange.includes(i) ? analysis.peakChangeRate : 0
1308
- }
1309
- });
1310
- }
1311
- imageBuffer.addGroup({
1312
- groupId,
1313
- intent: "tap-verification",
1314
- source: "tap-burst",
1315
- timestamp: Date.now(),
1316
- frameCount: frames.length,
1317
- summary: {
1318
- peakChangeRate: analysis.peakChangeRate,
1319
- peakFrame: analysis.peakFrame,
1320
- framesWithChange: analysis.framesWithChange,
1321
- transientChangeDetected: analysis.transientChangeDetected,
1322
- persistentChangeRate: analysis.persistentChangeRate
1323
- }
1324
- });
1325
- // Get dimensions from the last frame using sharp
1326
- const sharp = (await import("sharp")).default;
1327
- const meta = await sharp(markedFrames[markedFrames.length - 1]).metadata();
1328
- const screenshot = {
1329
- image: screenshotToBase64(markedFrames[markedFrames.length - 1]),
1330
- width: meta.width || 0,
1331
- height: meta.height || 0,
1332
- scaleFactor: 1
1333
- };
1334
- const verification = {
1335
- meaningful: analysis.meaningful,
1336
- changeRate: analysis.persistentChangeRate,
1337
- changedPixels: 0,
1338
- totalPixels: 0,
1339
- transientChangeDetected: analysis.transientChangeDetected,
1340
- peakChangeRate: analysis.peakChangeRate,
1341
- peakFrame: analysis.peakFrame,
1342
- burstGroupId: groupId,
1343
- explanation: buildVerificationExplanation({
1344
- meaningful: analysis.meaningful,
1345
- changeRate: analysis.persistentChangeRate,
1346
- changedPixels: 0,
1347
- totalPixels: 0,
1348
- transientChangeDetected: analysis.transientChangeDetected,
1349
- peakChangeRate: analysis.peakChangeRate,
1350
- peakFrame: analysis.peakFrame
1351
- })
1352
- };
1353
- const lastFrameIdx = frames.length - 1;
1354
- return {
1355
- screenshot,
1356
- verification,
1357
- afterWithMarkerBuffer: markedFrames[lastFrameIdx]
1358
- };
1359
- }
1360
1113
  // D2 (Step 6): in-memory ring buffer of recent coord-strategy artifact keys
1361
1114
  // `${round(x/20)}:${round(y/20)}`. Entries expire after 60s. The first
1362
1115
  // failed coord tap at a grid cell uploads its artifact; retries within the
@@ -1554,67 +1307,16 @@ export async function tap(options) {
1554
1307
  }
1555
1308
  // Native mode: bypass React Native connection, tap directly via ADB/simctl
1556
1309
  if (options.native && hasCoordinates) {
1557
- let platform = options.platform;
1558
- // Explicit udid forces iOS targeting. If platform is also explicit and conflicts,
1559
- // surface a clear error rather than silently overriding.
1560
- if (options.udid && platform === "android") {
1310
+ const nativeResolved = await resolveDeviceTarget(options.device);
1311
+ if (!nativeResolved.ok) {
1561
1312
  return {
1562
1313
  success: false,
1563
1314
  query,
1564
- error: 'udid is only valid for iOS. Got platform: "android".'
1315
+ error: formatResolverError(nativeResolved.error)
1565
1316
  };
1566
1317
  }
1567
- // Auto-detect platform if not specified
1568
- let nativeUdid;
1569
- if (options.udid) {
1570
- platform = "ios";
1571
- nativeUdid = options.udid;
1572
- }
1573
- else if (!platform) {
1574
- // If `device` is provided, try resolving it as an iOS simulator name first —
1575
- // otherwise the auto-detect would fall through to getActiveOrBootedSimulatorUdid()
1576
- // and ignore the caller's targeting hint when multiple simulators are booted.
1577
- const deviceMatchedUdid = options.device
1578
- ? await findSimulatorByName(options.device).catch(() => null)
1579
- : null;
1580
- if (deviceMatchedUdid) {
1581
- platform = "ios";
1582
- nativeUdid = deviceMatchedUdid;
1583
- }
1584
- else {
1585
- const [androidDevice, iosSimulator] = await Promise.all([
1586
- getDefaultAndroidDevice().catch(() => null),
1587
- getActiveOrBootedSimulatorUdid().catch(() => null)
1588
- ]);
1589
- if (androidDevice && iosSimulator) {
1590
- return {
1591
- success: false,
1592
- query,
1593
- error: 'Multiple platforms detected (both Android and iOS). Specify platform: "android" or platform: "ios" to target the correct device.'
1594
- };
1595
- }
1596
- if (androidDevice) {
1597
- platform = "android";
1598
- }
1599
- else if (iosSimulator) {
1600
- platform = "ios";
1601
- nativeUdid = iosSimulator;
1602
- }
1603
- else {
1604
- return {
1605
- success: false,
1606
- query,
1607
- error: "No Android device or iOS simulator found. Connect a device or start a simulator."
1608
- };
1609
- }
1610
- }
1611
- }
1612
- else if (platform === "ios") {
1613
- nativeUdid =
1614
- (options.device ? await findSimulatorByName(options.device) : null) ??
1615
- (await getActiveOrBootedSimulatorUdid()) ??
1616
- undefined;
1617
- }
1318
+ const platform = nativeResolved.target.platform;
1319
+ const nativeUdid = nativeResolved.target.iosUdid;
1618
1320
  const nativeShouldScreenshot = options.screenshot !== false;
1619
1321
  // Decoupled (I5, 2026-05-16): verify runs even when image bytes aren't returned.
1620
1322
  const nativeShouldVerify = options.verify !== false;
@@ -1672,12 +1374,26 @@ export async function tap(options) {
1672
1374
  screenshotScale: nativeScreenshotMeta?.scaleFactor || 1
1673
1375
  });
1674
1376
  if (options.burst && nativeShouldVerify && nativeBeforeBuffer) {
1675
- ({ screenshot, verification } = await burstCaptureAndVerify(platform, nativeBeforeBuffer, nativeUdid, nativeScreenshotMeta?.scaleFactor, nativeMarker));
1377
+ ({ screenshot, verification } = await burstCaptureAndVerify({
1378
+ platform,
1379
+ beforeBuffer: nativeBeforeBuffer,
1380
+ udid: nativeUdid,
1381
+ beforeScaleFactor: nativeScreenshotMeta?.scaleFactor,
1382
+ markerPx: nativeMarker
1383
+ }));
1676
1384
  if (!nativeShouldScreenshot)
1677
1385
  screenshot = undefined;
1678
1386
  }
1679
1387
  else {
1680
- ({ screenshot, verification } = await verifyAndCapture(platform, nativeShouldVerify, nativeShouldScreenshot, nativeBeforeBuffer, nativeUdid, nativeScreenshotMeta?.scaleFactor, nativeMarker));
1388
+ ({ screenshot, verification } = await verifyAndCapture({
1389
+ platform,
1390
+ shouldVerify: nativeShouldVerify,
1391
+ shouldScreenshot: nativeShouldScreenshot,
1392
+ beforeBuffer: nativeBeforeBuffer,
1393
+ udid: nativeUdid,
1394
+ beforeScaleFactor: nativeScreenshotMeta?.scaleFactor,
1395
+ markerPx: nativeMarker
1396
+ }));
1681
1397
  }
1682
1398
  return formatTapSuccess({
1683
1399
  method: "native-coordinate",
@@ -1695,46 +1411,48 @@ export async function tap(options) {
1695
1411
  suggestion: `Take a screenshot (${platform === "ios" ? "ios_screenshot" : "android_screenshot"}) to verify coordinates.`
1696
1412
  });
1697
1413
  }
1698
- // udid is iOS-only. Reject early if combined with android.
1699
- if (options.udid && options.platform === "android") {
1414
+ // Resolve device target via the unified resolver. Replaces the prior chain
1415
+ // of (udid → connectedApp → findSimulatorByName) lookups and the standalone
1416
+ // "no platform → probe OS for devices" branch. Returns a structured error
1417
+ // when ambiguous; surface that to the caller so they can disambiguate.
1418
+ const resolved = await resolveDeviceTarget(options.device);
1419
+ if (!resolved.ok) {
1700
1420
  return {
1701
1421
  success: false,
1702
1422
  query,
1703
- error: 'udid is only valid for iOS. Got platform: "android".'
1423
+ error: formatResolverError(resolved.error)
1704
1424
  };
1705
1425
  }
1706
- // Detect available capabilities
1426
+ const platform = resolved.target.platform;
1427
+ let targetUdid = resolved.target.iosUdid;
1428
+ // Pick the connected app to bias strategy selection. Prefer the registry
1429
+ // entry whose identifier matches the resolved target; fall back to a
1430
+ // device-name match (handy when registry entries lack identifiers yet);
1431
+ // last resort is the first connected app on the matching platform.
1707
1432
  const allApps = Array.from(connectedApps.values());
1708
1433
  let hasMetro = allApps.length > 0;
1709
- // Pick the connected app to bias platform/strategy selection.
1710
- // Precedence:
1711
- // 1. udid → match the iOS app whose simulatorUdid equals udid (if any)
1712
- // 2. device → substring match against connected app's deviceName/title
1713
- // 3. platform → first app on that platform
1714
- // 4. first connected app
1715
1434
  let app;
1716
- if (options.udid) {
1717
- app = allApps.find((a) => a.platform === "ios" && a.simulatorUdid === options.udid);
1435
+ if (resolved.target.iosUdid) {
1436
+ app = allApps.find((a) => a.platform === "ios" && a.simulatorUdid === resolved.target.iosUdid);
1437
+ }
1438
+ else if (resolved.target.androidSerial) {
1439
+ app = allApps.find((a) => a.platform === "android" && a.adbSerial === resolved.target.androidSerial);
1718
1440
  }
1719
1441
  if (!app && options.device) {
1720
1442
  try {
1721
1443
  app = getConnectedAppByDevice(options.device) ?? undefined;
1722
1444
  }
1723
- catch (err) {
1724
- // getConnectedAppByDevice throws on ambiguous matches — surface that to the caller
1725
- return {
1726
- success: false,
1727
- query,
1728
- error: err instanceof Error ? err.message : String(err)
1729
- };
1445
+ catch {
1446
+ // getConnectedAppByDevice throws on ambiguous matches; the resolver
1447
+ // would have already returned that error, so reaching here means
1448
+ // the resolved target's identifier just doesn't appear in the
1449
+ // registry yet — fall through to the platform-first pick below.
1730
1450
  }
1731
1451
  }
1732
1452
  if (!app) {
1733
- app = options.platform
1734
- ? (allApps.find((a) => a.platform === options.platform) ?? allApps[0])
1735
- : allApps[0];
1453
+ app = allApps.find((a) => a.platform === platform) ?? allApps[0];
1736
1454
  }
1737
- // Try to auto-connect to Metro (for fiber strategy), but don't fail if it doesn't work
1455
+ // Try to auto-connect to Metro (for fiber strategy), but don't fail if it doesn't work.
1738
1456
  if (!hasMetro) {
1739
1457
  try {
1740
1458
  await withTimeout((async () => {
@@ -1751,56 +1469,15 @@ export async function tap(options) {
1751
1469
  })(), Math.min(remainingMs(), 3000), "auto-connect");
1752
1470
  const apps = Array.from(connectedApps.values());
1753
1471
  hasMetro = apps.length > 0;
1754
- app = apps[0];
1472
+ app = apps.find((a) => a.platform === platform) ?? apps[0];
1473
+ if (platform === "ios" && !targetUdid) {
1474
+ targetUdid = app?.simulatorUdid ?? targetUdid;
1475
+ }
1755
1476
  }
1756
1477
  catch {
1757
1478
  // Auto-connect failed — Metro-dependent strategies will be skipped
1758
1479
  }
1759
1480
  }
1760
- // Detect platform — from Metro connection or from available devices
1761
- let platform = options.platform;
1762
- if (!platform) {
1763
- if (app) {
1764
- platform = app.platform;
1765
- }
1766
- else {
1767
- // No Metro — detect from available devices
1768
- const [androidDevice, iosSimulator] = await Promise.all([
1769
- getDefaultAndroidDevice().catch(() => null),
1770
- getActiveOrBootedSimulatorUdid().catch(() => null)
1771
- ]);
1772
- if (androidDevice && iosSimulator) {
1773
- return {
1774
- success: false,
1775
- query,
1776
- error: 'Both iOS and Android devices detected but no Metro connection. Specify platform="ios" or platform="android".'
1777
- };
1778
- }
1779
- if (androidDevice)
1780
- platform = "android";
1781
- else if (iosSimulator)
1782
- platform = "ios";
1783
- }
1784
- }
1785
- if (!platform) {
1786
- return {
1787
- success: false,
1788
- query,
1789
- error: "No devices found. Boot a simulator or connect an Android device."
1790
- };
1791
- }
1792
- // Resolve target iOS simulator UDID.
1793
- // Precedence: explicit udid → connected app's simulatorUdid → findSimulatorByName(device or app deviceName)
1794
- // This ensures taps go to the correct device when multiple simulators are booted.
1795
- let targetUdid;
1796
- if (platform === "ios") {
1797
- targetUdid =
1798
- options.udid ??
1799
- app?.simulatorUdid ??
1800
- (options.device ? await findSimulatorByName(options.device) : null) ??
1801
- (app?.deviceInfo?.deviceName ? await findSimulatorByName(app.deviceInfo.deviceName) : null) ??
1802
- undefined;
1803
- }
1804
1481
  const deviceName = app?.deviceInfo?.deviceName;
1805
1482
  // Determine strategies
1806
1483
  const strategies = getAvailableStrategies(query, strategy);
@@ -1995,7 +1672,13 @@ export async function tap(options) {
1995
1672
  devicePixelRatio: dprForMarker
1996
1673
  });
1997
1674
  if (options.burst && canVerify && beforeBuffer) {
1998
- ({ screenshot, verification, afterWithMarkerBuffer } = await burstCaptureAndVerify(platform, beforeBuffer, targetUdid, beforeScaleFactor, strategyMarker));
1675
+ ({ screenshot, verification, afterWithMarkerBuffer } = await burstCaptureAndVerify({
1676
+ platform,
1677
+ beforeBuffer,
1678
+ udid: targetUdid,
1679
+ beforeScaleFactor,
1680
+ markerPx: strategyMarker
1681
+ }));
1999
1682
  // Burst always produces a screenshot for diff visualization. Drop the
2000
1683
  // bytes when the caller didn't ask for them AND the tap was meaningful
2001
1684
  // — keeping the screenshot on unmeaningful taps saves the agent a
@@ -2005,8 +1688,15 @@ export async function tap(options) {
2005
1688
  screenshot = undefined;
2006
1689
  }
2007
1690
  else {
2008
- ({ screenshot, verification, afterWithMarkerBuffer } = await verifyAndCapture(platform, canVerify,
2009
- /* shouldScreenshot */ true, beforeBuffer, targetUdid, beforeScaleFactor, strategyMarker));
1691
+ ({ screenshot, verification, afterWithMarkerBuffer } = await verifyAndCapture({
1692
+ platform,
1693
+ shouldVerify: canVerify,
1694
+ shouldScreenshot: true,
1695
+ beforeBuffer,
1696
+ udid: targetUdid,
1697
+ beforeScaleFactor,
1698
+ markerPx: strategyMarker
1699
+ }));
2010
1700
  // Mirror the burst-path gate: drop the screenshot only when the
2011
1701
  // caller didn't ask for it AND the tap was meaningful.
2012
1702
  if (!shouldScreenshot && verification?.meaningful !== false)
@@ -2123,11 +1813,24 @@ export async function tap(options) {
2123
1813
  androidDensityScale: fnDensity
2124
1814
  });
2125
1815
  if (options.burst && canVerify && beforeBuffer) {
2126
- ({ screenshot, verification } = await burstCaptureAndVerify(platform, beforeBuffer, targetUdid, beforeScaleFactor, fiberMarker));
1816
+ ({ screenshot, verification } = await burstCaptureAndVerify({
1817
+ platform,
1818
+ beforeBuffer,
1819
+ udid: targetUdid,
1820
+ beforeScaleFactor,
1821
+ markerPx: fiberMarker
1822
+ }));
2127
1823
  }
2128
1824
  else {
2129
- ({ screenshot, verification } = await verifyAndCapture(platform, canVerify,
2130
- /* shouldScreenshot */ true, beforeBuffer, targetUdid, beforeScaleFactor, fiberMarker));
1825
+ ({ screenshot, verification } = await verifyAndCapture({
1826
+ platform,
1827
+ shouldVerify: canVerify,
1828
+ shouldScreenshot: true,
1829
+ beforeBuffer,
1830
+ udid: targetUdid,
1831
+ beforeScaleFactor,
1832
+ markerPx: fiberMarker
1833
+ }));
2131
1834
  }
2132
1835
  if (!shouldScreenshot && verification?.meaningful !== false)
2133
1836
  screenshot = undefined;
@@ -2161,7 +1864,7 @@ export async function tap(options) {
2161
1864
  // Do NOT fall through to other strategies (they can't resolve ambiguity).
2162
1865
  if (result.matches && result.ambiguous) {
2163
1866
  const { screenshot: matchScreenshot } = shouldScreenshot
2164
- ? await verifyAndCapture(platform, false, true, null, targetUdid)
1867
+ ? await verifyAndCapture({ platform, shouldVerify: false, shouldScreenshot: true, beforeBuffer: null, udid: targetUdid })
2165
1868
  : { screenshot: undefined };
2166
1869
  if (matchScreenshot && app) {
2167
1870
  app.lastScreenshot = {
@@ -2195,7 +1898,7 @@ export async function tap(options) {
2195
1898
  const allTimedOut = attempted.length > 0 && attempted.every(a => a.outcome === "timeout" || a.outcome === "skipped");
2196
1899
  let suggestion = buildSuggestion(query, strategies, platform);
2197
1900
  if (fiberSawInvisible) {
2198
- suggestion = `Element exists in the React tree but is not on screen. Try scrolling it into view (${platform === "ios" ? "ios_swipe / arrow keys" : "android_swipe"}), dismiss any overlay covering it, or wait for layout to settle. ` + suggestion;
1901
+ suggestion = `Element exists in the React tree but is not on screen. Try scrolling it into view with swipe(), dismiss any overlay covering it, or wait for layout to settle. ` + suggestion;
2199
1902
  }
2200
1903
  else if (allTimedOut) {
2201
1904
  // No strategy got to a definitive answer — don't let the agent conclude
@@ -2203,7 +1906,7 @@ export async function tap(options) {
2203
1906
  suggestion = `All strategies timed out — the element's presence is UNKNOWN. Retry the tap (transient slowness is common on dense screens), or try a different strategy explicitly (e.g. strategy='fiber' if accessibility timed out). ` + suggestion;
2204
1907
  }
2205
1908
  const { screenshot: failScreenshot } = shouldScreenshot
2206
- ? await verifyAndCapture(platform, false, true, null, targetUdid)
1909
+ ? await verifyAndCapture({ platform, shouldVerify: false, shouldScreenshot: true, beforeBuffer: null, udid: targetUdid })
2207
1910
  : { screenshot: undefined };
2208
1911
  if (failScreenshot && app) {
2209
1912
  app.lastScreenshot = {
@@ -2279,7 +1982,7 @@ function buildSuggestion(query, triedStrategies, platform) {
2279
1982
  // the error) keeps the existing message stable for tooling that parses it.
2280
1983
  if (query.testID && looksLikeVirtualizedListItem(query.testID)) {
2281
1984
  suggestions.push("testID looks like a virtualized list item — scroll it on-screen first " +
2282
- `(${platform === "ios" ? "ios_button arrow keys / swipe" : "android_swipe"}) before tapping`);
1985
+ `(swipe()${platform === "ios" ? " or ios_button arrow keys" : ""}) before tapping`);
2283
1986
  }
2284
1987
  suggestions.push(`Take a screenshot (${platform === "ios" ? "ios_screenshot" : "android_screenshot"}) ` +
2285
1988
  "to verify the element is visible, then use x/y coordinates");