react-native-ai-debugger 1.1.6 → 1.4.33

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (168) hide show
  1. package/LICENSE +1 -1
  2. package/README.md +386 -45
  3. package/build/__tests__/helpers/tap-test-helpers.d.ts +36 -0
  4. package/build/__tests__/helpers/tap-test-helpers.d.ts.map +1 -0
  5. package/build/__tests__/helpers/tap-test-helpers.js +130 -0
  6. package/build/__tests__/helpers/tap-test-helpers.js.map +1 -0
  7. package/build/__tests__/integration/connection-health.test.js +12 -6
  8. package/build/__tests__/integration/connection-health.test.js.map +1 -1
  9. package/build/__tests__/integration/execute-in-app.test.js +2 -1
  10. package/build/__tests__/integration/execute-in-app.test.js.map +1 -1
  11. package/build/__tests__/integration/tap-strategies.test.d.ts +2 -0
  12. package/build/__tests__/integration/tap-strategies.test.d.ts.map +1 -0
  13. package/build/__tests__/integration/tap-strategies.test.js +329 -0
  14. package/build/__tests__/integration/tap-strategies.test.js.map +1 -0
  15. package/build/__tests__/integration/tools.test.js +9 -10
  16. package/build/__tests__/integration/tools.test.js.map +1 -1
  17. package/build/__tests__/unit/burst.test.d.ts +2 -0
  18. package/build/__tests__/unit/burst.test.d.ts.map +1 -0
  19. package/build/__tests__/unit/burst.test.js +50 -0
  20. package/build/__tests__/unit/burst.test.js.map +1 -0
  21. package/build/__tests__/unit/connection-health.test.js +26 -16
  22. package/build/__tests__/unit/connection-health.test.js.map +1 -1
  23. package/build/__tests__/unit/fiberMatch.test.d.ts +2 -0
  24. package/build/__tests__/unit/fiberMatch.test.d.ts.map +1 -0
  25. package/build/__tests__/unit/fiberMatch.test.js +114 -0
  26. package/build/__tests__/unit/fiberMatch.test.js.map +1 -0
  27. package/build/__tests__/unit/fingerprint.test.d.ts +2 -0
  28. package/build/__tests__/unit/fingerprint.test.d.ts.map +1 -0
  29. package/build/__tests__/unit/fingerprint.test.js +89 -0
  30. package/build/__tests__/unit/fingerprint.test.js.map +1 -0
  31. package/build/__tests__/unit/imageBuffer.test.d.ts +2 -0
  32. package/build/__tests__/unit/imageBuffer.test.d.ts.map +1 -0
  33. package/build/__tests__/unit/imageBuffer.test.js +128 -0
  34. package/build/__tests__/unit/imageBuffer.test.js.map +1 -0
  35. package/build/__tests__/unit/logs.test.js +9 -0
  36. package/build/__tests__/unit/logs.test.js.map +1 -1
  37. package/build/__tests__/unit/networkInterceptor.test.d.ts +2 -0
  38. package/build/__tests__/unit/networkInterceptor.test.d.ts.map +1 -0
  39. package/build/__tests__/unit/networkInterceptor.test.js +104 -0
  40. package/build/__tests__/unit/networkInterceptor.test.js.map +1 -0
  41. package/build/__tests__/unit/screenshot-diff.test.d.ts +2 -0
  42. package/build/__tests__/unit/screenshot-diff.test.d.ts.map +1 -0
  43. package/build/__tests__/unit/screenshot-diff.test.js +97 -0
  44. package/build/__tests__/unit/screenshot-diff.test.js.map +1 -0
  45. package/build/__tests__/unit/tap.test.js +542 -25
  46. package/build/__tests__/unit/tap.test.js.map +1 -1
  47. package/build/core/android.d.ts.map +1 -1
  48. package/build/core/android.js +104 -113
  49. package/build/core/android.js.map +1 -1
  50. package/build/core/appDetection.d.ts +8 -0
  51. package/build/core/appDetection.d.ts.map +1 -0
  52. package/build/core/appDetection.js +115 -0
  53. package/build/core/appDetection.js.map +1 -0
  54. package/build/core/bundle.d.ts.map +1 -1
  55. package/build/core/bundle.js +15 -23
  56. package/build/core/bundle.js.map +1 -1
  57. package/build/core/config.d.ts +8 -0
  58. package/build/core/config.d.ts.map +1 -0
  59. package/build/core/config.js +27 -0
  60. package/build/core/config.js.map +1 -0
  61. package/build/core/connection.d.ts +47 -3
  62. package/build/core/connection.d.ts.map +1 -1
  63. package/build/core/connection.js +519 -124
  64. package/build/core/connection.js.map +1 -1
  65. package/build/core/executor.d.ts +82 -14
  66. package/build/core/executor.d.ts.map +1 -1
  67. package/build/core/executor.js +1869 -418
  68. package/build/core/executor.js.map +1 -1
  69. package/build/core/feedback.d.ts +18 -0
  70. package/build/core/feedback.d.ts.map +1 -0
  71. package/build/core/feedback.js +73 -0
  72. package/build/core/feedback.js.map +1 -0
  73. package/build/core/fiberMatch.d.ts +25 -0
  74. package/build/core/fiberMatch.d.ts.map +1 -0
  75. package/build/core/fiberMatch.js +35 -0
  76. package/build/core/fiberMatch.js.map +1 -0
  77. package/build/core/fingerprint.d.ts +4 -0
  78. package/build/core/fingerprint.d.ts.map +1 -0
  79. package/build/core/fingerprint.js +47 -0
  80. package/build/core/fingerprint.js.map +1 -0
  81. package/build/core/guides.d.ts.map +1 -1
  82. package/build/core/guides.js +66 -3
  83. package/build/core/guides.js.map +1 -1
  84. package/build/core/imageBuffer.d.ts +37 -0
  85. package/build/core/imageBuffer.d.ts.map +1 -0
  86. package/build/core/imageBuffer.js +58 -0
  87. package/build/core/imageBuffer.js.map +1 -0
  88. package/build/core/index.d.ts +22 -17
  89. package/build/core/index.d.ts.map +1 -1
  90. package/build/core/index.js +23 -19
  91. package/build/core/index.js.map +1 -1
  92. package/build/core/ios.d.ts +36 -7
  93. package/build/core/ios.d.ts.map +1 -1
  94. package/build/core/ios.js +561 -326
  95. package/build/core/ios.js.map +1 -1
  96. package/build/core/license.d.ts +38 -0
  97. package/build/core/license.d.ts.map +1 -0
  98. package/build/core/license.js +243 -0
  99. package/build/core/license.js.map +1 -0
  100. package/build/core/logbox.d.ts +57 -0
  101. package/build/core/logbox.d.ts.map +1 -0
  102. package/build/core/logbox.js +282 -0
  103. package/build/core/logbox.js.map +1 -0
  104. package/build/core/logs.d.ts +1 -0
  105. package/build/core/logs.d.ts.map +1 -1
  106. package/build/core/logs.js +5 -0
  107. package/build/core/logs.js.map +1 -1
  108. package/build/core/metro.d.ts +8 -0
  109. package/build/core/metro.d.ts.map +1 -1
  110. package/build/core/metro.js +28 -0
  111. package/build/core/metro.js.map +1 -1
  112. package/build/core/networkInterceptor.d.ts +29 -2
  113. package/build/core/networkInterceptor.d.ts.map +1 -1
  114. package/build/core/networkInterceptor.js +195 -191
  115. package/build/core/networkInterceptor.js.map +1 -1
  116. package/build/core/ocr.d.ts +8 -3
  117. package/build/core/ocr.d.ts.map +1 -1
  118. package/build/core/ocr.js +116 -15
  119. package/build/core/ocr.js.map +1 -1
  120. package/build/core/posthog.d.ts +5 -0
  121. package/build/core/posthog.d.ts.map +1 -0
  122. package/build/core/posthog.js +46 -0
  123. package/build/core/posthog.js.map +1 -0
  124. package/build/core/sdkBridge.d.ts +67 -0
  125. package/build/core/sdkBridge.d.ts.map +1 -0
  126. package/build/core/sdkBridge.js +148 -0
  127. package/build/core/sdkBridge.js.map +1 -0
  128. package/build/core/state.d.ts +25 -5
  129. package/build/core/state.d.ts.map +1 -1
  130. package/build/core/state.js +79 -10
  131. package/build/core/state.js.map +1 -1
  132. package/build/core/telemetry.d.ts +21 -1
  133. package/build/core/telemetry.d.ts.map +1 -1
  134. package/build/core/telemetry.js +166 -10
  135. package/build/core/telemetry.js.map +1 -1
  136. package/build/core/types.d.ts +11 -0
  137. package/build/core/types.d.ts.map +1 -1
  138. package/build/index.js +1490 -324
  139. package/build/index.js.map +1 -1
  140. package/build/pro/screenshot-diff.d.ts +10 -0
  141. package/build/pro/screenshot-diff.d.ts.map +1 -0
  142. package/build/pro/screenshot-diff.js +38 -0
  143. package/build/pro/screenshot-diff.js.map +1 -0
  144. package/build/pro/tap.d.ts +84 -2
  145. package/build/pro/tap.d.ts.map +1 -1
  146. package/build/pro/tap.js +835 -129
  147. package/build/pro/tap.js.map +1 -1
  148. package/build/tools/accountTools.d.ts +44 -0
  149. package/build/tools/accountTools.d.ts.map +1 -0
  150. package/build/tools/accountTools.js +217 -0
  151. package/build/tools/accountTools.js.map +1 -0
  152. package/package.json +8 -7
  153. package/build/core/httpServer.d.ts +0 -14
  154. package/build/core/httpServer.d.ts.map +0 -1
  155. package/build/core/httpServer.js +0 -2459
  156. package/build/core/httpServer.js.map +0 -1
  157. package/build/core/httpServerProcess.d.ts +0 -25
  158. package/build/core/httpServerProcess.d.ts.map +0 -1
  159. package/build/core/httpServerProcess.js +0 -153
  160. package/build/core/httpServerProcess.js.map +0 -1
  161. package/build/core/tap.d.ts +0 -91
  162. package/build/core/tap.d.ts.map +0 -1
  163. package/build/core/tap.js +0 -542
  164. package/build/core/tap.js.map +0 -1
  165. package/build/httpServerStandalone.d.ts +0 -7
  166. package/build/httpServerStandalone.d.ts.map +0 -1
  167. package/build/httpServerStandalone.js +0 -31
  168. package/build/httpServerStandalone.js.map +0 -1
package/build/pro/tap.js CHANGED
@@ -1,11 +1,67 @@
1
- import { connectedApps } from "../core/state.js";
2
- import { inferIOSDevicePixelRatio } from "../core/ocr.js";
1
+ import { connectedApps, imageBuffer } from "../core/state.js";
3
2
  import { executeInApp } from "../core/executor.js";
4
3
  import { pressElement } from "../core/executor.js";
5
- import { iosTap, iosFindElement, iosScreenshot } from "../core/ios.js";
6
- import { androidTap, androidFindElement } from "../core/android.js";
4
+ import { iosTap, iosFindElement, iosScreenshot, getActiveOrBootedSimulatorUdid, findSimulatorByName, isUiDriverAvailable, getUiDriverInstallHint } from "../core/ios.js";
5
+ import { androidTap, androidFindElement, getDefaultAndroidDevice, androidScreenshot } from "../core/android.js";
6
+ import { compareScreenshots } from "./screenshot-diff.js";
7
7
  import { scanMetroPorts, fetchDevices, selectMainDevice } from "../core/metro.js";
8
8
  import { connectToDevice, clearReconnectionSuppression } from "../core/connection.js";
9
+ import { notifyDriverMissing } from "../core/logbox.js";
10
+ export function buildVerificationExplanation(v) {
11
+ const pct = (rate) => (rate * 100).toFixed(1) + "%";
12
+ if (v.meaningful && !v.transientChangeDetected) {
13
+ return `Tap caused a visible UI change (${pct(v.changeRate)} pixel diff). The screen updated as expected.`;
14
+ }
15
+ if (v.meaningful && v.transientChangeDetected) {
16
+ return (`No persistent change, but transient visual feedback detected ` +
17
+ `(frame ${v.peakFrame} peak ${pct(v.peakChangeRate || 0)} diff). ` +
18
+ `Tap triggered a press animation that settled back to original state.`);
19
+ }
20
+ if (v.transientChangeDetected === false) {
21
+ return (`No visual change detected — neither persistent nor transient across burst frames. ` +
22
+ `The element may not respond visually or the tap may have missed.`);
23
+ }
24
+ return (`No visual change detected between before and after screenshots. ` +
25
+ `The element may not respond visually or the tap may have missed.`);
26
+ }
27
+ const BURST_CHANGE_THRESHOLD = 0.005;
28
+ export async function analyzeBurstFrames(frames, options) {
29
+ if (frames.length < 2) {
30
+ return {
31
+ meaningful: false,
32
+ persistentChangeRate: 0,
33
+ transientChangeDetected: false,
34
+ peakChangeRate: 0,
35
+ peakFrame: 0,
36
+ framesWithChange: []
37
+ };
38
+ }
39
+ let peakChangeRate = 0;
40
+ let peakFrame = 0;
41
+ const framesWithChange = [];
42
+ for (let i = 1; i < frames.length; i++) {
43
+ const diff = await compareScreenshots(frames[i - 1], frames[i], options);
44
+ if (diff.changeRate > BURST_CHANGE_THRESHOLD) {
45
+ framesWithChange.push(i);
46
+ }
47
+ if (diff.changeRate > peakChangeRate) {
48
+ peakChangeRate = diff.changeRate;
49
+ peakFrame = i;
50
+ }
51
+ }
52
+ const persistentDiff = await compareScreenshots(frames[0], frames[frames.length - 1], options);
53
+ const persistentChangeRate = persistentDiff.changeRate;
54
+ const transientChangeDetected = !persistentDiff.changed && framesWithChange.length > 0;
55
+ const meaningful = persistentDiff.changed || transientChangeDetected;
56
+ return {
57
+ meaningful,
58
+ persistentChangeRate,
59
+ transientChangeDetected,
60
+ peakChangeRate,
61
+ peakFrame,
62
+ framesWithChange
63
+ };
64
+ }
9
65
  // --- Helpers ---
10
66
  export function buildQuery(options) {
11
67
  const query = {};
@@ -21,43 +77,89 @@ export function buildQuery(options) {
21
77
  query.y = options.y;
22
78
  return query;
23
79
  }
24
- export function isNonAscii(text) {
25
- return /[^\x00-\x7F]/.test(text);
80
+ /**
81
+ * Check if text contains characters that break Hermes Runtime.evaluate.
82
+ * Standard accented Latin characters (Polish, Vietnamese, French, German, etc.)
83
+ * and Cyrillic work fine in Hermes. Only emoji and special Unicode ranges cause issues.
84
+ */
85
+ export function hasProblematicUnicode(text) {
86
+ const emojiPattern = /[\u{1F600}-\u{1F64F}]|[\u{1F300}-\u{1F5FF}]|[\u{1F680}-\u{1F6FF}]|[\u{1F1E0}-\u{1F1FF}]|[\u{2600}-\u{26FF}]|[\u{2700}-\u{27BF}]|[\u{FE00}-\u{FE0F}]|[\u{1F900}-\u{1F9FF}]|[\u{200D}]|[\u{20E3}]|[\u{E0020}-\u{E007F}]/u;
87
+ return emojiPattern.test(text);
88
+ }
89
+ function normalizeForMatch(text) {
90
+ return text.trim().replace(/\s+/g, " ").toLowerCase();
91
+ }
92
+ export function findOcrMatch(ocrResult, query) {
93
+ const needle = normalizeForMatch(query);
94
+ if (!needle)
95
+ return null;
96
+ const words = ocrResult.words ?? [];
97
+ const lines = ocrResult.lines ?? [];
98
+ const exactWord = words.find((w) => normalizeForMatch(w.text) === needle);
99
+ if (exactWord)
100
+ return { text: exactWord.text, tapCenter: exactWord.tapCenter };
101
+ const exactLine = lines.find((l) => normalizeForMatch(l.text) === needle);
102
+ if (exactLine)
103
+ return { text: exactLine.text, tapCenter: exactLine.tapCenter };
104
+ const substringLine = lines.find((l) => normalizeForMatch(l.text).includes(needle));
105
+ if (substringLine)
106
+ return { text: substringLine.text, tapCenter: substringLine.tapCenter };
107
+ const substringWord = words.find((w) => normalizeForMatch(w.text).includes(needle));
108
+ if (substringWord)
109
+ return { text: substringWord.text, tapCenter: substringWord.tapCenter };
110
+ return null;
26
111
  }
27
112
  export function getAvailableStrategies(query, strategy) {
28
113
  if (query.x !== undefined && query.y !== undefined) {
29
114
  return ["coordinate"];
30
115
  }
31
116
  if (strategy !== "auto") {
117
+ // Always fallback to OCR for text queries — explicit strategy may miss visible text
118
+ if (query.text && strategy !== "ocr" && strategy !== "coordinate") {
119
+ return [strategy, "ocr"];
120
+ }
32
121
  return [strategy];
33
122
  }
34
123
  if (query.component && !query.text && !query.testID) {
35
124
  return ["fiber"];
36
125
  }
37
126
  if (query.testID && !query.text) {
38
- return ["fiber", "accessibility"];
127
+ return ["accessibility", "fiber"];
39
128
  }
40
129
  if (query.text) {
41
130
  const strategies = [];
42
- if (!isNonAscii(query.text)) {
131
+ strategies.push("accessibility");
132
+ if (!hasProblematicUnicode(query.text)) {
43
133
  strategies.push("fiber");
44
134
  }
45
- strategies.push("accessibility", "ocr");
135
+ strategies.push("ocr");
46
136
  return strategies;
47
137
  }
48
138
  return ["fiber", "accessibility", "ocr"];
49
139
  }
50
- export function convertPixelsToPoints(pixelX, pixelY, platform, devicePixelRatio, scaleFactor = 1) {
51
- const originalX = pixelX * scaleFactor;
52
- const originalY = pixelY * scaleFactor;
140
+ /**
141
+ * Convert screenshot image coordinates to platform-native tap coordinates.
142
+ *
143
+ * For iOS: screenshot pixels → device pixels (undo downscale) → points (÷ DPR)
144
+ * For Android: screenshot pixels → device pixels (undo downscale)
145
+ *
146
+ * IMPORTANT: Only use this for EXTERNAL coordinates from screenshots.
147
+ * Internal strategies (OCR, accessibility, fiber) produce tap-ready coordinates
148
+ * and call iosTap/androidTap directly — they must NOT go through this function.
149
+ */
150
+ export function convertScreenshotToTapCoords(pixelX, pixelY, platform, devicePixelRatio, scaleFactor = 1) {
151
+ const deviceX = pixelX * scaleFactor;
152
+ const deviceY = pixelY * scaleFactor;
53
153
  if (platform === "android") {
54
- return { x: Math.round(originalX), y: Math.round(originalY) };
154
+ return { x: Math.round(deviceX), y: Math.round(deviceY) };
55
155
  }
56
156
  return {
57
- x: Math.round(originalX / devicePixelRatio),
58
- y: Math.round(originalY / devicePixelRatio),
157
+ x: Math.round(deviceX / devicePixelRatio),
158
+ y: Math.round(deviceY / devicePixelRatio)
59
159
  };
60
160
  }
161
+ /** @deprecated Use convertScreenshotToTapCoords instead */
162
+ export const convertPixelsToPoints = convertScreenshotToTapCoords;
61
163
  export async function getCurrentScreen() {
62
164
  try {
63
165
  // Note: Uses var instead of let/const because Hermes Runtime.evaluate
@@ -75,7 +177,7 @@ export async function getCurrentScreen() {
75
177
  if (roots.length === 0) return null;
76
178
 
77
179
  function findScreen(fiber, depth) {
78
- if (!fiber || depth > 30) return null;
180
+ if (!fiber || depth > 5000) return null;
79
181
  var name = fiber.type && (fiber.type.displayName || fiber.type.name || (typeof fiber.type === 'string' ? fiber.type : null));
80
182
 
81
183
  if (name === 'RNSScreen') {
@@ -107,7 +209,7 @@ export async function getCurrentScreen() {
107
209
  }
108
210
 
109
211
  function findFirstUserComponent(fiber, depth) {
110
- if (!fiber || depth > 10) return null;
212
+ if (!fiber || depth > 5000) return null;
111
213
  var name = fiber.type && (fiber.type.displayName || fiber.type.name);
112
214
  if (name && typeof fiber.type !== 'string') return name;
113
215
  var child = fiber.child;
@@ -136,21 +238,33 @@ export async function getCurrentScreen() {
136
238
  }
137
239
  }
138
240
  export function formatTapSuccess(data) {
241
+ const { screenshot, verification, ...rest } = data;
139
242
  return {
140
243
  success: true,
141
- ...data,
244
+ ...rest,
245
+ ...(verification && { verification }),
246
+ ...(screenshot && { screenshot })
142
247
  };
143
248
  }
144
249
  export function formatTapFailure(data) {
145
250
  const errorMsg = data.error || buildErrorMessage(data.query);
251
+ const warning = data.verification && !data.verification.meaningful
252
+ ? "Tap executed but no visual change detected. The element may not exist at these coordinates. Examine the screenshot to verify and retry with adjusted coordinates."
253
+ : undefined;
254
+ const lastStrategy = data.attempted.length > 0 ? data.attempted[data.attempted.length - 1].strategy : undefined;
146
255
  return {
147
256
  success: false,
257
+ method: lastStrategy,
148
258
  query: data.query,
149
259
  screen: data.screen,
150
260
  error: errorMsg,
151
261
  attempted: data.attempted,
152
262
  suggestion: data.suggestion,
153
263
  matches: data.matches,
264
+ ...(data.device && { device: data.device }),
265
+ ...(data.verification && { verification: data.verification }),
266
+ ...(data.screenshot && { screenshot: data.screenshot }),
267
+ ...(warning && { warning })
154
268
  };
155
269
  }
156
270
  function buildErrorMessage(query) {
@@ -165,13 +279,34 @@ function buildErrorMessage(query) {
165
279
  }
166
280
  // --- Strategy Functions ---
167
281
  async function tryFiberStrategy(query, index, maxTraversalDepth) {
282
+ // Retry with increasing depth if the initial traversal finds nothing
283
+ const baseDepth = maxTraversalDepth ?? 15;
284
+ const depthAttempts = [baseDepth];
285
+ // Only add deeper retries if user didn't explicitly set a high depth
286
+ if (baseDepth <= 15) {
287
+ depthAttempts.push(30, 45);
288
+ }
289
+ else if (baseDepth <= 30) {
290
+ depthAttempts.push(baseDepth * 2);
291
+ }
292
+ let lastResult = null;
293
+ for (const depth of depthAttempts) {
294
+ const result = await tryFiberAtDepth(query, index, depth);
295
+ if (result.success || result.matches) {
296
+ return result;
297
+ }
298
+ lastResult = result;
299
+ }
300
+ return lastResult;
301
+ }
302
+ async function tryFiberAtDepth(query, index, maxTraversalDepth) {
168
303
  try {
169
304
  const result = await pressElement({
170
305
  text: query.text,
171
306
  testID: query.testID,
172
307
  component: query.component,
173
308
  index,
174
- maxTraversalDepth,
309
+ maxTraversalDepth
175
310
  });
176
311
  if (!result.success) {
177
312
  return { success: false, reason: result.error || "pressElement failed" };
@@ -183,67 +318,115 @@ async function tryFiberStrategy(query, index, maxTraversalDepth) {
183
318
  if (parsed.error) {
184
319
  const strategyResult = {
185
320
  success: false,
186
- reason: parsed.error,
321
+ reason: parsed.error
187
322
  };
188
323
  if (parsed.matches) {
189
324
  strategyResult.matches = parsed.matches;
190
325
  }
191
326
  return strategyResult;
192
327
  }
193
- // Input elements found via fiber can't be pressed — need native tap
194
- // Return as failure so orchestrator falls through to accessibility/coordinate
328
+ // Fiber finds the element by text/testID/component, then measures its
329
+ // host component's screen position for a native tap. This ensures the tap
330
+ // goes through React's event pipeline, executing any onPress wrappers
331
+ // (analytics, debouncing, state tracking) inside the component.
195
332
  if (parsed.needsNativeTap) {
333
+ const elementType = parsed.isInput ? "input element" : "pressable element";
334
+ if (parsed.nativeTapTarget && parsed.nativeTapTarget.x && parsed.nativeTapTarget.y) {
335
+ return {
336
+ success: false,
337
+ reason: `Found ${parsed.pressed} (${elementType}) — measured coordinates for native tap`,
338
+ pressed: parsed.pressed,
339
+ text: parsed.text,
340
+ path: parsed.path || null,
341
+ component: parsed.pressed || null,
342
+ convertedTo: {
343
+ x: parsed.nativeTapTarget.x,
344
+ y: parsed.nativeTapTarget.y,
345
+ unit: parsed.nativeTapTarget.unit || "points"
346
+ }
347
+ };
348
+ }
196
349
  return {
197
350
  success: false,
198
- reason: `Found ${parsed.pressed} (input element) but it requires native tap — falling through to next strategy`,
351
+ reason: `Found ${parsed.pressed} (${elementType}) but could not measure coordinates — falling through to next strategy`
199
352
  };
200
353
  }
354
+ // All elements now use needsNativeTap — this shouldn't be reached
201
355
  return {
202
- success: true,
203
- reason: "Pressed via React fiber tree",
204
- pressed: parsed.pressed,
205
- text: parsed.text,
206
- path: parsed.path || null,
207
- component: parsed.pressed || null,
356
+ success: false,
357
+ reason: "Unexpected: element did not request native tap"
208
358
  };
209
359
  }
210
360
  catch (err) {
211
361
  return {
212
362
  success: false,
213
- reason: `Fiber strategy error: ${err instanceof Error ? err.message : String(err)}`,
363
+ reason: `Fiber strategy error: ${err instanceof Error ? err.message : String(err)}`
214
364
  };
215
365
  }
216
366
  }
217
- async function tryAccessibilityStrategy(query, index, platform) {
367
+ async function tryAccessibilityStrategy(query, index, platform, udid) {
218
368
  try {
219
369
  const hasTestID = !!query.testID;
220
370
  const hasText = !!query.text;
221
371
  if (!hasTestID && !hasText) {
222
- return { success: false, reason: "No text or testID for accessibility search" };
372
+ return {
373
+ success: false,
374
+ reason: "No text or testID for accessibility search"
375
+ };
223
376
  }
224
377
  if (platform === "ios") {
225
- // iOS: IDB does not expose accessibilityIdentifier (testID),
226
- // so search by labelContains as best-effort fallback
227
- const searchText = query.text || query.testID;
228
- const result = await iosFindElement({
229
- labelContains: searchText,
230
- index,
231
- });
378
+ // iOS: testID maps to accessibilityIdentifier — search by identifier first,
379
+ // then fall back to labelContains for text-based searches
380
+ let result;
381
+ if (hasTestID && !hasText) {
382
+ // Try exact identifier match first (testID → accessibilityIdentifier)
383
+ result = await iosFindElement({
384
+ identifier: query.testID,
385
+ index
386
+ }, udid);
387
+ // Fall back to identifierContains if exact match fails
388
+ if (!result.success || !result.allMatches || result.allMatches.length === 0) {
389
+ result = await iosFindElement({
390
+ identifierContains: query.testID,
391
+ index
392
+ }, udid);
393
+ }
394
+ // Last resort: try labelContains in case testID is reflected in label
395
+ if (!result.success || !result.allMatches || result.allMatches.length === 0) {
396
+ result = await iosFindElement({
397
+ labelContains: query.testID,
398
+ index
399
+ }, udid);
400
+ }
401
+ }
402
+ else {
403
+ const searchText = query.text;
404
+ result = await iosFindElement({
405
+ labelContains: searchText,
406
+ index
407
+ }, udid);
408
+ }
232
409
  if (!result.success || !result.allMatches || result.allMatches.length === 0) {
233
- return { success: false, reason: result.error ?? "No iOS accessibility match" };
410
+ return {
411
+ success: false,
412
+ reason: result.error ?? "No iOS accessibility match"
413
+ };
234
414
  }
235
415
  const match = result.allMatches[index ?? 0];
236
416
  if (!match) {
237
- return { success: false, reason: `Index ${index} out of bounds (${result.allMatches.length} matches)` };
417
+ return {
418
+ success: false,
419
+ reason: `Index ${index} out of bounds (${result.allMatches.length} matches)`
420
+ };
238
421
  }
239
- await iosTap(match.center.x, match.center.y);
422
+ await iosTap(match.center.x, match.center.y, { udid });
240
423
  return {
241
424
  success: true,
242
425
  reason: "Tapped via iOS accessibility",
243
426
  pressed: match.label || match.type,
244
427
  text: match.label || undefined,
245
428
  component: match.type || null,
246
- convertedTo: { x: match.center.x, y: match.center.y, unit: "points" },
429
+ convertedTo: { x: match.center.x, y: match.center.y, unit: "points" }
247
430
  };
248
431
  }
249
432
  else {
@@ -258,19 +441,24 @@ async function tryAccessibilityStrategy(query, index, platform) {
258
441
  let result = await androidFindElement(searchOptions);
259
442
  // If testID search via resourceId failed, try contentDescContains
260
443
  // (older RN versions map testID to content-description)
261
- if (hasTestID && !hasText &&
262
- (!result.success || !result.allMatches || result.allMatches.length === 0)) {
444
+ if (hasTestID && !hasText && (!result.success || !result.allMatches || result.allMatches.length === 0)) {
263
445
  result = await androidFindElement({
264
446
  contentDescContains: query.testID,
265
- index,
447
+ index
266
448
  });
267
449
  }
268
450
  if (!result.success || !result.allMatches || result.allMatches.length === 0) {
269
- return { success: false, reason: result.error ?? "No Android accessibility match" };
451
+ return {
452
+ success: false,
453
+ reason: result.error ?? "No Android accessibility match"
454
+ };
270
455
  }
271
456
  const match = result.allMatches[index ?? 0];
272
457
  if (!match) {
273
- return { success: false, reason: `Index ${index} out of bounds (${result.allMatches.length} matches)` };
458
+ return {
459
+ success: false,
460
+ reason: `Index ${index} out of bounds (${result.allMatches.length} matches)`
461
+ };
274
462
  }
275
463
  await androidTap(match.center.x, match.center.y);
276
464
  return {
@@ -279,18 +467,18 @@ async function tryAccessibilityStrategy(query, index, platform) {
279
467
  pressed: match.text || match.className || undefined,
280
468
  text: match.text || undefined,
281
469
  component: match.className || undefined,
282
- convertedTo: { x: match.center.x, y: match.center.y, unit: "pixels" },
470
+ convertedTo: { x: match.center.x, y: match.center.y, unit: "pixels" }
283
471
  };
284
472
  }
285
473
  }
286
474
  catch (err) {
287
475
  return {
288
476
  success: false,
289
- reason: `Accessibility strategy error: ${err instanceof Error ? err.message : String(err)}`,
477
+ reason: `Accessibility strategy error: ${err instanceof Error ? err.message : String(err)}`
290
478
  };
291
479
  }
292
480
  }
293
- async function tryOcrStrategy(query, platform) {
481
+ async function tryOcrStrategy(query, platform, udid) {
294
482
  try {
295
483
  const searchText = query.text;
296
484
  if (!searchText) {
@@ -298,78 +486,89 @@ async function tryOcrStrategy(query, platform) {
298
486
  }
299
487
  let imageBuffer;
300
488
  let scaleFactor = 1;
301
- let originalWidth;
302
- let originalHeight;
303
489
  if (platform === "ios") {
304
- const screenshot = await iosScreenshot();
490
+ const screenshot = await iosScreenshot(undefined, udid);
305
491
  if (!screenshot.success || !screenshot.data) {
306
- return { success: false, reason: "Failed to capture iOS screenshot for OCR" };
492
+ return {
493
+ success: false,
494
+ reason: "Failed to capture iOS screenshot for OCR"
495
+ };
307
496
  }
308
497
  imageBuffer = screenshot.data;
309
498
  scaleFactor = screenshot.scaleFactor ?? 1;
310
- originalWidth = screenshot.originalWidth;
311
- originalHeight = screenshot.originalHeight;
312
499
  }
313
500
  else {
314
501
  const { androidScreenshot } = await import("../core/android.js");
315
502
  const screenshot = await androidScreenshot();
316
503
  if (!screenshot.success || !screenshot.data) {
317
- return { success: false, reason: "Failed to capture Android screenshot for OCR" };
504
+ return {
505
+ success: false,
506
+ reason: "Failed to capture Android screenshot for OCR"
507
+ };
318
508
  }
319
509
  imageBuffer = screenshot.data;
320
510
  scaleFactor = screenshot.scaleFactor ?? 1;
321
- originalWidth = screenshot.originalWidth;
322
- originalHeight = screenshot.originalHeight;
323
511
  }
324
- const devicePixelRatio = (platform === "ios" && originalWidth && originalHeight)
325
- ? inferIOSDevicePixelRatio(originalWidth, originalHeight)
326
- : 3;
327
512
  const { recognizeText } = await import("../core/ocr.js");
328
513
  const ocrResult = await recognizeText(imageBuffer, {
329
514
  scaleFactor,
330
- platform,
331
- devicePixelRatio,
515
+ platform
332
516
  });
333
- const lowerSearch = searchText.toLowerCase();
334
- const matchingWord = ocrResult.words.find((w) => w.text.toLowerCase() === lowerSearch) || ocrResult.words.find((w) => w.text.toLowerCase().includes(lowerSearch));
335
- if (!matchingWord) {
336
- return { success: false, reason: `OCR did not find text "${searchText}" on screen` };
517
+ const match = findOcrMatch(ocrResult, searchText);
518
+ if (!match) {
519
+ return {
520
+ success: false,
521
+ reason: `OCR did not find text "${searchText}" on screen`
522
+ };
337
523
  }
338
524
  if (platform === "ios") {
339
- await iosTap(matchingWord.tapCenter.x, matchingWord.tapCenter.y);
525
+ // tapCenter is in image-pixel space (downscaled) — convert to points
526
+ const { getDevicePixelRatio } = await import("../core/ios.js");
527
+ const dpr = await getDevicePixelRatio(udid);
528
+ const tapResult = await iosTap(Math.round((match.tapCenter.x * scaleFactor) / dpr), Math.round((match.tapCenter.y * scaleFactor) / dpr), { udid });
529
+ if (!tapResult.success) {
530
+ return {
531
+ success: false,
532
+ reason: `OCR found "${match.text}" but tap failed: ${tapResult.error}`
533
+ };
534
+ }
340
535
  }
341
536
  else {
342
- await androidTap(matchingWord.tapCenter.x, matchingWord.tapCenter.y);
537
+ // Android: image-pixel → device-pixel (undo downscale), ADB accepts pixels
538
+ await androidTap(Math.round(match.tapCenter.x * scaleFactor), Math.round(match.tapCenter.y * scaleFactor));
343
539
  }
344
540
  return {
345
541
  success: true,
346
542
  reason: "Tapped via OCR text recognition",
347
- text: matchingWord.text,
543
+ text: match.text,
348
544
  convertedTo: {
349
- x: matchingWord.tapCenter.x,
350
- y: matchingWord.tapCenter.y,
351
- unit: platform === "ios" ? "points" : "pixels",
352
- },
545
+ x: match.tapCenter.x,
546
+ y: match.tapCenter.y,
547
+ unit: "pixels"
548
+ }
353
549
  };
354
550
  }
355
551
  catch (err) {
356
552
  return {
357
553
  success: false,
358
- reason: `OCR strategy error: ${err instanceof Error ? err.message : String(err)}`,
554
+ reason: `OCR strategy error: ${err instanceof Error ? err.message : String(err)}`
359
555
  };
360
556
  }
361
557
  }
362
- async function tryCoordinateStrategy(pixelX, pixelY, platform, lastScreenshot) {
558
+ async function tryCoordinateStrategy(pixelX, pixelY, platform, lastScreenshot, udid) {
363
559
  try {
364
560
  if (platform === "ios") {
365
561
  const scaleFactor = lastScreenshot?.scaleFactor ?? 1;
366
- const originalWidth = lastScreenshot?.originalWidth;
367
- const originalHeight = lastScreenshot?.originalHeight;
368
- const devicePixelRatio = (originalWidth && originalHeight)
369
- ? inferIOSDevicePixelRatio(originalWidth, originalHeight)
370
- : 3;
371
- const converted = convertPixelsToPoints(pixelX, pixelY, "ios", devicePixelRatio, scaleFactor);
372
- await iosTap(converted.x, converted.y);
562
+ const { getDevicePixelRatio } = await import("../core/ios.js");
563
+ const devicePixelRatio = await getDevicePixelRatio(udid);
564
+ const converted = convertScreenshotToTapCoords(pixelX, pixelY, "ios", devicePixelRatio, scaleFactor);
565
+ const tapResult = await iosTap(converted.x, converted.y, { udid });
566
+ if (!tapResult.success) {
567
+ return {
568
+ success: false,
569
+ reason: `Coordinate tap failed: ${tapResult.error}`
570
+ };
571
+ }
373
572
  // Best-effort: identify what was tapped via fiber tree
374
573
  let screen = null;
375
574
  try {
@@ -382,12 +581,12 @@ async function tryCoordinateStrategy(pixelX, pixelY, platform, lastScreenshot) {
382
581
  success: true,
383
582
  reason: "Tapped at coordinates (iOS)",
384
583
  screen,
385
- convertedTo: { x: converted.x, y: converted.y, unit: "points" },
584
+ convertedTo: { x: converted.x, y: converted.y, unit: "points" }
386
585
  };
387
586
  }
388
587
  else {
389
588
  const scaleFactor = lastScreenshot?.scaleFactor ?? 1;
390
- const converted = convertPixelsToPoints(pixelX, pixelY, "android", 1, scaleFactor);
589
+ const converted = convertScreenshotToTapCoords(pixelX, pixelY, "android", 1, scaleFactor);
391
590
  await androidTap(converted.x, converted.y);
392
591
  // Best-effort: identify what was tapped via fiber tree
393
592
  let screen = null;
@@ -401,16 +600,204 @@ async function tryCoordinateStrategy(pixelX, pixelY, platform, lastScreenshot) {
401
600
  success: true,
402
601
  reason: "Tapped at coordinates (Android)",
403
602
  screen,
404
- convertedTo: { x: converted.x, y: converted.y, unit: "pixels" },
603
+ convertedTo: { x: converted.x, y: converted.y, unit: "pixels" }
405
604
  };
406
605
  }
407
606
  }
408
607
  catch (err) {
409
608
  return {
410
609
  success: false,
411
- reason: `Coordinate strategy error: ${err instanceof Error ? err.message : String(err)}`,
610
+ reason: `Coordinate strategy error: ${err instanceof Error ? err.message : String(err)}`
611
+ };
612
+ }
613
+ }
614
+ const SETTLE_DELAY_MS = 800;
615
+ const TAP_TIMEOUT_MS = 20000;
616
+ const MIN_STRATEGY_BUDGET_MS = 500;
617
+ const MAX_STRATEGY_MS = {
618
+ fiber: 5000,
619
+ accessibility: 3000,
620
+ ocr: 5000,
621
+ coordinate: 3000
622
+ };
623
+ // Matches only the outer withTimeout wrapper message for a tap strategy.
624
+ // Nested sub-operation errors inside a strategy (e.g. "CDP getProperties timed out after 150ms")
625
+ // must NOT be classified as a tap-level timeout.
626
+ const STRATEGY_TIMEOUT_RE = /^(fiber|accessibility|ocr|coordinate) timed out after \d+ms$/;
627
+ export function isTapTimeout(attempted) {
628
+ return attempted.some((a) => STRATEGY_TIMEOUT_RE.test(a.reason) || a.reason.startsWith("Skipped —"));
629
+ }
630
+ function withTimeout(promise, ms, label) {
631
+ return new Promise((resolve, reject) => {
632
+ const timer = setTimeout(() => reject(new Error(`${label} timed out after ${ms}ms`)), ms);
633
+ promise.then((val) => {
634
+ clearTimeout(timer);
635
+ resolve(val);
636
+ }, (err) => {
637
+ clearTimeout(timer);
638
+ reject(err);
639
+ });
640
+ });
641
+ }
642
+ async function captureScreenshot(platform, udid) {
643
+ try {
644
+ const result = platform === "ios" ? await iosScreenshot(undefined, udid) : await androidScreenshot();
645
+ if (!result.success || !result.data)
646
+ return null;
647
+ return {
648
+ buffer: result.data,
649
+ width: result.originalWidth || 0,
650
+ height: result.originalHeight || 0,
651
+ scaleFactor: result.scaleFactor || 1
412
652
  };
413
653
  }
654
+ catch {
655
+ return null;
656
+ }
657
+ }
658
+ function screenshotToBase64(buffer) {
659
+ return buffer.toString("base64");
660
+ }
661
+ async function verifyAndCapture(platform, shouldVerify, shouldScreenshot, beforeBuffer, udid, beforeScaleFactor) {
662
+ if (!shouldScreenshot)
663
+ return {};
664
+ await new Promise((resolve) => setTimeout(resolve, SETTLE_DELAY_MS));
665
+ const after = await captureScreenshot(platform, udid);
666
+ if (!after)
667
+ return {};
668
+ const screenshot = {
669
+ image: screenshotToBase64(after.buffer),
670
+ width: after.width,
671
+ height: after.height,
672
+ scaleFactor: after.scaleFactor
673
+ };
674
+ let verification;
675
+ if (shouldVerify && beforeBuffer) {
676
+ try {
677
+ const rawStatusBar = platform === "ios" ? 177 : 142; // pixels in original screenshot space
678
+ const scale = beforeScaleFactor || after.scaleFactor || 1;
679
+ const statusBarHeight = Math.round(rawStatusBar / scale);
680
+ const diff = await compareScreenshots(beforeBuffer, after.buffer, {
681
+ statusBarHeight
682
+ });
683
+ verification = {
684
+ meaningful: diff.changed,
685
+ changeRate: diff.changeRate,
686
+ changedPixels: diff.changedPixels,
687
+ totalPixels: diff.totalPixels,
688
+ explanation: buildVerificationExplanation({
689
+ meaningful: diff.changed,
690
+ changeRate: diff.changeRate,
691
+ changedPixels: diff.changedPixels,
692
+ totalPixels: diff.totalPixels
693
+ })
694
+ };
695
+ }
696
+ catch {
697
+ // Diff failed — still return the screenshot
698
+ }
699
+ }
700
+ const verifyGroupId = `verify-${Date.now()}-${Math.random().toString(36).slice(2, 8)}`;
701
+ if (beforeBuffer) {
702
+ imageBuffer.add({
703
+ id: `${verifyGroupId}-before`,
704
+ image: beforeBuffer,
705
+ timestamp: Date.now(),
706
+ source: "tap-verify",
707
+ groupId: verifyGroupId,
708
+ metadata: { phase: "before" }
709
+ });
710
+ }
711
+ imageBuffer.add({
712
+ id: `${verifyGroupId}-after`,
713
+ image: after.buffer,
714
+ timestamp: Date.now(),
715
+ source: "tap-verify",
716
+ groupId: verifyGroupId,
717
+ metadata: { phase: "after", changeRate: verification?.changeRate }
718
+ });
719
+ return { screenshot, verification };
720
+ }
721
+ // --- Burst Capture ---
722
+ const BURST_FRAME_COUNT = 4;
723
+ const BURST_FRAME_INTERVAL_MS = 150;
724
+ async function burstCaptureAndVerify(platform, beforeBuffer, udid, beforeScaleFactor) {
725
+ if (!beforeBuffer)
726
+ return {};
727
+ const frames = [beforeBuffer];
728
+ let capturedScaleFactor = beforeScaleFactor || 1;
729
+ for (let i = 0; i < BURST_FRAME_COUNT; i++) {
730
+ await new Promise((resolve) => setTimeout(resolve, BURST_FRAME_INTERVAL_MS));
731
+ const capture = await captureScreenshot(platform, udid);
732
+ if (capture) {
733
+ frames.push(capture.buffer);
734
+ if (i === 0)
735
+ capturedScaleFactor = capture.scaleFactor || capturedScaleFactor;
736
+ }
737
+ }
738
+ if (frames.length < 2)
739
+ return {};
740
+ const rawStatusBar = platform === "ios" ? 177 : 142; // pixels in original screenshot space
741
+ const statusBarHeight = Math.round(rawStatusBar / capturedScaleFactor);
742
+ const analysis = await analyzeBurstFrames(frames, { statusBarHeight });
743
+ const groupId = `burst-${Date.now()}-${Math.random().toString(36).slice(2, 8)}`;
744
+ for (let i = 0; i < frames.length; i++) {
745
+ imageBuffer.add({
746
+ id: `${groupId}-f${i}`,
747
+ image: frames[i],
748
+ timestamp: Date.now(),
749
+ source: "tap-burst",
750
+ groupId,
751
+ metadata: {
752
+ frameIndex: i,
753
+ isBefore: i === 0,
754
+ changeRate: i === 0 ? 0 : analysis.framesWithChange.includes(i) ? analysis.peakChangeRate : 0
755
+ }
756
+ });
757
+ }
758
+ imageBuffer.addGroup({
759
+ groupId,
760
+ intent: "tap-verification",
761
+ source: "tap-burst",
762
+ timestamp: Date.now(),
763
+ frameCount: frames.length,
764
+ summary: {
765
+ peakChangeRate: analysis.peakChangeRate,
766
+ peakFrame: analysis.peakFrame,
767
+ framesWithChange: analysis.framesWithChange,
768
+ transientChangeDetected: analysis.transientChangeDetected,
769
+ persistentChangeRate: analysis.persistentChangeRate
770
+ }
771
+ });
772
+ // Get dimensions from the last frame using sharp
773
+ const sharp = (await import("sharp")).default;
774
+ const meta = await sharp(frames[frames.length - 1]).metadata();
775
+ const screenshot = {
776
+ image: screenshotToBase64(frames[frames.length - 1]),
777
+ width: meta.width || 0,
778
+ height: meta.height || 0,
779
+ scaleFactor: 1
780
+ };
781
+ const verification = {
782
+ meaningful: analysis.meaningful,
783
+ changeRate: analysis.persistentChangeRate,
784
+ changedPixels: 0,
785
+ totalPixels: 0,
786
+ transientChangeDetected: analysis.transientChangeDetected,
787
+ peakChangeRate: analysis.peakChangeRate,
788
+ peakFrame: analysis.peakFrame,
789
+ burstGroupId: groupId,
790
+ explanation: buildVerificationExplanation({
791
+ meaningful: analysis.meaningful,
792
+ changeRate: analysis.persistentChangeRate,
793
+ changedPixels: 0,
794
+ totalPixels: 0,
795
+ transientChangeDetected: analysis.transientChangeDetected,
796
+ peakChangeRate: analysis.peakChangeRate,
797
+ peakFrame: analysis.peakFrame
798
+ })
799
+ };
800
+ return { screenshot, verification };
414
801
  }
415
802
  // --- Orchestrator ---
416
803
  export async function tap(options) {
@@ -418,6 +805,8 @@ export async function tap(options) {
418
805
  const strategy = options.strategy || "auto";
419
806
  const index = options.index;
420
807
  const maxTraversalDepth = options.maxTraversalDepth;
808
+ const deadline = Date.now() + TAP_TIMEOUT_MS;
809
+ const remainingMs = () => Math.max(0, deadline - Date.now());
421
810
  // Validate inputs
422
811
  const hasSearchParam = query.text || query.testID || query.component;
423
812
  const hasCoordinates = query.x !== undefined || query.y !== undefined;
@@ -425,68 +814,302 @@ export async function tap(options) {
425
814
  return {
426
815
  success: false,
427
816
  query,
428
- error: "Must provide at least one of: text, testID, component, or x/y coordinates",
817
+ error: "Must provide at least one of: text, testID, component, or x/y coordinates"
429
818
  };
430
819
  }
431
820
  if (hasCoordinates && (query.x === undefined || query.y === undefined)) {
432
821
  return {
433
822
  success: false,
434
823
  query,
435
- error: "Both x and y coordinates must be provided",
824
+ error: "Both x and y coordinates must be provided"
436
825
  };
437
826
  }
438
- // Get connected app — auto-connect if none
439
- let apps = Array.from(connectedApps.values());
440
- if (apps.length === 0) {
441
- try {
442
- clearReconnectionSuppression();
443
- const openPorts = await scanMetroPorts();
444
- for (const port of openPorts) {
445
- const devices = await fetchDevices(port);
446
- const mainDevice = selectMainDevice(devices);
447
- if (mainDevice) {
448
- await connectToDevice(mainDevice, port);
449
- break;
827
+ // Native mode: bypass React Native connection, tap directly via ADB/simctl
828
+ if (options.native && hasCoordinates) {
829
+ let platform = options.platform;
830
+ // Auto-detect platform if not specified
831
+ let nativeUdid;
832
+ if (!platform) {
833
+ const [androidDevice, iosSimulator] = await Promise.all([
834
+ getDefaultAndroidDevice().catch(() => null),
835
+ getActiveOrBootedSimulatorUdid().catch(() => null)
836
+ ]);
837
+ if (androidDevice && iosSimulator) {
838
+ return {
839
+ success: false,
840
+ query,
841
+ error: 'Multiple platforms detected (both Android and iOS). Specify platform: "android" or platform: "ios" to target the correct device.'
842
+ };
843
+ }
844
+ if (androidDevice) {
845
+ platform = "android";
846
+ }
847
+ else if (iosSimulator) {
848
+ platform = "ios";
849
+ nativeUdid = iosSimulator;
850
+ }
851
+ else {
852
+ return {
853
+ success: false,
854
+ query,
855
+ error: "No Android device or iOS simulator found. Connect a device or start a simulator."
856
+ };
857
+ }
858
+ }
859
+ else if (platform === "ios") {
860
+ nativeUdid = (await getActiveOrBootedSimulatorUdid()) ?? undefined;
861
+ }
862
+ const nativeShouldScreenshot = options.screenshot !== false;
863
+ const nativeShouldVerify = nativeShouldScreenshot && options.verify !== false;
864
+ let nativeBeforeBuffer = null;
865
+ let nativeScreenshotMeta;
866
+ if (nativeShouldVerify) {
867
+ const before = await captureScreenshot(platform, nativeUdid);
868
+ nativeBeforeBuffer = before?.buffer || null;
869
+ if (before) {
870
+ nativeScreenshotMeta = {
871
+ originalWidth: before.width,
872
+ originalHeight: before.height,
873
+ scaleFactor: before.scaleFactor
874
+ };
875
+ }
876
+ }
877
+ // If no screenshot was taken for verification, take one just for scaleFactor
878
+ if (!nativeScreenshotMeta) {
879
+ const ref = await captureScreenshot(platform, nativeUdid);
880
+ if (ref) {
881
+ nativeScreenshotMeta = {
882
+ originalWidth: ref.width,
883
+ originalHeight: ref.height,
884
+ scaleFactor: ref.scaleFactor
885
+ };
886
+ // Also use it for verification if buffer is needed
887
+ if (!nativeBeforeBuffer) {
888
+ nativeBeforeBuffer = ref.buffer;
450
889
  }
451
890
  }
452
- apps = Array.from(connectedApps.values());
891
+ }
892
+ let result;
893
+ try {
894
+ result = await withTimeout(tryCoordinateStrategy(query.x, query.y, platform, nativeScreenshotMeta, nativeUdid), remainingMs(), "native-coordinate");
895
+ }
896
+ catch (err) {
897
+ return formatTapFailure({
898
+ query,
899
+ attempted: [
900
+ {
901
+ strategy: "native-coordinate",
902
+ reason: err instanceof Error ? err.message : String(err)
903
+ }
904
+ ],
905
+ suggestion: `Tap timed out. Take a screenshot (${platform === "ios" ? "ios_screenshot" : "android_screenshot"}) and retry with coordinates.`
906
+ });
907
+ }
908
+ if (result.success) {
909
+ let screenshot;
910
+ let verification;
911
+ if (options.burst && nativeShouldVerify && nativeBeforeBuffer) {
912
+ ({ screenshot, verification } = await burstCaptureAndVerify(platform, nativeBeforeBuffer, nativeUdid, nativeScreenshotMeta?.scaleFactor));
913
+ }
914
+ else {
915
+ ({ screenshot, verification } = await verifyAndCapture(platform, nativeShouldVerify, nativeShouldScreenshot, nativeBeforeBuffer, nativeUdid, nativeScreenshotMeta?.scaleFactor));
916
+ }
917
+ return formatTapSuccess({
918
+ method: "native-coordinate",
919
+ query,
920
+ pressed: result.pressed,
921
+ convertedTo: result.convertedTo,
922
+ platform,
923
+ screenshot,
924
+ verification
925
+ });
926
+ }
927
+ return formatTapFailure({
928
+ query,
929
+ attempted: [{ strategy: "native-coordinate", reason: result.reason }],
930
+ suggestion: `Take a screenshot (${platform === "ios" ? "ios_screenshot" : "android_screenshot"}) to verify coordinates.`
931
+ });
932
+ }
933
+ // Detect available capabilities
934
+ const allApps = Array.from(connectedApps.values());
935
+ let hasMetro = allApps.length > 0;
936
+ // If platform is specified, prefer an app matching that platform
937
+ let app = options.platform
938
+ ? (allApps.find((a) => a.platform === options.platform) ?? allApps[0])
939
+ : allApps[0];
940
+ // Try to auto-connect to Metro (for fiber strategy), but don't fail if it doesn't work
941
+ if (!hasMetro) {
942
+ try {
943
+ await withTimeout((async () => {
944
+ clearReconnectionSuppression();
945
+ const openPorts = await scanMetroPorts();
946
+ for (const port of openPorts) {
947
+ const devices = await fetchDevices(port);
948
+ const mainDevice = selectMainDevice(devices);
949
+ if (mainDevice) {
950
+ await connectToDevice(mainDevice, port);
951
+ break;
952
+ }
953
+ }
954
+ })(), Math.min(remainingMs(), 3000), "auto-connect");
955
+ const apps = Array.from(connectedApps.values());
956
+ hasMetro = apps.length > 0;
957
+ app = apps[0];
453
958
  }
454
959
  catch {
455
- // Auto-connect failed, will fall through to error below
960
+ // Auto-connect failed — Metro-dependent strategies will be skipped
961
+ }
962
+ }
963
+ // Detect platform — from Metro connection or from available devices
964
+ let platform = options.platform;
965
+ if (!platform) {
966
+ if (app) {
967
+ platform = app.platform;
968
+ }
969
+ else {
970
+ // No Metro — detect from available devices
971
+ const [androidDevice, iosSimulator] = await Promise.all([
972
+ getDefaultAndroidDevice().catch(() => null),
973
+ getActiveOrBootedSimulatorUdid().catch(() => null)
974
+ ]);
975
+ if (androidDevice && iosSimulator) {
976
+ return {
977
+ success: false,
978
+ query,
979
+ error: 'Both iOS and Android devices detected but no Metro connection. Specify platform="ios" or platform="android".'
980
+ };
981
+ }
982
+ if (androidDevice)
983
+ platform = "android";
984
+ else if (iosSimulator)
985
+ platform = "ios";
456
986
  }
457
987
  }
458
- if (apps.length === 0) {
988
+ if (!platform) {
459
989
  return {
460
990
  success: false,
461
991
  query,
462
- error: "No connected app. Auto-connect failed — no Metro servers found. Run scan_metro manually.",
992
+ error: "No devices found. Boot a simulator or connect an Android device."
463
993
  };
464
994
  }
465
- const app = apps[0];
466
- const platform = app.platform;
995
+ // Resolve target iOS simulator UDID from the connected app
996
+ // This ensures taps go to the correct device when multiple simulators are booted
997
+ let targetUdid;
998
+ if (platform === "ios") {
999
+ targetUdid =
1000
+ app?.simulatorUdid ??
1001
+ (app?.deviceInfo?.deviceName ? await findSimulatorByName(app.deviceInfo.deviceName) : null) ??
1002
+ undefined;
1003
+ }
1004
+ const deviceName = app?.deviceInfo?.deviceName;
467
1005
  // Determine strategies
468
1006
  const strategies = getAvailableStrategies(query, strategy);
469
1007
  const attempted = [];
470
- // Execute strategies in order
471
- for (const strat of strategies) {
1008
+ // Early UI driver check for iOS — fail fast instead of falling through every strategy
1009
+ const UI_DRIVER_REQUIRED_STRATEGIES = ["accessibility", "ocr", "coordinate"];
1010
+ let uiDriverMissing = false;
1011
+ if (platform === "ios") {
1012
+ uiDriverMissing = !(await isUiDriverAvailable());
1013
+ }
1014
+ // Filter strategies by available capabilities
1015
+ const filteredStrategies = strategies.filter((strat) => {
1016
+ if (strat === "fiber" && !hasMetro) {
1017
+ attempted.push({
1018
+ strategy: "fiber",
1019
+ reason: "Skipped — no Metro connection (required for fiber)"
1020
+ });
1021
+ return false;
1022
+ }
1023
+ if (uiDriverMissing && UI_DRIVER_REQUIRED_STRATEGIES.includes(strat)) {
1024
+ attempted.push({
1025
+ strategy: strat,
1026
+ reason: "Skipped — iOS UI driver is not installed (required for iOS tap/accessibility/OCR)"
1027
+ });
1028
+ return false;
1029
+ }
1030
+ return true;
1031
+ });
1032
+ if (filteredStrategies.length === 0) {
1033
+ if (uiDriverMissing) {
1034
+ notifyDriverMissing("ios");
1035
+ }
1036
+ const errorMessage = uiDriverMissing
1037
+ ? `Cannot tap on iOS Simulator — ${getUiDriverInstallHint()}\n\nThe iOS UI driver is required for tapping, swiping, text input, and accessibility queries on iOS Simulators.\n\nAfter installing, retry the tap.`
1038
+ : "All strategies require Metro connection, which is unavailable.\n\nTo fix:\n1. Make sure your React Native app is running\n2. Run scan_metro to connect\n3. Or use tap(x, y, native=true) for coordinate-based taps";
1039
+ return {
1040
+ success: false,
1041
+ query,
1042
+ attempted,
1043
+ error: errorMessage
1044
+ };
1045
+ }
1046
+ // Determine screenshot and verification behavior
1047
+ const shouldScreenshot = options.screenshot !== false;
1048
+ // Always capture before screenshot when screenshots are enabled and verify isn't explicitly off
1049
+ // Verification decision is deferred until we know which strategy actually succeeded
1050
+ const canVerify = shouldScreenshot && options.verify !== false;
1051
+ // Capture "before" screenshot for verification
1052
+ let beforeBuffer = null;
1053
+ let beforeScaleFactor;
1054
+ if (canVerify) {
1055
+ const before = await captureScreenshot(platform, targetUdid);
1056
+ beforeBuffer = before?.buffer || null;
1057
+ beforeScaleFactor = before?.scaleFactor;
1058
+ }
1059
+ // Execute strategies in order with per-strategy caps and overall budget
1060
+ for (const strat of filteredStrategies) {
1061
+ const remaining = remainingMs();
1062
+ if (remaining < MIN_STRATEGY_BUDGET_MS) {
1063
+ attempted.push({
1064
+ strategy: strat,
1065
+ reason: `Skipped — only ${remaining}ms remaining (budget ${TAP_TIMEOUT_MS}ms)`
1066
+ });
1067
+ continue;
1068
+ }
1069
+ const cap = MAX_STRATEGY_MS[strat] ?? 5000;
1070
+ const budget = Math.min(cap, remaining);
472
1071
  let result;
473
- switch (strat) {
474
- case "fiber":
475
- result = await tryFiberStrategy(query, index, maxTraversalDepth);
476
- break;
477
- case "accessibility":
478
- result = await tryAccessibilityStrategy(query, index, platform);
479
- break;
480
- case "ocr":
481
- result = await tryOcrStrategy(query, platform);
482
- break;
483
- case "coordinate":
484
- result = await tryCoordinateStrategy(query.x, query.y, platform, app.lastScreenshot);
485
- break;
486
- default:
487
- result = { success: false, reason: `Unknown strategy: ${strat}` };
1072
+ try {
1073
+ switch (strat) {
1074
+ case "fiber":
1075
+ result = await withTimeout(tryFiberStrategy(query, index, maxTraversalDepth), budget, `fiber`);
1076
+ break;
1077
+ case "accessibility":
1078
+ result = await withTimeout(tryAccessibilityStrategy(query, index, platform, targetUdid), budget, `accessibility`);
1079
+ break;
1080
+ case "ocr":
1081
+ result = await withTimeout(tryOcrStrategy(query, platform, targetUdid), budget, `ocr`);
1082
+ break;
1083
+ case "coordinate":
1084
+ result = await withTimeout(tryCoordinateStrategy(query.x, query.y, platform, app?.lastScreenshot, targetUdid), budget, `coordinate`);
1085
+ break;
1086
+ default:
1087
+ result = { success: false, reason: `Unknown strategy: ${strat}` };
1088
+ }
1089
+ }
1090
+ catch (err) {
1091
+ attempted.push({
1092
+ strategy: strat,
1093
+ reason: err instanceof Error ? err.message : String(err)
1094
+ });
1095
+ continue;
488
1096
  }
489
1097
  if (result.success) {
1098
+ let screenshot;
1099
+ let verification;
1100
+ if (options.burst && canVerify && beforeBuffer) {
1101
+ ({ screenshot, verification } = await burstCaptureAndVerify(platform, beforeBuffer, targetUdid, beforeScaleFactor));
1102
+ }
1103
+ else {
1104
+ ({ screenshot, verification } = await verifyAndCapture(platform, canVerify, shouldScreenshot, beforeBuffer, targetUdid, beforeScaleFactor));
1105
+ }
1106
+ if (screenshot && app) {
1107
+ app.lastScreenshot = {
1108
+ originalWidth: screenshot.width,
1109
+ originalHeight: screenshot.height,
1110
+ scaleFactor: screenshot.scaleFactor
1111
+ };
1112
+ }
490
1113
  return formatTapSuccess({
491
1114
  method: strat,
492
1115
  query,
@@ -497,25 +1120,108 @@ export async function tap(options) {
497
1120
  component: result.component,
498
1121
  convertedTo: result.convertedTo,
499
1122
  platform,
1123
+ device: deviceName,
1124
+ screenshot,
1125
+ verification
500
1126
  });
501
1127
  }
502
1128
  attempted.push({ strategy: strat, reason: result.reason });
1129
+ // If fiber found an element with measured coordinates, do a native tap directly
1130
+ if (strat === "fiber" && result.convertedTo && result.pressed) {
1131
+ try {
1132
+ const coords = result.convertedTo;
1133
+ if (platform === "ios") {
1134
+ // Fabric returns points — iosTap expects points
1135
+ await iosTap(coords.x, coords.y, { udid: targetUdid });
1136
+ }
1137
+ else {
1138
+ // Fabric returns dp — androidTap expects pixels
1139
+ // Convert dp to pixels using device density
1140
+ const { androidGetDensity } = await import("../core/android.js");
1141
+ const densityResult = await androidGetDensity();
1142
+ const densityScale = (densityResult.density || 420) / 160;
1143
+ await androidTap(Math.round(coords.x * densityScale), Math.round(coords.y * densityScale));
1144
+ }
1145
+ // fiber+native uses native tap — always verify
1146
+ let screenshot;
1147
+ let verification;
1148
+ if (options.burst && canVerify && beforeBuffer) {
1149
+ ({ screenshot, verification } = await burstCaptureAndVerify(platform, beforeBuffer, targetUdid, beforeScaleFactor));
1150
+ }
1151
+ else {
1152
+ ({ screenshot, verification } = await verifyAndCapture(platform, canVerify, shouldScreenshot, beforeBuffer, targetUdid, beforeScaleFactor));
1153
+ }
1154
+ if (screenshot && app) {
1155
+ app.lastScreenshot = {
1156
+ originalWidth: screenshot.width,
1157
+ originalHeight: screenshot.height,
1158
+ scaleFactor: screenshot.scaleFactor
1159
+ };
1160
+ }
1161
+ return formatTapSuccess({
1162
+ method: "fiber+native",
1163
+ query,
1164
+ pressed: result.pressed,
1165
+ text: result.text,
1166
+ path: result.path,
1167
+ component: result.component,
1168
+ convertedTo: coords,
1169
+ platform,
1170
+ device: deviceName,
1171
+ screenshot,
1172
+ verification
1173
+ });
1174
+ }
1175
+ catch {
1176
+ // Native tap at fiber coordinates failed — continue to next strategy
1177
+ }
1178
+ }
503
1179
  // If we got match suggestions from fiber, carry them forward
504
1180
  if (result.matches) {
1181
+ const { screenshot: matchScreenshot } = shouldScreenshot
1182
+ ? await verifyAndCapture(platform, false, true, null, targetUdid)
1183
+ : { screenshot: undefined };
1184
+ if (matchScreenshot && app) {
1185
+ app.lastScreenshot = {
1186
+ originalWidth: matchScreenshot.width,
1187
+ originalHeight: matchScreenshot.height,
1188
+ scaleFactor: matchScreenshot.scaleFactor
1189
+ };
1190
+ }
505
1191
  return formatTapFailure({
506
1192
  query,
507
1193
  attempted,
508
1194
  suggestion: `Found ${result.matches.length} match(es) — specify index to select one`,
1195
+ device: deviceName,
509
1196
  matches: result.matches,
1197
+ screenshot: matchScreenshot
510
1198
  });
511
1199
  }
512
1200
  }
513
- // All strategies failed
1201
+ // All strategies failed — check if the tap budget was the cause.
1202
+ // Only outer withTimeout wrapper messages or Skipped entries count; nested
1203
+ // sub-op errors that happen to contain "timed out" do not imply the tap
1204
+ // itself ran out of time.
1205
+ const hitTimeout = isTapTimeout(attempted);
1206
+ const elapsed = TAP_TIMEOUT_MS - remainingMs();
514
1207
  const suggestion = buildSuggestion(query, strategies, platform);
1208
+ const { screenshot: failScreenshot } = shouldScreenshot
1209
+ ? await verifyAndCapture(platform, false, true, null, targetUdid)
1210
+ : { screenshot: undefined };
1211
+ if (failScreenshot && app) {
1212
+ app.lastScreenshot = {
1213
+ originalWidth: failScreenshot.width,
1214
+ originalHeight: failScreenshot.height,
1215
+ scaleFactor: failScreenshot.scaleFactor
1216
+ };
1217
+ }
515
1218
  return formatTapFailure({
516
1219
  query,
517
1220
  attempted,
1221
+ error: hitTimeout ? `Tap timed out after ${elapsed}ms (budget ${TAP_TIMEOUT_MS}ms)` : undefined,
518
1222
  suggestion,
1223
+ device: deviceName,
1224
+ screenshot: failScreenshot
519
1225
  });
520
1226
  }
521
1227
  function buildSuggestion(query, triedStrategies, platform) {
@@ -526,8 +1232,8 @@ function buildSuggestion(query, triedStrategies, platform) {
526
1232
  if (query.text && query.text.length <= 2) {
527
1233
  suggestions.push("Very short text is unreliable for OCR — use testID or coordinates instead");
528
1234
  }
529
- if (query.text && isNonAscii(query.text)) {
530
- suggestions.push("Non-ASCII text cannot use fiber strategy — use testID or coordinates instead");
1235
+ if (query.text && hasProblematicUnicode(query.text)) {
1236
+ suggestions.push("Emoji text cannot use fiber strategy — use testID or coordinates instead");
531
1237
  }
532
1238
  if (query.component && triedStrategies.includes("fiber")) {
533
1239
  suggestions.push("Component not found or has no onPress handler — use find_components to discover exact component names, or use text/coordinates instead");