react-native-ai-debugger 1.1.6 → 1.4.33
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +1 -1
- package/README.md +386 -45
- package/build/__tests__/helpers/tap-test-helpers.d.ts +36 -0
- package/build/__tests__/helpers/tap-test-helpers.d.ts.map +1 -0
- package/build/__tests__/helpers/tap-test-helpers.js +130 -0
- package/build/__tests__/helpers/tap-test-helpers.js.map +1 -0
- package/build/__tests__/integration/connection-health.test.js +12 -6
- package/build/__tests__/integration/connection-health.test.js.map +1 -1
- package/build/__tests__/integration/execute-in-app.test.js +2 -1
- package/build/__tests__/integration/execute-in-app.test.js.map +1 -1
- package/build/__tests__/integration/tap-strategies.test.d.ts +2 -0
- package/build/__tests__/integration/tap-strategies.test.d.ts.map +1 -0
- package/build/__tests__/integration/tap-strategies.test.js +329 -0
- package/build/__tests__/integration/tap-strategies.test.js.map +1 -0
- package/build/__tests__/integration/tools.test.js +9 -10
- package/build/__tests__/integration/tools.test.js.map +1 -1
- package/build/__tests__/unit/burst.test.d.ts +2 -0
- package/build/__tests__/unit/burst.test.d.ts.map +1 -0
- package/build/__tests__/unit/burst.test.js +50 -0
- package/build/__tests__/unit/burst.test.js.map +1 -0
- package/build/__tests__/unit/connection-health.test.js +26 -16
- package/build/__tests__/unit/connection-health.test.js.map +1 -1
- package/build/__tests__/unit/fiberMatch.test.d.ts +2 -0
- package/build/__tests__/unit/fiberMatch.test.d.ts.map +1 -0
- package/build/__tests__/unit/fiberMatch.test.js +114 -0
- package/build/__tests__/unit/fiberMatch.test.js.map +1 -0
- package/build/__tests__/unit/fingerprint.test.d.ts +2 -0
- package/build/__tests__/unit/fingerprint.test.d.ts.map +1 -0
- package/build/__tests__/unit/fingerprint.test.js +89 -0
- package/build/__tests__/unit/fingerprint.test.js.map +1 -0
- package/build/__tests__/unit/imageBuffer.test.d.ts +2 -0
- package/build/__tests__/unit/imageBuffer.test.d.ts.map +1 -0
- package/build/__tests__/unit/imageBuffer.test.js +128 -0
- package/build/__tests__/unit/imageBuffer.test.js.map +1 -0
- package/build/__tests__/unit/logs.test.js +9 -0
- package/build/__tests__/unit/logs.test.js.map +1 -1
- package/build/__tests__/unit/networkInterceptor.test.d.ts +2 -0
- package/build/__tests__/unit/networkInterceptor.test.d.ts.map +1 -0
- package/build/__tests__/unit/networkInterceptor.test.js +104 -0
- package/build/__tests__/unit/networkInterceptor.test.js.map +1 -0
- package/build/__tests__/unit/screenshot-diff.test.d.ts +2 -0
- package/build/__tests__/unit/screenshot-diff.test.d.ts.map +1 -0
- package/build/__tests__/unit/screenshot-diff.test.js +97 -0
- package/build/__tests__/unit/screenshot-diff.test.js.map +1 -0
- package/build/__tests__/unit/tap.test.js +542 -25
- package/build/__tests__/unit/tap.test.js.map +1 -1
- package/build/core/android.d.ts.map +1 -1
- package/build/core/android.js +104 -113
- package/build/core/android.js.map +1 -1
- package/build/core/appDetection.d.ts +8 -0
- package/build/core/appDetection.d.ts.map +1 -0
- package/build/core/appDetection.js +115 -0
- package/build/core/appDetection.js.map +1 -0
- package/build/core/bundle.d.ts.map +1 -1
- package/build/core/bundle.js +15 -23
- package/build/core/bundle.js.map +1 -1
- package/build/core/config.d.ts +8 -0
- package/build/core/config.d.ts.map +1 -0
- package/build/core/config.js +27 -0
- package/build/core/config.js.map +1 -0
- package/build/core/connection.d.ts +47 -3
- package/build/core/connection.d.ts.map +1 -1
- package/build/core/connection.js +519 -124
- package/build/core/connection.js.map +1 -1
- package/build/core/executor.d.ts +82 -14
- package/build/core/executor.d.ts.map +1 -1
- package/build/core/executor.js +1869 -418
- package/build/core/executor.js.map +1 -1
- package/build/core/feedback.d.ts +18 -0
- package/build/core/feedback.d.ts.map +1 -0
- package/build/core/feedback.js +73 -0
- package/build/core/feedback.js.map +1 -0
- package/build/core/fiberMatch.d.ts +25 -0
- package/build/core/fiberMatch.d.ts.map +1 -0
- package/build/core/fiberMatch.js +35 -0
- package/build/core/fiberMatch.js.map +1 -0
- package/build/core/fingerprint.d.ts +4 -0
- package/build/core/fingerprint.d.ts.map +1 -0
- package/build/core/fingerprint.js +47 -0
- package/build/core/fingerprint.js.map +1 -0
- package/build/core/guides.d.ts.map +1 -1
- package/build/core/guides.js +66 -3
- package/build/core/guides.js.map +1 -1
- package/build/core/imageBuffer.d.ts +37 -0
- package/build/core/imageBuffer.d.ts.map +1 -0
- package/build/core/imageBuffer.js +58 -0
- package/build/core/imageBuffer.js.map +1 -0
- package/build/core/index.d.ts +22 -17
- package/build/core/index.d.ts.map +1 -1
- package/build/core/index.js +23 -19
- package/build/core/index.js.map +1 -1
- package/build/core/ios.d.ts +36 -7
- package/build/core/ios.d.ts.map +1 -1
- package/build/core/ios.js +561 -326
- package/build/core/ios.js.map +1 -1
- package/build/core/license.d.ts +38 -0
- package/build/core/license.d.ts.map +1 -0
- package/build/core/license.js +243 -0
- package/build/core/license.js.map +1 -0
- package/build/core/logbox.d.ts +57 -0
- package/build/core/logbox.d.ts.map +1 -0
- package/build/core/logbox.js +282 -0
- package/build/core/logbox.js.map +1 -0
- package/build/core/logs.d.ts +1 -0
- package/build/core/logs.d.ts.map +1 -1
- package/build/core/logs.js +5 -0
- package/build/core/logs.js.map +1 -1
- package/build/core/metro.d.ts +8 -0
- package/build/core/metro.d.ts.map +1 -1
- package/build/core/metro.js +28 -0
- package/build/core/metro.js.map +1 -1
- package/build/core/networkInterceptor.d.ts +29 -2
- package/build/core/networkInterceptor.d.ts.map +1 -1
- package/build/core/networkInterceptor.js +195 -191
- package/build/core/networkInterceptor.js.map +1 -1
- package/build/core/ocr.d.ts +8 -3
- package/build/core/ocr.d.ts.map +1 -1
- package/build/core/ocr.js +116 -15
- package/build/core/ocr.js.map +1 -1
- package/build/core/posthog.d.ts +5 -0
- package/build/core/posthog.d.ts.map +1 -0
- package/build/core/posthog.js +46 -0
- package/build/core/posthog.js.map +1 -0
- package/build/core/sdkBridge.d.ts +67 -0
- package/build/core/sdkBridge.d.ts.map +1 -0
- package/build/core/sdkBridge.js +148 -0
- package/build/core/sdkBridge.js.map +1 -0
- package/build/core/state.d.ts +25 -5
- package/build/core/state.d.ts.map +1 -1
- package/build/core/state.js +79 -10
- package/build/core/state.js.map +1 -1
- package/build/core/telemetry.d.ts +21 -1
- package/build/core/telemetry.d.ts.map +1 -1
- package/build/core/telemetry.js +166 -10
- package/build/core/telemetry.js.map +1 -1
- package/build/core/types.d.ts +11 -0
- package/build/core/types.d.ts.map +1 -1
- package/build/index.js +1490 -324
- package/build/index.js.map +1 -1
- package/build/pro/screenshot-diff.d.ts +10 -0
- package/build/pro/screenshot-diff.d.ts.map +1 -0
- package/build/pro/screenshot-diff.js +38 -0
- package/build/pro/screenshot-diff.js.map +1 -0
- package/build/pro/tap.d.ts +84 -2
- package/build/pro/tap.d.ts.map +1 -1
- package/build/pro/tap.js +835 -129
- package/build/pro/tap.js.map +1 -1
- package/build/tools/accountTools.d.ts +44 -0
- package/build/tools/accountTools.d.ts.map +1 -0
- package/build/tools/accountTools.js +217 -0
- package/build/tools/accountTools.js.map +1 -0
- package/package.json +8 -7
- package/build/core/httpServer.d.ts +0 -14
- package/build/core/httpServer.d.ts.map +0 -1
- package/build/core/httpServer.js +0 -2459
- package/build/core/httpServer.js.map +0 -1
- package/build/core/httpServerProcess.d.ts +0 -25
- package/build/core/httpServerProcess.d.ts.map +0 -1
- package/build/core/httpServerProcess.js +0 -153
- package/build/core/httpServerProcess.js.map +0 -1
- package/build/core/tap.d.ts +0 -91
- package/build/core/tap.d.ts.map +0 -1
- package/build/core/tap.js +0 -542
- package/build/core/tap.js.map +0 -1
- package/build/httpServerStandalone.d.ts +0 -7
- package/build/httpServerStandalone.d.ts.map +0 -1
- package/build/httpServerStandalone.js +0 -31
- package/build/httpServerStandalone.js.map +0 -1
package/build/pro/tap.js
CHANGED
|
@@ -1,11 +1,67 @@
|
|
|
1
|
-
import { connectedApps } from "../core/state.js";
|
|
2
|
-
import { inferIOSDevicePixelRatio } from "../core/ocr.js";
|
|
1
|
+
import { connectedApps, imageBuffer } from "../core/state.js";
|
|
3
2
|
import { executeInApp } from "../core/executor.js";
|
|
4
3
|
import { pressElement } from "../core/executor.js";
|
|
5
|
-
import { iosTap, iosFindElement, iosScreenshot } from "../core/ios.js";
|
|
6
|
-
import { androidTap, androidFindElement } from "../core/android.js";
|
|
4
|
+
import { iosTap, iosFindElement, iosScreenshot, getActiveOrBootedSimulatorUdid, findSimulatorByName, isUiDriverAvailable, getUiDriverInstallHint } from "../core/ios.js";
|
|
5
|
+
import { androidTap, androidFindElement, getDefaultAndroidDevice, androidScreenshot } from "../core/android.js";
|
|
6
|
+
import { compareScreenshots } from "./screenshot-diff.js";
|
|
7
7
|
import { scanMetroPorts, fetchDevices, selectMainDevice } from "../core/metro.js";
|
|
8
8
|
import { connectToDevice, clearReconnectionSuppression } from "../core/connection.js";
|
|
9
|
+
import { notifyDriverMissing } from "../core/logbox.js";
|
|
10
|
+
export function buildVerificationExplanation(v) {
|
|
11
|
+
const pct = (rate) => (rate * 100).toFixed(1) + "%";
|
|
12
|
+
if (v.meaningful && !v.transientChangeDetected) {
|
|
13
|
+
return `Tap caused a visible UI change (${pct(v.changeRate)} pixel diff). The screen updated as expected.`;
|
|
14
|
+
}
|
|
15
|
+
if (v.meaningful && v.transientChangeDetected) {
|
|
16
|
+
return (`No persistent change, but transient visual feedback detected ` +
|
|
17
|
+
`(frame ${v.peakFrame} peak ${pct(v.peakChangeRate || 0)} diff). ` +
|
|
18
|
+
`Tap triggered a press animation that settled back to original state.`);
|
|
19
|
+
}
|
|
20
|
+
if (v.transientChangeDetected === false) {
|
|
21
|
+
return (`No visual change detected — neither persistent nor transient across burst frames. ` +
|
|
22
|
+
`The element may not respond visually or the tap may have missed.`);
|
|
23
|
+
}
|
|
24
|
+
return (`No visual change detected between before and after screenshots. ` +
|
|
25
|
+
`The element may not respond visually or the tap may have missed.`);
|
|
26
|
+
}
|
|
27
|
+
const BURST_CHANGE_THRESHOLD = 0.005;
|
|
28
|
+
export async function analyzeBurstFrames(frames, options) {
|
|
29
|
+
if (frames.length < 2) {
|
|
30
|
+
return {
|
|
31
|
+
meaningful: false,
|
|
32
|
+
persistentChangeRate: 0,
|
|
33
|
+
transientChangeDetected: false,
|
|
34
|
+
peakChangeRate: 0,
|
|
35
|
+
peakFrame: 0,
|
|
36
|
+
framesWithChange: []
|
|
37
|
+
};
|
|
38
|
+
}
|
|
39
|
+
let peakChangeRate = 0;
|
|
40
|
+
let peakFrame = 0;
|
|
41
|
+
const framesWithChange = [];
|
|
42
|
+
for (let i = 1; i < frames.length; i++) {
|
|
43
|
+
const diff = await compareScreenshots(frames[i - 1], frames[i], options);
|
|
44
|
+
if (diff.changeRate > BURST_CHANGE_THRESHOLD) {
|
|
45
|
+
framesWithChange.push(i);
|
|
46
|
+
}
|
|
47
|
+
if (diff.changeRate > peakChangeRate) {
|
|
48
|
+
peakChangeRate = diff.changeRate;
|
|
49
|
+
peakFrame = i;
|
|
50
|
+
}
|
|
51
|
+
}
|
|
52
|
+
const persistentDiff = await compareScreenshots(frames[0], frames[frames.length - 1], options);
|
|
53
|
+
const persistentChangeRate = persistentDiff.changeRate;
|
|
54
|
+
const transientChangeDetected = !persistentDiff.changed && framesWithChange.length > 0;
|
|
55
|
+
const meaningful = persistentDiff.changed || transientChangeDetected;
|
|
56
|
+
return {
|
|
57
|
+
meaningful,
|
|
58
|
+
persistentChangeRate,
|
|
59
|
+
transientChangeDetected,
|
|
60
|
+
peakChangeRate,
|
|
61
|
+
peakFrame,
|
|
62
|
+
framesWithChange
|
|
63
|
+
};
|
|
64
|
+
}
|
|
9
65
|
// --- Helpers ---
|
|
10
66
|
export function buildQuery(options) {
|
|
11
67
|
const query = {};
|
|
@@ -21,43 +77,89 @@ export function buildQuery(options) {
|
|
|
21
77
|
query.y = options.y;
|
|
22
78
|
return query;
|
|
23
79
|
}
|
|
24
|
-
|
|
25
|
-
|
|
80
|
+
/**
|
|
81
|
+
* Check if text contains characters that break Hermes Runtime.evaluate.
|
|
82
|
+
* Standard accented Latin characters (Polish, Vietnamese, French, German, etc.)
|
|
83
|
+
* and Cyrillic work fine in Hermes. Only emoji and special Unicode ranges cause issues.
|
|
84
|
+
*/
|
|
85
|
+
export function hasProblematicUnicode(text) {
|
|
86
|
+
const emojiPattern = /[\u{1F600}-\u{1F64F}]|[\u{1F300}-\u{1F5FF}]|[\u{1F680}-\u{1F6FF}]|[\u{1F1E0}-\u{1F1FF}]|[\u{2600}-\u{26FF}]|[\u{2700}-\u{27BF}]|[\u{FE00}-\u{FE0F}]|[\u{1F900}-\u{1F9FF}]|[\u{200D}]|[\u{20E3}]|[\u{E0020}-\u{E007F}]/u;
|
|
87
|
+
return emojiPattern.test(text);
|
|
88
|
+
}
|
|
89
|
+
function normalizeForMatch(text) {
|
|
90
|
+
return text.trim().replace(/\s+/g, " ").toLowerCase();
|
|
91
|
+
}
|
|
92
|
+
export function findOcrMatch(ocrResult, query) {
|
|
93
|
+
const needle = normalizeForMatch(query);
|
|
94
|
+
if (!needle)
|
|
95
|
+
return null;
|
|
96
|
+
const words = ocrResult.words ?? [];
|
|
97
|
+
const lines = ocrResult.lines ?? [];
|
|
98
|
+
const exactWord = words.find((w) => normalizeForMatch(w.text) === needle);
|
|
99
|
+
if (exactWord)
|
|
100
|
+
return { text: exactWord.text, tapCenter: exactWord.tapCenter };
|
|
101
|
+
const exactLine = lines.find((l) => normalizeForMatch(l.text) === needle);
|
|
102
|
+
if (exactLine)
|
|
103
|
+
return { text: exactLine.text, tapCenter: exactLine.tapCenter };
|
|
104
|
+
const substringLine = lines.find((l) => normalizeForMatch(l.text).includes(needle));
|
|
105
|
+
if (substringLine)
|
|
106
|
+
return { text: substringLine.text, tapCenter: substringLine.tapCenter };
|
|
107
|
+
const substringWord = words.find((w) => normalizeForMatch(w.text).includes(needle));
|
|
108
|
+
if (substringWord)
|
|
109
|
+
return { text: substringWord.text, tapCenter: substringWord.tapCenter };
|
|
110
|
+
return null;
|
|
26
111
|
}
|
|
27
112
|
export function getAvailableStrategies(query, strategy) {
|
|
28
113
|
if (query.x !== undefined && query.y !== undefined) {
|
|
29
114
|
return ["coordinate"];
|
|
30
115
|
}
|
|
31
116
|
if (strategy !== "auto") {
|
|
117
|
+
// Always fallback to OCR for text queries — explicit strategy may miss visible text
|
|
118
|
+
if (query.text && strategy !== "ocr" && strategy !== "coordinate") {
|
|
119
|
+
return [strategy, "ocr"];
|
|
120
|
+
}
|
|
32
121
|
return [strategy];
|
|
33
122
|
}
|
|
34
123
|
if (query.component && !query.text && !query.testID) {
|
|
35
124
|
return ["fiber"];
|
|
36
125
|
}
|
|
37
126
|
if (query.testID && !query.text) {
|
|
38
|
-
return ["
|
|
127
|
+
return ["accessibility", "fiber"];
|
|
39
128
|
}
|
|
40
129
|
if (query.text) {
|
|
41
130
|
const strategies = [];
|
|
42
|
-
|
|
131
|
+
strategies.push("accessibility");
|
|
132
|
+
if (!hasProblematicUnicode(query.text)) {
|
|
43
133
|
strategies.push("fiber");
|
|
44
134
|
}
|
|
45
|
-
strategies.push("
|
|
135
|
+
strategies.push("ocr");
|
|
46
136
|
return strategies;
|
|
47
137
|
}
|
|
48
138
|
return ["fiber", "accessibility", "ocr"];
|
|
49
139
|
}
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
140
|
+
/**
|
|
141
|
+
* Convert screenshot image coordinates to platform-native tap coordinates.
|
|
142
|
+
*
|
|
143
|
+
* For iOS: screenshot pixels → device pixels (undo downscale) → points (÷ DPR)
|
|
144
|
+
* For Android: screenshot pixels → device pixels (undo downscale)
|
|
145
|
+
*
|
|
146
|
+
* IMPORTANT: Only use this for EXTERNAL coordinates from screenshots.
|
|
147
|
+
* Internal strategies (OCR, accessibility, fiber) produce tap-ready coordinates
|
|
148
|
+
* and call iosTap/androidTap directly — they must NOT go through this function.
|
|
149
|
+
*/
|
|
150
|
+
export function convertScreenshotToTapCoords(pixelX, pixelY, platform, devicePixelRatio, scaleFactor = 1) {
|
|
151
|
+
const deviceX = pixelX * scaleFactor;
|
|
152
|
+
const deviceY = pixelY * scaleFactor;
|
|
53
153
|
if (platform === "android") {
|
|
54
|
-
return { x: Math.round(
|
|
154
|
+
return { x: Math.round(deviceX), y: Math.round(deviceY) };
|
|
55
155
|
}
|
|
56
156
|
return {
|
|
57
|
-
x: Math.round(
|
|
58
|
-
y: Math.round(
|
|
157
|
+
x: Math.round(deviceX / devicePixelRatio),
|
|
158
|
+
y: Math.round(deviceY / devicePixelRatio)
|
|
59
159
|
};
|
|
60
160
|
}
|
|
161
|
+
/** @deprecated Use convertScreenshotToTapCoords instead */
|
|
162
|
+
export const convertPixelsToPoints = convertScreenshotToTapCoords;
|
|
61
163
|
export async function getCurrentScreen() {
|
|
62
164
|
try {
|
|
63
165
|
// Note: Uses var instead of let/const because Hermes Runtime.evaluate
|
|
@@ -75,7 +177,7 @@ export async function getCurrentScreen() {
|
|
|
75
177
|
if (roots.length === 0) return null;
|
|
76
178
|
|
|
77
179
|
function findScreen(fiber, depth) {
|
|
78
|
-
if (!fiber || depth >
|
|
180
|
+
if (!fiber || depth > 5000) return null;
|
|
79
181
|
var name = fiber.type && (fiber.type.displayName || fiber.type.name || (typeof fiber.type === 'string' ? fiber.type : null));
|
|
80
182
|
|
|
81
183
|
if (name === 'RNSScreen') {
|
|
@@ -107,7 +209,7 @@ export async function getCurrentScreen() {
|
|
|
107
209
|
}
|
|
108
210
|
|
|
109
211
|
function findFirstUserComponent(fiber, depth) {
|
|
110
|
-
if (!fiber || depth >
|
|
212
|
+
if (!fiber || depth > 5000) return null;
|
|
111
213
|
var name = fiber.type && (fiber.type.displayName || fiber.type.name);
|
|
112
214
|
if (name && typeof fiber.type !== 'string') return name;
|
|
113
215
|
var child = fiber.child;
|
|
@@ -136,21 +238,33 @@ export async function getCurrentScreen() {
|
|
|
136
238
|
}
|
|
137
239
|
}
|
|
138
240
|
export function formatTapSuccess(data) {
|
|
241
|
+
const { screenshot, verification, ...rest } = data;
|
|
139
242
|
return {
|
|
140
243
|
success: true,
|
|
141
|
-
...
|
|
244
|
+
...rest,
|
|
245
|
+
...(verification && { verification }),
|
|
246
|
+
...(screenshot && { screenshot })
|
|
142
247
|
};
|
|
143
248
|
}
|
|
144
249
|
export function formatTapFailure(data) {
|
|
145
250
|
const errorMsg = data.error || buildErrorMessage(data.query);
|
|
251
|
+
const warning = data.verification && !data.verification.meaningful
|
|
252
|
+
? "Tap executed but no visual change detected. The element may not exist at these coordinates. Examine the screenshot to verify and retry with adjusted coordinates."
|
|
253
|
+
: undefined;
|
|
254
|
+
const lastStrategy = data.attempted.length > 0 ? data.attempted[data.attempted.length - 1].strategy : undefined;
|
|
146
255
|
return {
|
|
147
256
|
success: false,
|
|
257
|
+
method: lastStrategy,
|
|
148
258
|
query: data.query,
|
|
149
259
|
screen: data.screen,
|
|
150
260
|
error: errorMsg,
|
|
151
261
|
attempted: data.attempted,
|
|
152
262
|
suggestion: data.suggestion,
|
|
153
263
|
matches: data.matches,
|
|
264
|
+
...(data.device && { device: data.device }),
|
|
265
|
+
...(data.verification && { verification: data.verification }),
|
|
266
|
+
...(data.screenshot && { screenshot: data.screenshot }),
|
|
267
|
+
...(warning && { warning })
|
|
154
268
|
};
|
|
155
269
|
}
|
|
156
270
|
function buildErrorMessage(query) {
|
|
@@ -165,13 +279,34 @@ function buildErrorMessage(query) {
|
|
|
165
279
|
}
|
|
166
280
|
// --- Strategy Functions ---
|
|
167
281
|
async function tryFiberStrategy(query, index, maxTraversalDepth) {
|
|
282
|
+
// Retry with increasing depth if the initial traversal finds nothing
|
|
283
|
+
const baseDepth = maxTraversalDepth ?? 15;
|
|
284
|
+
const depthAttempts = [baseDepth];
|
|
285
|
+
// Only add deeper retries if user didn't explicitly set a high depth
|
|
286
|
+
if (baseDepth <= 15) {
|
|
287
|
+
depthAttempts.push(30, 45);
|
|
288
|
+
}
|
|
289
|
+
else if (baseDepth <= 30) {
|
|
290
|
+
depthAttempts.push(baseDepth * 2);
|
|
291
|
+
}
|
|
292
|
+
let lastResult = null;
|
|
293
|
+
for (const depth of depthAttempts) {
|
|
294
|
+
const result = await tryFiberAtDepth(query, index, depth);
|
|
295
|
+
if (result.success || result.matches) {
|
|
296
|
+
return result;
|
|
297
|
+
}
|
|
298
|
+
lastResult = result;
|
|
299
|
+
}
|
|
300
|
+
return lastResult;
|
|
301
|
+
}
|
|
302
|
+
async function tryFiberAtDepth(query, index, maxTraversalDepth) {
|
|
168
303
|
try {
|
|
169
304
|
const result = await pressElement({
|
|
170
305
|
text: query.text,
|
|
171
306
|
testID: query.testID,
|
|
172
307
|
component: query.component,
|
|
173
308
|
index,
|
|
174
|
-
maxTraversalDepth
|
|
309
|
+
maxTraversalDepth
|
|
175
310
|
});
|
|
176
311
|
if (!result.success) {
|
|
177
312
|
return { success: false, reason: result.error || "pressElement failed" };
|
|
@@ -183,67 +318,115 @@ async function tryFiberStrategy(query, index, maxTraversalDepth) {
|
|
|
183
318
|
if (parsed.error) {
|
|
184
319
|
const strategyResult = {
|
|
185
320
|
success: false,
|
|
186
|
-
reason: parsed.error
|
|
321
|
+
reason: parsed.error
|
|
187
322
|
};
|
|
188
323
|
if (parsed.matches) {
|
|
189
324
|
strategyResult.matches = parsed.matches;
|
|
190
325
|
}
|
|
191
326
|
return strategyResult;
|
|
192
327
|
}
|
|
193
|
-
//
|
|
194
|
-
//
|
|
328
|
+
// Fiber finds the element by text/testID/component, then measures its
|
|
329
|
+
// host component's screen position for a native tap. This ensures the tap
|
|
330
|
+
// goes through React's event pipeline, executing any onPress wrappers
|
|
331
|
+
// (analytics, debouncing, state tracking) inside the component.
|
|
195
332
|
if (parsed.needsNativeTap) {
|
|
333
|
+
const elementType = parsed.isInput ? "input element" : "pressable element";
|
|
334
|
+
if (parsed.nativeTapTarget && parsed.nativeTapTarget.x && parsed.nativeTapTarget.y) {
|
|
335
|
+
return {
|
|
336
|
+
success: false,
|
|
337
|
+
reason: `Found ${parsed.pressed} (${elementType}) — measured coordinates for native tap`,
|
|
338
|
+
pressed: parsed.pressed,
|
|
339
|
+
text: parsed.text,
|
|
340
|
+
path: parsed.path || null,
|
|
341
|
+
component: parsed.pressed || null,
|
|
342
|
+
convertedTo: {
|
|
343
|
+
x: parsed.nativeTapTarget.x,
|
|
344
|
+
y: parsed.nativeTapTarget.y,
|
|
345
|
+
unit: parsed.nativeTapTarget.unit || "points"
|
|
346
|
+
}
|
|
347
|
+
};
|
|
348
|
+
}
|
|
196
349
|
return {
|
|
197
350
|
success: false,
|
|
198
|
-
reason: `Found ${parsed.pressed} (
|
|
351
|
+
reason: `Found ${parsed.pressed} (${elementType}) but could not measure coordinates — falling through to next strategy`
|
|
199
352
|
};
|
|
200
353
|
}
|
|
354
|
+
// All elements now use needsNativeTap — this shouldn't be reached
|
|
201
355
|
return {
|
|
202
|
-
success:
|
|
203
|
-
reason: "
|
|
204
|
-
pressed: parsed.pressed,
|
|
205
|
-
text: parsed.text,
|
|
206
|
-
path: parsed.path || null,
|
|
207
|
-
component: parsed.pressed || null,
|
|
356
|
+
success: false,
|
|
357
|
+
reason: "Unexpected: element did not request native tap"
|
|
208
358
|
};
|
|
209
359
|
}
|
|
210
360
|
catch (err) {
|
|
211
361
|
return {
|
|
212
362
|
success: false,
|
|
213
|
-
reason: `Fiber strategy error: ${err instanceof Error ? err.message : String(err)}
|
|
363
|
+
reason: `Fiber strategy error: ${err instanceof Error ? err.message : String(err)}`
|
|
214
364
|
};
|
|
215
365
|
}
|
|
216
366
|
}
|
|
217
|
-
async function tryAccessibilityStrategy(query, index, platform) {
|
|
367
|
+
async function tryAccessibilityStrategy(query, index, platform, udid) {
|
|
218
368
|
try {
|
|
219
369
|
const hasTestID = !!query.testID;
|
|
220
370
|
const hasText = !!query.text;
|
|
221
371
|
if (!hasTestID && !hasText) {
|
|
222
|
-
return {
|
|
372
|
+
return {
|
|
373
|
+
success: false,
|
|
374
|
+
reason: "No text or testID for accessibility search"
|
|
375
|
+
};
|
|
223
376
|
}
|
|
224
377
|
if (platform === "ios") {
|
|
225
|
-
// iOS:
|
|
226
|
-
//
|
|
227
|
-
|
|
228
|
-
|
|
229
|
-
|
|
230
|
-
|
|
231
|
-
|
|
378
|
+
// iOS: testID maps to accessibilityIdentifier — search by identifier first,
|
|
379
|
+
// then fall back to labelContains for text-based searches
|
|
380
|
+
let result;
|
|
381
|
+
if (hasTestID && !hasText) {
|
|
382
|
+
// Try exact identifier match first (testID → accessibilityIdentifier)
|
|
383
|
+
result = await iosFindElement({
|
|
384
|
+
identifier: query.testID,
|
|
385
|
+
index
|
|
386
|
+
}, udid);
|
|
387
|
+
// Fall back to identifierContains if exact match fails
|
|
388
|
+
if (!result.success || !result.allMatches || result.allMatches.length === 0) {
|
|
389
|
+
result = await iosFindElement({
|
|
390
|
+
identifierContains: query.testID,
|
|
391
|
+
index
|
|
392
|
+
}, udid);
|
|
393
|
+
}
|
|
394
|
+
// Last resort: try labelContains in case testID is reflected in label
|
|
395
|
+
if (!result.success || !result.allMatches || result.allMatches.length === 0) {
|
|
396
|
+
result = await iosFindElement({
|
|
397
|
+
labelContains: query.testID,
|
|
398
|
+
index
|
|
399
|
+
}, udid);
|
|
400
|
+
}
|
|
401
|
+
}
|
|
402
|
+
else {
|
|
403
|
+
const searchText = query.text;
|
|
404
|
+
result = await iosFindElement({
|
|
405
|
+
labelContains: searchText,
|
|
406
|
+
index
|
|
407
|
+
}, udid);
|
|
408
|
+
}
|
|
232
409
|
if (!result.success || !result.allMatches || result.allMatches.length === 0) {
|
|
233
|
-
return {
|
|
410
|
+
return {
|
|
411
|
+
success: false,
|
|
412
|
+
reason: result.error ?? "No iOS accessibility match"
|
|
413
|
+
};
|
|
234
414
|
}
|
|
235
415
|
const match = result.allMatches[index ?? 0];
|
|
236
416
|
if (!match) {
|
|
237
|
-
return {
|
|
417
|
+
return {
|
|
418
|
+
success: false,
|
|
419
|
+
reason: `Index ${index} out of bounds (${result.allMatches.length} matches)`
|
|
420
|
+
};
|
|
238
421
|
}
|
|
239
|
-
await iosTap(match.center.x, match.center.y);
|
|
422
|
+
await iosTap(match.center.x, match.center.y, { udid });
|
|
240
423
|
return {
|
|
241
424
|
success: true,
|
|
242
425
|
reason: "Tapped via iOS accessibility",
|
|
243
426
|
pressed: match.label || match.type,
|
|
244
427
|
text: match.label || undefined,
|
|
245
428
|
component: match.type || null,
|
|
246
|
-
convertedTo: { x: match.center.x, y: match.center.y, unit: "points" }
|
|
429
|
+
convertedTo: { x: match.center.x, y: match.center.y, unit: "points" }
|
|
247
430
|
};
|
|
248
431
|
}
|
|
249
432
|
else {
|
|
@@ -258,19 +441,24 @@ async function tryAccessibilityStrategy(query, index, platform) {
|
|
|
258
441
|
let result = await androidFindElement(searchOptions);
|
|
259
442
|
// If testID search via resourceId failed, try contentDescContains
|
|
260
443
|
// (older RN versions map testID to content-description)
|
|
261
|
-
if (hasTestID && !hasText &&
|
|
262
|
-
(!result.success || !result.allMatches || result.allMatches.length === 0)) {
|
|
444
|
+
if (hasTestID && !hasText && (!result.success || !result.allMatches || result.allMatches.length === 0)) {
|
|
263
445
|
result = await androidFindElement({
|
|
264
446
|
contentDescContains: query.testID,
|
|
265
|
-
index
|
|
447
|
+
index
|
|
266
448
|
});
|
|
267
449
|
}
|
|
268
450
|
if (!result.success || !result.allMatches || result.allMatches.length === 0) {
|
|
269
|
-
return {
|
|
451
|
+
return {
|
|
452
|
+
success: false,
|
|
453
|
+
reason: result.error ?? "No Android accessibility match"
|
|
454
|
+
};
|
|
270
455
|
}
|
|
271
456
|
const match = result.allMatches[index ?? 0];
|
|
272
457
|
if (!match) {
|
|
273
|
-
return {
|
|
458
|
+
return {
|
|
459
|
+
success: false,
|
|
460
|
+
reason: `Index ${index} out of bounds (${result.allMatches.length} matches)`
|
|
461
|
+
};
|
|
274
462
|
}
|
|
275
463
|
await androidTap(match.center.x, match.center.y);
|
|
276
464
|
return {
|
|
@@ -279,18 +467,18 @@ async function tryAccessibilityStrategy(query, index, platform) {
|
|
|
279
467
|
pressed: match.text || match.className || undefined,
|
|
280
468
|
text: match.text || undefined,
|
|
281
469
|
component: match.className || undefined,
|
|
282
|
-
convertedTo: { x: match.center.x, y: match.center.y, unit: "pixels" }
|
|
470
|
+
convertedTo: { x: match.center.x, y: match.center.y, unit: "pixels" }
|
|
283
471
|
};
|
|
284
472
|
}
|
|
285
473
|
}
|
|
286
474
|
catch (err) {
|
|
287
475
|
return {
|
|
288
476
|
success: false,
|
|
289
|
-
reason: `Accessibility strategy error: ${err instanceof Error ? err.message : String(err)}
|
|
477
|
+
reason: `Accessibility strategy error: ${err instanceof Error ? err.message : String(err)}`
|
|
290
478
|
};
|
|
291
479
|
}
|
|
292
480
|
}
|
|
293
|
-
async function tryOcrStrategy(query, platform) {
|
|
481
|
+
async function tryOcrStrategy(query, platform, udid) {
|
|
294
482
|
try {
|
|
295
483
|
const searchText = query.text;
|
|
296
484
|
if (!searchText) {
|
|
@@ -298,78 +486,89 @@ async function tryOcrStrategy(query, platform) {
|
|
|
298
486
|
}
|
|
299
487
|
let imageBuffer;
|
|
300
488
|
let scaleFactor = 1;
|
|
301
|
-
let originalWidth;
|
|
302
|
-
let originalHeight;
|
|
303
489
|
if (platform === "ios") {
|
|
304
|
-
const screenshot = await iosScreenshot();
|
|
490
|
+
const screenshot = await iosScreenshot(undefined, udid);
|
|
305
491
|
if (!screenshot.success || !screenshot.data) {
|
|
306
|
-
return {
|
|
492
|
+
return {
|
|
493
|
+
success: false,
|
|
494
|
+
reason: "Failed to capture iOS screenshot for OCR"
|
|
495
|
+
};
|
|
307
496
|
}
|
|
308
497
|
imageBuffer = screenshot.data;
|
|
309
498
|
scaleFactor = screenshot.scaleFactor ?? 1;
|
|
310
|
-
originalWidth = screenshot.originalWidth;
|
|
311
|
-
originalHeight = screenshot.originalHeight;
|
|
312
499
|
}
|
|
313
500
|
else {
|
|
314
501
|
const { androidScreenshot } = await import("../core/android.js");
|
|
315
502
|
const screenshot = await androidScreenshot();
|
|
316
503
|
if (!screenshot.success || !screenshot.data) {
|
|
317
|
-
return {
|
|
504
|
+
return {
|
|
505
|
+
success: false,
|
|
506
|
+
reason: "Failed to capture Android screenshot for OCR"
|
|
507
|
+
};
|
|
318
508
|
}
|
|
319
509
|
imageBuffer = screenshot.data;
|
|
320
510
|
scaleFactor = screenshot.scaleFactor ?? 1;
|
|
321
|
-
originalWidth = screenshot.originalWidth;
|
|
322
|
-
originalHeight = screenshot.originalHeight;
|
|
323
511
|
}
|
|
324
|
-
const devicePixelRatio = (platform === "ios" && originalWidth && originalHeight)
|
|
325
|
-
? inferIOSDevicePixelRatio(originalWidth, originalHeight)
|
|
326
|
-
: 3;
|
|
327
512
|
const { recognizeText } = await import("../core/ocr.js");
|
|
328
513
|
const ocrResult = await recognizeText(imageBuffer, {
|
|
329
514
|
scaleFactor,
|
|
330
|
-
platform
|
|
331
|
-
devicePixelRatio,
|
|
515
|
+
platform
|
|
332
516
|
});
|
|
333
|
-
const
|
|
334
|
-
|
|
335
|
-
|
|
336
|
-
|
|
517
|
+
const match = findOcrMatch(ocrResult, searchText);
|
|
518
|
+
if (!match) {
|
|
519
|
+
return {
|
|
520
|
+
success: false,
|
|
521
|
+
reason: `OCR did not find text "${searchText}" on screen`
|
|
522
|
+
};
|
|
337
523
|
}
|
|
338
524
|
if (platform === "ios") {
|
|
339
|
-
|
|
525
|
+
// tapCenter is in image-pixel space (downscaled) — convert to points
|
|
526
|
+
const { getDevicePixelRatio } = await import("../core/ios.js");
|
|
527
|
+
const dpr = await getDevicePixelRatio(udid);
|
|
528
|
+
const tapResult = await iosTap(Math.round((match.tapCenter.x * scaleFactor) / dpr), Math.round((match.tapCenter.y * scaleFactor) / dpr), { udid });
|
|
529
|
+
if (!tapResult.success) {
|
|
530
|
+
return {
|
|
531
|
+
success: false,
|
|
532
|
+
reason: `OCR found "${match.text}" but tap failed: ${tapResult.error}`
|
|
533
|
+
};
|
|
534
|
+
}
|
|
340
535
|
}
|
|
341
536
|
else {
|
|
342
|
-
|
|
537
|
+
// Android: image-pixel → device-pixel (undo downscale), ADB accepts pixels
|
|
538
|
+
await androidTap(Math.round(match.tapCenter.x * scaleFactor), Math.round(match.tapCenter.y * scaleFactor));
|
|
343
539
|
}
|
|
344
540
|
return {
|
|
345
541
|
success: true,
|
|
346
542
|
reason: "Tapped via OCR text recognition",
|
|
347
|
-
text:
|
|
543
|
+
text: match.text,
|
|
348
544
|
convertedTo: {
|
|
349
|
-
x:
|
|
350
|
-
y:
|
|
351
|
-
unit:
|
|
352
|
-
}
|
|
545
|
+
x: match.tapCenter.x,
|
|
546
|
+
y: match.tapCenter.y,
|
|
547
|
+
unit: "pixels"
|
|
548
|
+
}
|
|
353
549
|
};
|
|
354
550
|
}
|
|
355
551
|
catch (err) {
|
|
356
552
|
return {
|
|
357
553
|
success: false,
|
|
358
|
-
reason: `OCR strategy error: ${err instanceof Error ? err.message : String(err)}
|
|
554
|
+
reason: `OCR strategy error: ${err instanceof Error ? err.message : String(err)}`
|
|
359
555
|
};
|
|
360
556
|
}
|
|
361
557
|
}
|
|
362
|
-
async function tryCoordinateStrategy(pixelX, pixelY, platform, lastScreenshot) {
|
|
558
|
+
async function tryCoordinateStrategy(pixelX, pixelY, platform, lastScreenshot, udid) {
|
|
363
559
|
try {
|
|
364
560
|
if (platform === "ios") {
|
|
365
561
|
const scaleFactor = lastScreenshot?.scaleFactor ?? 1;
|
|
366
|
-
const
|
|
367
|
-
const
|
|
368
|
-
const
|
|
369
|
-
|
|
370
|
-
|
|
371
|
-
|
|
372
|
-
|
|
562
|
+
const { getDevicePixelRatio } = await import("../core/ios.js");
|
|
563
|
+
const devicePixelRatio = await getDevicePixelRatio(udid);
|
|
564
|
+
const converted = convertScreenshotToTapCoords(pixelX, pixelY, "ios", devicePixelRatio, scaleFactor);
|
|
565
|
+
const tapResult = await iosTap(converted.x, converted.y, { udid });
|
|
566
|
+
if (!tapResult.success) {
|
|
567
|
+
return {
|
|
568
|
+
success: false,
|
|
569
|
+
reason: `Coordinate tap failed: ${tapResult.error}`
|
|
570
|
+
};
|
|
571
|
+
}
|
|
373
572
|
// Best-effort: identify what was tapped via fiber tree
|
|
374
573
|
let screen = null;
|
|
375
574
|
try {
|
|
@@ -382,12 +581,12 @@ async function tryCoordinateStrategy(pixelX, pixelY, platform, lastScreenshot) {
|
|
|
382
581
|
success: true,
|
|
383
582
|
reason: "Tapped at coordinates (iOS)",
|
|
384
583
|
screen,
|
|
385
|
-
convertedTo: { x: converted.x, y: converted.y, unit: "points" }
|
|
584
|
+
convertedTo: { x: converted.x, y: converted.y, unit: "points" }
|
|
386
585
|
};
|
|
387
586
|
}
|
|
388
587
|
else {
|
|
389
588
|
const scaleFactor = lastScreenshot?.scaleFactor ?? 1;
|
|
390
|
-
const converted =
|
|
589
|
+
const converted = convertScreenshotToTapCoords(pixelX, pixelY, "android", 1, scaleFactor);
|
|
391
590
|
await androidTap(converted.x, converted.y);
|
|
392
591
|
// Best-effort: identify what was tapped via fiber tree
|
|
393
592
|
let screen = null;
|
|
@@ -401,16 +600,204 @@ async function tryCoordinateStrategy(pixelX, pixelY, platform, lastScreenshot) {
|
|
|
401
600
|
success: true,
|
|
402
601
|
reason: "Tapped at coordinates (Android)",
|
|
403
602
|
screen,
|
|
404
|
-
convertedTo: { x: converted.x, y: converted.y, unit: "pixels" }
|
|
603
|
+
convertedTo: { x: converted.x, y: converted.y, unit: "pixels" }
|
|
405
604
|
};
|
|
406
605
|
}
|
|
407
606
|
}
|
|
408
607
|
catch (err) {
|
|
409
608
|
return {
|
|
410
609
|
success: false,
|
|
411
|
-
reason: `Coordinate strategy error: ${err instanceof Error ? err.message : String(err)}
|
|
610
|
+
reason: `Coordinate strategy error: ${err instanceof Error ? err.message : String(err)}`
|
|
611
|
+
};
|
|
612
|
+
}
|
|
613
|
+
}
|
|
614
|
+
const SETTLE_DELAY_MS = 800;
|
|
615
|
+
const TAP_TIMEOUT_MS = 20000;
|
|
616
|
+
const MIN_STRATEGY_BUDGET_MS = 500;
|
|
617
|
+
const MAX_STRATEGY_MS = {
|
|
618
|
+
fiber: 5000,
|
|
619
|
+
accessibility: 3000,
|
|
620
|
+
ocr: 5000,
|
|
621
|
+
coordinate: 3000
|
|
622
|
+
};
|
|
623
|
+
// Matches only the outer withTimeout wrapper message for a tap strategy.
|
|
624
|
+
// Nested sub-operation errors inside a strategy (e.g. "CDP getProperties timed out after 150ms")
|
|
625
|
+
// must NOT be classified as a tap-level timeout.
|
|
626
|
+
const STRATEGY_TIMEOUT_RE = /^(fiber|accessibility|ocr|coordinate) timed out after \d+ms$/;
|
|
627
|
+
export function isTapTimeout(attempted) {
|
|
628
|
+
return attempted.some((a) => STRATEGY_TIMEOUT_RE.test(a.reason) || a.reason.startsWith("Skipped —"));
|
|
629
|
+
}
|
|
630
|
+
function withTimeout(promise, ms, label) {
|
|
631
|
+
return new Promise((resolve, reject) => {
|
|
632
|
+
const timer = setTimeout(() => reject(new Error(`${label} timed out after ${ms}ms`)), ms);
|
|
633
|
+
promise.then((val) => {
|
|
634
|
+
clearTimeout(timer);
|
|
635
|
+
resolve(val);
|
|
636
|
+
}, (err) => {
|
|
637
|
+
clearTimeout(timer);
|
|
638
|
+
reject(err);
|
|
639
|
+
});
|
|
640
|
+
});
|
|
641
|
+
}
|
|
642
|
+
async function captureScreenshot(platform, udid) {
|
|
643
|
+
try {
|
|
644
|
+
const result = platform === "ios" ? await iosScreenshot(undefined, udid) : await androidScreenshot();
|
|
645
|
+
if (!result.success || !result.data)
|
|
646
|
+
return null;
|
|
647
|
+
return {
|
|
648
|
+
buffer: result.data,
|
|
649
|
+
width: result.originalWidth || 0,
|
|
650
|
+
height: result.originalHeight || 0,
|
|
651
|
+
scaleFactor: result.scaleFactor || 1
|
|
412
652
|
};
|
|
413
653
|
}
|
|
654
|
+
catch {
|
|
655
|
+
return null;
|
|
656
|
+
}
|
|
657
|
+
}
|
|
658
|
+
function screenshotToBase64(buffer) {
|
|
659
|
+
return buffer.toString("base64");
|
|
660
|
+
}
|
|
661
|
+
async function verifyAndCapture(platform, shouldVerify, shouldScreenshot, beforeBuffer, udid, beforeScaleFactor) {
|
|
662
|
+
if (!shouldScreenshot)
|
|
663
|
+
return {};
|
|
664
|
+
await new Promise((resolve) => setTimeout(resolve, SETTLE_DELAY_MS));
|
|
665
|
+
const after = await captureScreenshot(platform, udid);
|
|
666
|
+
if (!after)
|
|
667
|
+
return {};
|
|
668
|
+
const screenshot = {
|
|
669
|
+
image: screenshotToBase64(after.buffer),
|
|
670
|
+
width: after.width,
|
|
671
|
+
height: after.height,
|
|
672
|
+
scaleFactor: after.scaleFactor
|
|
673
|
+
};
|
|
674
|
+
let verification;
|
|
675
|
+
if (shouldVerify && beforeBuffer) {
|
|
676
|
+
try {
|
|
677
|
+
const rawStatusBar = platform === "ios" ? 177 : 142; // pixels in original screenshot space
|
|
678
|
+
const scale = beforeScaleFactor || after.scaleFactor || 1;
|
|
679
|
+
const statusBarHeight = Math.round(rawStatusBar / scale);
|
|
680
|
+
const diff = await compareScreenshots(beforeBuffer, after.buffer, {
|
|
681
|
+
statusBarHeight
|
|
682
|
+
});
|
|
683
|
+
verification = {
|
|
684
|
+
meaningful: diff.changed,
|
|
685
|
+
changeRate: diff.changeRate,
|
|
686
|
+
changedPixels: diff.changedPixels,
|
|
687
|
+
totalPixels: diff.totalPixels,
|
|
688
|
+
explanation: buildVerificationExplanation({
|
|
689
|
+
meaningful: diff.changed,
|
|
690
|
+
changeRate: diff.changeRate,
|
|
691
|
+
changedPixels: diff.changedPixels,
|
|
692
|
+
totalPixels: diff.totalPixels
|
|
693
|
+
})
|
|
694
|
+
};
|
|
695
|
+
}
|
|
696
|
+
catch {
|
|
697
|
+
// Diff failed — still return the screenshot
|
|
698
|
+
}
|
|
699
|
+
}
|
|
700
|
+
const verifyGroupId = `verify-${Date.now()}-${Math.random().toString(36).slice(2, 8)}`;
|
|
701
|
+
if (beforeBuffer) {
|
|
702
|
+
imageBuffer.add({
|
|
703
|
+
id: `${verifyGroupId}-before`,
|
|
704
|
+
image: beforeBuffer,
|
|
705
|
+
timestamp: Date.now(),
|
|
706
|
+
source: "tap-verify",
|
|
707
|
+
groupId: verifyGroupId,
|
|
708
|
+
metadata: { phase: "before" }
|
|
709
|
+
});
|
|
710
|
+
}
|
|
711
|
+
imageBuffer.add({
|
|
712
|
+
id: `${verifyGroupId}-after`,
|
|
713
|
+
image: after.buffer,
|
|
714
|
+
timestamp: Date.now(),
|
|
715
|
+
source: "tap-verify",
|
|
716
|
+
groupId: verifyGroupId,
|
|
717
|
+
metadata: { phase: "after", changeRate: verification?.changeRate }
|
|
718
|
+
});
|
|
719
|
+
return { screenshot, verification };
|
|
720
|
+
}
|
|
721
|
+
// --- Burst Capture ---
|
|
722
|
+
const BURST_FRAME_COUNT = 4;
|
|
723
|
+
const BURST_FRAME_INTERVAL_MS = 150;
|
|
724
|
+
async function burstCaptureAndVerify(platform, beforeBuffer, udid, beforeScaleFactor) {
|
|
725
|
+
if (!beforeBuffer)
|
|
726
|
+
return {};
|
|
727
|
+
const frames = [beforeBuffer];
|
|
728
|
+
let capturedScaleFactor = beforeScaleFactor || 1;
|
|
729
|
+
for (let i = 0; i < BURST_FRAME_COUNT; i++) {
|
|
730
|
+
await new Promise((resolve) => setTimeout(resolve, BURST_FRAME_INTERVAL_MS));
|
|
731
|
+
const capture = await captureScreenshot(platform, udid);
|
|
732
|
+
if (capture) {
|
|
733
|
+
frames.push(capture.buffer);
|
|
734
|
+
if (i === 0)
|
|
735
|
+
capturedScaleFactor = capture.scaleFactor || capturedScaleFactor;
|
|
736
|
+
}
|
|
737
|
+
}
|
|
738
|
+
if (frames.length < 2)
|
|
739
|
+
return {};
|
|
740
|
+
const rawStatusBar = platform === "ios" ? 177 : 142; // pixels in original screenshot space
|
|
741
|
+
const statusBarHeight = Math.round(rawStatusBar / capturedScaleFactor);
|
|
742
|
+
const analysis = await analyzeBurstFrames(frames, { statusBarHeight });
|
|
743
|
+
const groupId = `burst-${Date.now()}-${Math.random().toString(36).slice(2, 8)}`;
|
|
744
|
+
for (let i = 0; i < frames.length; i++) {
|
|
745
|
+
imageBuffer.add({
|
|
746
|
+
id: `${groupId}-f${i}`,
|
|
747
|
+
image: frames[i],
|
|
748
|
+
timestamp: Date.now(),
|
|
749
|
+
source: "tap-burst",
|
|
750
|
+
groupId,
|
|
751
|
+
metadata: {
|
|
752
|
+
frameIndex: i,
|
|
753
|
+
isBefore: i === 0,
|
|
754
|
+
changeRate: i === 0 ? 0 : analysis.framesWithChange.includes(i) ? analysis.peakChangeRate : 0
|
|
755
|
+
}
|
|
756
|
+
});
|
|
757
|
+
}
|
|
758
|
+
imageBuffer.addGroup({
|
|
759
|
+
groupId,
|
|
760
|
+
intent: "tap-verification",
|
|
761
|
+
source: "tap-burst",
|
|
762
|
+
timestamp: Date.now(),
|
|
763
|
+
frameCount: frames.length,
|
|
764
|
+
summary: {
|
|
765
|
+
peakChangeRate: analysis.peakChangeRate,
|
|
766
|
+
peakFrame: analysis.peakFrame,
|
|
767
|
+
framesWithChange: analysis.framesWithChange,
|
|
768
|
+
transientChangeDetected: analysis.transientChangeDetected,
|
|
769
|
+
persistentChangeRate: analysis.persistentChangeRate
|
|
770
|
+
}
|
|
771
|
+
});
|
|
772
|
+
// Get dimensions from the last frame using sharp
|
|
773
|
+
const sharp = (await import("sharp")).default;
|
|
774
|
+
const meta = await sharp(frames[frames.length - 1]).metadata();
|
|
775
|
+
const screenshot = {
|
|
776
|
+
image: screenshotToBase64(frames[frames.length - 1]),
|
|
777
|
+
width: meta.width || 0,
|
|
778
|
+
height: meta.height || 0,
|
|
779
|
+
scaleFactor: 1
|
|
780
|
+
};
|
|
781
|
+
const verification = {
|
|
782
|
+
meaningful: analysis.meaningful,
|
|
783
|
+
changeRate: analysis.persistentChangeRate,
|
|
784
|
+
changedPixels: 0,
|
|
785
|
+
totalPixels: 0,
|
|
786
|
+
transientChangeDetected: analysis.transientChangeDetected,
|
|
787
|
+
peakChangeRate: analysis.peakChangeRate,
|
|
788
|
+
peakFrame: analysis.peakFrame,
|
|
789
|
+
burstGroupId: groupId,
|
|
790
|
+
explanation: buildVerificationExplanation({
|
|
791
|
+
meaningful: analysis.meaningful,
|
|
792
|
+
changeRate: analysis.persistentChangeRate,
|
|
793
|
+
changedPixels: 0,
|
|
794
|
+
totalPixels: 0,
|
|
795
|
+
transientChangeDetected: analysis.transientChangeDetected,
|
|
796
|
+
peakChangeRate: analysis.peakChangeRate,
|
|
797
|
+
peakFrame: analysis.peakFrame
|
|
798
|
+
})
|
|
799
|
+
};
|
|
800
|
+
return { screenshot, verification };
|
|
414
801
|
}
|
|
415
802
|
// --- Orchestrator ---
|
|
416
803
|
export async function tap(options) {
|
|
@@ -418,6 +805,8 @@ export async function tap(options) {
|
|
|
418
805
|
const strategy = options.strategy || "auto";
|
|
419
806
|
const index = options.index;
|
|
420
807
|
const maxTraversalDepth = options.maxTraversalDepth;
|
|
808
|
+
const deadline = Date.now() + TAP_TIMEOUT_MS;
|
|
809
|
+
const remainingMs = () => Math.max(0, deadline - Date.now());
|
|
421
810
|
// Validate inputs
|
|
422
811
|
const hasSearchParam = query.text || query.testID || query.component;
|
|
423
812
|
const hasCoordinates = query.x !== undefined || query.y !== undefined;
|
|
@@ -425,68 +814,302 @@ export async function tap(options) {
|
|
|
425
814
|
return {
|
|
426
815
|
success: false,
|
|
427
816
|
query,
|
|
428
|
-
error: "Must provide at least one of: text, testID, component, or x/y coordinates"
|
|
817
|
+
error: "Must provide at least one of: text, testID, component, or x/y coordinates"
|
|
429
818
|
};
|
|
430
819
|
}
|
|
431
820
|
if (hasCoordinates && (query.x === undefined || query.y === undefined)) {
|
|
432
821
|
return {
|
|
433
822
|
success: false,
|
|
434
823
|
query,
|
|
435
|
-
error: "Both x and y coordinates must be provided"
|
|
824
|
+
error: "Both x and y coordinates must be provided"
|
|
436
825
|
};
|
|
437
826
|
}
|
|
438
|
-
//
|
|
439
|
-
|
|
440
|
-
|
|
441
|
-
|
|
442
|
-
|
|
443
|
-
|
|
444
|
-
|
|
445
|
-
|
|
446
|
-
|
|
447
|
-
|
|
448
|
-
|
|
449
|
-
|
|
827
|
+
// Native mode: bypass React Native connection, tap directly via ADB/simctl
|
|
828
|
+
if (options.native && hasCoordinates) {
|
|
829
|
+
let platform = options.platform;
|
|
830
|
+
// Auto-detect platform if not specified
|
|
831
|
+
let nativeUdid;
|
|
832
|
+
if (!platform) {
|
|
833
|
+
const [androidDevice, iosSimulator] = await Promise.all([
|
|
834
|
+
getDefaultAndroidDevice().catch(() => null),
|
|
835
|
+
getActiveOrBootedSimulatorUdid().catch(() => null)
|
|
836
|
+
]);
|
|
837
|
+
if (androidDevice && iosSimulator) {
|
|
838
|
+
return {
|
|
839
|
+
success: false,
|
|
840
|
+
query,
|
|
841
|
+
error: 'Multiple platforms detected (both Android and iOS). Specify platform: "android" or platform: "ios" to target the correct device.'
|
|
842
|
+
};
|
|
843
|
+
}
|
|
844
|
+
if (androidDevice) {
|
|
845
|
+
platform = "android";
|
|
846
|
+
}
|
|
847
|
+
else if (iosSimulator) {
|
|
848
|
+
platform = "ios";
|
|
849
|
+
nativeUdid = iosSimulator;
|
|
850
|
+
}
|
|
851
|
+
else {
|
|
852
|
+
return {
|
|
853
|
+
success: false,
|
|
854
|
+
query,
|
|
855
|
+
error: "No Android device or iOS simulator found. Connect a device or start a simulator."
|
|
856
|
+
};
|
|
857
|
+
}
|
|
858
|
+
}
|
|
859
|
+
else if (platform === "ios") {
|
|
860
|
+
nativeUdid = (await getActiveOrBootedSimulatorUdid()) ?? undefined;
|
|
861
|
+
}
|
|
862
|
+
const nativeShouldScreenshot = options.screenshot !== false;
|
|
863
|
+
const nativeShouldVerify = nativeShouldScreenshot && options.verify !== false;
|
|
864
|
+
let nativeBeforeBuffer = null;
|
|
865
|
+
let nativeScreenshotMeta;
|
|
866
|
+
if (nativeShouldVerify) {
|
|
867
|
+
const before = await captureScreenshot(platform, nativeUdid);
|
|
868
|
+
nativeBeforeBuffer = before?.buffer || null;
|
|
869
|
+
if (before) {
|
|
870
|
+
nativeScreenshotMeta = {
|
|
871
|
+
originalWidth: before.width,
|
|
872
|
+
originalHeight: before.height,
|
|
873
|
+
scaleFactor: before.scaleFactor
|
|
874
|
+
};
|
|
875
|
+
}
|
|
876
|
+
}
|
|
877
|
+
// If no screenshot was taken for verification, take one just for scaleFactor
|
|
878
|
+
if (!nativeScreenshotMeta) {
|
|
879
|
+
const ref = await captureScreenshot(platform, nativeUdid);
|
|
880
|
+
if (ref) {
|
|
881
|
+
nativeScreenshotMeta = {
|
|
882
|
+
originalWidth: ref.width,
|
|
883
|
+
originalHeight: ref.height,
|
|
884
|
+
scaleFactor: ref.scaleFactor
|
|
885
|
+
};
|
|
886
|
+
// Also use it for verification if buffer is needed
|
|
887
|
+
if (!nativeBeforeBuffer) {
|
|
888
|
+
nativeBeforeBuffer = ref.buffer;
|
|
450
889
|
}
|
|
451
890
|
}
|
|
452
|
-
|
|
891
|
+
}
|
|
892
|
+
let result;
|
|
893
|
+
try {
|
|
894
|
+
result = await withTimeout(tryCoordinateStrategy(query.x, query.y, platform, nativeScreenshotMeta, nativeUdid), remainingMs(), "native-coordinate");
|
|
895
|
+
}
|
|
896
|
+
catch (err) {
|
|
897
|
+
return formatTapFailure({
|
|
898
|
+
query,
|
|
899
|
+
attempted: [
|
|
900
|
+
{
|
|
901
|
+
strategy: "native-coordinate",
|
|
902
|
+
reason: err instanceof Error ? err.message : String(err)
|
|
903
|
+
}
|
|
904
|
+
],
|
|
905
|
+
suggestion: `Tap timed out. Take a screenshot (${platform === "ios" ? "ios_screenshot" : "android_screenshot"}) and retry with coordinates.`
|
|
906
|
+
});
|
|
907
|
+
}
|
|
908
|
+
if (result.success) {
|
|
909
|
+
let screenshot;
|
|
910
|
+
let verification;
|
|
911
|
+
if (options.burst && nativeShouldVerify && nativeBeforeBuffer) {
|
|
912
|
+
({ screenshot, verification } = await burstCaptureAndVerify(platform, nativeBeforeBuffer, nativeUdid, nativeScreenshotMeta?.scaleFactor));
|
|
913
|
+
}
|
|
914
|
+
else {
|
|
915
|
+
({ screenshot, verification } = await verifyAndCapture(platform, nativeShouldVerify, nativeShouldScreenshot, nativeBeforeBuffer, nativeUdid, nativeScreenshotMeta?.scaleFactor));
|
|
916
|
+
}
|
|
917
|
+
return formatTapSuccess({
|
|
918
|
+
method: "native-coordinate",
|
|
919
|
+
query,
|
|
920
|
+
pressed: result.pressed,
|
|
921
|
+
convertedTo: result.convertedTo,
|
|
922
|
+
platform,
|
|
923
|
+
screenshot,
|
|
924
|
+
verification
|
|
925
|
+
});
|
|
926
|
+
}
|
|
927
|
+
return formatTapFailure({
|
|
928
|
+
query,
|
|
929
|
+
attempted: [{ strategy: "native-coordinate", reason: result.reason }],
|
|
930
|
+
suggestion: `Take a screenshot (${platform === "ios" ? "ios_screenshot" : "android_screenshot"}) to verify coordinates.`
|
|
931
|
+
});
|
|
932
|
+
}
|
|
933
|
+
// Detect available capabilities
|
|
934
|
+
const allApps = Array.from(connectedApps.values());
|
|
935
|
+
let hasMetro = allApps.length > 0;
|
|
936
|
+
// If platform is specified, prefer an app matching that platform
|
|
937
|
+
let app = options.platform
|
|
938
|
+
? (allApps.find((a) => a.platform === options.platform) ?? allApps[0])
|
|
939
|
+
: allApps[0];
|
|
940
|
+
// Try to auto-connect to Metro (for fiber strategy), but don't fail if it doesn't work
|
|
941
|
+
if (!hasMetro) {
|
|
942
|
+
try {
|
|
943
|
+
await withTimeout((async () => {
|
|
944
|
+
clearReconnectionSuppression();
|
|
945
|
+
const openPorts = await scanMetroPorts();
|
|
946
|
+
for (const port of openPorts) {
|
|
947
|
+
const devices = await fetchDevices(port);
|
|
948
|
+
const mainDevice = selectMainDevice(devices);
|
|
949
|
+
if (mainDevice) {
|
|
950
|
+
await connectToDevice(mainDevice, port);
|
|
951
|
+
break;
|
|
952
|
+
}
|
|
953
|
+
}
|
|
954
|
+
})(), Math.min(remainingMs(), 3000), "auto-connect");
|
|
955
|
+
const apps = Array.from(connectedApps.values());
|
|
956
|
+
hasMetro = apps.length > 0;
|
|
957
|
+
app = apps[0];
|
|
453
958
|
}
|
|
454
959
|
catch {
|
|
455
|
-
// Auto-connect failed
|
|
960
|
+
// Auto-connect failed — Metro-dependent strategies will be skipped
|
|
961
|
+
}
|
|
962
|
+
}
|
|
963
|
+
// Detect platform — from Metro connection or from available devices
|
|
964
|
+
let platform = options.platform;
|
|
965
|
+
if (!platform) {
|
|
966
|
+
if (app) {
|
|
967
|
+
platform = app.platform;
|
|
968
|
+
}
|
|
969
|
+
else {
|
|
970
|
+
// No Metro — detect from available devices
|
|
971
|
+
const [androidDevice, iosSimulator] = await Promise.all([
|
|
972
|
+
getDefaultAndroidDevice().catch(() => null),
|
|
973
|
+
getActiveOrBootedSimulatorUdid().catch(() => null)
|
|
974
|
+
]);
|
|
975
|
+
if (androidDevice && iosSimulator) {
|
|
976
|
+
return {
|
|
977
|
+
success: false,
|
|
978
|
+
query,
|
|
979
|
+
error: 'Both iOS and Android devices detected but no Metro connection. Specify platform="ios" or platform="android".'
|
|
980
|
+
};
|
|
981
|
+
}
|
|
982
|
+
if (androidDevice)
|
|
983
|
+
platform = "android";
|
|
984
|
+
else if (iosSimulator)
|
|
985
|
+
platform = "ios";
|
|
456
986
|
}
|
|
457
987
|
}
|
|
458
|
-
if (
|
|
988
|
+
if (!platform) {
|
|
459
989
|
return {
|
|
460
990
|
success: false,
|
|
461
991
|
query,
|
|
462
|
-
error: "No
|
|
992
|
+
error: "No devices found. Boot a simulator or connect an Android device."
|
|
463
993
|
};
|
|
464
994
|
}
|
|
465
|
-
|
|
466
|
-
|
|
995
|
+
// Resolve target iOS simulator UDID from the connected app
|
|
996
|
+
// This ensures taps go to the correct device when multiple simulators are booted
|
|
997
|
+
let targetUdid;
|
|
998
|
+
if (platform === "ios") {
|
|
999
|
+
targetUdid =
|
|
1000
|
+
app?.simulatorUdid ??
|
|
1001
|
+
(app?.deviceInfo?.deviceName ? await findSimulatorByName(app.deviceInfo.deviceName) : null) ??
|
|
1002
|
+
undefined;
|
|
1003
|
+
}
|
|
1004
|
+
const deviceName = app?.deviceInfo?.deviceName;
|
|
467
1005
|
// Determine strategies
|
|
468
1006
|
const strategies = getAvailableStrategies(query, strategy);
|
|
469
1007
|
const attempted = [];
|
|
470
|
-
//
|
|
471
|
-
|
|
1008
|
+
// Early UI driver check for iOS — fail fast instead of falling through every strategy
|
|
1009
|
+
const UI_DRIVER_REQUIRED_STRATEGIES = ["accessibility", "ocr", "coordinate"];
|
|
1010
|
+
let uiDriverMissing = false;
|
|
1011
|
+
if (platform === "ios") {
|
|
1012
|
+
uiDriverMissing = !(await isUiDriverAvailable());
|
|
1013
|
+
}
|
|
1014
|
+
// Filter strategies by available capabilities
|
|
1015
|
+
const filteredStrategies = strategies.filter((strat) => {
|
|
1016
|
+
if (strat === "fiber" && !hasMetro) {
|
|
1017
|
+
attempted.push({
|
|
1018
|
+
strategy: "fiber",
|
|
1019
|
+
reason: "Skipped — no Metro connection (required for fiber)"
|
|
1020
|
+
});
|
|
1021
|
+
return false;
|
|
1022
|
+
}
|
|
1023
|
+
if (uiDriverMissing && UI_DRIVER_REQUIRED_STRATEGIES.includes(strat)) {
|
|
1024
|
+
attempted.push({
|
|
1025
|
+
strategy: strat,
|
|
1026
|
+
reason: "Skipped — iOS UI driver is not installed (required for iOS tap/accessibility/OCR)"
|
|
1027
|
+
});
|
|
1028
|
+
return false;
|
|
1029
|
+
}
|
|
1030
|
+
return true;
|
|
1031
|
+
});
|
|
1032
|
+
if (filteredStrategies.length === 0) {
|
|
1033
|
+
if (uiDriverMissing) {
|
|
1034
|
+
notifyDriverMissing("ios");
|
|
1035
|
+
}
|
|
1036
|
+
const errorMessage = uiDriverMissing
|
|
1037
|
+
? `Cannot tap on iOS Simulator — ${getUiDriverInstallHint()}\n\nThe iOS UI driver is required for tapping, swiping, text input, and accessibility queries on iOS Simulators.\n\nAfter installing, retry the tap.`
|
|
1038
|
+
: "All strategies require Metro connection, which is unavailable.\n\nTo fix:\n1. Make sure your React Native app is running\n2. Run scan_metro to connect\n3. Or use tap(x, y, native=true) for coordinate-based taps";
|
|
1039
|
+
return {
|
|
1040
|
+
success: false,
|
|
1041
|
+
query,
|
|
1042
|
+
attempted,
|
|
1043
|
+
error: errorMessage
|
|
1044
|
+
};
|
|
1045
|
+
}
|
|
1046
|
+
// Determine screenshot and verification behavior
|
|
1047
|
+
const shouldScreenshot = options.screenshot !== false;
|
|
1048
|
+
// Always capture before screenshot when screenshots are enabled and verify isn't explicitly off
|
|
1049
|
+
// Verification decision is deferred until we know which strategy actually succeeded
|
|
1050
|
+
const canVerify = shouldScreenshot && options.verify !== false;
|
|
1051
|
+
// Capture "before" screenshot for verification
|
|
1052
|
+
let beforeBuffer = null;
|
|
1053
|
+
let beforeScaleFactor;
|
|
1054
|
+
if (canVerify) {
|
|
1055
|
+
const before = await captureScreenshot(platform, targetUdid);
|
|
1056
|
+
beforeBuffer = before?.buffer || null;
|
|
1057
|
+
beforeScaleFactor = before?.scaleFactor;
|
|
1058
|
+
}
|
|
1059
|
+
// Execute strategies in order with per-strategy caps and overall budget
|
|
1060
|
+
for (const strat of filteredStrategies) {
|
|
1061
|
+
const remaining = remainingMs();
|
|
1062
|
+
if (remaining < MIN_STRATEGY_BUDGET_MS) {
|
|
1063
|
+
attempted.push({
|
|
1064
|
+
strategy: strat,
|
|
1065
|
+
reason: `Skipped — only ${remaining}ms remaining (budget ${TAP_TIMEOUT_MS}ms)`
|
|
1066
|
+
});
|
|
1067
|
+
continue;
|
|
1068
|
+
}
|
|
1069
|
+
const cap = MAX_STRATEGY_MS[strat] ?? 5000;
|
|
1070
|
+
const budget = Math.min(cap, remaining);
|
|
472
1071
|
let result;
|
|
473
|
-
|
|
474
|
-
|
|
475
|
-
|
|
476
|
-
|
|
477
|
-
|
|
478
|
-
|
|
479
|
-
|
|
480
|
-
|
|
481
|
-
|
|
482
|
-
|
|
483
|
-
|
|
484
|
-
|
|
485
|
-
|
|
486
|
-
|
|
487
|
-
|
|
1072
|
+
try {
|
|
1073
|
+
switch (strat) {
|
|
1074
|
+
case "fiber":
|
|
1075
|
+
result = await withTimeout(tryFiberStrategy(query, index, maxTraversalDepth), budget, `fiber`);
|
|
1076
|
+
break;
|
|
1077
|
+
case "accessibility":
|
|
1078
|
+
result = await withTimeout(tryAccessibilityStrategy(query, index, platform, targetUdid), budget, `accessibility`);
|
|
1079
|
+
break;
|
|
1080
|
+
case "ocr":
|
|
1081
|
+
result = await withTimeout(tryOcrStrategy(query, platform, targetUdid), budget, `ocr`);
|
|
1082
|
+
break;
|
|
1083
|
+
case "coordinate":
|
|
1084
|
+
result = await withTimeout(tryCoordinateStrategy(query.x, query.y, platform, app?.lastScreenshot, targetUdid), budget, `coordinate`);
|
|
1085
|
+
break;
|
|
1086
|
+
default:
|
|
1087
|
+
result = { success: false, reason: `Unknown strategy: ${strat}` };
|
|
1088
|
+
}
|
|
1089
|
+
}
|
|
1090
|
+
catch (err) {
|
|
1091
|
+
attempted.push({
|
|
1092
|
+
strategy: strat,
|
|
1093
|
+
reason: err instanceof Error ? err.message : String(err)
|
|
1094
|
+
});
|
|
1095
|
+
continue;
|
|
488
1096
|
}
|
|
489
1097
|
if (result.success) {
|
|
1098
|
+
let screenshot;
|
|
1099
|
+
let verification;
|
|
1100
|
+
if (options.burst && canVerify && beforeBuffer) {
|
|
1101
|
+
({ screenshot, verification } = await burstCaptureAndVerify(platform, beforeBuffer, targetUdid, beforeScaleFactor));
|
|
1102
|
+
}
|
|
1103
|
+
else {
|
|
1104
|
+
({ screenshot, verification } = await verifyAndCapture(platform, canVerify, shouldScreenshot, beforeBuffer, targetUdid, beforeScaleFactor));
|
|
1105
|
+
}
|
|
1106
|
+
if (screenshot && app) {
|
|
1107
|
+
app.lastScreenshot = {
|
|
1108
|
+
originalWidth: screenshot.width,
|
|
1109
|
+
originalHeight: screenshot.height,
|
|
1110
|
+
scaleFactor: screenshot.scaleFactor
|
|
1111
|
+
};
|
|
1112
|
+
}
|
|
490
1113
|
return formatTapSuccess({
|
|
491
1114
|
method: strat,
|
|
492
1115
|
query,
|
|
@@ -497,25 +1120,108 @@ export async function tap(options) {
|
|
|
497
1120
|
component: result.component,
|
|
498
1121
|
convertedTo: result.convertedTo,
|
|
499
1122
|
platform,
|
|
1123
|
+
device: deviceName,
|
|
1124
|
+
screenshot,
|
|
1125
|
+
verification
|
|
500
1126
|
});
|
|
501
1127
|
}
|
|
502
1128
|
attempted.push({ strategy: strat, reason: result.reason });
|
|
1129
|
+
// If fiber found an element with measured coordinates, do a native tap directly
|
|
1130
|
+
if (strat === "fiber" && result.convertedTo && result.pressed) {
|
|
1131
|
+
try {
|
|
1132
|
+
const coords = result.convertedTo;
|
|
1133
|
+
if (platform === "ios") {
|
|
1134
|
+
// Fabric returns points — iosTap expects points
|
|
1135
|
+
await iosTap(coords.x, coords.y, { udid: targetUdid });
|
|
1136
|
+
}
|
|
1137
|
+
else {
|
|
1138
|
+
// Fabric returns dp — androidTap expects pixels
|
|
1139
|
+
// Convert dp to pixels using device density
|
|
1140
|
+
const { androidGetDensity } = await import("../core/android.js");
|
|
1141
|
+
const densityResult = await androidGetDensity();
|
|
1142
|
+
const densityScale = (densityResult.density || 420) / 160;
|
|
1143
|
+
await androidTap(Math.round(coords.x * densityScale), Math.round(coords.y * densityScale));
|
|
1144
|
+
}
|
|
1145
|
+
// fiber+native uses native tap — always verify
|
|
1146
|
+
let screenshot;
|
|
1147
|
+
let verification;
|
|
1148
|
+
if (options.burst && canVerify && beforeBuffer) {
|
|
1149
|
+
({ screenshot, verification } = await burstCaptureAndVerify(platform, beforeBuffer, targetUdid, beforeScaleFactor));
|
|
1150
|
+
}
|
|
1151
|
+
else {
|
|
1152
|
+
({ screenshot, verification } = await verifyAndCapture(platform, canVerify, shouldScreenshot, beforeBuffer, targetUdid, beforeScaleFactor));
|
|
1153
|
+
}
|
|
1154
|
+
if (screenshot && app) {
|
|
1155
|
+
app.lastScreenshot = {
|
|
1156
|
+
originalWidth: screenshot.width,
|
|
1157
|
+
originalHeight: screenshot.height,
|
|
1158
|
+
scaleFactor: screenshot.scaleFactor
|
|
1159
|
+
};
|
|
1160
|
+
}
|
|
1161
|
+
return formatTapSuccess({
|
|
1162
|
+
method: "fiber+native",
|
|
1163
|
+
query,
|
|
1164
|
+
pressed: result.pressed,
|
|
1165
|
+
text: result.text,
|
|
1166
|
+
path: result.path,
|
|
1167
|
+
component: result.component,
|
|
1168
|
+
convertedTo: coords,
|
|
1169
|
+
platform,
|
|
1170
|
+
device: deviceName,
|
|
1171
|
+
screenshot,
|
|
1172
|
+
verification
|
|
1173
|
+
});
|
|
1174
|
+
}
|
|
1175
|
+
catch {
|
|
1176
|
+
// Native tap at fiber coordinates failed — continue to next strategy
|
|
1177
|
+
}
|
|
1178
|
+
}
|
|
503
1179
|
// If we got match suggestions from fiber, carry them forward
|
|
504
1180
|
if (result.matches) {
|
|
1181
|
+
const { screenshot: matchScreenshot } = shouldScreenshot
|
|
1182
|
+
? await verifyAndCapture(platform, false, true, null, targetUdid)
|
|
1183
|
+
: { screenshot: undefined };
|
|
1184
|
+
if (matchScreenshot && app) {
|
|
1185
|
+
app.lastScreenshot = {
|
|
1186
|
+
originalWidth: matchScreenshot.width,
|
|
1187
|
+
originalHeight: matchScreenshot.height,
|
|
1188
|
+
scaleFactor: matchScreenshot.scaleFactor
|
|
1189
|
+
};
|
|
1190
|
+
}
|
|
505
1191
|
return formatTapFailure({
|
|
506
1192
|
query,
|
|
507
1193
|
attempted,
|
|
508
1194
|
suggestion: `Found ${result.matches.length} match(es) — specify index to select one`,
|
|
1195
|
+
device: deviceName,
|
|
509
1196
|
matches: result.matches,
|
|
1197
|
+
screenshot: matchScreenshot
|
|
510
1198
|
});
|
|
511
1199
|
}
|
|
512
1200
|
}
|
|
513
|
-
// All strategies failed
|
|
1201
|
+
// All strategies failed — check if the tap budget was the cause.
|
|
1202
|
+
// Only outer withTimeout wrapper messages or Skipped entries count; nested
|
|
1203
|
+
// sub-op errors that happen to contain "timed out" do not imply the tap
|
|
1204
|
+
// itself ran out of time.
|
|
1205
|
+
const hitTimeout = isTapTimeout(attempted);
|
|
1206
|
+
const elapsed = TAP_TIMEOUT_MS - remainingMs();
|
|
514
1207
|
const suggestion = buildSuggestion(query, strategies, platform);
|
|
1208
|
+
const { screenshot: failScreenshot } = shouldScreenshot
|
|
1209
|
+
? await verifyAndCapture(platform, false, true, null, targetUdid)
|
|
1210
|
+
: { screenshot: undefined };
|
|
1211
|
+
if (failScreenshot && app) {
|
|
1212
|
+
app.lastScreenshot = {
|
|
1213
|
+
originalWidth: failScreenshot.width,
|
|
1214
|
+
originalHeight: failScreenshot.height,
|
|
1215
|
+
scaleFactor: failScreenshot.scaleFactor
|
|
1216
|
+
};
|
|
1217
|
+
}
|
|
515
1218
|
return formatTapFailure({
|
|
516
1219
|
query,
|
|
517
1220
|
attempted,
|
|
1221
|
+
error: hitTimeout ? `Tap timed out after ${elapsed}ms (budget ${TAP_TIMEOUT_MS}ms)` : undefined,
|
|
518
1222
|
suggestion,
|
|
1223
|
+
device: deviceName,
|
|
1224
|
+
screenshot: failScreenshot
|
|
519
1225
|
});
|
|
520
1226
|
}
|
|
521
1227
|
function buildSuggestion(query, triedStrategies, platform) {
|
|
@@ -526,8 +1232,8 @@ function buildSuggestion(query, triedStrategies, platform) {
|
|
|
526
1232
|
if (query.text && query.text.length <= 2) {
|
|
527
1233
|
suggestions.push("Very short text is unreliable for OCR — use testID or coordinates instead");
|
|
528
1234
|
}
|
|
529
|
-
if (query.text &&
|
|
530
|
-
suggestions.push("
|
|
1235
|
+
if (query.text && hasProblematicUnicode(query.text)) {
|
|
1236
|
+
suggestions.push("Emoji text cannot use fiber strategy — use testID or coordinates instead");
|
|
531
1237
|
}
|
|
532
1238
|
if (query.component && triedStrategies.includes("fiber")) {
|
|
533
1239
|
suggestions.push("Component not found or has no onPress handler — use find_components to discover exact component names, or use text/coordinates instead");
|