react-native-ai-devtools 1.4.29 → 1.4.31
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +0 -1
- package/build/__tests__/unit/fiberMatch.test.d.ts +2 -0
- package/build/__tests__/unit/fiberMatch.test.d.ts.map +1 -0
- package/build/__tests__/unit/fiberMatch.test.js +114 -0
- package/build/__tests__/unit/fiberMatch.test.js.map +1 -0
- package/build/__tests__/unit/tap.test.js +122 -10
- package/build/__tests__/unit/tap.test.js.map +1 -1
- package/build/core/executor.d.ts.map +1 -1
- package/build/core/executor.js +172 -37
- package/build/core/executor.js.map +1 -1
- package/build/core/fiberMatch.d.ts +25 -0
- package/build/core/fiberMatch.d.ts.map +1 -0
- package/build/core/fiberMatch.js +35 -0
- package/build/core/fiberMatch.js.map +1 -0
- package/build/core/guides.d.ts.map +1 -1
- package/build/core/guides.js +0 -2
- package/build/core/guides.js.map +1 -1
- package/build/core/telemetry.d.ts +1 -1
- package/build/core/telemetry.d.ts.map +1 -1
- package/build/core/telemetry.js +12 -3
- package/build/core/telemetry.js.map +1 -1
- package/build/index.js +2 -2
- package/build/index.js.map +1 -1
- package/build/pro/tap.d.ts +13 -0
- package/build/pro/tap.d.ts.map +1 -1
- package/build/pro/tap.js +177 -99
- package/build/pro/tap.js.map +1 -1
- package/package.json +1 -1
package/build/pro/tap.d.ts
CHANGED
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import type { OCRResult } from "../core/ocr.js";
|
|
1
2
|
export type TapStrategy = "auto" | "fiber" | "accessibility" | "ocr" | "coordinate";
|
|
2
3
|
export interface TapQuery {
|
|
3
4
|
text?: string;
|
|
@@ -101,6 +102,14 @@ export declare function buildQuery(options: TapOptions): TapQuery;
|
|
|
101
102
|
* and Cyrillic work fine in Hermes. Only emoji and special Unicode ranges cause issues.
|
|
102
103
|
*/
|
|
103
104
|
export declare function hasProblematicUnicode(text: string): boolean;
|
|
105
|
+
export interface OcrMatch {
|
|
106
|
+
text: string;
|
|
107
|
+
tapCenter: {
|
|
108
|
+
x: number;
|
|
109
|
+
y: number;
|
|
110
|
+
};
|
|
111
|
+
}
|
|
112
|
+
export declare function findOcrMatch(ocrResult: OCRResult, query: string): OcrMatch | null;
|
|
104
113
|
export declare function getAvailableStrategies(query: TapQuery, strategy: TapStrategy): string[];
|
|
105
114
|
/**
|
|
106
115
|
* Convert screenshot image coordinates to platform-native tap coordinates.
|
|
@@ -156,5 +165,9 @@ export declare function formatTapFailure(data: {
|
|
|
156
165
|
screenshot?: TapScreenshot;
|
|
157
166
|
verification?: TapVerification;
|
|
158
167
|
}): TapResult;
|
|
168
|
+
export declare function isTapTimeout(attempted: readonly {
|
|
169
|
+
reason: string;
|
|
170
|
+
strategy?: string;
|
|
171
|
+
}[]): boolean;
|
|
159
172
|
export declare function tap(options: TapOptions): Promise<TapResult>;
|
|
160
173
|
//# sourceMappingURL=tap.d.ts.map
|
package/build/pro/tap.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"tap.d.ts","sourceRoot":"","sources":["../../src/pro/tap.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"tap.d.ts","sourceRoot":"","sources":["../../src/pro/tap.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,gBAAgB,CAAC;AAoBhD,MAAM,MAAM,WAAW,GAAG,MAAM,GAAG,OAAO,GAAG,eAAe,GAAG,KAAK,GAAG,YAAY,CAAC;AAEpF,MAAM,WAAW,QAAQ;IACrB,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,CAAC,CAAC,EAAE,MAAM,CAAC;IACX,CAAC,CAAC,EAAE,MAAM,CAAC;CACd;AAED,MAAM,WAAW,UAAU;IACvB,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,CAAC,CAAC,EAAE,MAAM,CAAC;IACX,CAAC,CAAC,EAAE,MAAM,CAAC;IACX,QAAQ,CAAC,EAAE,WAAW,CAAC;IACvB,iBAAiB,CAAC,EAAE,MAAM,CAAC;IAC3B,MAAM,CAAC,EAAE,OAAO,CAAC;IACjB,QAAQ,CAAC,EAAE,KAAK,GAAG,SAAS,CAAC;IAC7B,UAAU,CAAC,EAAE,OAAO,CAAC;IACrB,MAAM,CAAC,EAAE,OAAO,CAAC;IACjB,KAAK,CAAC,EAAE,OAAO,CAAC;CACnB;AAED,MAAM,WAAW,UAAU;IACvB,QAAQ,EAAE,MAAM,CAAC;IACjB,MAAM,EAAE,MAAM,CAAC;CAClB;AAED,MAAM,WAAW,aAAa;IAC1B,KAAK,EAAE,MAAM,CAAC;IACd,KAAK,EAAE,MAAM,CAAC;IACd,MAAM,EAAE,MAAM,CAAC;IACf,WAAW,EAAE,MAAM,CAAC;CACvB;AAED,MAAM,WAAW,eAAe;IAC5B,UAAU,EAAE,OAAO,CAAC;IACpB,UAAU,EAAE,MAAM,CAAC;IACnB,aAAa,EAAE,MAAM,CAAC;IACtB,WAAW,EAAE,MAAM,CAAC;IACpB,uBAAuB,CAAC,EAAE,OAAO,CAAC;IAClC,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,WAAW,EAAE,MAAM,CAAC;CACvB;AAED,wBAAgB,4BAA4B,CAAC,CAAC,EAAE;IAC5C,UAAU,EAAE,OAAO,CAAC;IACpB,UAAU,EAAE,MAAM,CAAC;IACnB,aAAa,EAAE,MAAM,CAAC;IACtB,WAAW,EAAE,MAAM,CAAC;IACpB,uBAAuB,CAAC,EAAE,OAAO,CAAC;IAClC,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,SAAS,CAAC,EAAE,MAAM,CAAC;CACtB,GAAG,MAAM,CA0BT;AAED,MAAM,WAAW,aAAa;IAC1B,UAAU,EAAE,OAAO,CAAC;IACpB,oBAAoB,EAAE,MAAM,CAAC;IAC7B,uBAAuB,EAAE,OAAO,CAAC;IACjC,cAAc,EAAE,MAAM,CAAC;IACvB,SAAS,EAAE,MAAM,CAAC;IAClB,gBAAgB,EAAE,MAAM,EAAE,CAAC;CAC9B;AAID,wBAAsB,kBAAkB,CACpC,MAAM,EAAE,MAAM,EAAE,EAChB,OAAO,CAAC,EAAE;IAAE,eAAe,CAAC,EAAE,MAAM,CAAA;CAAE,GACvC,OAAO,CAAC,aAAa,CAAC,CAwCxB;AAED,MAAM,WAAW,SAAS;IACtB,OAAO,EAAE,OAAO,CAAC;IACjB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,KAAK,EAAE,QAAQ,CAAC;IAChB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,MAAM,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IACvB,IAAI,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IACrB,SAAS,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IAC1B,QAAQ,CAAC,EAAE;QAAE,CAAC,EAAE,MAAM,CAAC;QAAC,CAAC,EAAE,MAAM,CAAA;KAAE,CAAC;IACpC,WAAW,CAAC,EAAE;QAAE,CAAC,EAAE,MAAM,CAAC;QAAC,CAAC,EAAE,MAAM,CAAC;QAAC,IAAI,EAAE,MAAM,CAAA;KAAE,CAAC;IACrD,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,SAAS,CAAC,EAAE,UAAU,EAAE,CAAC;IACzB,OAAO,CAAC,EAAE,KAAK,CAAC;QAAE,KAAK,EAAE,MAAM,CAAC;QAAC,SAAS,EAAE,MAAM,CAAC;QAAC,IAAI,EAAE,MAAM,CAAA;KAAE,CAAC,CAAC;IACpE,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,UAAU,CAAC,EAAE,aAAa,CAAC;IAC3B,YAAY,CAAC,EAAE,eAAe,CAAC;IAC/B,OAAO,CAAC,EAAE,MAAM,CAAC;CACpB;AAID,wBAAgB,UAAU,CAAC,OAAO,EAAE,UAAU,GAAG,QAAQ,CAQxD;AAED;;;;GAIG;AACH,wBAAgB,qBAAqB,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAI3D;AAED,MAAM,WAAW,QAAQ;IACrB,IAAI,EAAE,MAAM,CAAC;IACb,SAAS,EAAE;QAAE,CAAC,EAAE,MAAM,CAAC;QAAC,CAAC,EAAE,MAAM,CAAA;KAAE,CAAC;CACvC;AAMD,wBAAgB,YAAY,CAAC,SAAS,EAAE,SAAS,EAAE,KAAK,EAAE,MAAM,GAAG,QAAQ,GAAG,IAAI,CAoBjF;AAED,wBAAgB,sBAAsB,CAAC,KAAK,EAAE,QAAQ,EAAE,QAAQ,EAAE,WAAW,GAAG,MAAM,EAAE,CA2BvF;AAED;;;;;;;;;GASG;AACH,wBAAgB,4BAA4B,CACxC,MAAM,EAAE,MAAM,EACd,MAAM,EAAE,MAAM,EACd,QAAQ,EAAE,KAAK,GAAG,SAAS,EAC3B,gBAAgB,EAAE,MAAM,EACxB,WAAW,GAAE,MAAU,GACxB;IAAE,CAAC,EAAE,MAAM,CAAC;IAAC,CAAC,EAAE,MAAM,CAAA;CAAE,CAY1B;AAED,2DAA2D;AAC3D,eAAO,MAAM,qBAAqB,qCAA+B,CAAC;AAElE,wBAAsB,gBAAgB,IAAI,OAAO,CAAC,MAAM,GAAG,IAAI,CAAC,CA4E/D;AAED,wBAAgB,gBAAgB,CAAC,IAAI,EAAE;IACnC,MAAM,EAAE,MAAM,CAAC;IACf,KAAK,EAAE,QAAQ,CAAC;IAChB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,MAAM,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IACvB,IAAI,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IACrB,SAAS,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IAC1B,QAAQ,CAAC,EAAE;QAAE,CAAC,EAAE,MAAM,CAAC;QAAC,CAAC,EAAE,MAAM,CAAA;KAAE,CAAC;IACpC,WAAW,CAAC,EAAE;QAAE,CAAC,EAAE,MAAM,CAAC;QAAC,CAAC,EAAE,MAAM,CAAC;QAAC,IAAI,EAAE,MAAM,CAAA;KAAE,CAAC;IACrD,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,UAAU,CAAC,EAAE,aAAa,CAAC;IAC3B,YAAY,CAAC,EAAE,eAAe,CAAC;CAClC,GAAG,SAAS,CAQZ;AAED,wBAAgB,gBAAgB,CAAC,IAAI,EAAE;IACnC,KAAK,EAAE,QAAQ,CAAC;IAChB,MAAM,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IACvB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,SAAS,EAAE,UAAU,EAAE,CAAC;IACxB,UAAU,EAAE,MAAM,CAAC;IACnB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,OAAO,CAAC,EAAE,KAAK,CAAC;QAAE,KAAK,EAAE,MAAM,CAAC;QAAC,SAAS,EAAE,MAAM,CAAC;QAAC,IAAI,EAAE,MAAM,CAAA;KAAE,CAAC,CAAC;IACpE,UAAU,CAAC,EAAE,aAAa,CAAC;IAC3B,YAAY,CAAC,EAAE,eAAe,CAAC;CAClC,GAAG,SAAS,CAqBZ;AAybD,wBAAgB,YAAY,CAAC,SAAS,EAAE,SAAS;IAAE,MAAM,EAAE,MAAM,CAAC;IAAC,QAAQ,CAAC,EAAE,MAAM,CAAA;CAAE,EAAE,GAAG,OAAO,CAIjG;AAqND,wBAAsB,GAAG,CAAC,OAAO,EAAE,UAAU,GAAG,OAAO,CAAC,SAAS,CAAC,CA2ejE"}
|
package/build/pro/tap.js
CHANGED
|
@@ -86,6 +86,29 @@ export function hasProblematicUnicode(text) {
|
|
|
86
86
|
const emojiPattern = /[\u{1F600}-\u{1F64F}]|[\u{1F300}-\u{1F5FF}]|[\u{1F680}-\u{1F6FF}]|[\u{1F1E0}-\u{1F1FF}]|[\u{2600}-\u{26FF}]|[\u{2700}-\u{27BF}]|[\u{FE00}-\u{FE0F}]|[\u{1F900}-\u{1F9FF}]|[\u{200D}]|[\u{20E3}]|[\u{E0020}-\u{E007F}]/u;
|
|
87
87
|
return emojiPattern.test(text);
|
|
88
88
|
}
|
|
89
|
+
function normalizeForMatch(text) {
|
|
90
|
+
return text.trim().replace(/\s+/g, " ").toLowerCase();
|
|
91
|
+
}
|
|
92
|
+
export function findOcrMatch(ocrResult, query) {
|
|
93
|
+
const needle = normalizeForMatch(query);
|
|
94
|
+
if (!needle)
|
|
95
|
+
return null;
|
|
96
|
+
const words = ocrResult.words ?? [];
|
|
97
|
+
const lines = ocrResult.lines ?? [];
|
|
98
|
+
const exactWord = words.find((w) => normalizeForMatch(w.text) === needle);
|
|
99
|
+
if (exactWord)
|
|
100
|
+
return { text: exactWord.text, tapCenter: exactWord.tapCenter };
|
|
101
|
+
const exactLine = lines.find((l) => normalizeForMatch(l.text) === needle);
|
|
102
|
+
if (exactLine)
|
|
103
|
+
return { text: exactLine.text, tapCenter: exactLine.tapCenter };
|
|
104
|
+
const substringLine = lines.find((l) => normalizeForMatch(l.text).includes(needle));
|
|
105
|
+
if (substringLine)
|
|
106
|
+
return { text: substringLine.text, tapCenter: substringLine.tapCenter };
|
|
107
|
+
const substringWord = words.find((w) => normalizeForMatch(w.text).includes(needle));
|
|
108
|
+
if (substringWord)
|
|
109
|
+
return { text: substringWord.text, tapCenter: substringWord.tapCenter };
|
|
110
|
+
return null;
|
|
111
|
+
}
|
|
89
112
|
export function getAvailableStrategies(query, strategy) {
|
|
90
113
|
if (query.x !== undefined && query.y !== undefined) {
|
|
91
114
|
return ["coordinate"];
|
|
@@ -105,15 +128,10 @@ export function getAvailableStrategies(query, strategy) {
|
|
|
105
128
|
}
|
|
106
129
|
if (query.text) {
|
|
107
130
|
const strategies = [];
|
|
108
|
-
|
|
109
|
-
// (e.g. TouchableWithoutFeedback), making accessibility match the
|
|
110
|
-
// full-screen parent instead of the actual button. Fiber uses
|
|
111
|
-
// measureInWindow on the exact pressable and taps via coordinates,
|
|
112
|
-
// bypassing the flattened accessibility tree entirely.
|
|
131
|
+
strategies.push("accessibility");
|
|
113
132
|
if (!hasProblematicUnicode(query.text)) {
|
|
114
133
|
strategies.push("fiber");
|
|
115
134
|
}
|
|
116
|
-
strategies.push("accessibility");
|
|
117
135
|
strategies.push("ocr");
|
|
118
136
|
return strategies;
|
|
119
137
|
}
|
|
@@ -260,15 +278,35 @@ function buildErrorMessage(query) {
|
|
|
260
278
|
return `No element found matching ${parts.join(", ")}`;
|
|
261
279
|
}
|
|
262
280
|
// --- Strategy Functions ---
|
|
263
|
-
async function tryFiberStrategy(query, index, maxTraversalDepth
|
|
281
|
+
async function tryFiberStrategy(query, index, maxTraversalDepth) {
|
|
282
|
+
// Retry with increasing depth if the initial traversal finds nothing
|
|
283
|
+
const baseDepth = maxTraversalDepth ?? 15;
|
|
284
|
+
const depthAttempts = [baseDepth];
|
|
285
|
+
// Only add deeper retries if user didn't explicitly set a high depth
|
|
286
|
+
if (baseDepth <= 15) {
|
|
287
|
+
depthAttempts.push(30, 45);
|
|
288
|
+
}
|
|
289
|
+
else if (baseDepth <= 30) {
|
|
290
|
+
depthAttempts.push(baseDepth * 2);
|
|
291
|
+
}
|
|
292
|
+
let lastResult = null;
|
|
293
|
+
for (const depth of depthAttempts) {
|
|
294
|
+
const result = await tryFiberAtDepth(query, index, depth);
|
|
295
|
+
if (result.success || result.matches) {
|
|
296
|
+
return result;
|
|
297
|
+
}
|
|
298
|
+
lastResult = result;
|
|
299
|
+
}
|
|
300
|
+
return lastResult;
|
|
301
|
+
}
|
|
302
|
+
async function tryFiberAtDepth(query, index, maxTraversalDepth) {
|
|
264
303
|
try {
|
|
265
304
|
const result = await pressElement({
|
|
266
305
|
text: query.text,
|
|
267
306
|
testID: query.testID,
|
|
268
307
|
component: query.component,
|
|
269
308
|
index,
|
|
270
|
-
maxTraversalDepth
|
|
271
|
-
device: deviceName
|
|
309
|
+
maxTraversalDepth
|
|
272
310
|
});
|
|
273
311
|
if (!result.success) {
|
|
274
312
|
return { success: false, reason: result.error || "pressElement failed" };
|
|
@@ -287,30 +325,36 @@ async function tryFiberStrategy(query, index, maxTraversalDepth, platform, targe
|
|
|
287
325
|
}
|
|
288
326
|
return strategyResult;
|
|
289
327
|
}
|
|
290
|
-
|
|
328
|
+
// Fiber finds the element by text/testID/component, then measures its
|
|
329
|
+
// host component's screen position for a native tap. This ensures the tap
|
|
330
|
+
// goes through React's event pipeline, executing any onPress wrappers
|
|
331
|
+
// (analytics, debouncing, state tracking) inside the component.
|
|
332
|
+
if (parsed.needsNativeTap) {
|
|
333
|
+
const elementType = parsed.isInput ? "input element" : "pressable element";
|
|
334
|
+
if (parsed.nativeTapTarget && parsed.nativeTapTarget.x && parsed.nativeTapTarget.y) {
|
|
335
|
+
return {
|
|
336
|
+
success: false,
|
|
337
|
+
reason: `Found ${parsed.pressed} (${elementType}) — measured coordinates for native tap`,
|
|
338
|
+
pressed: parsed.pressed,
|
|
339
|
+
text: parsed.text,
|
|
340
|
+
path: parsed.path || null,
|
|
341
|
+
component: parsed.pressed || null,
|
|
342
|
+
convertedTo: {
|
|
343
|
+
x: parsed.nativeTapTarget.x,
|
|
344
|
+
y: parsed.nativeTapTarget.y,
|
|
345
|
+
unit: parsed.nativeTapTarget.unit || "points"
|
|
346
|
+
}
|
|
347
|
+
};
|
|
348
|
+
}
|
|
291
349
|
return {
|
|
292
350
|
success: false,
|
|
293
|
-
reason: parsed.
|
|
351
|
+
reason: `Found ${parsed.pressed} (${elementType}) but could not measure coordinates — falling through to next strategy`
|
|
294
352
|
};
|
|
295
353
|
}
|
|
296
|
-
//
|
|
297
|
-
if (platform === "ios") {
|
|
298
|
-
await iosTap(parsed.x, parsed.y, { udid: targetUdid });
|
|
299
|
-
}
|
|
300
|
-
else {
|
|
301
|
-
const { androidGetDensity } = await import("../core/android.js");
|
|
302
|
-
const densityResult = await androidGetDensity();
|
|
303
|
-
const densityScale = (densityResult.density || 420) / 160;
|
|
304
|
-
await androidTap(Math.round(parsed.x * densityScale), Math.round(parsed.y * densityScale));
|
|
305
|
-
}
|
|
354
|
+
// All elements now use needsNativeTap — this shouldn't be reached
|
|
306
355
|
return {
|
|
307
|
-
success:
|
|
308
|
-
reason: "
|
|
309
|
-
pressed: parsed.pressed,
|
|
310
|
-
text: parsed.text,
|
|
311
|
-
path: parsed.path || null,
|
|
312
|
-
component: parsed.pressed || null,
|
|
313
|
-
convertedTo: { x: parsed.x, y: parsed.y, unit: "points" }
|
|
356
|
+
success: false,
|
|
357
|
+
reason: "Unexpected: element did not request native tap"
|
|
314
358
|
};
|
|
315
359
|
}
|
|
316
360
|
catch (err) {
|
|
@@ -470,10 +514,8 @@ async function tryOcrStrategy(query, platform, udid) {
|
|
|
470
514
|
scaleFactor,
|
|
471
515
|
platform
|
|
472
516
|
});
|
|
473
|
-
const
|
|
474
|
-
|
|
475
|
-
ocrResult.words.find((w) => w.text.toLowerCase().includes(lowerSearch));
|
|
476
|
-
if (!matchingWord) {
|
|
517
|
+
const match = findOcrMatch(ocrResult, searchText);
|
|
518
|
+
if (!match) {
|
|
477
519
|
return {
|
|
478
520
|
success: false,
|
|
479
521
|
reason: `OCR did not find text "${searchText}" on screen`
|
|
@@ -483,25 +525,25 @@ async function tryOcrStrategy(query, platform, udid) {
|
|
|
483
525
|
// tapCenter is in image-pixel space (downscaled) — convert to points
|
|
484
526
|
const { getDevicePixelRatio } = await import("../core/ios.js");
|
|
485
527
|
const dpr = await getDevicePixelRatio(udid);
|
|
486
|
-
const tapResult = await iosTap(Math.round((
|
|
528
|
+
const tapResult = await iosTap(Math.round((match.tapCenter.x * scaleFactor) / dpr), Math.round((match.tapCenter.y * scaleFactor) / dpr), { udid });
|
|
487
529
|
if (!tapResult.success) {
|
|
488
530
|
return {
|
|
489
531
|
success: false,
|
|
490
|
-
reason: `OCR found "${
|
|
532
|
+
reason: `OCR found "${match.text}" but tap failed: ${tapResult.error}`
|
|
491
533
|
};
|
|
492
534
|
}
|
|
493
535
|
}
|
|
494
536
|
else {
|
|
495
537
|
// Android: image-pixel → device-pixel (undo downscale), ADB accepts pixels
|
|
496
|
-
await androidTap(Math.round(
|
|
538
|
+
await androidTap(Math.round(match.tapCenter.x * scaleFactor), Math.round(match.tapCenter.y * scaleFactor));
|
|
497
539
|
}
|
|
498
540
|
return {
|
|
499
541
|
success: true,
|
|
500
542
|
reason: "Tapped via OCR text recognition",
|
|
501
|
-
text:
|
|
543
|
+
text: match.text,
|
|
502
544
|
convertedTo: {
|
|
503
|
-
x:
|
|
504
|
-
y:
|
|
545
|
+
x: match.tapCenter.x,
|
|
546
|
+
y: match.tapCenter.y,
|
|
505
547
|
unit: "pixels"
|
|
506
548
|
}
|
|
507
549
|
};
|
|
@@ -515,20 +557,11 @@ async function tryOcrStrategy(query, platform, udid) {
|
|
|
515
557
|
}
|
|
516
558
|
async function tryCoordinateStrategy(pixelX, pixelY, platform, lastScreenshot, udid) {
|
|
517
559
|
try {
|
|
518
|
-
// If no screenshot metadata is available (e.g. first tap in session,
|
|
519
|
-
// or user called ios_screenshot which doesn't update lastScreenshot),
|
|
520
|
-
// capture a fresh screenshot to determine the correct scaleFactor.
|
|
521
|
-
// Without this, scaleFactor defaults to 1 and conversion is wrong
|
|
522
|
-
// for downscaled screenshots.
|
|
523
|
-
let resolvedScaleFactor = lastScreenshot?.scaleFactor;
|
|
524
|
-
if (resolvedScaleFactor == null) {
|
|
525
|
-
const ref = await captureScreenshot(platform, udid);
|
|
526
|
-
resolvedScaleFactor = ref?.scaleFactor ?? 1;
|
|
527
|
-
}
|
|
528
560
|
if (platform === "ios") {
|
|
561
|
+
const scaleFactor = lastScreenshot?.scaleFactor ?? 1;
|
|
529
562
|
const { getDevicePixelRatio } = await import("../core/ios.js");
|
|
530
563
|
const devicePixelRatio = await getDevicePixelRatio(udid);
|
|
531
|
-
const converted = convertScreenshotToTapCoords(pixelX, pixelY, "ios", devicePixelRatio,
|
|
564
|
+
const converted = convertScreenshotToTapCoords(pixelX, pixelY, "ios", devicePixelRatio, scaleFactor);
|
|
532
565
|
const tapResult = await iosTap(converted.x, converted.y, { udid });
|
|
533
566
|
if (!tapResult.success) {
|
|
534
567
|
return {
|
|
@@ -552,7 +585,8 @@ async function tryCoordinateStrategy(pixelX, pixelY, platform, lastScreenshot, u
|
|
|
552
585
|
};
|
|
553
586
|
}
|
|
554
587
|
else {
|
|
555
|
-
const
|
|
588
|
+
const scaleFactor = lastScreenshot?.scaleFactor ?? 1;
|
|
589
|
+
const converted = convertScreenshotToTapCoords(pixelX, pixelY, "android", 1, scaleFactor);
|
|
556
590
|
await androidTap(converted.x, converted.y);
|
|
557
591
|
// Best-effort: identify what was tapped via fiber tree
|
|
558
592
|
let screen = null;
|
|
@@ -586,6 +620,13 @@ const MAX_STRATEGY_MS = {
|
|
|
586
620
|
ocr: 5000,
|
|
587
621
|
coordinate: 3000
|
|
588
622
|
};
|
|
623
|
+
// Matches only the outer withTimeout wrapper message for a tap strategy.
|
|
624
|
+
// Nested sub-operation errors inside a strategy (e.g. "CDP getProperties timed out after 150ms")
|
|
625
|
+
// must NOT be classified as a tap-level timeout.
|
|
626
|
+
const STRATEGY_TIMEOUT_RE = /^(fiber|accessibility|ocr|coordinate) timed out after \d+ms$/;
|
|
627
|
+
export function isTapTimeout(attempted) {
|
|
628
|
+
return attempted.some((a) => STRATEGY_TIMEOUT_RE.test(a.reason) || a.reason.startsWith("Skipped —"));
|
|
629
|
+
}
|
|
589
630
|
function withTimeout(promise, ms, label) {
|
|
590
631
|
return new Promise((resolve, reject) => {
|
|
591
632
|
const timer = setTimeout(() => reject(new Error(`${label} timed out after ${ms}ms`)), ms);
|
|
@@ -821,39 +862,37 @@ export async function tap(options) {
|
|
|
821
862
|
const nativeShouldScreenshot = options.screenshot !== false;
|
|
822
863
|
const nativeShouldVerify = nativeShouldScreenshot && options.verify !== false;
|
|
823
864
|
let nativeBeforeBuffer = null;
|
|
865
|
+
let nativeScreenshotMeta;
|
|
824
866
|
if (nativeShouldVerify) {
|
|
825
867
|
const before = await captureScreenshot(platform, nativeUdid);
|
|
826
868
|
nativeBeforeBuffer = before?.buffer || null;
|
|
827
|
-
|
|
828
|
-
|
|
829
|
-
|
|
830
|
-
|
|
831
|
-
|
|
832
|
-
|
|
833
|
-
const tapX = Math.round(query.x);
|
|
834
|
-
const tapY = Math.round(query.y);
|
|
835
|
-
if (platform === "ios") {
|
|
836
|
-
const tapResult = await iosTap(tapX, tapY, { udid: nativeUdid });
|
|
837
|
-
if (!tapResult.success) {
|
|
838
|
-
result = { success: false, reason: `Native tap failed: ${tapResult.error}` };
|
|
839
|
-
}
|
|
840
|
-
else {
|
|
841
|
-
result = {
|
|
842
|
-
success: true,
|
|
843
|
-
reason: "Tapped at native coordinates (iOS)",
|
|
844
|
-
convertedTo: { x: tapX, y: tapY, unit: "points" }
|
|
845
|
-
};
|
|
846
|
-
}
|
|
869
|
+
if (before) {
|
|
870
|
+
nativeScreenshotMeta = {
|
|
871
|
+
originalWidth: before.width,
|
|
872
|
+
originalHeight: before.height,
|
|
873
|
+
scaleFactor: before.scaleFactor
|
|
874
|
+
};
|
|
847
875
|
}
|
|
848
|
-
|
|
849
|
-
|
|
850
|
-
|
|
851
|
-
|
|
852
|
-
|
|
853
|
-
|
|
876
|
+
}
|
|
877
|
+
// If no screenshot was taken for verification, take one just for scaleFactor
|
|
878
|
+
if (!nativeScreenshotMeta) {
|
|
879
|
+
const ref = await captureScreenshot(platform, nativeUdid);
|
|
880
|
+
if (ref) {
|
|
881
|
+
nativeScreenshotMeta = {
|
|
882
|
+
originalWidth: ref.width,
|
|
883
|
+
originalHeight: ref.height,
|
|
884
|
+
scaleFactor: ref.scaleFactor
|
|
854
885
|
};
|
|
886
|
+
// Also use it for verification if buffer is needed
|
|
887
|
+
if (!nativeBeforeBuffer) {
|
|
888
|
+
nativeBeforeBuffer = ref.buffer;
|
|
889
|
+
}
|
|
855
890
|
}
|
|
856
891
|
}
|
|
892
|
+
let result;
|
|
893
|
+
try {
|
|
894
|
+
result = await withTimeout(tryCoordinateStrategy(query.x, query.y, platform, nativeScreenshotMeta, nativeUdid), remainingMs(), "native-coordinate");
|
|
895
|
+
}
|
|
857
896
|
catch (err) {
|
|
858
897
|
return formatTapFailure({
|
|
859
898
|
query,
|
|
@@ -870,10 +909,10 @@ export async function tap(options) {
|
|
|
870
909
|
let screenshot;
|
|
871
910
|
let verification;
|
|
872
911
|
if (options.burst && nativeShouldVerify && nativeBeforeBuffer) {
|
|
873
|
-
({ screenshot, verification } = await burstCaptureAndVerify(platform, nativeBeforeBuffer, nativeUdid));
|
|
912
|
+
({ screenshot, verification } = await burstCaptureAndVerify(platform, nativeBeforeBuffer, nativeUdid, nativeScreenshotMeta?.scaleFactor));
|
|
874
913
|
}
|
|
875
914
|
else {
|
|
876
|
-
({ screenshot, verification } = await verifyAndCapture(platform, nativeShouldVerify, nativeShouldScreenshot, nativeBeforeBuffer, nativeUdid));
|
|
915
|
+
({ screenshot, verification } = await verifyAndCapture(platform, nativeShouldVerify, nativeShouldScreenshot, nativeBeforeBuffer, nativeUdid, nativeScreenshotMeta?.scaleFactor));
|
|
877
916
|
}
|
|
878
917
|
return formatTapSuccess({
|
|
879
918
|
method: "native-coordinate",
|
|
@@ -894,22 +933,10 @@ export async function tap(options) {
|
|
|
894
933
|
// Detect available capabilities
|
|
895
934
|
const allApps = Array.from(connectedApps.values());
|
|
896
935
|
let hasMetro = allApps.length > 0;
|
|
897
|
-
// If platform is specified, prefer an app matching that platform
|
|
898
|
-
|
|
899
|
-
|
|
900
|
-
|
|
901
|
-
if (options.platform && allApps.length > 1) {
|
|
902
|
-
app = allApps.find((a) => a.platform === options.platform);
|
|
903
|
-
if (!app && options.platform === "ios") {
|
|
904
|
-
app = allApps.find((a) => !!a.simulatorUdid);
|
|
905
|
-
}
|
|
906
|
-
if (!app) {
|
|
907
|
-
app = allApps[0];
|
|
908
|
-
}
|
|
909
|
-
}
|
|
910
|
-
else {
|
|
911
|
-
app = allApps[0];
|
|
912
|
-
}
|
|
936
|
+
// If platform is specified, prefer an app matching that platform
|
|
937
|
+
let app = options.platform
|
|
938
|
+
? (allApps.find((a) => a.platform === options.platform) ?? allApps[0])
|
|
939
|
+
: allApps[0];
|
|
913
940
|
// Try to auto-connect to Metro (for fiber strategy), but don't fail if it doesn't work
|
|
914
941
|
if (!hasMetro) {
|
|
915
942
|
try {
|
|
@@ -1045,7 +1072,7 @@ export async function tap(options) {
|
|
|
1045
1072
|
try {
|
|
1046
1073
|
switch (strat) {
|
|
1047
1074
|
case "fiber":
|
|
1048
|
-
result = await withTimeout(tryFiberStrategy(query, index, maxTraversalDepth
|
|
1075
|
+
result = await withTimeout(tryFiberStrategy(query, index, maxTraversalDepth), budget, `fiber`);
|
|
1049
1076
|
break;
|
|
1050
1077
|
case "accessibility":
|
|
1051
1078
|
result = await withTimeout(tryAccessibilityStrategy(query, index, platform, targetUdid), budget, `accessibility`);
|
|
@@ -1099,6 +1126,56 @@ export async function tap(options) {
|
|
|
1099
1126
|
});
|
|
1100
1127
|
}
|
|
1101
1128
|
attempted.push({ strategy: strat, reason: result.reason });
|
|
1129
|
+
// If fiber found an element with measured coordinates, do a native tap directly
|
|
1130
|
+
if (strat === "fiber" && result.convertedTo && result.pressed) {
|
|
1131
|
+
try {
|
|
1132
|
+
const coords = result.convertedTo;
|
|
1133
|
+
if (platform === "ios") {
|
|
1134
|
+
// Fabric returns points — iosTap expects points
|
|
1135
|
+
await iosTap(coords.x, coords.y, { udid: targetUdid });
|
|
1136
|
+
}
|
|
1137
|
+
else {
|
|
1138
|
+
// Fabric returns dp — androidTap expects pixels
|
|
1139
|
+
// Convert dp to pixels using device density
|
|
1140
|
+
const { androidGetDensity } = await import("../core/android.js");
|
|
1141
|
+
const densityResult = await androidGetDensity();
|
|
1142
|
+
const densityScale = (densityResult.density || 420) / 160;
|
|
1143
|
+
await androidTap(Math.round(coords.x * densityScale), Math.round(coords.y * densityScale));
|
|
1144
|
+
}
|
|
1145
|
+
// fiber+native uses native tap — always verify
|
|
1146
|
+
let screenshot;
|
|
1147
|
+
let verification;
|
|
1148
|
+
if (options.burst && canVerify && beforeBuffer) {
|
|
1149
|
+
({ screenshot, verification } = await burstCaptureAndVerify(platform, beforeBuffer, targetUdid, beforeScaleFactor));
|
|
1150
|
+
}
|
|
1151
|
+
else {
|
|
1152
|
+
({ screenshot, verification } = await verifyAndCapture(platform, canVerify, shouldScreenshot, beforeBuffer, targetUdid, beforeScaleFactor));
|
|
1153
|
+
}
|
|
1154
|
+
if (screenshot && app) {
|
|
1155
|
+
app.lastScreenshot = {
|
|
1156
|
+
originalWidth: screenshot.width,
|
|
1157
|
+
originalHeight: screenshot.height,
|
|
1158
|
+
scaleFactor: screenshot.scaleFactor
|
|
1159
|
+
};
|
|
1160
|
+
}
|
|
1161
|
+
return formatTapSuccess({
|
|
1162
|
+
method: "fiber+native",
|
|
1163
|
+
query,
|
|
1164
|
+
pressed: result.pressed,
|
|
1165
|
+
text: result.text,
|
|
1166
|
+
path: result.path,
|
|
1167
|
+
component: result.component,
|
|
1168
|
+
convertedTo: coords,
|
|
1169
|
+
platform,
|
|
1170
|
+
device: deviceName,
|
|
1171
|
+
screenshot,
|
|
1172
|
+
verification
|
|
1173
|
+
});
|
|
1174
|
+
}
|
|
1175
|
+
catch {
|
|
1176
|
+
// Native tap at fiber coordinates failed — continue to next strategy
|
|
1177
|
+
}
|
|
1178
|
+
}
|
|
1102
1179
|
// If we got match suggestions from fiber, carry them forward
|
|
1103
1180
|
if (result.matches) {
|
|
1104
1181
|
const { screenshot: matchScreenshot } = shouldScreenshot
|
|
@@ -1121,10 +1198,11 @@ export async function tap(options) {
|
|
|
1121
1198
|
});
|
|
1122
1199
|
}
|
|
1123
1200
|
}
|
|
1124
|
-
// All strategies failed — check if
|
|
1125
|
-
|
|
1126
|
-
|
|
1127
|
-
|
|
1201
|
+
// All strategies failed — check if the tap budget was the cause.
|
|
1202
|
+
// Only outer withTimeout wrapper messages or Skipped entries count; nested
|
|
1203
|
+
// sub-op errors that happen to contain "timed out" do not imply the tap
|
|
1204
|
+
// itself ran out of time.
|
|
1205
|
+
const hitTimeout = isTapTimeout(attempted);
|
|
1128
1206
|
const elapsed = TAP_TIMEOUT_MS - remainingMs();
|
|
1129
1207
|
const suggestion = buildSuggestion(query, strategies, platform);
|
|
1130
1208
|
const { screenshot: failScreenshot } = shouldScreenshot
|