react-native-ai-devtools 1.4.17 → 1.4.18
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +36 -13
- package/build/core/index.d.ts +18 -18
- package/build/core/index.d.ts.map +1 -1
- package/build/core/index.js +16 -16
- package/build/core/index.js.map +1 -1
- package/build/core/ios.d.ts +20 -7
- package/build/core/ios.d.ts.map +1 -1
- package/build/core/ios.js +453 -339
- package/build/core/ios.js.map +1 -1
- package/build/core/telemetry.d.ts +1 -1
- package/build/core/telemetry.d.ts.map +1 -1
- package/build/core/telemetry.js +5 -1
- package/build/core/telemetry.js.map +1 -1
- package/build/index.js +5 -1
- package/build/index.js.map +1 -1
- package/build/pro/tap.d.ts.map +1 -1
- package/build/pro/tap.js +157 -99
- package/build/pro/tap.js.map +1 -1
- package/package.json +1 -1
package/build/pro/tap.js
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import { connectedApps, imageBuffer } from "../core/state.js";
|
|
2
2
|
import { executeInApp } from "../core/executor.js";
|
|
3
3
|
import { pressElement } from "../core/executor.js";
|
|
4
|
-
import { iosTap, iosFindElement, iosScreenshot, getActiveOrBootedSimulatorUdid, findSimulatorByName,
|
|
4
|
+
import { iosTap, iosFindElement, iosScreenshot, getActiveOrBootedSimulatorUdid, findSimulatorByName, isUiDriverAvailable, getUiDriverInstallHint } from "../core/ios.js";
|
|
5
5
|
import { androidTap, androidFindElement, getDefaultAndroidDevice, androidScreenshot } from "../core/android.js";
|
|
6
6
|
import { compareScreenshots } from "./screenshot-diff.js";
|
|
7
7
|
import { scanMetroPorts, fetchDevices, selectMainDevice } from "../core/metro.js";
|
|
@@ -32,7 +32,7 @@ export async function analyzeBurstFrames(frames, options) {
|
|
|
32
32
|
transientChangeDetected: false,
|
|
33
33
|
peakChangeRate: 0,
|
|
34
34
|
peakFrame: 0,
|
|
35
|
-
framesWithChange: []
|
|
35
|
+
framesWithChange: []
|
|
36
36
|
};
|
|
37
37
|
}
|
|
38
38
|
let peakChangeRate = 0;
|
|
@@ -58,7 +58,7 @@ export async function analyzeBurstFrames(frames, options) {
|
|
|
58
58
|
transientChangeDetected,
|
|
59
59
|
peakChangeRate,
|
|
60
60
|
peakFrame,
|
|
61
|
-
framesWithChange
|
|
61
|
+
framesWithChange
|
|
62
62
|
};
|
|
63
63
|
}
|
|
64
64
|
// --- Helpers ---
|
|
@@ -131,7 +131,7 @@ export function convertScreenshotToTapCoords(pixelX, pixelY, platform, devicePix
|
|
|
131
131
|
}
|
|
132
132
|
return {
|
|
133
133
|
x: Math.round(deviceX / devicePixelRatio),
|
|
134
|
-
y: Math.round(deviceY / devicePixelRatio)
|
|
134
|
+
y: Math.round(deviceY / devicePixelRatio)
|
|
135
135
|
};
|
|
136
136
|
}
|
|
137
137
|
/** @deprecated Use convertScreenshotToTapCoords instead */
|
|
@@ -219,7 +219,7 @@ export function formatTapSuccess(data) {
|
|
|
219
219
|
success: true,
|
|
220
220
|
...rest,
|
|
221
221
|
...(verification && { verification }),
|
|
222
|
-
...(screenshot && { screenshot })
|
|
222
|
+
...(screenshot && { screenshot })
|
|
223
223
|
};
|
|
224
224
|
}
|
|
225
225
|
export function formatTapFailure(data) {
|
|
@@ -240,7 +240,7 @@ export function formatTapFailure(data) {
|
|
|
240
240
|
...(data.device && { device: data.device }),
|
|
241
241
|
...(data.verification && { verification: data.verification }),
|
|
242
242
|
...(data.screenshot && { screenshot: data.screenshot }),
|
|
243
|
-
...(warning && { warning })
|
|
243
|
+
...(warning && { warning })
|
|
244
244
|
};
|
|
245
245
|
}
|
|
246
246
|
function buildErrorMessage(query) {
|
|
@@ -282,7 +282,7 @@ async function tryFiberAtDepth(query, index, maxTraversalDepth) {
|
|
|
282
282
|
testID: query.testID,
|
|
283
283
|
component: query.component,
|
|
284
284
|
index,
|
|
285
|
-
maxTraversalDepth
|
|
285
|
+
maxTraversalDepth
|
|
286
286
|
});
|
|
287
287
|
if (!result.success) {
|
|
288
288
|
return { success: false, reason: result.error || "pressElement failed" };
|
|
@@ -294,7 +294,7 @@ async function tryFiberAtDepth(query, index, maxTraversalDepth) {
|
|
|
294
294
|
if (parsed.error) {
|
|
295
295
|
const strategyResult = {
|
|
296
296
|
success: false,
|
|
297
|
-
reason: parsed.error
|
|
297
|
+
reason: parsed.error
|
|
298
298
|
};
|
|
299
299
|
if (parsed.matches) {
|
|
300
300
|
strategyResult.matches = parsed.matches;
|
|
@@ -318,22 +318,25 @@ async function tryFiberAtDepth(query, index, maxTraversalDepth) {
|
|
|
318
318
|
convertedTo: {
|
|
319
319
|
x: parsed.nativeTapTarget.x,
|
|
320
320
|
y: parsed.nativeTapTarget.y,
|
|
321
|
-
unit: parsed.nativeTapTarget.unit || "points"
|
|
322
|
-
}
|
|
321
|
+
unit: parsed.nativeTapTarget.unit || "points"
|
|
322
|
+
}
|
|
323
323
|
};
|
|
324
324
|
}
|
|
325
325
|
return {
|
|
326
326
|
success: false,
|
|
327
|
-
reason: `Found ${parsed.pressed} (${elementType}) but could not measure coordinates — falling through to next strategy
|
|
327
|
+
reason: `Found ${parsed.pressed} (${elementType}) but could not measure coordinates — falling through to next strategy`
|
|
328
328
|
};
|
|
329
329
|
}
|
|
330
330
|
// All elements now use needsNativeTap — this shouldn't be reached
|
|
331
|
-
return {
|
|
331
|
+
return {
|
|
332
|
+
success: false,
|
|
333
|
+
reason: "Unexpected: element did not request native tap"
|
|
334
|
+
};
|
|
332
335
|
}
|
|
333
336
|
catch (err) {
|
|
334
337
|
return {
|
|
335
338
|
success: false,
|
|
336
|
-
reason: `Fiber strategy error: ${err instanceof Error ? err.message : String(err)}
|
|
339
|
+
reason: `Fiber strategy error: ${err instanceof Error ? err.message : String(err)}`
|
|
337
340
|
};
|
|
338
341
|
}
|
|
339
342
|
}
|
|
@@ -342,7 +345,10 @@ async function tryAccessibilityStrategy(query, index, platform, udid) {
|
|
|
342
345
|
const hasTestID = !!query.testID;
|
|
343
346
|
const hasText = !!query.text;
|
|
344
347
|
if (!hasTestID && !hasText) {
|
|
345
|
-
return {
|
|
348
|
+
return {
|
|
349
|
+
success: false,
|
|
350
|
+
reason: "No text or testID for accessibility search"
|
|
351
|
+
};
|
|
346
352
|
}
|
|
347
353
|
if (platform === "ios") {
|
|
348
354
|
// iOS: testID maps to accessibilityIdentifier — search by identifier first,
|
|
@@ -352,20 +358,20 @@ async function tryAccessibilityStrategy(query, index, platform, udid) {
|
|
|
352
358
|
// Try exact identifier match first (testID → accessibilityIdentifier)
|
|
353
359
|
result = await iosFindElement({
|
|
354
360
|
identifier: query.testID,
|
|
355
|
-
index
|
|
361
|
+
index
|
|
356
362
|
}, udid);
|
|
357
363
|
// Fall back to identifierContains if exact match fails
|
|
358
364
|
if (!result.success || !result.allMatches || result.allMatches.length === 0) {
|
|
359
365
|
result = await iosFindElement({
|
|
360
366
|
identifierContains: query.testID,
|
|
361
|
-
index
|
|
367
|
+
index
|
|
362
368
|
}, udid);
|
|
363
369
|
}
|
|
364
370
|
// Last resort: try labelContains in case testID is reflected in label
|
|
365
371
|
if (!result.success || !result.allMatches || result.allMatches.length === 0) {
|
|
366
372
|
result = await iosFindElement({
|
|
367
373
|
labelContains: query.testID,
|
|
368
|
-
index
|
|
374
|
+
index
|
|
369
375
|
}, udid);
|
|
370
376
|
}
|
|
371
377
|
}
|
|
@@ -373,15 +379,21 @@ async function tryAccessibilityStrategy(query, index, platform, udid) {
|
|
|
373
379
|
const searchText = query.text;
|
|
374
380
|
result = await iosFindElement({
|
|
375
381
|
labelContains: searchText,
|
|
376
|
-
index
|
|
382
|
+
index
|
|
377
383
|
}, udid);
|
|
378
384
|
}
|
|
379
385
|
if (!result.success || !result.allMatches || result.allMatches.length === 0) {
|
|
380
|
-
return {
|
|
386
|
+
return {
|
|
387
|
+
success: false,
|
|
388
|
+
reason: result.error ?? "No iOS accessibility match"
|
|
389
|
+
};
|
|
381
390
|
}
|
|
382
391
|
const match = result.allMatches[index ?? 0];
|
|
383
392
|
if (!match) {
|
|
384
|
-
return {
|
|
393
|
+
return {
|
|
394
|
+
success: false,
|
|
395
|
+
reason: `Index ${index} out of bounds (${result.allMatches.length} matches)`
|
|
396
|
+
};
|
|
385
397
|
}
|
|
386
398
|
await iosTap(match.center.x, match.center.y, { udid });
|
|
387
399
|
return {
|
|
@@ -390,7 +402,7 @@ async function tryAccessibilityStrategy(query, index, platform, udid) {
|
|
|
390
402
|
pressed: match.label || match.type,
|
|
391
403
|
text: match.label || undefined,
|
|
392
404
|
component: match.type || null,
|
|
393
|
-
convertedTo: { x: match.center.x, y: match.center.y, unit: "points" }
|
|
405
|
+
convertedTo: { x: match.center.x, y: match.center.y, unit: "points" }
|
|
394
406
|
};
|
|
395
407
|
}
|
|
396
408
|
else {
|
|
@@ -405,19 +417,24 @@ async function tryAccessibilityStrategy(query, index, platform, udid) {
|
|
|
405
417
|
let result = await androidFindElement(searchOptions);
|
|
406
418
|
// If testID search via resourceId failed, try contentDescContains
|
|
407
419
|
// (older RN versions map testID to content-description)
|
|
408
|
-
if (hasTestID && !hasText &&
|
|
409
|
-
(!result.success || !result.allMatches || result.allMatches.length === 0)) {
|
|
420
|
+
if (hasTestID && !hasText && (!result.success || !result.allMatches || result.allMatches.length === 0)) {
|
|
410
421
|
result = await androidFindElement({
|
|
411
422
|
contentDescContains: query.testID,
|
|
412
|
-
index
|
|
423
|
+
index
|
|
413
424
|
});
|
|
414
425
|
}
|
|
415
426
|
if (!result.success || !result.allMatches || result.allMatches.length === 0) {
|
|
416
|
-
return {
|
|
427
|
+
return {
|
|
428
|
+
success: false,
|
|
429
|
+
reason: result.error ?? "No Android accessibility match"
|
|
430
|
+
};
|
|
417
431
|
}
|
|
418
432
|
const match = result.allMatches[index ?? 0];
|
|
419
433
|
if (!match) {
|
|
420
|
-
return {
|
|
434
|
+
return {
|
|
435
|
+
success: false,
|
|
436
|
+
reason: `Index ${index} out of bounds (${result.allMatches.length} matches)`
|
|
437
|
+
};
|
|
421
438
|
}
|
|
422
439
|
await androidTap(match.center.x, match.center.y);
|
|
423
440
|
return {
|
|
@@ -426,14 +443,14 @@ async function tryAccessibilityStrategy(query, index, platform, udid) {
|
|
|
426
443
|
pressed: match.text || match.className || undefined,
|
|
427
444
|
text: match.text || undefined,
|
|
428
445
|
component: match.className || undefined,
|
|
429
|
-
convertedTo: { x: match.center.x, y: match.center.y, unit: "pixels" }
|
|
446
|
+
convertedTo: { x: match.center.x, y: match.center.y, unit: "pixels" }
|
|
430
447
|
};
|
|
431
448
|
}
|
|
432
449
|
}
|
|
433
450
|
catch (err) {
|
|
434
451
|
return {
|
|
435
452
|
success: false,
|
|
436
|
-
reason: `Accessibility strategy error: ${err instanceof Error ? err.message : String(err)}
|
|
453
|
+
reason: `Accessibility strategy error: ${err instanceof Error ? err.message : String(err)}`
|
|
437
454
|
};
|
|
438
455
|
}
|
|
439
456
|
}
|
|
@@ -448,7 +465,10 @@ async function tryOcrStrategy(query, platform, udid) {
|
|
|
448
465
|
if (platform === "ios") {
|
|
449
466
|
const screenshot = await iosScreenshot(undefined, udid);
|
|
450
467
|
if (!screenshot.success || !screenshot.data) {
|
|
451
|
-
return {
|
|
468
|
+
return {
|
|
469
|
+
success: false,
|
|
470
|
+
reason: "Failed to capture iOS screenshot for OCR"
|
|
471
|
+
};
|
|
452
472
|
}
|
|
453
473
|
imageBuffer = screenshot.data;
|
|
454
474
|
scaleFactor = screenshot.scaleFactor ?? 1;
|
|
@@ -457,7 +477,10 @@ async function tryOcrStrategy(query, platform, udid) {
|
|
|
457
477
|
const { androidScreenshot } = await import("../core/android.js");
|
|
458
478
|
const screenshot = await androidScreenshot();
|
|
459
479
|
if (!screenshot.success || !screenshot.data) {
|
|
460
|
-
return {
|
|
480
|
+
return {
|
|
481
|
+
success: false,
|
|
482
|
+
reason: "Failed to capture Android screenshot for OCR"
|
|
483
|
+
};
|
|
461
484
|
}
|
|
462
485
|
imageBuffer = screenshot.data;
|
|
463
486
|
scaleFactor = screenshot.scaleFactor ?? 1;
|
|
@@ -465,12 +488,16 @@ async function tryOcrStrategy(query, platform, udid) {
|
|
|
465
488
|
const { recognizeText } = await import("../core/ocr.js");
|
|
466
489
|
const ocrResult = await recognizeText(imageBuffer, {
|
|
467
490
|
scaleFactor,
|
|
468
|
-
platform
|
|
491
|
+
platform
|
|
469
492
|
});
|
|
470
493
|
const lowerSearch = searchText.toLowerCase();
|
|
471
|
-
const matchingWord = ocrResult.words.find((w) => w.text.toLowerCase() === lowerSearch) ||
|
|
494
|
+
const matchingWord = ocrResult.words.find((w) => w.text.toLowerCase() === lowerSearch) ||
|
|
495
|
+
ocrResult.words.find((w) => w.text.toLowerCase().includes(lowerSearch));
|
|
472
496
|
if (!matchingWord) {
|
|
473
|
-
return {
|
|
497
|
+
return {
|
|
498
|
+
success: false,
|
|
499
|
+
reason: `OCR did not find text "${searchText}" on screen`
|
|
500
|
+
};
|
|
474
501
|
}
|
|
475
502
|
if (platform === "ios") {
|
|
476
503
|
// tapCenter is in image-pixel space (downscaled) — convert to points
|
|
@@ -478,7 +505,10 @@ async function tryOcrStrategy(query, platform, udid) {
|
|
|
478
505
|
const dpr = await getDevicePixelRatio(udid);
|
|
479
506
|
const tapResult = await iosTap(Math.round((matchingWord.tapCenter.x * scaleFactor) / dpr), Math.round((matchingWord.tapCenter.y * scaleFactor) / dpr), { udid });
|
|
480
507
|
if (!tapResult.success) {
|
|
481
|
-
return {
|
|
508
|
+
return {
|
|
509
|
+
success: false,
|
|
510
|
+
reason: `OCR found "${matchingWord.text}" but tap failed: ${tapResult.error}`
|
|
511
|
+
};
|
|
482
512
|
}
|
|
483
513
|
}
|
|
484
514
|
else {
|
|
@@ -492,14 +522,14 @@ async function tryOcrStrategy(query, platform, udid) {
|
|
|
492
522
|
convertedTo: {
|
|
493
523
|
x: matchingWord.tapCenter.x,
|
|
494
524
|
y: matchingWord.tapCenter.y,
|
|
495
|
-
unit: "pixels"
|
|
496
|
-
}
|
|
525
|
+
unit: "pixels"
|
|
526
|
+
}
|
|
497
527
|
};
|
|
498
528
|
}
|
|
499
529
|
catch (err) {
|
|
500
530
|
return {
|
|
501
531
|
success: false,
|
|
502
|
-
reason: `OCR strategy error: ${err instanceof Error ? err.message : String(err)}
|
|
532
|
+
reason: `OCR strategy error: ${err instanceof Error ? err.message : String(err)}`
|
|
503
533
|
};
|
|
504
534
|
}
|
|
505
535
|
}
|
|
@@ -512,7 +542,10 @@ async function tryCoordinateStrategy(pixelX, pixelY, platform, lastScreenshot, u
|
|
|
512
542
|
const converted = convertScreenshotToTapCoords(pixelX, pixelY, "ios", devicePixelRatio, scaleFactor);
|
|
513
543
|
const tapResult = await iosTap(converted.x, converted.y, { udid });
|
|
514
544
|
if (!tapResult.success) {
|
|
515
|
-
return {
|
|
545
|
+
return {
|
|
546
|
+
success: false,
|
|
547
|
+
reason: `Coordinate tap failed: ${tapResult.error}`
|
|
548
|
+
};
|
|
516
549
|
}
|
|
517
550
|
// Best-effort: identify what was tapped via fiber tree
|
|
518
551
|
let screen = null;
|
|
@@ -526,7 +559,7 @@ async function tryCoordinateStrategy(pixelX, pixelY, platform, lastScreenshot, u
|
|
|
526
559
|
success: true,
|
|
527
560
|
reason: "Tapped at coordinates (iOS)",
|
|
528
561
|
screen,
|
|
529
|
-
convertedTo: { x: converted.x, y: converted.y, unit: "points" }
|
|
562
|
+
convertedTo: { x: converted.x, y: converted.y, unit: "points" }
|
|
530
563
|
};
|
|
531
564
|
}
|
|
532
565
|
else {
|
|
@@ -545,14 +578,14 @@ async function tryCoordinateStrategy(pixelX, pixelY, platform, lastScreenshot, u
|
|
|
545
578
|
success: true,
|
|
546
579
|
reason: "Tapped at coordinates (Android)",
|
|
547
580
|
screen,
|
|
548
|
-
convertedTo: { x: converted.x, y: converted.y, unit: "pixels" }
|
|
581
|
+
convertedTo: { x: converted.x, y: converted.y, unit: "pixels" }
|
|
549
582
|
};
|
|
550
583
|
}
|
|
551
584
|
}
|
|
552
585
|
catch (err) {
|
|
553
586
|
return {
|
|
554
587
|
success: false,
|
|
555
|
-
reason: `Coordinate strategy error: ${err instanceof Error ? err.message : String(err)}
|
|
588
|
+
reason: `Coordinate strategy error: ${err instanceof Error ? err.message : String(err)}`
|
|
556
589
|
};
|
|
557
590
|
}
|
|
558
591
|
}
|
|
@@ -563,12 +596,18 @@ const MAX_STRATEGY_MS = {
|
|
|
563
596
|
fiber: 5000,
|
|
564
597
|
accessibility: 3000,
|
|
565
598
|
ocr: 5000,
|
|
566
|
-
coordinate: 3000
|
|
599
|
+
coordinate: 3000
|
|
567
600
|
};
|
|
568
601
|
function withTimeout(promise, ms, label) {
|
|
569
602
|
return new Promise((resolve, reject) => {
|
|
570
603
|
const timer = setTimeout(() => reject(new Error(`${label} timed out after ${ms}ms`)), ms);
|
|
571
|
-
promise.then((val) => {
|
|
604
|
+
promise.then((val) => {
|
|
605
|
+
clearTimeout(timer);
|
|
606
|
+
resolve(val);
|
|
607
|
+
}, (err) => {
|
|
608
|
+
clearTimeout(timer);
|
|
609
|
+
reject(err);
|
|
610
|
+
});
|
|
572
611
|
});
|
|
573
612
|
}
|
|
574
613
|
async function captureScreenshot(platform, udid) {
|
|
@@ -580,7 +619,7 @@ async function captureScreenshot(platform, udid) {
|
|
|
580
619
|
buffer: result.data,
|
|
581
620
|
width: result.originalWidth || 0,
|
|
582
621
|
height: result.originalHeight || 0,
|
|
583
|
-
scaleFactor: result.scaleFactor || 1
|
|
622
|
+
scaleFactor: result.scaleFactor || 1
|
|
584
623
|
};
|
|
585
624
|
}
|
|
586
625
|
catch {
|
|
@@ -601,7 +640,7 @@ async function verifyAndCapture(platform, shouldVerify, shouldScreenshot, before
|
|
|
601
640
|
image: screenshotToBase64(after.buffer),
|
|
602
641
|
width: after.width,
|
|
603
642
|
height: after.height,
|
|
604
|
-
scaleFactor: after.scaleFactor
|
|
643
|
+
scaleFactor: after.scaleFactor
|
|
605
644
|
};
|
|
606
645
|
let verification;
|
|
607
646
|
if (shouldVerify && beforeBuffer) {
|
|
@@ -609,7 +648,9 @@ async function verifyAndCapture(platform, shouldVerify, shouldScreenshot, before
|
|
|
609
648
|
const rawStatusBar = platform === "ios" ? 177 : 142; // pixels in original screenshot space
|
|
610
649
|
const scale = beforeScaleFactor || after.scaleFactor || 1;
|
|
611
650
|
const statusBarHeight = Math.round(rawStatusBar / scale);
|
|
612
|
-
const diff = await compareScreenshots(beforeBuffer, after.buffer, {
|
|
651
|
+
const diff = await compareScreenshots(beforeBuffer, after.buffer, {
|
|
652
|
+
statusBarHeight
|
|
653
|
+
});
|
|
613
654
|
verification = {
|
|
614
655
|
meaningful: diff.changed,
|
|
615
656
|
changeRate: diff.changeRate,
|
|
@@ -619,8 +660,8 @@ async function verifyAndCapture(platform, shouldVerify, shouldScreenshot, before
|
|
|
619
660
|
meaningful: diff.changed,
|
|
620
661
|
changeRate: diff.changeRate,
|
|
621
662
|
changedPixels: diff.changedPixels,
|
|
622
|
-
totalPixels: diff.totalPixels
|
|
623
|
-
})
|
|
663
|
+
totalPixels: diff.totalPixels
|
|
664
|
+
})
|
|
624
665
|
};
|
|
625
666
|
}
|
|
626
667
|
catch {
|
|
@@ -635,7 +676,7 @@ async function verifyAndCapture(platform, shouldVerify, shouldScreenshot, before
|
|
|
635
676
|
timestamp: Date.now(),
|
|
636
677
|
source: "tap-verify",
|
|
637
678
|
groupId: verifyGroupId,
|
|
638
|
-
metadata: { phase: "before" }
|
|
679
|
+
metadata: { phase: "before" }
|
|
639
680
|
});
|
|
640
681
|
}
|
|
641
682
|
imageBuffer.add({
|
|
@@ -644,7 +685,7 @@ async function verifyAndCapture(platform, shouldVerify, shouldScreenshot, before
|
|
|
644
685
|
timestamp: Date.now(),
|
|
645
686
|
source: "tap-verify",
|
|
646
687
|
groupId: verifyGroupId,
|
|
647
|
-
metadata: { phase: "after", changeRate: verification?.changeRate }
|
|
688
|
+
metadata: { phase: "after", changeRate: verification?.changeRate }
|
|
648
689
|
});
|
|
649
690
|
return { screenshot, verification };
|
|
650
691
|
}
|
|
@@ -681,8 +722,8 @@ async function burstCaptureAndVerify(platform, beforeBuffer, udid, beforeScaleFa
|
|
|
681
722
|
metadata: {
|
|
682
723
|
frameIndex: i,
|
|
683
724
|
isBefore: i === 0,
|
|
684
|
-
changeRate: i === 0 ? 0 :
|
|
685
|
-
}
|
|
725
|
+
changeRate: i === 0 ? 0 : analysis.framesWithChange.includes(i) ? analysis.peakChangeRate : 0
|
|
726
|
+
}
|
|
686
727
|
});
|
|
687
728
|
}
|
|
688
729
|
imageBuffer.addGroup({
|
|
@@ -696,8 +737,8 @@ async function burstCaptureAndVerify(platform, beforeBuffer, udid, beforeScaleFa
|
|
|
696
737
|
peakFrame: analysis.peakFrame,
|
|
697
738
|
framesWithChange: analysis.framesWithChange,
|
|
698
739
|
transientChangeDetected: analysis.transientChangeDetected,
|
|
699
|
-
persistentChangeRate: analysis.persistentChangeRate
|
|
700
|
-
}
|
|
740
|
+
persistentChangeRate: analysis.persistentChangeRate
|
|
741
|
+
}
|
|
701
742
|
});
|
|
702
743
|
// Get dimensions from the last frame using sharp
|
|
703
744
|
const sharp = (await import("sharp")).default;
|
|
@@ -706,7 +747,7 @@ async function burstCaptureAndVerify(platform, beforeBuffer, udid, beforeScaleFa
|
|
|
706
747
|
image: screenshotToBase64(frames[frames.length - 1]),
|
|
707
748
|
width: meta.width || 0,
|
|
708
749
|
height: meta.height || 0,
|
|
709
|
-
scaleFactor: 1
|
|
750
|
+
scaleFactor: 1
|
|
710
751
|
};
|
|
711
752
|
const verification = {
|
|
712
753
|
meaningful: analysis.meaningful,
|
|
@@ -724,8 +765,8 @@ async function burstCaptureAndVerify(platform, beforeBuffer, udid, beforeScaleFa
|
|
|
724
765
|
totalPixels: 0,
|
|
725
766
|
transientChangeDetected: analysis.transientChangeDetected,
|
|
726
767
|
peakChangeRate: analysis.peakChangeRate,
|
|
727
|
-
peakFrame: analysis.peakFrame
|
|
728
|
-
})
|
|
768
|
+
peakFrame: analysis.peakFrame
|
|
769
|
+
})
|
|
729
770
|
};
|
|
730
771
|
return { screenshot, verification };
|
|
731
772
|
}
|
|
@@ -744,14 +785,14 @@ export async function tap(options) {
|
|
|
744
785
|
return {
|
|
745
786
|
success: false,
|
|
746
787
|
query,
|
|
747
|
-
error: "Must provide at least one of: text, testID, component, or x/y coordinates"
|
|
788
|
+
error: "Must provide at least one of: text, testID, component, or x/y coordinates"
|
|
748
789
|
};
|
|
749
790
|
}
|
|
750
791
|
if (hasCoordinates && (query.x === undefined || query.y === undefined)) {
|
|
751
792
|
return {
|
|
752
793
|
success: false,
|
|
753
794
|
query,
|
|
754
|
-
error: "Both x and y coordinates must be provided"
|
|
795
|
+
error: "Both x and y coordinates must be provided"
|
|
755
796
|
};
|
|
756
797
|
}
|
|
757
798
|
// Native mode: bypass React Native connection, tap directly via ADB/simctl
|
|
@@ -762,13 +803,13 @@ export async function tap(options) {
|
|
|
762
803
|
if (!platform) {
|
|
763
804
|
const [androidDevice, iosSimulator] = await Promise.all([
|
|
764
805
|
getDefaultAndroidDevice().catch(() => null),
|
|
765
|
-
getActiveOrBootedSimulatorUdid().catch(() => null)
|
|
806
|
+
getActiveOrBootedSimulatorUdid().catch(() => null)
|
|
766
807
|
]);
|
|
767
808
|
if (androidDevice && iosSimulator) {
|
|
768
809
|
return {
|
|
769
810
|
success: false,
|
|
770
811
|
query,
|
|
771
|
-
error:
|
|
812
|
+
error: 'Multiple platforms detected (both Android and iOS). Specify platform: "android" or platform: "ios" to target the correct device.'
|
|
772
813
|
};
|
|
773
814
|
}
|
|
774
815
|
if (androidDevice) {
|
|
@@ -782,7 +823,7 @@ export async function tap(options) {
|
|
|
782
823
|
return {
|
|
783
824
|
success: false,
|
|
784
825
|
query,
|
|
785
|
-
error: "No Android device or iOS simulator found. Connect a device or start a simulator."
|
|
826
|
+
error: "No Android device or iOS simulator found. Connect a device or start a simulator."
|
|
786
827
|
};
|
|
787
828
|
}
|
|
788
829
|
}
|
|
@@ -800,7 +841,7 @@ export async function tap(options) {
|
|
|
800
841
|
nativeScreenshotMeta = {
|
|
801
842
|
originalWidth: before.width,
|
|
802
843
|
originalHeight: before.height,
|
|
803
|
-
scaleFactor: before.scaleFactor
|
|
844
|
+
scaleFactor: before.scaleFactor
|
|
804
845
|
};
|
|
805
846
|
}
|
|
806
847
|
}
|
|
@@ -811,7 +852,7 @@ export async function tap(options) {
|
|
|
811
852
|
nativeScreenshotMeta = {
|
|
812
853
|
originalWidth: ref.width,
|
|
813
854
|
originalHeight: ref.height,
|
|
814
|
-
scaleFactor: ref.scaleFactor
|
|
855
|
+
scaleFactor: ref.scaleFactor
|
|
815
856
|
};
|
|
816
857
|
// Also use it for verification if buffer is needed
|
|
817
858
|
if (!nativeBeforeBuffer) {
|
|
@@ -826,8 +867,13 @@ export async function tap(options) {
|
|
|
826
867
|
catch (err) {
|
|
827
868
|
return formatTapFailure({
|
|
828
869
|
query,
|
|
829
|
-
attempted: [
|
|
830
|
-
|
|
870
|
+
attempted: [
|
|
871
|
+
{
|
|
872
|
+
strategy: "native-coordinate",
|
|
873
|
+
reason: err instanceof Error ? err.message : String(err)
|
|
874
|
+
}
|
|
875
|
+
],
|
|
876
|
+
suggestion: `Tap timed out. Take a screenshot (${platform === "ios" ? "ios_screenshot" : "android_screenshot"}) and retry with coordinates.`
|
|
831
877
|
});
|
|
832
878
|
}
|
|
833
879
|
if (result.success) {
|
|
@@ -846,13 +892,13 @@ export async function tap(options) {
|
|
|
846
892
|
convertedTo: result.convertedTo,
|
|
847
893
|
platform,
|
|
848
894
|
screenshot,
|
|
849
|
-
verification
|
|
895
|
+
verification
|
|
850
896
|
});
|
|
851
897
|
}
|
|
852
898
|
return formatTapFailure({
|
|
853
899
|
query,
|
|
854
900
|
attempted: [{ strategy: "native-coordinate", reason: result.reason }],
|
|
855
|
-
suggestion: `Take a screenshot (${platform === "ios" ? "ios_screenshot" : "android_screenshot"}) to verify coordinates
|
|
901
|
+
suggestion: `Take a screenshot (${platform === "ios" ? "ios_screenshot" : "android_screenshot"}) to verify coordinates.`
|
|
856
902
|
});
|
|
857
903
|
}
|
|
858
904
|
// Detect available capabilities
|
|
@@ -860,7 +906,7 @@ export async function tap(options) {
|
|
|
860
906
|
let hasMetro = allApps.length > 0;
|
|
861
907
|
// If platform is specified, prefer an app matching that platform
|
|
862
908
|
let app = options.platform
|
|
863
|
-
? allApps.find((a) => a.platform === options.platform) ?? allApps[0]
|
|
909
|
+
? (allApps.find((a) => a.platform === options.platform) ?? allApps[0])
|
|
864
910
|
: allApps[0];
|
|
865
911
|
// Try to auto-connect to Metro (for fiber strategy), but don't fail if it doesn't work
|
|
866
912
|
if (!hasMetro) {
|
|
@@ -895,13 +941,13 @@ export async function tap(options) {
|
|
|
895
941
|
// No Metro — detect from available devices
|
|
896
942
|
const [androidDevice, iosSimulator] = await Promise.all([
|
|
897
943
|
getDefaultAndroidDevice().catch(() => null),
|
|
898
|
-
getActiveOrBootedSimulatorUdid().catch(() => null)
|
|
944
|
+
getActiveOrBootedSimulatorUdid().catch(() => null)
|
|
899
945
|
]);
|
|
900
946
|
if (androidDevice && iosSimulator) {
|
|
901
947
|
return {
|
|
902
948
|
success: false,
|
|
903
949
|
query,
|
|
904
|
-
error:
|
|
950
|
+
error: 'Both iOS and Android devices detected but no Metro connection. Specify platform="ios" or platform="android".'
|
|
905
951
|
};
|
|
906
952
|
}
|
|
907
953
|
if (androidDevice)
|
|
@@ -914,47 +960,55 @@ export async function tap(options) {
|
|
|
914
960
|
return {
|
|
915
961
|
success: false,
|
|
916
962
|
query,
|
|
917
|
-
error: "No devices found. Boot a simulator or connect an Android device."
|
|
963
|
+
error: "No devices found. Boot a simulator or connect an Android device."
|
|
918
964
|
};
|
|
919
965
|
}
|
|
920
966
|
// Resolve target iOS simulator UDID from the connected app
|
|
921
967
|
// This ensures taps go to the correct device when multiple simulators are booted
|
|
922
968
|
let targetUdid;
|
|
923
969
|
if (platform === "ios") {
|
|
924
|
-
targetUdid =
|
|
925
|
-
|
|
970
|
+
targetUdid =
|
|
971
|
+
app?.simulatorUdid ??
|
|
972
|
+
(app?.deviceInfo?.deviceName ? await findSimulatorByName(app.deviceInfo.deviceName) : null) ??
|
|
973
|
+
undefined;
|
|
926
974
|
}
|
|
927
975
|
const deviceName = app?.deviceInfo?.deviceName;
|
|
928
976
|
// Determine strategies
|
|
929
977
|
const strategies = getAvailableStrategies(query, strategy);
|
|
930
978
|
const attempted = [];
|
|
931
|
-
// Early
|
|
932
|
-
const
|
|
933
|
-
let
|
|
979
|
+
// Early UI driver check for iOS — fail fast instead of falling through every strategy
|
|
980
|
+
const UI_DRIVER_REQUIRED_STRATEGIES = ["accessibility", "ocr", "coordinate"];
|
|
981
|
+
let uiDriverMissing = false;
|
|
934
982
|
if (platform === "ios") {
|
|
935
|
-
|
|
983
|
+
uiDriverMissing = !(await isUiDriverAvailable());
|
|
936
984
|
}
|
|
937
985
|
// Filter strategies by available capabilities
|
|
938
986
|
const filteredStrategies = strategies.filter((strat) => {
|
|
939
987
|
if (strat === "fiber" && !hasMetro) {
|
|
940
|
-
attempted.push({
|
|
988
|
+
attempted.push({
|
|
989
|
+
strategy: "fiber",
|
|
990
|
+
reason: "Skipped — no Metro connection (required for fiber)"
|
|
991
|
+
});
|
|
941
992
|
return false;
|
|
942
993
|
}
|
|
943
|
-
if (
|
|
944
|
-
attempted.push({
|
|
994
|
+
if (uiDriverMissing && UI_DRIVER_REQUIRED_STRATEGIES.includes(strat)) {
|
|
995
|
+
attempted.push({
|
|
996
|
+
strategy: strat,
|
|
997
|
+
reason: "Skipped — iOS UI driver is not installed (required for iOS tap/accessibility/OCR)"
|
|
998
|
+
});
|
|
945
999
|
return false;
|
|
946
1000
|
}
|
|
947
1001
|
return true;
|
|
948
1002
|
});
|
|
949
1003
|
if (filteredStrategies.length === 0) {
|
|
950
|
-
const errorMessage =
|
|
951
|
-
?
|
|
1004
|
+
const errorMessage = uiDriverMissing
|
|
1005
|
+
? `Cannot tap on iOS Simulator — ${getUiDriverInstallHint()}\n\nThe iOS UI driver is required for tapping, swiping, text input, and accessibility queries on iOS Simulators.\n\nAfter installing, retry the tap.`
|
|
952
1006
|
: "All strategies require Metro connection, which is unavailable.\n\nTo fix:\n1. Make sure your React Native app is running\n2. Run scan_metro to connect\n3. Or use tap(x, y, native=true) for coordinate-based taps";
|
|
953
1007
|
return {
|
|
954
1008
|
success: false,
|
|
955
1009
|
query,
|
|
956
1010
|
attempted,
|
|
957
|
-
error: errorMessage
|
|
1011
|
+
error: errorMessage
|
|
958
1012
|
};
|
|
959
1013
|
}
|
|
960
1014
|
// Determine screenshot and verification behavior
|
|
@@ -974,7 +1028,10 @@ export async function tap(options) {
|
|
|
974
1028
|
for (const strat of filteredStrategies) {
|
|
975
1029
|
const remaining = remainingMs();
|
|
976
1030
|
if (remaining < MIN_STRATEGY_BUDGET_MS) {
|
|
977
|
-
attempted.push({
|
|
1031
|
+
attempted.push({
|
|
1032
|
+
strategy: strat,
|
|
1033
|
+
reason: `Skipped — only ${remaining}ms remaining (budget ${TAP_TIMEOUT_MS}ms)`
|
|
1034
|
+
});
|
|
978
1035
|
continue;
|
|
979
1036
|
}
|
|
980
1037
|
const cap = MAX_STRATEGY_MS[strat] ?? 5000;
|
|
@@ -999,7 +1056,10 @@ export async function tap(options) {
|
|
|
999
1056
|
}
|
|
1000
1057
|
}
|
|
1001
1058
|
catch (err) {
|
|
1002
|
-
attempted.push({
|
|
1059
|
+
attempted.push({
|
|
1060
|
+
strategy: strat,
|
|
1061
|
+
reason: err instanceof Error ? err.message : String(err)
|
|
1062
|
+
});
|
|
1003
1063
|
continue;
|
|
1004
1064
|
}
|
|
1005
1065
|
if (result.success) {
|
|
@@ -1015,7 +1075,7 @@ export async function tap(options) {
|
|
|
1015
1075
|
app.lastScreenshot = {
|
|
1016
1076
|
originalWidth: screenshot.width,
|
|
1017
1077
|
originalHeight: screenshot.height,
|
|
1018
|
-
scaleFactor: screenshot.scaleFactor
|
|
1078
|
+
scaleFactor: screenshot.scaleFactor
|
|
1019
1079
|
};
|
|
1020
1080
|
}
|
|
1021
1081
|
return formatTapSuccess({
|
|
@@ -1030,7 +1090,7 @@ export async function tap(options) {
|
|
|
1030
1090
|
platform,
|
|
1031
1091
|
device: deviceName,
|
|
1032
1092
|
screenshot,
|
|
1033
|
-
verification
|
|
1093
|
+
verification
|
|
1034
1094
|
});
|
|
1035
1095
|
}
|
|
1036
1096
|
attempted.push({ strategy: strat, reason: result.reason });
|
|
@@ -1063,7 +1123,7 @@ export async function tap(options) {
|
|
|
1063
1123
|
app.lastScreenshot = {
|
|
1064
1124
|
originalWidth: screenshot.width,
|
|
1065
1125
|
originalHeight: screenshot.height,
|
|
1066
|
-
scaleFactor: screenshot.scaleFactor
|
|
1126
|
+
scaleFactor: screenshot.scaleFactor
|
|
1067
1127
|
};
|
|
1068
1128
|
}
|
|
1069
1129
|
return formatTapSuccess({
|
|
@@ -1077,7 +1137,7 @@ export async function tap(options) {
|
|
|
1077
1137
|
platform,
|
|
1078
1138
|
device: deviceName,
|
|
1079
1139
|
screenshot,
|
|
1080
|
-
verification
|
|
1140
|
+
verification
|
|
1081
1141
|
});
|
|
1082
1142
|
}
|
|
1083
1143
|
catch {
|
|
@@ -1093,7 +1153,7 @@ export async function tap(options) {
|
|
|
1093
1153
|
app.lastScreenshot = {
|
|
1094
1154
|
originalWidth: matchScreenshot.width,
|
|
1095
1155
|
originalHeight: matchScreenshot.height,
|
|
1096
|
-
scaleFactor: matchScreenshot.scaleFactor
|
|
1156
|
+
scaleFactor: matchScreenshot.scaleFactor
|
|
1097
1157
|
};
|
|
1098
1158
|
}
|
|
1099
1159
|
return formatTapFailure({
|
|
@@ -1102,13 +1162,13 @@ export async function tap(options) {
|
|
|
1102
1162
|
suggestion: `Found ${result.matches.length} match(es) — specify index to select one`,
|
|
1103
1163
|
device: deviceName,
|
|
1104
1164
|
matches: result.matches,
|
|
1105
|
-
screenshot: matchScreenshot
|
|
1165
|
+
screenshot: matchScreenshot
|
|
1106
1166
|
});
|
|
1107
1167
|
}
|
|
1108
1168
|
}
|
|
1109
1169
|
// All strategies failed — check if timeout was the cause
|
|
1110
|
-
const timedOut = attempted.some(a => a.reason.includes("timed out"));
|
|
1111
|
-
const skipped = attempted.some(a => a.reason.includes("Skipped"));
|
|
1170
|
+
const timedOut = attempted.some((a) => a.reason.includes("timed out"));
|
|
1171
|
+
const skipped = attempted.some((a) => a.reason.includes("Skipped"));
|
|
1112
1172
|
const hitTimeout = timedOut || skipped;
|
|
1113
1173
|
const elapsed = TAP_TIMEOUT_MS - remainingMs();
|
|
1114
1174
|
const suggestion = buildSuggestion(query, strategies, platform);
|
|
@@ -1119,18 +1179,16 @@ export async function tap(options) {
|
|
|
1119
1179
|
app.lastScreenshot = {
|
|
1120
1180
|
originalWidth: failScreenshot.width,
|
|
1121
1181
|
originalHeight: failScreenshot.height,
|
|
1122
|
-
scaleFactor: failScreenshot.scaleFactor
|
|
1182
|
+
scaleFactor: failScreenshot.scaleFactor
|
|
1123
1183
|
};
|
|
1124
1184
|
}
|
|
1125
1185
|
return formatTapFailure({
|
|
1126
1186
|
query,
|
|
1127
1187
|
attempted,
|
|
1128
|
-
error: hitTimeout
|
|
1129
|
-
? `Tap timed out after ${elapsed}ms (budget ${TAP_TIMEOUT_MS}ms)`
|
|
1130
|
-
: undefined,
|
|
1188
|
+
error: hitTimeout ? `Tap timed out after ${elapsed}ms (budget ${TAP_TIMEOUT_MS}ms)` : undefined,
|
|
1131
1189
|
suggestion,
|
|
1132
1190
|
device: deviceName,
|
|
1133
|
-
screenshot: failScreenshot
|
|
1191
|
+
screenshot: failScreenshot
|
|
1134
1192
|
});
|
|
1135
1193
|
}
|
|
1136
1194
|
function buildSuggestion(query, triedStrategies, platform) {
|