react-native-ai-debugger 1.6.3 → 1.6.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +10 -0
- package/build/__tests__/unit/failureArtifact.test.d.ts +2 -0
- package/build/__tests__/unit/failureArtifact.test.d.ts.map +1 -0
- package/build/__tests__/unit/failureArtifact.test.js +207 -0
- package/build/__tests__/unit/failureArtifact.test.js.map +1 -0
- package/build/__tests__/unit/tap-evidence.test.d.ts +2 -0
- package/build/__tests__/unit/tap-evidence.test.d.ts.map +1 -0
- package/build/__tests__/unit/tap-evidence.test.js +89 -0
- package/build/__tests__/unit/tap-evidence.test.js.map +1 -0
- package/build/core/executor.d.ts.map +1 -1
- package/build/core/executor.js +3 -3
- package/build/core/executor.js.map +1 -1
- package/build/core/failureArtifact.d.ts +122 -0
- package/build/core/failureArtifact.d.ts.map +1 -0
- package/build/core/failureArtifact.js +117 -0
- package/build/core/failureArtifact.js.map +1 -0
- package/build/core/telemetry.d.ts +3 -1
- package/build/core/telemetry.d.ts.map +1 -1
- package/build/core/telemetry.js +23 -1
- package/build/core/telemetry.js.map +1 -1
- package/build/index.js +21 -1
- package/build/index.js.map +1 -1
- package/build/pro/tap.d.ts +55 -0
- package/build/pro/tap.d.ts.map +1 -1
- package/build/pro/tap.js +313 -29
- package/build/pro/tap.js.map +1 -1
- package/package.json +1 -1
package/build/pro/tap.d.ts
CHANGED
|
@@ -110,6 +110,11 @@ export interface TapResult {
|
|
|
110
110
|
screenshot?: TapScreenshot;
|
|
111
111
|
verification?: TapVerification;
|
|
112
112
|
warning?: string;
|
|
113
|
+
artifactKey?: string;
|
|
114
|
+
ocrClosestMatch?: string;
|
|
115
|
+
fiberPressableCount?: string;
|
|
116
|
+
accessibilityMatchCount?: string;
|
|
117
|
+
appRoute?: string;
|
|
113
118
|
}
|
|
114
119
|
export declare function buildQuery(options: TapOptions): TapQuery;
|
|
115
120
|
/**
|
|
@@ -185,6 +190,56 @@ export declare function formatTapFailure(data: {
|
|
|
185
190
|
screenshot?: TapScreenshot;
|
|
186
191
|
verification?: TapVerification;
|
|
187
192
|
}): TapResult;
|
|
193
|
+
export interface EvidenceSink {
|
|
194
|
+
fiber: {
|
|
195
|
+
ran: boolean;
|
|
196
|
+
durationMs: number;
|
|
197
|
+
metroConnected: boolean;
|
|
198
|
+
pressables: Array<{
|
|
199
|
+
label?: string;
|
|
200
|
+
testID?: string;
|
|
201
|
+
componentName?: string;
|
|
202
|
+
bounds?: {
|
|
203
|
+
x: number;
|
|
204
|
+
y: number;
|
|
205
|
+
width: number;
|
|
206
|
+
height: number;
|
|
207
|
+
};
|
|
208
|
+
}>;
|
|
209
|
+
};
|
|
210
|
+
accessibility: {
|
|
211
|
+
ran: boolean;
|
|
212
|
+
durationMs: number;
|
|
213
|
+
elements: Array<{
|
|
214
|
+
label?: string;
|
|
215
|
+
testID?: string;
|
|
216
|
+
frame?: {
|
|
217
|
+
x: number;
|
|
218
|
+
y: number;
|
|
219
|
+
width: number;
|
|
220
|
+
height: number;
|
|
221
|
+
};
|
|
222
|
+
}>;
|
|
223
|
+
};
|
|
224
|
+
ocr: {
|
|
225
|
+
ran: boolean;
|
|
226
|
+
durationMs: number;
|
|
227
|
+
detections: Array<{
|
|
228
|
+
text: string;
|
|
229
|
+
bbox: [number, number, number, number];
|
|
230
|
+
conf: number;
|
|
231
|
+
}>;
|
|
232
|
+
closestMatch: {
|
|
233
|
+
text: string;
|
|
234
|
+
score: number;
|
|
235
|
+
} | null;
|
|
236
|
+
};
|
|
237
|
+
}
|
|
238
|
+
export declare function makeEmptyEvidenceSink(): EvidenceSink;
|
|
239
|
+
export declare function findClosestOcrText(ocrResult: OCRResult, query: string): {
|
|
240
|
+
text: string;
|
|
241
|
+
score: number;
|
|
242
|
+
} | null;
|
|
188
243
|
export declare function isTapTimeout(attempted: readonly {
|
|
189
244
|
reason: string;
|
|
190
245
|
strategy?: string;
|
package/build/pro/tap.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"tap.d.ts","sourceRoot":"","sources":["../../src/pro/tap.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,gBAAgB,CAAC;
|
|
1
|
+
{"version":3,"file":"tap.d.ts","sourceRoot":"","sources":["../../src/pro/tap.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,gBAAgB,CAAC;AAsBhD,MAAM,MAAM,WAAW,GAAG,MAAM,GAAG,OAAO,GAAG,eAAe,GAAG,KAAK,GAAG,YAAY,CAAC;AAEpF,MAAM,WAAW,QAAQ;IACrB,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,CAAC,CAAC,EAAE,MAAM,CAAC;IACX,CAAC,CAAC,EAAE,MAAM,CAAC;CACd;AAED,MAAM,WAAW,UAAU;IACvB,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,CAAC,CAAC,EAAE,MAAM,CAAC;IACX,CAAC,CAAC,EAAE,MAAM,CAAC;IACX,QAAQ,CAAC,EAAE,WAAW,CAAC;IACvB,iBAAiB,CAAC,EAAE,MAAM,CAAC;IAC3B,MAAM,CAAC,EAAE,OAAO,CAAC;IACjB,QAAQ,CAAC,EAAE,KAAK,GAAG,SAAS,CAAC;IAC7B,UAAU,CAAC,EAAE,OAAO,CAAC;IACrB,MAAM,CAAC,EAAE,OAAO,CAAC;IACjB,KAAK,CAAC,EAAE,OAAO,CAAC;IAChB;;;;OAIG;IACH,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB;;;;OAIG;IACH,IAAI,CAAC,EAAE,MAAM,CAAC;CACjB;AAED,MAAM,WAAW,UAAU;IACvB,QAAQ,EAAE,MAAM,CAAC;IACjB,MAAM,EAAE,MAAM,CAAC;CAClB;AAED,MAAM,WAAW,aAAa;IAC1B,KAAK,EAAE,MAAM,CAAC;IACd,KAAK,EAAE,MAAM,CAAC;IACd,MAAM,EAAE,MAAM,CAAC;IACf,WAAW,EAAE,MAAM,CAAC;CACvB;AAED,MAAM,WAAW,eAAe;IAC5B,UAAU,EAAE,OAAO,CAAC;IACpB,UAAU,EAAE,MAAM,CAAC;IACnB,aAAa,EAAE,MAAM,CAAC;IACtB,WAAW,EAAE,MAAM,CAAC;IACpB,uBAAuB,CAAC,EAAE,OAAO,CAAC;IAClC,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,WAAW,EAAE,MAAM,CAAC;CACvB;AAED,wBAAgB,4BAA4B,CAAC,CAAC,EAAE;IAC5C,UAAU,EAAE,OAAO,CAAC;IACpB,UAAU,EAAE,MAAM,CAAC;IACnB,aAAa,EAAE,MAAM,CAAC;IACtB,WAAW,EAAE,MAAM,CAAC;IACpB,uBAAuB,CAAC,EAAE,OAAO,CAAC;IAClC,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,SAAS,CAAC,EAAE,MAAM,CAAC;CACtB,GAAG,MAAM,CA0BT;AAED,MAAM,WAAW,aAAa;IAC1B,UAAU,EAAE,OAAO,CAAC;IACpB,oBAAoB,EAAE,MAAM,CAAC;IAC7B,uBAAuB,EAAE,OAAO,CAAC;IACjC,cAAc,EAAE,MAAM,CAAC;IACvB,SAAS,EAAE,MAAM,CAAC;IAClB,gBAAgB,EAAE,MAAM,EAAE,CAAC;CAC9B;AAID,wBAAsB,kBAAkB,CACpC,MAAM,EAAE,MAAM,EAAE,EAChB,OAAO,CAAC,EAAE;IAAE,eAAe,CAAC,EAAE,MAAM,CAAA;CAAE,GACvC,OAAO,CAAC,aAAa,CAAC,CAwCxB;AAED,MAAM,WAAW,SAAS;IACtB,OAAO,EAAE,OAAO,CAAC;IACjB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,KAAK,EAAE,QAAQ,CAAC;IAChB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,MAAM,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IACvB,IAAI,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IACrB,SAAS,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IAC1B,QAAQ,CAAC,EAAE;QAAE,CAAC,EAAE,MAAM,CAAC;QAAC,CAAC,EAAE,MAAM,CAAA;KAAE,CAAC;IACpC,WAAW,CAAC,EAAE;QAAE,CAAC,EAAE,MAAM,CAAC;QAAC,CAAC,EAAE,MAAM,CAAC;QAAC,IAAI,EAAE,MAAM,CAAA;KAAE,CAAC;IACrD,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,SAAS,CAAC,EAAE,UAAU,EAAE,CAAC;IACzB,OAAO,CAAC,EAAE,KAAK,CAAC;QAAE,KAAK,EAAE,MAAM,CAAC;QAAC,SAAS,EAAE,MAAM,CAAC;QAAC,IAAI,EAAE,MAAM,CAAC;QAAC,MAAM,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;QAAC,CAAC,CAAC,EAAE,MAAM,CAAC;QAAC,CAAC,CAAC,EAAE,MAAM,CAAA;KAAE,CAAC,CAAC;IACpH,SAAS,CAAC,EAAE,OAAO,CAAC;IACpB,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,UAAU,CAAC,EAAE,aAAa,CAAC;IAC3B,YAAY,CAAC,EAAE,eAAe,CAAC;IAC/B,OAAO,CAAC,EAAE,MAAM,CAAC;IAGjB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB,mBAAmB,CAAC,EAAE,MAAM,CAAC;IAC7B,uBAAuB,CAAC,EAAE,MAAM,CAAC;IACjC,QAAQ,CAAC,EAAE,MAAM,CAAC;CACrB;AAID,wBAAgB,UAAU,CAAC,OAAO,EAAE,UAAU,GAAG,QAAQ,CAQxD;AAED;;;;GAIG;AACH,wBAAgB,qBAAqB,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAI3D;AAED,MAAM,WAAW,QAAQ;IACrB,IAAI,EAAE,MAAM,CAAC;IACb,SAAS,EAAE;QAAE,CAAC,EAAE,MAAM,CAAC;QAAC,CAAC,EAAE,MAAM,CAAA;KAAE,CAAC;CACvC;AAMD,wBAAgB,YAAY,CAAC,SAAS,EAAE,SAAS,EAAE,KAAK,EAAE,MAAM,GAAG,QAAQ,GAAG,IAAI,CAoBjF;AAED,wBAAgB,sBAAsB,CAAC,KAAK,EAAE,QAAQ,EAAE,QAAQ,EAAE,WAAW,GAAG,MAAM,EAAE,CA2BvF;AAED;;;;;;;;;GASG;AACH,wBAAgB,4BAA4B,CACxC,MAAM,EAAE,MAAM,EACd,MAAM,EAAE,MAAM,EACd,QAAQ,EAAE,KAAK,GAAG,SAAS,EAC3B,gBAAgB,EAAE,MAAM,EACxB,WAAW,GAAE,MAAU,GACxB;IAAE,CAAC,EAAE,MAAM,CAAC;IAAC,CAAC,EAAE,MAAM,CAAA;CAAE,CAY1B;AAED,2DAA2D;AAC3D,eAAO,MAAM,qBAAqB,qCAA+B,CAAC;AAElE,wBAAsB,gBAAgB,IAAI,OAAO,CAAC,MAAM,GAAG,IAAI,CAAC,CA4E/D;AAED,wBAAgB,gBAAgB,CAAC,IAAI,EAAE;IACnC,MAAM,EAAE,MAAM,CAAC;IACf,KAAK,EAAE,QAAQ,CAAC;IAChB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,MAAM,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IACvB,IAAI,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IACrB,SAAS,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IAC1B,QAAQ,CAAC,EAAE;QAAE,CAAC,EAAE,MAAM,CAAC;QAAC,CAAC,EAAE,MAAM,CAAA;KAAE,CAAC;IACpC,WAAW,CAAC,EAAE;QAAE,CAAC,EAAE,MAAM,CAAC;QAAC,CAAC,EAAE,MAAM,CAAC;QAAC,IAAI,EAAE,MAAM,CAAA;KAAE,CAAC;IACrD,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,UAAU,CAAC,EAAE,aAAa,CAAC;IAC3B,YAAY,CAAC,EAAE,eAAe,CAAC;CAClC,GAAG,SAAS,CAQZ;AAED,wBAAgB,gBAAgB,CAAC,IAAI,EAAE;IACnC,KAAK,EAAE,QAAQ,CAAC;IAChB,MAAM,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IACvB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,SAAS,EAAE,UAAU,EAAE,CAAC;IACxB,UAAU,EAAE,MAAM,CAAC;IACnB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,OAAO,CAAC,EAAE,KAAK,CAAC;QAAE,KAAK,EAAE,MAAM,CAAC;QAAC,SAAS,EAAE,MAAM,CAAC;QAAC,IAAI,EAAE,MAAM,CAAC;QAAC,MAAM,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;QAAC,CAAC,CAAC,EAAE,MAAM,CAAC;QAAC,CAAC,CAAC,EAAE,MAAM,CAAA;KAAE,CAAC,CAAC;IACpH,SAAS,CAAC,EAAE,OAAO,CAAC;IACpB,UAAU,CAAC,EAAE,aAAa,CAAC;IAC3B,YAAY,CAAC,EAAE,eAAe,CAAC;CAClC,GAAG,SAAS,CAsBZ;AAyBD,MAAM,WAAW,YAAY;IACzB,KAAK,EAAE;QACH,GAAG,EAAE,OAAO,CAAC;QACb,UAAU,EAAE,MAAM,CAAC;QACnB,cAAc,EAAE,OAAO,CAAC;QACxB,UAAU,EAAE,KAAK,CAAC;YACd,KAAK,CAAC,EAAE,MAAM,CAAC;YACf,MAAM,CAAC,EAAE,MAAM,CAAC;YAChB,aAAa,CAAC,EAAE,MAAM,CAAC;YACvB,MAAM,CAAC,EAAE;gBAAE,CAAC,EAAE,MAAM,CAAC;gBAAC,CAAC,EAAE,MAAM,CAAC;gBAAC,KAAK,EAAE,MAAM,CAAC;gBAAC,MAAM,EAAE,MAAM,CAAA;aAAE,CAAC;SACpE,CAAC,CAAC;KACN,CAAC;IACF,aAAa,EAAE;QACX,GAAG,EAAE,OAAO,CAAC;QACb,UAAU,EAAE,MAAM,CAAC;QACnB,QAAQ,EAAE,KAAK,CAAC;YACZ,KAAK,CAAC,EAAE,MAAM,CAAC;YACf,MAAM,CAAC,EAAE,MAAM,CAAC;YAChB,KAAK,CAAC,EAAE;gBAAE,CAAC,EAAE,MAAM,CAAC;gBAAC,CAAC,EAAE,MAAM,CAAC;gBAAC,KAAK,EAAE,MAAM,CAAC;gBAAC,MAAM,EAAE,MAAM,CAAA;aAAE,CAAC;SACnE,CAAC,CAAC;KACN,CAAC;IACF,GAAG,EAAE;QACD,GAAG,EAAE,OAAO,CAAC;QACb,UAAU,EAAE,MAAM,CAAC;QACnB,UAAU,EAAE,KAAK,CAAC;YACd,IAAI,EAAE,MAAM,CAAC;YACb,IAAI,EAAE,CAAC,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,CAAC,CAAC;YACvC,IAAI,EAAE,MAAM,CAAC;SAChB,CAAC,CAAC;QACH,YAAY,EAAE;YAAE,IAAI,EAAE,MAAM,CAAC;YAAC,KAAK,EAAE,MAAM,CAAA;SAAE,GAAG,IAAI,CAAC;KACxD,CAAC;CACL;AAED,wBAAgB,qBAAqB,IAAI,YAAY,CAMpD;AA+BD,wBAAgB,kBAAkB,CAC9B,SAAS,EAAE,SAAS,EACpB,KAAK,EAAE,MAAM,GACd;IAAE,IAAI,EAAE,MAAM,CAAC;IAAC,KAAK,EAAE,MAAM,CAAA;CAAE,GAAG,IAAI,CAgBxC;AAsfD,wBAAgB,YAAY,CAAC,SAAS,EAAE,SAAS;IAAE,MAAM,EAAE,MAAM,CAAC;IAAC,QAAQ,CAAC,EAAE,MAAM,CAAA;CAAE,EAAE,GAAG,OAAO,CAIjG;AAwbD,wBAAsB,GAAG,CAAC,OAAO,EAAE,UAAU,GAAG,OAAO,CAAC,SAAS,CAAC,CA2oBjE"}
|
package/build/pro/tap.js
CHANGED
|
@@ -7,6 +7,7 @@ import { compareScreenshots } from "./screenshot-diff.js";
|
|
|
7
7
|
import { scanMetroPorts, fetchDevices, selectMainDevice } from "../core/metro.js";
|
|
8
8
|
import { connectToDevice, clearReconnectionSuppression, getConnectedAppByDevice } from "../core/connection.js";
|
|
9
9
|
import { notifyDriverMissing } from "../core/logbox.js";
|
|
10
|
+
import { captureFailureArtifact } from "../core/failureArtifact.js";
|
|
10
11
|
export function buildVerificationExplanation(v) {
|
|
11
12
|
const pct = (rate) => (rate * 100).toFixed(1) + "%";
|
|
12
13
|
if (v.meaningful && !v.transientChangeDetected) {
|
|
@@ -278,27 +279,112 @@ function buildErrorMessage(query) {
|
|
|
278
279
|
parts.push(`component="${query.component}"`);
|
|
279
280
|
return `No element found matching ${parts.join(", ")}`;
|
|
280
281
|
}
|
|
282
|
+
export function makeEmptyEvidenceSink() {
|
|
283
|
+
return {
|
|
284
|
+
fiber: { ran: false, durationMs: 0, metroConnected: false, pressables: [] },
|
|
285
|
+
accessibility: { ran: false, durationMs: 0, elements: [] },
|
|
286
|
+
ocr: { ran: false, durationMs: 0, detections: [], closestMatch: null }
|
|
287
|
+
};
|
|
288
|
+
}
|
|
289
|
+
function ocrSimilarity(a, b) {
|
|
290
|
+
if (!a || !b)
|
|
291
|
+
return 0;
|
|
292
|
+
if (a === b)
|
|
293
|
+
return 1;
|
|
294
|
+
// Dice coefficient over character bigrams (case/space-insensitive)
|
|
295
|
+
const bigrams = (s) => {
|
|
296
|
+
const m = new Map();
|
|
297
|
+
if (s.length < 2) {
|
|
298
|
+
if (s)
|
|
299
|
+
m.set(s, 1);
|
|
300
|
+
return m;
|
|
301
|
+
}
|
|
302
|
+
for (let i = 0; i < s.length - 1; i++) {
|
|
303
|
+
const bg = s.slice(i, i + 2);
|
|
304
|
+
m.set(bg, (m.get(bg) ?? 0) + 1);
|
|
305
|
+
}
|
|
306
|
+
return m;
|
|
307
|
+
};
|
|
308
|
+
const ag = bigrams(a);
|
|
309
|
+
const bg = bigrams(b);
|
|
310
|
+
let intersection = 0;
|
|
311
|
+
for (const [k, va] of ag) {
|
|
312
|
+
const vb = bg.get(k);
|
|
313
|
+
if (vb)
|
|
314
|
+
intersection += Math.min(va, vb);
|
|
315
|
+
}
|
|
316
|
+
const total = Array.from(ag.values()).reduce((s, v) => s + v, 0)
|
|
317
|
+
+ Array.from(bg.values()).reduce((s, v) => s + v, 0);
|
|
318
|
+
if (!total)
|
|
319
|
+
return 0;
|
|
320
|
+
return (2 * intersection) / total;
|
|
321
|
+
}
|
|
322
|
+
export function findClosestOcrText(ocrResult, query) {
|
|
323
|
+
if (!query)
|
|
324
|
+
return null;
|
|
325
|
+
const needle = normalizeForMatch(query);
|
|
326
|
+
if (!needle)
|
|
327
|
+
return null;
|
|
328
|
+
const candidates = [];
|
|
329
|
+
if (ocrResult?.words?.length)
|
|
330
|
+
candidates.push(...ocrResult.words.map(w => ({ text: w.text })));
|
|
331
|
+
if (ocrResult?.lines?.length)
|
|
332
|
+
candidates.push(...ocrResult.lines.map(l => ({ text: l.text })));
|
|
333
|
+
if (!candidates.length)
|
|
334
|
+
return null;
|
|
335
|
+
let best = null;
|
|
336
|
+
for (const c of candidates) {
|
|
337
|
+
const norm = normalizeForMatch(c.text);
|
|
338
|
+
if (!norm)
|
|
339
|
+
continue;
|
|
340
|
+
const score = ocrSimilarity(needle, norm);
|
|
341
|
+
if (!best || score > best.score)
|
|
342
|
+
best = { text: c.text, score };
|
|
343
|
+
}
|
|
344
|
+
return best;
|
|
345
|
+
}
|
|
281
346
|
// --- Strategy Functions ---
|
|
282
|
-
async function tryFiberStrategy(query, index, maxTraversalDepth) {
|
|
283
|
-
|
|
284
|
-
|
|
285
|
-
|
|
286
|
-
|
|
287
|
-
|
|
288
|
-
|
|
289
|
-
|
|
290
|
-
|
|
291
|
-
depthAttempts
|
|
292
|
-
|
|
293
|
-
|
|
294
|
-
|
|
295
|
-
const result = await tryFiberAtDepth(query, index, depth);
|
|
296
|
-
if (result.success || result.matches) {
|
|
297
|
-
return result;
|
|
347
|
+
async function tryFiberStrategy(query, index, maxTraversalDepth, sink) {
|
|
348
|
+
if (sink) {
|
|
349
|
+
sink.fiber.ran = true;
|
|
350
|
+
sink.fiber.metroConnected = connectedApps.size > 0;
|
|
351
|
+
}
|
|
352
|
+
const startedAt = Date.now();
|
|
353
|
+
try {
|
|
354
|
+
// Retry with increasing depth if the initial traversal finds nothing
|
|
355
|
+
const baseDepth = maxTraversalDepth ?? 15;
|
|
356
|
+
const depthAttempts = [baseDepth];
|
|
357
|
+
// Only add deeper retries if user didn't explicitly set a high depth
|
|
358
|
+
if (baseDepth <= 15) {
|
|
359
|
+
depthAttempts.push(30, 45);
|
|
298
360
|
}
|
|
299
|
-
|
|
361
|
+
else if (baseDepth <= 30) {
|
|
362
|
+
depthAttempts.push(baseDepth * 2);
|
|
363
|
+
}
|
|
364
|
+
let lastResult = null;
|
|
365
|
+
for (const depth of depthAttempts) {
|
|
366
|
+
const result = await tryFiberAtDepth(query, index, depth);
|
|
367
|
+
if (sink && result.matches?.length) {
|
|
368
|
+
sink.fiber.pressables = result.matches.slice(0, 50).map(m => ({
|
|
369
|
+
label: m.text || undefined,
|
|
370
|
+
testID: m.testID ?? undefined,
|
|
371
|
+
componentName: m.component,
|
|
372
|
+
bounds: (m.x !== undefined && m.y !== undefined)
|
|
373
|
+
? { x: m.x, y: m.y, width: 0, height: 0 }
|
|
374
|
+
: undefined
|
|
375
|
+
}));
|
|
376
|
+
}
|
|
377
|
+
if (result.success || result.matches) {
|
|
378
|
+
return result;
|
|
379
|
+
}
|
|
380
|
+
lastResult = result;
|
|
381
|
+
}
|
|
382
|
+
return lastResult;
|
|
383
|
+
}
|
|
384
|
+
finally {
|
|
385
|
+
if (sink)
|
|
386
|
+
sink.fiber.durationMs = Date.now() - startedAt;
|
|
300
387
|
}
|
|
301
|
-
return lastResult;
|
|
302
388
|
}
|
|
303
389
|
async function tryFiberAtDepth(query, index, maxTraversalDepth) {
|
|
304
390
|
try {
|
|
@@ -376,7 +462,10 @@ async function tryFiberAtDepth(query, index, maxTraversalDepth) {
|
|
|
376
462
|
};
|
|
377
463
|
}
|
|
378
464
|
}
|
|
379
|
-
async function tryAccessibilityStrategy(query, index, platform, udid) {
|
|
465
|
+
async function tryAccessibilityStrategy(query, index, platform, udid, sink) {
|
|
466
|
+
if (sink)
|
|
467
|
+
sink.accessibility.ran = true;
|
|
468
|
+
const startedAt = Date.now();
|
|
380
469
|
try {
|
|
381
470
|
const hasTestID = !!query.testID;
|
|
382
471
|
const hasText = !!query.text;
|
|
@@ -426,6 +515,15 @@ async function tryAccessibilityStrategy(query, index, platform, udid) {
|
|
|
426
515
|
}, udid);
|
|
427
516
|
}
|
|
428
517
|
}
|
|
518
|
+
if (sink && result.allMatches?.length) {
|
|
519
|
+
sink.accessibility.elements = result.allMatches.slice(0, 50).map(el => ({
|
|
520
|
+
label: el.label || undefined,
|
|
521
|
+
testID: el.identifier || undefined,
|
|
522
|
+
frame: el.frame
|
|
523
|
+
? { x: el.frame.x, y: el.frame.y, width: el.frame.width, height: el.frame.height }
|
|
524
|
+
: undefined
|
|
525
|
+
}));
|
|
526
|
+
}
|
|
429
527
|
if (!result.success || !result.allMatches || result.allMatches.length === 0) {
|
|
430
528
|
return {
|
|
431
529
|
success: false,
|
|
@@ -467,6 +565,15 @@ async function tryAccessibilityStrategy(query, index, platform, udid) {
|
|
|
467
565
|
index
|
|
468
566
|
});
|
|
469
567
|
}
|
|
568
|
+
if (sink && result.allMatches?.length) {
|
|
569
|
+
sink.accessibility.elements = result.allMatches.slice(0, 50).map(el => ({
|
|
570
|
+
label: el.text || el.contentDesc || undefined,
|
|
571
|
+
testID: el.resourceId || undefined,
|
|
572
|
+
frame: el.bounds
|
|
573
|
+
? { x: el.bounds.left, y: el.bounds.top, width: el.bounds.width, height: el.bounds.height }
|
|
574
|
+
: undefined
|
|
575
|
+
}));
|
|
576
|
+
}
|
|
470
577
|
if (!result.success || !result.allMatches || result.allMatches.length === 0) {
|
|
471
578
|
return {
|
|
472
579
|
success: false,
|
|
@@ -497,8 +604,15 @@ async function tryAccessibilityStrategy(query, index, platform, udid) {
|
|
|
497
604
|
reason: `Accessibility strategy error: ${err instanceof Error ? err.message : String(err)}`
|
|
498
605
|
};
|
|
499
606
|
}
|
|
607
|
+
finally {
|
|
608
|
+
if (sink)
|
|
609
|
+
sink.accessibility.durationMs = Date.now() - startedAt;
|
|
610
|
+
}
|
|
500
611
|
}
|
|
501
|
-
async function tryOcrStrategy(query, platform, udid) {
|
|
612
|
+
async function tryOcrStrategy(query, platform, udid, sink) {
|
|
613
|
+
if (sink)
|
|
614
|
+
sink.ocr.ran = true;
|
|
615
|
+
const ocrStartedAt = Date.now();
|
|
502
616
|
try {
|
|
503
617
|
const searchText = query.text;
|
|
504
618
|
if (!searchText) {
|
|
@@ -534,6 +648,18 @@ async function tryOcrStrategy(query, platform, udid) {
|
|
|
534
648
|
scaleFactor,
|
|
535
649
|
platform
|
|
536
650
|
});
|
|
651
|
+
if (sink && ocrResult) {
|
|
652
|
+
const allDetections = [
|
|
653
|
+
...(ocrResult.words ?? []),
|
|
654
|
+
...(ocrResult.lines ?? [])
|
|
655
|
+
];
|
|
656
|
+
sink.ocr.detections = allDetections.slice(0, 100).map(r => ({
|
|
657
|
+
text: r.text,
|
|
658
|
+
bbox: [r.bbox.x0, r.bbox.y0, r.bbox.x1 - r.bbox.x0, r.bbox.y1 - r.bbox.y0],
|
|
659
|
+
conf: r.confidence ?? 0
|
|
660
|
+
}));
|
|
661
|
+
sink.ocr.closestMatch = findClosestOcrText(ocrResult, searchText);
|
|
662
|
+
}
|
|
537
663
|
const match = findOcrMatch(ocrResult, searchText);
|
|
538
664
|
if (!match) {
|
|
539
665
|
return {
|
|
@@ -574,6 +700,10 @@ async function tryOcrStrategy(query, platform, udid) {
|
|
|
574
700
|
reason: `OCR strategy error: ${err instanceof Error ? err.message : String(err)}`
|
|
575
701
|
};
|
|
576
702
|
}
|
|
703
|
+
finally {
|
|
704
|
+
if (sink)
|
|
705
|
+
sink.ocr.durationMs = Date.now() - ocrStartedAt;
|
|
706
|
+
}
|
|
577
707
|
}
|
|
578
708
|
async function tryCoordinateStrategy(pixelX, pixelY, platform, lastScreenshot, udid) {
|
|
579
709
|
try {
|
|
@@ -810,7 +940,7 @@ async function verifyAndCapture(platform, shouldVerify, shouldScreenshot, before
|
|
|
810
940
|
groupId: verifyGroupId,
|
|
811
941
|
metadata: { phase: "after", changeRate: verification?.changeRate }
|
|
812
942
|
});
|
|
813
|
-
return { screenshot, verification };
|
|
943
|
+
return { screenshot, verification, afterBuffer: after.buffer, afterWithMarkerBuffer: afterWithMarker };
|
|
814
944
|
}
|
|
815
945
|
// --- Burst Capture ---
|
|
816
946
|
const BURST_FRAME_COUNT = 4;
|
|
@@ -902,7 +1032,122 @@ async function burstCaptureAndVerify(platform, beforeBuffer, udid, beforeScaleFa
|
|
|
902
1032
|
peakFrame: analysis.peakFrame
|
|
903
1033
|
})
|
|
904
1034
|
};
|
|
905
|
-
|
|
1035
|
+
const lastFrameIdx = frames.length - 1;
|
|
1036
|
+
return {
|
|
1037
|
+
screenshot,
|
|
1038
|
+
verification,
|
|
1039
|
+
afterBuffer: frames[lastFrameIdx],
|
|
1040
|
+
afterWithMarkerBuffer: markedFrames[lastFrameIdx]
|
|
1041
|
+
};
|
|
1042
|
+
}
|
|
1043
|
+
async function captureTapArtifact(ctx) {
|
|
1044
|
+
try {
|
|
1045
|
+
const { getServerVersion } = await import("../core/telemetry.js");
|
|
1046
|
+
const strategyChain = ctx.attempted.map(a => `${a.strategy}:${a.reason.slice(0, 40)}`).join("|");
|
|
1047
|
+
const result = await captureFailureArtifact({
|
|
1048
|
+
outcome: ctx.outcome,
|
|
1049
|
+
predicate: ctx.query,
|
|
1050
|
+
errorMessage: ctx.errorMessage,
|
|
1051
|
+
errorCategory: ctx.errorCategory,
|
|
1052
|
+
strategyChain,
|
|
1053
|
+
sessionId: "",
|
|
1054
|
+
version: getServerVersion(),
|
|
1055
|
+
changeRate: ctx.verification?.changeRate,
|
|
1056
|
+
meaningful: ctx.verification?.meaningful,
|
|
1057
|
+
senses: ctx.evidence
|
|
1058
|
+
? {
|
|
1059
|
+
ocr: ctx.evidence.ocr,
|
|
1060
|
+
fiber: {
|
|
1061
|
+
ran: ctx.evidence.fiber.ran,
|
|
1062
|
+
durationMs: ctx.evidence.fiber.durationMs,
|
|
1063
|
+
metroConnected: ctx.evidence.fiber.metroConnected,
|
|
1064
|
+
pressables: ctx.evidence.fiber.pressables.map(p => ({
|
|
1065
|
+
label: p.label,
|
|
1066
|
+
testID: p.testID,
|
|
1067
|
+
componentName: p.componentName,
|
|
1068
|
+
bounds: p.bounds
|
|
1069
|
+
? [p.bounds.x, p.bounds.y, p.bounds.width, p.bounds.height]
|
|
1070
|
+
: undefined
|
|
1071
|
+
}))
|
|
1072
|
+
},
|
|
1073
|
+
accessibility: {
|
|
1074
|
+
ran: ctx.evidence.accessibility.ran,
|
|
1075
|
+
durationMs: ctx.evidence.accessibility.durationMs,
|
|
1076
|
+
elements: ctx.evidence.accessibility.elements.map(el => ({
|
|
1077
|
+
label: el.label,
|
|
1078
|
+
testID: el.testID,
|
|
1079
|
+
frame: el.frame
|
|
1080
|
+
? [el.frame.x, el.frame.y, el.frame.width, el.frame.height]
|
|
1081
|
+
: undefined
|
|
1082
|
+
}))
|
|
1083
|
+
}
|
|
1084
|
+
}
|
|
1085
|
+
: {
|
|
1086
|
+
ocr: { ran: false, durationMs: 0, detections: [], closestMatch: null },
|
|
1087
|
+
fiber: {
|
|
1088
|
+
ran: ctx.attempted.some(a => a.strategy === "fiber"),
|
|
1089
|
+
durationMs: 0,
|
|
1090
|
+
metroConnected: connectedApps.size > 0,
|
|
1091
|
+
pressables: (ctx.fiberMatches || []).slice(0, 10).map(m => ({
|
|
1092
|
+
label: m.text || undefined,
|
|
1093
|
+
testID: m.testID || undefined,
|
|
1094
|
+
componentName: m.component
|
|
1095
|
+
}))
|
|
1096
|
+
},
|
|
1097
|
+
accessibility: {
|
|
1098
|
+
ran: ctx.attempted.some(a => a.strategy === "accessibility"),
|
|
1099
|
+
durationMs: 0,
|
|
1100
|
+
elements: []
|
|
1101
|
+
}
|
|
1102
|
+
},
|
|
1103
|
+
chosenTapPoint: ctx.chosenTapPoint ?? null,
|
|
1104
|
+
chosenElement: null,
|
|
1105
|
+
screenshots: {
|
|
1106
|
+
before: ctx.screenshotBuffer ?? null,
|
|
1107
|
+
after: ctx.afterBuffer ?? null,
|
|
1108
|
+
afterWithMarker: ctx.afterWithMarker ?? null
|
|
1109
|
+
},
|
|
1110
|
+
deviceMeta: {
|
|
1111
|
+
platform: ctx.platform,
|
|
1112
|
+
driver: ctx.iosDriver,
|
|
1113
|
+
screenSize: { w: ctx.screenshotMeta?.width || 0, h: ctx.screenshotMeta?.height || 0 }
|
|
1114
|
+
}
|
|
1115
|
+
});
|
|
1116
|
+
return result.signals;
|
|
1117
|
+
}
|
|
1118
|
+
catch {
|
|
1119
|
+
return undefined;
|
|
1120
|
+
}
|
|
1121
|
+
}
|
|
1122
|
+
function attachArtifactSignals(result, signals) {
|
|
1123
|
+
if (!signals)
|
|
1124
|
+
return result;
|
|
1125
|
+
if (signals.artifactKey)
|
|
1126
|
+
result.artifactKey = signals.artifactKey;
|
|
1127
|
+
if (signals.ocrClosestMatch)
|
|
1128
|
+
result.ocrClosestMatch = signals.ocrClosestMatch;
|
|
1129
|
+
if (signals.fiberPressableCount)
|
|
1130
|
+
result.fiberPressableCount = signals.fiberPressableCount;
|
|
1131
|
+
if (signals.accessibilityMatchCount)
|
|
1132
|
+
result.accessibilityMatchCount = signals.accessibilityMatchCount;
|
|
1133
|
+
if (signals.appRoute)
|
|
1134
|
+
result.appRoute = signals.appRoute;
|
|
1135
|
+
// Append agent-facing hints to error message
|
|
1136
|
+
if (result.error) {
|
|
1137
|
+
if (signals.ocrClosestMatch) {
|
|
1138
|
+
result.error = `${result.error}\nClosest OCR match: ${signals.ocrClosestMatch}`;
|
|
1139
|
+
}
|
|
1140
|
+
if (signals.nearbyPressables.length > 0) {
|
|
1141
|
+
const labels = signals.nearbyPressables
|
|
1142
|
+
.map(p => p.testID || p.label)
|
|
1143
|
+
.filter(Boolean)
|
|
1144
|
+
.slice(0, 3)
|
|
1145
|
+
.join(", ");
|
|
1146
|
+
if (labels)
|
|
1147
|
+
result.error = `${result.error}\nNearby pressables: ${labels}`;
|
|
1148
|
+
}
|
|
1149
|
+
}
|
|
1150
|
+
return result;
|
|
906
1151
|
}
|
|
907
1152
|
// --- Orchestrator ---
|
|
908
1153
|
export async function tap(options) {
|
|
@@ -1179,6 +1424,7 @@ export async function tap(options) {
|
|
|
1179
1424
|
// Determine strategies
|
|
1180
1425
|
const strategies = getAvailableStrategies(query, strategy);
|
|
1181
1426
|
const attempted = [];
|
|
1427
|
+
const evidence = makeEmptyEvidenceSink();
|
|
1182
1428
|
// Early UI driver check for iOS — fail fast instead of falling through every strategy
|
|
1183
1429
|
const UI_DRIVER_REQUIRED_STRATEGIES = ["accessibility", "ocr", "coordinate"];
|
|
1184
1430
|
let uiDriverMissing = false;
|
|
@@ -1246,13 +1492,13 @@ export async function tap(options) {
|
|
|
1246
1492
|
try {
|
|
1247
1493
|
switch (strat) {
|
|
1248
1494
|
case "fiber":
|
|
1249
|
-
result = await withTimeout(tryFiberStrategy(query, index, maxTraversalDepth), budget, `fiber`);
|
|
1495
|
+
result = await withTimeout(tryFiberStrategy(query, index, maxTraversalDepth, evidence), budget, `fiber`);
|
|
1250
1496
|
break;
|
|
1251
1497
|
case "accessibility":
|
|
1252
|
-
result = await withTimeout(tryAccessibilityStrategy(query, index, platform, targetUdid), budget, `accessibility`);
|
|
1498
|
+
result = await withTimeout(tryAccessibilityStrategy(query, index, platform, targetUdid, evidence), budget, `accessibility`);
|
|
1253
1499
|
break;
|
|
1254
1500
|
case "ocr":
|
|
1255
|
-
result = await withTimeout(tryOcrStrategy(query, platform, targetUdid), budget, `ocr`);
|
|
1501
|
+
result = await withTimeout(tryOcrStrategy(query, platform, targetUdid, evidence), budget, `ocr`);
|
|
1256
1502
|
break;
|
|
1257
1503
|
case "coordinate":
|
|
1258
1504
|
result = await withTimeout(tryCoordinateStrategy(query.x, query.y, platform, app?.lastScreenshot, targetUdid), budget, `coordinate`);
|
|
@@ -1271,6 +1517,8 @@ export async function tap(options) {
|
|
|
1271
1517
|
if (result.success) {
|
|
1272
1518
|
let screenshot;
|
|
1273
1519
|
let verification;
|
|
1520
|
+
let afterBuffer;
|
|
1521
|
+
let afterWithMarkerBuffer;
|
|
1274
1522
|
let dprForMarker;
|
|
1275
1523
|
if (platform === "ios") {
|
|
1276
1524
|
try {
|
|
@@ -1290,10 +1538,10 @@ export async function tap(options) {
|
|
|
1290
1538
|
devicePixelRatio: dprForMarker
|
|
1291
1539
|
});
|
|
1292
1540
|
if (options.burst && canVerify && beforeBuffer) {
|
|
1293
|
-
({ screenshot, verification } = await burstCaptureAndVerify(platform, beforeBuffer, targetUdid, beforeScaleFactor, strategyMarker));
|
|
1541
|
+
({ screenshot, verification, afterBuffer, afterWithMarkerBuffer } = await burstCaptureAndVerify(platform, beforeBuffer, targetUdid, beforeScaleFactor, strategyMarker));
|
|
1294
1542
|
}
|
|
1295
1543
|
else {
|
|
1296
|
-
({ screenshot, verification } = await verifyAndCapture(platform, canVerify, shouldScreenshot, beforeBuffer, targetUdid, beforeScaleFactor, strategyMarker));
|
|
1544
|
+
({ screenshot, verification, afterBuffer, afterWithMarkerBuffer } = await verifyAndCapture(platform, canVerify, shouldScreenshot, beforeBuffer, targetUdid, beforeScaleFactor, strategyMarker));
|
|
1297
1545
|
}
|
|
1298
1546
|
if (screenshot && app) {
|
|
1299
1547
|
app.lastScreenshot = {
|
|
@@ -1302,7 +1550,7 @@ export async function tap(options) {
|
|
|
1302
1550
|
scaleFactor: screenshot.scaleFactor
|
|
1303
1551
|
};
|
|
1304
1552
|
}
|
|
1305
|
-
|
|
1553
|
+
const successResult = formatTapSuccess({
|
|
1306
1554
|
method: strat,
|
|
1307
1555
|
query,
|
|
1308
1556
|
pressed: result.pressed,
|
|
@@ -1316,6 +1564,27 @@ export async function tap(options) {
|
|
|
1316
1564
|
screenshot,
|
|
1317
1565
|
verification
|
|
1318
1566
|
});
|
|
1567
|
+
// Capture an artifact for "successful but unmeaningful" taps so we can
|
|
1568
|
+
// diagnose taps that landed wrong or hit non-responsive elements.
|
|
1569
|
+
if (verification && verification.changeRate < 0.001) {
|
|
1570
|
+
const unmeaningfulSignals = await captureTapArtifact({
|
|
1571
|
+
query,
|
|
1572
|
+
outcome: "unmeaningful",
|
|
1573
|
+
attempted: [...attempted, { strategy: strat, reason: "success" }],
|
|
1574
|
+
platform,
|
|
1575
|
+
iosDriver: platform === "ios" ? (process.env.IOS_DRIVER?.toLowerCase() || "idb") : undefined,
|
|
1576
|
+
deviceName,
|
|
1577
|
+
screenshotMeta: screenshot ? { width: screenshot.width, height: screenshot.height } : undefined,
|
|
1578
|
+
screenshotBuffer: beforeBuffer,
|
|
1579
|
+
afterBuffer: afterBuffer ?? null,
|
|
1580
|
+
afterWithMarker: afterWithMarkerBuffer ?? null,
|
|
1581
|
+
chosenTapPoint: strategyMarker ? { x: strategyMarker.x, y: strategyMarker.y } : null,
|
|
1582
|
+
verification,
|
|
1583
|
+
evidence
|
|
1584
|
+
});
|
|
1585
|
+
attachArtifactSignals(successResult, unmeaningfulSignals);
|
|
1586
|
+
}
|
|
1587
|
+
return successResult;
|
|
1319
1588
|
}
|
|
1320
1589
|
attempted.push({ strategy: strat, reason: result.reason });
|
|
1321
1590
|
// If fiber found an element with measured coordinates, do a native tap directly
|
|
@@ -1445,7 +1714,7 @@ export async function tap(options) {
|
|
|
1445
1714
|
scaleFactor: failScreenshot.scaleFactor
|
|
1446
1715
|
};
|
|
1447
1716
|
}
|
|
1448
|
-
|
|
1717
|
+
const failureResult = formatTapFailure({
|
|
1449
1718
|
query,
|
|
1450
1719
|
attempted,
|
|
1451
1720
|
error: hitTimeout ? `Tap timed out after ${elapsed}ms (budget ${TAP_TIMEOUT_MS}ms)` : undefined,
|
|
@@ -1453,6 +1722,21 @@ export async function tap(options) {
|
|
|
1453
1722
|
device: deviceName,
|
|
1454
1723
|
screenshot: failScreenshot
|
|
1455
1724
|
});
|
|
1725
|
+
const failSignals = await captureTapArtifact({
|
|
1726
|
+
query,
|
|
1727
|
+
outcome: "failure",
|
|
1728
|
+
errorMessage: failureResult.error,
|
|
1729
|
+
attempted,
|
|
1730
|
+
platform,
|
|
1731
|
+
iosDriver: platform === "ios" ? (process.env.IOS_DRIVER?.toLowerCase() || "idb") : undefined,
|
|
1732
|
+
deviceName,
|
|
1733
|
+
screenshotMeta: failScreenshot ? { width: failScreenshot.width, height: failScreenshot.height } : undefined,
|
|
1734
|
+
screenshotBuffer: beforeBuffer,
|
|
1735
|
+
afterWithMarker: null,
|
|
1736
|
+
chosenTapPoint: null,
|
|
1737
|
+
evidence
|
|
1738
|
+
});
|
|
1739
|
+
return attachArtifactSignals(failureResult, failSignals);
|
|
1456
1740
|
}
|
|
1457
1741
|
function buildSuggestion(query, triedStrategies, platform) {
|
|
1458
1742
|
const suggestions = [];
|