visor-ai 0.2.8 → 0.2.9
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/adapters.js +224 -22
- package/dist/appMap.js +1902 -137
- package/dist/cli.js +304 -45
- package/dist/localRuntime.js +8 -0
- package/package.json +1 -1
package/dist/appMap.js
CHANGED
|
@@ -22,6 +22,10 @@ const RISKY_WORDS = [
|
|
|
22
22
|
];
|
|
23
23
|
const INPUT_WORDS = ['password', 'email', 'username', 'search', 'input', 'text field', 'textfield'];
|
|
24
24
|
const AUTH_WORDS = ['login', 'log in', 'sign in', 'password', 'username', 'auth', 'authentication'];
|
|
25
|
+
const DEFAULT_ACTION_SETTLE_MS = 400;
|
|
26
|
+
const DEFAULT_ACTION_SETTLE_POLL_MS = 400;
|
|
27
|
+
const DEFAULT_CRAWL_SETTLE_MS = 1500;
|
|
28
|
+
const DEFAULT_CRAWL_SETTLE_POLL_MS = 300;
|
|
25
29
|
function envNoMap() {
|
|
26
30
|
const raw = process.env.VISOR_NO_MAP ?? process.env.VISOR_DISABLE_MAP;
|
|
27
31
|
return ['1', 'true', 'yes', 'on'].includes(String(raw ?? '').trim().toLowerCase());
|
|
@@ -33,9 +37,21 @@ function resolveOptions(platform, options) {
|
|
|
33
37
|
rootDir: options?.rootDir ?? process.env.VISOR_APP_MAP_DIR ?? path.join(process.cwd(), '.visor', 'maps'),
|
|
34
38
|
appId: options?.appId ?? `default-${platform}`,
|
|
35
39
|
repairDepth: options?.repairDepth ?? 2,
|
|
36
|
-
repairTimeoutMs: options?.repairTimeoutMs ?? 30000
|
|
40
|
+
repairTimeoutMs: options?.repairTimeoutMs ?? 30000,
|
|
41
|
+
repair: options?.repair === true,
|
|
42
|
+
crawl: options?.crawl === true,
|
|
43
|
+
crawlDepth: options?.crawlDepth ?? 2,
|
|
44
|
+
crawlLimit: options?.crawlLimit ?? 24,
|
|
45
|
+
crawlSettleMs: nonNegativeNumber(options?.crawlSettleMs, DEFAULT_CRAWL_SETTLE_MS),
|
|
46
|
+
crawlSettlePollMs: positiveNumber(options?.crawlSettlePollMs, DEFAULT_CRAWL_SETTLE_POLL_MS)
|
|
37
47
|
};
|
|
38
48
|
}
|
|
49
|
+
function nonNegativeNumber(value, defaultValue) {
|
|
50
|
+
return typeof value === 'number' && Number.isFinite(value) && value >= 0 ? value : defaultValue;
|
|
51
|
+
}
|
|
52
|
+
function positiveNumber(value, defaultValue) {
|
|
53
|
+
return typeof value === 'number' && Number.isFinite(value) && value > 0 ? value : defaultValue;
|
|
54
|
+
}
|
|
39
55
|
function mapIdentity(platform, appId) {
|
|
40
56
|
return `${platform}:${appId}`;
|
|
41
57
|
}
|
|
@@ -67,8 +83,15 @@ function readMap(filePath, platform, appId) {
|
|
|
67
83
|
return { map: newMap(platform, appId), scrubbed: false };
|
|
68
84
|
}
|
|
69
85
|
const originalEdgeCount = parsed.edges.length;
|
|
86
|
+
let enrichedActions = false;
|
|
87
|
+
for (const variant of parsed.variants) {
|
|
88
|
+
if (!Array.isArray(variant.actions)) {
|
|
89
|
+
variant.actions = actionAffordancesForElements(variant.elements);
|
|
90
|
+
enrichedActions = true;
|
|
91
|
+
}
|
|
92
|
+
}
|
|
70
93
|
parsed.edges = parsed.edges.filter((edge) => !isValueBearingAction(edge.command, edge.args));
|
|
71
|
-
return { map: parsed, scrubbed: parsed.edges.length !== originalEdgeCount };
|
|
94
|
+
return { map: parsed, scrubbed: parsed.edges.length !== originalEdgeCount || enrichedActions };
|
|
72
95
|
}
|
|
73
96
|
catch {
|
|
74
97
|
return { map: newMap(platform, appId), scrubbed: false };
|
|
@@ -130,10 +153,20 @@ function attributeMap(input) {
|
|
|
130
153
|
const attrs = {};
|
|
131
154
|
const pattern = /([A-Za-z_:][-A-Za-z0-9_:.]*)\s*=\s*("([^"]*)"|'([^']*)')/g;
|
|
132
155
|
for (const match of input.matchAll(pattern)) {
|
|
133
|
-
attrs[match[1]] = match[3] ?? match[4] ?? '';
|
|
156
|
+
attrs[match[1]] = decodeXmlEntities(match[3] ?? match[4] ?? '');
|
|
134
157
|
}
|
|
135
158
|
return attrs;
|
|
136
159
|
}
|
|
160
|
+
function decodeXmlEntities(value) {
|
|
161
|
+
return value
|
|
162
|
+
.replace(/&#x([0-9a-f]+);/gi, (_, hex) => String.fromCodePoint(Number.parseInt(hex, 16)))
|
|
163
|
+
.replace(/&#(\d+);/g, (_, decimal) => String.fromCodePoint(Number.parseInt(decimal, 10)))
|
|
164
|
+
.replace(/"/g, '"')
|
|
165
|
+
.replace(/'/g, "'")
|
|
166
|
+
.replace(/</g, '<')
|
|
167
|
+
.replace(/>/g, '>')
|
|
168
|
+
.replace(/&/g, '&');
|
|
169
|
+
}
|
|
137
170
|
function normalizeLabel(value) {
|
|
138
171
|
return value
|
|
139
172
|
.replace(/[0-9a-f]{8,}/gi, '<id>')
|
|
@@ -146,6 +179,40 @@ function normalizeLabel(value) {
|
|
|
146
179
|
function unique(values) {
|
|
147
180
|
return Array.from(new Set(values.filter((value) => value.trim() !== '')));
|
|
148
181
|
}
|
|
182
|
+
function isScrollableTag(tag) {
|
|
183
|
+
const normalized = tag.toLowerCase();
|
|
184
|
+
return (normalized.includes('scrollview') ||
|
|
185
|
+
normalized.includes('scroll') ||
|
|
186
|
+
normalized.includes('list') ||
|
|
187
|
+
normalized.includes('table'));
|
|
188
|
+
}
|
|
189
|
+
function attrBool(attrs, key, defaultValue) {
|
|
190
|
+
const raw = attrs[key];
|
|
191
|
+
if (raw === undefined) {
|
|
192
|
+
return defaultValue;
|
|
193
|
+
}
|
|
194
|
+
return raw.trim().toLowerCase() === 'true';
|
|
195
|
+
}
|
|
196
|
+
function attrNumber(attrs, key) {
|
|
197
|
+
const parsed = Number(attrs[key]);
|
|
198
|
+
return Number.isFinite(parsed) ? parsed : null;
|
|
199
|
+
}
|
|
200
|
+
function expandSemanticLabels(labels) {
|
|
201
|
+
return labels.flatMap((label) => {
|
|
202
|
+
const lines = unique(label.split(/\n+/).map((line) => line.trim()));
|
|
203
|
+
if (lines.length === 1 && lines[0] !== label) {
|
|
204
|
+
return [label, lines[0]];
|
|
205
|
+
}
|
|
206
|
+
if (lines.length > 1 && label.split(/\n+/).filter((line) => line.trim() !== '').every((line) => line.trim() === lines[0])) {
|
|
207
|
+
return [label, lines[0]];
|
|
208
|
+
}
|
|
209
|
+
const rawLines = label.split(/\n+/).map((line) => line.trim()).filter(Boolean);
|
|
210
|
+
if (rawLines.length >= 3 && rawLines[0] === rawLines[rawLines.length - 1]) {
|
|
211
|
+
return [label, rawLines[0]];
|
|
212
|
+
}
|
|
213
|
+
return [label];
|
|
214
|
+
});
|
|
215
|
+
}
|
|
149
216
|
function isLikelyFormControl(tag, attrs) {
|
|
150
217
|
const haystack = [
|
|
151
218
|
tag,
|
|
@@ -172,6 +239,19 @@ function isLikelyFormControl(tag, attrs) {
|
|
|
172
239
|
'card'
|
|
173
240
|
].some((word) => haystack.includes(word));
|
|
174
241
|
}
|
|
242
|
+
function structuralElementTarget(tag, attrs) {
|
|
243
|
+
if (!isScrollableTag(tag)) {
|
|
244
|
+
return '';
|
|
245
|
+
}
|
|
246
|
+
return [
|
|
247
|
+
'scroll-container',
|
|
248
|
+
tag,
|
|
249
|
+
attrs.x ?? '',
|
|
250
|
+
attrs.y ?? '',
|
|
251
|
+
attrs.width ?? '',
|
|
252
|
+
attrs.height ?? ''
|
|
253
|
+
].join('=');
|
|
254
|
+
}
|
|
175
255
|
function redactLabel(value) {
|
|
176
256
|
return value
|
|
177
257
|
.replace(/[A-Z0-9._%+-]+@[A-Z0-9.-]+\.[A-Z]{2,}/gi, '<redacted-email>')
|
|
@@ -202,7 +282,7 @@ function extractElements(source) {
|
|
|
202
282
|
const attrs = attributeMap(match[2] ?? '');
|
|
203
283
|
const bodyText = (match[3] ?? '').trim();
|
|
204
284
|
const formControl = isLikelyFormControl(tag, attrs);
|
|
205
|
-
const rawLabels = unique([
|
|
285
|
+
const rawLabels = unique(expandSemanticLabels([
|
|
206
286
|
redactLabel(attrs.name ?? ''),
|
|
207
287
|
redactLabel(attrs.label ?? ''),
|
|
208
288
|
formControl ? '' : redactLabel(attrs.text ?? ''),
|
|
@@ -210,17 +290,19 @@ function extractElements(source) {
|
|
|
210
290
|
redactLabel(attrs['content-desc'] ?? ''),
|
|
211
291
|
redactLabel(attrs['contentDescription'] ?? ''),
|
|
212
292
|
formControl ? '' : redactLabel(bodyText)
|
|
213
|
-
]);
|
|
293
|
+
]));
|
|
214
294
|
const id = attrs['resource-id'] ?? attrs.id ?? attrs.identifier ?? '';
|
|
295
|
+
const structuralTarget = structuralElementTarget(tag, attrs);
|
|
215
296
|
const targets = unique([
|
|
216
297
|
...rawLabels,
|
|
217
298
|
...rawLabels.map((label) => `text=${label}`),
|
|
218
|
-
id ? `id=${id}` : ''
|
|
299
|
+
id ? `id=${id}` : '',
|
|
300
|
+
structuralTarget
|
|
219
301
|
]);
|
|
220
302
|
if (targets.length === 0) {
|
|
221
303
|
continue;
|
|
222
304
|
}
|
|
223
|
-
const stable = Boolean(rawLabels.length > 0 || id);
|
|
305
|
+
const stable = Boolean(rawLabels.length > 0 || id || structuralTarget);
|
|
224
306
|
const selector = id ? `id=${id}` : rawLabels[0] ?? targets[0];
|
|
225
307
|
elements.push({
|
|
226
308
|
tag,
|
|
@@ -228,7 +310,14 @@ function extractElements(source) {
|
|
|
228
310
|
targets,
|
|
229
311
|
labels: rawLabels,
|
|
230
312
|
stable,
|
|
231
|
-
safety: classifyControl(tag, rawLabels, attrs)
|
|
313
|
+
safety: classifyControl(tag, rawLabels, attrs),
|
|
314
|
+
enabled: attrBool(attrs, 'enabled', true),
|
|
315
|
+
visible: attrBool(attrs, 'visible', true),
|
|
316
|
+
clickable: attrBool(attrs, 'clickable', false),
|
|
317
|
+
x: attrNumber(attrs, 'x'),
|
|
318
|
+
y: attrNumber(attrs, 'y'),
|
|
319
|
+
width: attrNumber(attrs, 'width'),
|
|
320
|
+
height: attrNumber(attrs, 'height')
|
|
232
321
|
});
|
|
233
322
|
}
|
|
234
323
|
return elements;
|
|
@@ -263,14 +352,49 @@ function similarity(left, right) {
|
|
|
263
352
|
const containment = smaller === 0 ? 0 : intersection / smaller;
|
|
264
353
|
return containment >= 0.8 ? Math.max(jaccard, containment) : jaccard;
|
|
265
354
|
}
|
|
266
|
-
function
|
|
355
|
+
function identityKeys(elements, includeStatic) {
|
|
356
|
+
const navigationElements = new Set(navigationControlsForElements(elements).map((control) => control.element));
|
|
267
357
|
return unique(elements.flatMap((element) => {
|
|
358
|
+
if (navigationElements.has(element)) {
|
|
359
|
+
return [];
|
|
360
|
+
}
|
|
268
361
|
const tag = element.tag.toLowerCase();
|
|
269
|
-
|
|
362
|
+
if (!includeStatic && isStaticIdentityElement(element)) {
|
|
363
|
+
return [];
|
|
364
|
+
}
|
|
365
|
+
const labels = element.labels
|
|
366
|
+
.filter((label) => !isGlobalIdentityLabel(label, element))
|
|
367
|
+
.map(normalizeLabel);
|
|
270
368
|
const ids = element.targets.filter((target) => target.startsWith('id=')).map(normalizeLabel);
|
|
271
369
|
return [...labels, ...ids].map((value) => `${tag}:${element.safety}:${value}`);
|
|
272
370
|
})).sort();
|
|
273
371
|
}
|
|
372
|
+
function isStaticIdentityElement(element) {
|
|
373
|
+
const tag = element.tag.toLowerCase();
|
|
374
|
+
return element.safety === 'unknown' || tag.includes('statictext') || tag.includes('image') || isScrollableTag(tag);
|
|
375
|
+
}
|
|
376
|
+
function screenSimilarity(left, right) {
|
|
377
|
+
const leftControlKeys = identityKeys(left, false);
|
|
378
|
+
const rightControlKeys = identityKeys(right, false);
|
|
379
|
+
const fullScore = similarity(identityKeys(left, true), identityKeys(right, true));
|
|
380
|
+
if (leftControlKeys.length === 0 && rightControlKeys.length === 0) {
|
|
381
|
+
return fullScore;
|
|
382
|
+
}
|
|
383
|
+
return Math.max(fullScore, similarity(leftControlKeys, rightControlKeys));
|
|
384
|
+
}
|
|
385
|
+
function isGlobalIdentityLabel(label, element) {
|
|
386
|
+
const normalized = normalizeControlTarget(label);
|
|
387
|
+
if (isBundleLikeLabel(normalized)) {
|
|
388
|
+
return true;
|
|
389
|
+
}
|
|
390
|
+
if (['settings settings', 'notifications notifications'].includes(normalized)) {
|
|
391
|
+
return true;
|
|
392
|
+
}
|
|
393
|
+
return false;
|
|
394
|
+
}
|
|
395
|
+
function isBundleLikeLabel(normalized) {
|
|
396
|
+
return /^[a-z][a-z0-9_]*(?:\.[a-z][a-z0-9_]*){2,}$/.test(normalized);
|
|
397
|
+
}
|
|
274
398
|
function newVariant(map, observation) {
|
|
275
399
|
const screen = {
|
|
276
400
|
id: `screen_${map.screens.length + 1}`,
|
|
@@ -283,6 +407,7 @@ function newVariant(map, observation) {
|
|
|
283
407
|
normalized_fingerprint: observation.normalized_fingerprint,
|
|
284
408
|
elements: observation.elements,
|
|
285
409
|
element_keys: observation.element_keys,
|
|
410
|
+
actions: actionAffordancesForElements(observation.elements),
|
|
286
411
|
auth_required: observation.auth_required,
|
|
287
412
|
observations: 1,
|
|
288
413
|
confidence: 0.6,
|
|
@@ -296,16 +421,38 @@ function newVariant(map, observation) {
|
|
|
296
421
|
function upsertVariant(map, observation) {
|
|
297
422
|
let best = null;
|
|
298
423
|
let bestScore = 0;
|
|
424
|
+
const observedNavigationTarget = preferredNavigationTargetForElements(observation.elements);
|
|
425
|
+
const observedHasBottomNavigation = navigationControlsForElements(observation.elements).length >= 2;
|
|
426
|
+
const observedHasScrollableSurface = elementsHaveScrollableSurface(observation.elements);
|
|
427
|
+
const observedControlKeys = identityKeys(observation.elements, false);
|
|
299
428
|
for (const variant of map.variants) {
|
|
429
|
+
const variantNavigationTarget = preferredNavigationTarget(variant);
|
|
430
|
+
if (observedNavigationTarget &&
|
|
431
|
+
variantNavigationTarget &&
|
|
432
|
+
observedNavigationTarget !== variantNavigationTarget) {
|
|
433
|
+
continue;
|
|
434
|
+
}
|
|
435
|
+
if (observedHasScrollableSurface || hasScrollableSurface(variant)) {
|
|
436
|
+
const variantControlKeys = identityKeys(variant.elements, false);
|
|
437
|
+
if ((observedControlKeys.length > 0 || variantControlKeys.length > 0) &&
|
|
438
|
+
similarity(observedControlKeys, variantControlKeys) < 0.3) {
|
|
439
|
+
continue;
|
|
440
|
+
}
|
|
441
|
+
}
|
|
300
442
|
const score = variant.normalized_fingerprint === observation.normalized_fingerprint
|
|
301
443
|
? 1
|
|
302
|
-
:
|
|
444
|
+
: screenSimilarity(variant.elements, observation.elements);
|
|
303
445
|
if (score > bestScore) {
|
|
304
446
|
best = variant;
|
|
305
447
|
bestScore = score;
|
|
306
448
|
}
|
|
307
449
|
}
|
|
308
|
-
|
|
450
|
+
const requiredScore = observedHasBottomNavigation || (best ? hasBottomNavigation(best) : false)
|
|
451
|
+
? 0.62
|
|
452
|
+
: observedHasScrollableSurface || (best ? hasScrollableSurface(best) : false)
|
|
453
|
+
? 0.62
|
|
454
|
+
: 0.45;
|
|
455
|
+
if (!best || bestScore < requiredScore) {
|
|
309
456
|
return {
|
|
310
457
|
observation,
|
|
311
458
|
variant: newVariant(map, observation),
|
|
@@ -316,6 +463,7 @@ function upsertVariant(map, observation) {
|
|
|
316
463
|
best.normalized_fingerprint = observation.normalized_fingerprint;
|
|
317
464
|
best.elements = observation.elements;
|
|
318
465
|
best.element_keys = observation.element_keys;
|
|
466
|
+
best.actions = actionAffordancesForElements(observation.elements);
|
|
319
467
|
best.auth_required = observation.auth_required;
|
|
320
468
|
best.observations += 1;
|
|
321
469
|
best.confidence = Math.min(1, best.confidence + 0.05);
|
|
@@ -326,7 +474,7 @@ function upsertVariant(map, observation) {
|
|
|
326
474
|
created: false
|
|
327
475
|
};
|
|
328
476
|
}
|
|
329
|
-
async function
|
|
477
|
+
async function captureCurrentSource(context) {
|
|
330
478
|
const sourceDir = ensureDir(path.join(os.tmpdir(), 'visor-app-map-source'));
|
|
331
479
|
const sourcePath = path.join(sourceDir, `${process.pid}-${Date.now()}-${randomUUID()}.xml`);
|
|
332
480
|
const details = await context.adapter.source({ label: 'app-map-source', path: sourcePath });
|
|
@@ -344,10 +492,143 @@ async function observeCurrentScreen(context) {
|
|
|
344
492
|
catch {
|
|
345
493
|
// Source observations are transient; stale temp files should not fail a run.
|
|
346
494
|
}
|
|
347
|
-
|
|
495
|
+
return { source };
|
|
496
|
+
}
|
|
497
|
+
function isTransientObservation(observation) {
|
|
498
|
+
return observation.elements.length === 0 || observation.element_keys.length === 0;
|
|
499
|
+
}
|
|
500
|
+
async function captureCurrentObservation(context) {
|
|
501
|
+
let observation = null;
|
|
502
|
+
for (let attempt = 0; attempt < 5; attempt += 1) {
|
|
503
|
+
const captured = await captureCurrentSource(context);
|
|
504
|
+
observation = observeSource(captured.source);
|
|
505
|
+
if (!isTransientObservation(observation) || attempt === 4) {
|
|
506
|
+
break;
|
|
507
|
+
}
|
|
508
|
+
await delay(500);
|
|
509
|
+
}
|
|
510
|
+
return observation ?? observeSource('');
|
|
511
|
+
}
|
|
512
|
+
function observeScreenObservation(context, observation) {
|
|
513
|
+
const observed = upsertVariant(context.map, observation);
|
|
348
514
|
context.changed = true;
|
|
349
515
|
return observed;
|
|
350
516
|
}
|
|
517
|
+
async function observeCurrentScreen(context) {
|
|
518
|
+
return observeScreenObservation(context, await captureCurrentObservation(context));
|
|
519
|
+
}
|
|
520
|
+
async function observeAfterAction(context, target, options = {}) {
|
|
521
|
+
const observed = await observeCurrentScreen(context);
|
|
522
|
+
if (target && restoreMatch(observed, target)) {
|
|
523
|
+
return observed;
|
|
524
|
+
}
|
|
525
|
+
if (!shouldSettleObservation(observed.observation)) {
|
|
526
|
+
return observed;
|
|
527
|
+
}
|
|
528
|
+
const settleMs = nonNegativeNumber(options.settleMs, DEFAULT_ACTION_SETTLE_MS);
|
|
529
|
+
if (settleMs === 0) {
|
|
530
|
+
return observed;
|
|
531
|
+
}
|
|
532
|
+
const pollMs = positiveNumber(options.pollMs, DEFAULT_ACTION_SETTLE_POLL_MS);
|
|
533
|
+
let best = observed;
|
|
534
|
+
let previous = observed;
|
|
535
|
+
let elapsed = 0;
|
|
536
|
+
while (elapsed < settleMs) {
|
|
537
|
+
const waitMs = Math.min(pollMs, settleMs - elapsed);
|
|
538
|
+
await settleDelay(context, waitMs);
|
|
539
|
+
elapsed += waitMs;
|
|
540
|
+
let next;
|
|
541
|
+
try {
|
|
542
|
+
next = await observeCurrentScreen(context);
|
|
543
|
+
}
|
|
544
|
+
catch {
|
|
545
|
+
return best;
|
|
546
|
+
}
|
|
547
|
+
if (target && restoreMatch(next, target)) {
|
|
548
|
+
return next;
|
|
549
|
+
}
|
|
550
|
+
if (isRicherObservation(next.observation, best.observation)) {
|
|
551
|
+
best = next;
|
|
552
|
+
}
|
|
553
|
+
if (next.observation.fingerprint === previous.observation.fingerprint && !shouldSettleObservation(next.observation)) {
|
|
554
|
+
return isRicherObservation(next.observation, best.observation) ? next : best;
|
|
555
|
+
}
|
|
556
|
+
previous = next;
|
|
557
|
+
}
|
|
558
|
+
return best;
|
|
559
|
+
}
|
|
560
|
+
async function observeAfterCrawlAction(context) {
|
|
561
|
+
const observation = await captureCurrentObservation(context);
|
|
562
|
+
const settled = await settleCrawlObservation(context, observation);
|
|
563
|
+
return observeScreenObservation(context, settled);
|
|
564
|
+
}
|
|
565
|
+
async function settleCrawlObservation(context, observation) {
|
|
566
|
+
const settleMs = context.options.crawlSettleMs;
|
|
567
|
+
if (settleMs === 0) {
|
|
568
|
+
return observation;
|
|
569
|
+
}
|
|
570
|
+
const pollMs = context.options.crawlSettlePollMs;
|
|
571
|
+
let best = observation;
|
|
572
|
+
let previous = observation;
|
|
573
|
+
let elapsed = 0;
|
|
574
|
+
while (elapsed < settleMs) {
|
|
575
|
+
const waitMs = Math.min(pollMs, settleMs - elapsed);
|
|
576
|
+
await settleDelay(context, waitMs);
|
|
577
|
+
elapsed += waitMs;
|
|
578
|
+
let next;
|
|
579
|
+
try {
|
|
580
|
+
next = await captureCurrentObservation(context);
|
|
581
|
+
}
|
|
582
|
+
catch {
|
|
583
|
+
return best;
|
|
584
|
+
}
|
|
585
|
+
if (isRicherObservation(next, best)) {
|
|
586
|
+
best = next;
|
|
587
|
+
}
|
|
588
|
+
if (next.fingerprint === previous.fingerprint && !shouldSettleObservation(next)) {
|
|
589
|
+
return isRicherObservation(next, best) ? next : best;
|
|
590
|
+
}
|
|
591
|
+
previous = next;
|
|
592
|
+
}
|
|
593
|
+
return best;
|
|
594
|
+
}
|
|
595
|
+
async function settleDelay(context, ms) {
|
|
596
|
+
if (ms <= 0) {
|
|
597
|
+
return;
|
|
598
|
+
}
|
|
599
|
+
try {
|
|
600
|
+
await context.adapter.wait({ ms, label: 'app-map-settle' });
|
|
601
|
+
}
|
|
602
|
+
catch {
|
|
603
|
+
await delay(ms);
|
|
604
|
+
}
|
|
605
|
+
}
|
|
606
|
+
function isRicherObservation(candidate, current) {
|
|
607
|
+
if (isTransientObservation(current) && !isTransientObservation(candidate)) {
|
|
608
|
+
return true;
|
|
609
|
+
}
|
|
610
|
+
if (isTransientObservation(candidate)) {
|
|
611
|
+
return false;
|
|
612
|
+
}
|
|
613
|
+
const candidateScore = observationRichness(candidate);
|
|
614
|
+
const currentScore = observationRichness(current);
|
|
615
|
+
return candidateScore > currentScore;
|
|
616
|
+
}
|
|
617
|
+
function observationRichness(observation) {
|
|
618
|
+
const stableControls = observation.elements.filter((element) => element.stable && element.visible !== false && element.safety === 'safe').length;
|
|
619
|
+
return observation.element_keys.length + observation.elements.length * 2 + stableControls * 3;
|
|
620
|
+
}
|
|
621
|
+
function shouldSettleObservation(observation) {
|
|
622
|
+
if (isTransientObservation(observation)) {
|
|
623
|
+
return true;
|
|
624
|
+
}
|
|
625
|
+
const labels = observation.elements.flatMap((element) => element.labels.map(normalizeControlTarget));
|
|
626
|
+
const hasTransitionLabel = labels.some((label) => /\b(loading|syncing|refreshing|please wait|progress)\b/.test(label));
|
|
627
|
+
return hasTransitionLabel && observation.elements.length <= 3;
|
|
628
|
+
}
|
|
629
|
+
function delay(ms) {
|
|
630
|
+
return new Promise((resolve) => setTimeout(resolve, ms));
|
|
631
|
+
}
|
|
351
632
|
function targetDescriptor(command, args) {
|
|
352
633
|
if (isValueBearingAction(command, args)) {
|
|
353
634
|
return { kind: 'unknown', confidence: 0 };
|
|
@@ -355,17 +636,26 @@ function targetDescriptor(command, args) {
|
|
|
355
636
|
if (command === 'tap') {
|
|
356
637
|
try {
|
|
357
638
|
if (resolveTapMode(args) === 'coordinates') {
|
|
358
|
-
return { kind: 'coordinate', confidence: 0 };
|
|
639
|
+
return { kind: 'coordinate', target: coordinateTarget(args), confidence: 0.5 };
|
|
359
640
|
}
|
|
360
641
|
}
|
|
361
642
|
catch {
|
|
362
643
|
return { kind: 'unknown', confidence: 0 };
|
|
363
644
|
}
|
|
364
645
|
}
|
|
646
|
+
if (command === 'scroll') {
|
|
647
|
+
return { kind: 'stable', target: scrollTarget(args), confidence: 0.4 };
|
|
648
|
+
}
|
|
365
649
|
const target = typeof args.target === 'string' ? args.target : undefined;
|
|
366
650
|
if (!target) {
|
|
367
651
|
return { kind: 'unknown', confidence: 0 };
|
|
368
652
|
}
|
|
653
|
+
if (target.startsWith('first-in-section=')) {
|
|
654
|
+
return { kind: 'section-first', target, value: target.slice(17), confidence: 0.55 };
|
|
655
|
+
}
|
|
656
|
+
if (target.startsWith('text~=')) {
|
|
657
|
+
return { kind: 'text-contains', target, value: target.slice(6), confidence: 0.45 };
|
|
658
|
+
}
|
|
369
659
|
if (target.startsWith('text=')) {
|
|
370
660
|
return { kind: 'text', target, value: target.slice(5), confidence: 0.6 };
|
|
371
661
|
}
|
|
@@ -374,30 +664,75 @@ function targetDescriptor(command, args) {
|
|
|
374
664
|
}
|
|
375
665
|
return { kind: 'stable', target, value: target.replace(/^(id=|accessibility=)/, ''), confidence: 0.9 };
|
|
376
666
|
}
|
|
667
|
+
function coordinateTarget(args) {
|
|
668
|
+
const x = Number(args.x);
|
|
669
|
+
const y = Number(args.y);
|
|
670
|
+
const base = `x=${Number.isFinite(x) ? x : String(args.x)},y=${Number.isFinite(y) ? y : String(args.y)}`;
|
|
671
|
+
return args.normalized === true ? `${base},normalized=true` : base;
|
|
672
|
+
}
|
|
673
|
+
function scrollTarget(args) {
|
|
674
|
+
const direction = String(args.direction ?? 'down').trim().toLowerCase() || 'down';
|
|
675
|
+
const percent = args.percent === undefined ? '' : `,percent=${String(args.percent)}`;
|
|
676
|
+
return `scroll=${direction}${percent}`;
|
|
677
|
+
}
|
|
377
678
|
function isValueBearingAction(command, args) {
|
|
378
679
|
return command === 'act' && args.name === 'type' && args.value !== undefined;
|
|
379
680
|
}
|
|
380
681
|
function replayableEdge(edge) {
|
|
381
682
|
return !isValueBearingAction(edge.command, edge.args);
|
|
382
683
|
}
|
|
383
|
-
function
|
|
684
|
+
function elementContainsTarget(element, descriptor) {
|
|
685
|
+
if (descriptor.kind === 'section-first') {
|
|
686
|
+
return false;
|
|
687
|
+
}
|
|
384
688
|
if (!descriptor.target && !descriptor.value) {
|
|
385
689
|
return false;
|
|
386
690
|
}
|
|
387
691
|
const target = descriptor.target ?? '';
|
|
388
692
|
const value = descriptor.value ?? target;
|
|
389
|
-
|
|
390
|
-
|
|
391
|
-
|
|
392
|
-
|
|
393
|
-
|
|
394
|
-
|
|
395
|
-
|
|
396
|
-
|
|
397
|
-
|
|
693
|
+
if (element.targets.includes(target) || element.targets.includes(value)) {
|
|
694
|
+
return true;
|
|
695
|
+
}
|
|
696
|
+
if (descriptor.kind === 'text-contains') {
|
|
697
|
+
const normalizedValue = normalizeControlTarget(value);
|
|
698
|
+
return element.labels.some((label) => normalizeControlTarget(label).includes(normalizedValue));
|
|
699
|
+
}
|
|
700
|
+
return element.labels.some((label) => label === value);
|
|
701
|
+
}
|
|
702
|
+
function variantContainsTarget(variant, descriptor) {
|
|
703
|
+
if (descriptor.kind === 'section-first') {
|
|
704
|
+
return sectionFirstTapArgs(variant, descriptor) !== null;
|
|
705
|
+
}
|
|
706
|
+
return variant.elements.some((element) => elementContainsTarget(element, descriptor));
|
|
707
|
+
}
|
|
708
|
+
async function liveContainsTarget(context, descriptor) {
|
|
709
|
+
if (descriptor.kind === 'section-first') {
|
|
710
|
+
return false;
|
|
711
|
+
}
|
|
712
|
+
const target = descriptor.target ?? descriptor.value;
|
|
713
|
+
if (!target) {
|
|
714
|
+
return false;
|
|
715
|
+
}
|
|
716
|
+
try {
|
|
717
|
+
return await context.adapter.exists(target);
|
|
718
|
+
}
|
|
719
|
+
catch {
|
|
720
|
+
return false;
|
|
721
|
+
}
|
|
398
722
|
}
|
|
399
723
|
function candidateVariants(map, descriptor) {
|
|
400
|
-
return map.variants.filter((variant) => variantContainsTarget(variant, descriptor));
|
|
724
|
+
return map.variants.filter((variant) => variantContainsTarget(variant, descriptor) && variantMatchesDescriptorContext(map, variant, descriptor));
|
|
725
|
+
}
|
|
726
|
+
function variantMatchesDescriptorContext(map, variant, descriptor) {
|
|
727
|
+
if (descriptor.kind !== 'section-first') {
|
|
728
|
+
return true;
|
|
729
|
+
}
|
|
730
|
+
const navigationHint = sectionNavigationHint(descriptor.value ?? descriptor.target ?? '', map);
|
|
731
|
+
if (!navigationHint) {
|
|
732
|
+
return true;
|
|
733
|
+
}
|
|
734
|
+
const navigationTarget = preferredNavigationTarget(variant);
|
|
735
|
+
return !navigationTarget || navigationTarget === navigationHint;
|
|
401
736
|
}
|
|
402
737
|
function summarizeEdge(edge) {
|
|
403
738
|
return {
|
|
@@ -414,29 +749,39 @@ function summarizeControl(command, target, confidence) {
|
|
|
414
749
|
};
|
|
415
750
|
}
|
|
416
751
|
function planRoute(map, fromVariantId, descriptor) {
|
|
417
|
-
if (descriptor.kind === 'coordinate' || descriptor.kind === 'unknown') {
|
|
752
|
+
if (descriptor.kind === 'coordinate' || descriptor.kind === 'unknown' || !routeableDescriptor(descriptor)) {
|
|
418
753
|
return { edges: [], ambiguous: false, authRequired: false };
|
|
419
754
|
}
|
|
420
755
|
const destinations = candidateVariants(map, descriptor);
|
|
421
756
|
if (destinations.length === 0) {
|
|
422
757
|
return { edges: [], ambiguous: false, authRequired: false };
|
|
423
758
|
}
|
|
424
|
-
if (descriptor.kind === 'text' && destinations.length > 1) {
|
|
425
|
-
return { edges: [], ambiguous: true, authRequired: false };
|
|
426
|
-
}
|
|
427
759
|
const destinationIds = new Set(destinations.map((variant) => variant.id));
|
|
428
760
|
const visited = new Set([fromVariantId]);
|
|
429
761
|
const queue = [{ variantId: fromVariantId, edges: [] }];
|
|
762
|
+
const reachableRoutes = [];
|
|
763
|
+
let shortestDestinationLength = null;
|
|
430
764
|
while (queue.length > 0) {
|
|
431
765
|
const current = queue.shift();
|
|
432
766
|
if (!current) {
|
|
433
767
|
break;
|
|
434
768
|
}
|
|
435
769
|
if (destinationIds.has(current.variantId)) {
|
|
436
|
-
|
|
437
|
-
|
|
770
|
+
if (shortestDestinationLength === null) {
|
|
771
|
+
shortestDestinationLength = current.edges.length;
|
|
772
|
+
}
|
|
773
|
+
if (current.edges.length === shortestDestinationLength) {
|
|
774
|
+
reachableRoutes.push(current);
|
|
775
|
+
}
|
|
776
|
+
continue;
|
|
777
|
+
}
|
|
778
|
+
if (shortestDestinationLength !== null && current.edges.length >= shortestDestinationLength) {
|
|
779
|
+
continue;
|
|
438
780
|
}
|
|
439
|
-
const outgoing =
|
|
781
|
+
const outgoing = [
|
|
782
|
+
...map.edges,
|
|
783
|
+
...navigationVirtualEdges(map, map.variants.find((variant) => variant.id === current.variantId))
|
|
784
|
+
]
|
|
440
785
|
.filter((edge) => edge.from_variant_id === current.variantId && !edge.stale && edge.confidence >= 0.25 && replayableEdge(edge))
|
|
441
786
|
.sort((left, right) => right.confidence - left.confidence);
|
|
442
787
|
for (const edge of outgoing) {
|
|
@@ -447,13 +792,227 @@ function planRoute(map, fromVariantId, descriptor) {
|
|
|
447
792
|
queue.push({ variantId: edge.to_variant_id, edges: [...current.edges, edge] });
|
|
448
793
|
}
|
|
449
794
|
}
|
|
795
|
+
if (reachableRoutes.length > 0) {
|
|
796
|
+
const rankedRoutes = reachableRoutes
|
|
797
|
+
.map((route) => ({
|
|
798
|
+
...route,
|
|
799
|
+
score: destinationTargetScore(map, map.variants.find((variant) => variant.id === route.variantId), descriptor) + routeTargetScore(map, route.edges, descriptor)
|
|
800
|
+
}))
|
|
801
|
+
.sort((left, right) => right.score - left.score);
|
|
802
|
+
const best = rankedRoutes[0];
|
|
803
|
+
if (!best) {
|
|
804
|
+
return { edges: [], ambiguous: false, authRequired: false };
|
|
805
|
+
}
|
|
806
|
+
const tiedBest = rankedRoutes.filter((route) => route.score === best.score);
|
|
807
|
+
const ambiguous = (descriptor.kind === 'text' || descriptor.kind === 'text-contains' || descriptor.kind === 'section-first') &&
|
|
808
|
+
tiedBest.length > 1;
|
|
809
|
+
const diagnostics = routePlanDiagnostics(map, descriptor, rankedRoutes, ambiguous ? undefined : best, ambiguous);
|
|
810
|
+
if (ambiguous) {
|
|
811
|
+
return { edges: [], ambiguous: true, authRequired: false, diagnostics };
|
|
812
|
+
}
|
|
813
|
+
const route = best;
|
|
814
|
+
const destination = map.variants.find((variant) => variant.id === route?.variantId);
|
|
815
|
+
return {
|
|
816
|
+
edges: route?.edges ?? [],
|
|
817
|
+
ambiguous: false,
|
|
818
|
+
authRequired: Boolean(destination?.auth_required),
|
|
819
|
+
diagnostics
|
|
820
|
+
};
|
|
821
|
+
}
|
|
450
822
|
return { edges: [], ambiguous: false, authRequired: destinations.some((variant) => variant.auth_required) };
|
|
451
823
|
}
|
|
824
|
+
function routePlanDiagnostics(map, descriptor, rankedRoutes, selected, ambiguous) {
|
|
825
|
+
const bestScore = rankedRoutes[0]?.score;
|
|
826
|
+
return {
|
|
827
|
+
target: descriptor.target ?? descriptor.value ?? descriptor.kind,
|
|
828
|
+
selected_variant_id: selected?.variantId,
|
|
829
|
+
ambiguous,
|
|
830
|
+
route_candidates: rankedRoutes.map((route) => {
|
|
831
|
+
const variant = map.variants.find((candidate) => candidate.id === route.variantId);
|
|
832
|
+
const isSelected = selected?.variantId === route.variantId;
|
|
833
|
+
return {
|
|
834
|
+
variant_id: route.variantId,
|
|
835
|
+
screen_id: variant?.screen_id,
|
|
836
|
+
score: Math.round(route.score * 100) / 100,
|
|
837
|
+
route: route.edges.map(summarizeEdge),
|
|
838
|
+
selected: isSelected,
|
|
839
|
+
...(isSelected
|
|
840
|
+
? {}
|
|
841
|
+
: { rejected_reason: ambiguous && route.score === bestScore ? 'ambiguous_tie' : 'lower_score' })
|
|
842
|
+
};
|
|
843
|
+
})
|
|
844
|
+
};
|
|
845
|
+
}
|
|
846
|
+
function routeableDescriptor(descriptor) {
|
|
847
|
+
if (descriptor.kind === 'section-first') {
|
|
848
|
+
const value = descriptor.value ?? descriptor.target ?? '';
|
|
849
|
+
return /[a-z]/i.test(value);
|
|
850
|
+
}
|
|
851
|
+
if (descriptor.kind !== 'text-contains') {
|
|
852
|
+
return true;
|
|
853
|
+
}
|
|
854
|
+
const value = descriptor.value ?? descriptor.target ?? '';
|
|
855
|
+
return /[a-z]/i.test(value);
|
|
856
|
+
}
|
|
857
|
+
function navigationVirtualEdges(map, variant) {
|
|
858
|
+
if (!variant) {
|
|
859
|
+
return [];
|
|
860
|
+
}
|
|
861
|
+
const currentNavigationTarget = preferredNavigationTarget(variant);
|
|
862
|
+
const edges = [];
|
|
863
|
+
const destinationsByTarget = new Map();
|
|
864
|
+
for (const destination of map.variants) {
|
|
865
|
+
if (destination.id === variant.id) {
|
|
866
|
+
continue;
|
|
867
|
+
}
|
|
868
|
+
const target = preferredNavigationTarget(destination);
|
|
869
|
+
if (!target || target === currentNavigationTarget || destinationsByTarget.has(target)) {
|
|
870
|
+
continue;
|
|
871
|
+
}
|
|
872
|
+
destinationsByTarget.set(target, destination);
|
|
873
|
+
}
|
|
874
|
+
for (const candidate of safeTapCandidates(variant, { skipCurrentNavigation: true })) {
|
|
875
|
+
const target = normalizeControlTarget(candidate.target);
|
|
876
|
+
const destination = destinationsByTarget.get(target);
|
|
877
|
+
if (!destination) {
|
|
878
|
+
continue;
|
|
879
|
+
}
|
|
880
|
+
if (hasRealEdgeForTap(map, variant.id, candidate)) {
|
|
881
|
+
continue;
|
|
882
|
+
}
|
|
883
|
+
edges.push({
|
|
884
|
+
id: `virtual_nav_${variant.id}_${target}`,
|
|
885
|
+
from_variant_id: variant.id,
|
|
886
|
+
to_variant_id: destination.id,
|
|
887
|
+
command: 'tap',
|
|
888
|
+
args: structuredClone(candidate.args),
|
|
889
|
+
target: targetDescriptor('tap', candidate.args).target,
|
|
890
|
+
confidence: 0.4,
|
|
891
|
+
successes: 0,
|
|
892
|
+
failures: 0,
|
|
893
|
+
stale: false,
|
|
894
|
+
candidate: true,
|
|
895
|
+
destination_contract: {
|
|
896
|
+
variant_id: destination.id,
|
|
897
|
+
required_targets: contractTargets(destination),
|
|
898
|
+
normalized_fingerprint: destination.normalized_fingerprint
|
|
899
|
+
},
|
|
900
|
+
last_observed_at: utcNowIso()
|
|
901
|
+
});
|
|
902
|
+
}
|
|
903
|
+
return edges;
|
|
904
|
+
}
|
|
905
|
+
function hasRealEdgeForTap(map, fromVariantId, candidate) {
|
|
906
|
+
const descriptor = targetDescriptor('tap', candidate.args);
|
|
907
|
+
const candidateTargets = new Set([candidate.key, candidate.target, descriptor.target, descriptor.value].filter(Boolean));
|
|
908
|
+
return map.edges.some((edge) => edge.from_variant_id === fromVariantId &&
|
|
909
|
+
!edge.stale &&
|
|
910
|
+
edge.confidence >= 0.25 &&
|
|
911
|
+
replayableEdge(edge) &&
|
|
912
|
+
candidateTargets.has(edge.target));
|
|
913
|
+
}
|
|
914
|
+
function destinationTargetScore(map, variant, descriptor) {
|
|
915
|
+
if (!variant) {
|
|
916
|
+
return 0;
|
|
917
|
+
}
|
|
918
|
+
if (descriptor.kind === 'section-first') {
|
|
919
|
+
const element = firstItemInSection(variant, descriptor);
|
|
920
|
+
if (!element) {
|
|
921
|
+
return 0;
|
|
922
|
+
}
|
|
923
|
+
let score = isHighValueContentCard(element) ? 10 : 7;
|
|
924
|
+
const navigationHint = sectionNavigationHint(descriptor.value ?? descriptor.target ?? '', map);
|
|
925
|
+
const navigationTarget = preferredNavigationTarget(variant);
|
|
926
|
+
if (navigationHint && navigationTarget === navigationHint) {
|
|
927
|
+
score += 5;
|
|
928
|
+
}
|
|
929
|
+
else if (navigationHint && navigationTarget && navigationTarget !== navigationHint) {
|
|
930
|
+
score -= 5;
|
|
931
|
+
}
|
|
932
|
+
return score;
|
|
933
|
+
}
|
|
934
|
+
let score = 0;
|
|
935
|
+
for (const element of variant.elements) {
|
|
936
|
+
if (!elementContainsTarget(element, descriptor)) {
|
|
937
|
+
continue;
|
|
938
|
+
}
|
|
939
|
+
let elementScore = 1;
|
|
940
|
+
if (isLikelyTapElement(element)) {
|
|
941
|
+
elementScore += 4;
|
|
942
|
+
}
|
|
943
|
+
else if (element.visible === false) {
|
|
944
|
+
elementScore -= 2;
|
|
945
|
+
}
|
|
946
|
+
if (isHighValueContentCard(element)) {
|
|
947
|
+
elementScore += 3;
|
|
948
|
+
}
|
|
949
|
+
const value = normalizeControlTarget(descriptor.value ?? descriptor.target ?? '');
|
|
950
|
+
if (value &&
|
|
951
|
+
element.labels.some((label) => normalizeControlTarget(label) === value ||
|
|
952
|
+
label.split(/\n+/).some((line) => normalizeControlTarget(line) === value))) {
|
|
953
|
+
elementScore += 2;
|
|
954
|
+
}
|
|
955
|
+
score = Math.max(score, elementScore);
|
|
956
|
+
}
|
|
957
|
+
if (preferredNavigationTarget(variant) && score > 0) {
|
|
958
|
+
score += 1;
|
|
959
|
+
}
|
|
960
|
+
if (score > 0 && isFeedLikeVariant(variant) && !descriptorTargetsFeed(descriptor)) {
|
|
961
|
+
score -= 3;
|
|
962
|
+
}
|
|
963
|
+
return score;
|
|
964
|
+
}
|
|
965
|
+
function isFeedLikeVariant(variant) {
|
|
966
|
+
return contentLabels(variant).some((label) => /^(for you|activity|feed|latest|timeline|updates?)$/.test(label));
|
|
967
|
+
}
|
|
968
|
+
function descriptorTargetsFeed(descriptor) {
|
|
969
|
+
const value = normalizeControlTarget(descriptor.value ?? descriptor.target ?? '');
|
|
970
|
+
return /\b(activity|feed|post|comment|update|timeline)\b/.test(value);
|
|
971
|
+
}
|
|
972
|
+
function routeTargetScore(map, route, descriptor) {
|
|
973
|
+
if (descriptor.kind !== 'section-first') {
|
|
974
|
+
return 0;
|
|
975
|
+
}
|
|
976
|
+
const navigationHint = sectionNavigationHint(descriptor.value ?? descriptor.target ?? '', map);
|
|
977
|
+
if (!navigationHint) {
|
|
978
|
+
return 0;
|
|
979
|
+
}
|
|
980
|
+
return route.reduce((score, edge) => {
|
|
981
|
+
const target = edgeNavigationTarget(map, edge);
|
|
982
|
+
if (!target) {
|
|
983
|
+
return score;
|
|
984
|
+
}
|
|
985
|
+
return score + (target === navigationHint ? 3 : -3);
|
|
986
|
+
}, 0);
|
|
987
|
+
}
|
|
988
|
+
function edgeNavigationTarget(map, edge) {
|
|
989
|
+
const normalizedTarget = normalizeControlTarget(edge.target ?? '');
|
|
990
|
+
const from = map.variants.find((variant) => variant.id === edge.from_variant_id);
|
|
991
|
+
if (from && isNavigationTarget(from, normalizedTarget)) {
|
|
992
|
+
return normalizedTarget;
|
|
993
|
+
}
|
|
994
|
+
if (!from || edge.command !== 'tap') {
|
|
995
|
+
return null;
|
|
996
|
+
}
|
|
997
|
+
const edgeCoordinateTarget = coordinateTarget(edge.args);
|
|
998
|
+
for (const control of navigationControls(from)) {
|
|
999
|
+
const elementCoordinateTarget = coordinateArgsForElement(control.element);
|
|
1000
|
+
if (elementCoordinateTarget && coordinateTarget(elementCoordinateTarget) === edgeCoordinateTarget) {
|
|
1001
|
+
return control.target;
|
|
1002
|
+
}
|
|
1003
|
+
}
|
|
1004
|
+
return null;
|
|
1005
|
+
}
|
|
452
1006
|
function contractTargets(variant) {
|
|
453
|
-
|
|
1007
|
+
const navigationElements = new Set(navigationControls(variant).map((control) => control.element));
|
|
1008
|
+
const contentTargets = variant.elements
|
|
454
1009
|
.filter((element) => element.stable)
|
|
455
|
-
.
|
|
456
|
-
.
|
|
1010
|
+
.filter((element) => !navigationElements.has(element))
|
|
1011
|
+
.flatMap((element) => element.targets.filter((target) => !isGlobalIdentityLabel(normalizeControlTarget(target), element)));
|
|
1012
|
+
const fallbackTargets = variant.elements
|
|
1013
|
+
.filter((element) => element.stable)
|
|
1014
|
+
.flatMap((element) => element.targets);
|
|
1015
|
+
return unique([...contentTargets, ...fallbackTargets]).slice(0, 8);
|
|
457
1016
|
}
|
|
458
1017
|
function matchingEdge(map, fromVariantId, command, target) {
|
|
459
1018
|
return map.edges.find((edge) => edge.from_variant_id === fromVariantId &&
|
|
@@ -465,7 +1024,7 @@ function recordEdgeSuccess(context, from, to, command, args, candidate) {
|
|
|
465
1024
|
return;
|
|
466
1025
|
}
|
|
467
1026
|
const descriptor = targetDescriptor(command, args);
|
|
468
|
-
if (descriptor.kind === '
|
|
1027
|
+
if (descriptor.kind === 'unknown') {
|
|
469
1028
|
return;
|
|
470
1029
|
}
|
|
471
1030
|
const edge = matchingEdge(context.map, from.id, command, descriptor.target);
|
|
@@ -515,25 +1074,87 @@ function demoteEdge(context, edge) {
|
|
|
515
1074
|
edge.last_observed_at = utcNowIso();
|
|
516
1075
|
context.changed = true;
|
|
517
1076
|
}
|
|
518
|
-
function
|
|
519
|
-
|
|
1077
|
+
function shouldRequireObservedEffect(command, args) {
|
|
1078
|
+
return command === 'tap' || command === 'navigate' || command === 'scroll' || (command === 'act' && args.name === 'back');
|
|
1079
|
+
}
|
|
1080
|
+
function assertObservedEffect(command, args, before, after, descriptor) {
|
|
1081
|
+
if (!shouldRequireObservedEffect(command, args)) {
|
|
1082
|
+
return;
|
|
1083
|
+
}
|
|
1084
|
+
if (before.observation.fingerprint !== after.observation.fingerprint) {
|
|
1085
|
+
return;
|
|
1086
|
+
}
|
|
1087
|
+
const target = descriptor.target ?? descriptor.value ?? command;
|
|
1088
|
+
throw new Error(`Action '${command}' on '${target}' had no effect: screen source did not change after the command`);
|
|
1089
|
+
}
|
|
1090
|
+
function variantSatisfiesContract(variant, contract) {
|
|
1091
|
+
if (variant.id === contract.variant_id) {
|
|
520
1092
|
return true;
|
|
521
1093
|
}
|
|
522
|
-
const observedTargets = new Set(
|
|
523
|
-
const required =
|
|
1094
|
+
const observedTargets = new Set(variant.elements.flatMap((element) => element.targets));
|
|
1095
|
+
const required = contract.required_targets;
|
|
524
1096
|
if (required.length === 0) {
|
|
525
|
-
return
|
|
1097
|
+
return variant.normalized_fingerprint === contract.normalized_fingerprint;
|
|
526
1098
|
}
|
|
527
1099
|
const matched = required.filter((target) => observedTargets.has(target)).length;
|
|
528
1100
|
return matched / required.length >= 0.6;
|
|
529
1101
|
}
|
|
1102
|
+
function restoreContractHasSharedAnchors(observed, target) {
|
|
1103
|
+
const observedTargets = new Set(observed.elements.flatMap((element) => element.targets));
|
|
1104
|
+
const required = contractTargets(target);
|
|
1105
|
+
if (required.length === 0) {
|
|
1106
|
+
return false;
|
|
1107
|
+
}
|
|
1108
|
+
const matched = unique(required.filter((targetValue) => observedTargets.has(targetValue)));
|
|
1109
|
+
return matched.length >= 2 && matched.length / required.length >= 0.25;
|
|
1110
|
+
}
|
|
1111
|
+
function restoreMatch(observed, target) {
|
|
1112
|
+
if (observed.variant.id === target.variant.id) {
|
|
1113
|
+
return { acceptedBy: 'exact' };
|
|
1114
|
+
}
|
|
1115
|
+
const observedNavigationTarget = preferredNavigationTarget(observed.variant);
|
|
1116
|
+
const targetNavigationTarget = preferredNavigationTarget(target.variant);
|
|
1117
|
+
if (observedNavigationTarget && targetNavigationTarget && observedNavigationTarget !== targetNavigationTarget) {
|
|
1118
|
+
return null;
|
|
1119
|
+
}
|
|
1120
|
+
if (hasScrollableSurface(observed.variant) || hasScrollableSurface(target.variant)) {
|
|
1121
|
+
const observedControlKeys = identityKeys(observed.variant.elements, false);
|
|
1122
|
+
const targetControlKeys = identityKeys(target.variant.elements, false);
|
|
1123
|
+
if ((observedControlKeys.length > 0 || targetControlKeys.length > 0) &&
|
|
1124
|
+
similarity(observedControlKeys, targetControlKeys) < 0.3) {
|
|
1125
|
+
return null;
|
|
1126
|
+
}
|
|
1127
|
+
}
|
|
1128
|
+
const contract = {
|
|
1129
|
+
variant_id: target.variant.id,
|
|
1130
|
+
required_targets: contractTargets(target.variant),
|
|
1131
|
+
normalized_fingerprint: target.variant.normalized_fingerprint
|
|
1132
|
+
};
|
|
1133
|
+
if (variantSatisfiesContract(observed.variant, contract)) {
|
|
1134
|
+
return { acceptedBy: 'contract' };
|
|
1135
|
+
}
|
|
1136
|
+
if (observedNavigationTarget &&
|
|
1137
|
+
targetNavigationTarget &&
|
|
1138
|
+
observedNavigationTarget === targetNavigationTarget &&
|
|
1139
|
+
restoreContractHasSharedAnchors(observed.variant, target.variant)) {
|
|
1140
|
+
return { acceptedBy: 'contract' };
|
|
1141
|
+
}
|
|
1142
|
+
const score = screenSimilarity(target.variant.elements, observed.variant.elements);
|
|
1143
|
+
if (score >= 0.72) {
|
|
1144
|
+
return { acceptedBy: 'similarity', score };
|
|
1145
|
+
}
|
|
1146
|
+
return null;
|
|
1147
|
+
}
|
|
1148
|
+
function satisfiesContract(edge, observed) {
|
|
1149
|
+
return variantSatisfiesContract(observed.variant, edge.destination_contract);
|
|
1150
|
+
}
|
|
530
1151
|
async function driveRoute(context, current, route) {
|
|
531
1152
|
for (const edge of route) {
|
|
532
1153
|
const args = structuredClone(edge.args);
|
|
533
1154
|
let observed;
|
|
534
1155
|
try {
|
|
535
1156
|
await context.adapter[edge.command](args);
|
|
536
|
-
observed = await
|
|
1157
|
+
observed = await observeAfterAction(context);
|
|
537
1158
|
}
|
|
538
1159
|
catch {
|
|
539
1160
|
demoteEdge(context, edge);
|
|
@@ -554,63 +1175,731 @@ async function driveRoute(context, current, route) {
|
|
|
554
1175
|
}
|
|
555
1176
|
return { current };
|
|
556
1177
|
}
|
|
557
|
-
function
|
|
558
|
-
|
|
559
|
-
|
|
560
|
-
|
|
561
|
-
|
|
562
|
-
|
|
563
|
-
|
|
1178
|
+
function safeTapCandidates(variant, options = {}) {
|
|
1179
|
+
const candidates = [];
|
|
1180
|
+
const seen = new Set();
|
|
1181
|
+
const currentNavigationTarget = options.skipCurrentNavigation ? preferredNavigationTarget(variant) : null;
|
|
1182
|
+
const elements = [...variant.elements].sort((left, right) => {
|
|
1183
|
+
const priority = tapElementPriority(left, variant, options) - tapElementPriority(right, variant, options);
|
|
1184
|
+
if (priority !== 0) {
|
|
1185
|
+
return priority;
|
|
1186
|
+
}
|
|
1187
|
+
return (left.y ?? 0) - (right.y ?? 0) || (left.x ?? 0) - (right.x ?? 0);
|
|
1188
|
+
});
|
|
1189
|
+
for (const element of elements) {
|
|
1190
|
+
if (!(element.safety === 'safe' && element.stable && isLikelyTapElement(element))) {
|
|
1191
|
+
continue;
|
|
1192
|
+
}
|
|
1193
|
+
const target = plainTapTargets(element)[0] ?? element.selector;
|
|
1194
|
+
if (!target) {
|
|
1195
|
+
continue;
|
|
1196
|
+
}
|
|
1197
|
+
const normalizedTarget = normalizeControlTarget(target);
|
|
1198
|
+
if (options.excludeNavigation && isNavigationTarget(variant, normalizedTarget)) {
|
|
1199
|
+
continue;
|
|
1200
|
+
}
|
|
1201
|
+
if (currentNavigationTarget && normalizedTarget === currentNavigationTarget) {
|
|
1202
|
+
continue;
|
|
1203
|
+
}
|
|
1204
|
+
const coordinateArgs = coordinateArgsForElement(element);
|
|
1205
|
+
const args = coordinateArgs ?? { target };
|
|
1206
|
+
const key = coordinateArgs ? coordinateTarget(coordinateArgs) : target;
|
|
1207
|
+
if (seen.has(key)) {
|
|
1208
|
+
continue;
|
|
1209
|
+
}
|
|
1210
|
+
seen.add(key);
|
|
1211
|
+
candidates.push({ target, args, key });
|
|
1212
|
+
}
|
|
1213
|
+
return candidates;
|
|
564
1214
|
}
|
|
565
|
-
|
|
566
|
-
|
|
567
|
-
|
|
1215
|
+
function tapElementPriority(element, variant, options = {}) {
|
|
1216
|
+
const target = plainTapTargets(element)[0] ?? element.selector;
|
|
1217
|
+
const normalized = normalizeControlTarget(target);
|
|
1218
|
+
if (options.preferContent && isHighValueContentCard(element)) {
|
|
1219
|
+
return 5;
|
|
568
1220
|
}
|
|
569
|
-
const
|
|
570
|
-
|
|
571
|
-
|
|
572
|
-
|
|
573
|
-
let routeSoFar = route;
|
|
574
|
-
while (Date.now() <= deadline) {
|
|
575
|
-
if (variantContainsTarget(cursor.variant, descriptor)) {
|
|
576
|
-
return { found: true, current: cursor, route: routeSoFar, remainingDepth: depthLeft };
|
|
577
|
-
}
|
|
578
|
-
if (depthLeft <= 0) {
|
|
579
|
-
return { found: false, current: cursor, route: routeSoFar, remainingDepth: depthLeft };
|
|
580
|
-
}
|
|
581
|
-
const attemptsForVariant = attemptedTargets.get(cursor.variant.id) ?? new Set();
|
|
582
|
-
attemptedTargets.set(cursor.variant.id, attemptsForVariant);
|
|
583
|
-
const target = safeTapTargets(cursor.variant).find((candidate) => !blockedTargets.has(candidate) && !attemptsForVariant.has(candidate));
|
|
584
|
-
if (!target) {
|
|
585
|
-
return { found: false, current: cursor, route: routeSoFar, remainingDepth: depthLeft };
|
|
586
|
-
}
|
|
587
|
-
attemptsForVariant.add(target);
|
|
588
|
-
const args = { target };
|
|
589
|
-
let observed;
|
|
590
|
-
try {
|
|
591
|
-
await context.adapter.tap(args);
|
|
592
|
-
observed = await observeCurrentScreen(context);
|
|
593
|
-
}
|
|
594
|
-
catch {
|
|
595
|
-
try {
|
|
596
|
-
cursor = await observeCurrentScreen(context);
|
|
597
|
-
}
|
|
598
|
-
catch {
|
|
599
|
-
// If both the exploratory tap and source read fail, keep the last observed screen.
|
|
600
|
-
}
|
|
601
|
-
continue;
|
|
602
|
-
}
|
|
603
|
-
recordEdgeSuccess(context, cursor.variant, observed.variant, 'tap', args, true);
|
|
604
|
-
const nextRoute = [...routeSoFar, summarizeControl('tap', target, 0.45)];
|
|
605
|
-
const nested = await attemptFrom(observed, depthLeft - 1, nextRoute);
|
|
606
|
-
if (nested.found) {
|
|
607
|
-
return nested;
|
|
608
|
-
}
|
|
609
|
-
cursor = nested.current;
|
|
610
|
-
depthLeft = nested.remainingDepth;
|
|
611
|
-
routeSoFar = nested.route;
|
|
1221
|
+
const navPriority = navigationControlPriority(variant, normalized);
|
|
1222
|
+
if (navPriority !== undefined) {
|
|
1223
|
+
if (options.preferContent) {
|
|
1224
|
+
return 80 + navPriority;
|
|
612
1225
|
}
|
|
613
|
-
return
|
|
1226
|
+
return navPriority;
|
|
1227
|
+
}
|
|
1228
|
+
if (normalized === 'see all') {
|
|
1229
|
+
return 10;
|
|
1230
|
+
}
|
|
1231
|
+
if (isBackControl(element)) {
|
|
1232
|
+
return 90;
|
|
1233
|
+
}
|
|
1234
|
+
if (element.tag.toLowerCase().includes('button')) {
|
|
1235
|
+
return 40;
|
|
1236
|
+
}
|
|
1237
|
+
if (element.tag.toLowerCase().includes('other')) {
|
|
1238
|
+
return 60;
|
|
1239
|
+
}
|
|
1240
|
+
return 50;
|
|
1241
|
+
}
|
|
1242
|
+
function isHighValueContentCard(element) {
|
|
1243
|
+
if (!isLikelyTapElement(element)) {
|
|
1244
|
+
return false;
|
|
1245
|
+
}
|
|
1246
|
+
const target = plainTapTargets(element)[0] ?? element.selector;
|
|
1247
|
+
const normalizedTarget = normalizeControlTarget(target);
|
|
1248
|
+
if (normalizedTarget === 'see all' || /^(back|close|dismiss|done)$/i.test(target)) {
|
|
1249
|
+
return false;
|
|
1250
|
+
}
|
|
1251
|
+
const lines = unique(element.labels.flatMap((label) => label.split(/\n+/).map((line) => line.trim())));
|
|
1252
|
+
const area = (element.width ?? 0) * (element.height ?? 0);
|
|
1253
|
+
return (lines.length >= 2 ||
|
|
1254
|
+
(element.tag.toLowerCase().includes('other') && area >= 3000) ||
|
|
1255
|
+
((element.width ?? 0) >= 120 && (element.height ?? 0) >= 64 && element.labels.length > 0));
|
|
1256
|
+
}
|
|
1257
|
+
function actionAffordancesForElements(elements) {
|
|
1258
|
+
const navigationElements = new Set(navigationControlsForElements(elements).map((control) => control.element));
|
|
1259
|
+
const navigationTarget = preferredNavigationTargetForElements(elements) ?? undefined;
|
|
1260
|
+
const actions = [];
|
|
1261
|
+
const seen = new Set();
|
|
1262
|
+
const candidates = [...elements]
|
|
1263
|
+
.filter((element) => element.safety === 'safe' && element.stable && isLikelyTapElement(element))
|
|
1264
|
+
.filter((element) => !navigationElements.has(element) && !isBackControl(element))
|
|
1265
|
+
.sort((left, right) => (left.y ?? 0) - (right.y ?? 0) || (left.x ?? 0) - (right.x ?? 0));
|
|
1266
|
+
for (const element of candidates) {
|
|
1267
|
+
const label = actionLabel(element);
|
|
1268
|
+
if (!label) {
|
|
1269
|
+
continue;
|
|
1270
|
+
}
|
|
1271
|
+
const coordinateArgs = coordinateArgsForElement(element);
|
|
1272
|
+
const stableTarget = plainTapTargets(element)[0] ?? element.selector;
|
|
1273
|
+
const args = coordinateArgs ?? { target: stableTarget };
|
|
1274
|
+
const target = coordinateArgs ? coordinateTarget(coordinateArgs) : stableTarget;
|
|
1275
|
+
const scope = actionScopeForElement(elements, element);
|
|
1276
|
+
const dedupeKey = [
|
|
1277
|
+
actionIntent(label),
|
|
1278
|
+
target,
|
|
1279
|
+
scope?.kind ?? '',
|
|
1280
|
+
scope?.label ?? ''
|
|
1281
|
+
].join('|');
|
|
1282
|
+
if (seen.has(dedupeKey)) {
|
|
1283
|
+
continue;
|
|
1284
|
+
}
|
|
1285
|
+
seen.add(dedupeKey);
|
|
1286
|
+
actions.push({
|
|
1287
|
+
command: 'tap',
|
|
1288
|
+
intent: actionIntent(label),
|
|
1289
|
+
label,
|
|
1290
|
+
target,
|
|
1291
|
+
args,
|
|
1292
|
+
safety: element.safety,
|
|
1293
|
+
...(scope ? { scope } : {}),
|
|
1294
|
+
...(navigationTarget ? { navigation_target: navigationTarget } : {}),
|
|
1295
|
+
source: {
|
|
1296
|
+
tag: element.tag,
|
|
1297
|
+
x: element.x,
|
|
1298
|
+
y: element.y,
|
|
1299
|
+
width: element.width,
|
|
1300
|
+
height: element.height
|
|
1301
|
+
}
|
|
1302
|
+
});
|
|
1303
|
+
}
|
|
1304
|
+
return actions;
|
|
1305
|
+
}
|
|
1306
|
+
function actionLabel(element) {
|
|
1307
|
+
const target = plainTapTargets(element)[0] ?? element.labels[0] ?? element.selector;
|
|
1308
|
+
return target.split(/\n+/).map((line) => line.trim()).filter(Boolean)[0] ?? target.trim();
|
|
1309
|
+
}
|
|
1310
|
+
function actionIntent(label) {
|
|
1311
|
+
const normalized = normalizeControlTarget(label);
|
|
1312
|
+
const intentPatterns = [
|
|
1313
|
+
['like', /\blike\b/],
|
|
1314
|
+
['comment', /\bcomment\b/],
|
|
1315
|
+
['share', /\bshare\b/],
|
|
1316
|
+
['save', /\b(save|bookmark)\b/],
|
|
1317
|
+
['follow', /\bfollow\b/],
|
|
1318
|
+
['watchlist', /\bwatchlist\b/],
|
|
1319
|
+
['search', /\bsearch\b/],
|
|
1320
|
+
['create', /\b(create|new)\b/],
|
|
1321
|
+
['filter', /\bfilter\b/],
|
|
1322
|
+
['sort', /\bsort\b/],
|
|
1323
|
+
['dismiss', /\b(close|dismiss|skip)\b/]
|
|
1324
|
+
];
|
|
1325
|
+
const matched = intentPatterns.find(([, pattern]) => pattern.test(normalized));
|
|
1326
|
+
if (matched) {
|
|
1327
|
+
return matched[0];
|
|
1328
|
+
}
|
|
1329
|
+
return normalized.split(/[^a-z0-9]+/).filter(Boolean)[0] ?? 'tap';
|
|
1330
|
+
}
|
|
1331
|
+
function actionScopeForElement(elements, element) {
|
|
1332
|
+
const content = nearestContentScopeElement(elements, element);
|
|
1333
|
+
if (content) {
|
|
1334
|
+
return {
|
|
1335
|
+
kind: 'content',
|
|
1336
|
+
label: content.labels[0]
|
|
1337
|
+
};
|
|
1338
|
+
}
|
|
1339
|
+
const section = nearestActionSectionElement(elements, element);
|
|
1340
|
+
if (section) {
|
|
1341
|
+
return {
|
|
1342
|
+
kind: 'section',
|
|
1343
|
+
label: section.labels[0]
|
|
1344
|
+
};
|
|
1345
|
+
}
|
|
1346
|
+
return { kind: 'screen' };
|
|
1347
|
+
}
|
|
1348
|
+
function nearestContentScopeElement(elements, element) {
|
|
1349
|
+
if (element.y === null) {
|
|
1350
|
+
return null;
|
|
1351
|
+
}
|
|
1352
|
+
return elements
|
|
1353
|
+
.filter((candidate) => candidate !== element && candidate.y !== null && candidate.height !== null)
|
|
1354
|
+
.filter((candidate) => isHighValueContentCard(candidate))
|
|
1355
|
+
.filter((candidate) => {
|
|
1356
|
+
const candidateTop = candidate.y;
|
|
1357
|
+
const candidateBottom = candidateTop + Math.max(0, candidate.height ?? 0);
|
|
1358
|
+
return candidateTop <= element.y && candidateBottom + 96 >= element.y;
|
|
1359
|
+
})
|
|
1360
|
+
.sort((left, right) => (right.y ?? 0) - (left.y ?? 0))[0] ?? null;
|
|
1361
|
+
}
|
|
1362
|
+
function nearestActionSectionElement(elements, element) {
|
|
1363
|
+
if (element.y === null) {
|
|
1364
|
+
return null;
|
|
1365
|
+
}
|
|
1366
|
+
return elements
|
|
1367
|
+
.filter((candidate) => candidate.y !== null && candidate.y < element.y && isActionSectionElement(candidate))
|
|
1368
|
+
.sort((left, right) => (right.y ?? 0) - (left.y ?? 0))[0] ?? null;
|
|
1369
|
+
}
|
|
1370
|
+
function isActionSectionElement(element) {
|
|
1371
|
+
if (isLikelyTapElement(element) || element.labels.length === 0) {
|
|
1372
|
+
return false;
|
|
1373
|
+
}
|
|
1374
|
+
return element.labels.some((label) => {
|
|
1375
|
+
const normalized = normalizeControlTarget(label);
|
|
1376
|
+
if (!/[a-z]/i.test(normalized)) {
|
|
1377
|
+
return false;
|
|
1378
|
+
}
|
|
1379
|
+
const words = normalized.split(/\s+/).filter(Boolean);
|
|
1380
|
+
return words.length <= 8;
|
|
1381
|
+
});
|
|
1382
|
+
}
|
|
1383
|
+
function normalizeControlTarget(target) {
|
|
1384
|
+
return target.replace(/\s+/g, ' ').trim().toLowerCase();
|
|
1385
|
+
}
|
|
1386
|
+
function sectionNavigationHint(value, map) {
|
|
1387
|
+
const normalized = normalizeControlTarget(value);
|
|
1388
|
+
const targets = unique(map.variants.flatMap((variant) => navigationControls(variant).map((control) => control.target)))
|
|
1389
|
+
.sort((left, right) => right.length - left.length);
|
|
1390
|
+
for (const target of targets) {
|
|
1391
|
+
if (normalized.includes(target)) {
|
|
1392
|
+
return target;
|
|
1393
|
+
}
|
|
1394
|
+
}
|
|
1395
|
+
return null;
|
|
1396
|
+
}
|
|
1397
|
+
function crawlCandidateOptions(variant) {
|
|
1398
|
+
const preferredTarget = preferredNavigationTarget(variant);
|
|
1399
|
+
const preferContent = preferredTarget !== null && !isPrimaryNavigationTarget(variant, preferredTarget);
|
|
1400
|
+
return {
|
|
1401
|
+
preferContent,
|
|
1402
|
+
excludeNavigation: preferContent,
|
|
1403
|
+
skipCurrentNavigation: true
|
|
1404
|
+
};
|
|
1405
|
+
}
|
|
1406
|
+
function crawlTemplateKey(variant) {
|
|
1407
|
+
const navigationTarget = preferredNavigationTarget(variant) ?? 'none';
|
|
1408
|
+
const elementKeys = unique(variant.elements.flatMap((element) => crawlTemplateElementKeys(variant, element))).sort();
|
|
1409
|
+
return signatureFor([`nav=${navigationTarget}`, ...elementKeys]);
|
|
1410
|
+
}
|
|
1411
|
+
function crawlTemplateElementKeys(variant, element) {
|
|
1412
|
+
if (element.visible === false || element.stable === false) {
|
|
1413
|
+
return [];
|
|
1414
|
+
}
|
|
1415
|
+
const tag = element.tag.toLowerCase();
|
|
1416
|
+
const layout = `${bucketNumber(element.width, 24)}x${bucketNumber(element.height, 24)}`;
|
|
1417
|
+
if (isScrollableTag(tag)) {
|
|
1418
|
+
return [`scroll:${layout}`];
|
|
1419
|
+
}
|
|
1420
|
+
if (isNavigationElement(variant, element)) {
|
|
1421
|
+
return [`nav-control:${normalizeControlTarget(plainTapTargets(element)[0] ?? element.selector)}:${layout}`];
|
|
1422
|
+
}
|
|
1423
|
+
if (isHighValueContentCard(element)) {
|
|
1424
|
+
return [`content-card:${contentLineCount(element)}:${layout}`];
|
|
1425
|
+
}
|
|
1426
|
+
if (isLikelyTapElement(element)) {
|
|
1427
|
+
return [`tap:${tag}:${templateControlLabel(element)}:${layout}`];
|
|
1428
|
+
}
|
|
1429
|
+
return [];
|
|
1430
|
+
}
|
|
1431
|
+
function templateControlLabel(element) {
|
|
1432
|
+
if (isBackControl(element)) {
|
|
1433
|
+
return 'back';
|
|
1434
|
+
}
|
|
1435
|
+
const target = normalizeControlTarget(plainTapTargets(element)[0] ?? element.selector);
|
|
1436
|
+
const words = target.split(/\s+/).filter(Boolean);
|
|
1437
|
+
if (target && words.length <= 4 && target.length <= 48) {
|
|
1438
|
+
return target;
|
|
1439
|
+
}
|
|
1440
|
+
return 'control';
|
|
1441
|
+
}
|
|
1442
|
+
function repeatedContentCandidateKey(variant, candidate) {
|
|
1443
|
+
const element = elementForTapCandidate(variant, candidate);
|
|
1444
|
+
if (!element || !isHighValueContentCard(element)) {
|
|
1445
|
+
return null;
|
|
1446
|
+
}
|
|
1447
|
+
const section = nearestSectionHeadingLabel(variant, element) ?? 'section=none';
|
|
1448
|
+
const tag = element.tag.toLowerCase();
|
|
1449
|
+
const shape = [
|
|
1450
|
+
crawlTemplateKey(variant),
|
|
1451
|
+
section,
|
|
1452
|
+
tag,
|
|
1453
|
+
`lines=${contentLineCount(element)}`,
|
|
1454
|
+
`size=${bucketNumber(element.width, 24)}x${bucketNumber(element.height, 24)}`
|
|
1455
|
+
];
|
|
1456
|
+
return signatureFor(shape);
|
|
1457
|
+
}
|
|
1458
|
+
function elementForTapCandidate(variant, candidate) {
|
|
1459
|
+
const candidateCoordinate = typeof candidate.args.x === 'number' && typeof candidate.args.y === 'number'
|
|
1460
|
+
? coordinateTarget(candidate.args)
|
|
1461
|
+
: null;
|
|
1462
|
+
for (const element of variant.elements) {
|
|
1463
|
+
if (candidateCoordinate) {
|
|
1464
|
+
const elementCoordinate = coordinateArgsForElement(element);
|
|
1465
|
+
if (elementCoordinate && coordinateTarget(elementCoordinate) === candidateCoordinate) {
|
|
1466
|
+
return element;
|
|
1467
|
+
}
|
|
1468
|
+
}
|
|
1469
|
+
const target = plainTapTargets(element)[0] ?? element.selector;
|
|
1470
|
+
if (target === candidate.target || element.targets.includes(candidate.target)) {
|
|
1471
|
+
return element;
|
|
1472
|
+
}
|
|
1473
|
+
}
|
|
1474
|
+
return null;
|
|
1475
|
+
}
|
|
1476
|
+
function nearestSectionHeadingLabel(variant, element) {
|
|
1477
|
+
if (element.y === null) {
|
|
1478
|
+
return null;
|
|
1479
|
+
}
|
|
1480
|
+
const heading = variant.elements
|
|
1481
|
+
.filter((candidate) => candidate.y !== null && candidate.y < element.y && isLikelySectionHeading(variant, candidate))
|
|
1482
|
+
.sort((left, right) => (right.y ?? 0) - (left.y ?? 0))[0];
|
|
1483
|
+
const label = heading?.labels[0];
|
|
1484
|
+
return label ? `section=${normalizeControlTarget(label)}` : null;
|
|
1485
|
+
}
|
|
1486
|
+
function contentLineCount(element) {
|
|
1487
|
+
const lines = unique(element.labels.flatMap((label) => label.split(/\n+/).map((line) => line.trim()).filter(Boolean)));
|
|
1488
|
+
return lines.length;
|
|
1489
|
+
}
|
|
1490
|
+
function bucketNumber(value, size) {
|
|
1491
|
+
if (value === null || !Number.isFinite(value)) {
|
|
1492
|
+
return 0;
|
|
1493
|
+
}
|
|
1494
|
+
return Math.round(value / size) * size;
|
|
1495
|
+
}
|
|
1496
|
+
function crawlScrollCandidates(context, variant) {
|
|
1497
|
+
if (!context.adapter.capability().commands.includes('scroll') || !shouldExploreScroll(variant)) {
|
|
1498
|
+
return [];
|
|
1499
|
+
}
|
|
1500
|
+
const args = { direction: 'down', percent: 70 };
|
|
1501
|
+
return [
|
|
1502
|
+
{
|
|
1503
|
+
target: scrollTarget(args),
|
|
1504
|
+
args,
|
|
1505
|
+
key: 'scroll:down:70'
|
|
1506
|
+
}
|
|
1507
|
+
];
|
|
1508
|
+
}
|
|
1509
|
+
function shouldExploreScroll(variant) {
|
|
1510
|
+
return hasScrollableSurface(variant) || hasBottomReachableContent(variant);
|
|
1511
|
+
}
|
|
1512
|
+
function hasScrollableSurface(variant) {
|
|
1513
|
+
return elementsHaveScrollableSurface(variant.elements);
|
|
1514
|
+
}
|
|
1515
|
+
function elementsHaveScrollableSurface(elements) {
|
|
1516
|
+
return elements.some((element) => isScrollableTag(element.tag));
|
|
1517
|
+
}
|
|
1518
|
+
function hasBottomReachableContent(variant) {
|
|
1519
|
+
const content = scrollProbeElements(variant);
|
|
1520
|
+
if (content.length < 3) {
|
|
1521
|
+
return false;
|
|
1522
|
+
}
|
|
1523
|
+
const navControls = navigationControls(variant);
|
|
1524
|
+
const navTop = navControls
|
|
1525
|
+
.map((control) => control.element.y)
|
|
1526
|
+
.filter((value) => value !== null)
|
|
1527
|
+
.sort((left, right) => left - right)[0] ?? null;
|
|
1528
|
+
if (navTop === null) {
|
|
1529
|
+
return false;
|
|
1530
|
+
}
|
|
1531
|
+
const viewportBottom = navTop ?? Math.max(...content.map((element) => (element.y ?? 0) + (element.height ?? 0)));
|
|
1532
|
+
return content.some((element) => ((element.y ?? 0) + (element.height ?? 0)) >= viewportBottom - 48);
|
|
1533
|
+
}
|
|
1534
|
+
function scrollProbeElements(variant) {
|
|
1535
|
+
const navigationElements = new Set(navigationControls(variant).map((control) => control.element));
|
|
1536
|
+
return variant.elements.filter((element) => {
|
|
1537
|
+
if (element.visible === false || element.y === null || element.height === null || navigationElements.has(element)) {
|
|
1538
|
+
return false;
|
|
1539
|
+
}
|
|
1540
|
+
if (isBackControl(element)) {
|
|
1541
|
+
return false;
|
|
1542
|
+
}
|
|
1543
|
+
const tag = element.tag.toLowerCase();
|
|
1544
|
+
return !isScrollableTag(tag);
|
|
1545
|
+
});
|
|
1546
|
+
}
|
|
1547
|
+
function preferredNavigationTarget(variant) {
|
|
1548
|
+
return preferredNavigationTargetForElements(variant.elements);
|
|
1549
|
+
}
|
|
1550
|
+
function preferredNavigationTargetForElements(elements) {
|
|
1551
|
+
const controls = navigationControlsForElements(elements);
|
|
1552
|
+
if (controls.length === 0) {
|
|
1553
|
+
return null;
|
|
1554
|
+
}
|
|
1555
|
+
const labels = contentLabelsForElements(elements);
|
|
1556
|
+
const matched = controls.find((control) => labels.some((label) => labelMatchesNavigationTarget(label, control.target)));
|
|
1557
|
+
if (matched) {
|
|
1558
|
+
return matched.target;
|
|
1559
|
+
}
|
|
1560
|
+
return controls[0]?.target ?? null;
|
|
1561
|
+
}
|
|
1562
|
+
function hasBottomNavigation(variant) {
|
|
1563
|
+
return navigationControls(variant).length >= 2;
|
|
1564
|
+
}
|
|
1565
|
+
function contentLabels(variant) {
|
|
1566
|
+
return contentLabelsForElements(variant.elements);
|
|
1567
|
+
}
|
|
1568
|
+
function contentLabelsForElements(elements) {
|
|
1569
|
+
const navigationElements = new Set(navigationControlsForElements(elements).map((control) => control.element));
|
|
1570
|
+
return elements.flatMap((element) => element.labels
|
|
1571
|
+
.filter(() => !navigationElements.has(element))
|
|
1572
|
+
.filter((label) => !isGlobalIdentityLabel(label, element))
|
|
1573
|
+
.map(normalizeControlTarget));
|
|
1574
|
+
}
|
|
1575
|
+
function navigationControls(variant) {
|
|
1576
|
+
return navigationControlsForElements(variant.elements);
|
|
1577
|
+
}
|
|
1578
|
+
function navigationControlsForElements(elements) {
|
|
1579
|
+
const candidates = elements
|
|
1580
|
+
.filter((element) => element.safety === 'safe' && element.stable && isLikelyTapElement(element))
|
|
1581
|
+
.filter((element) => !isBackControl(element))
|
|
1582
|
+
.filter((element) => coordinateArgsForElement(element) !== null)
|
|
1583
|
+
.map((element) => ({ element, target: normalizeControlTarget(plainTapTargets(element)[0] ?? element.selector) }))
|
|
1584
|
+
.filter((candidate) => isCompactNavigationLabel(candidate.element, candidate.target));
|
|
1585
|
+
if (candidates.length < 2) {
|
|
1586
|
+
return [];
|
|
1587
|
+
}
|
|
1588
|
+
const maxY = Math.max(...candidates.map((candidate) => candidate.element.y ?? 0));
|
|
1589
|
+
const bottomRow = candidates
|
|
1590
|
+
.filter((candidate) => (candidate.element.y ?? 0) >= maxY - 12)
|
|
1591
|
+
.sort((left, right) => (left.element.x ?? 0) - (right.element.x ?? 0));
|
|
1592
|
+
if (bottomRow.length < 2) {
|
|
1593
|
+
return [];
|
|
1594
|
+
}
|
|
1595
|
+
return bottomRow.map((candidate, priority) => ({
|
|
1596
|
+
...candidate,
|
|
1597
|
+
priority
|
|
1598
|
+
}));
|
|
1599
|
+
}
|
|
1600
|
+
function isCompactNavigationLabel(element, target) {
|
|
1601
|
+
if (!target || target.includes('\n') || target.length > 32 || target === 'see all') {
|
|
1602
|
+
return false;
|
|
1603
|
+
}
|
|
1604
|
+
if ((element.height ?? 0) > 72) {
|
|
1605
|
+
return false;
|
|
1606
|
+
}
|
|
1607
|
+
const lines = unique(element.labels.flatMap((label) => label.split(/\n+/).map((line) => normalizeControlTarget(line))));
|
|
1608
|
+
return lines.length <= 2 && lines.some((line) => line === target);
|
|
1609
|
+
}
|
|
1610
|
+
function isNavigationElement(variant, element) {
|
|
1611
|
+
return navigationControls(variant).some((control) => control.element === element);
|
|
1612
|
+
}
|
|
1613
|
+
function isNavigationTarget(variant, normalizedTarget) {
|
|
1614
|
+
return navigationControls(variant).some((control) => control.target === normalizedTarget);
|
|
1615
|
+
}
|
|
1616
|
+
function navigationControlPriority(variant, normalizedTarget) {
|
|
1617
|
+
return navigationControls(variant).find((control) => control.target === normalizedTarget)?.priority;
|
|
1618
|
+
}
|
|
1619
|
+
function isPrimaryNavigationTarget(variant, normalizedTarget) {
|
|
1620
|
+
return navigationControls(variant)[0]?.target === normalizedTarget;
|
|
1621
|
+
}
|
|
1622
|
+
function labelMatchesNavigationTarget(label, target) {
|
|
1623
|
+
if (label === target) {
|
|
1624
|
+
return true;
|
|
1625
|
+
}
|
|
1626
|
+
const labelWords = new Set(label.split(/[^a-z0-9]+/).filter(Boolean));
|
|
1627
|
+
const targetWords = target.split(/[^a-z0-9]+/).filter(Boolean);
|
|
1628
|
+
if (targetWords.length === 1) {
|
|
1629
|
+
const words = Array.from(labelWords);
|
|
1630
|
+
return words.length <= 2 && words[0] === targetWords[0];
|
|
1631
|
+
}
|
|
1632
|
+
return targetWords.length > 0 && targetWords.every((word) => labelWords.has(word));
|
|
1633
|
+
}
|
|
1634
|
+
function currentScreenTapArgs(variant, descriptor) {
|
|
1635
|
+
if (descriptor.kind === 'section-first') {
|
|
1636
|
+
return sectionFirstTapArgs(variant, descriptor);
|
|
1637
|
+
}
|
|
1638
|
+
const action = matchingAction(variant, descriptor);
|
|
1639
|
+
if (action) {
|
|
1640
|
+
return structuredClone(action.args);
|
|
1641
|
+
}
|
|
1642
|
+
for (const element of variant.elements) {
|
|
1643
|
+
if (!elementContainsTarget(element, descriptor)) {
|
|
1644
|
+
continue;
|
|
1645
|
+
}
|
|
1646
|
+
if (!(element.safety === 'safe' && element.stable && isLikelyTapElement(element))) {
|
|
1647
|
+
continue;
|
|
1648
|
+
}
|
|
1649
|
+
const coordinateArgs = coordinateArgsForElement(element);
|
|
1650
|
+
if (coordinateArgs) {
|
|
1651
|
+
return coordinateArgs;
|
|
1652
|
+
}
|
|
1653
|
+
const stableTarget = plainTapTargets(element)[0] ?? element.selector;
|
|
1654
|
+
if (stableTarget && shouldUseStableTargetFallback(element, descriptor)) {
|
|
1655
|
+
return { target: stableTarget };
|
|
1656
|
+
}
|
|
1657
|
+
}
|
|
1658
|
+
return null;
|
|
1659
|
+
}
|
|
1660
|
+
function currentScreenLiveTapArgs(variant, descriptor) {
|
|
1661
|
+
const action = matchingAction(variant, descriptor);
|
|
1662
|
+
return action && shouldUseActionForLiveSemanticTap(action)
|
|
1663
|
+
? structuredClone(action.args)
|
|
1664
|
+
: null;
|
|
1665
|
+
}
|
|
1666
|
+
function matchingAction(variant, descriptor) {
|
|
1667
|
+
return variant.actions.find((candidate) => actionMatchesDescriptor(candidate, descriptor));
|
|
1668
|
+
}
|
|
1669
|
+
function actionMatchesDescriptor(action, descriptor) {
|
|
1670
|
+
if (action.command !== 'tap' || action.safety !== 'safe') {
|
|
1671
|
+
return false;
|
|
1672
|
+
}
|
|
1673
|
+
if (descriptor.kind === 'text-contains') {
|
|
1674
|
+
return false;
|
|
1675
|
+
}
|
|
1676
|
+
const target = descriptor.target ?? '';
|
|
1677
|
+
const value = descriptor.value ?? target;
|
|
1678
|
+
const normalizedValue = normalizeControlTarget(value);
|
|
1679
|
+
const normalizedTarget = normalizeControlTarget(target);
|
|
1680
|
+
const normalizedActionLabel = normalizeControlTarget(action.label);
|
|
1681
|
+
const normalizedActionTarget = normalizeControlTarget(action.target ?? '');
|
|
1682
|
+
if (!normalizedValue && !normalizedTarget) {
|
|
1683
|
+
return false;
|
|
1684
|
+
}
|
|
1685
|
+
return [normalizedActionLabel, normalizedActionTarget].some((candidate) => candidate !== '' && (candidate === normalizedValue || candidate === normalizedTarget));
|
|
1686
|
+
}
|
|
1687
|
+
function shouldUseActionForLiveSemanticTap(action) {
|
|
1688
|
+
return (typeof action.args.x === 'number' &&
|
|
1689
|
+
typeof action.args.y === 'number' &&
|
|
1690
|
+
action.source.tag.toLowerCase().includes('xcuielementtype'));
|
|
1691
|
+
}
|
|
1692
|
+
function shouldUseStableTargetFallback(element, descriptor) {
|
|
1693
|
+
if (descriptor.kind !== 'text-contains') {
|
|
1694
|
+
return false;
|
|
1695
|
+
}
|
|
1696
|
+
const value = descriptor.value ?? '';
|
|
1697
|
+
return value !== '' && !element.labels.some((label) => label.includes(value));
|
|
1698
|
+
}
|
|
1699
|
+
function sectionFirstTapArgs(variant, descriptor) {
|
|
1700
|
+
const element = firstItemInSection(variant, descriptor);
|
|
1701
|
+
return element ? coordinateArgsForElement(element) : null;
|
|
1702
|
+
}
|
|
1703
|
+
function firstItemInSection(variant, descriptor) {
|
|
1704
|
+
const value = descriptor.value ?? descriptor.target ?? '';
|
|
1705
|
+
if (!/[a-z]/i.test(value)) {
|
|
1706
|
+
return null;
|
|
1707
|
+
}
|
|
1708
|
+
const section = sectionHeadingElement(variant, value);
|
|
1709
|
+
if (!section || section.y === null || section.height === null) {
|
|
1710
|
+
return null;
|
|
1711
|
+
}
|
|
1712
|
+
const sectionY = section.y;
|
|
1713
|
+
const sectionBottom = sectionY + Math.max(0, section.height);
|
|
1714
|
+
const nextSectionY = variant.elements
|
|
1715
|
+
.filter((element) => element !== section &&
|
|
1716
|
+
element.y !== null &&
|
|
1717
|
+
element.y > sectionY &&
|
|
1718
|
+
isLikelySectionHeading(variant, element))
|
|
1719
|
+
.map((element) => element.y)
|
|
1720
|
+
.sort((left, right) => left - right)[0] ?? null;
|
|
1721
|
+
const candidates = variant.elements
|
|
1722
|
+
.filter((element) => isSectionItemCandidate(variant, element, value) &&
|
|
1723
|
+
element.y !== null &&
|
|
1724
|
+
element.y > sectionBottom &&
|
|
1725
|
+
(nextSectionY === null || element.y < nextSectionY))
|
|
1726
|
+
.sort((left, right) => (left.y ?? 0) - (right.y ?? 0) || (left.x ?? 0) - (right.x ?? 0));
|
|
1727
|
+
return candidates[0] ?? null;
|
|
1728
|
+
}
|
|
1729
|
+
function sectionHeadingElement(variant, value) {
|
|
1730
|
+
const matches = variant.elements
|
|
1731
|
+
.filter((element) => element.visible !== false &&
|
|
1732
|
+
element.y !== null &&
|
|
1733
|
+
element.height !== null &&
|
|
1734
|
+
elementLabelMatchesValue(element, value))
|
|
1735
|
+
.sort((left, right) => (left.y ?? 0) - (right.y ?? 0) || (left.x ?? 0) - (right.x ?? 0));
|
|
1736
|
+
return matches[0] ?? null;
|
|
1737
|
+
}
|
|
1738
|
+
function isSectionItemCandidate(variant, element, sectionValue) {
|
|
1739
|
+
if (!(element.safety === 'safe' && element.stable && isLikelyTapElement(element))) {
|
|
1740
|
+
return false;
|
|
1741
|
+
}
|
|
1742
|
+
if (!coordinateArgsForElement(element)) {
|
|
1743
|
+
return false;
|
|
1744
|
+
}
|
|
1745
|
+
if (isBackControl(element) || elementLabelMatchesValue(element, sectionValue)) {
|
|
1746
|
+
return false;
|
|
1747
|
+
}
|
|
1748
|
+
const target = normalizeControlTarget(plainTapTargets(element)[0] ?? element.selector);
|
|
1749
|
+
if (isNavigationTarget(variant, target) || target === 'see all') {
|
|
1750
|
+
return false;
|
|
1751
|
+
}
|
|
1752
|
+
return true;
|
|
1753
|
+
}
|
|
1754
|
+
function isLikelySectionHeading(variant, element) {
|
|
1755
|
+
if (isHighValueContentCard(element)) {
|
|
1756
|
+
return false;
|
|
1757
|
+
}
|
|
1758
|
+
const labels = element.labels.map(normalizeControlTarget);
|
|
1759
|
+
if (labels.some((label) => label === 'see all' || isNavigationTarget(variant, label))) {
|
|
1760
|
+
return false;
|
|
1761
|
+
}
|
|
1762
|
+
return labels.some((label) => {
|
|
1763
|
+
if (!/[a-z]/i.test(label)) {
|
|
1764
|
+
return false;
|
|
1765
|
+
}
|
|
1766
|
+
const words = label.split(/\s+/).filter(Boolean);
|
|
1767
|
+
const lineCount = label.split(/\n+/).filter(Boolean).length;
|
|
1768
|
+
return words.length <= 8 && lineCount <= 2;
|
|
1769
|
+
});
|
|
1770
|
+
}
|
|
1771
|
+
function elementLabelMatchesValue(element, value) {
|
|
1772
|
+
const normalizedValue = normalizeControlTarget(value);
|
|
1773
|
+
if (!normalizedValue) {
|
|
1774
|
+
return false;
|
|
1775
|
+
}
|
|
1776
|
+
return element.labels.some((label) => {
|
|
1777
|
+
const normalizedLabel = normalizeControlTarget(label);
|
|
1778
|
+
return (normalizedLabel === normalizedValue ||
|
|
1779
|
+
normalizedLabel.includes(normalizedValue) ||
|
|
1780
|
+
label.split(/\n+/).some((line) => normalizeControlTarget(line) === normalizedValue));
|
|
1781
|
+
});
|
|
1782
|
+
}
|
|
1783
|
+
function coordinateArgsForElement(element) {
|
|
1784
|
+
if (element.x === null ||
|
|
1785
|
+
element.y === null ||
|
|
1786
|
+
element.width === null ||
|
|
1787
|
+
element.height === null ||
|
|
1788
|
+
element.width <= 0 ||
|
|
1789
|
+
element.height <= 0) {
|
|
1790
|
+
return null;
|
|
1791
|
+
}
|
|
1792
|
+
if (isBackControl(element)) {
|
|
1793
|
+
return {
|
|
1794
|
+
x: Math.round(element.x + Math.min(40, element.width / 2)),
|
|
1795
|
+
y: Math.round(element.y + element.height / 2)
|
|
1796
|
+
};
|
|
1797
|
+
}
|
|
1798
|
+
return {
|
|
1799
|
+
x: Math.round(element.x + element.width / 2),
|
|
1800
|
+
y: Math.round(element.y + element.height / 2)
|
|
1801
|
+
};
|
|
1802
|
+
}
|
|
1803
|
+
function isBackControl(element) {
|
|
1804
|
+
if (!element.tag.toLowerCase().includes('button')) {
|
|
1805
|
+
return false;
|
|
1806
|
+
}
|
|
1807
|
+
return element.labels.some((label) => label.split(/\n+/).some((line) => line.trim().toLowerCase() === 'back'));
|
|
1808
|
+
}
|
|
1809
|
+
function plainTapTargets(element) {
|
|
1810
|
+
return element.targets
|
|
1811
|
+
.filter((target) => !target.includes('='))
|
|
1812
|
+
.sort((left, right) => tapTargetScore(left) - tapTargetScore(right));
|
|
1813
|
+
}
|
|
1814
|
+
function tapTargetScore(target) {
|
|
1815
|
+
let score = target.length;
|
|
1816
|
+
if (target.includes('\n')) {
|
|
1817
|
+
score += 1000;
|
|
1818
|
+
}
|
|
1819
|
+
return score;
|
|
1820
|
+
}
|
|
1821
|
+
function isLikelyTapElement(element) {
|
|
1822
|
+
if (element.enabled === false || element.visible === false) {
|
|
1823
|
+
return false;
|
|
1824
|
+
}
|
|
1825
|
+
if (element.width !== null &&
|
|
1826
|
+
element.height !== null &&
|
|
1827
|
+
(element.width <= 0 || element.height <= 0)) {
|
|
1828
|
+
return false;
|
|
1829
|
+
}
|
|
1830
|
+
const tag = element.tag.toLowerCase();
|
|
1831
|
+
if (tag.includes('statictext') ||
|
|
1832
|
+
tag.includes('image') ||
|
|
1833
|
+
tag.includes('textfield') ||
|
|
1834
|
+
isScrollableTag(tag)) {
|
|
1835
|
+
return false;
|
|
1836
|
+
}
|
|
1837
|
+
if (tag.includes('button') || element.clickable) {
|
|
1838
|
+
return true;
|
|
1839
|
+
}
|
|
1840
|
+
if (tag.includes('other')) {
|
|
1841
|
+
const width = element.width ?? 0;
|
|
1842
|
+
const height = element.height ?? 0;
|
|
1843
|
+
return width >= 20 && height >= 20 && element.labels.length > 0;
|
|
1844
|
+
}
|
|
1845
|
+
return false;
|
|
1846
|
+
}
|
|
1847
|
+
async function repairToTarget(context, current, descriptor, depth, deadline, blockedTargets) {
|
|
1848
|
+
if (Date.now() > deadline || depth <= 0) {
|
|
1849
|
+
return null;
|
|
1850
|
+
}
|
|
1851
|
+
const attemptedTargets = new Map();
|
|
1852
|
+
const attemptFrom = async (start, remainingDepth, route) => {
|
|
1853
|
+
let cursor = start;
|
|
1854
|
+
let depthLeft = remainingDepth;
|
|
1855
|
+
let routeSoFar = route;
|
|
1856
|
+
while (Date.now() <= deadline) {
|
|
1857
|
+
if (variantContainsTarget(cursor.variant, descriptor)) {
|
|
1858
|
+
return { found: true, current: cursor, route: routeSoFar, remainingDepth: depthLeft };
|
|
1859
|
+
}
|
|
1860
|
+
if (depthLeft <= 0) {
|
|
1861
|
+
return { found: false, current: cursor, route: routeSoFar, remainingDepth: depthLeft };
|
|
1862
|
+
}
|
|
1863
|
+
const attemptsForVariant = attemptedTargets.get(cursor.variant.id) ?? new Set();
|
|
1864
|
+
attemptedTargets.set(cursor.variant.id, attemptsForVariant);
|
|
1865
|
+
const candidate = safeTapCandidates(cursor.variant).find((tapCandidate) => !blockedTargets.has(tapCandidate.target) &&
|
|
1866
|
+
!blockedTargets.has(tapCandidate.key) &&
|
|
1867
|
+
!attemptsForVariant.has(tapCandidate.key));
|
|
1868
|
+
if (!candidate) {
|
|
1869
|
+
return { found: false, current: cursor, route: routeSoFar, remainingDepth: depthLeft };
|
|
1870
|
+
}
|
|
1871
|
+
attemptsForVariant.add(candidate.key);
|
|
1872
|
+
const args = candidate.args;
|
|
1873
|
+
let observed;
|
|
1874
|
+
try {
|
|
1875
|
+
await context.adapter.tap(args);
|
|
1876
|
+
observed = await observeCurrentScreen(context);
|
|
1877
|
+
}
|
|
1878
|
+
catch {
|
|
1879
|
+
try {
|
|
1880
|
+
cursor = await observeCurrentScreen(context);
|
|
1881
|
+
}
|
|
1882
|
+
catch {
|
|
1883
|
+
// If both the exploratory tap and source read fail, keep the last observed screen.
|
|
1884
|
+
}
|
|
1885
|
+
continue;
|
|
1886
|
+
}
|
|
1887
|
+
recordEdgeSuccess(context, cursor.variant, observed.variant, 'tap', args, true);
|
|
1888
|
+
const nextRoute = [...routeSoFar, summarizeControl('tap', candidate.target, 0.45)];
|
|
1889
|
+
const nested = await attemptFrom(observed, depthLeft - 1, nextRoute);
|
|
1890
|
+
if (nested.found) {
|
|
1891
|
+
return nested;
|
|
1892
|
+
}
|
|
1893
|
+
if (nested.current.variant.id === cursor.variant.id) {
|
|
1894
|
+
cursor = nested.current;
|
|
1895
|
+
routeSoFar = nested.route;
|
|
1896
|
+
continue;
|
|
1897
|
+
}
|
|
1898
|
+
cursor = nested.current;
|
|
1899
|
+
depthLeft = nested.remainingDepth;
|
|
1900
|
+
routeSoFar = nested.route;
|
|
1901
|
+
}
|
|
1902
|
+
return { found: false, current: cursor, route: routeSoFar, remainingDepth: depthLeft };
|
|
614
1903
|
};
|
|
615
1904
|
const attempt = await attemptFrom(current, depth, []);
|
|
616
1905
|
return attempt.found ? { current: attempt.current, route: attempt.route } : null;
|
|
@@ -625,63 +1914,537 @@ export async function runMappedCommand(context, command, args) {
|
|
|
625
1914
|
repairs: 0,
|
|
626
1915
|
path: context.filePath
|
|
627
1916
|
};
|
|
628
|
-
if (descriptor.kind === 'unknown'
|
|
1917
|
+
if (descriptor.kind === 'unknown') {
|
|
629
1918
|
const details = await context.adapter[command](args);
|
|
630
1919
|
return { ...details, map: metadata };
|
|
631
1920
|
}
|
|
632
1921
|
let before = await observeCurrentScreen(context);
|
|
633
|
-
if (
|
|
1922
|
+
if (descriptor.kind === 'coordinate' || command === 'scroll') {
|
|
1923
|
+
const details = await context.adapter[command](args);
|
|
1924
|
+
const after = await observeCurrentScreen(context);
|
|
1925
|
+
assertObservedEffect(command, args, before, after, descriptor);
|
|
1926
|
+
recordEdgeSuccess(context, before.variant, after.variant, command, args, false);
|
|
1927
|
+
return { ...details, map: metadata };
|
|
1928
|
+
}
|
|
1929
|
+
let liveTargetVisible = await liveContainsTarget(context, descriptor);
|
|
1930
|
+
if (!variantContainsTarget(before.variant, descriptor) && !liveTargetVisible) {
|
|
634
1931
|
if (before.variant.auth_required) {
|
|
635
1932
|
throw new Error(`Target '${descriptor.target ?? descriptor.value}' requires authentication from the current screen`);
|
|
636
1933
|
}
|
|
637
|
-
|
|
638
|
-
|
|
639
|
-
|
|
640
|
-
|
|
641
|
-
if (route.authRequired) {
|
|
642
|
-
throw new Error(`Target '${descriptor.target ?? descriptor.value}' is behind an auth-required screen`);
|
|
643
|
-
}
|
|
644
|
-
if (route.edges.length > 0) {
|
|
645
|
-
context.summary.used = true;
|
|
646
|
-
metadata.routed = true;
|
|
647
|
-
metadata.route = route.edges.map(summarizeEdge);
|
|
648
|
-
const driven = await driveRoute(context, before, route.edges);
|
|
649
|
-
before = driven.current;
|
|
650
|
-
if (driven.failedEdge) {
|
|
651
|
-
const repair = await repairToTarget(context, before, descriptor, context.options.repairDepth, Date.now() + context.options.repairTimeoutMs, new Set(driven.failedEdge.target ? [driven.failedEdge.target] : []));
|
|
652
|
-
if (!repair) {
|
|
653
|
-
throw new Error(`Cached app-map route for '${driven.failedEdge.target ?? driven.failedEdge.command}' could not be repaired within the bounded repair budget`);
|
|
654
|
-
}
|
|
655
|
-
context.summary.repaired = true;
|
|
656
|
-
context.summary.repairs += 1;
|
|
657
|
-
metadata.repaired = true;
|
|
658
|
-
metadata.repairs = context.summary.repairs;
|
|
659
|
-
metadata.route = [...metadata.route, ...repair.route];
|
|
660
|
-
before = repair.current;
|
|
1934
|
+
if (routeableDescriptor(descriptor)) {
|
|
1935
|
+
const route = planRoute(context.map, before.variant.id, descriptor);
|
|
1936
|
+
if (route.ambiguous) {
|
|
1937
|
+
throw new Error(`Target '${descriptor.target ?? descriptor.value}' is ambiguous in the app map`);
|
|
661
1938
|
}
|
|
662
|
-
|
|
663
|
-
|
|
664
|
-
|
|
665
|
-
if (
|
|
1939
|
+
if (route.authRequired) {
|
|
1940
|
+
throw new Error(`Target '${descriptor.target ?? descriptor.value}' is behind an auth-required screen`);
|
|
1941
|
+
}
|
|
1942
|
+
if (route.edges.length > 0) {
|
|
666
1943
|
context.summary.used = true;
|
|
667
|
-
|
|
668
|
-
|
|
669
|
-
|
|
670
|
-
|
|
671
|
-
|
|
672
|
-
|
|
673
|
-
before =
|
|
1944
|
+
metadata.routed = true;
|
|
1945
|
+
metadata.route = route.edges.map(summarizeEdge);
|
|
1946
|
+
if (route.diagnostics) {
|
|
1947
|
+
metadata.diagnostics = route.diagnostics;
|
|
1948
|
+
}
|
|
1949
|
+
const driven = await driveRoute(context, before, route.edges);
|
|
1950
|
+
before = driven.current;
|
|
1951
|
+
liveTargetVisible = await liveContainsTarget(context, descriptor);
|
|
1952
|
+
const targetReachedDespiteStaleContract = variantContainsTarget(before.variant, descriptor) || liveTargetVisible;
|
|
1953
|
+
if (driven.failedEdge && !targetReachedDespiteStaleContract) {
|
|
1954
|
+
if (!context.options.repair) {
|
|
1955
|
+
throw new Error(`Cached app-map route for '${driven.failedEdge.target ?? driven.failedEdge.command}' failed; rerun with repair enabled to explore an alternate route`);
|
|
1956
|
+
}
|
|
1957
|
+
const repair = await repairToTarget(context, before, descriptor, context.options.repairDepth, Date.now() + context.options.repairTimeoutMs, new Set(driven.failedEdge.target ? [driven.failedEdge.target] : []));
|
|
1958
|
+
if (!repair) {
|
|
1959
|
+
throw new Error(`Cached app-map route for '${driven.failedEdge.target ?? driven.failedEdge.command}' could not be repaired within the bounded repair budget`);
|
|
1960
|
+
}
|
|
1961
|
+
context.summary.repaired = true;
|
|
1962
|
+
context.summary.repairs += 1;
|
|
1963
|
+
metadata.repaired = true;
|
|
1964
|
+
metadata.repairs = context.summary.repairs;
|
|
1965
|
+
metadata.route = [...metadata.route, ...repair.route];
|
|
1966
|
+
before = repair.current;
|
|
1967
|
+
liveTargetVisible = await liveContainsTarget(context, descriptor);
|
|
1968
|
+
}
|
|
1969
|
+
}
|
|
1970
|
+
else if (context.options.repair) {
|
|
1971
|
+
const repair = await repairToTarget(context, before, descriptor, context.options.repairDepth, Date.now() + context.options.repairTimeoutMs, new Set());
|
|
1972
|
+
if (repair) {
|
|
1973
|
+
context.summary.used = true;
|
|
1974
|
+
context.summary.repaired = true;
|
|
1975
|
+
context.summary.repairs += 1;
|
|
1976
|
+
metadata.routed = repair.route.length > 0;
|
|
1977
|
+
metadata.repaired = true;
|
|
1978
|
+
metadata.repairs = context.summary.repairs;
|
|
1979
|
+
metadata.route = repair.route;
|
|
1980
|
+
before = repair.current;
|
|
1981
|
+
liveTargetVisible = await liveContainsTarget(context, descriptor);
|
|
1982
|
+
}
|
|
674
1983
|
}
|
|
675
1984
|
}
|
|
676
1985
|
}
|
|
677
|
-
if (!variantContainsTarget(before.variant, descriptor)) {
|
|
1986
|
+
if (!variantContainsTarget(before.variant, descriptor) && !liveTargetVisible) {
|
|
678
1987
|
throw new Error(`Target '${descriptor.target ?? descriptor.value}' is not reachable from the current app-map state`);
|
|
679
1988
|
}
|
|
680
|
-
const
|
|
1989
|
+
const executableArgs = command === 'tap'
|
|
1990
|
+
? liveTargetVisible
|
|
1991
|
+
? currentScreenLiveTapArgs(before.variant, descriptor) ?? args
|
|
1992
|
+
: currentScreenTapArgs(before.variant, descriptor) ?? args
|
|
1993
|
+
: args;
|
|
1994
|
+
const details = await context.adapter[command](executableArgs);
|
|
681
1995
|
const after = await observeCurrentScreen(context);
|
|
682
|
-
|
|
1996
|
+
assertObservedEffect(command, executableArgs, before, after, descriptor);
|
|
1997
|
+
recordEdgeSuccess(context, before.variant, after.variant, command, executableArgs, false);
|
|
683
1998
|
return { ...details, map: metadata };
|
|
684
1999
|
}
|
|
2000
|
+
function restoreAttemptDiagnostic(strategy, result, observed, match, target, command, error) {
|
|
2001
|
+
return {
|
|
2002
|
+
strategy,
|
|
2003
|
+
result,
|
|
2004
|
+
...(command ? { command } : {}),
|
|
2005
|
+
...(target ? { target } : {}),
|
|
2006
|
+
...(observed
|
|
2007
|
+
? {
|
|
2008
|
+
observed_variant_id: observed.variant.id,
|
|
2009
|
+
observed_screen_id: observed.variant.screen_id
|
|
2010
|
+
}
|
|
2011
|
+
: {}),
|
|
2012
|
+
...(match ? { accepted_by: match.acceptedBy } : {}),
|
|
2013
|
+
...(error instanceof Error ? { error: error.message } : {})
|
|
2014
|
+
};
|
|
2015
|
+
}
|
|
2016
|
+
function makeRestoreDiagnostic(from, target) {
|
|
2017
|
+
return {
|
|
2018
|
+
from_variant_id: from.variant.id,
|
|
2019
|
+
target_variant_id: target.variant.id,
|
|
2020
|
+
result: 'failed',
|
|
2021
|
+
attempts: []
|
|
2022
|
+
};
|
|
2023
|
+
}
|
|
2024
|
+
async function tryTapRestoreCandidate(context, current, target, candidate, strategy, diagnostic) {
|
|
2025
|
+
const args = candidate.args;
|
|
2026
|
+
try {
|
|
2027
|
+
await context.adapter.tap(args);
|
|
2028
|
+
const observed = await observeAfterAction(context, target);
|
|
2029
|
+
recordEdgeSuccess(context, current.variant, observed.variant, 'tap', args, true);
|
|
2030
|
+
const match = restoreMatch(observed, target);
|
|
2031
|
+
diagnostic.attempts.push(restoreAttemptDiagnostic(strategy, match ? 'matched' : 'mismatched', observed, match, targetDescriptor('tap', args).target ?? candidate.target, 'tap'));
|
|
2032
|
+
return { current: observed, match };
|
|
2033
|
+
}
|
|
2034
|
+
catch (error) {
|
|
2035
|
+
diagnostic.attempts.push(restoreAttemptDiagnostic(strategy, 'error', undefined, null, candidate.target, 'tap', error));
|
|
2036
|
+
try {
|
|
2037
|
+
return { current: await observeCurrentScreen(context), match: null };
|
|
2038
|
+
}
|
|
2039
|
+
catch {
|
|
2040
|
+
return { current, match: null };
|
|
2041
|
+
}
|
|
2042
|
+
}
|
|
2043
|
+
}
|
|
2044
|
+
async function tryScrollRestoreCandidate(context, current, target, candidate, diagnostic) {
|
|
2045
|
+
const args = candidate.args;
|
|
2046
|
+
try {
|
|
2047
|
+
await context.adapter.scroll(args);
|
|
2048
|
+
const observed = await observeAfterAction(context, target);
|
|
2049
|
+
recordEdgeSuccess(context, current.variant, observed.variant, 'scroll', args, true);
|
|
2050
|
+
const match = restoreMatch(observed, target);
|
|
2051
|
+
diagnostic.attempts.push(restoreAttemptDiagnostic('reverse-scroll', match ? 'matched' : 'mismatched', observed, match, candidate.target, 'scroll'));
|
|
2052
|
+
return { current: observed, match };
|
|
2053
|
+
}
|
|
2054
|
+
catch (error) {
|
|
2055
|
+
diagnostic.attempts.push(restoreAttemptDiagnostic('reverse-scroll', 'error', undefined, null, candidate.target, 'scroll', error));
|
|
2056
|
+
try {
|
|
2057
|
+
return { current: await observeCurrentScreen(context), match: null };
|
|
2058
|
+
}
|
|
2059
|
+
catch {
|
|
2060
|
+
return { current, match: null };
|
|
2061
|
+
}
|
|
2062
|
+
}
|
|
2063
|
+
}
|
|
2064
|
+
function targetRestoreCandidates(current, target) {
|
|
2065
|
+
const preferredTarget = preferredNavigationTarget(target.variant);
|
|
2066
|
+
return safeTapCandidates(current.variant).filter((candidate) => {
|
|
2067
|
+
if (/^(back|close|dismiss|done)$/i.test(candidate.target)) {
|
|
2068
|
+
return false;
|
|
2069
|
+
}
|
|
2070
|
+
const normalizedTarget = normalizeControlTarget(candidate.target);
|
|
2071
|
+
if (preferredTarget) {
|
|
2072
|
+
return normalizedTarget === preferredTarget;
|
|
2073
|
+
}
|
|
2074
|
+
if (isGlobalIdentityLabel(candidate.target)) {
|
|
2075
|
+
return false;
|
|
2076
|
+
}
|
|
2077
|
+
const descriptor = targetDescriptor('tap', { target: candidate.target });
|
|
2078
|
+
return descriptor.kind !== 'unknown' && variantContainsTarget(target.variant, descriptor);
|
|
2079
|
+
});
|
|
2080
|
+
}
|
|
2081
|
+
function reverseScrollRestoreCandidates(current, target) {
|
|
2082
|
+
if (!shouldTryReverseScrollRestore(current.variant, target.variant)) {
|
|
2083
|
+
return [];
|
|
2084
|
+
}
|
|
2085
|
+
const args = { direction: 'up', percent: 70 };
|
|
2086
|
+
return [
|
|
2087
|
+
{
|
|
2088
|
+
target: scrollTarget(args),
|
|
2089
|
+
args,
|
|
2090
|
+
key: 'scroll:up:70'
|
|
2091
|
+
}
|
|
2092
|
+
];
|
|
2093
|
+
}
|
|
2094
|
+
function shouldTryReverseScrollRestore(current, target) {
|
|
2095
|
+
if (hasScrollableSurface(current) || hasScrollableSurface(target)) {
|
|
2096
|
+
return true;
|
|
2097
|
+
}
|
|
2098
|
+
const currentNavigationTarget = preferredNavigationTarget(current);
|
|
2099
|
+
const targetNavigationTarget = preferredNavigationTarget(target);
|
|
2100
|
+
if (currentNavigationTarget && targetNavigationTarget && currentNavigationTarget === targetNavigationTarget) {
|
|
2101
|
+
return true;
|
|
2102
|
+
}
|
|
2103
|
+
return similarity(current.element_keys, target.element_keys) >= 0.2;
|
|
2104
|
+
}
|
|
2105
|
+
function navigationResetCandidates(current, root) {
|
|
2106
|
+
const rootTarget = preferredNavigationTarget(root.variant);
|
|
2107
|
+
if (!rootTarget) {
|
|
2108
|
+
return [];
|
|
2109
|
+
}
|
|
2110
|
+
return safeTapCandidates(current.variant).filter((candidate) => normalizeControlTarget(candidate.target) === rootTarget);
|
|
2111
|
+
}
|
|
2112
|
+
function shortestRouteToVariant(map, fromVariantId, toVariantId) {
|
|
2113
|
+
if (fromVariantId === toVariantId) {
|
|
2114
|
+
return [];
|
|
2115
|
+
}
|
|
2116
|
+
const visited = new Set([fromVariantId]);
|
|
2117
|
+
const queue = [{ variantId: fromVariantId, edges: [] }];
|
|
2118
|
+
while (queue.length > 0) {
|
|
2119
|
+
const current = queue.shift();
|
|
2120
|
+
if (!current) {
|
|
2121
|
+
break;
|
|
2122
|
+
}
|
|
2123
|
+
const variant = map.variants.find((candidate) => candidate.id === current.variantId);
|
|
2124
|
+
const outgoing = [
|
|
2125
|
+
...map.edges,
|
|
2126
|
+
...navigationVirtualEdges(map, variant)
|
|
2127
|
+
]
|
|
2128
|
+
.filter((edge) => edge.from_variant_id === current.variantId && !edge.stale && edge.confidence >= 0.25 && replayableEdge(edge))
|
|
2129
|
+
.sort((left, right) => right.confidence - left.confidence);
|
|
2130
|
+
for (const edge of outgoing) {
|
|
2131
|
+
if (visited.has(edge.to_variant_id)) {
|
|
2132
|
+
continue;
|
|
2133
|
+
}
|
|
2134
|
+
const edges = [...current.edges, edge];
|
|
2135
|
+
if (edge.to_variant_id === toVariantId) {
|
|
2136
|
+
return edges;
|
|
2137
|
+
}
|
|
2138
|
+
visited.add(edge.to_variant_id);
|
|
2139
|
+
queue.push({ variantId: edge.to_variant_id, edges });
|
|
2140
|
+
}
|
|
2141
|
+
}
|
|
2142
|
+
return null;
|
|
2143
|
+
}
|
|
2144
|
+
async function restoreViaRootRoute(context, current, target, root, diagnostic) {
|
|
2145
|
+
let resetCurrent = current;
|
|
2146
|
+
const rootMatch = restoreMatch(resetCurrent, root);
|
|
2147
|
+
if (!rootMatch) {
|
|
2148
|
+
for (const candidate of navigationResetCandidates(resetCurrent, root)) {
|
|
2149
|
+
const restored = await tryTapRestoreCandidate(context, resetCurrent, root, candidate, 'root-navigation', diagnostic);
|
|
2150
|
+
resetCurrent = restored.current;
|
|
2151
|
+
if (restored.match) {
|
|
2152
|
+
break;
|
|
2153
|
+
}
|
|
2154
|
+
}
|
|
2155
|
+
}
|
|
2156
|
+
const matchedRoot = restoreMatch(resetCurrent, root);
|
|
2157
|
+
if (!matchedRoot) {
|
|
2158
|
+
return null;
|
|
2159
|
+
}
|
|
2160
|
+
if (restoreMatch(resetCurrent, target)) {
|
|
2161
|
+
diagnostic.result = 'restored';
|
|
2162
|
+
diagnostic.accepted_by = 'root';
|
|
2163
|
+
return { current: target, restored: true, acceptedBy: 'root', diagnostic };
|
|
2164
|
+
}
|
|
2165
|
+
const route = shortestRouteToVariant(context.map, root.variant.id, target.variant.id);
|
|
2166
|
+
if (!route || route.length === 0) {
|
|
2167
|
+
return null;
|
|
2168
|
+
}
|
|
2169
|
+
const driven = await driveRoute(context, root, route);
|
|
2170
|
+
const match = restoreMatch(driven.current, target);
|
|
2171
|
+
diagnostic.attempts.push(restoreAttemptDiagnostic('root-route', match ? 'matched' : 'mismatched', driven.current, match ? { ...match, acceptedBy: 'root-route' } : null, route.map((edge) => edge.target ?? edge.command).join(' -> '), 'tap'));
|
|
2172
|
+
if (!match) {
|
|
2173
|
+
return { current: driven.current, restored: false, diagnostic };
|
|
2174
|
+
}
|
|
2175
|
+
diagnostic.result = 'restored';
|
|
2176
|
+
diagnostic.accepted_by = 'root-route';
|
|
2177
|
+
return { current: target, restored: true, acceptedBy: 'root-route', diagnostic };
|
|
2178
|
+
}
|
|
2179
|
+
async function restoreToVariant(context, current, target, root) {
|
|
2180
|
+
const diagnostic = makeRestoreDiagnostic(current, target);
|
|
2181
|
+
const initialMatch = restoreMatch(current, target);
|
|
2182
|
+
if (initialMatch) {
|
|
2183
|
+
diagnostic.result = 'restored';
|
|
2184
|
+
diagnostic.accepted_by = initialMatch.acceptedBy;
|
|
2185
|
+
return { current: target, restored: true, acceptedBy: initialMatch.acceptedBy, diagnostic };
|
|
2186
|
+
}
|
|
2187
|
+
const restoreTargets = safeTapCandidates(current.variant).filter((candidate) => /^(back|close|dismiss|done)$/i.test(candidate.target));
|
|
2188
|
+
for (const restoreCandidate of restoreTargets) {
|
|
2189
|
+
const restored = await tryTapRestoreCandidate(context, current, target, restoreCandidate, 'dismiss-control', diagnostic);
|
|
2190
|
+
current = restored.current;
|
|
2191
|
+
if (restored.match) {
|
|
2192
|
+
diagnostic.result = 'restored';
|
|
2193
|
+
diagnostic.accepted_by = restored.match.acceptedBy;
|
|
2194
|
+
return { current: target, restored: true, acceptedBy: restored.match.acceptedBy, diagnostic };
|
|
2195
|
+
}
|
|
2196
|
+
}
|
|
2197
|
+
for (const restoreCandidate of targetRestoreCandidates(current, target)) {
|
|
2198
|
+
const restored = await tryTapRestoreCandidate(context, current, target, restoreCandidate, 'target-navigation', diagnostic);
|
|
2199
|
+
current = restored.current;
|
|
2200
|
+
if (restored.match) {
|
|
2201
|
+
diagnostic.result = 'restored';
|
|
2202
|
+
diagnostic.accepted_by = restored.match.acceptedBy;
|
|
2203
|
+
return { current: target, restored: true, acceptedBy: restored.match.acceptedBy, diagnostic };
|
|
2204
|
+
}
|
|
2205
|
+
}
|
|
2206
|
+
for (const restoreCandidate of reverseScrollRestoreCandidates(current, target)) {
|
|
2207
|
+
const restored = await tryScrollRestoreCandidate(context, current, target, restoreCandidate, diagnostic);
|
|
2208
|
+
current = restored.current;
|
|
2209
|
+
if (restored.match) {
|
|
2210
|
+
diagnostic.result = 'restored';
|
|
2211
|
+
diagnostic.accepted_by = restored.match.acceptedBy;
|
|
2212
|
+
return { current: target, restored: true, acceptedBy: restored.match.acceptedBy, diagnostic };
|
|
2213
|
+
}
|
|
2214
|
+
}
|
|
2215
|
+
try {
|
|
2216
|
+
await context.adapter.act({ name: 'back' });
|
|
2217
|
+
const observed = await observeAfterAction(context, target);
|
|
2218
|
+
const match = restoreMatch(observed, target);
|
|
2219
|
+
diagnostic.attempts.push(restoreAttemptDiagnostic('platform-back', match ? 'matched' : 'mismatched', observed, match, 'back', 'act'));
|
|
2220
|
+
if (match) {
|
|
2221
|
+
diagnostic.result = 'restored';
|
|
2222
|
+
diagnostic.accepted_by = match.acceptedBy;
|
|
2223
|
+
return { current: target, restored: true, acceptedBy: match.acceptedBy, diagnostic };
|
|
2224
|
+
}
|
|
2225
|
+
current = observed;
|
|
2226
|
+
}
|
|
2227
|
+
catch (error) {
|
|
2228
|
+
diagnostic.attempts.push(restoreAttemptDiagnostic('platform-back', 'error', undefined, null, 'back', 'act', error));
|
|
2229
|
+
}
|
|
2230
|
+
if (root) {
|
|
2231
|
+
const rooted = await restoreViaRootRoute(context, current, target, root, diagnostic);
|
|
2232
|
+
if (rooted) {
|
|
2233
|
+
return rooted;
|
|
2234
|
+
}
|
|
2235
|
+
}
|
|
2236
|
+
return { current, restored: false, diagnostic };
|
|
2237
|
+
}
|
|
2238
|
+
async function crawlAppMap(context, start) {
|
|
2239
|
+
const visited = new Set();
|
|
2240
|
+
const visitedTemplates = new Set();
|
|
2241
|
+
const exploredRepeatedContentCandidates = new Set();
|
|
2242
|
+
const attemptedTargets = new Map();
|
|
2243
|
+
const restoreDiagnostics = [];
|
|
2244
|
+
let restoreFailures = 0;
|
|
2245
|
+
let actions = 0;
|
|
2246
|
+
let stoppedReason = 'complete';
|
|
2247
|
+
const restore = async (current, target) => {
|
|
2248
|
+
const result = await restoreToVariant(context, current, target, start);
|
|
2249
|
+
if (result.diagnostic.attempts.length > 0 || result.diagnostic.accepted_by) {
|
|
2250
|
+
restoreDiagnostics.push(result.diagnostic);
|
|
2251
|
+
}
|
|
2252
|
+
if (!result.restored) {
|
|
2253
|
+
restoreFailures += 1;
|
|
2254
|
+
}
|
|
2255
|
+
return result;
|
|
2256
|
+
};
|
|
2257
|
+
const markPartial = () => {
|
|
2258
|
+
if (stoppedReason === 'complete') {
|
|
2259
|
+
stoppedReason = 'partial';
|
|
2260
|
+
}
|
|
2261
|
+
};
|
|
2262
|
+
const visit = async (origin, depth, allowScrollFirst = true) => {
|
|
2263
|
+
if (depth <= 0) {
|
|
2264
|
+
return origin;
|
|
2265
|
+
}
|
|
2266
|
+
if (actions >= context.options.crawlLimit) {
|
|
2267
|
+
stoppedReason = 'limit';
|
|
2268
|
+
return origin;
|
|
2269
|
+
}
|
|
2270
|
+
const originTemplateKey = crawlTemplateKey(origin.variant);
|
|
2271
|
+
if (visitedTemplates.has(originTemplateKey) && !visited.has(origin.variant.id)) {
|
|
2272
|
+
return origin;
|
|
2273
|
+
}
|
|
2274
|
+
visitedTemplates.add(originTemplateKey);
|
|
2275
|
+
visited.add(origin.variant.id);
|
|
2276
|
+
const attemptsForVariant = attemptedTargets.get(origin.variant.id) ?? new Set();
|
|
2277
|
+
attemptedTargets.set(origin.variant.id, attemptsForVariant);
|
|
2278
|
+
const candidates = safeTapCandidates(origin.variant, crawlCandidateOptions(origin.variant))
|
|
2279
|
+
.filter((candidate) => !attemptsForVariant.has(candidate.key));
|
|
2280
|
+
let current = origin;
|
|
2281
|
+
const exploreScrollCandidates = async (scrollCandidatesToExplore) => {
|
|
2282
|
+
for (const candidate of scrollCandidatesToExplore) {
|
|
2283
|
+
if (actions >= context.options.crawlLimit) {
|
|
2284
|
+
stoppedReason = 'limit';
|
|
2285
|
+
return true;
|
|
2286
|
+
}
|
|
2287
|
+
if (!restoreMatch(current, origin)) {
|
|
2288
|
+
if (restoreMatch(current, start)) {
|
|
2289
|
+
markPartial();
|
|
2290
|
+
current = start;
|
|
2291
|
+
return true;
|
|
2292
|
+
}
|
|
2293
|
+
stoppedReason = 'restore_failed';
|
|
2294
|
+
return true;
|
|
2295
|
+
}
|
|
2296
|
+
current = origin;
|
|
2297
|
+
attemptsForVariant.add(candidate.key);
|
|
2298
|
+
const args = candidate.args;
|
|
2299
|
+
let observed;
|
|
2300
|
+
try {
|
|
2301
|
+
await context.adapter.scroll(args);
|
|
2302
|
+
actions += 1;
|
|
2303
|
+
observed = await observeAfterCrawlAction(context);
|
|
2304
|
+
}
|
|
2305
|
+
catch {
|
|
2306
|
+
try {
|
|
2307
|
+
current = await observeCurrentScreen(context);
|
|
2308
|
+
}
|
|
2309
|
+
catch {
|
|
2310
|
+
// Stay on the last known origin if a failed scroll also prevents source capture.
|
|
2311
|
+
}
|
|
2312
|
+
continue;
|
|
2313
|
+
}
|
|
2314
|
+
if (observed.variant.id === origin.variant.id || observed.observation.fingerprint === origin.observation.fingerprint) {
|
|
2315
|
+
current = origin;
|
|
2316
|
+
continue;
|
|
2317
|
+
}
|
|
2318
|
+
recordEdgeSuccess(context, origin.variant, observed.variant, 'scroll', args, true);
|
|
2319
|
+
visited.add(observed.variant.id);
|
|
2320
|
+
if (observed.variant.id !== origin.variant.id && depth > 1) {
|
|
2321
|
+
const nestedCurrent = await visit(observed, depth - 1, false);
|
|
2322
|
+
const restored = await restore(nestedCurrent, origin);
|
|
2323
|
+
current = restored.current;
|
|
2324
|
+
if (!restored.restored) {
|
|
2325
|
+
if (restoreMatch(current, start)) {
|
|
2326
|
+
markPartial();
|
|
2327
|
+
current = start;
|
|
2328
|
+
return true;
|
|
2329
|
+
}
|
|
2330
|
+
stoppedReason = 'restore_failed';
|
|
2331
|
+
return true;
|
|
2332
|
+
}
|
|
2333
|
+
}
|
|
2334
|
+
else {
|
|
2335
|
+
const restored = await restore(observed, origin);
|
|
2336
|
+
current = restored.current;
|
|
2337
|
+
if (!restored.restored) {
|
|
2338
|
+
if (restoreMatch(current, start)) {
|
|
2339
|
+
markPartial();
|
|
2340
|
+
if (restoreMatch(origin, start)) {
|
|
2341
|
+
current = origin;
|
|
2342
|
+
continue;
|
|
2343
|
+
}
|
|
2344
|
+
current = start;
|
|
2345
|
+
return true;
|
|
2346
|
+
}
|
|
2347
|
+
stoppedReason = 'restore_failed';
|
|
2348
|
+
return true;
|
|
2349
|
+
}
|
|
2350
|
+
}
|
|
2351
|
+
}
|
|
2352
|
+
return false;
|
|
2353
|
+
};
|
|
2354
|
+
const scrollCandidates = crawlScrollCandidates(context, origin.variant)
|
|
2355
|
+
.filter((candidate) => !attemptsForVariant.has(candidate.key));
|
|
2356
|
+
if (allowScrollFirst && hasScrollableSurface(origin.variant) && await exploreScrollCandidates(scrollCandidates)) {
|
|
2357
|
+
return current;
|
|
2358
|
+
}
|
|
2359
|
+
for (const candidate of candidates) {
|
|
2360
|
+
const repeatedCandidateKey = repeatedContentCandidateKey(origin.variant, candidate);
|
|
2361
|
+
if (repeatedCandidateKey && exploredRepeatedContentCandidates.has(repeatedCandidateKey)) {
|
|
2362
|
+
continue;
|
|
2363
|
+
}
|
|
2364
|
+
if (actions >= context.options.crawlLimit) {
|
|
2365
|
+
stoppedReason = 'limit';
|
|
2366
|
+
return current;
|
|
2367
|
+
}
|
|
2368
|
+
if (!restoreMatch(current, origin)) {
|
|
2369
|
+
if (restoreMatch(current, start)) {
|
|
2370
|
+
markPartial();
|
|
2371
|
+
return start;
|
|
2372
|
+
}
|
|
2373
|
+
stoppedReason = 'restore_failed';
|
|
2374
|
+
return current;
|
|
2375
|
+
}
|
|
2376
|
+
current = origin;
|
|
2377
|
+
attemptsForVariant.add(candidate.key);
|
|
2378
|
+
const args = candidate.args;
|
|
2379
|
+
let observed;
|
|
2380
|
+
try {
|
|
2381
|
+
await context.adapter.tap(args);
|
|
2382
|
+
actions += 1;
|
|
2383
|
+
observed = await observeAfterCrawlAction(context);
|
|
2384
|
+
}
|
|
2385
|
+
catch {
|
|
2386
|
+
try {
|
|
2387
|
+
current = await observeCurrentScreen(context);
|
|
2388
|
+
}
|
|
2389
|
+
catch {
|
|
2390
|
+
// Stay on the last known origin if a failed tap also prevents source capture.
|
|
2391
|
+
}
|
|
2392
|
+
continue;
|
|
2393
|
+
}
|
|
2394
|
+
recordEdgeSuccess(context, origin.variant, observed.variant, 'tap', args, true);
|
|
2395
|
+
visited.add(observed.variant.id);
|
|
2396
|
+
if (repeatedCandidateKey && observed.variant.id !== origin.variant.id) {
|
|
2397
|
+
exploredRepeatedContentCandidates.add(repeatedCandidateKey);
|
|
2398
|
+
}
|
|
2399
|
+
if (observed.variant.id !== origin.variant.id && depth > 1) {
|
|
2400
|
+
const nestedCurrent = await visit(observed, depth - 1);
|
|
2401
|
+
const restored = await restore(nestedCurrent, origin);
|
|
2402
|
+
current = restored.current;
|
|
2403
|
+
if (!restored.restored) {
|
|
2404
|
+
if (restoreMatch(current, start)) {
|
|
2405
|
+
markPartial();
|
|
2406
|
+
return start;
|
|
2407
|
+
}
|
|
2408
|
+
stoppedReason = 'restore_failed';
|
|
2409
|
+
return current;
|
|
2410
|
+
}
|
|
2411
|
+
}
|
|
2412
|
+
else {
|
|
2413
|
+
const restored = restoreMatch(observed, origin)
|
|
2414
|
+
? { current: origin, restored: true }
|
|
2415
|
+
: await restore(observed, origin);
|
|
2416
|
+
current = restored.current;
|
|
2417
|
+
if (!restored.restored) {
|
|
2418
|
+
if (restoreMatch(current, start)) {
|
|
2419
|
+
markPartial();
|
|
2420
|
+
if (restoreMatch(origin, start)) {
|
|
2421
|
+
current = origin;
|
|
2422
|
+
continue;
|
|
2423
|
+
}
|
|
2424
|
+
return start;
|
|
2425
|
+
}
|
|
2426
|
+
stoppedReason = 'restore_failed';
|
|
2427
|
+
return current;
|
|
2428
|
+
}
|
|
2429
|
+
}
|
|
2430
|
+
}
|
|
2431
|
+
const remainingScrollCandidates = crawlScrollCandidates(context, origin.variant)
|
|
2432
|
+
.filter((candidate) => !attemptsForVariant.has(candidate.key));
|
|
2433
|
+
if (await exploreScrollCandidates(remainingScrollCandidates)) {
|
|
2434
|
+
return current;
|
|
2435
|
+
}
|
|
2436
|
+
return current;
|
|
2437
|
+
};
|
|
2438
|
+
await visit(start, context.options.crawlDepth);
|
|
2439
|
+
return {
|
|
2440
|
+
enabled: true,
|
|
2441
|
+
actions,
|
|
2442
|
+
variants: visited.size,
|
|
2443
|
+
stopped_reason: stoppedReason,
|
|
2444
|
+
...(restoreFailures > 0 ? { restore_failures: restoreFailures } : {}),
|
|
2445
|
+
...(restoreDiagnostics.length > 0 ? { restore_diagnostics: restoreDiagnostics } : {})
|
|
2446
|
+
};
|
|
2447
|
+
}
|
|
685
2448
|
export async function discoverAppMap(adapter, options) {
|
|
686
2449
|
const context = createAppMapContext(adapter, options);
|
|
687
2450
|
if (!context) {
|
|
@@ -693,10 +2456,12 @@ export async function discoverAppMap(adapter, options) {
|
|
|
693
2456
|
}
|
|
694
2457
|
try {
|
|
695
2458
|
const observed = await observeCurrentScreen(context);
|
|
2459
|
+
const crawl = context.options.crawl ? await crawlAppMap(context, observed) : undefined;
|
|
696
2460
|
const summary = persistAppMapContext(context);
|
|
697
2461
|
return {
|
|
698
2462
|
action: 'discover',
|
|
699
2463
|
map: summary,
|
|
2464
|
+
...(crawl ? { crawl } : {}),
|
|
700
2465
|
screen: {
|
|
701
2466
|
variant_id: observed.variant.id,
|
|
702
2467
|
screen_id: observed.variant.screen_id,
|