visor-ai 0.2.8 → 0.2.9

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/appMap.js CHANGED
@@ -22,6 +22,10 @@ const RISKY_WORDS = [
22
22
  ];
23
23
  const INPUT_WORDS = ['password', 'email', 'username', 'search', 'input', 'text field', 'textfield'];
24
24
  const AUTH_WORDS = ['login', 'log in', 'sign in', 'password', 'username', 'auth', 'authentication'];
25
+ const DEFAULT_ACTION_SETTLE_MS = 400;
26
+ const DEFAULT_ACTION_SETTLE_POLL_MS = 400;
27
+ const DEFAULT_CRAWL_SETTLE_MS = 1500;
28
+ const DEFAULT_CRAWL_SETTLE_POLL_MS = 300;
25
29
  function envNoMap() {
26
30
  const raw = process.env.VISOR_NO_MAP ?? process.env.VISOR_DISABLE_MAP;
27
31
  return ['1', 'true', 'yes', 'on'].includes(String(raw ?? '').trim().toLowerCase());
@@ -33,9 +37,21 @@ function resolveOptions(platform, options) {
33
37
  rootDir: options?.rootDir ?? process.env.VISOR_APP_MAP_DIR ?? path.join(process.cwd(), '.visor', 'maps'),
34
38
  appId: options?.appId ?? `default-${platform}`,
35
39
  repairDepth: options?.repairDepth ?? 2,
36
- repairTimeoutMs: options?.repairTimeoutMs ?? 30000
40
+ repairTimeoutMs: options?.repairTimeoutMs ?? 30000,
41
+ repair: options?.repair === true,
42
+ crawl: options?.crawl === true,
43
+ crawlDepth: options?.crawlDepth ?? 2,
44
+ crawlLimit: options?.crawlLimit ?? 24,
45
+ crawlSettleMs: nonNegativeNumber(options?.crawlSettleMs, DEFAULT_CRAWL_SETTLE_MS),
46
+ crawlSettlePollMs: positiveNumber(options?.crawlSettlePollMs, DEFAULT_CRAWL_SETTLE_POLL_MS)
37
47
  };
38
48
  }
49
+ function nonNegativeNumber(value, defaultValue) {
50
+ return typeof value === 'number' && Number.isFinite(value) && value >= 0 ? value : defaultValue;
51
+ }
52
+ function positiveNumber(value, defaultValue) {
53
+ return typeof value === 'number' && Number.isFinite(value) && value > 0 ? value : defaultValue;
54
+ }
39
55
  function mapIdentity(platform, appId) {
40
56
  return `${platform}:${appId}`;
41
57
  }
@@ -67,8 +83,15 @@ function readMap(filePath, platform, appId) {
67
83
  return { map: newMap(platform, appId), scrubbed: false };
68
84
  }
69
85
  const originalEdgeCount = parsed.edges.length;
86
+ let enrichedActions = false;
87
+ for (const variant of parsed.variants) {
88
+ if (!Array.isArray(variant.actions)) {
89
+ variant.actions = actionAffordancesForElements(variant.elements);
90
+ enrichedActions = true;
91
+ }
92
+ }
70
93
  parsed.edges = parsed.edges.filter((edge) => !isValueBearingAction(edge.command, edge.args));
71
- return { map: parsed, scrubbed: parsed.edges.length !== originalEdgeCount };
94
+ return { map: parsed, scrubbed: parsed.edges.length !== originalEdgeCount || enrichedActions };
72
95
  }
73
96
  catch {
74
97
  return { map: newMap(platform, appId), scrubbed: false };
@@ -130,10 +153,20 @@ function attributeMap(input) {
130
153
  const attrs = {};
131
154
  const pattern = /([A-Za-z_:][-A-Za-z0-9_:.]*)\s*=\s*("([^"]*)"|'([^']*)')/g;
132
155
  for (const match of input.matchAll(pattern)) {
133
- attrs[match[1]] = match[3] ?? match[4] ?? '';
156
+ attrs[match[1]] = decodeXmlEntities(match[3] ?? match[4] ?? '');
134
157
  }
135
158
  return attrs;
136
159
  }
160
+ function decodeXmlEntities(value) {
161
+ return value
162
+ .replace(/&#x([0-9a-f]+);/gi, (_, hex) => String.fromCodePoint(Number.parseInt(hex, 16)))
163
+ .replace(/&#(\d+);/g, (_, decimal) => String.fromCodePoint(Number.parseInt(decimal, 10)))
164
+ .replace(/"/g, '"')
165
+ .replace(/'/g, "'")
166
+ .replace(/&lt;/g, '<')
167
+ .replace(/&gt;/g, '>')
168
+ .replace(/&amp;/g, '&');
169
+ }
137
170
  function normalizeLabel(value) {
138
171
  return value
139
172
  .replace(/[0-9a-f]{8,}/gi, '<id>')
@@ -146,6 +179,40 @@ function normalizeLabel(value) {
146
179
  function unique(values) {
147
180
  return Array.from(new Set(values.filter((value) => value.trim() !== '')));
148
181
  }
182
+ function isScrollableTag(tag) {
183
+ const normalized = tag.toLowerCase();
184
+ return (normalized.includes('scrollview') ||
185
+ normalized.includes('scroll') ||
186
+ normalized.includes('list') ||
187
+ normalized.includes('table'));
188
+ }
189
+ function attrBool(attrs, key, defaultValue) {
190
+ const raw = attrs[key];
191
+ if (raw === undefined) {
192
+ return defaultValue;
193
+ }
194
+ return raw.trim().toLowerCase() === 'true';
195
+ }
196
+ function attrNumber(attrs, key) {
197
+ const parsed = Number(attrs[key]);
198
+ return Number.isFinite(parsed) ? parsed : null;
199
+ }
200
+ function expandSemanticLabels(labels) {
201
+ return labels.flatMap((label) => {
202
+ const lines = unique(label.split(/\n+/).map((line) => line.trim()));
203
+ if (lines.length === 1 && lines[0] !== label) {
204
+ return [label, lines[0]];
205
+ }
206
+ if (lines.length > 1 && label.split(/\n+/).filter((line) => line.trim() !== '').every((line) => line.trim() === lines[0])) {
207
+ return [label, lines[0]];
208
+ }
209
+ const rawLines = label.split(/\n+/).map((line) => line.trim()).filter(Boolean);
210
+ if (rawLines.length >= 3 && rawLines[0] === rawLines[rawLines.length - 1]) {
211
+ return [label, rawLines[0]];
212
+ }
213
+ return [label];
214
+ });
215
+ }
149
216
  function isLikelyFormControl(tag, attrs) {
150
217
  const haystack = [
151
218
  tag,
@@ -172,6 +239,19 @@ function isLikelyFormControl(tag, attrs) {
172
239
  'card'
173
240
  ].some((word) => haystack.includes(word));
174
241
  }
242
+ function structuralElementTarget(tag, attrs) {
243
+ if (!isScrollableTag(tag)) {
244
+ return '';
245
+ }
246
+ return [
247
+ 'scroll-container',
248
+ tag,
249
+ attrs.x ?? '',
250
+ attrs.y ?? '',
251
+ attrs.width ?? '',
252
+ attrs.height ?? ''
253
+ ].join('=');
254
+ }
175
255
  function redactLabel(value) {
176
256
  return value
177
257
  .replace(/[A-Z0-9._%+-]+@[A-Z0-9.-]+\.[A-Z]{2,}/gi, '<redacted-email>')
@@ -202,7 +282,7 @@ function extractElements(source) {
202
282
  const attrs = attributeMap(match[2] ?? '');
203
283
  const bodyText = (match[3] ?? '').trim();
204
284
  const formControl = isLikelyFormControl(tag, attrs);
205
- const rawLabels = unique([
285
+ const rawLabels = unique(expandSemanticLabels([
206
286
  redactLabel(attrs.name ?? ''),
207
287
  redactLabel(attrs.label ?? ''),
208
288
  formControl ? '' : redactLabel(attrs.text ?? ''),
@@ -210,17 +290,19 @@ function extractElements(source) {
210
290
  redactLabel(attrs['content-desc'] ?? ''),
211
291
  redactLabel(attrs['contentDescription'] ?? ''),
212
292
  formControl ? '' : redactLabel(bodyText)
213
- ]);
293
+ ]));
214
294
  const id = attrs['resource-id'] ?? attrs.id ?? attrs.identifier ?? '';
295
+ const structuralTarget = structuralElementTarget(tag, attrs);
215
296
  const targets = unique([
216
297
  ...rawLabels,
217
298
  ...rawLabels.map((label) => `text=${label}`),
218
- id ? `id=${id}` : ''
299
+ id ? `id=${id}` : '',
300
+ structuralTarget
219
301
  ]);
220
302
  if (targets.length === 0) {
221
303
  continue;
222
304
  }
223
- const stable = Boolean(rawLabels.length > 0 || id);
305
+ const stable = Boolean(rawLabels.length > 0 || id || structuralTarget);
224
306
  const selector = id ? `id=${id}` : rawLabels[0] ?? targets[0];
225
307
  elements.push({
226
308
  tag,
@@ -228,7 +310,14 @@ function extractElements(source) {
228
310
  targets,
229
311
  labels: rawLabels,
230
312
  stable,
231
- safety: classifyControl(tag, rawLabels, attrs)
313
+ safety: classifyControl(tag, rawLabels, attrs),
314
+ enabled: attrBool(attrs, 'enabled', true),
315
+ visible: attrBool(attrs, 'visible', true),
316
+ clickable: attrBool(attrs, 'clickable', false),
317
+ x: attrNumber(attrs, 'x'),
318
+ y: attrNumber(attrs, 'y'),
319
+ width: attrNumber(attrs, 'width'),
320
+ height: attrNumber(attrs, 'height')
232
321
  });
233
322
  }
234
323
  return elements;
@@ -263,14 +352,49 @@ function similarity(left, right) {
263
352
  const containment = smaller === 0 ? 0 : intersection / smaller;
264
353
  return containment >= 0.8 ? Math.max(jaccard, containment) : jaccard;
265
354
  }
266
- function elementIdentityKeys(elements) {
355
+ function identityKeys(elements, includeStatic) {
356
+ const navigationElements = new Set(navigationControlsForElements(elements).map((control) => control.element));
267
357
  return unique(elements.flatMap((element) => {
358
+ if (navigationElements.has(element)) {
359
+ return [];
360
+ }
268
361
  const tag = element.tag.toLowerCase();
269
- const labels = element.labels.map(normalizeLabel);
362
+ if (!includeStatic && isStaticIdentityElement(element)) {
363
+ return [];
364
+ }
365
+ const labels = element.labels
366
+ .filter((label) => !isGlobalIdentityLabel(label, element))
367
+ .map(normalizeLabel);
270
368
  const ids = element.targets.filter((target) => target.startsWith('id=')).map(normalizeLabel);
271
369
  return [...labels, ...ids].map((value) => `${tag}:${element.safety}:${value}`);
272
370
  })).sort();
273
371
  }
372
+ function isStaticIdentityElement(element) {
373
+ const tag = element.tag.toLowerCase();
374
+ return element.safety === 'unknown' || tag.includes('statictext') || tag.includes('image') || isScrollableTag(tag);
375
+ }
376
+ function screenSimilarity(left, right) {
377
+ const leftControlKeys = identityKeys(left, false);
378
+ const rightControlKeys = identityKeys(right, false);
379
+ const fullScore = similarity(identityKeys(left, true), identityKeys(right, true));
380
+ if (leftControlKeys.length === 0 && rightControlKeys.length === 0) {
381
+ return fullScore;
382
+ }
383
+ return Math.max(fullScore, similarity(leftControlKeys, rightControlKeys));
384
+ }
385
+ function isGlobalIdentityLabel(label, element) {
386
+ const normalized = normalizeControlTarget(label);
387
+ if (isBundleLikeLabel(normalized)) {
388
+ return true;
389
+ }
390
+ if (['settings settings', 'notifications notifications'].includes(normalized)) {
391
+ return true;
392
+ }
393
+ return false;
394
+ }
395
+ function isBundleLikeLabel(normalized) {
396
+ return /^[a-z][a-z0-9_]*(?:\.[a-z][a-z0-9_]*){2,}$/.test(normalized);
397
+ }
274
398
  function newVariant(map, observation) {
275
399
  const screen = {
276
400
  id: `screen_${map.screens.length + 1}`,
@@ -283,6 +407,7 @@ function newVariant(map, observation) {
283
407
  normalized_fingerprint: observation.normalized_fingerprint,
284
408
  elements: observation.elements,
285
409
  element_keys: observation.element_keys,
410
+ actions: actionAffordancesForElements(observation.elements),
286
411
  auth_required: observation.auth_required,
287
412
  observations: 1,
288
413
  confidence: 0.6,
@@ -296,16 +421,38 @@ function newVariant(map, observation) {
296
421
  function upsertVariant(map, observation) {
297
422
  let best = null;
298
423
  let bestScore = 0;
424
+ const observedNavigationTarget = preferredNavigationTargetForElements(observation.elements);
425
+ const observedHasBottomNavigation = navigationControlsForElements(observation.elements).length >= 2;
426
+ const observedHasScrollableSurface = elementsHaveScrollableSurface(observation.elements);
427
+ const observedControlKeys = identityKeys(observation.elements, false);
299
428
  for (const variant of map.variants) {
429
+ const variantNavigationTarget = preferredNavigationTarget(variant);
430
+ if (observedNavigationTarget &&
431
+ variantNavigationTarget &&
432
+ observedNavigationTarget !== variantNavigationTarget) {
433
+ continue;
434
+ }
435
+ if (observedHasScrollableSurface || hasScrollableSurface(variant)) {
436
+ const variantControlKeys = identityKeys(variant.elements, false);
437
+ if ((observedControlKeys.length > 0 || variantControlKeys.length > 0) &&
438
+ similarity(observedControlKeys, variantControlKeys) < 0.3) {
439
+ continue;
440
+ }
441
+ }
300
442
  const score = variant.normalized_fingerprint === observation.normalized_fingerprint
301
443
  ? 1
302
- : similarity(elementIdentityKeys(variant.elements), elementIdentityKeys(observation.elements));
444
+ : screenSimilarity(variant.elements, observation.elements);
303
445
  if (score > bestScore) {
304
446
  best = variant;
305
447
  bestScore = score;
306
448
  }
307
449
  }
308
- if (!best || bestScore < 0.45) {
450
+ const requiredScore = observedHasBottomNavigation || (best ? hasBottomNavigation(best) : false)
451
+ ? 0.62
452
+ : observedHasScrollableSurface || (best ? hasScrollableSurface(best) : false)
453
+ ? 0.62
454
+ : 0.45;
455
+ if (!best || bestScore < requiredScore) {
309
456
  return {
310
457
  observation,
311
458
  variant: newVariant(map, observation),
@@ -316,6 +463,7 @@ function upsertVariant(map, observation) {
316
463
  best.normalized_fingerprint = observation.normalized_fingerprint;
317
464
  best.elements = observation.elements;
318
465
  best.element_keys = observation.element_keys;
466
+ best.actions = actionAffordancesForElements(observation.elements);
319
467
  best.auth_required = observation.auth_required;
320
468
  best.observations += 1;
321
469
  best.confidence = Math.min(1, best.confidence + 0.05);
@@ -326,7 +474,7 @@ function upsertVariant(map, observation) {
326
474
  created: false
327
475
  };
328
476
  }
329
- async function observeCurrentScreen(context) {
477
+ async function captureCurrentSource(context) {
330
478
  const sourceDir = ensureDir(path.join(os.tmpdir(), 'visor-app-map-source'));
331
479
  const sourcePath = path.join(sourceDir, `${process.pid}-${Date.now()}-${randomUUID()}.xml`);
332
480
  const details = await context.adapter.source({ label: 'app-map-source', path: sourcePath });
@@ -344,10 +492,143 @@ async function observeCurrentScreen(context) {
344
492
  catch {
345
493
  // Source observations are transient; stale temp files should not fail a run.
346
494
  }
347
- const observed = upsertVariant(context.map, observeSource(source));
495
+ return { source };
496
+ }
497
+ function isTransientObservation(observation) {
498
+ return observation.elements.length === 0 || observation.element_keys.length === 0;
499
+ }
500
+ async function captureCurrentObservation(context) {
501
+ let observation = null;
502
+ for (let attempt = 0; attempt < 5; attempt += 1) {
503
+ const captured = await captureCurrentSource(context);
504
+ observation = observeSource(captured.source);
505
+ if (!isTransientObservation(observation) || attempt === 4) {
506
+ break;
507
+ }
508
+ await delay(500);
509
+ }
510
+ return observation ?? observeSource('');
511
+ }
512
+ function observeScreenObservation(context, observation) {
513
+ const observed = upsertVariant(context.map, observation);
348
514
  context.changed = true;
349
515
  return observed;
350
516
  }
517
+ async function observeCurrentScreen(context) {
518
+ return observeScreenObservation(context, await captureCurrentObservation(context));
519
+ }
520
+ async function observeAfterAction(context, target, options = {}) {
521
+ const observed = await observeCurrentScreen(context);
522
+ if (target && restoreMatch(observed, target)) {
523
+ return observed;
524
+ }
525
+ if (!shouldSettleObservation(observed.observation)) {
526
+ return observed;
527
+ }
528
+ const settleMs = nonNegativeNumber(options.settleMs, DEFAULT_ACTION_SETTLE_MS);
529
+ if (settleMs === 0) {
530
+ return observed;
531
+ }
532
+ const pollMs = positiveNumber(options.pollMs, DEFAULT_ACTION_SETTLE_POLL_MS);
533
+ let best = observed;
534
+ let previous = observed;
535
+ let elapsed = 0;
536
+ while (elapsed < settleMs) {
537
+ const waitMs = Math.min(pollMs, settleMs - elapsed);
538
+ await settleDelay(context, waitMs);
539
+ elapsed += waitMs;
540
+ let next;
541
+ try {
542
+ next = await observeCurrentScreen(context);
543
+ }
544
+ catch {
545
+ return best;
546
+ }
547
+ if (target && restoreMatch(next, target)) {
548
+ return next;
549
+ }
550
+ if (isRicherObservation(next.observation, best.observation)) {
551
+ best = next;
552
+ }
553
+ if (next.observation.fingerprint === previous.observation.fingerprint && !shouldSettleObservation(next.observation)) {
554
+ return isRicherObservation(next.observation, best.observation) ? next : best;
555
+ }
556
+ previous = next;
557
+ }
558
+ return best;
559
+ }
560
+ async function observeAfterCrawlAction(context) {
561
+ const observation = await captureCurrentObservation(context);
562
+ const settled = await settleCrawlObservation(context, observation);
563
+ return observeScreenObservation(context, settled);
564
+ }
565
+ async function settleCrawlObservation(context, observation) {
566
+ const settleMs = context.options.crawlSettleMs;
567
+ if (settleMs === 0) {
568
+ return observation;
569
+ }
570
+ const pollMs = context.options.crawlSettlePollMs;
571
+ let best = observation;
572
+ let previous = observation;
573
+ let elapsed = 0;
574
+ while (elapsed < settleMs) {
575
+ const waitMs = Math.min(pollMs, settleMs - elapsed);
576
+ await settleDelay(context, waitMs);
577
+ elapsed += waitMs;
578
+ let next;
579
+ try {
580
+ next = await captureCurrentObservation(context);
581
+ }
582
+ catch {
583
+ return best;
584
+ }
585
+ if (isRicherObservation(next, best)) {
586
+ best = next;
587
+ }
588
+ if (next.fingerprint === previous.fingerprint && !shouldSettleObservation(next)) {
589
+ return isRicherObservation(next, best) ? next : best;
590
+ }
591
+ previous = next;
592
+ }
593
+ return best;
594
+ }
595
+ async function settleDelay(context, ms) {
596
+ if (ms <= 0) {
597
+ return;
598
+ }
599
+ try {
600
+ await context.adapter.wait({ ms, label: 'app-map-settle' });
601
+ }
602
+ catch {
603
+ await delay(ms);
604
+ }
605
+ }
606
+ function isRicherObservation(candidate, current) {
607
+ if (isTransientObservation(current) && !isTransientObservation(candidate)) {
608
+ return true;
609
+ }
610
+ if (isTransientObservation(candidate)) {
611
+ return false;
612
+ }
613
+ const candidateScore = observationRichness(candidate);
614
+ const currentScore = observationRichness(current);
615
+ return candidateScore > currentScore;
616
+ }
617
+ function observationRichness(observation) {
618
+ const stableControls = observation.elements.filter((element) => element.stable && element.visible !== false && element.safety === 'safe').length;
619
+ return observation.element_keys.length + observation.elements.length * 2 + stableControls * 3;
620
+ }
621
+ function shouldSettleObservation(observation) {
622
+ if (isTransientObservation(observation)) {
623
+ return true;
624
+ }
625
+ const labels = observation.elements.flatMap((element) => element.labels.map(normalizeControlTarget));
626
+ const hasTransitionLabel = labels.some((label) => /\b(loading|syncing|refreshing|please wait|progress)\b/.test(label));
627
+ return hasTransitionLabel && observation.elements.length <= 3;
628
+ }
629
+ function delay(ms) {
630
+ return new Promise((resolve) => setTimeout(resolve, ms));
631
+ }
351
632
  function targetDescriptor(command, args) {
352
633
  if (isValueBearingAction(command, args)) {
353
634
  return { kind: 'unknown', confidence: 0 };
@@ -355,17 +636,26 @@ function targetDescriptor(command, args) {
355
636
  if (command === 'tap') {
356
637
  try {
357
638
  if (resolveTapMode(args) === 'coordinates') {
358
- return { kind: 'coordinate', confidence: 0 };
639
+ return { kind: 'coordinate', target: coordinateTarget(args), confidence: 0.5 };
359
640
  }
360
641
  }
361
642
  catch {
362
643
  return { kind: 'unknown', confidence: 0 };
363
644
  }
364
645
  }
646
+ if (command === 'scroll') {
647
+ return { kind: 'stable', target: scrollTarget(args), confidence: 0.4 };
648
+ }
365
649
  const target = typeof args.target === 'string' ? args.target : undefined;
366
650
  if (!target) {
367
651
  return { kind: 'unknown', confidence: 0 };
368
652
  }
653
+ if (target.startsWith('first-in-section=')) {
654
+ return { kind: 'section-first', target, value: target.slice(17), confidence: 0.55 };
655
+ }
656
+ if (target.startsWith('text~=')) {
657
+ return { kind: 'text-contains', target, value: target.slice(6), confidence: 0.45 };
658
+ }
369
659
  if (target.startsWith('text=')) {
370
660
  return { kind: 'text', target, value: target.slice(5), confidence: 0.6 };
371
661
  }
@@ -374,30 +664,75 @@ function targetDescriptor(command, args) {
374
664
  }
375
665
  return { kind: 'stable', target, value: target.replace(/^(id=|accessibility=)/, ''), confidence: 0.9 };
376
666
  }
667
+ function coordinateTarget(args) {
668
+ const x = Number(args.x);
669
+ const y = Number(args.y);
670
+ const base = `x=${Number.isFinite(x) ? x : String(args.x)},y=${Number.isFinite(y) ? y : String(args.y)}`;
671
+ return args.normalized === true ? `${base},normalized=true` : base;
672
+ }
673
+ function scrollTarget(args) {
674
+ const direction = String(args.direction ?? 'down').trim().toLowerCase() || 'down';
675
+ const percent = args.percent === undefined ? '' : `,percent=${String(args.percent)}`;
676
+ return `scroll=${direction}${percent}`;
677
+ }
377
678
  function isValueBearingAction(command, args) {
378
679
  return command === 'act' && args.name === 'type' && args.value !== undefined;
379
680
  }
380
681
  function replayableEdge(edge) {
381
682
  return !isValueBearingAction(edge.command, edge.args);
382
683
  }
383
- function variantContainsTarget(variant, descriptor) {
684
+ function elementContainsTarget(element, descriptor) {
685
+ if (descriptor.kind === 'section-first') {
686
+ return false;
687
+ }
384
688
  if (!descriptor.target && !descriptor.value) {
385
689
  return false;
386
690
  }
387
691
  const target = descriptor.target ?? '';
388
692
  const value = descriptor.value ?? target;
389
- return variant.elements.some((element) => {
390
- if (element.targets.includes(target) || element.targets.includes(value)) {
391
- return true;
392
- }
393
- if (descriptor.kind === 'text') {
394
- return element.labels.some((label) => label === value);
395
- }
396
- return element.labels.some((label) => label === value);
397
- });
693
+ if (element.targets.includes(target) || element.targets.includes(value)) {
694
+ return true;
695
+ }
696
+ if (descriptor.kind === 'text-contains') {
697
+ const normalizedValue = normalizeControlTarget(value);
698
+ return element.labels.some((label) => normalizeControlTarget(label).includes(normalizedValue));
699
+ }
700
+ return element.labels.some((label) => label === value);
701
+ }
702
+ function variantContainsTarget(variant, descriptor) {
703
+ if (descriptor.kind === 'section-first') {
704
+ return sectionFirstTapArgs(variant, descriptor) !== null;
705
+ }
706
+ return variant.elements.some((element) => elementContainsTarget(element, descriptor));
707
+ }
708
+ async function liveContainsTarget(context, descriptor) {
709
+ if (descriptor.kind === 'section-first') {
710
+ return false;
711
+ }
712
+ const target = descriptor.target ?? descriptor.value;
713
+ if (!target) {
714
+ return false;
715
+ }
716
+ try {
717
+ return await context.adapter.exists(target);
718
+ }
719
+ catch {
720
+ return false;
721
+ }
398
722
  }
399
723
  function candidateVariants(map, descriptor) {
400
- return map.variants.filter((variant) => variantContainsTarget(variant, descriptor));
724
+ return map.variants.filter((variant) => variantContainsTarget(variant, descriptor) && variantMatchesDescriptorContext(map, variant, descriptor));
725
+ }
726
+ function variantMatchesDescriptorContext(map, variant, descriptor) {
727
+ if (descriptor.kind !== 'section-first') {
728
+ return true;
729
+ }
730
+ const navigationHint = sectionNavigationHint(descriptor.value ?? descriptor.target ?? '', map);
731
+ if (!navigationHint) {
732
+ return true;
733
+ }
734
+ const navigationTarget = preferredNavigationTarget(variant);
735
+ return !navigationTarget || navigationTarget === navigationHint;
401
736
  }
402
737
  function summarizeEdge(edge) {
403
738
  return {
@@ -414,29 +749,39 @@ function summarizeControl(command, target, confidence) {
414
749
  };
415
750
  }
416
751
  function planRoute(map, fromVariantId, descriptor) {
417
- if (descriptor.kind === 'coordinate' || descriptor.kind === 'unknown') {
752
+ if (descriptor.kind === 'coordinate' || descriptor.kind === 'unknown' || !routeableDescriptor(descriptor)) {
418
753
  return { edges: [], ambiguous: false, authRequired: false };
419
754
  }
420
755
  const destinations = candidateVariants(map, descriptor);
421
756
  if (destinations.length === 0) {
422
757
  return { edges: [], ambiguous: false, authRequired: false };
423
758
  }
424
- if (descriptor.kind === 'text' && destinations.length > 1) {
425
- return { edges: [], ambiguous: true, authRequired: false };
426
- }
427
759
  const destinationIds = new Set(destinations.map((variant) => variant.id));
428
760
  const visited = new Set([fromVariantId]);
429
761
  const queue = [{ variantId: fromVariantId, edges: [] }];
762
+ const reachableRoutes = [];
763
+ let shortestDestinationLength = null;
430
764
  while (queue.length > 0) {
431
765
  const current = queue.shift();
432
766
  if (!current) {
433
767
  break;
434
768
  }
435
769
  if (destinationIds.has(current.variantId)) {
436
- const destination = map.variants.find((variant) => variant.id === current.variantId);
437
- return { edges: current.edges, ambiguous: false, authRequired: Boolean(destination?.auth_required) };
770
+ if (shortestDestinationLength === null) {
771
+ shortestDestinationLength = current.edges.length;
772
+ }
773
+ if (current.edges.length === shortestDestinationLength) {
774
+ reachableRoutes.push(current);
775
+ }
776
+ continue;
777
+ }
778
+ if (shortestDestinationLength !== null && current.edges.length >= shortestDestinationLength) {
779
+ continue;
438
780
  }
439
- const outgoing = map.edges
781
+ const outgoing = [
782
+ ...map.edges,
783
+ ...navigationVirtualEdges(map, map.variants.find((variant) => variant.id === current.variantId))
784
+ ]
440
785
  .filter((edge) => edge.from_variant_id === current.variantId && !edge.stale && edge.confidence >= 0.25 && replayableEdge(edge))
441
786
  .sort((left, right) => right.confidence - left.confidence);
442
787
  for (const edge of outgoing) {
@@ -447,13 +792,227 @@ function planRoute(map, fromVariantId, descriptor) {
447
792
  queue.push({ variantId: edge.to_variant_id, edges: [...current.edges, edge] });
448
793
  }
449
794
  }
795
+ if (reachableRoutes.length > 0) {
796
+ const rankedRoutes = reachableRoutes
797
+ .map((route) => ({
798
+ ...route,
799
+ score: destinationTargetScore(map, map.variants.find((variant) => variant.id === route.variantId), descriptor) + routeTargetScore(map, route.edges, descriptor)
800
+ }))
801
+ .sort((left, right) => right.score - left.score);
802
+ const best = rankedRoutes[0];
803
+ if (!best) {
804
+ return { edges: [], ambiguous: false, authRequired: false };
805
+ }
806
+ const tiedBest = rankedRoutes.filter((route) => route.score === best.score);
807
+ const ambiguous = (descriptor.kind === 'text' || descriptor.kind === 'text-contains' || descriptor.kind === 'section-first') &&
808
+ tiedBest.length > 1;
809
+ const diagnostics = routePlanDiagnostics(map, descriptor, rankedRoutes, ambiguous ? undefined : best, ambiguous);
810
+ if (ambiguous) {
811
+ return { edges: [], ambiguous: true, authRequired: false, diagnostics };
812
+ }
813
+ const route = best;
814
+ const destination = map.variants.find((variant) => variant.id === route?.variantId);
815
+ return {
816
+ edges: route?.edges ?? [],
817
+ ambiguous: false,
818
+ authRequired: Boolean(destination?.auth_required),
819
+ diagnostics
820
+ };
821
+ }
450
822
  return { edges: [], ambiguous: false, authRequired: destinations.some((variant) => variant.auth_required) };
451
823
  }
824
+ function routePlanDiagnostics(map, descriptor, rankedRoutes, selected, ambiguous) {
825
+ const bestScore = rankedRoutes[0]?.score;
826
+ return {
827
+ target: descriptor.target ?? descriptor.value ?? descriptor.kind,
828
+ selected_variant_id: selected?.variantId,
829
+ ambiguous,
830
+ route_candidates: rankedRoutes.map((route) => {
831
+ const variant = map.variants.find((candidate) => candidate.id === route.variantId);
832
+ const isSelected = selected?.variantId === route.variantId;
833
+ return {
834
+ variant_id: route.variantId,
835
+ screen_id: variant?.screen_id,
836
+ score: Math.round(route.score * 100) / 100,
837
+ route: route.edges.map(summarizeEdge),
838
+ selected: isSelected,
839
+ ...(isSelected
840
+ ? {}
841
+ : { rejected_reason: ambiguous && route.score === bestScore ? 'ambiguous_tie' : 'lower_score' })
842
+ };
843
+ })
844
+ };
845
+ }
846
+ function routeableDescriptor(descriptor) {
847
+ if (descriptor.kind === 'section-first') {
848
+ const value = descriptor.value ?? descriptor.target ?? '';
849
+ return /[a-z]/i.test(value);
850
+ }
851
+ if (descriptor.kind !== 'text-contains') {
852
+ return true;
853
+ }
854
+ const value = descriptor.value ?? descriptor.target ?? '';
855
+ return /[a-z]/i.test(value);
856
+ }
857
+ function navigationVirtualEdges(map, variant) {
858
+ if (!variant) {
859
+ return [];
860
+ }
861
+ const currentNavigationTarget = preferredNavigationTarget(variant);
862
+ const edges = [];
863
+ const destinationsByTarget = new Map();
864
+ for (const destination of map.variants) {
865
+ if (destination.id === variant.id) {
866
+ continue;
867
+ }
868
+ const target = preferredNavigationTarget(destination);
869
+ if (!target || target === currentNavigationTarget || destinationsByTarget.has(target)) {
870
+ continue;
871
+ }
872
+ destinationsByTarget.set(target, destination);
873
+ }
874
+ for (const candidate of safeTapCandidates(variant, { skipCurrentNavigation: true })) {
875
+ const target = normalizeControlTarget(candidate.target);
876
+ const destination = destinationsByTarget.get(target);
877
+ if (!destination) {
878
+ continue;
879
+ }
880
+ if (hasRealEdgeForTap(map, variant.id, candidate)) {
881
+ continue;
882
+ }
883
+ edges.push({
884
+ id: `virtual_nav_${variant.id}_${target}`,
885
+ from_variant_id: variant.id,
886
+ to_variant_id: destination.id,
887
+ command: 'tap',
888
+ args: structuredClone(candidate.args),
889
+ target: targetDescriptor('tap', candidate.args).target,
890
+ confidence: 0.4,
891
+ successes: 0,
892
+ failures: 0,
893
+ stale: false,
894
+ candidate: true,
895
+ destination_contract: {
896
+ variant_id: destination.id,
897
+ required_targets: contractTargets(destination),
898
+ normalized_fingerprint: destination.normalized_fingerprint
899
+ },
900
+ last_observed_at: utcNowIso()
901
+ });
902
+ }
903
+ return edges;
904
+ }
905
+ function hasRealEdgeForTap(map, fromVariantId, candidate) {
906
+ const descriptor = targetDescriptor('tap', candidate.args);
907
+ const candidateTargets = new Set([candidate.key, candidate.target, descriptor.target, descriptor.value].filter(Boolean));
908
+ return map.edges.some((edge) => edge.from_variant_id === fromVariantId &&
909
+ !edge.stale &&
910
+ edge.confidence >= 0.25 &&
911
+ replayableEdge(edge) &&
912
+ candidateTargets.has(edge.target));
913
+ }
914
+ function destinationTargetScore(map, variant, descriptor) {
915
+ if (!variant) {
916
+ return 0;
917
+ }
918
+ if (descriptor.kind === 'section-first') {
919
+ const element = firstItemInSection(variant, descriptor);
920
+ if (!element) {
921
+ return 0;
922
+ }
923
+ let score = isHighValueContentCard(element) ? 10 : 7;
924
+ const navigationHint = sectionNavigationHint(descriptor.value ?? descriptor.target ?? '', map);
925
+ const navigationTarget = preferredNavigationTarget(variant);
926
+ if (navigationHint && navigationTarget === navigationHint) {
927
+ score += 5;
928
+ }
929
+ else if (navigationHint && navigationTarget && navigationTarget !== navigationHint) {
930
+ score -= 5;
931
+ }
932
+ return score;
933
+ }
934
+ let score = 0;
935
+ for (const element of variant.elements) {
936
+ if (!elementContainsTarget(element, descriptor)) {
937
+ continue;
938
+ }
939
+ let elementScore = 1;
940
+ if (isLikelyTapElement(element)) {
941
+ elementScore += 4;
942
+ }
943
+ else if (element.visible === false) {
944
+ elementScore -= 2;
945
+ }
946
+ if (isHighValueContentCard(element)) {
947
+ elementScore += 3;
948
+ }
949
+ const value = normalizeControlTarget(descriptor.value ?? descriptor.target ?? '');
950
+ if (value &&
951
+ element.labels.some((label) => normalizeControlTarget(label) === value ||
952
+ label.split(/\n+/).some((line) => normalizeControlTarget(line) === value))) {
953
+ elementScore += 2;
954
+ }
955
+ score = Math.max(score, elementScore);
956
+ }
957
+ if (preferredNavigationTarget(variant) && score > 0) {
958
+ score += 1;
959
+ }
960
+ if (score > 0 && isFeedLikeVariant(variant) && !descriptorTargetsFeed(descriptor)) {
961
+ score -= 3;
962
+ }
963
+ return score;
964
+ }
965
+ function isFeedLikeVariant(variant) {
966
+ return contentLabels(variant).some((label) => /^(for you|activity|feed|latest|timeline|updates?)$/.test(label));
967
+ }
968
+ function descriptorTargetsFeed(descriptor) {
969
+ const value = normalizeControlTarget(descriptor.value ?? descriptor.target ?? '');
970
+ return /\b(activity|feed|post|comment|update|timeline)\b/.test(value);
971
+ }
972
+ function routeTargetScore(map, route, descriptor) {
973
+ if (descriptor.kind !== 'section-first') {
974
+ return 0;
975
+ }
976
+ const navigationHint = sectionNavigationHint(descriptor.value ?? descriptor.target ?? '', map);
977
+ if (!navigationHint) {
978
+ return 0;
979
+ }
980
+ return route.reduce((score, edge) => {
981
+ const target = edgeNavigationTarget(map, edge);
982
+ if (!target) {
983
+ return score;
984
+ }
985
+ return score + (target === navigationHint ? 3 : -3);
986
+ }, 0);
987
+ }
988
+ function edgeNavigationTarget(map, edge) {
989
+ const normalizedTarget = normalizeControlTarget(edge.target ?? '');
990
+ const from = map.variants.find((variant) => variant.id === edge.from_variant_id);
991
+ if (from && isNavigationTarget(from, normalizedTarget)) {
992
+ return normalizedTarget;
993
+ }
994
+ if (!from || edge.command !== 'tap') {
995
+ return null;
996
+ }
997
+ const edgeCoordinateTarget = coordinateTarget(edge.args);
998
+ for (const control of navigationControls(from)) {
999
+ const elementCoordinateTarget = coordinateArgsForElement(control.element);
1000
+ if (elementCoordinateTarget && coordinateTarget(elementCoordinateTarget) === edgeCoordinateTarget) {
1001
+ return control.target;
1002
+ }
1003
+ }
1004
+ return null;
1005
+ }
452
1006
  function contractTargets(variant) {
453
- return variant.elements
1007
+ const navigationElements = new Set(navigationControls(variant).map((control) => control.element));
1008
+ const contentTargets = variant.elements
454
1009
  .filter((element) => element.stable)
455
- .flatMap((element) => element.targets)
456
- .slice(0, 8);
1010
+ .filter((element) => !navigationElements.has(element))
1011
+ .flatMap((element) => element.targets.filter((target) => !isGlobalIdentityLabel(normalizeControlTarget(target), element)));
1012
+ const fallbackTargets = variant.elements
1013
+ .filter((element) => element.stable)
1014
+ .flatMap((element) => element.targets);
1015
+ return unique([...contentTargets, ...fallbackTargets]).slice(0, 8);
457
1016
  }
458
1017
  function matchingEdge(map, fromVariantId, command, target) {
459
1018
  return map.edges.find((edge) => edge.from_variant_id === fromVariantId &&
@@ -465,7 +1024,7 @@ function recordEdgeSuccess(context, from, to, command, args, candidate) {
465
1024
  return;
466
1025
  }
467
1026
  const descriptor = targetDescriptor(command, args);
468
- if (descriptor.kind === 'coordinate' || descriptor.kind === 'unknown') {
1027
+ if (descriptor.kind === 'unknown') {
469
1028
  return;
470
1029
  }
471
1030
  const edge = matchingEdge(context.map, from.id, command, descriptor.target);
@@ -515,25 +1074,87 @@ function demoteEdge(context, edge) {
515
1074
  edge.last_observed_at = utcNowIso();
516
1075
  context.changed = true;
517
1076
  }
518
- function satisfiesContract(edge, observed) {
519
- if (observed.variant.id === edge.destination_contract.variant_id) {
1077
+ function shouldRequireObservedEffect(command, args) {
1078
+ return command === 'tap' || command === 'navigate' || command === 'scroll' || (command === 'act' && args.name === 'back');
1079
+ }
1080
+ function assertObservedEffect(command, args, before, after, descriptor) {
1081
+ if (!shouldRequireObservedEffect(command, args)) {
1082
+ return;
1083
+ }
1084
+ if (before.observation.fingerprint !== after.observation.fingerprint) {
1085
+ return;
1086
+ }
1087
+ const target = descriptor.target ?? descriptor.value ?? command;
1088
+ throw new Error(`Action '${command}' on '${target}' had no effect: screen source did not change after the command`);
1089
+ }
1090
+ function variantSatisfiesContract(variant, contract) {
1091
+ if (variant.id === contract.variant_id) {
520
1092
  return true;
521
1093
  }
522
- const observedTargets = new Set(observed.variant.elements.flatMap((element) => element.targets));
523
- const required = edge.destination_contract.required_targets;
1094
+ const observedTargets = new Set(variant.elements.flatMap((element) => element.targets));
1095
+ const required = contract.required_targets;
524
1096
  if (required.length === 0) {
525
- return observed.variant.normalized_fingerprint === edge.destination_contract.normalized_fingerprint;
1097
+ return variant.normalized_fingerprint === contract.normalized_fingerprint;
526
1098
  }
527
1099
  const matched = required.filter((target) => observedTargets.has(target)).length;
528
1100
  return matched / required.length >= 0.6;
529
1101
  }
1102
+ function restoreContractHasSharedAnchors(observed, target) {
1103
+ const observedTargets = new Set(observed.elements.flatMap((element) => element.targets));
1104
+ const required = contractTargets(target);
1105
+ if (required.length === 0) {
1106
+ return false;
1107
+ }
1108
+ const matched = unique(required.filter((targetValue) => observedTargets.has(targetValue)));
1109
+ return matched.length >= 2 && matched.length / required.length >= 0.25;
1110
+ }
1111
+ function restoreMatch(observed, target) {
1112
+ if (observed.variant.id === target.variant.id) {
1113
+ return { acceptedBy: 'exact' };
1114
+ }
1115
+ const observedNavigationTarget = preferredNavigationTarget(observed.variant);
1116
+ const targetNavigationTarget = preferredNavigationTarget(target.variant);
1117
+ if (observedNavigationTarget && targetNavigationTarget && observedNavigationTarget !== targetNavigationTarget) {
1118
+ return null;
1119
+ }
1120
+ if (hasScrollableSurface(observed.variant) || hasScrollableSurface(target.variant)) {
1121
+ const observedControlKeys = identityKeys(observed.variant.elements, false);
1122
+ const targetControlKeys = identityKeys(target.variant.elements, false);
1123
+ if ((observedControlKeys.length > 0 || targetControlKeys.length > 0) &&
1124
+ similarity(observedControlKeys, targetControlKeys) < 0.3) {
1125
+ return null;
1126
+ }
1127
+ }
1128
+ const contract = {
1129
+ variant_id: target.variant.id,
1130
+ required_targets: contractTargets(target.variant),
1131
+ normalized_fingerprint: target.variant.normalized_fingerprint
1132
+ };
1133
+ if (variantSatisfiesContract(observed.variant, contract)) {
1134
+ return { acceptedBy: 'contract' };
1135
+ }
1136
+ if (observedNavigationTarget &&
1137
+ targetNavigationTarget &&
1138
+ observedNavigationTarget === targetNavigationTarget &&
1139
+ restoreContractHasSharedAnchors(observed.variant, target.variant)) {
1140
+ return { acceptedBy: 'contract' };
1141
+ }
1142
+ const score = screenSimilarity(target.variant.elements, observed.variant.elements);
1143
+ if (score >= 0.72) {
1144
+ return { acceptedBy: 'similarity', score };
1145
+ }
1146
+ return null;
1147
+ }
1148
+ function satisfiesContract(edge, observed) {
1149
+ return variantSatisfiesContract(observed.variant, edge.destination_contract);
1150
+ }
530
1151
  async function driveRoute(context, current, route) {
531
1152
  for (const edge of route) {
532
1153
  const args = structuredClone(edge.args);
533
1154
  let observed;
534
1155
  try {
535
1156
  await context.adapter[edge.command](args);
536
- observed = await observeCurrentScreen(context);
1157
+ observed = await observeAfterAction(context);
537
1158
  }
538
1159
  catch {
539
1160
  demoteEdge(context, edge);
@@ -554,63 +1175,731 @@ async function driveRoute(context, current, route) {
554
1175
  }
555
1176
  return { current };
556
1177
  }
557
- function safeTapTargets(variant) {
558
- return unique(variant.elements
559
- .filter((element) => element.safety === 'safe' && element.stable)
560
- .map((element) => {
561
- const plainLabel = element.targets.find((target) => !target.includes('='));
562
- return plainLabel ?? element.targets[0] ?? '';
563
- }));
1178
+ function safeTapCandidates(variant, options = {}) {
1179
+ const candidates = [];
1180
+ const seen = new Set();
1181
+ const currentNavigationTarget = options.skipCurrentNavigation ? preferredNavigationTarget(variant) : null;
1182
+ const elements = [...variant.elements].sort((left, right) => {
1183
+ const priority = tapElementPriority(left, variant, options) - tapElementPriority(right, variant, options);
1184
+ if (priority !== 0) {
1185
+ return priority;
1186
+ }
1187
+ return (left.y ?? 0) - (right.y ?? 0) || (left.x ?? 0) - (right.x ?? 0);
1188
+ });
1189
+ for (const element of elements) {
1190
+ if (!(element.safety === 'safe' && element.stable && isLikelyTapElement(element))) {
1191
+ continue;
1192
+ }
1193
+ const target = plainTapTargets(element)[0] ?? element.selector;
1194
+ if (!target) {
1195
+ continue;
1196
+ }
1197
+ const normalizedTarget = normalizeControlTarget(target);
1198
+ if (options.excludeNavigation && isNavigationTarget(variant, normalizedTarget)) {
1199
+ continue;
1200
+ }
1201
+ if (currentNavigationTarget && normalizedTarget === currentNavigationTarget) {
1202
+ continue;
1203
+ }
1204
+ const coordinateArgs = coordinateArgsForElement(element);
1205
+ const args = coordinateArgs ?? { target };
1206
+ const key = coordinateArgs ? coordinateTarget(coordinateArgs) : target;
1207
+ if (seen.has(key)) {
1208
+ continue;
1209
+ }
1210
+ seen.add(key);
1211
+ candidates.push({ target, args, key });
1212
+ }
1213
+ return candidates;
564
1214
  }
565
- async function repairToTarget(context, current, descriptor, depth, deadline, blockedTargets) {
566
- if (Date.now() > deadline || depth <= 0) {
567
- return null;
1215
+ function tapElementPriority(element, variant, options = {}) {
1216
+ const target = plainTapTargets(element)[0] ?? element.selector;
1217
+ const normalized = normalizeControlTarget(target);
1218
+ if (options.preferContent && isHighValueContentCard(element)) {
1219
+ return 5;
568
1220
  }
569
- const attemptedTargets = new Map();
570
- const attemptFrom = async (start, remainingDepth, route) => {
571
- let cursor = start;
572
- let depthLeft = remainingDepth;
573
- let routeSoFar = route;
574
- while (Date.now() <= deadline) {
575
- if (variantContainsTarget(cursor.variant, descriptor)) {
576
- return { found: true, current: cursor, route: routeSoFar, remainingDepth: depthLeft };
577
- }
578
- if (depthLeft <= 0) {
579
- return { found: false, current: cursor, route: routeSoFar, remainingDepth: depthLeft };
580
- }
581
- const attemptsForVariant = attemptedTargets.get(cursor.variant.id) ?? new Set();
582
- attemptedTargets.set(cursor.variant.id, attemptsForVariant);
583
- const target = safeTapTargets(cursor.variant).find((candidate) => !blockedTargets.has(candidate) && !attemptsForVariant.has(candidate));
584
- if (!target) {
585
- return { found: false, current: cursor, route: routeSoFar, remainingDepth: depthLeft };
586
- }
587
- attemptsForVariant.add(target);
588
- const args = { target };
589
- let observed;
590
- try {
591
- await context.adapter.tap(args);
592
- observed = await observeCurrentScreen(context);
593
- }
594
- catch {
595
- try {
596
- cursor = await observeCurrentScreen(context);
597
- }
598
- catch {
599
- // If both the exploratory tap and source read fail, keep the last observed screen.
600
- }
601
- continue;
602
- }
603
- recordEdgeSuccess(context, cursor.variant, observed.variant, 'tap', args, true);
604
- const nextRoute = [...routeSoFar, summarizeControl('tap', target, 0.45)];
605
- const nested = await attemptFrom(observed, depthLeft - 1, nextRoute);
606
- if (nested.found) {
607
- return nested;
608
- }
609
- cursor = nested.current;
610
- depthLeft = nested.remainingDepth;
611
- routeSoFar = nested.route;
1221
+ const navPriority = navigationControlPriority(variant, normalized);
1222
+ if (navPriority !== undefined) {
1223
+ if (options.preferContent) {
1224
+ return 80 + navPriority;
612
1225
  }
613
- return { found: false, current: cursor, route: routeSoFar, remainingDepth: depthLeft };
1226
+ return navPriority;
1227
+ }
1228
+ if (normalized === 'see all') {
1229
+ return 10;
1230
+ }
1231
+ if (isBackControl(element)) {
1232
+ return 90;
1233
+ }
1234
+ if (element.tag.toLowerCase().includes('button')) {
1235
+ return 40;
1236
+ }
1237
+ if (element.tag.toLowerCase().includes('other')) {
1238
+ return 60;
1239
+ }
1240
+ return 50;
1241
+ }
1242
+ function isHighValueContentCard(element) {
1243
+ if (!isLikelyTapElement(element)) {
1244
+ return false;
1245
+ }
1246
+ const target = plainTapTargets(element)[0] ?? element.selector;
1247
+ const normalizedTarget = normalizeControlTarget(target);
1248
+ if (normalizedTarget === 'see all' || /^(back|close|dismiss|done)$/i.test(target)) {
1249
+ return false;
1250
+ }
1251
+ const lines = unique(element.labels.flatMap((label) => label.split(/\n+/).map((line) => line.trim())));
1252
+ const area = (element.width ?? 0) * (element.height ?? 0);
1253
+ return (lines.length >= 2 ||
1254
+ (element.tag.toLowerCase().includes('other') && area >= 3000) ||
1255
+ ((element.width ?? 0) >= 120 && (element.height ?? 0) >= 64 && element.labels.length > 0));
1256
+ }
1257
+ function actionAffordancesForElements(elements) {
1258
+ const navigationElements = new Set(navigationControlsForElements(elements).map((control) => control.element));
1259
+ const navigationTarget = preferredNavigationTargetForElements(elements) ?? undefined;
1260
+ const actions = [];
1261
+ const seen = new Set();
1262
+ const candidates = [...elements]
1263
+ .filter((element) => element.safety === 'safe' && element.stable && isLikelyTapElement(element))
1264
+ .filter((element) => !navigationElements.has(element) && !isBackControl(element))
1265
+ .sort((left, right) => (left.y ?? 0) - (right.y ?? 0) || (left.x ?? 0) - (right.x ?? 0));
1266
+ for (const element of candidates) {
1267
+ const label = actionLabel(element);
1268
+ if (!label) {
1269
+ continue;
1270
+ }
1271
+ const coordinateArgs = coordinateArgsForElement(element);
1272
+ const stableTarget = plainTapTargets(element)[0] ?? element.selector;
1273
+ const args = coordinateArgs ?? { target: stableTarget };
1274
+ const target = coordinateArgs ? coordinateTarget(coordinateArgs) : stableTarget;
1275
+ const scope = actionScopeForElement(elements, element);
1276
+ const dedupeKey = [
1277
+ actionIntent(label),
1278
+ target,
1279
+ scope?.kind ?? '',
1280
+ scope?.label ?? ''
1281
+ ].join('|');
1282
+ if (seen.has(dedupeKey)) {
1283
+ continue;
1284
+ }
1285
+ seen.add(dedupeKey);
1286
+ actions.push({
1287
+ command: 'tap',
1288
+ intent: actionIntent(label),
1289
+ label,
1290
+ target,
1291
+ args,
1292
+ safety: element.safety,
1293
+ ...(scope ? { scope } : {}),
1294
+ ...(navigationTarget ? { navigation_target: navigationTarget } : {}),
1295
+ source: {
1296
+ tag: element.tag,
1297
+ x: element.x,
1298
+ y: element.y,
1299
+ width: element.width,
1300
+ height: element.height
1301
+ }
1302
+ });
1303
+ }
1304
+ return actions;
1305
+ }
1306
+ function actionLabel(element) {
1307
+ const target = plainTapTargets(element)[0] ?? element.labels[0] ?? element.selector;
1308
+ return target.split(/\n+/).map((line) => line.trim()).filter(Boolean)[0] ?? target.trim();
1309
+ }
1310
+ function actionIntent(label) {
1311
+ const normalized = normalizeControlTarget(label);
1312
+ const intentPatterns = [
1313
+ ['like', /\blike\b/],
1314
+ ['comment', /\bcomment\b/],
1315
+ ['share', /\bshare\b/],
1316
+ ['save', /\b(save|bookmark)\b/],
1317
+ ['follow', /\bfollow\b/],
1318
+ ['watchlist', /\bwatchlist\b/],
1319
+ ['search', /\bsearch\b/],
1320
+ ['create', /\b(create|new)\b/],
1321
+ ['filter', /\bfilter\b/],
1322
+ ['sort', /\bsort\b/],
1323
+ ['dismiss', /\b(close|dismiss|skip)\b/]
1324
+ ];
1325
+ const matched = intentPatterns.find(([, pattern]) => pattern.test(normalized));
1326
+ if (matched) {
1327
+ return matched[0];
1328
+ }
1329
+ return normalized.split(/[^a-z0-9]+/).filter(Boolean)[0] ?? 'tap';
1330
+ }
1331
+ function actionScopeForElement(elements, element) {
1332
+ const content = nearestContentScopeElement(elements, element);
1333
+ if (content) {
1334
+ return {
1335
+ kind: 'content',
1336
+ label: content.labels[0]
1337
+ };
1338
+ }
1339
+ const section = nearestActionSectionElement(elements, element);
1340
+ if (section) {
1341
+ return {
1342
+ kind: 'section',
1343
+ label: section.labels[0]
1344
+ };
1345
+ }
1346
+ return { kind: 'screen' };
1347
+ }
1348
+ function nearestContentScopeElement(elements, element) {
1349
+ if (element.y === null) {
1350
+ return null;
1351
+ }
1352
+ return elements
1353
+ .filter((candidate) => candidate !== element && candidate.y !== null && candidate.height !== null)
1354
+ .filter((candidate) => isHighValueContentCard(candidate))
1355
+ .filter((candidate) => {
1356
+ const candidateTop = candidate.y;
1357
+ const candidateBottom = candidateTop + Math.max(0, candidate.height ?? 0);
1358
+ return candidateTop <= element.y && candidateBottom + 96 >= element.y;
1359
+ })
1360
+ .sort((left, right) => (right.y ?? 0) - (left.y ?? 0))[0] ?? null;
1361
+ }
1362
+ function nearestActionSectionElement(elements, element) {
1363
+ if (element.y === null) {
1364
+ return null;
1365
+ }
1366
+ return elements
1367
+ .filter((candidate) => candidate.y !== null && candidate.y < element.y && isActionSectionElement(candidate))
1368
+ .sort((left, right) => (right.y ?? 0) - (left.y ?? 0))[0] ?? null;
1369
+ }
1370
+ function isActionSectionElement(element) {
1371
+ if (isLikelyTapElement(element) || element.labels.length === 0) {
1372
+ return false;
1373
+ }
1374
+ return element.labels.some((label) => {
1375
+ const normalized = normalizeControlTarget(label);
1376
+ if (!/[a-z]/i.test(normalized)) {
1377
+ return false;
1378
+ }
1379
+ const words = normalized.split(/\s+/).filter(Boolean);
1380
+ return words.length <= 8;
1381
+ });
1382
+ }
1383
+ function normalizeControlTarget(target) {
1384
+ return target.replace(/\s+/g, ' ').trim().toLowerCase();
1385
+ }
1386
+ function sectionNavigationHint(value, map) {
1387
+ const normalized = normalizeControlTarget(value);
1388
+ const targets = unique(map.variants.flatMap((variant) => navigationControls(variant).map((control) => control.target)))
1389
+ .sort((left, right) => right.length - left.length);
1390
+ for (const target of targets) {
1391
+ if (normalized.includes(target)) {
1392
+ return target;
1393
+ }
1394
+ }
1395
+ return null;
1396
+ }
1397
+ function crawlCandidateOptions(variant) {
1398
+ const preferredTarget = preferredNavigationTarget(variant);
1399
+ const preferContent = preferredTarget !== null && !isPrimaryNavigationTarget(variant, preferredTarget);
1400
+ return {
1401
+ preferContent,
1402
+ excludeNavigation: preferContent,
1403
+ skipCurrentNavigation: true
1404
+ };
1405
+ }
1406
+ function crawlTemplateKey(variant) {
1407
+ const navigationTarget = preferredNavigationTarget(variant) ?? 'none';
1408
+ const elementKeys = unique(variant.elements.flatMap((element) => crawlTemplateElementKeys(variant, element))).sort();
1409
+ return signatureFor([`nav=${navigationTarget}`, ...elementKeys]);
1410
+ }
1411
+ function crawlTemplateElementKeys(variant, element) {
1412
+ if (element.visible === false || element.stable === false) {
1413
+ return [];
1414
+ }
1415
+ const tag = element.tag.toLowerCase();
1416
+ const layout = `${bucketNumber(element.width, 24)}x${bucketNumber(element.height, 24)}`;
1417
+ if (isScrollableTag(tag)) {
1418
+ return [`scroll:${layout}`];
1419
+ }
1420
+ if (isNavigationElement(variant, element)) {
1421
+ return [`nav-control:${normalizeControlTarget(plainTapTargets(element)[0] ?? element.selector)}:${layout}`];
1422
+ }
1423
+ if (isHighValueContentCard(element)) {
1424
+ return [`content-card:${contentLineCount(element)}:${layout}`];
1425
+ }
1426
+ if (isLikelyTapElement(element)) {
1427
+ return [`tap:${tag}:${templateControlLabel(element)}:${layout}`];
1428
+ }
1429
+ return [];
1430
+ }
1431
+ function templateControlLabel(element) {
1432
+ if (isBackControl(element)) {
1433
+ return 'back';
1434
+ }
1435
+ const target = normalizeControlTarget(plainTapTargets(element)[0] ?? element.selector);
1436
+ const words = target.split(/\s+/).filter(Boolean);
1437
+ if (target && words.length <= 4 && target.length <= 48) {
1438
+ return target;
1439
+ }
1440
+ return 'control';
1441
+ }
1442
+ function repeatedContentCandidateKey(variant, candidate) {
1443
+ const element = elementForTapCandidate(variant, candidate);
1444
+ if (!element || !isHighValueContentCard(element)) {
1445
+ return null;
1446
+ }
1447
+ const section = nearestSectionHeadingLabel(variant, element) ?? 'section=none';
1448
+ const tag = element.tag.toLowerCase();
1449
+ const shape = [
1450
+ crawlTemplateKey(variant),
1451
+ section,
1452
+ tag,
1453
+ `lines=${contentLineCount(element)}`,
1454
+ `size=${bucketNumber(element.width, 24)}x${bucketNumber(element.height, 24)}`
1455
+ ];
1456
+ return signatureFor(shape);
1457
+ }
1458
+ function elementForTapCandidate(variant, candidate) {
1459
+ const candidateCoordinate = typeof candidate.args.x === 'number' && typeof candidate.args.y === 'number'
1460
+ ? coordinateTarget(candidate.args)
1461
+ : null;
1462
+ for (const element of variant.elements) {
1463
+ if (candidateCoordinate) {
1464
+ const elementCoordinate = coordinateArgsForElement(element);
1465
+ if (elementCoordinate && coordinateTarget(elementCoordinate) === candidateCoordinate) {
1466
+ return element;
1467
+ }
1468
+ }
1469
+ const target = plainTapTargets(element)[0] ?? element.selector;
1470
+ if (target === candidate.target || element.targets.includes(candidate.target)) {
1471
+ return element;
1472
+ }
1473
+ }
1474
+ return null;
1475
+ }
1476
+ function nearestSectionHeadingLabel(variant, element) {
1477
+ if (element.y === null) {
1478
+ return null;
1479
+ }
1480
+ const heading = variant.elements
1481
+ .filter((candidate) => candidate.y !== null && candidate.y < element.y && isLikelySectionHeading(variant, candidate))
1482
+ .sort((left, right) => (right.y ?? 0) - (left.y ?? 0))[0];
1483
+ const label = heading?.labels[0];
1484
+ return label ? `section=${normalizeControlTarget(label)}` : null;
1485
+ }
1486
+ function contentLineCount(element) {
1487
+ const lines = unique(element.labels.flatMap((label) => label.split(/\n+/).map((line) => line.trim()).filter(Boolean)));
1488
+ return lines.length;
1489
+ }
1490
+ function bucketNumber(value, size) {
1491
+ if (value === null || !Number.isFinite(value)) {
1492
+ return 0;
1493
+ }
1494
+ return Math.round(value / size) * size;
1495
+ }
1496
+ function crawlScrollCandidates(context, variant) {
1497
+ if (!context.adapter.capability().commands.includes('scroll') || !shouldExploreScroll(variant)) {
1498
+ return [];
1499
+ }
1500
+ const args = { direction: 'down', percent: 70 };
1501
+ return [
1502
+ {
1503
+ target: scrollTarget(args),
1504
+ args,
1505
+ key: 'scroll:down:70'
1506
+ }
1507
+ ];
1508
+ }
1509
+ function shouldExploreScroll(variant) {
1510
+ return hasScrollableSurface(variant) || hasBottomReachableContent(variant);
1511
+ }
1512
+ function hasScrollableSurface(variant) {
1513
+ return elementsHaveScrollableSurface(variant.elements);
1514
+ }
1515
+ function elementsHaveScrollableSurface(elements) {
1516
+ return elements.some((element) => isScrollableTag(element.tag));
1517
+ }
1518
+ function hasBottomReachableContent(variant) {
1519
+ const content = scrollProbeElements(variant);
1520
+ if (content.length < 3) {
1521
+ return false;
1522
+ }
1523
+ const navControls = navigationControls(variant);
1524
+ const navTop = navControls
1525
+ .map((control) => control.element.y)
1526
+ .filter((value) => value !== null)
1527
+ .sort((left, right) => left - right)[0] ?? null;
1528
+ if (navTop === null) {
1529
+ return false;
1530
+ }
1531
+ const viewportBottom = navTop ?? Math.max(...content.map((element) => (element.y ?? 0) + (element.height ?? 0)));
1532
+ return content.some((element) => ((element.y ?? 0) + (element.height ?? 0)) >= viewportBottom - 48);
1533
+ }
1534
+ function scrollProbeElements(variant) {
1535
+ const navigationElements = new Set(navigationControls(variant).map((control) => control.element));
1536
+ return variant.elements.filter((element) => {
1537
+ if (element.visible === false || element.y === null || element.height === null || navigationElements.has(element)) {
1538
+ return false;
1539
+ }
1540
+ if (isBackControl(element)) {
1541
+ return false;
1542
+ }
1543
+ const tag = element.tag.toLowerCase();
1544
+ return !isScrollableTag(tag);
1545
+ });
1546
+ }
1547
+ function preferredNavigationTarget(variant) {
1548
+ return preferredNavigationTargetForElements(variant.elements);
1549
+ }
1550
+ function preferredNavigationTargetForElements(elements) {
1551
+ const controls = navigationControlsForElements(elements);
1552
+ if (controls.length === 0) {
1553
+ return null;
1554
+ }
1555
+ const labels = contentLabelsForElements(elements);
1556
+ const matched = controls.find((control) => labels.some((label) => labelMatchesNavigationTarget(label, control.target)));
1557
+ if (matched) {
1558
+ return matched.target;
1559
+ }
1560
+ return controls[0]?.target ?? null;
1561
+ }
1562
+ function hasBottomNavigation(variant) {
1563
+ return navigationControls(variant).length >= 2;
1564
+ }
1565
+ function contentLabels(variant) {
1566
+ return contentLabelsForElements(variant.elements);
1567
+ }
1568
+ function contentLabelsForElements(elements) {
1569
+ const navigationElements = new Set(navigationControlsForElements(elements).map((control) => control.element));
1570
+ return elements.flatMap((element) => element.labels
1571
+ .filter(() => !navigationElements.has(element))
1572
+ .filter((label) => !isGlobalIdentityLabel(label, element))
1573
+ .map(normalizeControlTarget));
1574
+ }
1575
+ function navigationControls(variant) {
1576
+ return navigationControlsForElements(variant.elements);
1577
+ }
1578
+ function navigationControlsForElements(elements) {
1579
+ const candidates = elements
1580
+ .filter((element) => element.safety === 'safe' && element.stable && isLikelyTapElement(element))
1581
+ .filter((element) => !isBackControl(element))
1582
+ .filter((element) => coordinateArgsForElement(element) !== null)
1583
+ .map((element) => ({ element, target: normalizeControlTarget(plainTapTargets(element)[0] ?? element.selector) }))
1584
+ .filter((candidate) => isCompactNavigationLabel(candidate.element, candidate.target));
1585
+ if (candidates.length < 2) {
1586
+ return [];
1587
+ }
1588
+ const maxY = Math.max(...candidates.map((candidate) => candidate.element.y ?? 0));
1589
+ const bottomRow = candidates
1590
+ .filter((candidate) => (candidate.element.y ?? 0) >= maxY - 12)
1591
+ .sort((left, right) => (left.element.x ?? 0) - (right.element.x ?? 0));
1592
+ if (bottomRow.length < 2) {
1593
+ return [];
1594
+ }
1595
+ return bottomRow.map((candidate, priority) => ({
1596
+ ...candidate,
1597
+ priority
1598
+ }));
1599
+ }
1600
+ function isCompactNavigationLabel(element, target) {
1601
+ if (!target || target.includes('\n') || target.length > 32 || target === 'see all') {
1602
+ return false;
1603
+ }
1604
+ if ((element.height ?? 0) > 72) {
1605
+ return false;
1606
+ }
1607
+ const lines = unique(element.labels.flatMap((label) => label.split(/\n+/).map((line) => normalizeControlTarget(line))));
1608
+ return lines.length <= 2 && lines.some((line) => line === target);
1609
+ }
1610
+ function isNavigationElement(variant, element) {
1611
+ return navigationControls(variant).some((control) => control.element === element);
1612
+ }
1613
+ function isNavigationTarget(variant, normalizedTarget) {
1614
+ return navigationControls(variant).some((control) => control.target === normalizedTarget);
1615
+ }
1616
+ function navigationControlPriority(variant, normalizedTarget) {
1617
+ return navigationControls(variant).find((control) => control.target === normalizedTarget)?.priority;
1618
+ }
1619
+ function isPrimaryNavigationTarget(variant, normalizedTarget) {
1620
+ return navigationControls(variant)[0]?.target === normalizedTarget;
1621
+ }
1622
+ function labelMatchesNavigationTarget(label, target) {
1623
+ if (label === target) {
1624
+ return true;
1625
+ }
1626
+ const labelWords = new Set(label.split(/[^a-z0-9]+/).filter(Boolean));
1627
+ const targetWords = target.split(/[^a-z0-9]+/).filter(Boolean);
1628
+ if (targetWords.length === 1) {
1629
+ const words = Array.from(labelWords);
1630
+ return words.length <= 2 && words[0] === targetWords[0];
1631
+ }
1632
+ return targetWords.length > 0 && targetWords.every((word) => labelWords.has(word));
1633
+ }
1634
+ function currentScreenTapArgs(variant, descriptor) {
1635
+ if (descriptor.kind === 'section-first') {
1636
+ return sectionFirstTapArgs(variant, descriptor);
1637
+ }
1638
+ const action = matchingAction(variant, descriptor);
1639
+ if (action) {
1640
+ return structuredClone(action.args);
1641
+ }
1642
+ for (const element of variant.elements) {
1643
+ if (!elementContainsTarget(element, descriptor)) {
1644
+ continue;
1645
+ }
1646
+ if (!(element.safety === 'safe' && element.stable && isLikelyTapElement(element))) {
1647
+ continue;
1648
+ }
1649
+ const coordinateArgs = coordinateArgsForElement(element);
1650
+ if (coordinateArgs) {
1651
+ return coordinateArgs;
1652
+ }
1653
+ const stableTarget = plainTapTargets(element)[0] ?? element.selector;
1654
+ if (stableTarget && shouldUseStableTargetFallback(element, descriptor)) {
1655
+ return { target: stableTarget };
1656
+ }
1657
+ }
1658
+ return null;
1659
+ }
1660
+ function currentScreenLiveTapArgs(variant, descriptor) {
1661
+ const action = matchingAction(variant, descriptor);
1662
+ return action && shouldUseActionForLiveSemanticTap(action)
1663
+ ? structuredClone(action.args)
1664
+ : null;
1665
+ }
1666
+ function matchingAction(variant, descriptor) {
1667
+ return variant.actions.find((candidate) => actionMatchesDescriptor(candidate, descriptor));
1668
+ }
1669
+ function actionMatchesDescriptor(action, descriptor) {
1670
+ if (action.command !== 'tap' || action.safety !== 'safe') {
1671
+ return false;
1672
+ }
1673
+ if (descriptor.kind === 'text-contains') {
1674
+ return false;
1675
+ }
1676
+ const target = descriptor.target ?? '';
1677
+ const value = descriptor.value ?? target;
1678
+ const normalizedValue = normalizeControlTarget(value);
1679
+ const normalizedTarget = normalizeControlTarget(target);
1680
+ const normalizedActionLabel = normalizeControlTarget(action.label);
1681
+ const normalizedActionTarget = normalizeControlTarget(action.target ?? '');
1682
+ if (!normalizedValue && !normalizedTarget) {
1683
+ return false;
1684
+ }
1685
+ return [normalizedActionLabel, normalizedActionTarget].some((candidate) => candidate !== '' && (candidate === normalizedValue || candidate === normalizedTarget));
1686
+ }
1687
+ function shouldUseActionForLiveSemanticTap(action) {
1688
+ return (typeof action.args.x === 'number' &&
1689
+ typeof action.args.y === 'number' &&
1690
+ action.source.tag.toLowerCase().includes('xcuielementtype'));
1691
+ }
1692
+ function shouldUseStableTargetFallback(element, descriptor) {
1693
+ if (descriptor.kind !== 'text-contains') {
1694
+ return false;
1695
+ }
1696
+ const value = descriptor.value ?? '';
1697
+ return value !== '' && !element.labels.some((label) => label.includes(value));
1698
+ }
1699
+ function sectionFirstTapArgs(variant, descriptor) {
1700
+ const element = firstItemInSection(variant, descriptor);
1701
+ return element ? coordinateArgsForElement(element) : null;
1702
+ }
1703
+ function firstItemInSection(variant, descriptor) {
1704
+ const value = descriptor.value ?? descriptor.target ?? '';
1705
+ if (!/[a-z]/i.test(value)) {
1706
+ return null;
1707
+ }
1708
+ const section = sectionHeadingElement(variant, value);
1709
+ if (!section || section.y === null || section.height === null) {
1710
+ return null;
1711
+ }
1712
+ const sectionY = section.y;
1713
+ const sectionBottom = sectionY + Math.max(0, section.height);
1714
+ const nextSectionY = variant.elements
1715
+ .filter((element) => element !== section &&
1716
+ element.y !== null &&
1717
+ element.y > sectionY &&
1718
+ isLikelySectionHeading(variant, element))
1719
+ .map((element) => element.y)
1720
+ .sort((left, right) => left - right)[0] ?? null;
1721
+ const candidates = variant.elements
1722
+ .filter((element) => isSectionItemCandidate(variant, element, value) &&
1723
+ element.y !== null &&
1724
+ element.y > sectionBottom &&
1725
+ (nextSectionY === null || element.y < nextSectionY))
1726
+ .sort((left, right) => (left.y ?? 0) - (right.y ?? 0) || (left.x ?? 0) - (right.x ?? 0));
1727
+ return candidates[0] ?? null;
1728
+ }
1729
+ function sectionHeadingElement(variant, value) {
1730
+ const matches = variant.elements
1731
+ .filter((element) => element.visible !== false &&
1732
+ element.y !== null &&
1733
+ element.height !== null &&
1734
+ elementLabelMatchesValue(element, value))
1735
+ .sort((left, right) => (left.y ?? 0) - (right.y ?? 0) || (left.x ?? 0) - (right.x ?? 0));
1736
+ return matches[0] ?? null;
1737
+ }
1738
+ function isSectionItemCandidate(variant, element, sectionValue) {
1739
+ if (!(element.safety === 'safe' && element.stable && isLikelyTapElement(element))) {
1740
+ return false;
1741
+ }
1742
+ if (!coordinateArgsForElement(element)) {
1743
+ return false;
1744
+ }
1745
+ if (isBackControl(element) || elementLabelMatchesValue(element, sectionValue)) {
1746
+ return false;
1747
+ }
1748
+ const target = normalizeControlTarget(plainTapTargets(element)[0] ?? element.selector);
1749
+ if (isNavigationTarget(variant, target) || target === 'see all') {
1750
+ return false;
1751
+ }
1752
+ return true;
1753
+ }
1754
+ function isLikelySectionHeading(variant, element) {
1755
+ if (isHighValueContentCard(element)) {
1756
+ return false;
1757
+ }
1758
+ const labels = element.labels.map(normalizeControlTarget);
1759
+ if (labels.some((label) => label === 'see all' || isNavigationTarget(variant, label))) {
1760
+ return false;
1761
+ }
1762
+ return labels.some((label) => {
1763
+ if (!/[a-z]/i.test(label)) {
1764
+ return false;
1765
+ }
1766
+ const words = label.split(/\s+/).filter(Boolean);
1767
+ const lineCount = label.split(/\n+/).filter(Boolean).length;
1768
+ return words.length <= 8 && lineCount <= 2;
1769
+ });
1770
+ }
1771
+ function elementLabelMatchesValue(element, value) {
1772
+ const normalizedValue = normalizeControlTarget(value);
1773
+ if (!normalizedValue) {
1774
+ return false;
1775
+ }
1776
+ return element.labels.some((label) => {
1777
+ const normalizedLabel = normalizeControlTarget(label);
1778
+ return (normalizedLabel === normalizedValue ||
1779
+ normalizedLabel.includes(normalizedValue) ||
1780
+ label.split(/\n+/).some((line) => normalizeControlTarget(line) === normalizedValue));
1781
+ });
1782
+ }
1783
+ function coordinateArgsForElement(element) {
1784
+ if (element.x === null ||
1785
+ element.y === null ||
1786
+ element.width === null ||
1787
+ element.height === null ||
1788
+ element.width <= 0 ||
1789
+ element.height <= 0) {
1790
+ return null;
1791
+ }
1792
+ if (isBackControl(element)) {
1793
+ return {
1794
+ x: Math.round(element.x + Math.min(40, element.width / 2)),
1795
+ y: Math.round(element.y + element.height / 2)
1796
+ };
1797
+ }
1798
+ return {
1799
+ x: Math.round(element.x + element.width / 2),
1800
+ y: Math.round(element.y + element.height / 2)
1801
+ };
1802
+ }
1803
+ function isBackControl(element) {
1804
+ if (!element.tag.toLowerCase().includes('button')) {
1805
+ return false;
1806
+ }
1807
+ return element.labels.some((label) => label.split(/\n+/).some((line) => line.trim().toLowerCase() === 'back'));
1808
+ }
1809
+ function plainTapTargets(element) {
1810
+ return element.targets
1811
+ .filter((target) => !target.includes('='))
1812
+ .sort((left, right) => tapTargetScore(left) - tapTargetScore(right));
1813
+ }
1814
+ function tapTargetScore(target) {
1815
+ let score = target.length;
1816
+ if (target.includes('\n')) {
1817
+ score += 1000;
1818
+ }
1819
+ return score;
1820
+ }
1821
+ function isLikelyTapElement(element) {
1822
+ if (element.enabled === false || element.visible === false) {
1823
+ return false;
1824
+ }
1825
+ if (element.width !== null &&
1826
+ element.height !== null &&
1827
+ (element.width <= 0 || element.height <= 0)) {
1828
+ return false;
1829
+ }
1830
+ const tag = element.tag.toLowerCase();
1831
+ if (tag.includes('statictext') ||
1832
+ tag.includes('image') ||
1833
+ tag.includes('textfield') ||
1834
+ isScrollableTag(tag)) {
1835
+ return false;
1836
+ }
1837
+ if (tag.includes('button') || element.clickable) {
1838
+ return true;
1839
+ }
1840
+ if (tag.includes('other')) {
1841
+ const width = element.width ?? 0;
1842
+ const height = element.height ?? 0;
1843
+ return width >= 20 && height >= 20 && element.labels.length > 0;
1844
+ }
1845
+ return false;
1846
+ }
1847
+ async function repairToTarget(context, current, descriptor, depth, deadline, blockedTargets) {
1848
+ if (Date.now() > deadline || depth <= 0) {
1849
+ return null;
1850
+ }
1851
+ const attemptedTargets = new Map();
1852
+ const attemptFrom = async (start, remainingDepth, route) => {
1853
+ let cursor = start;
1854
+ let depthLeft = remainingDepth;
1855
+ let routeSoFar = route;
1856
+ while (Date.now() <= deadline) {
1857
+ if (variantContainsTarget(cursor.variant, descriptor)) {
1858
+ return { found: true, current: cursor, route: routeSoFar, remainingDepth: depthLeft };
1859
+ }
1860
+ if (depthLeft <= 0) {
1861
+ return { found: false, current: cursor, route: routeSoFar, remainingDepth: depthLeft };
1862
+ }
1863
+ const attemptsForVariant = attemptedTargets.get(cursor.variant.id) ?? new Set();
1864
+ attemptedTargets.set(cursor.variant.id, attemptsForVariant);
1865
+ const candidate = safeTapCandidates(cursor.variant).find((tapCandidate) => !blockedTargets.has(tapCandidate.target) &&
1866
+ !blockedTargets.has(tapCandidate.key) &&
1867
+ !attemptsForVariant.has(tapCandidate.key));
1868
+ if (!candidate) {
1869
+ return { found: false, current: cursor, route: routeSoFar, remainingDepth: depthLeft };
1870
+ }
1871
+ attemptsForVariant.add(candidate.key);
1872
+ const args = candidate.args;
1873
+ let observed;
1874
+ try {
1875
+ await context.adapter.tap(args);
1876
+ observed = await observeCurrentScreen(context);
1877
+ }
1878
+ catch {
1879
+ try {
1880
+ cursor = await observeCurrentScreen(context);
1881
+ }
1882
+ catch {
1883
+ // If both the exploratory tap and source read fail, keep the last observed screen.
1884
+ }
1885
+ continue;
1886
+ }
1887
+ recordEdgeSuccess(context, cursor.variant, observed.variant, 'tap', args, true);
1888
+ const nextRoute = [...routeSoFar, summarizeControl('tap', candidate.target, 0.45)];
1889
+ const nested = await attemptFrom(observed, depthLeft - 1, nextRoute);
1890
+ if (nested.found) {
1891
+ return nested;
1892
+ }
1893
+ if (nested.current.variant.id === cursor.variant.id) {
1894
+ cursor = nested.current;
1895
+ routeSoFar = nested.route;
1896
+ continue;
1897
+ }
1898
+ cursor = nested.current;
1899
+ depthLeft = nested.remainingDepth;
1900
+ routeSoFar = nested.route;
1901
+ }
1902
+ return { found: false, current: cursor, route: routeSoFar, remainingDepth: depthLeft };
614
1903
  };
615
1904
  const attempt = await attemptFrom(current, depth, []);
616
1905
  return attempt.found ? { current: attempt.current, route: attempt.route } : null;
@@ -625,63 +1914,537 @@ export async function runMappedCommand(context, command, args) {
625
1914
  repairs: 0,
626
1915
  path: context.filePath
627
1916
  };
628
- if (descriptor.kind === 'unknown' || descriptor.kind === 'coordinate') {
1917
+ if (descriptor.kind === 'unknown') {
629
1918
  const details = await context.adapter[command](args);
630
1919
  return { ...details, map: metadata };
631
1920
  }
632
1921
  let before = await observeCurrentScreen(context);
633
- if (!variantContainsTarget(before.variant, descriptor)) {
1922
+ if (descriptor.kind === 'coordinate' || command === 'scroll') {
1923
+ const details = await context.adapter[command](args);
1924
+ const after = await observeCurrentScreen(context);
1925
+ assertObservedEffect(command, args, before, after, descriptor);
1926
+ recordEdgeSuccess(context, before.variant, after.variant, command, args, false);
1927
+ return { ...details, map: metadata };
1928
+ }
1929
+ let liveTargetVisible = await liveContainsTarget(context, descriptor);
1930
+ if (!variantContainsTarget(before.variant, descriptor) && !liveTargetVisible) {
634
1931
  if (before.variant.auth_required) {
635
1932
  throw new Error(`Target '${descriptor.target ?? descriptor.value}' requires authentication from the current screen`);
636
1933
  }
637
- const route = planRoute(context.map, before.variant.id, descriptor);
638
- if (route.ambiguous) {
639
- throw new Error(`Target '${descriptor.target ?? descriptor.value}' is ambiguous in the app map`);
640
- }
641
- if (route.authRequired) {
642
- throw new Error(`Target '${descriptor.target ?? descriptor.value}' is behind an auth-required screen`);
643
- }
644
- if (route.edges.length > 0) {
645
- context.summary.used = true;
646
- metadata.routed = true;
647
- metadata.route = route.edges.map(summarizeEdge);
648
- const driven = await driveRoute(context, before, route.edges);
649
- before = driven.current;
650
- if (driven.failedEdge) {
651
- const repair = await repairToTarget(context, before, descriptor, context.options.repairDepth, Date.now() + context.options.repairTimeoutMs, new Set(driven.failedEdge.target ? [driven.failedEdge.target] : []));
652
- if (!repair) {
653
- throw new Error(`Cached app-map route for '${driven.failedEdge.target ?? driven.failedEdge.command}' could not be repaired within the bounded repair budget`);
654
- }
655
- context.summary.repaired = true;
656
- context.summary.repairs += 1;
657
- metadata.repaired = true;
658
- metadata.repairs = context.summary.repairs;
659
- metadata.route = [...metadata.route, ...repair.route];
660
- before = repair.current;
1934
+ if (routeableDescriptor(descriptor)) {
1935
+ const route = planRoute(context.map, before.variant.id, descriptor);
1936
+ if (route.ambiguous) {
1937
+ throw new Error(`Target '${descriptor.target ?? descriptor.value}' is ambiguous in the app map`);
661
1938
  }
662
- }
663
- else {
664
- const repair = await repairToTarget(context, before, descriptor, context.options.repairDepth, Date.now() + context.options.repairTimeoutMs, new Set());
665
- if (repair) {
1939
+ if (route.authRequired) {
1940
+ throw new Error(`Target '${descriptor.target ?? descriptor.value}' is behind an auth-required screen`);
1941
+ }
1942
+ if (route.edges.length > 0) {
666
1943
  context.summary.used = true;
667
- context.summary.repaired = true;
668
- context.summary.repairs += 1;
669
- metadata.routed = repair.route.length > 0;
670
- metadata.repaired = true;
671
- metadata.repairs = context.summary.repairs;
672
- metadata.route = repair.route;
673
- before = repair.current;
1944
+ metadata.routed = true;
1945
+ metadata.route = route.edges.map(summarizeEdge);
1946
+ if (route.diagnostics) {
1947
+ metadata.diagnostics = route.diagnostics;
1948
+ }
1949
+ const driven = await driveRoute(context, before, route.edges);
1950
+ before = driven.current;
1951
+ liveTargetVisible = await liveContainsTarget(context, descriptor);
1952
+ const targetReachedDespiteStaleContract = variantContainsTarget(before.variant, descriptor) || liveTargetVisible;
1953
+ if (driven.failedEdge && !targetReachedDespiteStaleContract) {
1954
+ if (!context.options.repair) {
1955
+ throw new Error(`Cached app-map route for '${driven.failedEdge.target ?? driven.failedEdge.command}' failed; rerun with repair enabled to explore an alternate route`);
1956
+ }
1957
+ const repair = await repairToTarget(context, before, descriptor, context.options.repairDepth, Date.now() + context.options.repairTimeoutMs, new Set(driven.failedEdge.target ? [driven.failedEdge.target] : []));
1958
+ if (!repair) {
1959
+ throw new Error(`Cached app-map route for '${driven.failedEdge.target ?? driven.failedEdge.command}' could not be repaired within the bounded repair budget`);
1960
+ }
1961
+ context.summary.repaired = true;
1962
+ context.summary.repairs += 1;
1963
+ metadata.repaired = true;
1964
+ metadata.repairs = context.summary.repairs;
1965
+ metadata.route = [...metadata.route, ...repair.route];
1966
+ before = repair.current;
1967
+ liveTargetVisible = await liveContainsTarget(context, descriptor);
1968
+ }
1969
+ }
1970
+ else if (context.options.repair) {
1971
+ const repair = await repairToTarget(context, before, descriptor, context.options.repairDepth, Date.now() + context.options.repairTimeoutMs, new Set());
1972
+ if (repair) {
1973
+ context.summary.used = true;
1974
+ context.summary.repaired = true;
1975
+ context.summary.repairs += 1;
1976
+ metadata.routed = repair.route.length > 0;
1977
+ metadata.repaired = true;
1978
+ metadata.repairs = context.summary.repairs;
1979
+ metadata.route = repair.route;
1980
+ before = repair.current;
1981
+ liveTargetVisible = await liveContainsTarget(context, descriptor);
1982
+ }
674
1983
  }
675
1984
  }
676
1985
  }
677
- if (!variantContainsTarget(before.variant, descriptor)) {
1986
+ if (!variantContainsTarget(before.variant, descriptor) && !liveTargetVisible) {
678
1987
  throw new Error(`Target '${descriptor.target ?? descriptor.value}' is not reachable from the current app-map state`);
679
1988
  }
680
- const details = await context.adapter[command](args);
1989
+ const executableArgs = command === 'tap'
1990
+ ? liveTargetVisible
1991
+ ? currentScreenLiveTapArgs(before.variant, descriptor) ?? args
1992
+ : currentScreenTapArgs(before.variant, descriptor) ?? args
1993
+ : args;
1994
+ const details = await context.adapter[command](executableArgs);
681
1995
  const after = await observeCurrentScreen(context);
682
- recordEdgeSuccess(context, before.variant, after.variant, command, args, false);
1996
+ assertObservedEffect(command, executableArgs, before, after, descriptor);
1997
+ recordEdgeSuccess(context, before.variant, after.variant, command, executableArgs, false);
683
1998
  return { ...details, map: metadata };
684
1999
  }
2000
+ function restoreAttemptDiagnostic(strategy, result, observed, match, target, command, error) {
2001
+ return {
2002
+ strategy,
2003
+ result,
2004
+ ...(command ? { command } : {}),
2005
+ ...(target ? { target } : {}),
2006
+ ...(observed
2007
+ ? {
2008
+ observed_variant_id: observed.variant.id,
2009
+ observed_screen_id: observed.variant.screen_id
2010
+ }
2011
+ : {}),
2012
+ ...(match ? { accepted_by: match.acceptedBy } : {}),
2013
+ ...(error instanceof Error ? { error: error.message } : {})
2014
+ };
2015
+ }
2016
+ function makeRestoreDiagnostic(from, target) {
2017
+ return {
2018
+ from_variant_id: from.variant.id,
2019
+ target_variant_id: target.variant.id,
2020
+ result: 'failed',
2021
+ attempts: []
2022
+ };
2023
+ }
2024
+ async function tryTapRestoreCandidate(context, current, target, candidate, strategy, diagnostic) {
2025
+ const args = candidate.args;
2026
+ try {
2027
+ await context.adapter.tap(args);
2028
+ const observed = await observeAfterAction(context, target);
2029
+ recordEdgeSuccess(context, current.variant, observed.variant, 'tap', args, true);
2030
+ const match = restoreMatch(observed, target);
2031
+ diagnostic.attempts.push(restoreAttemptDiagnostic(strategy, match ? 'matched' : 'mismatched', observed, match, targetDescriptor('tap', args).target ?? candidate.target, 'tap'));
2032
+ return { current: observed, match };
2033
+ }
2034
+ catch (error) {
2035
+ diagnostic.attempts.push(restoreAttemptDiagnostic(strategy, 'error', undefined, null, candidate.target, 'tap', error));
2036
+ try {
2037
+ return { current: await observeCurrentScreen(context), match: null };
2038
+ }
2039
+ catch {
2040
+ return { current, match: null };
2041
+ }
2042
+ }
2043
+ }
2044
+ async function tryScrollRestoreCandidate(context, current, target, candidate, diagnostic) {
2045
+ const args = candidate.args;
2046
+ try {
2047
+ await context.adapter.scroll(args);
2048
+ const observed = await observeAfterAction(context, target);
2049
+ recordEdgeSuccess(context, current.variant, observed.variant, 'scroll', args, true);
2050
+ const match = restoreMatch(observed, target);
2051
+ diagnostic.attempts.push(restoreAttemptDiagnostic('reverse-scroll', match ? 'matched' : 'mismatched', observed, match, candidate.target, 'scroll'));
2052
+ return { current: observed, match };
2053
+ }
2054
+ catch (error) {
2055
+ diagnostic.attempts.push(restoreAttemptDiagnostic('reverse-scroll', 'error', undefined, null, candidate.target, 'scroll', error));
2056
+ try {
2057
+ return { current: await observeCurrentScreen(context), match: null };
2058
+ }
2059
+ catch {
2060
+ return { current, match: null };
2061
+ }
2062
+ }
2063
+ }
2064
+ function targetRestoreCandidates(current, target) {
2065
+ const preferredTarget = preferredNavigationTarget(target.variant);
2066
+ return safeTapCandidates(current.variant).filter((candidate) => {
2067
+ if (/^(back|close|dismiss|done)$/i.test(candidate.target)) {
2068
+ return false;
2069
+ }
2070
+ const normalizedTarget = normalizeControlTarget(candidate.target);
2071
+ if (preferredTarget) {
2072
+ return normalizedTarget === preferredTarget;
2073
+ }
2074
+ if (isGlobalIdentityLabel(candidate.target)) {
2075
+ return false;
2076
+ }
2077
+ const descriptor = targetDescriptor('tap', { target: candidate.target });
2078
+ return descriptor.kind !== 'unknown' && variantContainsTarget(target.variant, descriptor);
2079
+ });
2080
+ }
2081
+ function reverseScrollRestoreCandidates(current, target) {
2082
+ if (!shouldTryReverseScrollRestore(current.variant, target.variant)) {
2083
+ return [];
2084
+ }
2085
+ const args = { direction: 'up', percent: 70 };
2086
+ return [
2087
+ {
2088
+ target: scrollTarget(args),
2089
+ args,
2090
+ key: 'scroll:up:70'
2091
+ }
2092
+ ];
2093
+ }
2094
+ function shouldTryReverseScrollRestore(current, target) {
2095
+ if (hasScrollableSurface(current) || hasScrollableSurface(target)) {
2096
+ return true;
2097
+ }
2098
+ const currentNavigationTarget = preferredNavigationTarget(current);
2099
+ const targetNavigationTarget = preferredNavigationTarget(target);
2100
+ if (currentNavigationTarget && targetNavigationTarget && currentNavigationTarget === targetNavigationTarget) {
2101
+ return true;
2102
+ }
2103
+ return similarity(current.element_keys, target.element_keys) >= 0.2;
2104
+ }
2105
+ function navigationResetCandidates(current, root) {
2106
+ const rootTarget = preferredNavigationTarget(root.variant);
2107
+ if (!rootTarget) {
2108
+ return [];
2109
+ }
2110
+ return safeTapCandidates(current.variant).filter((candidate) => normalizeControlTarget(candidate.target) === rootTarget);
2111
+ }
2112
+ function shortestRouteToVariant(map, fromVariantId, toVariantId) {
2113
+ if (fromVariantId === toVariantId) {
2114
+ return [];
2115
+ }
2116
+ const visited = new Set([fromVariantId]);
2117
+ const queue = [{ variantId: fromVariantId, edges: [] }];
2118
+ while (queue.length > 0) {
2119
+ const current = queue.shift();
2120
+ if (!current) {
2121
+ break;
2122
+ }
2123
+ const variant = map.variants.find((candidate) => candidate.id === current.variantId);
2124
+ const outgoing = [
2125
+ ...map.edges,
2126
+ ...navigationVirtualEdges(map, variant)
2127
+ ]
2128
+ .filter((edge) => edge.from_variant_id === current.variantId && !edge.stale && edge.confidence >= 0.25 && replayableEdge(edge))
2129
+ .sort((left, right) => right.confidence - left.confidence);
2130
+ for (const edge of outgoing) {
2131
+ if (visited.has(edge.to_variant_id)) {
2132
+ continue;
2133
+ }
2134
+ const edges = [...current.edges, edge];
2135
+ if (edge.to_variant_id === toVariantId) {
2136
+ return edges;
2137
+ }
2138
+ visited.add(edge.to_variant_id);
2139
+ queue.push({ variantId: edge.to_variant_id, edges });
2140
+ }
2141
+ }
2142
+ return null;
2143
+ }
2144
+ async function restoreViaRootRoute(context, current, target, root, diagnostic) {
2145
+ let resetCurrent = current;
2146
+ const rootMatch = restoreMatch(resetCurrent, root);
2147
+ if (!rootMatch) {
2148
+ for (const candidate of navigationResetCandidates(resetCurrent, root)) {
2149
+ const restored = await tryTapRestoreCandidate(context, resetCurrent, root, candidate, 'root-navigation', diagnostic);
2150
+ resetCurrent = restored.current;
2151
+ if (restored.match) {
2152
+ break;
2153
+ }
2154
+ }
2155
+ }
2156
+ const matchedRoot = restoreMatch(resetCurrent, root);
2157
+ if (!matchedRoot) {
2158
+ return null;
2159
+ }
2160
+ if (restoreMatch(resetCurrent, target)) {
2161
+ diagnostic.result = 'restored';
2162
+ diagnostic.accepted_by = 'root';
2163
+ return { current: target, restored: true, acceptedBy: 'root', diagnostic };
2164
+ }
2165
+ const route = shortestRouteToVariant(context.map, root.variant.id, target.variant.id);
2166
+ if (!route || route.length === 0) {
2167
+ return null;
2168
+ }
2169
+ const driven = await driveRoute(context, root, route);
2170
+ const match = restoreMatch(driven.current, target);
2171
+ diagnostic.attempts.push(restoreAttemptDiagnostic('root-route', match ? 'matched' : 'mismatched', driven.current, match ? { ...match, acceptedBy: 'root-route' } : null, route.map((edge) => edge.target ?? edge.command).join(' -> '), 'tap'));
2172
+ if (!match) {
2173
+ return { current: driven.current, restored: false, diagnostic };
2174
+ }
2175
+ diagnostic.result = 'restored';
2176
+ diagnostic.accepted_by = 'root-route';
2177
+ return { current: target, restored: true, acceptedBy: 'root-route', diagnostic };
2178
+ }
2179
+ async function restoreToVariant(context, current, target, root) {
2180
+ const diagnostic = makeRestoreDiagnostic(current, target);
2181
+ const initialMatch = restoreMatch(current, target);
2182
+ if (initialMatch) {
2183
+ diagnostic.result = 'restored';
2184
+ diagnostic.accepted_by = initialMatch.acceptedBy;
2185
+ return { current: target, restored: true, acceptedBy: initialMatch.acceptedBy, diagnostic };
2186
+ }
2187
+ const restoreTargets = safeTapCandidates(current.variant).filter((candidate) => /^(back|close|dismiss|done)$/i.test(candidate.target));
2188
+ for (const restoreCandidate of restoreTargets) {
2189
+ const restored = await tryTapRestoreCandidate(context, current, target, restoreCandidate, 'dismiss-control', diagnostic);
2190
+ current = restored.current;
2191
+ if (restored.match) {
2192
+ diagnostic.result = 'restored';
2193
+ diagnostic.accepted_by = restored.match.acceptedBy;
2194
+ return { current: target, restored: true, acceptedBy: restored.match.acceptedBy, diagnostic };
2195
+ }
2196
+ }
2197
+ for (const restoreCandidate of targetRestoreCandidates(current, target)) {
2198
+ const restored = await tryTapRestoreCandidate(context, current, target, restoreCandidate, 'target-navigation', diagnostic);
2199
+ current = restored.current;
2200
+ if (restored.match) {
2201
+ diagnostic.result = 'restored';
2202
+ diagnostic.accepted_by = restored.match.acceptedBy;
2203
+ return { current: target, restored: true, acceptedBy: restored.match.acceptedBy, diagnostic };
2204
+ }
2205
+ }
2206
+ for (const restoreCandidate of reverseScrollRestoreCandidates(current, target)) {
2207
+ const restored = await tryScrollRestoreCandidate(context, current, target, restoreCandidate, diagnostic);
2208
+ current = restored.current;
2209
+ if (restored.match) {
2210
+ diagnostic.result = 'restored';
2211
+ diagnostic.accepted_by = restored.match.acceptedBy;
2212
+ return { current: target, restored: true, acceptedBy: restored.match.acceptedBy, diagnostic };
2213
+ }
2214
+ }
2215
+ try {
2216
+ await context.adapter.act({ name: 'back' });
2217
+ const observed = await observeAfterAction(context, target);
2218
+ const match = restoreMatch(observed, target);
2219
+ diagnostic.attempts.push(restoreAttemptDiagnostic('platform-back', match ? 'matched' : 'mismatched', observed, match, 'back', 'act'));
2220
+ if (match) {
2221
+ diagnostic.result = 'restored';
2222
+ diagnostic.accepted_by = match.acceptedBy;
2223
+ return { current: target, restored: true, acceptedBy: match.acceptedBy, diagnostic };
2224
+ }
2225
+ current = observed;
2226
+ }
2227
+ catch (error) {
2228
+ diagnostic.attempts.push(restoreAttemptDiagnostic('platform-back', 'error', undefined, null, 'back', 'act', error));
2229
+ }
2230
+ if (root) {
2231
+ const rooted = await restoreViaRootRoute(context, current, target, root, diagnostic);
2232
+ if (rooted) {
2233
+ return rooted;
2234
+ }
2235
+ }
2236
+ return { current, restored: false, diagnostic };
2237
+ }
2238
+ async function crawlAppMap(context, start) {
2239
+ const visited = new Set();
2240
+ const visitedTemplates = new Set();
2241
+ const exploredRepeatedContentCandidates = new Set();
2242
+ const attemptedTargets = new Map();
2243
+ const restoreDiagnostics = [];
2244
+ let restoreFailures = 0;
2245
+ let actions = 0;
2246
+ let stoppedReason = 'complete';
2247
+ const restore = async (current, target) => {
2248
+ const result = await restoreToVariant(context, current, target, start);
2249
+ if (result.diagnostic.attempts.length > 0 || result.diagnostic.accepted_by) {
2250
+ restoreDiagnostics.push(result.diagnostic);
2251
+ }
2252
+ if (!result.restored) {
2253
+ restoreFailures += 1;
2254
+ }
2255
+ return result;
2256
+ };
2257
+ const markPartial = () => {
2258
+ if (stoppedReason === 'complete') {
2259
+ stoppedReason = 'partial';
2260
+ }
2261
+ };
2262
+ const visit = async (origin, depth, allowScrollFirst = true) => {
2263
+ if (depth <= 0) {
2264
+ return origin;
2265
+ }
2266
+ if (actions >= context.options.crawlLimit) {
2267
+ stoppedReason = 'limit';
2268
+ return origin;
2269
+ }
2270
+ const originTemplateKey = crawlTemplateKey(origin.variant);
2271
+ if (visitedTemplates.has(originTemplateKey) && !visited.has(origin.variant.id)) {
2272
+ return origin;
2273
+ }
2274
+ visitedTemplates.add(originTemplateKey);
2275
+ visited.add(origin.variant.id);
2276
+ const attemptsForVariant = attemptedTargets.get(origin.variant.id) ?? new Set();
2277
+ attemptedTargets.set(origin.variant.id, attemptsForVariant);
2278
+ const candidates = safeTapCandidates(origin.variant, crawlCandidateOptions(origin.variant))
2279
+ .filter((candidate) => !attemptsForVariant.has(candidate.key));
2280
+ let current = origin;
2281
+ const exploreScrollCandidates = async (scrollCandidatesToExplore) => {
2282
+ for (const candidate of scrollCandidatesToExplore) {
2283
+ if (actions >= context.options.crawlLimit) {
2284
+ stoppedReason = 'limit';
2285
+ return true;
2286
+ }
2287
+ if (!restoreMatch(current, origin)) {
2288
+ if (restoreMatch(current, start)) {
2289
+ markPartial();
2290
+ current = start;
2291
+ return true;
2292
+ }
2293
+ stoppedReason = 'restore_failed';
2294
+ return true;
2295
+ }
2296
+ current = origin;
2297
+ attemptsForVariant.add(candidate.key);
2298
+ const args = candidate.args;
2299
+ let observed;
2300
+ try {
2301
+ await context.adapter.scroll(args);
2302
+ actions += 1;
2303
+ observed = await observeAfterCrawlAction(context);
2304
+ }
2305
+ catch {
2306
+ try {
2307
+ current = await observeCurrentScreen(context);
2308
+ }
2309
+ catch {
2310
+ // Stay on the last known origin if a failed scroll also prevents source capture.
2311
+ }
2312
+ continue;
2313
+ }
2314
+ if (observed.variant.id === origin.variant.id || observed.observation.fingerprint === origin.observation.fingerprint) {
2315
+ current = origin;
2316
+ continue;
2317
+ }
2318
+ recordEdgeSuccess(context, origin.variant, observed.variant, 'scroll', args, true);
2319
+ visited.add(observed.variant.id);
2320
+ if (observed.variant.id !== origin.variant.id && depth > 1) {
2321
+ const nestedCurrent = await visit(observed, depth - 1, false);
2322
+ const restored = await restore(nestedCurrent, origin);
2323
+ current = restored.current;
2324
+ if (!restored.restored) {
2325
+ if (restoreMatch(current, start)) {
2326
+ markPartial();
2327
+ current = start;
2328
+ return true;
2329
+ }
2330
+ stoppedReason = 'restore_failed';
2331
+ return true;
2332
+ }
2333
+ }
2334
+ else {
2335
+ const restored = await restore(observed, origin);
2336
+ current = restored.current;
2337
+ if (!restored.restored) {
2338
+ if (restoreMatch(current, start)) {
2339
+ markPartial();
2340
+ if (restoreMatch(origin, start)) {
2341
+ current = origin;
2342
+ continue;
2343
+ }
2344
+ current = start;
2345
+ return true;
2346
+ }
2347
+ stoppedReason = 'restore_failed';
2348
+ return true;
2349
+ }
2350
+ }
2351
+ }
2352
+ return false;
2353
+ };
2354
+ const scrollCandidates = crawlScrollCandidates(context, origin.variant)
2355
+ .filter((candidate) => !attemptsForVariant.has(candidate.key));
2356
+ if (allowScrollFirst && hasScrollableSurface(origin.variant) && await exploreScrollCandidates(scrollCandidates)) {
2357
+ return current;
2358
+ }
2359
+ for (const candidate of candidates) {
2360
+ const repeatedCandidateKey = repeatedContentCandidateKey(origin.variant, candidate);
2361
+ if (repeatedCandidateKey && exploredRepeatedContentCandidates.has(repeatedCandidateKey)) {
2362
+ continue;
2363
+ }
2364
+ if (actions >= context.options.crawlLimit) {
2365
+ stoppedReason = 'limit';
2366
+ return current;
2367
+ }
2368
+ if (!restoreMatch(current, origin)) {
2369
+ if (restoreMatch(current, start)) {
2370
+ markPartial();
2371
+ return start;
2372
+ }
2373
+ stoppedReason = 'restore_failed';
2374
+ return current;
2375
+ }
2376
+ current = origin;
2377
+ attemptsForVariant.add(candidate.key);
2378
+ const args = candidate.args;
2379
+ let observed;
2380
+ try {
2381
+ await context.adapter.tap(args);
2382
+ actions += 1;
2383
+ observed = await observeAfterCrawlAction(context);
2384
+ }
2385
+ catch {
2386
+ try {
2387
+ current = await observeCurrentScreen(context);
2388
+ }
2389
+ catch {
2390
+ // Stay on the last known origin if a failed tap also prevents source capture.
2391
+ }
2392
+ continue;
2393
+ }
2394
+ recordEdgeSuccess(context, origin.variant, observed.variant, 'tap', args, true);
2395
+ visited.add(observed.variant.id);
2396
+ if (repeatedCandidateKey && observed.variant.id !== origin.variant.id) {
2397
+ exploredRepeatedContentCandidates.add(repeatedCandidateKey);
2398
+ }
2399
+ if (observed.variant.id !== origin.variant.id && depth > 1) {
2400
+ const nestedCurrent = await visit(observed, depth - 1);
2401
+ const restored = await restore(nestedCurrent, origin);
2402
+ current = restored.current;
2403
+ if (!restored.restored) {
2404
+ if (restoreMatch(current, start)) {
2405
+ markPartial();
2406
+ return start;
2407
+ }
2408
+ stoppedReason = 'restore_failed';
2409
+ return current;
2410
+ }
2411
+ }
2412
+ else {
2413
+ const restored = restoreMatch(observed, origin)
2414
+ ? { current: origin, restored: true }
2415
+ : await restore(observed, origin);
2416
+ current = restored.current;
2417
+ if (!restored.restored) {
2418
+ if (restoreMatch(current, start)) {
2419
+ markPartial();
2420
+ if (restoreMatch(origin, start)) {
2421
+ current = origin;
2422
+ continue;
2423
+ }
2424
+ return start;
2425
+ }
2426
+ stoppedReason = 'restore_failed';
2427
+ return current;
2428
+ }
2429
+ }
2430
+ }
2431
+ const remainingScrollCandidates = crawlScrollCandidates(context, origin.variant)
2432
+ .filter((candidate) => !attemptsForVariant.has(candidate.key));
2433
+ if (await exploreScrollCandidates(remainingScrollCandidates)) {
2434
+ return current;
2435
+ }
2436
+ return current;
2437
+ };
2438
+ await visit(start, context.options.crawlDepth);
2439
+ return {
2440
+ enabled: true,
2441
+ actions,
2442
+ variants: visited.size,
2443
+ stopped_reason: stoppedReason,
2444
+ ...(restoreFailures > 0 ? { restore_failures: restoreFailures } : {}),
2445
+ ...(restoreDiagnostics.length > 0 ? { restore_diagnostics: restoreDiagnostics } : {})
2446
+ };
2447
+ }
685
2448
  export async function discoverAppMap(adapter, options) {
686
2449
  const context = createAppMapContext(adapter, options);
687
2450
  if (!context) {
@@ -693,10 +2456,12 @@ export async function discoverAppMap(adapter, options) {
693
2456
  }
694
2457
  try {
695
2458
  const observed = await observeCurrentScreen(context);
2459
+ const crawl = context.options.crawl ? await crawlAppMap(context, observed) : undefined;
696
2460
  const summary = persistAppMapContext(context);
697
2461
  return {
698
2462
  action: 'discover',
699
2463
  map: summary,
2464
+ ...(crawl ? { crawl } : {}),
700
2465
  screen: {
701
2466
  variant_id: observed.variant.id,
702
2467
  screen_id: observed.variant.screen_id,