simframe 0.7.2 → 0.9.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/src/index.js CHANGED
@@ -19,6 +19,7 @@ import * as graph from './graph.js';
19
19
  import * as matching from './matching.js';
20
20
  import * as refs from './refs.js';
21
21
  import * as screenmap from './screenmap.js';
22
+ import * as metrics from './metrics.js';
22
23
  import { capabilitiesFor, resolveDevice, resize, screenshot } from './platform/index.js';
23
24
  import * as store from './store.js';
24
25
 
@@ -314,10 +315,24 @@ function stallNote(health) {
314
315
  const parts = [`capture: stalled — the display surface has been unreadable for ${Math.round(forMs / 1000)}s`];
315
316
  if (health.reattaches) parts.push(`${health.reattaches} re-attach${health.reattaches === 1 ? '' : 'es'} did not help`);
316
317
  if (health.reason) parts.push(String(health.reason).slice(0, 120));
317
- // The cure is the user's to apply. Saying so is the difference between an
318
- // agent that reports "the simulator is wedged" and one that retries a tap
319
- // twenty times because nothing appeared to change.
320
- parts.push('only restarting the device is known to cure it');
318
+ // What to do about it, and this line has been wrong until now.
319
+ //
320
+ // It said "only restarting the device is known to cure it". Then the same
321
+ // daemon's log turned out to contain nine "capture recovered on its own"
322
+ // lines, and a wedge that had survived 223 port re-resolves and 6 device
323
+ // rebinds cleared by itself while nobody touched it. Meanwhile Apple's own
324
+ // `simctl io screenshot` on a wedged device returns a valid PNG whose every
325
+ // pixel is black, in 16 s — so the display pipeline has stopped rendering
326
+ // and simframe's read is an accurate report of that, not a bug in it.
327
+ //
328
+ // So: it is the simulator, it often comes back, and restarting the device
329
+ // also cures it. `simframe doctor` runs a screenshot probe when this is
330
+ // published and says which of the two faults it is.
331
+ parts.push(
332
+ "this is the simulator's display, not simframe's read of it: Apple's own screenshot path "
333
+ + 'returns an all-black image on a wedged device. It frequently recovers on its own; '
334
+ + 'restarting the device also cures it, and `simframe doctor` will confirm which fault this is',
335
+ );
321
336
  return parts.join('; ');
322
337
  }
323
338
 
@@ -473,7 +488,7 @@ function pngSize(png) {
473
488
  return { width: png.readUInt32BE(16), height: png.readUInt32BE(20) };
474
489
  }
475
490
 
476
- export async function getState(deviceQuery, { since, options } = {}) {
491
+ export async function getState(deviceQuery, { since, options, inputHealth = false } = {}) {
477
492
  const { device, state } = await ensureDaemon(deviceQuery, options);
478
493
  return {
479
494
  device,
@@ -482,9 +497,54 @@ export async function getState(deviceQuery, { since, options } = {}) {
482
497
  map: regionMap(state.regions || [], REGION_COLS),
483
498
  since: compareToBaseline(state, resolveBaseline(state, since)),
484
499
  live: liveness(device.udid, state),
500
+ // Off by default and asked for by the state commands only. A flow step
501
+ // calls getState twice, and the fix for a stale session runs before every
502
+ // action anyway (input.ensureFreshSession) — this is the report, not the
503
+ // repair.
504
+ input: inputHealth ? await input.sessionHealth(device.udid) : undefined,
505
+ timing: inputHealth ? timingOfNow(device.udid, state) : undefined,
485
506
  };
486
507
  }
487
508
 
509
+ /**
510
+ * How long this screen has been moving, against how long it usually takes.
511
+ *
512
+ * Research §7 asks `sim_state` for this, and the point is a sentence an agent
513
+ * can act on: a screen 400 ms into a transition that normally takes 350 ms is
514
+ * fine, and the same screen four seconds in is not. Elapsed comes from the
515
+ * frame store's own clock; the distribution comes from the edge that was most
516
+ * recently traversed into this screen.
517
+ *
518
+ * Reads no frames and takes no perception pass: the layout hash and the
519
+ * structural hash screen memory filed under it are both already on disk.
520
+ */
521
+ function timingOfNow(udid, state) {
522
+ try {
523
+ const near = screenmap.recallNearest(udid, state.layoutHash);
524
+ const edge = graph.timingInto(udid, near?.entry?.structuralHash);
525
+ const elapsed = Number.isFinite(state.lastChangeAt) ? Date.now() - state.lastChangeAt : null;
526
+ const settled = state.stableForMs >= STRUCTURAL_SETTLE_MS;
527
+ const slower = graph.slowerThanUsual({
528
+ elapsedMs: elapsed,
529
+ p95: edge?.p95,
530
+ settled,
531
+ kind: state.transition?.kind,
532
+ });
533
+ return {
534
+ edge_p50: edge?.p50 ?? null,
535
+ edge_p95: edge?.p95 ?? null,
536
+ samples: edge?.samples ?? 0,
537
+ elapsed_ms: elapsed,
538
+ stable_for_ms: state.stableForMs ?? null,
539
+ slower_than_usual: Boolean(slower.slower),
540
+ note: slower.slower ? slower.note : null,
541
+ };
542
+ } catch {
543
+ // Timing is commentary. A screen with no history still has a state.
544
+ return null;
545
+ }
546
+ }
547
+
488
548
  /**
489
549
  * Wait for the screen to settle (`mode: 'stable'`) or to move away from what it
490
550
  * shows right now (`mode: 'change'`). Removes the screenshot-retry loop.
@@ -516,6 +576,21 @@ export async function waitFor(
516
576
  const deadline = startedAt + timeoutMs;
517
577
  const startSeq = first.seq;
518
578
  let last = first;
579
+ /**
580
+ * The longest pause *inside* this transition, in ms.
581
+ *
582
+ * This is the statistic a stillness window exists to defeat: a transition
583
+ * that pauses for 180 ms mid-flight will be mistaken for a finished screen
584
+ * by any window shorter than that. Nobody was measuring it, so the window
585
+ * was a constant — 500 ms, chosen once, paid by every step forever.
586
+ *
587
+ * Tracked as the highest `stableForMs` observed before the screen moved
588
+ * again. Reported so a caller can learn it per edge; a settle that ends
589
+ * without a second change has no gap to report and says 0.
590
+ */
591
+ let quietGapMs = 0;
592
+ let quietRun = 0;
593
+ let lastHash = first.hash;
519
594
  let sawChange = mode === 'stable' || first.hash !== baselineHashValue;
520
595
  const changedAtStart = sawChange && mode !== 'stable';
521
596
 
@@ -524,6 +599,7 @@ export async function waitFor(
524
599
  state: last,
525
600
  satisfied,
526
601
  mode,
602
+ quietGapMs,
527
603
  sawChange,
528
604
  changedBeforeWait: changedAtStart,
529
605
  baselineHash: baselineHashValue,
@@ -543,6 +619,17 @@ export async function waitFor(
543
619
 
544
620
  if (!sawChange && state.hash !== baselineHashValue) sawChange = true;
545
621
 
622
+ // A pause that turned out not to be the end of the transition. Only
623
+ // pauses followed by more movement count: the quiet at the end of a
624
+ // settle is the answer, not a gap.
625
+ if (state.hash !== lastHash) {
626
+ if (quietRun > quietGapMs) quietGapMs = quietRun;
627
+ quietRun = 0;
628
+ lastHash = state.hash;
629
+ } else if (Number.isFinite(state.stableForMs)) {
630
+ quietRun = Math.max(quietRun, state.stableForMs);
631
+ }
632
+
546
633
  // Some controls barely move the screen at all — a radio dot, a checkbox,
547
634
  // a button changing state. Waiting the full timeout for a change that
548
635
  // will never be visible turns a 100ms action into a 12s one, so give up
@@ -890,8 +977,15 @@ export async function locate(
890
977
  const list = outcome.alternatives
891
978
  .map((a, i) => `[${i}] "${a.label}" (${a.x},${a.y}) ${a.region ?? 'content'} ${a.score}`)
892
979
  .join(', ');
893
- throw new Error(
894
- `"${query}" matches ${outcome.alternatives.length} things on this screen — say which, or pass index: ${list}`,
980
+ // Tagged, not just thrown: the reason an escalation happened is known
981
+ // here and nowhere above here. See metrics.tag — it adds a property and
982
+ // changes nothing else about the error.
983
+ throw metrics.tag(
984
+ new Error(
985
+ `"${query}" matches ${outcome.alternatives.length} things on this screen — say which, or pass index: ${list}`,
986
+ ),
987
+ 'ambiguous_intent',
988
+ { candidates: outcome.alternatives },
895
989
  );
896
990
  }
897
991
  if (outcome.status === 'ok') {
@@ -905,24 +999,28 @@ export async function locate(
905
999
  // here undoes every guard above — it has no off-screen filter and no
906
1000
  // coverage weighting, and it is what returned a scrolled-away list row for
907
1001
  // "back". "Not found" is the correct answer.
908
- const sample = entry.targets
909
- .filter((t) => t.label && t.y >= 0 && t.y <= points.height)
910
- .slice(0, 12)
911
- .map((t) => t.label.slice(0, 24))
912
- .join(', ');
913
- throw new Error(`"${query}" is not on this screen. Visible: ${sample || '(nothing readable)'}`);
1002
+ const visible = entry.targets.filter((t) => t.label && t.y >= 0 && t.y <= points.height);
1003
+ const sample = visible.slice(0, 12).map((t) => t.label.slice(0, 24)).join(', ');
1004
+ // Which escalation this is depends on whether the screen was recognised.
1005
+ // Screen memory had nothing for it (`from` is one of the built values) and
1006
+ // the target is missing: that is not knowing the screen. On a screen
1007
+ // recalled from memory, the screen is known and the intent did not resolve.
1008
+ throw metrics.tag(
1009
+ new Error(`"${query}" is not on this screen. Visible: ${sample || '(nothing readable)'}`),
1010
+ from === 'memory' ? 'ambiguous_intent' : 'unknown_screen',
1011
+ { candidates: visible.slice(0, 8) },
1012
+ );
914
1013
  }
915
1014
 
916
1015
  const candidates = screenmap.rank(entry, query);
917
1016
  const target = index != null ? candidates[index] : candidates[0];
918
1017
  if (!target) {
919
- const sample = entry.targets
920
- .filter((t) => t.label)
921
- .slice(0, 12)
922
- .map((t) => t.label)
923
- .join(', ');
924
- throw new Error(
925
- `"${query}" is not on this screen. Visible: ${sample || '(nothing readable)'}`,
1018
+ const visible = entry.targets.filter((t) => t.label);
1019
+ const sample = visible.slice(0, 12).map((t) => t.label).join(', ');
1020
+ throw metrics.tag(
1021
+ new Error(`"${query}" is not on this screen. Visible: ${sample || '(nothing readable)'}`),
1022
+ from === 'memory' ? 'ambiguous_intent' : 'unknown_screen',
1023
+ { candidates: visible.slice(0, 8) },
926
1024
  );
927
1025
  }
928
1026
  return { device, state: current, entry, target, from, distance, settled, screens: screenmap.stats(udid).screens };
package/src/input.js CHANGED
@@ -2,8 +2,11 @@
2
2
  // capability layered on top, so every entry point here has to answer "is this
3
3
  // even available?" before it answers anything else.
4
4
  import { execFile } from 'node:child_process';
5
+ import fs from 'node:fs';
6
+ import path from 'node:path';
5
7
  import * as control from './control.js';
6
- import { capabilitiesFor, geometryFor, inputDriverFor, setPasteboard } from './platform/index.js';
8
+ import * as store from './store.js';
9
+ import { bootedAtFor, capabilitiesFor, geometryFor, inputDriverFor, setPasteboard } from './platform/index.js';
7
10
  import { promisify } from 'node:util';
8
11
 
9
12
  const run = promisify(execFile);
@@ -323,6 +326,7 @@ export function centerOf(node) {
323
326
  }
324
327
 
325
328
  export async function tapPoint(udid, x, y, { durationMs } = {}) {
329
+ await ensureFreshSession(udid);
326
330
  const point = { x: Math.round(x), y: Math.round(y) };
327
331
  const own = inputDriverFor(udid);
328
332
  if (own) {
@@ -347,6 +351,7 @@ export async function tapLabel(udid, query, { index, durationMs } = {}) {
347
351
  }
348
352
 
349
353
  export async function typeText(udid, value) {
354
+ await ensureFreshSession(udid);
350
355
  const own = inputDriverFor(udid);
351
356
  if (own) {
352
357
  // No pasteboard on Android (docs/DEFERRED.md), so exact text goes through
@@ -379,6 +384,7 @@ export async function typeText(udid, value) {
379
384
  * to say so; `type` still works on every device.
380
385
  */
381
386
  export async function pasteText(udid, value) {
387
+ await ensureFreshSession(udid);
382
388
  const own = inputDriverFor(udid);
383
389
  if (own?.key) {
384
390
  // The clipboard goes over gRPC; KEYCODE_PASTE is what puts it in the field.
@@ -413,6 +419,7 @@ export async function typeKeys(udid, value) {
413
419
  }
414
420
 
415
421
  export async function pressKey(udid, keycode) {
422
+ await ensureFreshSession(udid);
416
423
  const own = inputDriverFor(udid);
417
424
  if (own) {
418
425
  await own.key(udid, keycode);
@@ -433,10 +440,111 @@ export async function pressKey(udid, keycode) {
433
440
  *
434
441
  * @returns {Promise<boolean>} whether a session was actually reset.
435
442
  */
443
+ /**
444
+ * Is the daemon's HID session older than the device it talks to?
445
+ *
446
+ * Pure, so the comparison is testable without a device. `graceMs` covers the
447
+ * ordinary case where a daemon is started immediately after a boot and the two
448
+ * timestamps land within milliseconds of each other in either order.
449
+ */
450
+ export function sessionStaleness({ bootedAt, sessionSince, graceMs = 2000 }) {
451
+ if (!Number.isFinite(bootedAt)) {
452
+ return { stale: false, reason: 'cannot tell when the device booted' };
453
+ }
454
+ if (!Number.isFinite(sessionSince)) {
455
+ return { stale: false, reason: 'no capture daemon has recorded a start time, so there is no session to compare against' };
456
+ }
457
+ if (bootedAt <= sessionSince + graceMs) return { stale: false, reason: null };
458
+ return {
459
+ stale: true,
460
+ reason: `the device booted ${Math.round((bootedAt - sessionSince) / 1000)}s after the capture daemon started, `
461
+ + 'so the daemon holds an HID session for a device session that no longer exists',
462
+ bootedAt,
463
+ sessionSince,
464
+ };
465
+ }
466
+
467
+ /**
468
+ * What state the input path is in, for doctor and sim_state.
469
+ *
470
+ * Reads two timestamps off disk — the device's boot marker and the daemon's
471
+ * own `startedAt` — and costs a stat each. No input is dispatched to find out,
472
+ * because the whole failure being detected is input that reports success and
473
+ * does nothing.
474
+ */
475
+ const bootCache = new Map();
476
+ /**
477
+ * Boot time, cached for a moment.
478
+ *
479
+ * iOS answers with a stat; Android runs `adb shell cat /proc/uptime`, a
480
+ * subprocess of 20-40 ms, and `getState` runs twice per flow step. A few
481
+ * seconds of staleness in the staleness detector costs nothing — a device that
482
+ * rebooted three seconds ago is still rebooted at the next check.
483
+ */
484
+ async function bootedAtCached(udid, ttlMs = 3000) {
485
+ const hit = bootCache.get(udid);
486
+ if (hit && Date.now() - hit.at < ttlMs) return hit.value;
487
+ const value = await bootedAtFor(udid);
488
+ bootCache.set(udid, { at: Date.now(), value });
489
+ return value;
490
+ }
491
+
492
+ export async function sessionHealth(udid) {
493
+ if (!udid || !control.available(udid)) return { stale: false, reason: null };
494
+ let bootedAt = null;
495
+ try {
496
+ bootedAt = await bootedAtCached(udid);
497
+ } catch (err) {
498
+ return { stale: false, reason: `cannot tell when the device booted: ${err.message}` };
499
+ }
500
+ const meta = store.readJson(store.paths(udid).meta);
501
+ const rebuilt = store.readJson(sessionFile(udid))?.rebuiltAt;
502
+ // The newer of the two: the daemon starting creates a session, and rebuilding
503
+ // it replaces one. Either makes the session current as of that moment.
504
+ const sessionSince = Math.max(meta?.startedAt ?? 0, rebuilt ?? 0) || undefined;
505
+ return sessionStaleness({ bootedAt, sessionSince });
506
+ }
507
+
508
+ /**
509
+ * Rebuild the session if the device outlived it. Once per process, per device.
510
+ *
511
+ * Rebuild, and retry nothing: this runs *before* the action, so the action is
512
+ * delivered on a session known to be current. Retrying afterwards is how an
513
+ * action fires twice, which is the hazard the verify barrier exists to
514
+ * prevent — and it is why the existing recovery covers hardware buttons only.
515
+ */
516
+ const freshened = new Set();
517
+ export async function ensureFreshSession(udid) {
518
+ if (!udid || freshened.has(udid)) return null;
519
+ freshened.add(udid);
520
+ const health = await sessionHealth(udid);
521
+ if (!health.stale) return null;
522
+ const rebuilt = await resetSession(udid);
523
+ return { ...health, rebuilt };
524
+ }
525
+
526
+ /**
527
+ * When the HID session was last rebuilt, if it has been.
528
+ *
529
+ * The daemon's `startedAt` is the wrong clock on its own: rebuilding the
530
+ * session makes it current again without restarting the daemon, so comparing
531
+ * against the daemon's start left `doctor` reporting `stale` about a session
532
+ * that had just been rebuilt and was demonstrably working. It is a file rather
533
+ * than a variable because every CLI command is a new process and the daemon
534
+ * holding the session outlives all of them.
535
+ */
536
+ const sessionFile = (udid) => path.join(store.deviceDir(udid), 'input-session.json');
537
+
436
538
  export async function resetSession(udid) {
437
539
  if (!control.available(udid)) return false;
438
540
  try {
439
541
  await control.resetInput(udid);
542
+ try {
543
+ fs.mkdirSync(store.deviceDir(udid), { recursive: true });
544
+ store.writeAtomic(sessionFile(udid), JSON.stringify({ rebuiltAt: Date.now() }));
545
+ } catch {
546
+ /* the rebuild happened; failing to write it down only costs a stale report */
547
+ }
440
548
  return true;
441
549
  } catch {
442
550
  return false;
@@ -444,6 +552,7 @@ export async function resetSession(udid) {
444
552
  }
445
553
 
446
554
  export async function pressButton(udid, name) {
555
+ await ensureFreshSession(udid);
447
556
  const own = inputDriverFor(udid);
448
557
  if (own) {
449
558
  // Android's whole key vocabulary is safe to offer: `input keyevent` takes
@@ -464,6 +573,7 @@ export async function pressButton(udid, name) {
464
573
  }
465
574
 
466
575
  export async function swipe(udid, from, to, { durationMs = 300 } = {}) {
576
+ await ensureFreshSession(udid);
467
577
  const own = inputDriverFor(udid);
468
578
  if (own) {
469
579
  await own.swipe(udid, from, to, { durationMs });
package/src/intent.js CHANGED
@@ -5,6 +5,7 @@
5
5
  // instinct removes the expensive part of driving a UI with an agent: the model
6
6
  // round trip spent reasoning about controls it has never seen.
7
7
  import * as input from './input.js';
8
+ import * as metrics from './metrics.js';
8
9
 
9
10
  /** Confirming words, most specific first. The first tier with a hit wins. */
10
11
  const CONFIRM_TIERS = [
@@ -77,7 +78,11 @@ export function findOptions(nodes, geo) {
77
78
  export async function chooseAny(udid, { prefer, geo } = {}) {
78
79
  const nodes = await input.describeAll(udid);
79
80
  const options = findOptions(nodes, geo);
80
- if (!options.length) throw new Error('no selectable options found on screen');
81
+ if (!options.length) {
82
+ throw metrics.tag(new Error('no selectable options found on screen'), 'novel_dialog', {
83
+ candidates: nodes.filter((n) => n.label).slice(0, 8).map((n) => ({ label: n.label })),
84
+ });
85
+ }
81
86
  const chosen =
82
87
  (prefer && options.find((o) => o.label.toLowerCase().includes(String(prefer).toLowerCase()))) ||
83
88
  options[0];
@@ -89,7 +94,11 @@ export async function chooseAny(udid, { prefer, geo } = {}) {
89
94
  export async function confirm(udid, { geo } = {}) {
90
95
  const nodes = await input.describeAll(udid);
91
96
  const target = findConfirm(nodes, geo);
92
- if (!target) throw new Error('no confirming control on screen');
97
+ if (!target) {
98
+ throw metrics.tag(new Error('no confirming control on screen'), 'novel_dialog', {
99
+ candidates: nodes.filter((n) => n.label).slice(0, 8).map((n) => ({ label: n.label })),
100
+ });
101
+ }
93
102
  const point = input.centerOf(target);
94
103
  await input.tapPoint(udid, point.x, point.y);
95
104
  return { label: target.label, point, enabled: target.enabled };
package/src/matching.js CHANGED
@@ -182,6 +182,77 @@ export const SAME_CONTROL_POINTS = 12;
182
182
  */
183
183
  const INTERACTIVE_ROLE = /button|field|cell|row|link|switch|slider|tab|menu|segment|checkbox/i;
184
184
 
185
+ /**
186
+ * Is this target the accessibility tree's reading of a control?
187
+ *
188
+ * A merged target carries `ax|ocr`, because both sensors saw it. Every
189
+ * comparison against the string `'ax'` had to become this: an element does not
190
+ * stop being the tree's element because OCR agreed with it, and treating
191
+ * `ax|ocr` as "not ax" would have quietly demoted exactly the elements the
192
+ * merge is most confident about.
193
+ */
194
+ export const isAxTarget = (t) => /(^|\|)ax(\||$)/.test(t?.source ?? '');
195
+
196
+ /** How much of `inner` lies inside `outer`, as a fraction of inner's own area. */
197
+ export function containedFraction(inner, outer) {
198
+ if (!inner || !outer) return 0;
199
+ const iw = Math.max(0, inner.width ?? 0);
200
+ const ih = Math.max(0, inner.height ?? 0);
201
+ const innerArea = iw * ih;
202
+ if (innerArea <= 0) return 0;
203
+ const x = Math.max(inner.x, outer.x);
204
+ const y = Math.max(inner.y, outer.y);
205
+ const right = Math.min(inner.x + iw, outer.x + (outer.width ?? 0));
206
+ const bottom = Math.min(inner.y + ih, outer.y + (outer.height ?? 0));
207
+ const overlap = Math.max(0, right - x) * Math.max(0, bottom - y);
208
+ // Clamped: float arithmetic on sub-pixel OCR frames put a fully contained
209
+ // box at 1.0000000000000007, and a fraction of an area cannot exceed 1.
210
+ return Math.min(1, overlap / innerArea);
211
+ }
212
+
213
+ /**
214
+ * How much of the smaller box must sit inside the larger one to be the same
215
+ * element. OCR boxes sit a pixel or two outside the row they are printed on
216
+ * often enough that 1.0 would miss them.
217
+ */
218
+ export const CONTAINMENT = 0.9;
219
+
220
+ /**
221
+ * Do these two strings name the same thing?
222
+ *
223
+ * The discriminator that makes containment safe. A tab bar contains all five
224
+ * of its tab labels, and merging a container with its contents is the failure
225
+ * the old size cap was defending against — but a tab bar's own label is not
226
+ * "Assets", so the text test refuses that merge while allowing a row labelled
227
+ * "Kate Bell" to absorb OCR's reading of "Kate Bell".
228
+ *
229
+ * Substring counts because iOS labels carry state the printed text does not:
230
+ * a row reads "Larger Text" on screen and publishes "Larger Text, Off".
231
+ */
232
+ export function sameText(a, b) {
233
+ const x = norm(a);
234
+ const y = norm(b);
235
+ if (!x || !y) return false;
236
+ if (x === y) return true;
237
+ if (x.includes(y) || y.includes(x)) return Math.min(x.length, y.length) >= 3;
238
+ // Fuzzy, because OCR misreads a letter or two — measured: "Location (AII)"
239
+ // for "Location (All)", and a Cyrillic К for a K in a monogram.
240
+ return nameScore(x, y) >= 0.5;
241
+ }
242
+
243
+ /**
244
+ * The same element, seen by both sensors.
245
+ *
246
+ * `ax` is the tree's element, `ocr` a text box. True when the text sits
247
+ * (almost) wholly inside the element AND says the same thing as its label or
248
+ * value.
249
+ */
250
+ export function sameElementSeenTwice(ax, ocr) {
251
+ if (!ax?.frame || !ocr?.frame) return false;
252
+ if (containedFraction(ocr.frame, ax.frame) < CONTAINMENT) return false;
253
+ return sameText(ax.label, ocr.label ?? ocr.text) || sameText(ax.value, ocr.label ?? ocr.text);
254
+ }
255
+
185
256
  const contains = (frame, target) =>
186
257
  Boolean(frame)
187
258
  && target.x >= frame.x && target.x <= frame.x + (frame.width ?? 0)
@@ -205,6 +276,14 @@ function sameControl(a, b) {
205
276
  // tab bar from absorbing its own tabs.
206
277
  if (INTERACTIVE_ROLE.test(a.type ?? '') && contains(a.frame, b)) return true;
207
278
  if (INTERACTIVE_ROLE.test(b.type ?? '') && contains(b.frame, a)) return true;
279
+ // And a labelled accessibility element that is not an interactive role —
280
+ // a list row published as StaticText — with OCR's reading of its own label
281
+ // inside it. This is the pair that made `tap "Kate Bell"` refuse on every
282
+ // Contacts list: 16 escalations in the first instrumented run, all one
283
+ // screen. Defence in depth: the screen map now merges this pair at fusion,
284
+ // and a map built before that still resolves.
285
+ if (isAxTarget(a) && !isAxTarget(b) && sameElementSeenTwice(a, b)) return true;
286
+ if (isAxTarget(b) && !isAxTarget(a) && sameElementSeenTwice(b, a)) return true;
208
287
  return false;
209
288
  }
210
289
 
@@ -219,8 +298,8 @@ function collapseSamePlace(ranked) {
219
298
  // Prefer the real hit target: an accessibility element over OCR's reading of
220
299
  // it, and an interactive role over a caption sitting inside it.
221
300
  const better = (candidate, incumbent) => {
222
- if (candidate.target.source === 'ax' && incumbent.target.source !== 'ax') return true;
223
- if (candidate.target.source !== 'ax' && incumbent.target.source === 'ax') return false;
301
+ if (isAxTarget(candidate.target) && !isAxTarget(incumbent.target)) return true;
302
+ if (!isAxTarget(candidate.target) && isAxTarget(incumbent.target)) return false;
224
303
  return INTERACTIVE_ROLE.test(candidate.target.type ?? '')
225
304
  && !INTERACTIVE_ROLE.test(incumbent.target.type ?? '');
226
305
  };
package/src/mcp.js CHANGED
@@ -480,7 +480,7 @@ async function look(target, args, options) {
480
480
  async function state(target, args, options) {
481
481
  const { device } = await api.ensureDaemon(target, options);
482
482
  const requested = baselineFor(device.udid, args.since);
483
- const res = await api.getState(target, { since: requested, options });
483
+ const res = await api.getState(target, { since: requested, options, inputHealth: true });
484
484
  const s = res.state;
485
485
  const lines = [
486
486
  header(res.device, s, res.ageMs),
@@ -500,6 +500,19 @@ async function state(target, args, options) {
500
500
  }
501
501
  const warn = livenessLine(res.live);
502
502
  if (warn) lines.unshift(warn);
503
+ // An agent that cannot see this spends five turns wondering why a correct
504
+ // tap on a correct element did nothing. The repair happens before the next
505
+ // action either way; this is so the cause is visible when it does.
506
+ if (res.input?.stale) lines.unshift(`input: stale — ${res.input.reason}`);
507
+ // §7's timing line. Worth a line because "is it still coming or is it done"
508
+ // is a question an agent otherwise answers by waiting and guessing.
509
+ if (res.timing?.samples) {
510
+ lines.push(
511
+ `timing: usually ${res.timing.edge_p50}ms to arrive here (p95 ${res.timing.edge_p95}ms over `
512
+ + `${res.timing.samples} samples), ${res.timing.elapsed_ms}ms since the last change`
513
+ + (res.timing.slower_than_usual ? ` — ${res.timing.note}` : ''),
514
+ );
515
+ }
503
516
  remember(device.udid, s);
504
517
  return { content: [text(lines.filter(Boolean).join('\n'))] };
505
518
  }