simframe 0.8.0 → 0.10.0

package/src/input.js

@@ -2,8 +2,11 @@
 // capability layered on top, so every entry point here has to answer "is this
 // even available?" before it answers anything else.
 import { execFile } from 'node:child_process';
+import fs from 'node:fs';
+import path from 'node:path';
 import * as control from './control.js';
-import { capabilitiesFor, geometryFor, inputDriverFor, setPasteboard } from './platform/index.js';
+import * as store from './store.js';
+import { bootedAtFor, capabilitiesFor, geometryFor, inputDriverFor, setPasteboard } from './platform/index.js';
 import { promisify } from 'node:util';
 
 const run = promisify(execFile);
@@ -245,6 +248,11 @@ export function elementToNode(e) {
     type: e.role ?? null,
     identifier: e.identifier ?? null,
     enabled: e.state?.enabled ?? null,
+    // The daemon batches AXSelected and AXFocused alongside AXEnabled and has
+    // since 0.6.0. This converter took one of the three, and it is the one on
+    // the path that actually runs — `normalizeNode` below is the idb fallback.
+    selected: e.state?.selected ?? null,
+    focused: e.state?.focused ?? null,
     frame: e.frame ?? null,
     raw: e,
   };
@@ -270,6 +278,12 @@ function normalizeNode(node) {
     type: node.type ?? node.AXType ?? null,
     identifier: node.AXUniqueId ?? node.identifier ?? null,
     enabled: node.AXEnabled ?? node.enabled ?? null,
+    // The daemon has asked the tree for AXSelected and AXFocused since 0.6.0 —
+    // they are two of the eight attributes in its batched round trip — and this
+    // function dropped both. `view.renderRow` has printed `selected` for as
+    // long as it has existed, against a field nobody set.
+    selected: node.AXSelected ?? node.selected ?? null,
+    focused: node.AXFocused ?? node.focused ?? null,
     frame: frame
       ? {
           x: frame.x ?? frame.X ?? 0,
@@ -323,6 +337,7 @@ export function centerOf(node) {
 }
 
 export async function tapPoint(udid, x, y, { durationMs } = {}) {
+  await ensureFreshSession(udid);
   const point = { x: Math.round(x), y: Math.round(y) };
   const own = inputDriverFor(udid);
   if (own) {
@@ -347,6 +362,7 @@ export async function tapLabel(udid, query, { index, durationMs } = {}) {
 }
 
 export async function typeText(udid, value) {
+  await ensureFreshSession(udid);
   const own = inputDriverFor(udid);
   if (own) {
     // No pasteboard on Android (docs/DEFERRED.md), so exact text goes through
@@ -379,6 +395,7 @@ export async function typeText(udid, value) {
  * to say so; `type` still works on every device.
  */
 export async function pasteText(udid, value) {
+  await ensureFreshSession(udid);
   const own = inputDriverFor(udid);
   if (own?.key) {
     // The clipboard goes over gRPC; KEYCODE_PASTE is what puts it in the field.
@@ -413,6 +430,7 @@ export async function typeKeys(udid, value) {
 }
 
 export async function pressKey(udid, keycode) {
+  await ensureFreshSession(udid);
   const own = inputDriverFor(udid);
   if (own) {
     await own.key(udid, keycode);
@@ -433,10 +451,144 @@ export async function pressKey(udid, keycode) {
  *
  * @returns {Promise<boolean>} whether a session was actually reset.
  */
+/**
+ * Is the daemon's HID session older than the device it talks to?
+ *
+ * Pure, so the comparison is testable without a device. `graceMs` covers the
+ * ordinary case where a daemon is started immediately after a boot and the two
+ * timestamps land within milliseconds of each other in either order.
+ */
+export function sessionStaleness({ bootedAt, sessionSince, graceMs = 2000 }) {
+  if (!Number.isFinite(bootedAt)) {
+    return { stale: false, reason: 'cannot tell when the device booted' };
+  }
+  if (!Number.isFinite(sessionSince)) {
+    return { stale: false, reason: 'no capture daemon has recorded a start time, so there is no session to compare against' };
+  }
+  if (bootedAt <= sessionSince + graceMs) return { stale: false, reason: null };
+  return {
+    stale: true,
+    reason: `the device booted ${Math.round((bootedAt - sessionSince) / 1000)}s after the capture daemon started, `
+      + 'so the daemon holds an HID session for a device session that no longer exists',
+    bootedAt,
+    sessionSince,
+  };
+}
+
+/**
+ * What state the input path is in, for doctor and sim_state.
+ *
+ * Reads two timestamps off disk — the device's boot marker and the daemon's
+ * own `startedAt` — and costs a stat each. No input is dispatched to find out,
+ * because the whole failure being detected is input that reports success and
+ * does nothing.
+ */
+const bootCache = new Map();
+/**
+ * Boot time, cached for a moment.
+ *
+ * iOS answers with a stat; Android runs `adb shell cat /proc/uptime`, a
+ * subprocess of 20-40 ms, and `getState` runs twice per flow step. A few
+ * seconds of staleness in the staleness detector costs nothing — a device that
+ * rebooted three seconds ago is still rebooted at the next check.
+ */
+async function bootedAtCached(udid, ttlMs = 3000) {
+  const hit = bootCache.get(udid);
+  if (hit && Date.now() - hit.at < ttlMs) return hit.value;
+  const value = await bootedAtFor(udid);
+  bootCache.set(udid, { at: Date.now(), value });
+  return value;
+}
+
+export async function sessionHealth(udid) {
+  if (!udid || !control.available(udid)) return { stale: false, reason: null };
+  let bootedAt = null;
+  try {
+    bootedAt = await bootedAtCached(udid);
+  } catch (err) {
+    return { stale: false, reason: `cannot tell when the device booted: ${err.message}` };
+  }
+  const meta = store.readJson(store.paths(udid).meta);
+  const rebuilt = store.readJson(sessionFile(udid))?.rebuiltAt;
+  // The newer of the two: the daemon starting creates a session, and rebuilding
+  // it replaces one. Either makes the session current as of that moment.
+  const sessionSince = Math.max(meta?.startedAt ?? 0, rebuilt ?? 0) || undefined;
+  return sessionStaleness({ bootedAt, sessionSince });
+}
+
+/**
+ * Should this staleness be acted on, given what has already been rebuilt?
+ *
+ * Pure, and separate from the check because the *key* is the whole bug. This
+ * gate used to be a set of udids — "once per process, per device" — and the
+ * reasoning was to avoid statting on every action. What it actually bought was
+ * that the feature could not fire in the one process that matters. A CLI
+ * command is a new process every time, so per-process is per-call there and
+ * the gate never showed; the MCP server is a single process that lives for a
+ * whole session, so it checked once, at the first action, and then never
+ * again — and a device that reboots *mid-session* is precisely the case this
+ * exists to catch. Reported from a real session: capture kept working, input
+ * died, every tap returned `ok`, and about ten calls went into two wrong
+ * conclusions about the app.
+ *
+ * The right key is the boot the rebuild was for. One attempt per device boot:
+ * enough that a failed rebuild does not retry on every tap forever, and not so
+ * much that the next boot is invisible.
+ */
+export function shouldRebuildSession({ stale, bootedAt }, rebuiltFor) {
+  if (!stale) return false;
+  // A boot we cannot date cannot be memoised against, and re-attempting on
+  // every action would be worse than not detecting it. sessionStaleness only
+  // reports stale with a finite bootedAt, so this is a belt, not a case.
+  if (!Number.isFinite(bootedAt)) return false;
+  return rebuiltFor !== bootedAt;
+}
+
+/**
+ * Rebuild the session if the device outlived it. Once per device boot.
+ *
+ * Rebuild, and retry nothing: this runs *before* the action, so the action is
+ * delivered on a session known to be current. Retrying afterwards is how an
+ * action fires twice, which is the hazard the verify barrier exists to
+ * prevent — and it is why the existing recovery covers hardware buttons only.
+ *
+ * The check now runs on every dispatch rather than once. It costs two small
+ * `readJson`s and, at most every three seconds, one stat — `bootedAtCached`
+ * already caps the part that was expensive, which is what made the
+ * once-per-process gate unnecessary as well as wrong.
+ */
+const rebuiltForBoot = new Map();
+export async function ensureFreshSession(udid) {
+  if (!udid) return null;
+  const health = await sessionHealth(udid);
+  if (!shouldRebuildSession(health, rebuiltForBoot.get(udid))) return null;
+  rebuiltForBoot.set(udid, health.bootedAt);
+  const rebuilt = await resetSession(udid);
+  return { ...health, rebuilt };
+}
+
+/**
+ * When the HID session was last rebuilt, if it has been.
+ *
+ * The daemon's `startedAt` is the wrong clock on its own: rebuilding the
+ * session makes it current again without restarting the daemon, so comparing
+ * against the daemon's start left `doctor` reporting `stale` about a session
+ * that had just been rebuilt and was demonstrably working. It is a file rather
+ * than a variable because every CLI command is a new process and the daemon
+ * holding the session outlives all of them.
+ */
+const sessionFile = (udid) => path.join(store.deviceDir(udid), 'input-session.json');
+
 export async function resetSession(udid) {
   if (!control.available(udid)) return false;
   try {
     await control.resetInput(udid);
+    try {
+      fs.mkdirSync(store.deviceDir(udid), { recursive: true });
+      store.writeAtomic(sessionFile(udid), JSON.stringify({ rebuiltAt: Date.now() }));
+    } catch {
+      /* the rebuild happened; failing to write it down only costs a stale report */
+    }
     return true;
   } catch {
     return false;
@@ -444,6 +596,7 @@ export async function resetSession(udid) {
 }
 
 export async function pressButton(udid, name) {
+  await ensureFreshSession(udid);
   const own = inputDriverFor(udid);
   if (own) {
     // Android's whole key vocabulary is safe to offer: `input keyevent` takes
@@ -464,6 +617,7 @@ export async function pressButton(udid, name) {
 }
 
 export async function swipe(udid, from, to, { durationMs = 300 } = {}) {
+  await ensureFreshSession(udid);
   const own = inputDriverFor(udid);
   if (own) {
     await own.swipe(udid, from, to, { durationMs });

package/src/intent.js

@@ -5,6 +5,7 @@
 // instinct removes the expensive part of driving a UI with an agent: the model
 // round trip spent reasoning about controls it has never seen.
 import * as input from './input.js';
+import * as metrics from './metrics.js';
 
 /** Confirming words, most specific first. The first tier with a hit wins. */
 const CONFIRM_TIERS = [
@@ -77,7 +78,11 @@ export function findOptions(nodes, geo) {
 export async function chooseAny(udid, { prefer, geo } = {}) {
   const nodes = await input.describeAll(udid);
   const options = findOptions(nodes, geo);
-  if (!options.length) throw new Error('no selectable options found on screen');
+  if (!options.length) {
+    throw metrics.tag(new Error('no selectable options found on screen'), 'novel_dialog', {
+      candidates: nodes.filter((n) => n.label).slice(0, 8).map((n) => ({ label: n.label })),
+    });
+  }
   const chosen =
     (prefer && options.find((o) => o.label.toLowerCase().includes(String(prefer).toLowerCase()))) ||
     options[0];
@@ -89,7 +94,11 @@ export async function chooseAny(udid, { prefer, geo } = {}) {
 export async function confirm(udid, { geo } = {}) {
   const nodes = await input.describeAll(udid);
   const target = findConfirm(nodes, geo);
-  if (!target) throw new Error('no confirming control on screen');
+  if (!target) {
+    throw metrics.tag(new Error('no confirming control on screen'), 'novel_dialog', {
+      candidates: nodes.filter((n) => n.label).slice(0, 8).map((n) => ({ label: n.label })),
+    });
+  }
   const point = input.centerOf(target);
   await input.tapPoint(udid, point.x, point.y);
   return { label: target.label, point, enabled: target.enabled };

package/src/matching.js

@@ -59,7 +59,25 @@ export function nameScore(name, query) {
   const q = norm(query);
   if (!n || !q) return 0;
   if (n === q) return 1;
-  if (n.startsWith(q) || q.startsWith(n)) return 0.86;
+  // A prefix match is only as good as the share it covers, and this branch had
+  // to be taught that twice.
+  //
+  // `n.startsWith(q)` — the name begins with the query, "Acce" for
+  // "Accessibility" — is the ordinary case and keeps most of its score: a
+  // prefix of a name is how people abbreviate. `q.startsWith(n)` is the
+  // opposite direction, where the *name* is a fragment of the query, and it
+  // returned the same flat 0.86 no matter how little of the query it was. So
+  // the section-index letter "S" scored 0.86 against the query "Search" and
+  // beat the search field's own label "Q Search" at 0.585 — and simframe
+  // tapped a scrubber and typed into it.
+  //
+  // The sibling branch below already carries this lesson in a comment about
+  // "back" matching a list row. Only one of the two had learned it. Both scale
+  // now, and the floor differs by direction on purpose: a query that is a
+  // prefix of a name is usually deliberate, while a name that is a fragment of
+  // the query is usually a coincidence, and one character is always one.
+  if (n.startsWith(q)) return 0.86 * Math.max(PREFIX_FLOOR, Math.min(1, q.length / n.length + 0.35));
+  if (q.startsWith(n)) return 0.8 * Math.max(0.15, n.length / q.length);
   // A substring match is only as good as the share of the name it covers.
   // Without this, "back" scores 0.78 against a two-hundred-character list row
   // that happens to contain "Back of House", and beats the actual back button.
@@ -76,7 +94,31 @@ export function nameScore(name, query) {
   if (distance > cap) return 0;
   const longest = Math.max(n.length, q.length);
   const similarity = 1 - distance / longest;
-  return similarity >= 0.7 ? similarity * 0.72 : 0;
+  if (similarity >= 0.7) return similarity * 0.72;
+  // Last tier: OCR read a confusable character.
+  //
+  // Language correction is deliberately off, which is right for labels and
+  // wrong for exactly this. Measured on a real app, `(All)` reads back as
+  // `(AII)` and would fail an assert against the string it is; capital-I,
+  // lowercase-l, the digit one and a pipe are one shape in most UI fonts, as
+  // are capital-O and zero.
+  //
+  // Deliberately the *last* tier and discounted, not part of `norm`. Folding
+  // in `norm` would make it change what an exact match means — "Log in" and
+  // "1og in" would become the same string everywhere — and identity is not
+  // something to be fuzzy about. Here it only ever rescues a comparison that
+  // had already scored zero.
+  const foldedScore = confusableFold(n) === confusableFold(q) ? 0.62 : 0;
+  return foldedScore;
+}
+
+/** One shape per glyph family, for comparison only. Never for identity. */
+export function confusableFold(s) {
+  return String(s ?? '')
+    .replace(/[il1|!]/gi, '1')
+    .replace(/[o0]/gi, '0')
+    .replace(/[s5]/gi, '5')
+    .replace(/[b8]/gi, '8');
 }
 
 function synonymGroup(query) {
@@ -159,6 +201,17 @@ export function rank(targets, intent, { screen } = {}) {
 export const AMBIGUITY_MARGIN = 0.08;
 /** Below this, no candidate is worth acting on. */
 export const MINIMUM_SCORE = 0.45;
+/**
+ * How much of a name's score survives scaling a prefix by its coverage.
+ *
+ * Tied to `MINIMUM_SCORE` rather than chosen: at 0.5 the shortest useful
+ * abbreviation — "Ac" for "Accessibility" — scored 0.433 and fell *below* the
+ * threshold to resolve at all, which would have turned a ranking fix into a
+ * feature removal. 0.6 puts the worst case at 0.516, comfortably resolvable and
+ * still well under a fuller match. The unit test asserts the relationship so
+ * the cliff cannot come back by someone tuning one of the two numbers.
+ */
+export const PREFIX_FLOOR = 0.6;
 
 /**
  * How close two tap points have to be to mean the same control.
@@ -182,6 +235,77 @@ export const SAME_CONTROL_POINTS = 12;
  */
 const INTERACTIVE_ROLE = /button|field|cell|row|link|switch|slider|tab|menu|segment|checkbox/i;
 
+/**
+ * Is this target the accessibility tree's reading of a control?
+ *
+ * A merged target carries `ax|ocr`, because both sensors saw it. Every
+ * comparison against the string `'ax'` had to become this: an element does not
+ * stop being the tree's element because OCR agreed with it, and treating
+ * `ax|ocr` as "not ax" would have quietly demoted exactly the elements the
+ * merge is most confident about.
+ */
+export const isAxTarget = (t) => /(^|\|)ax(\||$)/.test(t?.source ?? '');
+
+/** How much of `inner` lies inside `outer`, as a fraction of inner's own area. */
+export function containedFraction(inner, outer) {
+  if (!inner || !outer) return 0;
+  const iw = Math.max(0, inner.width ?? 0);
+  const ih = Math.max(0, inner.height ?? 0);
+  const innerArea = iw * ih;
+  if (innerArea <= 0) return 0;
+  const x = Math.max(inner.x, outer.x);
+  const y = Math.max(inner.y, outer.y);
+  const right = Math.min(inner.x + iw, outer.x + (outer.width ?? 0));
+  const bottom = Math.min(inner.y + ih, outer.y + (outer.height ?? 0));
+  const overlap = Math.max(0, right - x) * Math.max(0, bottom - y);
+  // Clamped: float arithmetic on sub-pixel OCR frames put a fully contained
+  // box at 1.0000000000000007, and a fraction of an area cannot exceed 1.
+  return Math.min(1, overlap / innerArea);
+}
+
+/**
+ * How much of the smaller box must sit inside the larger one to be the same
+ * element. OCR boxes sit a pixel or two outside the row they are printed on
+ * often enough that 1.0 would miss them.
+ */
+export const CONTAINMENT = 0.9;
+
+/**
+ * Do these two strings name the same thing?
+ *
+ * The discriminator that makes containment safe. A tab bar contains all five
+ * of its tab labels, and merging a container with its contents is the failure
+ * the old size cap was defending against — but a tab bar's own label is not
+ * "Assets", so the text test refuses that merge while allowing a row labelled
+ * "Kate Bell" to absorb OCR's reading of "Kate Bell".
+ *
+ * Substring counts because iOS labels carry state the printed text does not:
+ * a row reads "Larger Text" on screen and publishes "Larger Text, Off".
+ */
+export function sameText(a, b) {
+  const x = norm(a);
+  const y = norm(b);
+  if (!x || !y) return false;
+  if (x === y) return true;
+  if (x.includes(y) || y.includes(x)) return Math.min(x.length, y.length) >= 3;
+  // Fuzzy, because OCR misreads a letter or two — measured: "Location (AII)"
+  // for "Location (All)", and a Cyrillic К for a K in a monogram.
+  return nameScore(x, y) >= 0.5;
+}
+
+/**
+ * The same element, seen by both sensors.
+ *
+ * `ax` is the tree's element, `ocr` a text box. True when the text sits
+ * (almost) wholly inside the element AND says the same thing as its label or
+ * value.
+ */
+export function sameElementSeenTwice(ax, ocr) {
+  if (!ax?.frame || !ocr?.frame) return false;
+  if (containedFraction(ocr.frame, ax.frame) < CONTAINMENT) return false;
+  return sameText(ax.label, ocr.label ?? ocr.text) || sameText(ax.value, ocr.label ?? ocr.text);
+}
+
 const contains = (frame, target) =>
   Boolean(frame)
   && target.x >= frame.x && target.x <= frame.x + (frame.width ?? 0)
@@ -205,6 +329,14 @@ function sameControl(a, b) {
   // tab bar from absorbing its own tabs.
   if (INTERACTIVE_ROLE.test(a.type ?? '') && contains(a.frame, b)) return true;
   if (INTERACTIVE_ROLE.test(b.type ?? '') && contains(b.frame, a)) return true;
+  // And a labelled accessibility element that is not an interactive role —
+  // a list row published as StaticText — with OCR's reading of its own label
+  // inside it. This is the pair that made `tap "Kate Bell"` refuse on every
+  // Contacts list: 16 escalations in the first instrumented run, all one
+  // screen. Defence in depth: the screen map now merges this pair at fusion,
+  // and a map built before that still resolves.
+  if (isAxTarget(a) && !isAxTarget(b) && sameElementSeenTwice(a, b)) return true;
+  if (isAxTarget(b) && !isAxTarget(a) && sameElementSeenTwice(b, a)) return true;
   return false;
 }
 
@@ -219,8 +351,8 @@ function collapseSamePlace(ranked) {
     // Prefer the real hit target: an accessibility element over OCR's reading of
     // it, and an interactive role over a caption sitting inside it.
     const better = (candidate, incumbent) => {
-      if (candidate.target.source === 'ax' && incumbent.target.source !== 'ax') return true;
-      if (candidate.target.source !== 'ax' && incumbent.target.source === 'ax') return false;
+      if (isAxTarget(candidate.target) && !isAxTarget(incumbent.target)) return true;
+      if (!isAxTarget(candidate.target) && isAxTarget(incumbent.target)) return false;
       return INTERACTIVE_ROLE.test(candidate.target.type ?? '')
         && !INTERACTIVE_ROLE.test(incumbent.target.type ?? '');
     };

package/src/mcp.js

@@ -480,7 +480,7 @@ async function look(target, args, options) {
 async function state(target, args, options) {
   const { device } = await api.ensureDaemon(target, options);
   const requested = baselineFor(device.udid, args.since);
-  const res = await api.getState(target, { since: requested, options });
+  const res = await api.getState(target, { since: requested, options, inputHealth: true });
   const s = res.state;
   const lines = [
     header(res.device, s, res.ageMs),
@@ -500,6 +500,19 @@ async function state(target, args, options) {
   }
   const warn = livenessLine(res.live);
   if (warn) lines.unshift(warn);
+  // An agent that cannot see this spends five turns wondering why a correct
+  // tap on a correct element did nothing. The repair happens before the next
+  // action either way; this is so the cause is visible when it does.
+  if (res.input?.stale) lines.unshift(`input: stale — ${res.input.reason}`);
+  // §7's timing line. Worth a line because "is it still coming or is it done"
+  // is a question an agent otherwise answers by waiting and guessing.
+  if (res.timing?.samples) {
+    lines.push(
+      `timing: usually ${res.timing.edge_p50}ms to arrive here (p95 ${res.timing.edge_p95}ms over `
+      + `${res.timing.samples} samples), ${res.timing.elapsed_ms}ms since the last change`
+      + (res.timing.slower_than_usual ? ` — ${res.timing.note}` : ''),
+    );
+  }
   remember(device.udid, s);
   return { content: [text(lines.filter(Boolean).join('\n'))] };
 }