simframe 0.9.0 → 0.11.0

package/src/input.js

@@ -248,6 +248,11 @@ export function elementToNode(e) {
     type: e.role ?? null,
     identifier: e.identifier ?? null,
     enabled: e.state?.enabled ?? null,
+    // The daemon batches AXSelected and AXFocused alongside AXEnabled and has
+    // since 0.6.0. This converter took one of the three, and it is the one on
+    // the path that actually runs — `normalizeNode` below is the idb fallback.
+    selected: e.state?.selected ?? null,
+    focused: e.state?.focused ?? null,
     frame: e.frame ?? null,
     raw: e,
   };
@@ -273,6 +278,12 @@ function normalizeNode(node) {
     type: node.type ?? node.AXType ?? null,
     identifier: node.AXUniqueId ?? node.identifier ?? null,
     enabled: node.AXEnabled ?? node.enabled ?? null,
+    // The daemon has asked the tree for AXSelected and AXFocused since 0.6.0 —
+    // they are two of the eight attributes in its batched round trip — and this
+    // function dropped both. `view.renderRow` has printed `selected` for as
+    // long as it has existed, against a field nobody set.
+    selected: node.AXSelected ?? node.selected ?? null,
+    focused: node.AXFocused ?? node.focused ?? null,
     frame: frame
       ? {
           x: frame.x ?? frame.X ?? 0,
@@ -418,14 +429,77 @@ export async function typeKeys(udid, value) {
   await idb(['ui', 'text', '--udid', udid, String(value)]);
 }
 
-export async function pressKey(udid, keycode) {
+/**
+ * Keyboard keys, by name.
+ *
+ * A peer was blocked outright for want of Return: half of mobile search fields
+ * submit on the keyboard return key, `button` covers only the hardware buttons,
+ * and `key` wanted a raw HID usage code that nobody should have to know. Typing
+ * "\n" as text is not a substitute — text goes through whatever keyboard layout
+ * iOS has active, and measured, it turned "Coke Display" into "Coke In Display".
+ *
+ * These are HID keyboard usage codes, which name a key *position* and are never
+ * translated by a layout. That property is the whole reason this path exists on
+ * a device whose own doctor warns that two extra layouts are installed.
+ */
+export const KEYS = {
+  return: 40, enter: 40, escape: 41, esc: 41, backspace: 42, delete: 42,
+  tab: 43, space: 44, up: 82, down: 81, left: 80, right: 79,
+  a: 4,
+};
+
+/** Modifier usage codes, held while another key is pressed. */
+export const MODIFIERS = { control: 224, shift: 225, alt: 226, option: 226, command: 227, cmd: 227, gui: 227 };
+
+/** The usage code for a name, a number, or null when it is neither. */
+export function keyUsage(key) {
+  if (Number.isFinite(Number(key))) return Number(key);
+  const name = String(key ?? '').trim().toLowerCase();
+  return Object.hasOwn(KEYS, name) ? KEYS[name] : null;
+}
+
+export async function pressKey(udid, keycode, { modifiers = [] } = {}) {
   await ensureFreshSession(udid);
+  const usage = keyUsage(keycode);
+  const held = modifiers
+    .map((m) => (Number.isFinite(Number(m)) ? Number(m) : MODIFIERS[String(m).trim().toLowerCase()]))
+    .filter((m) => Number.isFinite(m));
+  if (usage == null) {
+    throw new Error(`unknown key ${JSON.stringify(String(keycode))} — known names: ${Object.keys(KEYS).join(', ')}`
+      + ', or a HID usage code');
+  }
   const own = inputDriverFor(udid);
   if (own) {
-    await own.key(udid, keycode);
+    await own.key(udid, usage, held);
+    return;
+  }
+  // The daemon owns the keyboard usage path on iOS. It was implemented in the
+  // HID layer and never exposed as a verb, so this fell through to idb — which
+  // is absent on a machine using the daemon, and so there was no way to press a
+  // keyboard key at all.
+  if (control.available(udid)) {
+    await control.key(udid, usage, held);
     return;
   }
-  await idb(['ui', 'key', '--udid', udid, String(keycode)]);
+  if (held.length) throw new Error('modifier keys need the daemon; idb cannot hold one');
+  await idb(['ui', 'key', '--udid', udid, String(usage)]);
+}
+
+/**
+ * Empty the focused field.
+ *
+ * Command-A then Delete, over HID. There is no clear primitive anywhere —
+ * XCUITest, Appium and idb all lack one, and re-typing appends — so this is the
+ * standard answer rather than a trick of ours. It is layout-independent for the
+ * reason that matters on a device with Farsi and Armenian keyboards installed:
+ * a modifier and Delete are key *positions*, and so is the `a` in Command-A, so
+ * none of the three is translated by the active layout.
+ *
+ * It clears whatever has focus, which is why every caller focuses first.
+ */
+export async function clearField(udid) {
+  await pressKey(udid, 'a', { modifiers: ['command'] });
+  await pressKey(udid, 'delete');
 }
 
 /**
@@ -506,19 +580,52 @@ export async function sessionHealth(udid) {
 }
 
 /**
- * Rebuild the session if the device outlived it. Once per process, per device.
+ * Should this staleness be acted on, given what has already been rebuilt?
+ *
+ * Pure, and separate from the check because the *key* is the whole bug. This
+ * gate used to be a set of udids — "once per process, per device" — and the
+ * reasoning was to avoid statting on every action. What it actually bought was
+ * that the feature could not fire in the one process that matters. A CLI
+ * command is a new process every time, so per-process is per-call there and
+ * the gate never showed; the MCP server is a single process that lives for a
+ * whole session, so it checked once, at the first action, and then never
+ * again — and a device that reboots *mid-session* is precisely the case this
+ * exists to catch. Reported from a real session: capture kept working, input
+ * died, every tap returned `ok`, and about ten calls went into two wrong
+ * conclusions about the app.
+ *
+ * The right key is the boot the rebuild was for. One attempt per device boot:
+ * enough that a failed rebuild does not retry on every tap forever, and not so
+ * much that the next boot is invisible.
+ */
+export function shouldRebuildSession({ stale, bootedAt }, rebuiltFor) {
+  if (!stale) return false;
+  // A boot we cannot date cannot be memoised against, and re-attempting on
+  // every action would be worse than not detecting it. sessionStaleness only
+  // reports stale with a finite bootedAt, so this is a belt, not a case.
+  if (!Number.isFinite(bootedAt)) return false;
+  return rebuiltFor !== bootedAt;
+}
+
+/**
+ * Rebuild the session if the device outlived it. Once per device boot.
  *
  * Rebuild, and retry nothing: this runs *before* the action, so the action is
  * delivered on a session known to be current. Retrying afterwards is how an
  * action fires twice, which is the hazard the verify barrier exists to
  * prevent — and it is why the existing recovery covers hardware buttons only.
+ *
+ * The check now runs on every dispatch rather than once. It costs two small
+ * `readJson`s and, at most every three seconds, one stat — `bootedAtCached`
+ * already caps the part that was expensive, which is what made the
+ * once-per-process gate unnecessary as well as wrong.
  */
-const freshened = new Set();
+const rebuiltForBoot = new Map();
 export async function ensureFreshSession(udid) {
-  if (!udid || freshened.has(udid)) return null;
-  freshened.add(udid);
+  if (!udid) return null;
   const health = await sessionHealth(udid);
-  if (!health.stale) return null;
+  if (!shouldRebuildSession(health, rebuiltForBoot.get(udid))) return null;
+  rebuiltForBoot.set(udid, health.bootedAt);
   const rebuilt = await resetSession(udid);
   return { ...health, rebuilt };
 }

package/src/localhelper.js

@@ -0,0 +1,155 @@
+/**
+ * A warm, line-oriented local helper process.
+ *
+ * Extracted rather than duplicated, because writing this twice would mean
+ * risking the same two bugs twice — and both were subtle enough to look like
+ * something else entirely.
+ *
+ * A timed-out request left its waiter in the queue, so every later answer went
+ * to the wrong asker and the run simply never finished; it read as the model
+ * being slow. And unreferencing the child's stdout unreferenced the pipe every
+ * request waits on, so the process exited silently in the middle of an await
+ * and printed nothing at all, returning 0.
+ *
+ * The helper is kept warm because the first answer in a process pays model load
+ * — measured at ~880ms against ~560ms for every answer after it.
+ */
+import { spawn, execFile } from 'node:child_process';
+import fs from 'node:fs';
+import path from 'node:path';
+import { promisify } from 'node:util';
+
+const run = promisify(execFile);
+
+/**
+ * Compile a Swift source once and reuse the binary.
+ *
+ * `swiftc`, not `xcrun swiftc`: nothing above the platform boundary may name a
+ * platform tool, and the boundary test catches it. A compiler is not a device
+ * tool, which is the precedent `src/ocr.js` set.
+ */
+export function compiler({ source, binary, what }) {
+  let building = null;
+  return async function ensureBinary() {
+    if (building) return building;
+    building = (async () => {
+      try {
+        const src = fs.statSync(source).mtimeMs;
+        const bin = fs.existsSync(binary) ? fs.statSync(binary).mtimeMs : 0;
+        if (bin > src) return { available: true, binary };
+      } catch {
+        return { available: false, reason: `the ${what} source is missing from this install` };
+      }
+      try {
+        fs.mkdirSync(path.dirname(binary), { recursive: true });
+        await run('swiftc', ['-O', source, '-o', binary], { timeout: 180_000 });
+        return { available: true, binary };
+      } catch (err) {
+        building = null; // let a later call retry once a toolchain is present
+        return {
+          available: false,
+          reason: err.code === 'ENOENT'
+            ? `swiftc is not installed, so the ${what} cannot be built (install Xcode command line tools)`
+            : `could not build the ${what}: ${String(err.message).split('\n')[0]}`,
+        };
+      }
+    })();
+    return building;
+  };
+}
+
+/**
+ * Open a helper and speak JSON lines to it.
+ *
+ * @returns {{ask: (o: object, ms: number) => Promise<object|null>, close: () => void, ok: boolean, reason?: string}}
+ */
+export function lineServer({ ensureBinary, what }) {
+  let session = null;
+
+  async function open() {
+    if (session) return session;
+    const built = await ensureBinary();
+    if (!built.available) return { ok: false, reason: built.reason };
+    session = await new Promise((resolve) => {
+      const child = spawn(built.binary, [], { stdio: ['pipe', 'pipe', 'ignore'] });
+      // Deliberately NOT unref'd — see the note at the top of this file.
+      let buffer = '';
+      const waiters = [];
+      let settled = false;
+      const fail = (reason) => {
+        if (!settled) { settled = true; resolve({ ok: false, reason }); }
+        while (waiters.length) waiters.shift()(null);
+      };
+      child.on('error', (err) => fail(`the ${what} would not start: ${err.message}`));
+      child.on('exit', () => { session = null; fail(`the ${what} exited`); });
+      child.stdout.on('data', (chunk) => {
+        buffer += chunk;
+        let i = buffer.indexOf('\n');
+        while (i >= 0) {
+          const line = buffer.slice(0, i).trim();
+          buffer = buffer.slice(i + 1);
+          i = buffer.indexOf('\n');
+          if (!line) continue;
+          let msg;
+          try { msg = JSON.parse(line); } catch { continue; }
+          if (!settled) {
+            settled = true;
+            if (msg.ready) resolve({ ok: true, child, waiters });
+            else resolve({ ok: false, reason: msg.unavailable ?? `the ${what} did not become ready` });
+            continue;
+          }
+          const next = waiters.shift();
+          if (next) next(msg);
+        }
+      });
+    });
+    return session;
+  }
+
+  return {
+    async ask(question, timeoutMs = 3000) {
+      let live;
+      try {
+        live = await open();
+      } catch {
+        return null;
+      }
+      if (!live?.ok) return null;
+      return new Promise((resolve) => {
+        // A timed-out waiter is retired, not merely resolved. Leaving it queued
+        // sent the next answer to it instead of to the next asker, and every
+        // call after that was off by one.
+        let done = false;
+        const waiter = (msg) => {
+          if (done) return;
+          done = true;
+          clearTimeout(timer);
+          resolve(msg);
+        };
+        const timer = setTimeout(() => {
+          if (done) return;
+          done = true;
+          const i = live.waiters.indexOf(waiter);
+          if (i >= 0) live.waiters.splice(i, 1);
+          resolve(null);
+        }, timeoutMs);
+        live.waiters.push(waiter);
+        try {
+          live.child.stdin.write(`${JSON.stringify(question)}\n`);
+        } catch {
+          clearTimeout(timer);
+          done = true;
+          resolve(null);
+        }
+      });
+    },
+    async status() {
+      const live = await open();
+      return live?.ok ? { ok: true } : { ok: false, reason: live?.reason ?? 'unavailable' };
+    },
+    close() {
+      try { session?.child?.kill(); } catch { /* already gone */ }
+      session = null;
+    },
+  };
+}

package/src/matching.js

@@ -59,7 +59,25 @@ export function nameScore(name, query) {
   const q = norm(query);
   if (!n || !q) return 0;
   if (n === q) return 1;
-  if (n.startsWith(q) || q.startsWith(n)) return 0.86;
+  // A prefix match is only as good as the share it covers, and this branch had
+  // to be taught that twice.
+  //
+  // `n.startsWith(q)` — the name begins with the query, "Acce" for
+  // "Accessibility" — is the ordinary case and keeps most of its score: a
+  // prefix of a name is how people abbreviate. `q.startsWith(n)` is the
+  // opposite direction, where the *name* is a fragment of the query, and it
+  // returned the same flat 0.86 no matter how little of the query it was. So
+  // the section-index letter "S" scored 0.86 against the query "Search" and
+  // beat the search field's own label "Q Search" at 0.585 — and simframe
+  // tapped a scrubber and typed into it.
+  //
+  // The sibling branch below already carries this lesson in a comment about
+  // "back" matching a list row. Only one of the two had learned it. Both scale
+  // now, and the floor differs by direction on purpose: a query that is a
+  // prefix of a name is usually deliberate, while a name that is a fragment of
+  // the query is usually a coincidence, and one character is always one.
+  if (n.startsWith(q)) return 0.86 * Math.max(PREFIX_FLOOR, Math.min(1, q.length / n.length + 0.35));
+  if (q.startsWith(n)) return 0.8 * Math.max(0.15, n.length / q.length);
   // A substring match is only as good as the share of the name it covers.
   // Without this, "back" scores 0.78 against a two-hundred-character list row
   // that happens to contain "Back of House", and beats the actual back button.
@@ -76,7 +94,31 @@ export function nameScore(name, query) {
   if (distance > cap) return 0;
   const longest = Math.max(n.length, q.length);
   const similarity = 1 - distance / longest;
-  return similarity >= 0.7 ? similarity * 0.72 : 0;
+  if (similarity >= 0.7) return similarity * 0.72;
+  // Last tier: OCR read a confusable character.
+  //
+  // Language correction is deliberately off, which is right for labels and
+  // wrong for exactly this. Measured on a real app, `(All)` reads back as
+  // `(AII)` and would fail an assert against the string it is; capital-I,
+  // lowercase-l, the digit one and a pipe are one shape in most UI fonts, as
+  // are capital-O and zero.
+  //
+  // Deliberately the *last* tier and discounted, not part of `norm`. Folding
+  // in `norm` would make it change what an exact match means — "Log in" and
+  // "1og in" would become the same string everywhere — and identity is not
+  // something to be fuzzy about. Here it only ever rescues a comparison that
+  // had already scored zero.
+  const foldedScore = confusableFold(n) === confusableFold(q) ? 0.62 : 0;
+  return foldedScore;
+}
+
+/** One shape per glyph family, for comparison only. Never for identity. */
+export function confusableFold(s) {
+  return String(s ?? '')
+    .replace(/[il1|!]/gi, '1')
+    .replace(/[o0]/gi, '0')
+    .replace(/[s5]/gi, '5')
+    .replace(/[b8]/gi, '8');
 }
 
 function synonymGroup(query) {
@@ -159,6 +201,17 @@ export function rank(targets, intent, { screen } = {}) {
 export const AMBIGUITY_MARGIN = 0.08;
 /** Below this, no candidate is worth acting on. */
 export const MINIMUM_SCORE = 0.45;
+/**
+ * How much of a name's score survives scaling a prefix by its coverage.
+ *
+ * Tied to `MINIMUM_SCORE` rather than chosen: at 0.5 the shortest useful
+ * abbreviation — "Ac" for "Accessibility" — scored 0.433 and fell *below* the
+ * threshold to resolve at all, which would have turned a ranking fix into a
+ * feature removal. 0.6 puts the worst case at 0.516, comfortably resolvable and
+ * still well under a fuller match. The unit test asserts the relationship so
+ * the cliff cannot come back by someone tuning one of the two numbers.
+ */
+export const PREFIX_FLOOR = 0.6;
 
 /**
  * How close two tap points have to be to mean the same control.
@@ -284,9 +337,67 @@ function sameControl(a, b) {
   // and a map built before that still resolves.
   if (isAxTarget(a) && !isAxTarget(b) && sameElementSeenTwice(a, b)) return true;
   if (isAxTarget(b) && !isAxTarget(a) && sameElementSeenTwice(b, a)) return true;
+  // And a caption sitting on the control it names. Containment above requires
+  // the container to be a recognised hit target, and a React Native composite
+  // is a generic element — so a form row published its caption and its select
+  // with the same label, 152 points apart, and nothing merged them. They are
+  // one control seen twice, not two candidates: answering `ambiguous` here
+  // costs a round trip to choose between a thing and its own name.
+  if (isCaptionFor(a, b) || isCaptionFor(b, a)) return true;
   return false;
 }
 
+/** Does `caption` merely name `control`, overlapping it, with the same label? */
+function isCaptionFor(caption, control) {
+  if (!namesOnly(caption) || namesOnly(control)) return false;
+  if (norm(caption.label) !== norm(control.label)) return false;
+  return overlaps(caption.frame, control.frame);
+}
+
+/** Any shared area at all — a caption's box often clears its control's by a point or two. */
+function overlaps(a, b) {
+  if (!a || !b) return false;
+  return a.x < b.x + b.width && b.x < a.x + a.width
+    && a.y < b.y + b.height && b.y < a.y + a.height;
+}
+
+/**
+ * Roles that only ever *name* a control, never are one.
+ *
+ * The tree publishes a caption and the control it captions with the same label,
+ * and a React Native composite (a `native-base` Select, say) surfaces as a
+ * generic element that `INTERACTIVE_ROLE` does not recognise. So `tap "Problem"`
+ * resolved to the caption at (49,486) and did nothing, while the select sat at
+ * (201,496) — reported as half of the single largest cost in a real-app run,
+ * because it means neither selector can be trusted.
+ *
+ * `collapseSamePlace` already prefers a hit target over the text printed on it,
+ * but only once `sameControl` has decided they are the same place. These two
+ * were 152 points apart with a generic role, so nothing merged them.
+ */
+const NAMING_ONLY_ROLE = /^(statictext|text|label|heading|image)$/i;
+
+const namesOnly = (t) => NAMING_ONLY_ROLE.test(t?.type ?? '');
+
+/**
+ * A caption never wins over a control that answers the same name.
+ *
+ * Applied only when the two are close enough in score to be answering the same
+ * question — a heading is still reachable when nothing else matches, which is
+ * why this promotes rather than filters. Anything further apart than
+ * `CAPTION_MARGIN` is a different match, not the same match seen twice.
+ */
+export const CAPTION_MARGIN = 0.2;
+
+export function preferTheControl(ranked) {
+  if (!ranked.length || !namesOnly(ranked[0].target)) return ranked;
+  const lead = ranked[0].score;
+  const i = ranked.findIndex((c, idx) => idx > 0 && !namesOnly(c.target) && lead - c.score <= CAPTION_MARGIN);
+  if (i < 0) return ranked;
+  const promoted = { ...ranked[i], reasons: [...ranked[i].reasons, 'the control, not the caption naming it'] };
+  return [promoted, ...ranked.filter((_, idx) => idx !== i)];
+}
+
 function collapseSamePlace(ranked) {
   const kept = [];
   for (const c of ranked) {
@@ -298,6 +409,11 @@ function collapseSamePlace(ranked) {
     // Prefer the real hit target: an accessibility element over OCR's reading of
     // it, and an interactive role over a caption sitting inside it.
     const better = (candidate, incumbent) => {
+      // A caption loses to what it names before anything else is considered:
+      // a `StaticText` is never the tap target when the control it labels is
+      // right there, whatever either one's source.
+      if (namesOnly(incumbent.target) && !namesOnly(candidate.target)) return true;
+      if (namesOnly(candidate.target) && !namesOnly(incumbent.target)) return false;
       if (isAxTarget(candidate.target) && !isAxTarget(incumbent.target)) return true;
       if (!isAxTarget(candidate.target) && isAxTarget(incumbent.target)) return false;
       return INTERACTIVE_ROLE.test(candidate.target.type ?? '')
@@ -315,7 +431,7 @@ function collapseSamePlace(ranked) {
  * @returns {{status: 'ok'|'ambiguous'|'none', target?, score?, reasons?, alternatives?}}
  */
 export function resolve(targets, intent, options = {}) {
-  const ranked = collapseSamePlace(rank(targets, intent, options).filter((c) => c.score >= MINIMUM_SCORE));
+  const ranked = preferTheControl(collapseSamePlace(rank(targets, intent, options).filter((c) => c.score >= MINIMUM_SCORE)));
   if (!ranked.length) return { status: 'none', alternatives: [] };
   const [best, second] = ranked;
   if (second && best.score - second.score < AMBIGUITY_MARGIN) {