simframe 0.4.2 → 0.6.0-rc.1

package/src/input.js

@@ -2,6 +2,7 @@
 // capability layered on top, so every entry point here has to answer "is this
 // even available?" before it answers anything else.
 import { execFile } from 'node:child_process';
+import * as control from './control.js';
 import { promisify } from 'node:util';
 
 const run = promisify(execFile);
@@ -11,6 +12,77 @@ const IDB_HINT =
 
 let driverCache = null;
 
+/**
+ * Which input driver to use for a device.
+ *
+ * simframed is preferred when its control socket is live: it needs no install,
+ * speaks points natively, and is the path that survives idb breaking on a new
+ * iOS. idb remains the fallback so a machine without the daemon still works.
+ */
+export async function driverFor(udid) {
+  if (udid && control.available(udid)) {
+    try {
+      const status = await control.status(udid);
+      if (status.input?.available) {
+        return { name: 'simframed', available: true, version: status.input.detail, reason: null, viaSocket: true };
+      }
+      return { name: 'simframed', available: false, version: null, reason: status.input?.detail ?? 'input unavailable', viaSocket: true };
+    } catch {
+      /* daemon went away mid-call; fall through to idb */
+    }
+  }
+  return detectDriver();
+}
+
+/**
+ * An escape hatch back to idb for the tree.
+ *
+ * Every private-framework path here is version-coupled, and the host-side
+ * translator is no exception: an Xcode upgrade could break it on a machine
+ * where work still has to happen that day. Reading it per call rather than
+ * caching means the switch takes effect without restarting anything.
+ */
+function preferIdbTree() {
+  return process.env.SIMFRAME_AX_DRIVER === 'idb';
+}
+
+/**
+ * Which driver reads the accessibility tree for a device.
+ *
+ * Separate from `driverFor` because these are separate capabilities: a device
+ * can be perfectly touchable by the daemon while the translation framework is
+ * missing, and reporting one number for both hides which layer is down.
+ */
+export async function axDriverFor(udid) {
+  if (udid && control.available(udid) && !preferIdbTree()) {
+    try {
+      const status = await control.status(udid);
+      if (status.accessibility?.available) {
+        return { name: 'simframed', available: true, chosen: false, version: status.accessibility.detail, reason: null };
+      }
+      // The daemon is up and says it cannot read the tree. idb might still,
+      // so this is a reason to fall through rather than an answer.
+    } catch {
+      /* daemon went away mid-call; fall through to idb */
+    }
+  }
+  const idbDriver = await detectDriver();
+  // Asked for, or fallen back to? A driver someone chose is not a degradation,
+  // and grading it as one turns the documented escape hatch into a red CI run
+  // on exactly the day an Xcode upgrade makes you reach for it.
+  const chosen = preferIdbTree();
+  if (idbDriver.available) {
+    return {
+      name: 'idb',
+      available: true,
+      chosen,
+      version: chosen ? `${idbDriver.version} — selected by SIMFRAME_AX_DRIVER` : idbDriver.version,
+      reason: null,
+    };
+  }
+  return { name: null, available: false, chosen, version: null, reason: idbDriver.reason };
+}
+
 /** @returns {Promise<{name: string, available: boolean, version: string|null, reason: string|null}>} */
 export async function detectDriver({ refresh = false } = {}) {
   if (driverCache && !refresh) return driverCache;
@@ -68,6 +140,28 @@ export async function screenInfo(udid, { refresh = false } = {}) {
 }
 
 async function readScreenInfo(udid) {
+  // Ask the daemon first. It holds the device's own point size and scale, which
+  // makes it both authoritative and free — and it means geometry no longer
+  // needs idb at all. Going to idb first meant a machine without idb could
+  // capture and tap perfectly well but could not run a verified flow, because
+  // building a screen map needs the point size.
+  if (control.available(udid)) {
+    try {
+      const { device } = await control.status(udid);
+      if (device?.pointWidth && device?.pointHeight) {
+        const density = device.scale ?? 1;
+        return {
+          pixelWidth: Math.round(device.pointWidth * density),
+          pixelHeight: Math.round(device.pointHeight * density),
+          density,
+          pointWidth: device.pointWidth,
+          pointHeight: device.pointHeight,
+        };
+      }
+    } catch {
+      /* daemon went away mid-call; fall through to idb */
+    }
+  }
   const out = await idb(['describe', '--json', '--udid', udid]);
   const info = JSON.parse(out.trim().split('\n').filter(Boolean).pop());
   const dims = info.screen_dimensions || {};
@@ -83,6 +177,20 @@ async function readScreenInfo(udid) {
 
 /** The accessibility tree, flattened. This is what makes tap-by-label possible. */
 export async function describeAll(udid) {
+  // The daemon reads the tree host-side through AXPTranslator: no install, and
+  // measured at 45ms against idb's 203ms on the same screen. idb stays as the
+  // fallback, so a machine without the daemon still reads.
+  if (control.available(udid) && !preferIdbTree()) {
+    try {
+      const { screen } = await control.request(udid, { action: 'ui', ocr: false });
+      // An app mid-launch genuinely has no tree yet. Falling through to idb
+      // here would just ask a second time and report the same emptiness more
+      // slowly, so the honest answer is the empty one.
+      if (screen?.sources?.includes('ax')) return (screen.elements ?? []).map(elementToNode);
+    } catch {
+      /* daemon went away mid-call; fall through to idb */
+    }
+  }
   // Passing --json here yields empty output; the default already emits JSON.
   const out = await idb(['ui', 'describe-all', '--udid', udid]);
   const nodes = [];
@@ -104,6 +212,20 @@ export async function describeAll(udid) {
   return nodes.filter((n) => n.frame);
 }
 
+/** A daemon element back into the node shape every caller here expects. */
+export function elementToNode(e) {
+  return {
+    label: cleanLabel(e.label),
+    rawLabel: e.label ?? null,
+    value: e.value ?? null,
+    type: e.role ?? null,
+    identifier: e.identifier ?? null,
+    enabled: e.state?.enabled ?? null,
+    frame: e.frame ?? null,
+    raw: e,
+  };
+}
+
 // Icon fonts put glyphs in the Unicode private use areas, so a label arrives as
 // "<glyph>, My Tools". Matching has to see through that to the readable text.
 const PRIVATE_USE = /[\u{E000}-\u{F8FF}\u{F0000}-\u{FFFFD}\u{100000}-\u{10FFFD}]/gu;
@@ -177,10 +299,15 @@ export function centerOf(node) {
 }
 
 export async function tapPoint(udid, x, y, { durationMs } = {}) {
-  const args = ['ui', 'tap', '--udid', udid, String(Math.round(x)), String(Math.round(y))];
+  const point = { x: Math.round(x), y: Math.round(y) };
+  if (control.available(udid)) {
+    await control.tap(udid, point.x, point.y, durationMs ? { durationMs } : {});
+    return point;
+  }
+  const args = ['ui', 'tap', '--udid', udid, String(point.x), String(point.y)];
   if (durationMs) args.push('--duration', String(durationMs / 1000));
   await idb(args);
-  return { x: Math.round(x), y: Math.round(y) };
+  return point;
 }
 
 export async function tapLabel(udid, query, { index, durationMs } = {}) {
@@ -191,6 +318,21 @@ export async function tapLabel(udid, query, { index, durationMs } = {}) {
 }
 
 export async function typeText(udid, value) {
+  if (control.available(udid)) {
+    // The daemon's paste path carries characters rather than key positions, so
+    // it is not reinterpreted by the device's keyboard layout.
+    await control.paste(udid, String(value));
+    return;
+  }
+  await idb(['ui', 'text', '--udid', udid, String(value)]);
+}
+
+/** Key events rather than text: for shortcuts and search-as-you-type. */
+export async function typeKeys(udid, value) {
+  if (control.available(udid)) {
+    await control.type(udid, String(value));
+    return;
+  }
   await idb(['ui', 'text', '--udid', udid, String(value)]);
 }
 
@@ -198,11 +340,46 @@ export async function pressKey(udid, keycode) {
   await idb(['ui', 'key', '--udid', udid, String(keycode)]);
 }
 
+/**
+ * Rebuild the daemon's HID session.
+ *
+ * Input is the one path with no feedback: a dispatched Indigo message reports
+ * success when the send succeeds, and nothing asks the device whether anything
+ * happened. Measured on a long-running daemon, a HOME press returned in 66ms
+ * and the screen did not move; the same press on a freshly started daemon
+ * worked. Whoever holds the frames is the only one who can notice, which is why
+ * this is something callers invoke rather than something input does for itself.
+ *
+ * @returns {Promise<boolean>} whether a session was actually reset.
+ */
+export async function resetSession(udid) {
+  if (!control.available(udid)) return false;
+  try {
+    await control.resetInput(udid);
+    return true;
+  } catch {
+    return false;
+  }
+}
+
 export async function pressButton(udid, name) {
+  if (control.available(udid)) {
+    try {
+      await control.press(udid, String(name).toLowerCase());
+      return;
+    } catch (err) {
+      // Only home is verified through Indigo; anything else falls back.
+      if (!(await detectDriver()).available) throw err;
+    }
+  }
   await idb(['ui', 'button', '--udid', udid, String(name).toUpperCase()]);
 }
 
 export async function swipe(udid, from, to, { durationMs = 300 } = {}) {
+  if (control.available(udid)) {
+    await control.swipe(udid, from, to, { durationMs });
+    return;
+  }
   await idb([
     'ui', 'swipe', '--udid', udid,
     String(Math.round(from.x)), String(Math.round(from.y)),

package/src/matching.js

@@ -0,0 +1,265 @@
+// Resolving "what did they mean" against what is on screen.
+//
+// The rule throughout: never guess between two plausible answers. A wrong tap
+// is worse than a question, because a wrong tap can do something and the caller
+// will believe it did the right thing.
+
+/** Names for controls that carry an icon and no readable label. */
+const SYNONYMS = {
+  back: ['back', 'chevron.left', 'navigate back', 'previous', 'return'],
+  close: ['close', 'dismiss', 'xmark', 'cancel', 'done'],
+  search: ['search', 'find', 'magnifyingglass'],
+  add: ['add', 'new', 'create', 'plus', 'compose'],
+  more: ['more', 'options', 'ellipsis', 'overflow', 'menu'],
+  settings: ['settings', 'preferences', 'gear', 'configure'],
+  share: ['share', 'export', 'send'],
+  delete: ['delete', 'remove', 'trash', 'bin'],
+  edit: ['edit', 'modify', 'change'],
+  save: ['save', 'apply', 'confirm', 'ok', 'submit'],
+};
+
+/** Words that say what kind of control the caller means. */
+const ROLE_HINTS = [
+  { pattern: /\b(type|enter|fill|input)\b/i, roles: /field|textfield|textview|searchfield/i },
+  { pattern: /\b(tap|press|click|hit)\b/i, roles: /button|link|cell|tab/i },
+  { pattern: /\b(toggle|switch|enable|disable|turn)\b/i, roles: /switch|toggle|checkbox/i },
+  { pattern: /\b(tab)\b/i, roles: /tab/i },
+];
+
+/** Words that say where on screen the caller means. */
+const REGION_HINTS = [
+  { pattern: /\btab\b/i, region: 'tab-bar' },
+  { pattern: /\b(back|nav|title|toolbar)\b/i, region: 'nav-bar' },
+];
+
+const norm = (s) => String(s ?? '').toLowerCase().replace(/\s+/g, ' ').trim();
+
+/** Levenshtein distance, capped: beyond the cap the exact value is irrelevant. */
+export function editDistance(a, b, cap = 8) {
+  if (a === b) return 0;
+  if (Math.abs(a.length - b.length) > cap) return cap + 1;
+  let prev = Array.from({ length: b.length + 1 }, (_, i) => i);
+  for (let i = 1; i <= a.length; i++) {
+    const row = [i];
+    let best = i;
+    for (let j = 1; j <= b.length; j++) {
+      const cost = a[i - 1] === b[j - 1] ? 0 : 1;
+      row[j] = Math.min(prev[j] + 1, row[j - 1] + 1, prev[j - 1] + cost);
+      if (row[j] < best) best = row[j];
+    }
+    if (best > cap) return cap + 1;
+    prev = row;
+  }
+  return prev[b.length];
+}
+
+/** How well one name answers to a query, 0 to 1. */
+export function nameScore(name, query) {
+  const n = norm(name);
+  const q = norm(query);
+  if (!n || !q) return 0;
+  if (n === q) return 1;
+  if (n.startsWith(q) || q.startsWith(n)) return 0.86;
+  // A substring match is only as good as the share of the name it covers.
+  // Without this, "back" scores 0.78 against a two-hundred-character list row
+  // that happens to contain "Back of House", and beats the actual back button.
+  if (n.includes(q)) return 0.78 * Math.max(0.15, q.length / n.length);
+  if (q.includes(n)) return 0.7 * Math.max(0.15, n.length / q.length);
+  // Fuzzy, so a typo or a stray plural still resolves.
+  //
+  // editDistance returns cap+1 as a sentinel when it gives up early. Treating
+  // that as a measurement made every long string score well: 1 - 9/200 is
+  // 0.955, so a two-hundred-character list row scored 0.687 against any query
+  // at all. A bail-out is "no answer", not "nearly identical".
+  const cap = 8;
+  const distance = editDistance(n, q, cap);
+  if (distance > cap) return 0;
+  const longest = Math.max(n.length, q.length);
+  const similarity = 1 - distance / longest;
+  return similarity >= 0.7 ? similarity * 0.72 : 0;
+}
+
+function synonymGroup(query) {
+  const q = norm(query);
+  for (const [key, words] of Object.entries(SYNONYMS)) {
+    if (words.some((w) => w === q || q.includes(w))) return { key, words };
+  }
+  return null;
+}
+
+/**
+ * Rank what is on screen against an intent.
+ *
+ * Returns candidates sorted best first, each with the reasons behind its score
+ * so a caller — or a person reading a failure — can see why.
+ */
+export function rank(targets, intent, { screen } = {}) {
+  // Never offer something that is not on screen. A scrolled-away row still sits
+  // in the map with a negative y, and tapping it lands somewhere else entirely.
+  const visible = screen?.width && screen?.height
+    ? targets.filter((t) => {
+        const f = t.frame;
+        if (!f) return t.y >= 0 && t.y <= screen.height && t.x >= 0 && t.x <= screen.width;
+        return f.y + (f.height ?? 0) > 0 && f.y < screen.height
+          && f.x + (f.width ?? 0) > 0 && f.x < screen.width;
+      })
+    : targets;
+  const group = synonymGroup(intent);
+  const roleHint = ROLE_HINTS.find((h) => h.pattern.test(intent));
+  const regionHint = REGION_HINTS.find((h) => h.pattern.test(intent));
+  // Strip the verb: "tap the Save button" should match a control called "Save".
+  const bare = norm(intent)
+    .replace(/^(please\s+)?(tap|press|click|hit|type|enter|fill|open|select|choose|toggle|switch)\s+/i, '')
+    .replace(/^(the|a|an)\s+/i, '')
+    .replace(/\s+(button|tab|field|cell|link|icon)$/i, '');
+
+  const scored = [];
+  for (const t of visible) {
+    const names = [t.label, ...(t.aliases ?? [])].filter(Boolean);
+    let base = 0;
+    let matched = null;
+    for (const name of names) {
+      const s = Math.max(nameScore(name, intent), nameScore(name, bare));
+      if (s > base) {
+        base = s;
+        matched = name;
+      }
+    }
+    // An icon-only control has no readable name, so a synonym is the only way
+    // to reach it — this is how "back" finds a bare chevron.
+    if (group && base < 0.5 && !t.label && t.rawLabel) base = 0.55;
+    if (group && base < 0.5 && names.some((n) => group.words.includes(norm(n)))) base = 0.9;
+    if (base <= 0) continue;
+
+    const reasons = [matched ? `label "${matched}"` : 'icon-only'];
+    let score = base;
+    if (roleHint && roleHint.roles.test(t.type ?? '')) {
+      score += 0.12;
+      reasons.push(`role ${t.type}`);
+    }
+    if (regionHint && t.region === regionHint.region) {
+      score += 0.15;
+      reasons.push(`region ${t.region}`);
+    }
+    // A caption is not a control. Prefer something tappable when the names tie.
+    if (/button|link|cell|field|switch|tab/i.test(t.type ?? '')) {
+      score += 0.05;
+      reasons.push('interactive');
+    }
+    // Not capped here: clamping to 1 before comparing throws away exactly the
+    // signal the bonuses exist to provide. Two elements sharing a label both
+    // reach 1.0 on the name alone, and the region bonus that should separate
+    // them disappears into the ceiling.
+    scored.push({ target: t, score, reasons });
+  }
+  return scored.sort((a, b) => b.score - a.score);
+}
+
+/** How close two candidates may be before the answer counts as ambiguous. */
+export const AMBIGUITY_MARGIN = 0.08;
+/** Below this, no candidate is worth acting on. */
+export const MINIMUM_SCORE = 0.45;
+
+/**
+ * How close two tap points have to be to mean the same control.
+ *
+ * Deliberately small. Two genuinely different controls are not twelve points
+ * apart centre to centre on any screen iOS lays out; two *readings* of one
+ * control are one or two points apart, because the accessibility tree and OCR
+ * are describing the same rectangle. Measured on a real filter row: the tree
+ * published "Location (All)" at (201,181) and OCR read "Location (AII)" at
+ * (200,182), and the caller was asked which of the two it meant.
+ */
+export const SAME_CONTROL_POINTS = 12;
+
+/**
+ * Two candidates in the same place are one control read twice.
+ *
+ * Asking which one was meant is not caution here, it is a question with no
+ * answer — either tap lands on the same pixel. So the readings are collapsed,
+ * and the accessibility one wins, because it is the actual hit target and its
+ * label has not been through OCR.
+ */
+const INTERACTIVE_ROLE = /button|field|cell|row|link|switch|slider|tab|menu|segment|checkbox/i;
+
+const contains = (frame, target) =>
+  Boolean(frame)
+  && target.x >= frame.x && target.x <= frame.x + (frame.width ?? 0)
+  && target.y >= frame.y && target.y <= frame.y + (frame.height ?? 0);
+
+/**
+ * Are these two candidates the same control?
+ *
+ * Two ways, and the second one cost a measurement. Centres a couple of points
+ * apart are one rectangle read twice. But a full-width list cell and the
+ * left-aligned text printed inside it have centres a hundred points apart and
+ * are still one tap target — measured on a Settings list, where "General" came
+ * back as the cell at (201,326) and the OCR text at (102,327) and the caller
+ * was asked which of the two it meant. The screen map already folds that pair
+ * into one row; this is `locate` catching up with it.
+ */
+function sameControl(a, b) {
+  if (Math.abs(a.x - b.x) <= SAME_CONTROL_POINTS && Math.abs(a.y - b.y) <= SAME_CONTROL_POINTS) return true;
+  // Containment only counts when the container is a hit target. A group that
+  // merely encloses things is not the thing inside it, which is what stops a
+  // tab bar from absorbing its own tabs.
+  if (INTERACTIVE_ROLE.test(a.type ?? '') && contains(a.frame, b)) return true;
+  if (INTERACTIVE_ROLE.test(b.type ?? '') && contains(b.frame, a)) return true;
+  return false;
+}
+
+function collapseSamePlace(ranked) {
+  const kept = [];
+  for (const c of ranked) {
+    const twin = kept.find((k) => sameControl(k.target, c.target));
+    if (!twin) {
+      kept.push(c);
+      continue;
+    }
+    // Prefer the real hit target: an accessibility element over OCR's reading of
+    // it, and an interactive role over a caption sitting inside it.
+    const better = (candidate, incumbent) => {
+      if (candidate.target.source === 'ax' && incumbent.target.source !== 'ax') return true;
+      if (candidate.target.source !== 'ax' && incumbent.target.source === 'ax') return false;
+      return INTERACTIVE_ROLE.test(candidate.target.type ?? '')
+        && !INTERACTIVE_ROLE.test(incumbent.target.type ?? '');
+    };
+    if (better(c, twin)) {
+      kept[kept.indexOf(twin)] = { ...c, reasons: [...c.reasons, 'the hit target, not the text printed on it'] };
+    }
+  }
+  return kept;
+}
+
+/**
+ * Resolve an intent to one element, or say why not.
+ * @returns {{status: 'ok'|'ambiguous'|'none', target?, score?, reasons?, alternatives?}}
+ */
+export function resolve(targets, intent, options = {}) {
+  const ranked = collapseSamePlace(rank(targets, intent, options).filter((c) => c.score >= MINIMUM_SCORE));
+  if (!ranked.length) return { status: 'none', alternatives: [] };
+  const [best, second] = ranked;
+  if (second && best.score - second.score < AMBIGUITY_MARGIN) {
+    return {
+      status: 'ambiguous',
+      alternatives: ranked.slice(0, 5).map((c) => ({
+        label: c.target.label ?? '(icon-only)',
+        x: c.target.x,
+        y: c.target.y,
+        region: c.target.region,
+        score: Math.round(Math.min(1, c.score) * 100) / 100,
+        reasons: c.reasons,
+      })),
+    };
+  }
+  return {
+    status: 'ok',
+    target: best.target,
+    score: Math.round(Math.min(1, best.score) * 100) / 100,
+    reasons: best.reasons,
+    alternatives: ranked.slice(1, 4).map((c) => ({
+      label: c.target.label ?? '(icon-only)',
+      score: Math.round(Math.min(1, c.score) * 100) / 100,
+    })),
+  };
+}