simframe 0.5.0 → 0.6.0

package/src/index.js

@@ -17,6 +17,7 @@ import * as input from './input.js';
 import * as fingerprint from './fingerprint.js';
 import * as graph from './graph.js';
 import * as matching from './matching.js';
+import * as refs from './refs.js';
 import * as screenmap from './screenmap.js';
 import { resolveDevice, resize, screenshot } from './simctl.js';
 import * as store from './store.js';
@@ -25,7 +26,24 @@ const HERE = path.dirname(fileURLToPath(import.meta.url));
 const CLI = path.join(HERE, 'cli.js');
 const sleep = (ms) => new Promise((r) => setTimeout(r, ms));
 
-export const DETAIL_LEVELS = { low: 420, normal: 700, high: 1100, full: 0 };
+export const DETAIL_LEVELS = { low: 420, normal: 700, high: 1024, full: 0 };
+
+/**
+ * The ceiling on any image handed to a model.
+ *
+ * An image costs ~1,600 tokens when Claude Code handles it as a native image
+ * block, and 15,000–25,000 when the base64 is treated as text (claude-code
+ * issue #31208) — enough to trip the 25,000-token tool-result limit on its own.
+ * A native-resolution frame buys nothing at either price: 1024 px on the long
+ * edge is already more than a 393-point screen has to say. Only the CLI, which
+ * writes to a file rather than into a context window, may exceed it.
+ */
+export const MODEL_MAX_IMAGE_DIM = 1024;
+
+export function modelDetail(detail) {
+  const dim = resolveMaxDim(detail);
+  return dim === 0 || dim > MODEL_MAX_IMAGE_DIM ? MODEL_MAX_IMAGE_DIM : dim;
+}
 
 export function resolveMaxDim(detail) {
   if (typeof detail === 'number') return detail;
@@ -713,6 +731,40 @@ export async function locate(
 ) {
   const { device, state: firstState } = await ensureDaemon(deviceQuery, options);
   const udid = device.udid;
+
+  // Selectors resolve before any perception happens: `#3` is already an answer
+  // somebody numbered, and `@x,y` was never a question about the screen.
+  const selector = refs.parseSelector(query);
+  if (selector.kind === 'point') {
+    return {
+      device,
+      state: firstState,
+      target: { label: `(${selector.x},${selector.y})`, x: selector.x, y: selector.y, source: 'coordinates' },
+      from: 'selector',
+      distance: 0,
+      settled: true,
+    };
+  }
+  if (selector.kind === 'ref') {
+    // Screen memory answers "which screen is this?" from a file, so a ref can
+    // be checked against structural identity without paying for a perception
+    // pass — which is the whole reason a ref exists.
+    const near = screenmap.recallNearest(udid, firstState.layoutHash);
+    const hit = refs.resolveRef(udid, selector.ref, {
+      layoutHash: firstState.layoutHash,
+      structuralHash: near?.entry?.structuralHash ?? null,
+      screenKnown: Boolean(near),
+    });
+    return {
+      device,
+      state: firstState,
+      target: { ...hit, label: hit.label ?? `#${hit.ref}`, source: hit.source ?? 'ref' },
+      from: 'ref',
+      distance: 0,
+      settled: true,
+    };
+  }
+  if (selector.exact) query = selector.label;
   // Key memory off a settled frame, never off whichever frame happened to be
   // newest, so the capture rate cannot change what gets remembered.
   const { state, settled } = await settledState(udid, { settleMs });
@@ -839,7 +891,7 @@ const STRUCTURAL_SETTLE_SAMPLES = 3;
  * to. Novel fingerprints, and only those, are re-sampled until two consecutive
  * readings agree.
  */
-export async function screenIdentity(deviceQuery, { options, confirmNovel = true, settleMs, timeoutMs } = {}) {
+export async function screenIdentity(deviceQuery, { options, confirmNovel = true, settleMs, timeoutMs, fresh = false } = {}) {
   const { device, state } = await ensureDaemon(deviceQuery, options);
   const udid = device.udid;
 
@@ -855,8 +907,8 @@ export async function screenIdentity(deviceQuery, { options, confirmNovel = true
     });
     const current = settledFrame ?? state;
     let entry = fresh ? null : screenmap.recallNearest(udid, current.layoutHash)?.entry;
+    const geo = await deviceGeometry(udid, current);
     if (!entry) {
-      const geo = await deviceGeometry(udid, current);
       entry = await screenmap.build(udid, {
         hash: current.hash,
         layoutHash: current.layoutHash,
@@ -872,10 +924,16 @@ export async function screenIdentity(deviceQuery, { options, confirmNovel = true
       keyboard: Boolean(entry.keyboard),
       layoutHash: current.layoutHash,
       settled,
+      // Carried out so callers that want the elements as well as the identity
+      // do not pay for a second perception pass to get them. The compact
+      // screen map needs both, and reading twice was the whole cost of it.
+      entry,
+      state: current,
+      points: { width: geo.pointWidth, height: geo.pointHeight },
     };
   };
 
-  let identity = await read();
+  let identity = await read({ fresh });
   if (!confirmNovel) return { ...identity, confirmed: identity.settled };
   // Being recognised is stronger evidence than the pixel settle flag: a
   // fingerprint that matches a screen already trusted has nothing left to

package/src/input.js

@@ -34,6 +34,55 @@ export async function driverFor(udid) {
   return detectDriver();
 }
 
+/**
+ * An escape hatch back to idb for the tree.
+ *
+ * Every private-framework path here is version-coupled, and the host-side
+ * translator is no exception: an Xcode upgrade could break it on a machine
+ * where work still has to happen that day. Reading it per call rather than
+ * caching means the switch takes effect without restarting anything.
+ */
+function preferIdbTree() {
+  return process.env.SIMFRAME_AX_DRIVER === 'idb';
+}
+
+/**
+ * Which driver reads the accessibility tree for a device.
+ *
+ * Separate from `driverFor` because these are separate capabilities: a device
+ * can be perfectly touchable by the daemon while the translation framework is
+ * missing, and reporting one number for both hides which layer is down.
+ */
+export async function axDriverFor(udid) {
+  if (udid && control.available(udid) && !preferIdbTree()) {
+    try {
+      const status = await control.status(udid);
+      if (status.accessibility?.available) {
+        return { name: 'simframed', available: true, chosen: false, version: status.accessibility.detail, reason: null };
+      }
+      // The daemon is up and says it cannot read the tree. idb might still,
+      // so this is a reason to fall through rather than an answer.
+    } catch {
+      /* daemon went away mid-call; fall through to idb */
+    }
+  }
+  const idbDriver = await detectDriver();
+  // Asked for, or fallen back to? A driver someone chose is not a degradation,
+  // and grading it as one turns the documented escape hatch into a red CI run
+  // on exactly the day an Xcode upgrade makes you reach for it.
+  const chosen = preferIdbTree();
+  if (idbDriver.available) {
+    return {
+      name: 'idb',
+      available: true,
+      chosen,
+      version: chosen ? `${idbDriver.version} — selected by SIMFRAME_AX_DRIVER` : idbDriver.version,
+      reason: null,
+    };
+  }
+  return { name: null, available: false, chosen, version: null, reason: idbDriver.reason };
+}
+
 /** @returns {Promise<{name: string, available: boolean, version: string|null, reason: string|null}>} */
 export async function detectDriver({ refresh = false } = {}) {
   if (driverCache && !refresh) return driverCache;
@@ -128,6 +177,20 @@ async function readScreenInfo(udid) {
 
 /** The accessibility tree, flattened. This is what makes tap-by-label possible. */
 export async function describeAll(udid) {
+  // The daemon reads the tree host-side through AXPTranslator: no install, and
+  // measured at 45ms against idb's 203ms on the same screen. idb stays as the
+  // fallback, so a machine without the daemon still reads.
+  if (control.available(udid) && !preferIdbTree()) {
+    try {
+      const { screen } = await control.request(udid, { action: 'ui', ocr: false });
+      // An app mid-launch genuinely has no tree yet. Falling through to idb
+      // here would just ask a second time and report the same emptiness more
+      // slowly, so the honest answer is the empty one.
+      if (screen?.sources?.includes('ax')) return (screen.elements ?? []).map(elementToNode);
+    } catch {
+      /* daemon went away mid-call; fall through to idb */
+    }
+  }
   // Passing --json here yields empty output; the default already emits JSON.
   const out = await idb(['ui', 'describe-all', '--udid', udid]);
   const nodes = [];
@@ -149,6 +212,20 @@ export async function describeAll(udid) {
   return nodes.filter((n) => n.frame);
 }
 
+/** A daemon element back into the node shape every caller here expects. */
+export function elementToNode(e) {
+  return {
+    label: cleanLabel(e.label),
+    rawLabel: e.label ?? null,
+    value: e.value ?? null,
+    type: e.role ?? null,
+    identifier: e.identifier ?? null,
+    enabled: e.state?.enabled ?? null,
+    frame: e.frame ?? null,
+    raw: e,
+  };
+}
+
 // Icon fonts put glyphs in the Unicode private use areas, so a label arrives as
 // "<glyph>, My Tools". Matching has to see through that to the readable text.
 const PRIVATE_USE = /[\u{E000}-\u{F8FF}\u{F0000}-\u{FFFFD}\u{100000}-\u{10FFFD}]/gu;
@@ -263,6 +340,28 @@ export async function pressKey(udid, keycode) {
   await idb(['ui', 'key', '--udid', udid, String(keycode)]);
 }
 
+/**
+ * Rebuild the daemon's HID session.
+ *
+ * Input is the one path with no feedback: a dispatched Indigo message reports
+ * success when the send succeeds, and nothing asks the device whether anything
+ * happened. Measured on a long-running daemon, a HOME press returned in 66ms
+ * and the screen did not move; the same press on a freshly started daemon
+ * worked. Whoever holds the frames is the only one who can notice, which is why
+ * this is something callers invoke rather than something input does for itself.
+ *
+ * @returns {Promise<boolean>} whether a session was actually reset.
+ */
+export async function resetSession(udid) {
+  if (!control.available(udid)) return false;
+  try {
+    await control.resetInput(udid);
+    return true;
+  } catch {
+    return false;
+  }
+}
+
 export async function pressButton(udid, name) {
   if (control.available(udid)) {
     try {

package/src/matching.js

@@ -160,12 +160,83 @@ export const AMBIGUITY_MARGIN = 0.08;
 /** Below this, no candidate is worth acting on. */
 export const MINIMUM_SCORE = 0.45;
 
+/**
+ * How close two tap points have to be to mean the same control.
+ *
+ * Deliberately small. Two genuinely different controls are not twelve points
+ * apart centre to centre on any screen iOS lays out; two *readings* of one
+ * control are one or two points apart, because the accessibility tree and OCR
+ * are describing the same rectangle. Measured on a real filter row: the tree
+ * published "Location (All)" at (201,181) and OCR read "Location (AII)" at
+ * (200,182), and the caller was asked which of the two it meant.
+ */
+export const SAME_CONTROL_POINTS = 12;
+
+/**
+ * Two candidates in the same place are one control read twice.
+ *
+ * Asking which one was meant is not caution here, it is a question with no
+ * answer — either tap lands on the same pixel. So the readings are collapsed,
+ * and the accessibility one wins, because it is the actual hit target and its
+ * label has not been through OCR.
+ */
+const INTERACTIVE_ROLE = /button|field|cell|row|link|switch|slider|tab|menu|segment|checkbox/i;
+
+const contains = (frame, target) =>
+  Boolean(frame)
+  && target.x >= frame.x && target.x <= frame.x + (frame.width ?? 0)
+  && target.y >= frame.y && target.y <= frame.y + (frame.height ?? 0);
+
+/**
+ * Are these two candidates the same control?
+ *
+ * Two ways, and the second one cost a measurement. Centres a couple of points
+ * apart are one rectangle read twice. But a full-width list cell and the
+ * left-aligned text printed inside it have centres a hundred points apart and
+ * are still one tap target — measured on a Settings list, where "General" came
+ * back as the cell at (201,326) and the OCR text at (102,327) and the caller
+ * was asked which of the two it meant. The screen map already folds that pair
+ * into one row; this is `locate` catching up with it.
+ */
+function sameControl(a, b) {
+  if (Math.abs(a.x - b.x) <= SAME_CONTROL_POINTS && Math.abs(a.y - b.y) <= SAME_CONTROL_POINTS) return true;
+  // Containment only counts when the container is a hit target. A group that
+  // merely encloses things is not the thing inside it, which is what stops a
+  // tab bar from absorbing its own tabs.
+  if (INTERACTIVE_ROLE.test(a.type ?? '') && contains(a.frame, b)) return true;
+  if (INTERACTIVE_ROLE.test(b.type ?? '') && contains(b.frame, a)) return true;
+  return false;
+}
+
+function collapseSamePlace(ranked) {
+  const kept = [];
+  for (const c of ranked) {
+    const twin = kept.find((k) => sameControl(k.target, c.target));
+    if (!twin) {
+      kept.push(c);
+      continue;
+    }
+    // Prefer the real hit target: an accessibility element over OCR's reading of
+    // it, and an interactive role over a caption sitting inside it.
+    const better = (candidate, incumbent) => {
+      if (candidate.target.source === 'ax' && incumbent.target.source !== 'ax') return true;
+      if (candidate.target.source !== 'ax' && incumbent.target.source === 'ax') return false;
+      return INTERACTIVE_ROLE.test(candidate.target.type ?? '')
+        && !INTERACTIVE_ROLE.test(incumbent.target.type ?? '');
+    };
+    if (better(c, twin)) {
+      kept[kept.indexOf(twin)] = { ...c, reasons: [...c.reasons, 'the hit target, not the text printed on it'] };
+    }
+  }
+  return kept;
+}
+
 /**
  * Resolve an intent to one element, or say why not.
  * @returns {{status: 'ok'|'ambiguous'|'none', target?, score?, reasons?, alternatives?}}
  */
 export function resolve(targets, intent, options = {}) {
-  const ranked = rank(targets, intent, options).filter((c) => c.score >= MINIMUM_SCORE);
+  const ranked = collapseSamePlace(rank(targets, intent, options).filter((c) => c.score >= MINIMUM_SCORE));
   if (!ranked.length) return { status: 'none', alternatives: [] };
   const [best, second] = ranked;
   if (second && best.score - second.score < AMBIGUITY_MARGIN) {