simframe 0.4.1 → 0.5.0

package/src/matching.js

@@ -0,0 +1,194 @@
+// Resolving "what did they mean" against what is on screen.
+//
+// The rule throughout: never guess between two plausible answers. A wrong tap
+// is worse than a question, because a wrong tap can do something and the caller
+// will believe it did the right thing.
+
+/** Names for controls that carry an icon and no readable label. */
+const SYNONYMS = {
+  back: ['back', 'chevron.left', 'navigate back', 'previous', 'return'],
+  close: ['close', 'dismiss', 'xmark', 'cancel', 'done'],
+  search: ['search', 'find', 'magnifyingglass'],
+  add: ['add', 'new', 'create', 'plus', 'compose'],
+  more: ['more', 'options', 'ellipsis', 'overflow', 'menu'],
+  settings: ['settings', 'preferences', 'gear', 'configure'],
+  share: ['share', 'export', 'send'],
+  delete: ['delete', 'remove', 'trash', 'bin'],
+  edit: ['edit', 'modify', 'change'],
+  save: ['save', 'apply', 'confirm', 'ok', 'submit'],
+};
+
+/** Words that say what kind of control the caller means. */
+const ROLE_HINTS = [
+  { pattern: /\b(type|enter|fill|input)\b/i, roles: /field|textfield|textview|searchfield/i },
+  { pattern: /\b(tap|press|click|hit)\b/i, roles: /button|link|cell|tab/i },
+  { pattern: /\b(toggle|switch|enable|disable|turn)\b/i, roles: /switch|toggle|checkbox/i },
+  { pattern: /\b(tab)\b/i, roles: /tab/i },
+];
+
+/** Words that say where on screen the caller means. */
+const REGION_HINTS = [
+  { pattern: /\btab\b/i, region: 'tab-bar' },
+  { pattern: /\b(back|nav|title|toolbar)\b/i, region: 'nav-bar' },
+];
+
+const norm = (s) => String(s ?? '').toLowerCase().replace(/\s+/g, ' ').trim();
+
+/** Levenshtein distance, capped: beyond the cap the exact value is irrelevant. */
+export function editDistance(a, b, cap = 8) {
+  if (a === b) return 0;
+  if (Math.abs(a.length - b.length) > cap) return cap + 1;
+  let prev = Array.from({ length: b.length + 1 }, (_, i) => i);
+  for (let i = 1; i <= a.length; i++) {
+    const row = [i];
+    let best = i;
+    for (let j = 1; j <= b.length; j++) {
+      const cost = a[i - 1] === b[j - 1] ? 0 : 1;
+      row[j] = Math.min(prev[j] + 1, row[j - 1] + 1, prev[j - 1] + cost);
+      if (row[j] < best) best = row[j];
+    }
+    if (best > cap) return cap + 1;
+    prev = row;
+  }
+  return prev[b.length];
+}
+
+/** How well one name answers to a query, 0 to 1. */
+export function nameScore(name, query) {
+  const n = norm(name);
+  const q = norm(query);
+  if (!n || !q) return 0;
+  if (n === q) return 1;
+  if (n.startsWith(q) || q.startsWith(n)) return 0.86;
+  // A substring match is only as good as the share of the name it covers.
+  // Without this, "back" scores 0.78 against a two-hundred-character list row
+  // that happens to contain "Back of House", and beats the actual back button.
+  if (n.includes(q)) return 0.78 * Math.max(0.15, q.length / n.length);
+  if (q.includes(n)) return 0.7 * Math.max(0.15, n.length / q.length);
+  // Fuzzy, so a typo or a stray plural still resolves.
+  //
+  // editDistance returns cap+1 as a sentinel when it gives up early. Treating
+  // that as a measurement made every long string score well: 1 - 9/200 is
+  // 0.955, so a two-hundred-character list row scored 0.687 against any query
+  // at all. A bail-out is "no answer", not "nearly identical".
+  const cap = 8;
+  const distance = editDistance(n, q, cap);
+  if (distance > cap) return 0;
+  const longest = Math.max(n.length, q.length);
+  const similarity = 1 - distance / longest;
+  return similarity >= 0.7 ? similarity * 0.72 : 0;
+}
+
+function synonymGroup(query) {
+  const q = norm(query);
+  for (const [key, words] of Object.entries(SYNONYMS)) {
+    if (words.some((w) => w === q || q.includes(w))) return { key, words };
+  }
+  return null;
+}
+
+/**
+ * Rank what is on screen against an intent.
+ *
+ * Returns candidates sorted best first, each with the reasons behind its score
+ * so a caller — or a person reading a failure — can see why.
+ */
+export function rank(targets, intent, { screen } = {}) {
+  // Never offer something that is not on screen. A scrolled-away row still sits
+  // in the map with a negative y, and tapping it lands somewhere else entirely.
+  const visible = screen?.width && screen?.height
+    ? targets.filter((t) => {
+        const f = t.frame;
+        if (!f) return t.y >= 0 && t.y <= screen.height && t.x >= 0 && t.x <= screen.width;
+        return f.y + (f.height ?? 0) > 0 && f.y < screen.height
+          && f.x + (f.width ?? 0) > 0 && f.x < screen.width;
+      })
+    : targets;
+  const group = synonymGroup(intent);
+  const roleHint = ROLE_HINTS.find((h) => h.pattern.test(intent));
+  const regionHint = REGION_HINTS.find((h) => h.pattern.test(intent));
+  // Strip the verb: "tap the Save button" should match a control called "Save".
+  const bare = norm(intent)
+    .replace(/^(please\s+)?(tap|press|click|hit|type|enter|fill|open|select|choose|toggle|switch)\s+/i, '')
+    .replace(/^(the|a|an)\s+/i, '')
+    .replace(/\s+(button|tab|field|cell|link|icon)$/i, '');
+
+  const scored = [];
+  for (const t of visible) {
+    const names = [t.label, ...(t.aliases ?? [])].filter(Boolean);
+    let base = 0;
+    let matched = null;
+    for (const name of names) {
+      const s = Math.max(nameScore(name, intent), nameScore(name, bare));
+      if (s > base) {
+        base = s;
+        matched = name;
+      }
+    }
+    // An icon-only control has no readable name, so a synonym is the only way
+    // to reach it — this is how "back" finds a bare chevron.
+    if (group && base < 0.5 && !t.label && t.rawLabel) base = 0.55;
+    if (group && base < 0.5 && names.some((n) => group.words.includes(norm(n)))) base = 0.9;
+    if (base <= 0) continue;
+
+    const reasons = [matched ? `label "${matched}"` : 'icon-only'];
+    let score = base;
+    if (roleHint && roleHint.roles.test(t.type ?? '')) {
+      score += 0.12;
+      reasons.push(`role ${t.type}`);
+    }
+    if (regionHint && t.region === regionHint.region) {
+      score += 0.15;
+      reasons.push(`region ${t.region}`);
+    }
+    // A caption is not a control. Prefer something tappable when the names tie.
+    if (/button|link|cell|field|switch|tab/i.test(t.type ?? '')) {
+      score += 0.05;
+      reasons.push('interactive');
+    }
+    // Not capped here: clamping to 1 before comparing throws away exactly the
+    // signal the bonuses exist to provide. Two elements sharing a label both
+    // reach 1.0 on the name alone, and the region bonus that should separate
+    // them disappears into the ceiling.
+    scored.push({ target: t, score, reasons });
+  }
+  return scored.sort((a, b) => b.score - a.score);
+}
+
+/** How close two candidates may be before the answer counts as ambiguous. */
+export const AMBIGUITY_MARGIN = 0.08;
+/** Below this, no candidate is worth acting on. */
+export const MINIMUM_SCORE = 0.45;
+
+/**
+ * Resolve an intent to one element, or say why not.
+ * @returns {{status: 'ok'|'ambiguous'|'none', target?, score?, reasons?, alternatives?}}
+ */
+export function resolve(targets, intent, options = {}) {
+  const ranked = rank(targets, intent, options).filter((c) => c.score >= MINIMUM_SCORE);
+  if (!ranked.length) return { status: 'none', alternatives: [] };
+  const [best, second] = ranked;
+  if (second && best.score - second.score < AMBIGUITY_MARGIN) {
+    return {
+      status: 'ambiguous',
+      alternatives: ranked.slice(0, 5).map((c) => ({
+        label: c.target.label ?? '(icon-only)',
+        x: c.target.x,
+        y: c.target.y,
+        region: c.target.region,
+        score: Math.round(Math.min(1, c.score) * 100) / 100,
+        reasons: c.reasons,
+      })),
+    };
+  }
+  return {
+    status: 'ok',
+    target: best.target,
+    score: Math.round(Math.min(1, best.score) * 100) / 100,
+    reasons: best.reasons,
+    alternatives: ranked.slice(1, 4).map((c) => ({
+      label: c.target.label ?? '(icon-only)',
+      score: Math.round(Math.min(1, c.score) * 100) / 100,
+    })),
+  };
+}

package/src/mcp.js

@@ -7,6 +7,8 @@ import {
   ListToolsRequestSchema,
 } from '@modelcontextprotocol/sdk/types.js';
 import fs from 'node:fs';
+import path from 'node:path';
+import { fileURLToPath } from 'node:url';
 import { REGION_COLS, REGION_ROWS, regionMap } from './analyze.js';
 import * as actions from './actions.js';
 import * as api from './index.js';
@@ -153,6 +155,19 @@ const TOOLS = [
       },
     },
   },
+  {
+    name: 'sim_find',
+    description:
+      'Resolve an intent to one control on screen: "tap Save", "the Assets tab", "back". Understands verbs, typos, where on screen you meant, and icon-only controls by their common name. Returns the element with its tap point and the reasons it was chosen. When two things answer equally well it says so and lists them rather than guessing — a wrong tap is worse than a question, because it can do something and leave you believing it did the right thing. Cheaper and more reliable than reading a screenshot to find a control.',
+    inputSchema: {
+      type: 'object',
+      properties: {
+        ...deviceProp,
+        intent: { type: 'string', description: 'What you want to act on, in your own words.' },
+      },
+      required: ['intent'],
+    },
+  },
   {
     name: 'sim_capture',
     description:
@@ -180,6 +195,16 @@ const TOOLS = [
 // frame-to-frame delta look broken in practice.
 const lastSeen = new Map();
 
+/** Report the real version: a hardcoded one silently drifts every release. */
+function packageVersion() {
+  try {
+    const here = path.dirname(fileURLToPath(import.meta.url));
+    return JSON.parse(fs.readFileSync(path.join(here, '..', 'package.json'), 'utf8')).version;
+  } catch {
+    return '0.0.0';
+  }
+}
+
 function remember(udid, state) {
   lastSeen.set(udid, { hash: state.hash, seq: state.seq, at: state.capturedAt });
 }
@@ -221,7 +246,7 @@ function sinceLine(since) {
 
 export async function serve({ device: defaultDevice, options = {} } = {}) {
   const server = new Server(
-    { name: 'simframe', version: '0.1.0' },
+    { name: 'simframe', version: packageVersion() },
     { capabilities: { tools: {} } },
   );
 
@@ -242,6 +267,8 @@ export async function serve({ device: defaultDevice, options = {} } = {}) {
           return await strip(target, args, options);
         case 'sim_recall':
           return await recall(target, args, options);
+        case 'sim_find':
+          return await find(target, args, options);
         case 'sim_do':
           return await doScript(target, args, options);
         case 'sim_ui':
@@ -479,6 +506,23 @@ async function ui(target, args, options) {
   return { content: [text(`${head}\n${rows.join('\n')}${tail}`)] };
 }
 
+async function find(target, args, options) {
+  try {
+    const r = await api.locate(target, String(args.intent ?? ''), { options });
+    const lines = [
+      `${r.target.label ?? '(icon-only)'} — tap at (${r.target.x}, ${r.target.y})`,
+      `${r.target.region ?? 'content'} · ${r.target.type ?? '?'} · seen by ${r.target.source} · score ${r.score ?? '-'}`,
+    ];
+    if (r.reasons?.length) lines.push(`chosen because: ${r.reasons.join(', ')}`);
+    if (r.alternatives?.length) {
+      lines.push(`also considered: ${r.alternatives.map((a) => `"${a.label}" (${a.score})`).join(', ')}`);
+    }
+    return { content: [text(lines.join('\n'))] };
+  } catch (err) {
+    return { isError: true, content: [text(err.message)] };
+  }
+}
+
 async function capture(target, args, options) {
   if (args.action === 'start') {
     const res = await api.ensureDaemon(target, { ...options, fps: args.fps ?? options.fps });

package/src/navigate.js

@@ -0,0 +1,120 @@
+// Navigating by memory.
+//
+// Once the graph knows which screens exist and which action leads from one to
+// the next, getting somewhere is a search over known edges rather than a
+// question for a model. `goto` plans a path and walks it; a flow is a path
+// somebody already walked, saved so it can be walked again.
+import { runScript } from './actions.js';
+import * as graph from './graph.js';
+import * as api from './index.js';
+import * as store from './store.js';
+import fs from 'node:fs';
+import path from 'node:path';
+
+const flowDir = (udid) => path.join(store.deviceDir(udid), 'flows');
+
+/** The signature is lossy; older edges predate `step` and have to be reconstructed. */
+export function stepFor(edge) {
+  if (edge.step) return edge.step;
+  const [kind, rest] = [edge.action.slice(0, edge.action.indexOf(':')), edge.action.slice(edge.action.indexOf(':') + 1)];
+  if (kind === 'tap') return { tap: rest };
+  if (kind === 'scroll') return { scroll: rest };
+  if (kind === 'button') return { button: rest };
+  if (kind === 'tapAt') {
+    const [x, y] = rest.split(',').map(Number);
+    return Number.isFinite(x) && Number.isFinite(y) ? { tapAt: { x, y } } : null;
+  }
+  if (kind === 'swipe') {
+    const [from, to] = rest.split('->').map((p) => p.split(',').map(Number));
+    return from?.length === 2 && to?.length === 2 ? { swipe: { from, to } } : null;
+  }
+  return null;
+}
+
+/**
+ * Walk to a known screen.
+ *
+ * Fails rather than guesses: if the destination is not in the graph, or the
+ * query fits two screens equally, or no path of known edges reaches it, that is
+ * reported. A wrong route is worse than no route, because it taps things.
+ */
+export async function goto(deviceQuery, target, { options, ...runOptions } = {}) {
+  const { device } = await api.ensureDaemon(deviceQuery, options);
+  const udid = device.udid;
+
+  const found = graph.findScreen(udid, target);
+  if (!found) return { ok: false, reason: 'unknown-screen', known: knownScreens(udid) };
+  if (found.ambiguous) return { ok: false, reason: 'ambiguous', candidates: found.ambiguous };
+
+  const here = await api.screenIdentity(udid, {});
+  if (here.hash === found.node.hash) {
+    return { ok: true, already: true, screen: found.name, steps: [] };
+  }
+
+  const path_ = graph.route(udid, { hash: here.hash, tokens: here.tokens }, found.node.hash);
+  if (!path_) return { ok: false, reason: 'no-route', from: here.hash.slice(0, 8), to: found.name };
+
+  const steps = path_.map(stepFor);
+  if (steps.some((s) => !s)) return { ok: false, reason: 'unreplayable-edge', to: found.name };
+
+  const result = await runScript(udid, { steps, stopOnUnexpected: true, ...runOptions });
+  const arrived = await api.screenIdentity(udid, {});
+  return {
+    ok: arrived.hash === found.node.hash,
+    screen: found.name,
+    steps,
+    ranSteps: result.ranSteps,
+    results: result.results,
+    arrived: arrived.hash.slice(0, 8),
+  };
+}
+
+export function knownScreens(udid) {
+  return graph.allNodes(udid).map((n) => ({ name: graph.describe(n), hash: n.hash.slice(0, 8), edges: n.edges.length }));
+}
+
+/**
+ * Save a flow.
+ *
+ * Only flows that verified end to end are worth saving: a flow with an
+ * unverified step in it is a recording of something that may not have worked,
+ * and replaying it faithfully reproduces the doubt.
+ */
+export function saveFlow(udid, name, script, { force = false } = {}) {
+  const verdicts = (script.results ?? []).map((r) => r.verification?.verdict);
+  if (!force && verdicts.some((v) => v && v !== 'ok')) {
+    return { ok: false, reason: 'unverified-steps', verdicts };
+  }
+  const dir = flowDir(udid);
+  fs.mkdirSync(dir, { recursive: true });
+  const body = {
+    name,
+    savedAt: Date.now(),
+    steps: script.steps ?? (script.results ?? []).map((r) => r.step).filter(Boolean),
+    startScreen: script.startScreen ?? null,
+  };
+  store.writeAtomic(path.join(dir, `${encodeURIComponent(name)}.json`), JSON.stringify(body, null, 2));
+  return { ok: true, name, steps: body.steps.length };
+}
+
+export function loadFlow(udid, name) {
+  return store.readJson(path.join(flowDir(udid), `${encodeURIComponent(name)}.json`));
+}
+
+export function listFlows(udid) {
+  const dir = flowDir(udid);
+  if (!fs.existsSync(dir)) return [];
+  return fs.readdirSync(dir)
+    .filter((f) => f.endsWith('.json'))
+    .map((f) => store.readJson(path.join(dir, f)))
+    .filter(Boolean)
+    .map((f) => ({ name: f.name, steps: f.steps?.length ?? 0, savedAt: f.savedAt }));
+}
+
+export async function runFlow(deviceQuery, name, { options, ...runOptions } = {}) {
+  const { device } = await api.ensureDaemon(deviceQuery, options);
+  const flow = loadFlow(device.udid, name);
+  if (!flow) return { ok: false, reason: 'unknown-flow', known: listFlows(device.udid).map((f) => f.name) };
+  const result = await runScript(device.udid, { steps: flow.steps, stopOnUnexpected: true, ...runOptions });
+  return { ok: result.ranSteps === flow.steps.length, name, ...result };
+}

package/src/regions.js

@@ -0,0 +1,90 @@
+// Where on the screen something is, in the terms iOS itself uses.
+//
+// A label alone is ambiguous — "Assets" is both a screen title and a tab — but
+// a label plus a region rarely is. These are geometric priors from the Human
+// Interface Guidelines rather than anything read from the app, so they are
+// cheap, always available, and occasionally wrong in the same ways the
+// guidelines are.
+export const REGIONS = [
+  'status-bar',
+  'nav-bar',
+  'tab-bar',
+  'keyboard',
+  'content',
+];
+
+/**
+ * Fractions of screen height. Deliberately conservative: a band that is too
+ * greedy mislabels content as chrome, and content is the common case.
+ */
+const BANDS = {
+  statusBar: 0.065,   // through the notch / dynamic island
+  navBar: 0.14,       // title and its leading/trailing controls
+  tabBar: 0.92,       // home indicator sits below this
+};
+
+/** Keyboards occupy the bottom of the screen and are unusually tall. */
+const KEYBOARD_MIN_FRACTION = 0.28;
+
+/**
+ * Chrome is short. Position alone is not enough: the last row of a long list
+ * reaches into the tab-bar band, and calling a 90pt cell a tab item makes a
+ * seven-row list a different screen from a three-row one.
+ */
+const CHROME_MAX_HEIGHT_FRACTION = 0.075;
+
+export function regionFor(frame, screen, { keyboardTop } = {}) {
+  if (!frame || !screen?.height) return 'content';
+  const top = frame.y / screen.height;
+  const bottom = (frame.y + (frame.height ?? 0)) / screen.height;
+  if (keyboardTop != null && frame.y >= keyboardTop) return 'keyboard';
+  const short = (frame.height ?? 0) <= screen.height * CHROME_MAX_HEIGHT_FRACTION;
+  if (bottom <= BANDS.statusBar) return 'status-bar';
+  if (short && top < BANDS.navBar && bottom < BANDS.navBar * 1.6) return 'nav-bar';
+  if (short && top >= BANDS.tabBar - 0.06) return 'tab-bar';
+  return 'content';
+}
+
+/**
+ * Where a nav-bar control sits across the bar: leading, title or trailing.
+ * A back button is leading; an edit or done button is trailing. Callers use it
+ * to disambiguate two controls that share a label.
+ */
+export function navSlot(frame, screen) {
+  if (!frame || !screen?.width) return null;
+  const centre = frame.x + (frame.width ?? 0) / 2;
+  const third = screen.width / 3;
+  if (centre < third) return 'leading';
+  if (centre > third * 2) return 'trailing';
+  return 'title';
+}
+
+/**
+ * The top of the keyboard, if one appears to be up.
+ *
+ * Inferred from a dense band of similar-height elements filling the bottom of
+ * the screen — keys. Returns null when nothing looks like one, which is the
+ * common case and must stay cheap.
+ */
+export function detectKeyboardTop(elements, screen) {
+  if (!screen?.height || elements.length < 12) return null;
+  const threshold = screen.height * (1 - KEYBOARD_MIN_FRACTION);
+  const low = elements.filter((e) => e.frame && e.frame.y > threshold);
+  if (low.length < 12) return null;
+  const heights = low.map((e) => e.frame.height ?? 0).sort((a, b) => a - b);
+  const median = heights[heights.length >> 1];
+  // Keys are small and uniform; a list of cells down there is not.
+  const uniform = heights.filter((h) => Math.abs(h - median) <= Math.max(3, median * 0.4)).length;
+  if (uniform / low.length < 0.7 || median > screen.height * 0.07) return null;
+  return Math.min(...low.map((e) => e.frame.y));
+}
+
+/** Annotate a target list with region and nav slot. Mutates and returns it. */
+export function annotate(targets, screen) {
+  const keyboardTop = detectKeyboardTop(targets, screen);
+  for (const t of targets) {
+    t.region = regionFor(t.frame, screen, { keyboardTop });
+    if (t.region === 'nav-bar') t.navSlot = navSlot(t.frame, screen);
+  }
+  return targets;
+}

package/src/screenmap.js

@@ -8,11 +8,14 @@
 import fs from 'node:fs';
 import path from 'node:path';
 import { hashDistance } from './analyze.js';
+import * as control from './control.js';
+import * as fingerprint from './fingerprint.js';
 import * as input from './input.js';
 import * as ocr from './ocr.js';
+import * as regions from './regions.js';
 import * as store from './store.js';
 
-const MAP_VERSION = 2; // layout hash crop changed; old maps no longer comparable
+const MAP_VERSION = 5; // tab-band content no longer contributes labels
 
 function mapDir(udid) {
   return path.join(store.deviceDir(udid), 'screens');
@@ -20,11 +23,18 @@ function mapDir(udid) {
 
 /**
  * How many of the 288 layout bits may differ and still count as the same screen.
- * Measured on a real app: revisiting a screen (with different list rows and a
- * different clock) moved 0-3 bits; different screens were 77-96 apart. 12 sits
- * well clear of both.
+ *
+ * Re-measured across four visits to each of five screens: a revisit is usually
+ * identical (median 0) but the tail reaches 62 when list content has changed,
+ * while different screens sit at 74 and above. That margin is much narrower
+ * than the first calibration suggested, and it is the reason this number stays
+ * conservative rather than being raised to cover the tail.
+ *
+ * The consequence is deliberate: a heavily changed screen is rebuilt rather
+ * than recognised. A rebuild costs ~300ms; a false match taps the wrong
+ * control. See docs/DEFERRED.md on fingerprinting structure instead of pixels.
  */
-export const DEFAULT_TOLERANCE = 12;
+export const DEFAULT_TOLERANCE = 20;
 
 export function recall(udid, hash) {
   if (!hash) return null;
@@ -104,16 +114,25 @@ export async function build(udid, {
   density = 3,
   useAx = true,
   useOcr = true,
+  persist = true,
   screen,
 } = {}) {
   const targets = [];
   const sources = [];
-  // Kick OCR off before reading the tree: they are independent, and the OCR
-  // pass is pure computation on a file that already exists.
-  const ocrPromise =
-    useOcr && fullFrame && fs.existsSync(fullFrame)
-      ? ocr.readText(fullFrame, { density }).catch((err) => err)
-      : null;
+  // OCR starts before the tree read: they are independent, and running them in
+  // series costs the whole recognition pass.
+  //
+  // The daemon reads text straight off the framebuffer. The fallback encodes a
+  // PNG, writes it, spawns a helper and decodes it again — measured at 555ms
+  // against 174ms — so it is only used when no daemon is listening.
+  const viaDaemon = useOcr && control.available(udid);
+  const ocrPromise = !useOcr
+    ? null
+    : viaDaemon
+      ? control.request(udid, { action: 'ui' }).catch((err) => err)
+      : fullFrame && fs.existsSync(fullFrame)
+        ? ocr.readText(fullFrame, { density }).catch((err) => err)
+        : null;
   // With no geometry, treat every element as a potential control rather than
   // guessing a screen size and mis-classifying containers.
   const screenArea = screen?.width && screen?.height ? screen.width * screen.height : Infinity;
@@ -146,8 +165,24 @@ export async function build(udid, {
 
   if (ocrPromise) {
     try {
-      const words = await ocrPromise;
-      if (words instanceof Error) throw words;
+      const result = await ocrPromise;
+      if (result instanceof Error) throw result;
+      // The daemon answers in points; readText answers in points too, having
+      // divided by density. Normalise the daemon's element shape to match.
+      const words = viaDaemon
+        ? (result.screen?.elements ?? [])
+            .filter((e) => e.label?.trim())
+            .map((e) => ({
+              text: e.label,
+              confidence: e.confidence,
+              x: e.frame.x,
+              y: e.frame.y,
+              width: e.frame.width,
+              height: e.frame.height,
+              centerX: e.center.x,
+              centerY: e.center.y,
+            }))
+        : result;
       sources.push('ocr');
       for (const w of words) {
         if (!w.text.trim()) continue;
@@ -187,14 +222,35 @@ export async function build(udid, {
     }
   }
 
-  return remember(udid, {
-    version: MAP_VERSION,
-    hash,
-    layoutHash,
-    at: Date.now(),
-    sources,
-    targets,
-  });
+  return finish();
+
+  function finish() {
+    // Region priors are geometry, so they cost nothing and disambiguate a great
+    // deal: "Assets" the nav title and "Assets" the tab differ only by where
+    // they are.
+    if (screen?.width && screen?.height) regions.annotate(targets, screen);
+    // Two hashes, two jobs. The pixel layout hash indexes this entry, because
+    // it can be computed from a frame alone and so can find a map without
+    // building one. The structural hash identifies the screen, because content
+    // is pixels and a list with new rows is not a new screen.
+    const structure = screen?.width && screen?.height
+      ? fingerprint.fingerprint(targets, screen)
+      : { hash: null, tokens: [], keyboard: false };
+    const entry = {
+      version: MAP_VERSION,
+      hash,
+      layoutHash,
+      structuralHash: structure.hash,
+      structuralTokens: structure.tokens,
+      keyboard: structure.keyboard,
+      at: Date.now(),
+      sources,
+      targets,
+    };
+    // Only a map of a settled screen is worth keeping; remembering a transition
+    // fills the store with layouts that will never be seen again.
+    return persist ? remember(udid, entry) : entry;
+  }
 }
 
 const norm = (s) => String(s ?? '').toLowerCase().trim();

package/src/simctl.js

@@ -96,9 +96,17 @@ export function isBootedSync(udid) {
 }
 
 export async function screenshot(udid, outFile, { mask = 'ignored' } = {}) {
-  await run('xcrun', ['simctl', 'io', udid, 'screenshot', '--type=png', `--mask=${mask}`, outFile], {
-    timeout: 10_000,
-  });
+  try {
+    await run('xcrun', ['simctl', 'io', udid, 'screenshot', '--type=png', `--mask=${mask}`, outFile], {
+      timeout: 10_000,
+    });
+  } catch (err) {
+    // Same reason as launchApp: execFile's message is "Command failed: <the
+    // whole command>" and simctl's actual complaint is in stderr. A CI failure
+    // here reported the command and nothing about why it did not work.
+    const detail = (err.stderr || '').trim().split('\n').filter(Boolean).pop();
+    throw new Error(detail ? `simctl screenshot failed: ${detail}` : `simctl screenshot failed: ${err.message}`);
+  }
 }
 
 /** Resample with sips, which ships with macOS, so simframe needs no image deps. */
@@ -107,7 +115,15 @@ export async function resize(inFile, outFile, maxDim) {
 }
 
 export async function launchApp(udid, bundleId) {
-  await run('xcrun', ['simctl', 'launch', udid, bundleId], { timeout: 20_000 });
+  try {
+    await run('xcrun', ['simctl', 'launch', udid, bundleId], { timeout: 20_000 });
+  } catch (err) {
+    // execFile's message is just "Command failed: ..." with simctl's actual
+    // complaint left in stderr. A CI run failed here and said nothing about
+    // why, which is the same sin as a silent fallback.
+    const detail = (err.stderr || '').trim().split('\n').filter(Boolean).pop();
+    throw new Error(detail ? `could not launch ${bundleId}: ${detail}` : `could not launch ${bundleId}: ${err.message}`);
+  }
 }
 
 export async function terminateApp(udid, bundleId) {