simframe 0.9.0 → 0.11.0

package/src/metrics.js

@@ -40,8 +40,16 @@ export const FACULTY = {
   no_plan: 'exploration (Phase 14)',
 };
 
-/** Faculties that exist. Empty until Phase 11 lands the first one. */
-export const BUILT_FACULTIES = new Set();
+/**
+ * Faculties that exist.
+ *
+ * Phase 11 landed the first one, so `verification_failed` no longer maps to
+ * something unbuilt — which changes what those records *mean*. Before, they
+ * were a queue waiting on a phase. Now they are evidence that the phase which
+ * shipped is not sufficient, and that is a more useful thing for the log to be
+ * able to say than a count of things nobody has written yet.
+ */
+export const BUILT_FACULTIES = new Set(['sense of time (Phase 11)']);
 
 function metricPaths(udid) {
   const dir = store.deviceDir(udid);
@@ -111,9 +119,27 @@ export const readFlows = (udid, opts) => readJsonl(metricPaths(udid).flows, opts
  * matching error strings at the boundary — a regexed message is a reason that
  * silently becomes "unknown" the day somebody rewords it.
  */
-export function tag(err, reason, { candidates = [], tried = [] } = {}) {
+export function tag(err, reason, { candidates = [], tried = [], ambiguous = false, intent = null } = {}) {
   if (!REASONS.includes(reason)) throw new Error(`not an escalation reason: ${reason}`);
-  err.escalation = { reason, candidates, tried };
+  // `ambiguous` is narrower than the reason, and that is the point. Two very
+  // different failures both tag `ambiguous_intent`: the target is on screen
+  // several times over, and the target is not on screen at all on a screen we
+  // thought we knew. Only the first is resolvable by *choosing*, and only the
+  // first tells a waiting caller that waiting is pointless — the thing it is
+  // waiting for has already arrived.
+  // `intent` is the goal in the caller's own words, recorded as a field rather
+  // than left in the prose of `detail`.
+  //
+  // Phase 17's go/no-go asks whether an on-device model would pick the element
+  // Claude picked, given the goal and the element list. The element list is
+  // here as `candidates` and the eventual choice is recoverable from the
+  // graph — the tap that finally worked on this screen becomes a verified edge
+  // carrying its own step. The goal was the missing third, and it was sitting
+  // inside a sentence: `"X" matches 3 things on this screen — say which…`.
+  // Regexing it back out at export time is the exact habit this file exists to
+  // avoid, and it would silently return nothing the day that sentence is
+  // reworded.
+  err.escalation = { reason, candidates, tried, ambiguous, intent };
   return err;
 }
 
@@ -213,8 +239,64 @@ export function fingerprintNow(udid, screenmap) {
  * measurement's clothes, and every number in this project is supposed to say
  * where it came from.
  */
+/**
+ * Which run of which program wrote a record.
+ *
+ * The log is per-device and, until now, anonymous — so two agents driving one
+ * booted simulator wrote one interleaved file with no way to separate them.
+ * Measured on the bench device in a single evening: 57 records to 81, a third
+ * of the new ones naming screens from an app the suite has never launched.
+ *
+ * That is not a corrupted file, it is a corrupted instrument. CLAUDE.md makes
+ * the reason breakdown of this log the thing that chooses which faculty gets
+ * built next, and a breakdown that silently pools two sessions errs toward
+ * whichever of them made more mistakes — which is not the same question as
+ * which faculty is missing.
+ *
+ * A pid alone would not do: pids are reused, and the useful grouping is "one
+ * agent's run", which for the MCP server is the life of the process and for
+ * the CLI is a single command. So: the start time, the pid, and a random tail,
+ * computed once per process. `client` says what kind of process it was, since
+ * "the MCP server did this" and "somebody ran a CLI command" deserve different
+ * readings of the same reason.
+ *
+ * Deliberately not a device id, a username, or anything about the machine. This
+ * file is committed to a public repo in summary form, and the question it has
+ * to answer is "was this all one agent", which needs no identity to answer.
+ */
+const SESSION_ID = process.env.SIMFRAME_SESSION
+  ? String(process.env.SIMFRAME_SESSION).slice(0, 64)
+  : `${process.pid.toString(36)}-${Date.now().toString(36)}-${Math.random().toString(36).slice(2, 8)}`;
+
+/*
+ * Minting the id from the pid was right for the MCP server, which is one
+ * long-lived process, and wrong for everything else. A CLI-driven agent starts
+ * a process per command, so it got one "session" per command: on the benchmark
+ * device, 33 session ids for 46 records, 30 of them holding a single record.
+ * `escalations --session` was therefore unable to answer the one question it
+ * exists for, and Phase 17's go/no-go step 1 — "filter to one session id" —
+ * had nothing to filter. `SIMFRAME_SESSION` lets a caller that knows it is one
+ * session say so; the per-process id stays the default.
+ */
+
+/** How this process is being used, for reading a breakdown afterwards. */
+function clientKind() {
+  const argv = process.argv.join(' ');
+  if (/\bmcp\b/.test(argv)) return 'mcp';
+  if (/bench-hpi|scripts\//.test(argv)) return 'script';
+  if (/cli\.js|\bsimframe\b/.test(argv)) return 'cli';
+  return 'library';
+}
+const CLIENT = clientKind();
+
+/** The session this process's records belong to. Exported for `escalations`. */
+export const sessionId = () => SESSION_ID;
+export const clientName = () => CLIENT;
+
 export function recordEscalation(udid, {
   flowId = null,
+  flowName = null,
+  intent = null,
   stepIndex = null,
   fingerprint = null,
   reason,
@@ -229,7 +311,16 @@ export function recordEscalation(udid, {
   if (!OUTCOMES.includes(outcome)) throw new Error(`not an escalation outcome: ${outcome}`);
   const record = {
     timestamp: new Date().toISOString(),
+    // Added after the log turned out to pool two agents' work invisibly. Both
+    // are cheap and neither is derivable afterwards, which is the test for
+    // whether a field belongs in a log at all.
+    session_id: SESSION_ID,
+    client: CLIENT,
     flow_id: flowId,
+    flow_name: flowName,
+    // What was asked for, in the caller's words. Ground truth for Phase 17's
+    // go/no-go, and on its own it answers "what kind of decision is costing us".
+    intent: intent ? String(intent).slice(0, 120) : null,
     step_index: stepIndex,
     screen_fingerprint: fingerprint,
     reason,
@@ -408,14 +499,31 @@ export function hpi({ flows, baselines = {} }) {
  * rate is 1.0 by construction and says nothing. The per-reason breakdown is
  * the part that decides the next phase, and it is informative today.
  */
-export function breakdown(records) {
+export function breakdown(records, { session = null, flow = null } = {}) {
   const byReason = {};
   for (const r of REASONS) byReason[r] = 0;
   const byScreen = new Map();
   const byOutcome = {};
+  const bySession = new Map();
+  const byFlow = new Map();
   let avoidable = 0;
-  for (const r of records) {
+  let unattributed = 0;
+  // Filtering happens here rather than at the call site so `total` and every
+  // rate below it describe the same set of records.
+  const kept = records.filter((r) => (session ? r?.session_id === session : true))
+    .filter((r) => (flow ? r?.flow_name === flow : true));
+  for (const r of kept) {
     if (!REASONS.includes(r?.reason)) continue;
+    // Records written before sessions were recorded cannot be attributed, and
+    // saying how many there are is the difference between a breakdown that
+    // pools two agents and one that says it might be.
+    if (r.session_id) {
+      const key = `${r.session_id}|${r.client ?? '?'}`;
+      bySession.set(key, (bySession.get(key) ?? 0) + 1);
+    } else {
+      unattributed += 1;
+    }
+    if (r.flow_name) byFlow.set(r.flow_name, (byFlow.get(r.flow_name) ?? 0) + 1);
     byReason[r.reason] += 1;
     byOutcome[r.outcome] = (byOutcome[r.outcome] ?? 0) + 1;
     // Already avoided locally, so not avoidable by anything unbuilt.
@@ -423,9 +531,25 @@ export function breakdown(records) {
     const key = r.screen_fingerprint ?? '(no fingerprint)';
     byScreen.set(key, (byScreen.get(key) ?? 0) + 1);
   }
-  const total = records.filter((r) => REASONS.includes(r?.reason)).length;
+  const total = kept.filter((r) => REASONS.includes(r?.reason)).length;
+  const sessions = [...bySession.entries()]
+    .map(([key, count]) => {
+      const [id, client] = key.split('|');
+      return { session_id: id, client, count };
+    })
+    .sort((a, b) => b.count - a.count);
   return {
     total,
+    // The log is per-device and shared: two agents on one booted simulator
+    // write one interleaved file. More than one session here means the counts
+    // below are a pool, and CLAUDE.md uses those counts to choose a phase.
+    sessions,
+    session_count: sessions.length,
+    unattributed,
+    // Any unattributed record at all makes this a pool: the whole point is
+    // that they cannot be told apart, and 92 of them is not "one session".
+    pooled: sessions.length > 1 || unattributed > 0,
+    by_flow: Object.fromEntries([...byFlow.entries()].sort((a, b) => b[1] - a[1])),
     by_reason: byReason,
     by_outcome: byOutcome,
     faculty: Object.fromEntries(
@@ -437,7 +561,9 @@ export function breakdown(records) {
       .sort((a, b) => b[1] - a[1])
       .slice(0, 10)
       .map(([fingerprint, count]) => ({ fingerprint, count })),
-    model_turns_spent: records.reduce((acc, r) => acc + (r.model_turns_spent ?? 0), 0),
+    // `kept`, not `records` — a filtered breakdown that reports the whole
+    // log's model turns is the same class of mistake as pooling two sessions.
+    model_turns_spent: kept.reduce((acc, r) => acc + (r.model_turns_spent ?? 0), 0),
   };
 }
 

package/src/navigate.js

@@ -41,12 +41,15 @@ export function stepFor(edge) {
  * count. The reason mapping lives in metrics.PLAN_REASONS so the five reasons
  * have one owner.
  */
-function refuse(udid, result, { detail = null } = {}) {
+function refuse(udid, result, { detail = null, flowName = null } = {}) {
   try {
     const reason = metrics.PLAN_REASONS[result.reason];
     if (reason) {
       metrics.recordEscalation(udid, {
         reason,
+        // A refusal by `goto` is about a destination and one by `flow run` is
+        // about a named flow. Either is what a breakdown wants to group by.
+        flowName,
         fingerprint: metrics.fingerprintNow(udid, screenmap),
         outcome: 'escalated_to_model',
         detail: detail ?? result.reason,
@@ -70,8 +73,8 @@ export async function goto(deviceQuery, target, { options, ...runOptions } = {})
   const udid = device.udid;
 
   const found = graph.findScreen(udid, target);
-  if (!found) return refuse(udid, { ok: false, reason: 'unknown-screen', known: knownScreens(udid) }, { detail: `no screen matches "${target}"` });
-  if (found.ambiguous) return refuse(udid, { ok: false, reason: 'ambiguous', candidates: found.ambiguous }, { detail: `"${target}" fits ${found.ambiguous.length} screens` });
+  if (!found) return refuse(udid, { ok: false, reason: 'unknown-screen', known: knownScreens(udid) }, { detail: `no screen matches "${target}"`, flowName: `goto:${target}` });
+  if (found.ambiguous) return refuse(udid, { ok: false, reason: 'ambiguous', candidates: found.ambiguous }, { detail: `"${target}" fits ${found.ambiguous.length} screens`, flowName: `goto:${target}` });
 
   const here = await api.screenIdentity(udid, {});
   if (here.hash === found.node.hash) {
@@ -84,12 +87,12 @@ export async function goto(deviceQuery, target, { options, ...runOptions } = {})
   // `Cannot read properties of null (reading 'slice')` instead of answering.
   // Not hypothetical on Android, where README's own table puts the launcher at
   // one token.
-  if (!here.hash) return refuse(udid, { ok: false, reason: 'no-identity', to: found.name });
+  if (!here.hash) return refuse(udid, { ok: false, reason: 'no-identity', to: found.name }, { flowName: `goto:${target}` });
   const path_ = graph.route(udid, { hash: here.hash, tokens: here.tokens }, found.node.hash);
-  if (!path_) return refuse(udid, { ok: false, reason: 'no-route', from: here.hash.slice(0, 8), to: found.name });
+  if (!path_) return refuse(udid, { ok: false, reason: 'no-route', from: here.hash.slice(0, 8), to: found.name }, { flowName: `goto:${target}` });
 
   const steps = path_.map(stepFor);
-  if (steps.some((s) => !s)) return refuse(udid, { ok: false, reason: 'unreplayable-edge', to: found.name });
+  if (steps.some((s) => !s)) return refuse(udid, { ok: false, reason: 'unreplayable-edge', to: found.name }, { flowName: `goto:${target}` });
 
   const result = await runScript(udid, { steps, stopOnUnexpected: true, ...runOptions });
   const arrived = await api.screenIdentity(udid, {});
@@ -148,7 +151,7 @@ export function listFlows(udid) {
 export async function runFlow(deviceQuery, name, { options, ...runOptions } = {}) {
   const { device } = await api.ensureDaemon(deviceQuery, options);
   const flow = loadFlow(device.udid, name);
-  if (!flow) return refuse(device.udid, { ok: false, reason: 'unknown-flow', known: listFlows(device.udid).map((f) => f.name) }, { detail: `no saved flow "${name}"` });
+  if (!flow) return refuse(device.udid, { ok: false, reason: 'unknown-flow', known: listFlows(device.udid).map((f) => f.name) }, { detail: `no saved flow "${name}"`, flowName: name });
   // A replayed flow knows its own name, so its record can be compared against
   // a human doing the same thing. `minSteps` comes from the flow definition or
   // stays null — the step count of a recorded route is not a claim about the

package/src/ocr.js

@@ -19,6 +19,17 @@ const BIN = path.join(BIN_DIR, 'ocr');
 
 let ready = null;
 
+/**
+ * Which recognition level to ask Vision for.
+ *
+ * `accurate` is the default and what CLAUDE.md fixes; `fast` is the other thing
+ * Vision offers. Exposed so the pair can be scored against each other instead
+ * of one of them being a constant nobody measured.
+ */
+export function level() {
+  return String(process.env.SIMFRAME_OCR ?? '').toLowerCase() === 'fast' ? 'fast' : 'accurate';
+}
+
 /** Compile once, then reuse. Recompiles only if the source is newer than the binary. */
 export async function ensureBinary() {
   if (ready) return ready;
@@ -55,7 +66,13 @@ export async function ensureBinary() {
 export async function readText(pngFile, { density = 3 } = {}) {
   const built = await ensureBinary();
   if (!built.available) throw new Error(built.reason);
-  const { stdout } = await run(built.binary, [pngFile], { timeout: 30_000, maxBuffer: 16 << 20 });
+  // The recognition level rides in the environment rather than in argv, so the
+  // Swift side keeps its one-argument contract and an older binary still works.
+  const { stdout } = await run(built.binary, [pngFile], {
+    timeout: 30_000,
+    maxBuffer: 16 << 20,
+    env: { ...process.env, SIMFRAME_OCR: level() },
+  });
   const raw = JSON.parse(stdout || '[]');
   return raw.map((r) => ({
     text: r.text,

package/src/planner.js

@@ -0,0 +1,195 @@
+/**
+ * The local planner tier: a ranker, behind a flag, that may only reorder.
+ *
+ * **Why this exists at all, given Phase 17 was a no-go.** That phase asked a
+ * local model to *choose the next element*, and the answer was that the matcher
+ * already does — 37 of 40 real decisions. This is the complement and the one
+ * case a string matcher structurally cannot do: the goal matches **nothing** on
+ * screen, and something has to guess which container leads to it. "Change my
+ * username" shares no prefix, synonym or typo distance with "Account".
+ *
+ * Measured on this machine, six hand-written cases: 5 of 6 top-1, 6 of 6 top-3,
+ * median 564 ms warm. See `docs/BENCHMARKS.md`. Against a model round trip at
+ * 10–16 s that is roughly twenty times cheaper; against the honest baseline —
+ * breadth-first ordering, which needs no model — it won five of six.
+ *
+ * **What it is allowed to do, and it is deliberately almost nothing.** It
+ * reorders a list of candidates the caller has already permitted and will try
+ * in some order regardless. It cannot invent a label, cannot choose an action,
+ * cannot see pixels, and never runs on a destructive label because the caller
+ * filtered those out before asking (`src/vocabulary.js`). If it is wrong the
+ * exploration budget simply tries the next one. That is strictly weaker
+ * authority than Phase 17 proposed, which is what makes it safe to try.
+ *
+ * **Off unless asked.** `SIMFRAME_PLANNER=apple` turns it on; anything else,
+ * or any failure at all, degrades to `null` and the caller keeps its own order.
+ * `doctor` reports which. CI runs with it off.
+ */
+import { spawn, execFile } from 'node:child_process';
+import fs from 'node:fs';
+import path from 'node:path';
+import { fileURLToPath } from 'node:url';
+import { promisify } from 'node:util';
+import * as store from './store.js';
+
+const run = promisify(execFile);
+const SOURCE = path.join(path.dirname(fileURLToPath(import.meta.url)), '..', 'native', 'rank.swift');
+const BIN = path.join(store.ROOT, 'bin', 'rank');
+
+/** Which backend the caller asked for. Absent means no local planner. */
+export function requested(options) {
+  // Per call first, then the environment, for the reason in `sensorMode`: an
+  // MCP server's environment is fixed when it spawns, so a tester could not
+  // switch backends inside one session and a round came back with one arm of
+  // its A/B unrun.
+  const raw = String(options?.planner ?? process.env.SIMFRAME_PLANNER ?? '').trim().toLowerCase();
+  if (!raw || raw === 'none' || raw === 'off' || raw === '0' || raw === 'false') return null;
+  return raw;
+}
+
+let building = null;
+
+export async function ensureBinary() {
+  if (building) return building;
+  building = (async () => {
+    try {
+      const src = fs.statSync(SOURCE).mtimeMs;
+      const bin = fs.existsSync(BIN) ? fs.statSync(BIN).mtimeMs : 0;
+      if (bin > src) return { available: true, binary: BIN };
+    } catch {
+      return { available: false, reason: 'the ranker source is missing from this install' };
+    }
+    try {
+      fs.mkdirSync(path.dirname(BIN), { recursive: true });
+      // `swiftc`, not `xcrun swiftc`: nothing above the platform boundary may
+      // name a platform tool, and the boundary test catches it. `src/ocr.js`
+      // set this precedent — a compiler is not a device tool.
+      await run('swiftc', ['-O', SOURCE, '-o', BIN], { timeout: 180_000 });
+      return { available: true, binary: BIN };
+    } catch (err) {
+      building = null; // let a later call retry once a toolchain is present
+      return {
+        available: false,
+        reason: err.code === 'ENOENT'
+          ? 'swiftc is not installed, so the local planner cannot be built (install Xcode command line tools)'
+          : `could not build the local planner: ${String(err.message).split('\n')[0]}`,
+      };
+    }
+  })();
+  return building;
+}
+
+let session = null;
+
+/** Start the helper once and keep it, because the first answer pays model load. */
+async function open() {
+  if (session) return session;
+  const built = await ensureBinary();
+  if (!built.available) return { ok: false, reason: built.reason };
+  session = await new Promise((resolve) => {
+    const child = spawn(built.binary, [], { stdio: ['pipe', 'pipe', 'ignore'] });
+    // Deliberately NOT unref'd. Unreffing the child's stdout unreferences the
+    // very pipe every request waits on, so the process exited silently in the
+    // middle of an await — a flow that printed nothing and returned 0. The
+    // helper is closed explicitly instead, by whoever opened it.
+    let buffer = '';
+    const waiters = [];
+    let settled = false;
+    const fail = (reason) => {
+      if (!settled) { settled = true; resolve({ ok: false, reason }); }
+      while (waiters.length) waiters.shift()(null);
+    };
+    child.on('error', (err) => fail(`the local planner would not start: ${err.message}`));
+    child.on('exit', () => { session = null; fail('the local planner exited'); });
+    child.stdout.on('data', (chunk) => {
+      buffer += chunk;
+      let i = buffer.indexOf('\n');
+      while (i >= 0) {
+        const line = buffer.slice(0, i).trim();
+        buffer = buffer.slice(i + 1);
+        i = buffer.indexOf('\n');
+        if (!line) continue;
+        let msg;
+        try { msg = JSON.parse(line); } catch { continue; }
+        if (!settled) {
+          settled = true;
+          if (msg.ready) resolve({ ok: true, child, waiters });
+          else resolve({ ok: false, reason: msg.unavailable ?? 'the local planner did not become ready' });
+          continue;
+        }
+        const next = waiters.shift();
+        if (next) next(msg);
+      }
+    });
+  });
+  return session;
+}
+
+/**
+ * Reorder `options` by which is likeliest to lead to `goal`.
+ *
+ * @returns {Promise<string[]|null>} the caller's own order is correct when this
+ *   is null, which is every failure mode: flag off, no model, a timeout, a
+ *   parse problem, a paraphrasing answer. Never throws.
+ */
+export async function rank(goal, options, { timeoutMs = 3000, deviceOptions } = {}) {
+  if (!requested(deviceOptions)) return null;
+  if (!goal || !Array.isArray(options) || options.length < 2) return null;
+  let live;
+  try {
+    live = await open();
+  } catch {
+    return null;
+  }
+  if (!live?.ok) return null;
+  const answer = await new Promise((resolve) => {
+    // A timed-out waiter has to be *retired*, not merely resolved. Leaving it in
+    // the queue meant the next answer went to it instead of to the next asker,
+    // and every call after that was off by one — which showed up as an
+    // exploration run that never finished rather than as an error.
+    let done = false;
+    const waiter = (msg) => {
+      if (done) return;
+      done = true;
+      clearTimeout(timer);
+      resolve(msg);
+    };
+    const timer = setTimeout(() => {
+      if (done) return;
+      done = true;
+      const i = live.waiters.indexOf(waiter);
+      if (i >= 0) live.waiters.splice(i, 1);
+      resolve(null);
+    }, timeoutMs);
+    live.waiters.push(waiter);
+    try {
+      live.child.stdin.write(`${JSON.stringify({ goal: String(goal), options })}\n`);
+    } catch {
+      clearTimeout(timer);
+      resolve(null);
+    }
+  });
+  if (!answer?.order?.length) return null;
+  // It often returns a subset, so its order comes first and ours fills the tail.
+  // Trusting it to be exhaustive would silently drop candidates the budget was
+  // going to try.
+  const ranked = answer.order.filter((label) => options.includes(label));
+  const seen = new Set(ranked);
+  return [...ranked, ...options.filter((o) => !seen.has(o))];
+}
+
+/** For `doctor`: what the planner layer is, in one line. */
+export async function status(options) {
+  const want = requested(options);
+  if (!want) return { planner: 'none', detail: 'not requested (SIMFRAME_PLANNER is unset)' };
+  if (want !== 'apple') return { planner: 'none', detail: `no such planner backend: "${want}"` };
+  const live = await open();
+  if (!live?.ok) return { planner: 'none', detail: live?.reason ?? 'unavailable' };
+  return { planner: 'apple', detail: 'Apple Foundation Models, on-device, ranking only' };
+}
+
+/** Let a process exit without waiting on the helper. */
+export function close() {
+  try { session?.child?.kill(); } catch { /* already gone */ }
+  session = null;
+}

package/src/platform/android.js

@@ -183,7 +183,8 @@ async function resolveDevice(query, opts) {
         new Error(
           `${booted.length} emulators are running and none was named: ` +
             `${booted.map((d) => `${d.name} (${d.udid})`).join(', ')} — name one with --device, ` +
-            'or set SIMFRAME_DEVICE to pick a default for this shell',
+            'or set SIMFRAME_DEVICE to pick a default for this shell. Over MCP there is no shell: ' +
+            'pass "device" once on any call and the rest of the session remembers it',
         ),
         { ambiguous: true },
       );
@@ -827,7 +828,7 @@ async function launchApp(udid, bundleId, { args = [], env = {}, terminateFirst =
 async function terminateApp(udid, bundleId) {
   // `am` reports failure on stdout and still exits 0 — the same trap launchApp
   // and openUrl already check for. Without this, terminating a package that is
-  // not installed answered "terminated com.typo.app".
+  // not installed answered "terminated com.example.mistyped".
   const { stdout, stderr } = await adb(udid, ['shell', 'am', 'force-stop', bundleId]);
   const error = /^Error:.*$/m.exec(`${stdout}${stderr}`);
   if (error) {

package/src/platform/ios.js

@@ -87,7 +87,8 @@ async function resolveDevice(query, opts) {
         new Error(
           `${booted.length} simulators are booted and none was named: ` +
             `${booted.map((d) => `${d.name} (${d.udid})`).join(', ')} — name one with --device, ` +
-            'or set SIMFRAME_DEVICE to pick a default for this shell',
+            'or set SIMFRAME_DEVICE to pick a default for this shell. Over MCP there is no shell: ' +
+            'pass "device" once on any call and the rest of the session remembers it',
         ),
         { ambiguous: true },
       );

package/src/png.js

@@ -178,6 +178,32 @@ export function grayGrid(bmp, cols, rows) {
 }
 
 /** Nearest-neighbour scale. Only used for contact sheets, where speed beats quality. */
+/**
+ * A rectangle out of a bitmap, clamped to it.
+ *
+ * Exists because a whole screen at 1024px on the long edge cannot answer a
+ * question about one control. Reported from a real session: a selected filter
+ * chip and an unselected one are indistinguishable at that size, and selection
+ * state was the entire question the ticket turned on — so the agent shelled out
+ * to `simctl io` and PIL to crop and upscale the chip row, **for every single
+ * check**. Their estimate: six round trips.
+ *
+ * Coordinates are pixels; the caller converts from points, because only the
+ * caller knows the density it read them at.
+ */
+export function cropBitmap(bmp, x, y, width, height) {
+  const left = Math.max(0, Math.min(bmp.width - 1, Math.round(x)));
+  const top = Math.max(0, Math.min(bmp.height - 1, Math.round(y)));
+  const w = Math.max(1, Math.min(bmp.width - left, Math.round(width)));
+  const h = Math.max(1, Math.min(bmp.height - top, Math.round(height)));
+  const out = Buffer.alloc(w * h * 4);
+  for (let row = 0; row < h; row += 1) {
+    const from = ((top + row) * bmp.width + left) * 4;
+    bmp.data.copy(out, row * w * 4, from, from + w * 4);
+  }
+  return { width: w, height: h, data: out };
+}
+
 export function scaleBitmap(bmp, width, height) {
   const out = Buffer.allocUnsafe(width * height * 4);
   for (let y = 0; y < height; y++) {

package/src/refs.js

@@ -104,17 +104,46 @@ export function parseSelector(query) {
  * numbering introduces that labels do not have, and a ref resolved against the
  * wrong screen taps whatever now happens to sit at those coordinates.
  */
-export function resolveRef(udid, n, { structuralHash, layoutHash, screenKnown, tolerance = REF_TOLERANCE } = {}) {
+export function resolveRef(udid, n, { structuralHash, layoutHash, screenKnown, structuralDistance = 0, tolerance = REF_TOLERANCE } = {}) {
   const table = readRefs(udid);
   if (!table) throw new Error(`#${n} means nothing yet — read the screen first (sim_ui, or simframe ui)`);
-  const stale = (was, now) =>
-    `#${n} was numbered on a different screen (${was} → ${now}) — read the screen again before using refs`;
+  // The label this number was given to, when there is one. A stale ref is not
+  // nothing: the table records what it pointed at, which is enough for the
+  // caller to be offered the label instead of a bare refusal.
+  const labelFor = table.refs?.find((r) => r.ref === n)?.label ?? null;
+  // How long ago these numbers were handed out. Asked for by name: "refs
+  // expired (issued 4 calls ago) is actionable in a way this isn't".
+  const issued = Number.isFinite(table.at) ? ` refs were numbered ${Math.round((Date.now() - table.at) / 1000)}s ago;` : '';
+  // `staleKind` is the difference between "these numbers were drawn on a screen
+  // that has since shifted" and "you are somewhere else entirely", and only the
+  // first may be recovered by re-resolving the label the number stood for.
+  // Both wore the same flag once, and the caller re-resolved across an app
+  // switch: `#1` had been "Reminders" in Contacts, matched the status-bar
+  // back-to-app breadcrumb "• Reminders" at 0.64, and returned a tappable point
+  // in the status bar — a region the map itself refuses to offer. A refusal had
+  // become a confident wrong answer.
+  const staleError = (why, kind) => Object.assign(
+    new Error(`#${n} cannot be trusted here —${issued} ${why}. Read the screen again (sim_ui) to renumber`),
+    { staleRef: true, staleLabel: labelFor, staleKind: kind },
+  );
 
   // Structural identity first, because it is the question actually being asked:
   // is this the screen those numbers were assigned on? The caller gets it
   // cheaply — screen memory is a file read, not a perception pass.
-  if (table.structuralHash && structuralHash && table.structuralHash !== structuralHash) {
-    throw new Error(stale(table.structuralHash.slice(0, 8), structuralHash.slice(0, 8)));
+  //
+  // But only when the recall that produced it was exact. The identity arrives
+  // from `recallNearest`, which matches by layout within a tolerance so that a
+  // list with new rows stays one screen; above distance zero it is therefore a
+  // guess about *which* remembered screen this is, and a guess cannot be the
+  // sole reason to refuse. That mismatch was reported from the field as a
+  // refusal on unchanged state — the map had named the screen from the tolerant
+  // recall and printed the same header before and after, while this check read
+  // the same recall as exact and disagreed with it. Beyond distance zero the
+  // pixel backstop below is the one that decides, which is what it is for.
+  const exactRecall = structuralDistance === 0 || structuralDistance == null;
+  if (exactRecall && table.structuralHash && structuralHash && table.structuralHash !== structuralHash) {
+    throw staleError(`this is a different screen (${table.structuralHash.slice(0, 8)}`
+      + ` → ${structuralHash.slice(0, 8)})`, 'identity');
   }
   // Nothing recognises the screen we are on, so nothing can vouch for the
   // numbers. Refusing costs a re-read; guessing taps whatever is at those
@@ -128,9 +157,23 @@ export function resolveRef(udid, n, { structuralHash, layoutHash, screenKnown, t
   // other — measured: refs numbered on the springboard resolved happily on a
   // different screen because both hashes were degenerate. A hash with almost
   // no bits set is not evidence of anything.
-  if (layoutHash && table.layoutHash && informative(table.layoutHash) && informative(layoutHash)
-    && hashDistance(table.layoutHash, layoutHash) > tolerance) {
-    throw new Error(stale(table.layoutHash.slice(0, 8), layoutHash.slice(0, 8)));
+  // Reported three times in one session as `#4 was numbered on a different
+  // screen (03003714 → 03003714)` — a message that says the screen changed
+  // while showing that it did not, and left the reporter unable to tell a real
+  // move from a false positive. The cause was this branch printing eight
+  // characters of a **72-character** perceptual hash: its leading characters
+  // encode coarse structure, which is the very reason this comparison is a
+  // distance against a tolerance rather than an equality, so prefixes coincide
+  // routinely while the hashes differ. So this says the distance, and says that
+  // it is pixels rather than identity — a different thing from the branch above,
+  // which had been wearing the same sentence.
+  const drift = layoutHash && table.layoutHash && informative(table.layoutHash) && informative(layoutHash)
+    ? hashDistance(table.layoutHash, layoutHash)
+    : null;
+  if (drift != null && drift > tolerance) {
+    throw staleError(`the screen has moved too far from where these refs were numbered`
+      + ` (layout distance ${drift}, tolerance ${tolerance}) — the identity may be unchanged;`
+      + ' this is a pixel measurement, not a different screen', 'drift');
   }
   const hit = table.refs.find((r) => r.ref === n);
   if (!hit) {