simframe 0.10.0 → 0.12.0

package/src/metrics.js

@@ -24,6 +24,21 @@ export const REASONS = [
 
 export const OUTCOMES = ['resolved_locally', 'escalated_to_model', 'failed'];
 
+/**
+ * What the executor observed after a supervisor ruling — the field that makes a
+ * ruling scoreable rather than merely recorded.
+ *
+ * A ruling on its own says what the supervisor thought. Three items on the
+ * deferred list (101's p95 replay, 106's cost of the steps a `stop` skipped,
+ * 96's response variable) need what happened *next*, and until now nothing
+ * persisted a ruling at all: they went into a `supervisions` array on the
+ * result and died with the process. Three rulings had ever existed anywhere.
+ *
+ * Closed, and mandatory, for the same reason `REASONS` is: a vocabulary that
+ * admits "other" collects a pile of "other".
+ */
+export const RULING_OUTCOMES = ['recovered', 'still_failed', 'stopped', 'no_ruling'];
+
 /**
  * Which faculty would have removed this escalation.
  *
@@ -57,6 +72,7 @@ function metricPaths(udid) {
     dir,
     escalations: path.join(dir, 'escalations.jsonl'),
     flows: path.join(dir, 'flows.jsonl'),
+    supervisions: path.join(dir, 'supervisions.jsonl'),
     baselines: path.join(dir, 'baselines'),
   };
 }
@@ -109,6 +125,106 @@ export function readJsonl(file, { limit } = {}) {
 
 export const readEscalations = (udid, opts) => readJsonl(metricPaths(udid).escalations, opts);
 export const readFlows = (udid, opts) => readJsonl(metricPaths(udid).flows, opts);
+export const readSupervisions = (udid, opts) => readJsonl(metricPaths(udid).supervisions, opts);
+
+/**
+ * Write down one supervisor ruling and what came of it.
+ *
+ * The fields are chosen so the three items waiting on this can be answered
+ * **offline, from the log**, rather than by another device run:
+ *
+ * - `screen`/`edge` — which `(screen_hash, action)` the ruling was about, so
+ *   rulings can be grouped by edge the way the graph groups timings.
+ * - `p95`/`samples` — what the graph knew about that edge **at the moment of
+ *   the ruling**. Recording it now rather than looking it up later is the
+ *   difference between 101 being arithmetic and 101 being archaeology: the
+ *   graph keeps learning, so a p95 read next week is not the p95 the
+ *   supervisor was implicitly competing with.
+ * - `stillMs` — the same input the supervisor got, so a replay sees what it saw.
+ * - `expect` — the plan's own note, because a ruling made with a briefing and
+ *   one made without are not the same measurement (96's critical arm).
+ * - `outcome` — what the executor observed afterwards. Mandatory.
+ *
+ * `from` separates a deterministic rule from a model answer. Both are rulings
+ * and both have outcomes, but a comparison that mixed them would credit the
+ * model for what a two-line rule decided.
+ */
+export function recordSupervision(udid, {
+  session, index, step, edge, screen, decision, from, reason, ms,
+  stillMs, p95, samples, expect, failure, outcome,
+}) {
+  // Swallowed rather than thrown, unlike `recordEscalation`'s guard, and the
+  // difference is deliberate: this is called from inside a flow's failure
+  // handler, where a throw would turn a recoverable step failure into a crash.
+  // It still lands in `lastWriteError`, which `simframe escalations` prints.
+  if (!RULING_OUTCOMES.includes(outcome)) {
+    lastWriteError = `supervision outcome must be one of ${RULING_OUTCOMES.join('/')}, got ${JSON.stringify(outcome)}`;
+    return false;
+  }
+  return appendJsonl(metricPaths(udid).supervisions, {
+    timestamp: new Date().toISOString(),
+    session_id: session ?? SESSION_ID,
+    client: CLIENT,
+    step_index: index ?? null,
+    step: step ?? null,
+    edge: edge ?? null,
+    screen_fingerprint: screen ?? null,
+    decision,
+    from: from ?? 'model',
+    // Recorded, never presented as the ground for what happened: the supervisor
+    // has returned a correct decision with a reason citing a rule that did not
+    // apply. Keeping it is how that stays measurable instead of anecdotal.
+    reason: reason ? String(reason).slice(0, 200) : null,
+    latency_ms: Number.isFinite(ms) ? Math.round(ms) : null,
+    still_ms: Number.isFinite(stillMs) ? Math.round(stillMs) : null,
+    edge_p95_ms: Number.isFinite(p95) ? Math.round(p95) : null,
+    edge_samples: Number.isFinite(samples) ? samples : null,
+    expect: expect ? String(expect).slice(0, 200) : null,
+    failure: failure ? String(failure).slice(0, 300) : null,
+    outcome,
+  });
+}
+
+/**
+ * What the ruling log currently says, and whether it can yet answer 101.
+ *
+ * Deliberately counts rather than scores. Item 101 asks which `wait`/`retry`
+ * rulings a p95-per-edge lookup would have got right, and that is a separate
+ * piece of work; what this answers is the question that comes first and was
+ * embarrassing to get wrong once already — **is there a population to measure
+ * at all, and do its rows carry the fields the measurement needs.** `p95_known`
+ * is that readiness check: a ruling recorded on an edge the graph had never
+ * timed cannot take part in the comparison, however many of them there are.
+ */
+export function supervisionBreakdown(records) {
+  const by = (key) => {
+    const out = {};
+    for (const r of records) {
+      const k = r[key] ?? 'unknown';
+      out[k] = (out[k] ?? 0) + 1;
+    }
+    return out;
+  };
+  const matrix = {};
+  for (const r of records) {
+    const k = `${r.decision ?? 'unknown'} -> ${r.outcome ?? 'unknown'}`;
+    matrix[k] = (matrix[k] ?? 0) + 1;
+  }
+  const timed = records.filter((r) => Number.isFinite(r.edge_p95_ms));
+  const latencies = records.map((r) => r.latency_ms).filter((n) => Number.isFinite(n)).sort((a, b) => a - b);
+  return {
+    total: records.length,
+    by_decision: by('decision'),
+    by_outcome: by('outcome'),
+    by_from: by('from'),
+    decision_to_outcome: matrix,
+    // Readiness for 101, not a result for it.
+    p95_known: timed.length,
+    p95_unknown: records.length - timed.length,
+    sessions: [...new Set(records.map((r) => r.session_id).filter(Boolean))],
+    median_latency_ms: latencies.length ? latencies[latencies.length >> 1] : null,
+  };
+}
 
 /**
  * Mark an error as an escalation with a reason, at the site that knows why.
@@ -119,7 +235,7 @@ export const readFlows = (udid, opts) => readJsonl(metricPaths(udid).flows, opts
  * matching error strings at the boundary — a regexed message is a reason that
  * silently becomes "unknown" the day somebody rewords it.
  */
-export function tag(err, reason, { candidates = [], tried = [], ambiguous = false } = {}) {
+export function tag(err, reason, { candidates = [], tried = [], ambiguous = false, intent = null } = {}) {
   if (!REASONS.includes(reason)) throw new Error(`not an escalation reason: ${reason}`);
   // `ambiguous` is narrower than the reason, and that is the point. Two very
   // different failures both tag `ambiguous_intent`: the target is on screen
@@ -127,7 +243,19 @@ export function tag(err, reason, { candidates = [], tried = [], ambiguous = fals
   // thought we knew. Only the first is resolvable by *choosing*, and only the
   // first tells a waiting caller that waiting is pointless — the thing it is
   // waiting for has already arrived.
-  err.escalation = { reason, candidates, tried, ambiguous };
+  // `intent` is the goal in the caller's own words, recorded as a field rather
+  // than left in the prose of `detail`.
+  //
+  // Phase 17's go/no-go asks whether an on-device model would pick the element
+  // Claude picked, given the goal and the element list. The element list is
+  // here as `candidates` and the eventual choice is recoverable from the
+  // graph — the tap that finally worked on this screen becomes a verified edge
+  // carrying its own step. The goal was the missing third, and it was sitting
+  // inside a sentence: `"X" matches 3 things on this screen — say which…`.
+  // Regexing it back out at export time is the exact habit this file exists to
+  // avoid, and it would silently return nothing the day that sentence is
+  // reworded.
+  err.escalation = { reason, candidates, tried, ambiguous, intent };
   return err;
 }
 
@@ -252,7 +380,20 @@ export function fingerprintNow(udid, screenmap) {
  * file is committed to a public repo in summary form, and the question it has
  * to answer is "was this all one agent", which needs no identity to answer.
  */
-const SESSION_ID = `${process.pid.toString(36)}-${Date.now().toString(36)}-${Math.random().toString(36).slice(2, 8)}`;
+const SESSION_ID = process.env.SIMFRAME_SESSION
+  ? String(process.env.SIMFRAME_SESSION).slice(0, 64)
+  : `${process.pid.toString(36)}-${Date.now().toString(36)}-${Math.random().toString(36).slice(2, 8)}`;
+
+/*
+ * Minting the id from the pid was right for the MCP server, which is one
+ * long-lived process, and wrong for everything else. A CLI-driven agent starts
+ * a process per command, so it got one "session" per command: on the benchmark
+ * device, 33 session ids for 46 records, 30 of them holding a single record.
+ * `escalations --session` was therefore unable to answer the one question it
+ * exists for, and Phase 17's go/no-go step 1 — "filter to one session id" —
+ * had nothing to filter. `SIMFRAME_SESSION` lets a caller that knows it is one
+ * session say so; the per-process id stays the default.
+ */
 
 /** How this process is being used, for reading a breakdown afterwards. */
 function clientKind() {
@@ -271,6 +412,7 @@ export const clientName = () => CLIENT;
 export function recordEscalation(udid, {
   flowId = null,
   flowName = null,
+  intent = null,
   stepIndex = null,
   fingerprint = null,
   reason,
@@ -292,6 +434,9 @@ export function recordEscalation(udid, {
     client: CLIENT,
     flow_id: flowId,
     flow_name: flowName,
+    // What was asked for, in the caller's words. Ground truth for Phase 17's
+    // go/no-go, and on its own it answers "what kind of decision is costing us".
+    intent: intent ? String(intent).slice(0, 120) : null,
     step_index: stepIndex,
     screen_fingerprint: fingerprint,
     reason,

package/src/ocr.js

@@ -19,6 +19,17 @@ const BIN = path.join(BIN_DIR, 'ocr');
 
 let ready = null;
 
+/**
+ * Which recognition level to ask Vision for.
+ *
+ * `accurate` is the default and what CLAUDE.md fixes; `fast` is the other thing
+ * Vision offers. Exposed so the pair can be scored against each other instead
+ * of one of them being a constant nobody measured.
+ */
+export function level() {
+  return String(process.env.SIMFRAME_OCR ?? '').toLowerCase() === 'fast' ? 'fast' : 'accurate';
+}
+
 /** Compile once, then reuse. Recompiles only if the source is newer than the binary. */
 export async function ensureBinary() {
   if (ready) return ready;
@@ -55,7 +66,13 @@ export async function ensureBinary() {
 export async function readText(pngFile, { density = 3 } = {}) {
   const built = await ensureBinary();
   if (!built.available) throw new Error(built.reason);
-  const { stdout } = await run(built.binary, [pngFile], { timeout: 30_000, maxBuffer: 16 << 20 });
+  // The recognition level rides in the environment rather than in argv, so the
+  // Swift side keeps its one-argument contract and an older binary still works.
+  const { stdout } = await run(built.binary, [pngFile], {
+    timeout: 30_000,
+    maxBuffer: 16 << 20,
+    env: { ...process.env, SIMFRAME_OCR: level() },
+  });
   const raw = JSON.parse(stdout || '[]');
   return raw.map((r) => ({
     text: r.text,

package/src/planner.js

@@ -0,0 +1,195 @@
+/**
+ * The local planner tier: a ranker, behind a flag, that may only reorder.
+ *
+ * **Why this exists at all, given Phase 17 was a no-go.** That phase asked a
+ * local model to *choose the next element*, and the answer was that the matcher
+ * already does — 37 of 40 real decisions. This is the complement and the one
+ * case a string matcher structurally cannot do: the goal matches **nothing** on
+ * screen, and something has to guess which container leads to it. "Change my
+ * username" shares no prefix, synonym or typo distance with "Account".
+ *
+ * Measured on this machine, six hand-written cases: 5 of 6 top-1, 6 of 6 top-3,
+ * median 564 ms warm. See `docs/BENCHMARKS.md`. Against a model round trip at
+ * 10–16 s that is roughly twenty times cheaper; against the honest baseline —
+ * breadth-first ordering, which needs no model — it won five of six.
+ *
+ * **What it is allowed to do, and it is deliberately almost nothing.** It
+ * reorders a list of candidates the caller has already permitted and will try
+ * in some order regardless. It cannot invent a label, cannot choose an action,
+ * cannot see pixels, and never runs on a destructive label because the caller
+ * filtered those out before asking (`src/vocabulary.js`). If it is wrong the
+ * exploration budget simply tries the next one. That is strictly weaker
+ * authority than Phase 17 proposed, which is what makes it safe to try.
+ *
+ * **Off unless asked.** `SIMFRAME_PLANNER=apple` turns it on; anything else,
+ * or any failure at all, degrades to `null` and the caller keeps its own order.
+ * `doctor` reports which. CI runs with it off.
+ */
+import { spawn, execFile } from 'node:child_process';
+import fs from 'node:fs';
+import path from 'node:path';
+import { fileURLToPath } from 'node:url';
+import { promisify } from 'node:util';
+import * as store from './store.js';
+
+const run = promisify(execFile);
+const SOURCE = path.join(path.dirname(fileURLToPath(import.meta.url)), '..', 'native', 'rank.swift');
+const BIN = path.join(store.ROOT, 'bin', 'rank');
+
+/** Which backend the caller asked for. Absent means no local planner. */
+export function requested(options) {
+  // Per call first, then the environment, for the reason in `sensorMode`: an
+  // MCP server's environment is fixed when it spawns, so a tester could not
+  // switch backends inside one session and a round came back with one arm of
+  // its A/B unrun.
+  const raw = String(options?.planner ?? process.env.SIMFRAME_PLANNER ?? '').trim().toLowerCase();
+  if (!raw || raw === 'none' || raw === 'off' || raw === '0' || raw === 'false') return null;
+  return raw;
+}
+
+let building = null;
+
+export async function ensureBinary() {
+  if (building) return building;
+  building = (async () => {
+    try {
+      const src = fs.statSync(SOURCE).mtimeMs;
+      const bin = fs.existsSync(BIN) ? fs.statSync(BIN).mtimeMs : 0;
+      if (bin > src) return { available: true, binary: BIN };
+    } catch {
+      return { available: false, reason: 'the ranker source is missing from this install' };
+    }
+    try {
+      fs.mkdirSync(path.dirname(BIN), { recursive: true });
+      // `swiftc`, not `xcrun swiftc`: nothing above the platform boundary may
+      // name a platform tool, and the boundary test catches it. `src/ocr.js`
+      // set this precedent — a compiler is not a device tool.
+      await run('swiftc', ['-O', SOURCE, '-o', BIN], { timeout: 180_000 });
+      return { available: true, binary: BIN };
+    } catch (err) {
+      building = null; // let a later call retry once a toolchain is present
+      return {
+        available: false,
+        reason: err.code === 'ENOENT'
+          ? 'swiftc is not installed, so the local planner cannot be built (install Xcode command line tools)'
+          : `could not build the local planner: ${String(err.message).split('\n')[0]}`,
+      };
+    }
+  })();
+  return building;
+}
+
+let session = null;
+
+/** Start the helper once and keep it, because the first answer pays model load. */
+async function open() {
+  if (session) return session;
+  const built = await ensureBinary();
+  if (!built.available) return { ok: false, reason: built.reason };
+  session = await new Promise((resolve) => {
+    const child = spawn(built.binary, [], { stdio: ['pipe', 'pipe', 'ignore'] });
+    // Deliberately NOT unref'd. Unreffing the child's stdout unreferences the
+    // very pipe every request waits on, so the process exited silently in the
+    // middle of an await — a flow that printed nothing and returned 0. The
+    // helper is closed explicitly instead, by whoever opened it.
+    let buffer = '';
+    const waiters = [];
+    let settled = false;
+    const fail = (reason) => {
+      if (!settled) { settled = true; resolve({ ok: false, reason }); }
+      while (waiters.length) waiters.shift()(null);
+    };
+    child.on('error', (err) => fail(`the local planner would not start: ${err.message}`));
+    child.on('exit', () => { session = null; fail('the local planner exited'); });
+    child.stdout.on('data', (chunk) => {
+      buffer += chunk;
+      let i = buffer.indexOf('\n');
+      while (i >= 0) {
+        const line = buffer.slice(0, i).trim();
+        buffer = buffer.slice(i + 1);
+        i = buffer.indexOf('\n');
+        if (!line) continue;
+        let msg;
+        try { msg = JSON.parse(line); } catch { continue; }
+        if (!settled) {
+          settled = true;
+          if (msg.ready) resolve({ ok: true, child, waiters });
+          else resolve({ ok: false, reason: msg.unavailable ?? 'the local planner did not become ready' });
+          continue;
+        }
+        const next = waiters.shift();
+        if (next) next(msg);
+      }
+    });
+  });
+  return session;
+}
+
+/**
+ * Reorder `options` by which is likeliest to lead to `goal`.
+ *
+ * @returns {Promise<string[]|null>} the caller's own order is correct when this
+ *   is null, which is every failure mode: flag off, no model, a timeout, a
+ *   parse problem, a paraphrasing answer. Never throws.
+ */
+export async function rank(goal, options, { timeoutMs = 3000, deviceOptions } = {}) {
+  if (!requested(deviceOptions)) return null;
+  if (!goal || !Array.isArray(options) || options.length < 2) return null;
+  let live;
+  try {
+    live = await open();
+  } catch {
+    return null;
+  }
+  if (!live?.ok) return null;
+  const answer = await new Promise((resolve) => {
+    // A timed-out waiter has to be *retired*, not merely resolved. Leaving it in
+    // the queue meant the next answer went to it instead of to the next asker,
+    // and every call after that was off by one — which showed up as an
+    // exploration run that never finished rather than as an error.
+    let done = false;
+    const waiter = (msg) => {
+      if (done) return;
+      done = true;
+      clearTimeout(timer);
+      resolve(msg);
+    };
+    const timer = setTimeout(() => {
+      if (done) return;
+      done = true;
+      const i = live.waiters.indexOf(waiter);
+      if (i >= 0) live.waiters.splice(i, 1);
+      resolve(null);
+    }, timeoutMs);
+    live.waiters.push(waiter);
+    try {
+      live.child.stdin.write(`${JSON.stringify({ goal: String(goal), options })}\n`);
+    } catch {
+      clearTimeout(timer);
+      resolve(null);
+    }
+  });
+  if (!answer?.order?.length) return null;
+  // It often returns a subset, so its order comes first and ours fills the tail.
+  // Trusting it to be exhaustive would silently drop candidates the budget was
+  // going to try.
+  const ranked = answer.order.filter((label) => options.includes(label));
+  const seen = new Set(ranked);
+  return [...ranked, ...options.filter((o) => !seen.has(o))];
+}
+
+/** For `doctor`: what the planner layer is, in one line. */
+export async function status(options) {
+  const want = requested(options);
+  if (!want) return { planner: 'none', detail: 'not requested (SIMFRAME_PLANNER is unset)' };
+  if (want !== 'apple') return { planner: 'none', detail: `no such planner backend: "${want}"` };
+  const live = await open();
+  if (!live?.ok) return { planner: 'none', detail: live?.reason ?? 'unavailable' };
+  return { planner: 'apple', detail: 'Apple Foundation Models, on-device, ranking only' };
+}
+
+/** Let a process exit without waiting on the helper. */
+export function close() {
+  try { session?.child?.kill(); } catch { /* already gone */ }
+  session = null;
+}

package/src/platform/android.js

@@ -183,7 +183,8 @@ async function resolveDevice(query, opts) {
         new Error(
           `${booted.length} emulators are running and none was named: ` +
             `${booted.map((d) => `${d.name} (${d.udid})`).join(', ')} — name one with --device, ` +
-            'or set SIMFRAME_DEVICE to pick a default for this shell',
+            'or set SIMFRAME_DEVICE to pick a default for this shell. Over MCP there is no shell: ' +
+            'pass "device" once on any call and the rest of the session remembers it',
         ),
         { ambiguous: true },
       );
@@ -995,6 +996,28 @@ function capabilities() {
   };
 }
 
+/**
+ * Not implemented here, and it says so in its own vocabulary.
+ *
+ * `revive` exists for a display that has stopped rendering — a CoreSimulator
+ * pathology. An emulator's failure modes are its own (the console port going
+ * away, adb losing the device) and the remedy is not the same sequence, so
+ * borrowing the other platform's answer would be the mistake `doctor` made when
+ * it reported "input driver: idb" for an emulator: a claim about a tool that has
+ * never spoken to an Android device.
+ *
+ * When an emulator wedge is characterised rather than guessed at, this becomes
+ * a real implementation. Until then the honest answer is that this backend does
+ * not have the layer.
+ */
+async function restartDevice(serial) {
+  throw new Error(
+    `simframe cannot yet restart an emulator (${serial}) — that remedy is written for a`
+    + ' simulator display that stopped rendering, and an emulator fails differently.'
+    + ' Restart it from Android Studio, or `adb -s <serial> emu kill` and relaunch.',
+  );
+}
+
 /** @type {import('./index.js').Platform} */
 export const platform = {
   id: 'android',
@@ -1010,6 +1033,7 @@ export const platform = {
   launchApp,
   terminateApp,
   openUrl,
+  restartDevice,
   setPermission,
   setPasteboard,
   getPasteboard,

package/src/platform/index.js

@@ -61,7 +61,7 @@ export const PLATFORM_SURFACE = Object.freeze([
   'id', 'deviceNoun',
   'listDevices', 'bootedDevices', 'resolveDevice', 'isBootedSync', 'ownsUdid',
   'geometry', 'inputDriver',
-  'screenshot', 'launchApp', 'terminateApp', 'openUrl',
+  'screenshot', 'launchApp', 'terminateApp', 'openUrl', 'restartDevice',
   'setPermission', 'setPasteboard', 'permissionServices', 'capabilities', 'toolchain',
   'bootedAt',
 ]);
@@ -260,6 +260,16 @@ export const inputDriverFor = (udid) => platformFor(udid).inputDriver(udid);
  */
 export const capabilitiesFor = (udid) => platformFor(udid).capabilities();
 
+/**
+ * Power-cycle a device, on the backend that owns it.
+ *
+ * Reached only from `simframe revive`, never from the capture loop: the loop
+ * detects a stalled display and reports it, and restarting is the operator's
+ * call. A backend that does not have this remedy throws in its own terms rather
+ * than borrowing the other's.
+ */
+export const restartDevice = (udid) => platformFor(udid).restartDevice(udid);
+
 /** What `simframe doctor` should check: each registered backend's own toolchain. */
 export function toolchainChecks() {
   return backends().flatMap((backend) => backend.toolchain());

package/src/platform/ios.js

@@ -87,7 +87,8 @@ async function resolveDevice(query, opts) {
         new Error(
           `${booted.length} simulators are booted and none was named: ` +
             `${booted.map((d) => `${d.name} (${d.udid})`).join(', ')} — name one with --device, ` +
-            'or set SIMFRAME_DEVICE to pick a default for this shell',
+            'or set SIMFRAME_DEVICE to pick a default for this shell. Over MCP there is no shell: ' +
+            'pass "device" once on any call and the rest of the session remembers it',
         ),
         { ambiguous: true },
       );
@@ -188,6 +189,28 @@ async function openUrl(udid, url) {
   await run('xcrun', ['simctl', 'openurl', udid, url], { timeout: 20_000 });
 }
 
+/**
+ * Shut a device down and bring it back, waiting for the boot to finish.
+ *
+ * The remedy for a display that has stopped rendering, which the capture loop
+ * can detect and must not perform: it reports `stalled` and stops, because a
+ * capture loop that rebooted the device it was watching would be a tool
+ * reaching for the mains when a reading looks wrong. This is the operator's
+ * decision, reached by `simframe revive`.
+ *
+ * `bootstatus -b` and not `boot`, for the reason it is used in CI: `boot`
+ * returns before the device is usable, and everything downstream then races the
+ * boot. Timeboxed generously — a cold boot on a busy machine is slow, and a
+ * boot that never finishes should fail here with a reason rather than as a
+ * puzzle further down.
+ */
+async function restartDevice(udid) {
+  // Tolerated: a device that is already off cannot be shut down, and that is
+  // the state this command is most often reached from.
+  await run('xcrun', ['simctl', 'shutdown', udid], { timeout: 60_000 }).catch(() => null);
+  await run('xcrun', ['simctl', 'bootstatus', udid, '-b'], { timeout: 240_000 });
+}
+
 const PERMISSION_SERVICES = [
   'all', 'calendar', 'contacts-limited', 'contacts', 'location', 'location-always',
   'photos-add', 'photos', 'media-library', 'microphone', 'motion', 'reminders', 'siri',
@@ -346,6 +369,7 @@ export const platform = {
   launchApp,
   terminateApp,
   openUrl,
+  restartDevice,
   setPermission,
   setPasteboard,
   permissionServices: () => PERMISSION_SERVICES,

package/src/png.js

@@ -178,6 +178,32 @@ export function grayGrid(bmp, cols, rows) {
 }
 
 /** Nearest-neighbour scale. Only used for contact sheets, where speed beats quality. */
+/**
+ * A rectangle out of a bitmap, clamped to it.
+ *
+ * Exists because a whole screen at 1024px on the long edge cannot answer a
+ * question about one control. Reported from a real session: a selected filter
+ * chip and an unselected one are indistinguishable at that size, and selection
+ * state was the entire question the ticket turned on — so the agent shelled out
+ * to `simctl io` and PIL to crop and upscale the chip row, **for every single
+ * check**. Their estimate: six round trips.
+ *
+ * Coordinates are pixels; the caller converts from points, because only the
+ * caller knows the density it read them at.
+ */
+export function cropBitmap(bmp, x, y, width, height) {
+  const left = Math.max(0, Math.min(bmp.width - 1, Math.round(x)));
+  const top = Math.max(0, Math.min(bmp.height - 1, Math.round(y)));
+  const w = Math.max(1, Math.min(bmp.width - left, Math.round(width)));
+  const h = Math.max(1, Math.min(bmp.height - top, Math.round(height)));
+  const out = Buffer.alloc(w * h * 4);
+  for (let row = 0; row < h; row += 1) {
+    const from = ((top + row) * bmp.width + left) * 4;
+    bmp.data.copy(out, row * w * 4, from, from + w * 4);
+  }
+  return { width: w, height: h, data: out };
+}
+
 export function scaleBitmap(bmp, width, height) {
   const out = Buffer.allocUnsafe(width * height * 4);
   for (let y = 0; y < height; y++) {

package/src/refs.js

@@ -104,17 +104,46 @@ export function parseSelector(query) {
  * numbering introduces that labels do not have, and a ref resolved against the
  * wrong screen taps whatever now happens to sit at those coordinates.
  */
-export function resolveRef(udid, n, { structuralHash, layoutHash, screenKnown, tolerance = REF_TOLERANCE } = {}) {
+export function resolveRef(udid, n, { structuralHash, layoutHash, screenKnown, structuralDistance = 0, tolerance = REF_TOLERANCE } = {}) {
   const table = readRefs(udid);
   if (!table) throw new Error(`#${n} means nothing yet — read the screen first (sim_ui, or simframe ui)`);
-  const stale = (was, now) =>
-    `#${n} was numbered on a different screen (${was} → ${now}) — read the screen again before using refs`;
+  // The label this number was given to, when there is one. A stale ref is not
+  // nothing: the table records what it pointed at, which is enough for the
+  // caller to be offered the label instead of a bare refusal.
+  const labelFor = table.refs?.find((r) => r.ref === n)?.label ?? null;
+  // How long ago these numbers were handed out. Asked for by name: "refs
+  // expired (issued 4 calls ago) is actionable in a way this isn't".
+  const issued = Number.isFinite(table.at) ? ` refs were numbered ${Math.round((Date.now() - table.at) / 1000)}s ago;` : '';
+  // `staleKind` is the difference between "these numbers were drawn on a screen
+  // that has since shifted" and "you are somewhere else entirely", and only the
+  // first may be recovered by re-resolving the label the number stood for.
+  // Both wore the same flag once, and the caller re-resolved across an app
+  // switch: `#1` had been "Reminders" in Contacts, matched the status-bar
+  // back-to-app breadcrumb "• Reminders" at 0.64, and returned a tappable point
+  // in the status bar — a region the map itself refuses to offer. A refusal had
+  // become a confident wrong answer.
+  const staleError = (why, kind) => Object.assign(
+    new Error(`#${n} cannot be trusted here —${issued} ${why}. Read the screen again (sim_ui) to renumber`),
+    { staleRef: true, staleLabel: labelFor, staleKind: kind },
+  );
 
   // Structural identity first, because it is the question actually being asked:
   // is this the screen those numbers were assigned on? The caller gets it
   // cheaply — screen memory is a file read, not a perception pass.
-  if (table.structuralHash && structuralHash && table.structuralHash !== structuralHash) {
-    throw new Error(stale(table.structuralHash.slice(0, 8), structuralHash.slice(0, 8)));
+  //
+  // But only when the recall that produced it was exact. The identity arrives
+  // from `recallNearest`, which matches by layout within a tolerance so that a
+  // list with new rows stays one screen; above distance zero it is therefore a
+  // guess about *which* remembered screen this is, and a guess cannot be the
+  // sole reason to refuse. That mismatch was reported from the field as a
+  // refusal on unchanged state — the map had named the screen from the tolerant
+  // recall and printed the same header before and after, while this check read
+  // the same recall as exact and disagreed with it. Beyond distance zero the
+  // pixel backstop below is the one that decides, which is what it is for.
+  const exactRecall = structuralDistance === 0 || structuralDistance == null;
+  if (exactRecall && table.structuralHash && structuralHash && table.structuralHash !== structuralHash) {
+    throw staleError(`this is a different screen (${table.structuralHash.slice(0, 8)}`
+      + ` → ${structuralHash.slice(0, 8)})`, 'identity');
   }
   // Nothing recognises the screen we are on, so nothing can vouch for the
   // numbers. Refusing costs a re-read; guessing taps whatever is at those
@@ -128,9 +157,23 @@ export function resolveRef(udid, n, { structuralHash, layoutHash, screenKnown, t
   // other — measured: refs numbered on the springboard resolved happily on a
   // different screen because both hashes were degenerate. A hash with almost
   // no bits set is not evidence of anything.
-  if (layoutHash && table.layoutHash && informative(table.layoutHash) && informative(layoutHash)
-    && hashDistance(table.layoutHash, layoutHash) > tolerance) {
-    throw new Error(stale(table.layoutHash.slice(0, 8), layoutHash.slice(0, 8)));
+  // Reported three times in one session as `#4 was numbered on a different
+  // screen (03003714 → 03003714)` — a message that says the screen changed
+  // while showing that it did not, and left the reporter unable to tell a real
+  // move from a false positive. The cause was this branch printing eight
+  // characters of a **72-character** perceptual hash: its leading characters
+  // encode coarse structure, which is the very reason this comparison is a
+  // distance against a tolerance rather than an equality, so prefixes coincide
+  // routinely while the hashes differ. So this says the distance, and says that
+  // it is pixels rather than identity — a different thing from the branch above,
+  // which had been wearing the same sentence.
+  const drift = layoutHash && table.layoutHash && informative(table.layoutHash) && informative(layoutHash)
+    ? hashDistance(table.layoutHash, layoutHash)
+    : null;
+  if (drift != null && drift > tolerance) {
+    throw staleError(`the screen has moved too far from where these refs were numbered`
+      + ` (layout distance ${drift}, tolerance ${tolerance}) — the identity may be unchanged;`
+      + ' this is a pixel measurement, not a different screen', 'drift');
   }
   const hit = table.refs.find((r) => r.ref === n);
   if (!hit) {