simframe 0.10.0 → 0.11.0

package/src/cli.js

@@ -47,6 +47,10 @@ const USAGE = `simframe — always-warm iOS Simulator frames
   simframe baseline list             recorded runs per flow, and what is committed
   simframe hpi     [device]          Human Parity Index, per flow and overall
   simframe escalations [device]      why simframe handed decisions back, by reason
+                                     (--session=<id> narrows to one agent; the
+                                     ids are listed in the output. SIMFRAME_SESSION
+                                     names one, but only at process start — an
+                                     already-running MCP server cannot pick it up)
   simframe devices                   list simulators
   simframe doctor                    check that this machine can capture
                                      (--strict, or SIMFRAME_STRICT=1, makes any
@@ -162,6 +166,25 @@ const num = (v, fallback) => (v == null ? fallback : Number(v));
  * script that has to parse one command's prose and another's JSON will parse
  * the prose wrong exactly once and then be trusted anyway.
  */
+/**
+ * The machine-readable half of a failure.
+ *
+ * A refusal recognisable only by reading its prose is a refusal nobody can
+ * depend on. Our own CI asserted the stale-ref guard by matching three
+ * phrasings and went red when a fourth arrived — a *better* one, naming the
+ * label the number stood for. The sentence is for a person; these fields are
+ * the contract, and they live in one function because `find` reports its own
+ * failures and the top-level handler reports the rest.
+ */
+function failureJson(err) {
+  return {
+    ok: false,
+    error: err.message,
+    reason: metrics.escalationOf(err)?.reason ?? null,
+    ...(err.staleRef ? { staleRef: true, staleKind: err.staleKind ?? null, staleLabel: err.staleLabel ?? null } : {}),
+  };
+}
+
 function emit(flags, json, lines) {
   if (flags.json) {
     console.log(JSON.stringify(json, null, 2));
@@ -171,11 +194,26 @@ function emit(flags, json, lines) {
   if (body != null) console.log(Array.isArray(body) ? body.filter((l) => l != null).join('\n') : body);
 }
 
-/** The end-state screen map, rendered from a reading the flow already took. */
-async function mapText(device, options, identity) {
+/**
+ * The end-state screen map, re-read rather than recalled, with its hint.
+ *
+ * Both halves were reported against Phase 11.5 and both were right. The map was
+ * rendered from whatever reading the flow already had, which is memory-first —
+ * so a trailing map could describe the screen as it was seconds ago, and the
+ * remedy in practice was a `ui --refresh` after nearly every call, which is a
+ * whole extra turn to save a few hundred milliseconds. Wrong way round.
+ *
+ * And the hint was only ever printed by the MCP server, so no CLI user could
+ * see it and no CLI run could test it.
+ */
+async function mapText(device, options, identity, { flowOk = true, escalated = false, refresh = true } = {}) {
   try {
-    const m = await view.screenMap(device, { options, identity: identity?.entry ? identity : undefined });
-    return m.text;
+    const m = await view.screenMap(device, {
+      options,
+      refresh,
+      identity: refresh ? undefined : (identity?.entry ? identity : undefined),
+    });
+    return `${m.text}\n${view.hintFor(m, { flowOk, escalated })}`;
   } catch (err) {
     return `(could not read the screen: ${err.message})`;
   }
@@ -231,6 +269,11 @@ async function main() {
   if (flags.maxDim) options.maxDim = num(flags.maxDim);
   if (flags.ringSize) options.ringSize = num(flags.ringSize);
   if (flags.engine) options.engine = String(flags.engine);
+  // Per-command overrides for the two experiment knobs, so an A/B is an
+  // argument rather than a restart. `--sensor=ax-first`, `--planner=apple`.
+  if (flags.sensor) options.sensor = String(flags.sensor);
+  if (flags.planner) options.planner = String(flags.planner);
+  if (flags.supervisor) options.supervisor = String(flags.supervisor);
 
   switch (command) {
     case undefined:
@@ -533,6 +576,7 @@ async function main() {
         all: Boolean(flags.all),
         refresh: Boolean(flags.refresh),
       });
+      m.text = `${m.text}\n${view.hintFor(m)}`;
       emit(
         flags,
         {
@@ -585,6 +629,7 @@ async function main() {
         stableMs: num(flags.stableMs, 500),
         timeoutMs: num(flags.timeoutMs, 8000),
         continueOnError: Boolean(flags.continueOnError),
+        supervise: flags.supervise ? String(flags.supervise) : undefined,
         options,
       });
       const saved = flags.save
@@ -592,7 +637,16 @@ async function main() {
         : null;
       // `--map=false` arrives as the string "false"; `--no-map` as true.
       const wantMap = !flags.json && flags.noMap !== true && String(flags.map ?? 'true') !== 'false';
-      const map = wantMap ? await mapText(flags.device, options, res.endScreen) : null;
+      // Every local ruling, so a wrong one is correctable rather than
+      // mysterious — and the model's stated reason is shown as its claim.
+      for (const s_ of res.supervisions ?? []) {
+        console.log(`supervisor at step ${s_.index}: ${s_.decision} — ${s_.outcome}`
+          + (s_.reason ? ` (it said: "${s_.reason}")` : ''));
+      }
+      const escalated = (res.results ?? []).some((r) => metrics.ESCALATING_VERDICTS.has(r.verification?.verdict));
+      const map = wantMap
+        ? await mapText(flags.device, options, res.endScreen, { flowOk: res.ok, escalated })
+        : null;
       emit(
         flags,
         {
@@ -812,7 +866,7 @@ async function main() {
           ],
         );
       } catch (err) {
-        emit(flags, { ok: false, error: err.message }, err.message);
+        emit(flags, failureJson(err), err.message);
         process.exitCode = 1;
       }
       return;
@@ -1047,6 +1101,10 @@ async function main() {
         json: Boolean(flags.json),
         strict: Boolean(flags.strict) || process.env.SIMFRAME_STRICT === '1',
         device: flags.device,
+        // So `doctor --sensor=ax-first --planner=apple` reports the mode the
+        // caller is about to use, not the one the environment happens to hold.
+        // Confirming the mode before a run is the whole reason to read this.
+        options,
       });
       return;
     }
@@ -1123,7 +1181,7 @@ async function blackScreenProbe(udid) {
   }
 }
 
-async function doctor({ json = false, strict = false, device } = {}) {
+async function doctor({ json = false, strict = false, device, options = {} } = {}) {
   const checks = [];
   // `level` is 'ok' | 'warn' | 'fail'. A warn means it works but not the way it
   // should — the exact state that used to be invisible.
@@ -1172,6 +1230,55 @@ async function doctor({ json = false, strict = false, device } = {}) {
     add('on-device OCR', 'warn', err.message, { key: 'ocr.available', value: false });
   }
 
+  // Which sensors a read asks for, and what the recognition-level flag can and
+  // cannot reach. Stated because `SIMFRAME_OCR` looks like it configures the
+  // OCR everyone uses and does not: the daemon reads text in-process off the
+  // framebuffer and owns its own recognition level, so the flag only reaches
+  // the no-daemon fallback in `native/ocr.swift`.
+  try {
+    const api = await import('./index.js');
+    const ocrMod = await import('./ocr.js');
+    const mode = api.sensorMode(options);
+    add('sensor mode', 'ok',
+      mode === 'ax-first'
+        ? 'ax-first — the tree alone (~50ms), paying for OCR only when a resolve fails'
+        : 'full — accessibility and OCR fused on every read (~164ms)',
+      { key: 'sensor.mode', value: mode });
+    add('OCR level', 'ok', `${ocrMod.level()} (fallback helper only; the daemon owns its own)`, {
+      key: 'ocr.level',
+      value: ocrMod.level(),
+    });
+  } catch { /* reported by the layers above */ }
+
+  // The local supervisor. Behind the hands and in front of the reasoner, and
+  // able to say only wait/retry/stop.
+  try {
+    const supervisor = await import('./supervisor.js');
+    const st = await supervisor.status(options);
+    add('local supervisor', 'ok', `${st.supervisor} — ${st.detail}`, {
+      key: 'supervisor.backend',
+      value: st.supervisor,
+    });
+    supervisor.close();
+  } catch (err) {
+    add('local supervisor', 'ok', `none — ${err.message}`, { key: 'supervisor.backend', value: 'none' });
+  }
+
+  // The local planner tier. `none` is the normal answer and not a fault: it is
+  // off unless SIMFRAME_PLANNER asks for it, and it only ever reorders
+  // candidates that exploration was going to try anyway.
+  try {
+    const planner = await import('./planner.js');
+    const st = await planner.status(options);
+    add('local planner', 'ok', `${st.planner} — ${st.detail}`, {
+      key: 'planner.backend',
+      value: st.planner,
+    });
+    planner.close();
+  } catch (err) {
+    add('local planner', 'ok', `none — ${err.message}`, { key: 'planner.backend', value: 'none' });
+  }
+
   try {
     let booted = await bootedDevices();
     // Respect --device. Without this, doctor reports on every booted simulator,
@@ -1398,13 +1505,28 @@ async function doctor({ json = false, strict = false, device } = {}) {
   process.exitCode = failed.length || (strict && warned.length) ? 1 : 0;
 }
 
-main().catch((err) => {
+// A long-lived local helper must not decide when the CLI exits. It is closed
+// after every command, whether or not one was ever started — `close()` on an
+// unopened planner is a no-op, and leaving it open made a finished flow hang.
+const closeHelpers = async () => {
+  try {
+    const planner = await import('./planner.js');
+    planner.close();
+    const supervisor = await import('./supervisor.js');
+    supervisor.close();
+  } catch { /* nothing to close */ }
+};
+
+main().then(closeHelpers, async (err) => {
+  await closeHelpers();
+  throw err;
+}).catch((err) => {
   // A caller that asked for JSON gets JSON, failures included. Printing prose
   // here handed `JSON.parse` a SyntaxError instead of a reason, so a script
   // could not tell "the daemon lost the display" from "simframe is broken" —
   // which is the whole point of a machine-readable interface.
   if (process.argv.includes('--json')) {
-    process.stdout.write(`${JSON.stringify({ ok: false, error: err.message }, null, 2)}\n`);
+    process.stdout.write(`${JSON.stringify(failureJson(err), null, 2)}\n`);
   } else {
     process.stderr.write(`simframe: ${err.message}\n`);
   }

package/src/control.js

@@ -65,6 +65,7 @@ export const swipe = (udid, from, to, opts = {}) =>
 export const type = (udid, text) => request(udid, { action: 'type', text });
 export const paste = (udid, text) => request(udid, { action: 'paste', text });
 export const press = (udid, button) => request(udid, { action: 'press', button });
+export const key = (udid, usage, modifiers = []) => request(udid, { action: 'key', usage, modifiers });
 export const status = (udid) => request(udid, { action: 'status' });
 export const resetInput = (udid) => request(udid, { action: 'resetInput' });
 export const longPress = (udid, x, y, opts = {}) => request(udid, { action: 'longPress', x, y, ...opts });

package/src/fingerprint.js

@@ -30,8 +30,26 @@ import * as regions from './regions.js';
  *     list screen. Same rules, different input, therefore different hashes —
  *     and a stored hash that can never match again is the quietest kind of
  *     wrong, which is what this counter exists to prevent.
+ * 5 — a phantom keyboard was deleting screens' content from their identity. A
+ *     dozen short text rows of uniform height stacked low on a read-only
+ *     summary satisfied every size-and-uniformity test for a keyboard, and
+ *     `tokens` discards everything below `keyboardTop` — so two screens of one
+ *     wizard, sharing a nav title and a step indicator, collapsed onto a single
+ *     hash. `detectKeyboardTop` now requires the small uniform boxes to be
+ *     key-shaped. Every screen with content in its lower half hashes
+ *     differently, so the stored graph and maps must go.
+ * 6 — the opposite half of the same bug, and it took a recorded screen to see.
+ *     `KEYBOARD_MIN_FRACTION` is a *detection window*, not a keyboard's height,
+ *     and its edge was being used as the boundary — so on an iPhone 17 Pro with
+ *     the software keyboard up, the window starts at y=629 while the `q`–`p`
+ *     row's frame top is **590**, and that whole row fell outside it. Ten
+ *     keyboard keys were reported as page content and counted into the screen's
+ *     identity, in the same map that said `keyboard up`. The boundary now
+ *     extends upward while the rows above keep being key-shaped, which page
+ *     content is not. Any screen fingerprinted with a keyboard up hashes
+ *     differently, so the stored graph and maps must go again.
  */
-export const TOKEN_RULES_VERSION = 4;
+export const TOKEN_RULES_VERSION = 6;
 
 /** Frames are quantised to this, so sub-pixel drift and a nudged row do not matter. */
 export const GRID = 24;

package/src/graph.js

@@ -295,6 +295,38 @@ function replayable(step) {
   return step.launch != null || step.openUrl != null ? null : step;
 }
 
+/**
+ * What has worked from this screen before, in the caller's own vocabulary.
+ *
+ * The graph has always known this and never said it. A map reported
+ * `(known, 3 known exits)` — the *count* — so an agent on a screen simframe had
+ * driven successfully six times still had to read it to learn what was tappable.
+ * Measured across two peer rounds: a flow whose steps were known in advance ran
+ * 16 steps in **one** call, and the same agent on a screen the graph also knew
+ * but whose labels it did not spent 25 calls on 31 steps. The difference was not
+ * perception. It was whether a plan existed before execution started.
+ *
+ * Ordered by how often each has worked, because that is the order an agent
+ * should try them in.
+ */
+export function exitsOf(node, { limit = 8 } = {}) {
+  return (node?.edges ?? [])
+    .map((e) => ({
+      action: e.step?.action ?? (e.action ?? '').split(':')[0] ?? 'tap',
+      label: e.step?.value ?? e.step?.target ?? e.step?.label ?? e.step?.into ?? null,
+      to: e.to ?? null,
+      count: e.count ?? 0,
+      kind: e.kind ?? null,
+    }))
+    // A `#13` was a ref on the screen it was typed on and means nothing on the
+    // next visit; a raw coordinate is not a name either. Neither is reusable
+    // vocabulary, which is the whole point of this list.
+    .filter((e) => e.label != null && !/^#\d+$/.test(String(e.label).trim())
+      && !/^@?-?\d+\s*,\s*-?\d+$/.test(String(e.label).trim()))
+    .sort((a, b) => b.count - a.count)
+    .slice(0, limit);
+}
+
 /**
  * What to call this screen, for a human typing `goto`.
  *
@@ -756,12 +788,36 @@ export const VERDICTS = ['ok', 'no-visible-change', 'unexpected-screen', 'unveri
  * app and the verdict still said `unexpected-screen`, because the verdict never
  * asked the graph.
  */
+/**
+ * Are these two readings the same screen?
+ *
+ * `nearestScreen` has always had a token-similarity tolerance, precisely so
+ * that a screen whose *content* differs — a list with different rows, a form
+ * showing a different record — still resolves to the screen it is. The
+ * verification path threw that away: it passed bare hash strings, and a string
+ * carries no tokens, so only an exact hash could ever match.
+ *
+ * The cost was measured. `unexpected-screen` fired three times in one reported
+ * run and was wrong all three; two were this — the tester picked a different
+ * asset than earlier runs had, so the content differed, so the hash differed,
+ * so a correct navigation was called a wrong turn. Their conclusion: *"this
+ * will fire on every run that varies its test data — i.e. every useful run."*
+ * And because a failed step abandons the rest of its batch, each false alarm
+ * costs a round trip, which is the thing the whole design is trying to buy.
+ *
+ * So pass the reading, not just its name: `{hash, tokens}` lets the tolerance
+ * that already exists do its job. A string still works and still means "exact
+ * match only", which is right for a stored prediction that has no tokens.
+ */
 function sameScreen(udid, a, b) {
-  if (!a || !b) return false;
-  if (a === b) return true;
+  const hashOf = (v) => (typeof v === 'string' ? v : v?.hash);
+  const ha = hashOf(a);
+  const hb = hashOf(b);
+  if (!ha || !hb) return false;
+  if (ha === hb) return true;
   if (!udid) return false;
-  const nodeA = nearestScreen(udid, a)?.node;
-  const nodeB = nearestScreen(udid, b)?.node;
+  const nodeA = nearestScreen(udid, typeof a === 'string' ? a : { hash: ha, tokens: a?.tokens })?.node;
+  const nodeB = nearestScreen(udid, typeof b === 'string' ? b : { hash: hb, tokens: b?.tokens })?.node;
   return Boolean(nodeA && nodeB && nodeA.hash === nodeB.hash);
 }
 
@@ -777,9 +833,35 @@ function sameScreen(udid, a, b) {
  */
 export const CONFIDENT_OBSERVATIONS = 2;
 
-export function verdict({ udid, prediction, before, after, kind }) {
-  if (!before || !after) return { verdict: 'unverified', detail: 'no state to compare' };
-  const moved = before !== after;
+/**
+ * Actions whose correct outcome is that the screen stays where it is.
+ *
+ * Typing into a field does not navigate, so screen-identity movement cannot
+ * say whether it worked — and answering with `no-visible-change` was actively
+ * harmful three ways. It printed a verdict contradicting the wait's own
+ * observation on the same line (`[a small change, in one region only] …
+ * [no-visible-change]`, both true, of different questions). It is an escalating
+ * verdict, so a clean flow that typed anything told the caller to stop and
+ * think. And it invited a re-type, which doubles a field that cannot be
+ * cleared.
+ *
+ * These steps are verified by reading the field back instead — see
+ * `fieldContents` in actions.js.
+ */
+export const STAYS_ON_SCREEN = new Set(['type', 'paste', 'key']);
+
+export function verdict({ udid, prediction, before, after, kind, action }) {
+  // `before`/`after` may be a hash or a whole reading. A reading carries its
+  // tokens, which is what lets a content-varied screen still be recognised as
+  // the screen it is.
+  const hashOf = (v) => (typeof v === 'string' ? v : v?.hash);
+  const beforeHash = hashOf(before);
+  const afterHash = hashOf(after);
+  if (!beforeHash || !afterHash) return { verdict: 'unverified', detail: 'no state to compare' };
+  const moved = beforeHash !== afterHash;
+  if (STAYS_ON_SCREEN.has(action) && !moved) {
+    return { verdict: 'ok', detail: 'the screen was not expected to change, and did not' };
+  }
   if (!prediction) {
     if (!moved) return { verdict: 'no-visible-change', detail: 'the screen did not change, and nothing predicted it would' };
     return { verdict: 'unverified', detail: 'this action has not been seen on this screen before' };