simframe 0.14.1 → 0.14.3

package/scripts/eval-fingerprint.mjs

@@ -20,6 +20,7 @@
 import fs from 'node:fs';
 import * as actions from '../src/actions.js';
 import * as fingerprint from '../src/fingerprint.js';
+import { classifyStray } from './classify-stray.mjs';
 import * as graph from '../src/graph.js';
 import * as api from '../src/index.js';
 
@@ -152,12 +153,24 @@ const STALE_READ_RETRIES = 3;
  * Backed off rather than fixed, because the thing being waited for is a screen
  * finishing its draw, and the runner that needs this is the slow one.
  */
+/**
+ * How old a frame may be and still be a reading of NOW.
+ *
+ * `FRAME_IS_CURRENT_MS` in src/index.js is 2500 for the same question on the
+ * settle path. Doubled here because this harness deliberately reads cold and a
+ * hosted runner's capture p50 is 2.5x this laptop's — but bounded, because a
+ * 38-second-old frame is a different screen, not a slow one.
+ */
+const FRAME_TOO_OLD_MS = 5000;
+
 const SPARSE_READ_RETRIES = 3;
 const SPARSE_READ_WAIT_MS = 1500;
 const STALE_READ_WAIT_MS = 1000;
 
 /** When the steps that were supposed to change the screen finished. */
 let navigatedAt = 0;
+/** What the tour's own steps said, kept for the arrival report. */
+let lastSteps = [];
 /** Readings that never got a frame newer than their own navigation. */
 const staleReadings = [];
 
@@ -222,7 +235,37 @@ for (let round = 1; round <= rounds; round += 1) {
       // written out first, because a failing run is the one whose evidence
       // matters.
       try {
-        await actions.runScript(device, { steps: screen.steps, verify: false });
+        // Keep what the tour's own steps reported. On an arrival failure the
+        // decisive question is whether the last wait PASSED and on what — and
+        // that answer was being thrown away, so three CI failures in a row
+        // could only be reasoned about rather than read. A local reproduction
+        // ran the same sequence 14 times and landed correctly every time, so
+        // the runner is the only place this can be observed.
+        const run = await actions.runScript(device, { steps: screen.steps, verify: false });
+        lastSteps = (run?.results ?? []).map((r) => ({
+          action: r.action,
+          ok: r.ok !== false,
+          ms: r.ms,
+          detail: typeof r.detail === 'string' ? r.detail.slice(0, 200) : null,
+        }));
+        // **`runScript` does not throw on a failed step** — it returns
+        // `ok: !failed` — and this only ever caught throws. So a tour whose
+        // `waitFor` timed out was treated as having arrived: `navigatedAt` was
+        // set, the previous screen was read, and the arrival check then reported
+        // it as a fingerprint result. Three CI failures were that, and the
+        // diagnostic added one commit earlier is what showed it —
+        // `ok launch … / FAIL waitFor` sitting under a reading the harness had
+        // accepted.
+        //
+        // A tour that did not arrive is a precondition, not a measurement. It
+        // throws into the same handler as a step that threw, so the report is
+        // identical and the readings so far are still written out.
+        if (run && run.ok === false) {
+          const bad = (run.results ?? []).find((r) => r.ok === false);
+          throw new Error(
+            `step ${bad?.action ?? '?'} did not succeed: ${bad?.error ?? 'no reason given'}`,
+          );
+        }
         navigatedAt = Date.now();
       } catch (err) {
         save({ abandonedAt: { screen: screen.name, round, error: err.message } });
@@ -275,18 +318,38 @@ for (let round = 1; round <= rounds; round += 1) {
     // stale — an eval that scores a stale frame is measuring the runner, which
     // is the one thing this harness says it is not doing.
     if (navigatedAt) {
+      // TWO conditions, and the second was missing.
+      //
+      // Newer than the navigation is not the same as recent. Measured on a
+      // runner: a reading came off a frame **38,266 ms old** and passed this
+      // guard, because the navigation had also been more than 38 s earlier — so
+      // `capturedAt >= navigatedAt` held while the frame was half a minute
+      // stale and showed a screen two steps back. `settled: true` alongside it,
+      // because a starved capture looks exactly like a still one.
+      //
+      // A frame this old is capture starvation, not a slow screen, and no
+      // number of re-reads invents a frame the daemon is not producing. So it
+      // retries and then says which of the two it is, because "the frame
+      // predates the navigation" and "capture stopped producing frames" point
+      // at completely different things.
+      const tooOld = (r) => Date.now() - (r.state?.capturedAt ?? 0) > FRAME_TOO_OLD_MS;
+      const predatesNav = (r) => (r.state?.capturedAt ?? 0) < navigatedAt;
       for (let attempt = 0; attempt < STALE_READ_RETRIES; attempt += 1) {
-        const capturedAt = id.state?.capturedAt ?? 0;
-        if (capturedAt >= navigatedAt) break;
-        const age = Date.now() - capturedAt;
-        console.log(`         ("${screen.name}" read a frame from ${age}ms ago, older than the navigation`
+        if (!predatesNav(id) && !tooOld(id)) break;
+        const age = Date.now() - (id.state?.capturedAt ?? 0);
+        console.log(`         ("${screen.name}" read a frame from ${age}ms ago`
+          + `${predatesNav(id) ? ', older than the navigation' : `, over the ${FRAME_TOO_OLD_MS}ms freshness bound`}`
           + ` — reading again ${attempt + 1}/${STALE_READ_RETRIES})`);
         await new Promise((r) => setTimeout(r, STALE_READ_WAIT_MS));
         id = await api.screenIdentity(device, { fresh: true, confirmNovel: false });
       }
-      if ((id.state?.capturedAt ?? 0) < navigatedAt) {
+      if (predatesNav(id)) {
         staleReadings.push(`${screen.name} round ${round}: frame still predates the navigation`
           + ` by ${navigatedAt - (id.state?.capturedAt ?? 0)}ms after ${STALE_READ_RETRIES} re-reads`);
+      } else if (tooOld(id)) {
+        staleReadings.push(`${screen.name} round ${round}: newest frame is`
+          + ` ${Date.now() - (id.state?.capturedAt ?? 0)}ms old after ${STALE_READ_RETRIES} re-reads`
+          + ' — capture is starved, not the screen still');
       }
     }
     // A reading too sparse to be a screen is not a reading.
@@ -343,6 +406,7 @@ for (let round = 1; round <= rounds; round += 1) {
     // taken, and every distribution below it is then measuring the tour rather
     // than the fingerprint. The first version of this harness did exactly that
     // and reported that the distributions overlapped completely.
+    id.steps = lastSteps;
     const previous = readings[readings.length - 1];
     if (previous && previous.name !== screen.name) {
       // Not just an identical hash. Two readings of the same screen can differ
@@ -367,10 +431,20 @@ for (let round = 1; round <= rounds; round += 1) {
           `${previous.name} -> ${screen.name}: similarity ${s.toFixed(2)} — the screen did not change`
           + `\n         sensors: ${(id.entry?.sources ?? []).join('+') || 'none'}`
           + `, settled: ${id.settled}, frame ${Math.round(Date.now() - (id.state?.capturedAt ?? Date.now()))}ms old`
-          + `\n         on screen: ${labels.length ? labels.join(' · ') : '(nothing readable)'}`);
+          + `\n         on screen: ${labels.length ? labels.join(' · ') : '(nothing readable)'}`
+          // The decisive line, and it was missing. Whether the tour's last wait
+          // PASSED, and what it said, separates "the wait matched something it
+          // should not have" from "the screen changed after the wait was
+          // satisfied" — the two causes this message has always named as having
+          // opposite fixes, while giving the reader nothing to tell them apart.
+          + `\n         the tour's own steps: ${
+            (lastSteps ?? []).length
+              ? lastSteps.map((st) => `${st.ok ? 'ok' : 'FAIL'} ${st.action}${st.detail ? ` — ${st.detail}` : ''}`).join('\n                               ')
+              : '(not recorded)'}`);
       }
     }
     readings.push({
+      steps: lastSteps,
       name: screen.name,
       round,
       hash: id.hash,
@@ -506,6 +580,17 @@ function findStrays(all) {
 }
 
 const strays = findStrays(readings);
+// Readings grouped by screen, for the classification below.
+//
+// `findStrays` builds this too and it is LOCAL to that function — so reaching
+// for it here threw `byName is not defined` on a runner, in the middle of the
+// report, after the strays had already been printed. I had validated the
+// classification against a saved artifact in a standalone script, where I had
+// declared it myself, and never once in place. The logic was right and the
+// integration was never run: the unit tests do not reach this file at all.
+const readingsByName = new Map();
+for (const r of readings) readingsByName.set(r.name, [...(readingsByName.get(r.name) ?? []), r]);
+
 if (strays.length) {
   console.error(`\nFAIL ${strays.length} reading(s) do not resemble their own screen:`);
   // Two causes wear the same symptom, and until now the report asserted the
@@ -516,10 +601,25 @@ if (strays.length) {
   // label while matching one particular other screen almost exactly, and a
   // wrong turn is one whose tokens name a screen the tour did not ask for.
   let collisions = 0;
+  // Kept so the SUMMARY agrees with the lines above it. It did not: the summary
+  // tested `sparseAt` directly while each line used the classifier, so a stray
+  // correctly labelled WRONG SCREEN was followed by "every stray above was
+  // UNDER-READ". A report that contradicts itself in consecutive paragraphs is
+  // worse than either sentence alone.
+  const verdicts = [];
   for (const { reading, bestSelf, bestOther, match } of strays) {
     const named = namedTokens(reading.tokens);
     const matchNamed = match ? namedTokens(match.tokens) : [];
-    const collided = bestOther >= 0.99 && named.length === 0 && matchNamed.length === 0;
+    // One classifier, one source of truth. `collided` was computed here as well
+    // and the two could drift — which is how this block came to have a third
+    // cause landing in the wrong bucket in the first place.
+    const verdict = classifyStray({
+      reading, match, bestOther, named, matchNamed,
+      siblings: readingsByName.get(reading.name) ?? [],
+      wasSparse: sparseAt.has(`${reading.name}|${reading.round}`),
+    });
+    const { collided, underRead, wrongScreen } = verdict;
+    verdicts.push(verdict);
     if (collided) collisions += 1;
     // The third cause, and the one that produced this report on 2026-09-15.
     //
@@ -534,7 +634,21 @@ if (strays.length) {
     // nothing to work with, which is this harness's subject. This means we did
     // not look long enough, which is the harness's own fault and a different
     // remedy.
-    const underRead = sparseAt.has(`${reading.name}|${reading.round}`);
+    //
+    // **And sparseness alone is not enough to claim it**, which this
+    // misdiagnosed once. A wrong turn that lands on a screen which legitimately
+    // reads sparse gets flagged sparse too, and was then reported as our
+    // instrument's fault rather than the tour's. Measured: `settings-general`
+    // r3 came back with tokens byte-identical to the Settings root, including
+    // `heading:nav-bar:"settings"` — a *heading*, which is the root's title,
+    // where General publishes a *button* with the same word. It was read on the
+    // wrong screen, and this called it an under-read.
+    //
+    // The discriminator is the screen's own other rounds: if `settings-general`
+    // read differently and richly in rounds 1 and 2, then the screen IS
+    // distinguishable and a round matching another screen exactly went
+    // somewhere else. Only when no round of this screen can tell itself apart
+    // is "we did not look long enough" the honest reading.
     console.error(`       ${reading.name} r${reading.round}: own screen ${bestSelf.toFixed(2)}, `
       + `${match ? `${match.name} r${match.round}` : 'another screen'} ${bestOther.toFixed(2)} `
       + `(${reading.count} tokens, ${named.length} named, sources ${reading.sources.join('+') || 'none'})`);
@@ -542,6 +656,11 @@ if (strays.length) {
       console.error('              ^ a COLLISION, not a wrong turn: neither reading carries a chrome');
       console.error('                label, so both are structure with no name and the fingerprint has');
       console.error('                nothing left to tell two list screens apart.');
+    } else if (wrongScreen) {
+      console.error(`              ^ WRONG SCREEN: these tokens are ${match ? `identical to ${match.name}` : 'another screen'},`);
+      console.error('                and this screen reads differently in its other rounds — so it is');
+      console.error('                distinguishable and the tour was simply somewhere else. A tap that');
+      console.error('                missed, or a screen that went back before it was read.');
     } else if (underRead) {
       console.error('              ^ UNDER-READ, not a wrong turn: this reading stayed below the token');
       console.error('                floor after every retry, so it never saw enough of its screen to');
@@ -553,10 +672,16 @@ if (strays.length) {
     console.error(`\n${collisions} of ${strays.length} are fingerprint collisions. That is this harness's own subject,`);
     console.error('not a tour fault: a reading whose chrome label went missing cannot establish');
     console.error('identity, and comparing it as though it could is what produced the verdict above.');
-  } else if (strays.every((x) => sparseAt.has(`${x.reading.name}|${x.reading.round}`))) {
+  } else if (verdicts.length && verdicts.every((v) => v.underRead)) {
     console.error('\nEvery stray above was UNDER-READ, so this says nothing about the tour or the');
     console.error('fingerprint — the harness scored a look that was too short. The retries are in');
     console.error('SPARSE_READ_RETRIES; a runner that needs more than they allow is the finding.');
+  } else if (verdicts.some((v) => v.wrongScreen)) {
+    const n = verdicts.filter((v) => v.wrongScreen).length;
+    console.error(`\n${n} stray(s) were read on the WRONG SCREEN — the tokens match another screen`);
+    console.error('exactly while this one reads differently in its other rounds, so it is');
+    console.error('distinguishable and the tour was somewhere else. Not a fingerprint result:');
+    console.error('a tap that missed, or a screen that went back before it was read.');
   } else {
     console.error('\nThat is the tour going somewhere unintended, not the fingerprint drifting, and');
     console.error('measuring it as either distribution poisons both ends. Fix the tour — a tap that');

package/src/actions.js

@@ -90,16 +90,49 @@ const ACTION_STEPS = new Set([
   'launch', 'terminate', 'openUrl', 'confirm', 'chooseAny', 'permission',
 ]);
 
+/**
+ * Step keys that are really the MCP tool names, accepted as aliases.
+ *
+ * Two field reports guessed these independently and each wrong guess cost a
+ * round trip and aborted the rest of the batch. There is a `sim_wait` tool and
+ * a `sim_type_into` tool, so `wait` and `type_into` are what a caller reaches
+ * for inside `sim_do` — and the vocabulary answered "unknown step" without
+ * saying what the words are. A tool surface that names an action one way and
+ * accepts it another is charging the caller for our inconsistency.
+ */
+const STEP_ALIASES = Object.freeze({
+  wait: 'settle',
+  type_into: 'type',
+  typeInto: 'type',
+  scroll_to: 'scrollTo',
+  wait_for: 'waitFor',
+  waitfor: 'waitFor',
+  tap_at: 'tapAt',
+  open_url: 'openUrl',
+  assert_gone: 'assertGone',
+  assert_text: 'assertText',
+});
+
+/** Every step key `runStep` understands, for an error that can be acted on. */
+export const STEP_KEYS = Object.freeze([
+  'tap', 'tapAt', 'type', 'paste', 'clear', 'swipe', 'scroll', 'scrollTo',
+  'button', 'key', 'launch', 'terminate', 'openUrl', 'confirm', 'chooseAny',
+  'permission', 'settle', 'waitFor', 'waitText', 'assert', 'assertGone',
+  'assertText', 'visible', 'gone', 'enabled', 'disabled', 'value', 'look',
+  'seek', 'sweep', 'pause',
+]);
+
 /** Accept both `{tap: "Save"}` shorthand and `{action: "tap", target: "Save"}`. */
 export function normalizeStep(raw) {
-  if (typeof raw === 'string') return { action: raw };
-  if (raw.action) return { ...raw };
+  const canonical = (a) => STEP_ALIASES[a] ?? a;
+  if (typeof raw === 'string') return { action: canonical(raw) };
+  if (raw.action) return { ...raw, action: canonical(raw.action) };
   const [key] = Object.keys(raw);
   if (!key) throw new Error('empty step');
   // Siblings like timeoutMs sit alongside the shorthand key and must survive.
   const { [key]: value, ...rest } = raw;
   const inline = value && typeof value === 'object' && !Array.isArray(value) ? value : { value };
-  const step = { ...rest, ...inline, action: key };
+  const step = { ...rest, ...inline, action: canonical(key) };
   // Drop keys that are present but undefined. `simframe tap X` used to pass
   // `index: undefined`, which survived here and then crashed the signature
   // builder before the tap was ever sent.
@@ -699,7 +732,10 @@ export async function runScript(
         // eight times below its threshold, so it reads as no-visible-change and
         // its edge is no longer recorded. That is the right trade while the
         // detector cannot see it, and it comes back on its own once it can.
-        const noEvidence = Boolean(settled?.noVisibleChange);
+        // A changed control IS evidence, whatever the frame hash says. Without
+        // this the edge is not recorded either, so a toggle that worked teaches
+        // the graph nothing and stays unpredictable forever.
+        const noEvidence = Boolean(settled?.noVisibleChange) && !stateDelta(beforeScreen?.entry, afterReading?.entry);
         if (afterScreen.confirmed && afterScreen.hash && !noEvidence) {
           // The observed cost of this transition, which is what makes the next
           // one adaptive. Only from a settle that was actually satisfied: a
@@ -755,7 +791,22 @@ export async function runScript(
       // first — and six swipes reporting "no visible change" is what item 120
       // was reported against. Keying on one of them would have shipped a fix
       // that did not fire on its own bug report.
-      const wentNowhere = Boolean(settled?.noVisibleChange) || verification?.verdict === 'no-visible-change';
+      // Declared BEFORE `wentNowhere`, which reads it. It sat below and the
+      // unit tests were green because nothing without a device reaches this
+      // line — a `const` in the temporal dead zone throws on first use, which
+      // is the same trap the `afterReading` comment above records.
+      //
+      // Computed on EITHER signal, because they are two different sensors and
+      // the reported case came through the one this first version missed: the
+      // radio move settled in 2714ms with `sawChange` true, so
+      // `settled.noVisibleChange` was false and the verdict came from the
+      // fingerprint instead.
+      const changedState = (settled?.noVisibleChange || verification?.verdict === 'no-visible-change')
+        ? stateDelta(beforeScreen?.entry, afterReading?.entry)
+        : null;
+      if (changedState) verification = stateChanged(verification, changedState);
+      const wentNowhere = !changedState
+        && (Boolean(settled?.noVisibleChange) || verification?.verdict === 'no-visible-change');
       const aimNote = wentNowhere && aim.at
         ? screenmap.describePoint(
           // `beforeScreen` is null with verification off, and that is exactly
@@ -769,7 +820,9 @@ export async function runScript(
       const note = launchNote
         + (filling ? ` [settled, but ${filling} — waitFor content, do not act on this yet]` : '')
         + (settled?.smallChange ? ' [a small change, in one region only]' : '')
-        + (settled?.noVisibleChange ? ' [no visible change]' : '')
+        + (changedState
+          ? ` [the screen did not move, but ${changedState.detail} — this worked, do not retry]`
+          : (settled?.noVisibleChange ? ' [no visible change]' : ''))
         // After the symptom, because it is the explanation of it.
         + (aimNote ? ` [${aimNote}]` : '')
         + (settled?.staleBaseline ? ' [baseline had already settled; re-taken from the live screen]' : '')
@@ -1561,6 +1614,37 @@ async function confirmNoChange(deviceQuery, verification, { beforeScreen, option
   };
 }
 
+/**
+ * A control that changed state is not a step that did nothing.
+ *
+ * The sibling of `belowThreshold` on the same verdict, for the other kind of
+ * evidence. `no-visible-change` arrives by two routes — the pixel detector and
+ * the fingerprint agreeing we are on the same screen — and a switch or a radio
+ * satisfies both while having plainly worked. Measured on a device: a radio
+ * moving from "Top" to "Inline" reported `(settled 2714ms)
+ * [no-visible-change]` with `selected` moving between the two rows in the
+ * element list either side of it.
+ *
+ * This is the verdict an agent reads to decide whether to retry, and after item
+ * 148 made the tap actually land, a retry undoes the thing that worked. So the
+ * verdict carries the evidence rather than contradicting it.
+ *
+ * Still not `ok`: the screen genuinely did not become a different screen, and
+ * claiming a transition that did not happen would be the opposite error. It
+ * becomes `unverified` — the verdict this project uses for "something happened
+ * and nobody can vouch for what" — with the change named.
+ */
+export function stateChanged(verification, delta) {
+  if (verification?.verdict !== 'no-visible-change' || !delta) return verification;
+  return {
+    ...verification,
+    verdict: 'unverified',
+    detail: `the screen did not change, but ${delta.detail}`
+      + ' — the action worked; do not retry it, or you will undo it',
+    stateDelta: delta,
+  };
+}
+
 export function belowThreshold(verification, settled) {
   if (verification?.verdict !== 'no-visible-change' || !settled?.smallChange) return verification;
   return {
@@ -1608,6 +1692,37 @@ async function focusHint(deviceQuery, target, ctx) {
   }
 }
 
+/**
+ * Is anything focused? Asked before typing blind.
+ *
+ * `{"type": "text"}` with no `into` types into whatever has focus, and reported
+ * `ok … [unconfirmed — no field named, so nothing was read back]`. The note was
+ * honest and it was attached to a **green step**, so a chained `sim_do`
+ * continued on a premise that had never been checked — a field report lost text
+ * that went nowhere exactly this way.
+ *
+ * If nothing has focus, the keystrokes are lost and there is no version of that
+ * which is an `ok`. The tree already answers it: one accessibility read, no
+ * OCR, which is the same cost `focusHint` pays on the `into` path.
+ *
+ * Advisory in one direction only. A tree that cannot answer — no targets, a
+ * read that threw — returns `unknown`, and the step proceeds with its
+ * unconfirmed note rather than failing on our inability to look. Refusing to
+ * type because we could not read the screen would be a worse trade than the
+ * bug.
+ */
+async function focusedSomewhere(deviceQuery, ctx) {
+  try {
+    const { entry } = await api.readScreenWith(deviceQuery, { useOcr: false, options: ctx.options });
+    const targets = entry?.targets ?? [];
+    if (!targets.length) return { known: false };
+    const hit = targets.find((t) => t.focused === true);
+    return { known: true, focused: Boolean(hit), where: hit?.label ?? hit?.identifier ?? null };
+  } catch {
+    return { known: false };
+  }
+}
+
 /**
  * How long to wait for the tap to become focus before inserting text.
  *
@@ -2286,11 +2401,29 @@ async function runStep(deviceQuery, udid, step, ctx) {
         journalWrite(udid, step, sent, back, ctx);
         return `typed into ${field.where}${back.note}${back.landed ? '' : field.quiet}`;
       }
+      // **Refusing here was tried and reverted — see DEFERRED 161.** The idea
+      // was that "nothing focused" is knowable before typing, so typing into
+      // nothing should fail rather than report `ok`. On a device it produced a
+      // FALSE REFUSAL on the legitimate case: after tapping Settings' own
+      // search field, every element in the tree read `focused: false`. iOS does
+      // not reliably publish AXFocused here, which `focusHint` above already
+      // says in its own comment — *"with a hardware keyboard attached to the
+      // simulator no software keyboard appears, so there is often nothing to
+      // see"*. A gate on an unpublished signal blocks real work, which is worse
+      // than the verdict it was fixing.
+      //
+      // So the focus reading stays advisory and goes into the note. What the
+      // field report actually objected to was an honest caveat attached to a
+      // GREEN step, and the note is where that has to be fixed until there is a
+      // signal worth gating on.
+      const focus = await focusedSomewhere(deviceQuery, ctx);
       await input.typeText(udid, step.text ?? step.value);
-      // No selector, so there is nothing to read back — which is a fine trade
-      // for typing into whatever Safari's own form chevrons focused, but it
-      // must not be reported as though the text was seen to land.
-      return 'typed text [unconfirmed — no field named, so nothing was read back]';
+      return 'typed text [NOT CONFIRMED: no field named, so nothing was read back'
+        + (focus.known && !focus.focused
+          ? ' and nothing on this screen reports keyboard focus — the text may have gone nowhere'
+          : '')
+        + '. Name the field — {"type": {"into": "<field>", "text": "..."}} — to have it tapped'
+        + ' first and read back]';
     }
     case 'paste': {
       // Long strings are much faster on the pasteboard than through the
@@ -2486,11 +2619,47 @@ async function runStep(deviceQuery, udid, step, ctx) {
           return null;
         }
       };
+      /**
+       * Is this target actually on the screen?
+       *
+       * Resolving is not the same as being in view, and this returned `"X" is
+       * in view at 201,-35 after 1 scroll down` — 35pt ABOVE the viewport, so
+       * the tap that followed missed. Reported from the field. The tree carries
+       * scrolled-away rows with out-of-bounds coordinates, which is exactly the
+       * case `scrollTo` exists to resolve, so claiming success on one is the
+       * one answer it must never give.
+       *
+       * Generous rather than strict: any overlap with the viewport counts, so a
+       * row half over an edge is still reachable and still reported. Only a
+       * target entirely outside keeps the search going.
+       */
+      const inViewport = (t) => {
+        const w = points?.width;
+        const h = points?.height;
+        if (!Number.isFinite(w) || !Number.isFinite(h)) return true;
+        const f = t?.frame;
+        if (f && Number.isFinite(f.y) && Number.isFinite(f.height)) {
+          return f.y + f.height > 0 && f.y < h && f.x + (f.width ?? 0) > 0 && f.x < w;
+        }
+        return Number.isFinite(t?.y) && t.y >= 0 && t.y <= h
+          && Number.isFinite(t?.x) && t.x >= 0 && t.x <= w;
+      };
+      // The direction actually scrolled, which is not always the one asked for:
+      // `offsetSays()` may reverse it, and the message used to name the request
+      // rather than the act — reported as "after 1 scroll down" on a request
+      // for "up".
+      const scrolled = [];
       for (let i = 0; i <= max; i += 1) {
         try {
           const found = await api.locate(deviceQuery, query, { index: step.index, refresh: i > 0 });
-          return `"${found.target.label}" is in view at ${found.target.x},${found.target.y}` +
-            (i ? ` after ${i} scroll${i === 1 ? '' : 's'} ${dir}` : ' already');
+          if (!inViewport(found.target)) throw new Error(
+            `"${query}" is in the tree but not in view (at ${found.target.x},${found.target.y}`
+            + ` on a ${Math.round(points?.width ?? 0)}x${Math.round(points?.height ?? 0)}pt screen)`);
+          const how = scrolled.length
+            ? ` after ${scrolled.length} scroll${scrolled.length === 1 ? '' : 's'} `
+              + (new Set(scrolled).size === 1 ? scrolled[0] : scrolled.join(' then '))
+            : ' already';
+          return `"${found.target.label ?? query}" is in view at ${found.target.x},${found.target.y}${how}`;
         } catch (err) {
           if (i === max) {
             throw new Error(`scrolled ${dir} ${max}x without finding ${query}: ${err.message}`);
@@ -2499,6 +2668,7 @@ async function runStep(deviceQuery, udid, step, ctx) {
         const evidence = await offsetSays();
         if (evidence) dir = evidence;
         const wasAt = await hashNow(deviceQuery, ctx.options);
+        scrolled.push(dir);
         await runStep(deviceQuery, udid, { action: 'scroll', value: dir }, ctx);
         // A scroll either moves immediately or not at all, so it does not need a
         // transition's budget. Six iterations at 2,500ms was most of why this
@@ -2763,11 +2933,83 @@ async function runStep(deviceQuery, udid, step, ctx) {
       await sleep(ms);
       return `paused ${ms}ms`;
     }
-    default:
-      throw new Error(`unknown step "${step.action}"`);
+    default: {
+      // Say what the words are. "unknown step" on its own sent two testers
+      // guessing, and a guess costs a round trip and the rest of the batch.
+      const near = STEP_KEYS.filter((k) => {
+        const a = String(step.action ?? '').toLowerCase().replace(/[_-]/g, '');
+        const b = k.toLowerCase();
+        return a && (b.startsWith(a) || a.startsWith(b) || b.includes(a));
+      }).slice(0, 3);
+      throw new Error(
+        `unknown step "${step.action}"`
+        + (near.length ? ` — did you mean ${near.map((k) => `"${k}"`).join(' or ')}?` : '')
+        + ` Valid steps: ${STEP_KEYS.join(', ')}.`,
+      );
+    }
   }
 }
 
+/**
+ * What changed in the controls, when the pixels say nothing did.
+ *
+ * `settle()` decides `noVisibleChange` from frame hashes alone, and a control
+ * that flips is eight times below the change threshold (DEFERRED 4, measured on
+ * a switch: 0.00049 peak against 0.004). So a switch that genuinely flipped is
+ * reported as an action that did nothing — with the changed value printed in the
+ * same response.
+ *
+ * **That verdict became dangerous the moment item 148 made the tap land.** While
+ * the tap missed the frame centre, `no-visible-change` was true and harmless.
+ * Now the tap actuates, and `no-visible-change` is what an agent reads to decide
+ * whether to retry — so a retry silently un-flips the toggle. An external
+ * tester made exactly that point about my own commit message, which had claimed
+ * the verdict "was telling the truth".
+ *
+ * This is wiring, not calibration. The element values are already in hand on
+ * both sides of the action — `beforeScreen.entry` and the after-reading are read
+ * for verification anyway — so the comparison costs nothing on the device. A
+ * control whose value or selected state changed did something, whatever the
+ * frame hash says.
+ *
+ * Identity across the two readings goes by identifier, then label, then
+ * type-and-place: a row that moved is not the same reading of the same control,
+ * and pretending otherwise would invent changes.
+ */
+export function stateDelta(beforeEntry, afterEntry) {
+  const keyOf = (t) => t.identifier || t.label || `${t.type ?? '?'}@${Math.round(t.x ?? 0)},${Math.round(t.y ?? 0)}`;
+  const stateOf = (t) => `${t.value ?? ''}|${t.selected ?? ''}`;
+  const before = new Map();
+  for (const t of beforeEntry?.targets ?? []) {
+    if (t.value == null && t.selected == null) continue;
+    before.set(keyOf(t), { state: stateOf(t), target: t });
+  }
+  if (!before.size) return null;
+  const changes = [];
+  for (const t of afterEntry?.targets ?? []) {
+    if (t.value == null && t.selected == null) continue;
+    const was = before.get(keyOf(t));
+    if (!was || was.state === stateOf(t)) continue;
+    changes.push({
+      name: t.label || t.identifier || keyOf(t),
+      from: was.target.value ?? was.target.selected,
+      to: t.value ?? t.selected,
+    });
+  }
+  if (!changes.length) return null;
+  // Name the control that turned ON, not the one that turned off. A radio move
+  // changes two rows and reporting "Inline changed from true to false" is
+  // accurate and reads like a loss — the useful half is what is selected now.
+  const on = (v) => v === true || v === 'true' || v === '1';
+  const first = changes.find((c) => on(c.to) && !on(c.from)) ?? changes[0];
+  return {
+    count: changes.length,
+    detail: `${JSON.stringify(String(first.name).slice(0, 32))} changed from `
+      + `${JSON.stringify(String(first.from))} to ${JSON.stringify(String(first.to))}`
+      + (changes.length > 1 ? ` (and ${changes.length - 1} other control(s))` : ''),
+  };
+}
+
 /**
  * Why a settle gave up, in the numbers that decided it.
  *

package/src/cli.js

@@ -34,7 +34,7 @@ const USAGE = `simframe — always-warm iOS Simulator frames
   simframe tap     <selector>        tap #3, "Save", or @120,400
   simframe do      <script.json>     run a scripted flow (see below)
   simframe screens [device]          list screens this device has learned
-  simframe storage [bundle-id]       what the app saved (works on a shut-down device)
+  simframe storage [bundle-id] [--device=<name|udid>]   what the app saved (no boot needed)
   simframe goto    <screen>          walk to a known screen through known steps
   simframe flow    save <name> <script.json>   run a flow and save it if every step verifies
   simframe flow    run  <name>       replay a saved flow
@@ -867,7 +867,44 @@ async function main() {
       // whole value of this command is that it answers on a device that is not
       // running — measured: simctl itself cannot, on this Xcode. Routing it
       // through the daemon would throw that away for no gain.
-      const device = await resolveDevice(flags.device);
+      // Resolving a device here must NOT require one to be running, and it did.
+      //
+      // The whole claim of this command is that it answers before a boot, and
+      // `resolveDevice(undefined)` resolves against *booted* devices — so the
+      // documented form, `simframe storage <bundle-id>`, failed with "no booted
+      // simulator" at exactly the feature's headline use case. Reported by an
+      // external tester, who found it worked only when a device was named,
+      // which the help line did not mention either.
+      //
+      // Named: resolve as usual, booted or not. Unnamed: prefer a booted device
+      // when there is one, otherwise the only device that has app containers —
+      // and when that is ambiguous, say so and name the candidates rather than
+      // complaining about a boot nobody needs.
+      const named = flags.device ?? flags.udid ?? process.env.SIMFRAME_DEVICE;
+      let device;
+      if (named) {
+        device = await resolveDevice(String(named));
+      } else {
+        device = await resolveDevice(undefined).catch(() => null);
+        if (!device) {
+          const all = await listDevices({ all: true });
+          const withApps = [];
+          for (const d of all) {
+            const apps = await storage.apps(d.udid).catch(() => []);
+            if (apps.length) withApps.push({ device: d, apps: apps.length });
+          }
+          if (withApps.length === 1) {
+            device = withApps[0].device;
+          } else {
+            throw new Error(
+              withApps.length
+                ? 'storage reads a device that does not have to be running, so name which one:\n'
+                  + withApps.map((w) => `  --device=${w.device.udid}   ${w.device.name} (${w.apps} app(s))`).join('\n')
+                : 'no device on this host has any app data to read',
+            );
+          }
+        }
+      }
       const [bundleId] = positional;
       if (!bundleId) {
         const list = await storage.apps(device.udid);