simframe 0.12.2 → 0.14.0

package/src/index.js

@@ -13,6 +13,7 @@ import {
   isBlackFrame,
   maxCellDelta,
   CELL_CHANGE,
+  describeMotion,
   regionDeltas,
   regionMap,
   signatureDiff,
@@ -125,6 +126,33 @@ export const READY_TIMEOUT_MS = 20_000;
  */
 export const LIVE_DAEMON_CAP_MS = 60_000;
 
+/**
+ * How many gestures may land on a motionless screen before that is evidence.
+ *
+ * **This replaces a duration threshold, and the replacement is the point.** The
+ * first version asked "has the screen been still for a long time?" and needed
+ * 20 s before it would speak. A field report then caught a frame roughly three
+ * hours stale, presented as 130 ms old, on a screen still for **8.2 seconds** —
+ * so the check sat silently under its own gate during exactly the failure it
+ * was written for. A threshold chosen from one earlier example is not a
+ * mechanism.
+ *
+ * What says "dead surface" is not how long the screen has been quiet. It is
+ * that we kept touching it and *nothing moved at all*. One tap that changes
+ * nothing is ordinary — a disabled control, a form refusing a submit. Three in
+ * a row, with not one pixel of response, is a different claim.
+ */
+export const SURFACE_SUSPECT_INPUTS = 3;
+
+/**
+ * Slack between "input landed" and "the screen should have moved".
+ *
+ * A gesture only counts against the surface if it landed *inside* the quiet
+ * period with room to spare — otherwise the tap that ends the quiet period
+ * counts as evidence against it.
+ */
+export const SURFACE_MARGIN_MS = 400;
+
 export function daemonStatus(udid) {
   const meta = store.readJson(store.paths(udid).meta);
   const pid = meta?.pid ?? null;
@@ -149,10 +177,13 @@ export async function ensureDaemon(deviceQuery, options = {}) {
   // A dead daemon leaves its last state.json behind. Anything captured before
   // we (re)started the loop is not evidence of a live screen, so ignore it.
   const minCapturedAt = existing.alive ? 0 : Date.now();
+  // The pid we spawned, which is the only honest answer to "did a daemon come
+  // up" — see the comment on the wait below.
+  let spawnedPid = null;
   if (!existing.alive) {
     if (acquireSpawnLock(p.lock)) {
       try {
-        await startEngine(device.udid, options);
+        spawnedPid = (await startEngine(device.udid, options)).pid;
       } finally {
         // Hold the lock briefly so a burst of callers does not double-spawn.
         setTimeout(() => releaseSpawnLock(p.lock), 1500).unref?.();
@@ -178,25 +209,64 @@ export async function ensureDaemon(deviceQuery, options = {}) {
   // had read identically, which cost a log dive to tell apart.
   const liveDeadline = Date.now() + LIVE_DAEMON_CAP_MS;
   let sawLiveDaemon = false;
+  let sawProcess = false;
   for (;;) {
     const state = store.readJson(p.state);
     if (state && state.capturedAt >= minCapturedAt && Date.now() - state.capturedAt < 30_000) {
       return { device, state, started: !existing.alive };
     }
-    const alive = daemonStatus(device.udid).alive;
-    sawLiveDaemon = sawLiveDaemon || alive;
-    if (Date.now() >= (alive ? liveDeadline : deadline)) break;
+    // Two different questions, and for a long time one file answered both.
+    //
+    // `daemonStatus` reads meta.json, which the daemon writes in `claim()` —
+    // *after* `platform.attach`, its slowest startup step. So a daemon that has
+    // spawned and is attaching to CoreSimulator is indistinguishable, from
+    // here, from one that never started: both are `alive: false`. That is the
+    // condition the live cap below exists for, and it could never be reached in
+    // the case that produced it, because reaching it required the very file
+    // whose absence was the problem.
+    //
+    // Measured on a hosted runner, 2026-09-13: `simframe start` gave up after
+    // 20 s saying **"no daemon process came up"**, and the next step of the
+    // same job printed `● iPhone 16 Pro pid=9573 frame=#5 age=652ms` with a
+    // daemon log showing it had been capturing the whole time. The sentence was
+    // not merely unhelpful, it named the wrong condition — the fourth time this
+    // project has failed that way and the second time in two days.
+    //
+    // The pid we spawned is the honest signal and it was already in hand;
+    // `startEngine` was discarding it.
+    const claimed = daemonStatus(device.udid).alive;
+    const running = claimed || (spawnedPid != null && store.isProcessAlive(spawnedPid));
+    sawLiveDaemon = sawLiveDaemon || claimed;
+    sawProcess = sawProcess || running;
+    if (Date.now() >= (running ? liveDeadline : deadline)) break;
     await sleep(80);
   }
   const tail = readLogTail(p.log);
-  const waited = sawLiveDaemon
-    ? `the daemon is running and the display produced no frame in ${Math.round(LIVE_DAEMON_CAP_MS / 1000)}s`
-    : 'no daemon process came up';
   throw new Error(
-    `simframe daemon did not produce a frame for ${device.name} — ${waited}${tail ? `\n${tail}` : ''}`,
+    `simframe daemon did not produce a frame for ${device.name} — `
+    + `${readinessFailure({ sawLiveDaemon, sawProcess })}${tail ? `\n${tail}` : ''}`,
   );
 }
 
+/**
+ * Which of three failures this was.
+ *
+ * Extracted so it can be tested, for the reason `screenshotFailure` was: the
+ * whole defect here is a message naming the wrong condition, and a message is
+ * only checkable if something can ask for it without a device. Two of these
+ * three sentences did not exist until a runner produced each of them in turn and
+ * both arrived reading as the third.
+ */
+export function readinessFailure({ sawLiveDaemon, sawProcess }) {
+  const cap = Math.round(LIVE_DAEMON_CAP_MS / 1000);
+  if (sawLiveDaemon) return `the daemon is running and the display produced no frame in ${cap}s`;
+  if (sawProcess) {
+    return `the daemon process started but never claimed the device in ${cap}s`
+      + ' — it is stuck attaching to the simulator rather than failing to launch';
+  }
+  return 'no daemon process came up';
+}
+
 /** Why the daemon was not used, when it was not. Surfaced by doctor. */
 export let engineFallbackReason = null;
 
@@ -252,14 +322,16 @@ async function startEngine(udid, options) {
     if (built.ok) {
       engineFallbackReason = null;
       recordFallback(udid, null);
-      engine.spawnDaemon(udid, options);
-      return 'simframed';
+      // The pid, not just the name. Whether a daemon came up is a question
+      // about a process, and answering it from meta.json — which the daemon
+      // writes only after attaching — could not tell "never started" from
+      // "still starting". See the wait in `ensureDaemon`.
+      return { engine: 'simframed', pid: engine.spawnDaemon(udid, options)?.pid ?? null };
     }
     engineFallbackReason = built.reason ?? 'simframed unavailable';
     recordFallback(udid, engineFallbackReason);
   }
-  spawnNodeDaemon(udid, options);
-  return 'screenshot';
+  return { engine: 'screenshot', pid: spawnNodeDaemon(udid, options)?.pid ?? null };
 }
 
 function spawnNodeDaemon(udid, options) {
@@ -283,6 +355,7 @@ function spawnNodeDaemon(udid, options) {
     env: process.env,
   });
   child.unref();
+  return child;
 }
 
 function acquireSpawnLock(lockFile) {
@@ -423,6 +496,49 @@ export function liveness(udid, state) {
   if (!damageDriven && ageMs > STALE_FRAME_MS) {
     return { ok: false, ageMs, stalled: false, note: `capture loop is stalled: newest frame is ${ageMs}ms old` };
   }
+
+  // A live loop re-reading a dead surface, which every other signal calls fine.
+  //
+  // Reported from the field on 0.12.2, and it is the most expensive shape this
+  // tool has produced: `sim_look` served the *login screen* for three minutes
+  // while the app was four screens deep in a wizard, and announced the frame as
+  // 66ms old. The daemon was not lying — the frame really was new. It was a new
+  // read of a surface that had stopped updating. Every check above passes: the
+  // process is alive, no read failed, the age is tiny.
+  //
+  // The contradiction is between two numbers we already have. `stableForMs`
+  // says the screen has not changed in N ms; `lastInputAt` says we were
+  // delivering taps and swipes during that window. A screen that has not moved
+  // since *before* we last touched it, for long enough that several gestures
+  // landed inside the quiet period, is not a calm screen. The tester's own
+  // figures: stable for 159,753 ms across three screen transitions.
+  //
+  // Deliberately a WARNING and not a failure. A genuinely inert screen is
+  // possible — a form that rejects every tap, a disabled control tapped twice —
+  // and turning that into a hard error would be a confident wrong answer of
+  // exactly the kind this project keeps paying for. It says what it sees and
+  // names the arbiter.
+  const stableForMs = state?.stableForMs;
+  if (Number.isFinite(stableForMs)) {
+    // The moment the screen last moved. Every gesture delivered after it landed
+    // on a screen that did not react.
+    const lastChange = Date.now() - stableForMs + SURFACE_MARGIN_MS;
+    const ignored = store.inputTimes(udid).filter((t) => t > lastChange);
+    if (ignored.length >= SURFACE_SUSPECT_INPUTS) {
+      const quiet = Math.round(stableForMs / 1000);
+      return {
+        ok: true,
+        ageMs,
+        stalled: false,
+        suspectSurface: true,
+        ignoredInputs: ignored.length,
+        note: `${ignored.length} gestures have been delivered without the screen moving at all `
+          + `(still for ${quiet < 1 ? `${Math.round(stableForMs)}ms` : `${quiet}s`}), `
+          + 'so the frames are new and the surface behind them may be dead. '
+          + 'The accessibility tree is read separately and is the tiebreaker; `simframe revive` re-attaches capture.',
+      };
+    }
+  }
   return { ok: true, ageMs, stalled: false, note: null };
 }
 
@@ -748,6 +864,37 @@ function timingOfNow(udid, state) {
  * sampled after the fact is the single most common way to wait for a change
  * that has already happened.
  */
+/**
+ * How far back "still moving" looks, in milliseconds.
+ *
+ * **In time, not in frames**, and that distinction is the whole of it. The
+ * first version kept the last eight frame pairs, on the reasoning that at a
+ * 60 ms poll floor eight frames is about half a second. Frames are not a clock:
+ * capture is damage-driven with a slow idle floor, so on a quiet screen eight
+ * frames spanned *sixteen seconds* — and the window still held the transition
+ * that had brought us to the screen. A home screen with one animated widget
+ * duly reported movement in all thirty-two regions.
+ *
+ * One second, long enough that a stepping animation is not missed between two
+ * frames and short enough that the entry transition is gone by the time a
+ * settle gives up.
+ */
+export const MOTION_WINDOW_MS = 1000;
+
+/**
+ * How young a frame has to be to count as "the screen right now".
+ *
+ * The daemon's idle floor is 2,000 ms — on a screen with no damage it captures
+ * anyway, at that cadence, precisely so state stays fresh enough to reason
+ * about. So a frame younger than that floor plus a margin is the current
+ * screen; older than it means the loop has missed its own beat, which is what
+ * `liveness` is for.
+ *
+ * Tied to the daemon's floor rather than chosen, because a number chosen here
+ * would drift away from it silently the first time the floor moved.
+ */
+export const FRAME_IS_CURRENT_MS = 2500;
+
 export async function waitFor(
   deviceQuery,
   { mode = 'settle', since, stableMs = 600, timeoutMs = 8000, reactionMs = 2500, baselineHash, options } = {},
@@ -789,6 +936,29 @@ export async function waitFor(
     ? hexToSignature(resolved.entry.sig)
     : (requested == null ? hexToSignature(currentSig(first) ?? '') : null);
   let smallChange = false;
+  /**
+   * Where the screen moved while we waited — item 123.
+   *
+   * The per-region maximum over a trailing window of frame pairs, so a settle
+   * that gives up can say *where* rather than only that it did.
+   *
+   * A window and not a running max over the whole wait, which is what the first
+   * version did and it was useless: every wait begins with the transition that
+   * brought us here, so a home screen with one animated widget reported
+   * movement in all thirty-two regions and "spread across the screen". The
+   * question a settle failure asks is *what is still moving*, not what has
+   * moved at any point since we started looking.
+   *
+   * A window rather than the last pair alone, because a blinking or stepping
+   * animation is between frames as often as not, and a single pair would
+   * report a still screen that is not one.
+   *
+   * Costs one subtraction per region per frame over a signature that was
+   * already read and parsed for `smallChange`.
+   */
+  const motionRing = [];
+  let motionPrev = null;
+  let motionSeq = null;
   /** Frames the display was not rendering at all. See `isBlackFrame`. */
   let blackFrames = 0;
   let blackSinceStart = null;
@@ -834,6 +1004,46 @@ export async function waitFor(
     sawChange = false;
   }
 
+  /**
+   * What the daemon says is animating, if anything — and it has always known.
+   *
+   * `Motion.state` in the daemon localises a small persistent animation and
+   * publishes it as `motion.animating`, a bounding box, alongside its own
+   * `settled` flag. Nothing in JavaScript read either. `waitFor` decided
+   * stillness from `stableForMs` alone, which is derived from a *mean* over the
+   * grid, and a spinner does not move a mean.
+   *
+   * Measured on the testbed's Diagnostics screen, which now carries one on
+   * purpose: frames arriving every 85 ms, `motion.animating` boxed at 13x13,
+   * `motion.settled` **false** — and `stableForMs` **79,207**. Seventy-nine
+   * seconds of claimed stillness on a screen that had never once stopped. A
+   * field report had already called this in as `settled after 62ms` on a
+   * still-loading screen.
+   *
+   * Deliberately reported and **not** acted on. Requiring the daemon's `settled`
+   * here would be correct for a spinner and wrong for a text caret, which is
+   * also small, also persistent, and must never stop a screen from settling —
+   * and choosing between them needs a caret measured, not a threshold picked
+   * today. So the disagreement becomes visible instead of being resolved by
+   * guess: the caller is told a part of the screen is still moving and where,
+   * and can decide. The decision is an open item with a number attached to it.
+   */
+  const animatingNow = () => {
+    const box = last?.motion?.animating;
+    if (!box) return null;
+    return { ...box, daemonSettled: last.motion.settled === true };
+  };
+
+  /** The last second of movement, collapsed to one per-region maximum. */
+  function motionSummary() {
+    const cutoff = Date.now() - MOTION_WINDOW_MS;
+    const recent = motionRing.filter((e) => e.at >= cutoff);
+    if (!recent.length) return null;
+    const max = recent[0].deltas.map((_, i) => Math.max(...recent.map((e) => e.deltas[i] ?? 0)));
+    if (!max.some((d) => d > 0)) return null;
+    return { deltas: max, map: regionMap(max), windowMs: MOTION_WINDOW_MS, ...describeMotion(max) };
+  }
+
   const done = (satisfied, extra = {}) => ({
     device,
     state: last,
@@ -851,6 +1061,24 @@ export async function waitFor(
     // because the app drew black. What makes it the capture wedge is that it
     // stays black while input is being delivered, and the caller knows that.
     blackMs: blackSinceStart ? Date.now() - blackSinceStart : 0,
+    // Only worth carrying when something actually moved; a still screen that
+    // never satisfied the wait has nothing to point at.
+    motion: motionSummary(),
+    animating: animatingNow(),
+    /**
+     * Everything that decided this answer, for the failure message.
+     *
+     * Two CI cycles were spent guessing at a `screen did not settle within
+     * 25008ms` — a sentence that names the one number which is never the
+     * reason. The wait turns on three things and the log carried none of them:
+     * how long the screen had actually been still, how much stillness was
+     * being asked for, and whether any frame arrived at all. "Still for 1,200
+     * of the 1,400 ms required" and "still for 135,000 ms and nothing arrived"
+     * are opposite diagnoses and they had one sentence between them.
+     */
+    stillForMs: Number.isFinite(last?.stableForMs) ? last.stableForMs : null,
+    stableMsRequired: stableMs,
+    framesSeen: Number.isFinite(last?.seq) ? last.seq - startSeq : null,
     baselineHash: baselineHashValue,
     baselineResolved,
     waitedMs: Date.now() - startedAt,
@@ -904,6 +1132,23 @@ export async function waitFor(
         }
       }
 
+      // Track where it is moving, on new frames only — comparing a frame with
+      // itself is a row of zeros that would dilute nothing but waste the work.
+      if (state.seq !== motionSeq) {
+        const sigHex = currentSig(state);
+        const sig = sigHex ? hexToSignature(sigHex) : null;
+        if (sig) {
+          if (motionPrev) {
+            motionRing.push({ at: Date.now(), deltas: regionDeltas(sig, motionPrev) });
+            // Bounded in time by the summary and in length here, so a screen
+            // rendering at 60 fps for ten seconds cannot grow this without end.
+            while (motionRing.length > 240) motionRing.shift();
+          }
+          motionPrev = sig;
+        }
+        motionSeq = state.seq;
+      }
+
       // A pause that turned out not to be the end of the transition. Only
       // pauses followed by more movement count: the quiet at the end of a
       // settle is the answer, not a gap.
@@ -939,10 +1184,30 @@ export async function waitFor(
         // the caller acted can never satisfy it; once the change is seen,
         // stableForMs is measured from that change. Plain "stable" has no such
         // requirement — an already-still screen genuinely is stable.
-        // At least one frame must arrive during the call, so the answer is
-        // never derived purely from what was already on disk.
+        //
+        // The evidence must be *current*, and that used to be spelled "at least
+        // one frame must arrive during the call". The intent is right and the
+        // spelling made the wait unsatisfiable on exactly the screens it is
+        // asked about most. Capture is damage-driven: a screen that is not
+        // moving produces no new frame by design, so on a still screen the
+        // condition asks for evidence the system has deliberately chosen not to
+        // generate, and the wait burns its whole budget and reports *"screen did
+        // not settle"* about a screen that has been motionless throughout.
+        //
+        // Measured here rather than argued: a flow step reported `did not settle
+        // within 1547ms` while its own evidence line read **still for 3548ms of
+        // the 1400ms required; NO frame arrived while waiting**. Three and a half
+        // seconds of stillness, two and a half times what was asked, refused.
+        //
+        // So currency is a question about time, not about a counter — the same
+        // correction the motion window needed. A frame younger than the capture
+        // loop's own idle floor *is* the screen as it is now. `liveness` is
+        // checked at the top of every iteration and already rejects a stale file
+        // from a dead daemon, which is the case the counter was really guarding.
         const freshFrames = state.seq - startSeq;
-        if (sawChange && freshFrames >= 1 && state.stableForMs >= stableMs) {
+        const age = Date.now() - (state.capturedAt ?? 0);
+        const current = freshFrames >= 1 || age <= FRAME_IS_CURRENT_MS;
+        if (sawChange && current && state.stableForMs >= stableMs) {
           return done(true);
         }
       }
@@ -1470,7 +1735,28 @@ async function locateWith(
   const target = index != null ? candidates[index] : candidates[0];
   if (!target) {
     const visible = entry.targets.filter((t) => t.label);
-    const sample = visible.slice(0, 12).map((t) => t.label).join(', ');
+    const shown = visible.slice(0, 12).map((t) => t.label);
+    // Say when the list is cut. A field report found `"Work Orders" is not on
+    // this screen. Visible: …` on a screen whose own element map, three lines
+    // below in the same reply, listed `#25 text 200,836 Work Orders` — it was
+    // simply past the twelfth entry. A truncated list that does not announce
+    // its truncation reads as an exhaustive one.
+    const sample = shown.join(', ') + (visible.length > shown.length ? `, …and ${visible.length - shown.length} more` : '');
+    // An index past the end is not an absent element, and reporting it as one
+    // sent a tester looking for a control that was on the screen all along.
+    // `{"tap": "Work Orders", "index": 1}` on a screen with exactly one match
+    // said "not on this screen" while the map listed it.
+    if (index != null && candidates.length) {
+      throw metrics.tag(
+        new Error(
+          `"${query}" matches ${candidates.length} thing${candidates.length === 1 ? '' : 's'} on this screen,`
+          + ` so index ${index} is out of range — valid indices are 0..${candidates.length - 1}.`
+          + ` Nearest: ${candidates.slice(0, 4).map((c) => `[${candidates.indexOf(c)}] ${JSON.stringify(String(c.label ?? '').slice(0, 28))}`).join(', ')}`,
+        ),
+        'ambiguous_intent',
+        { candidates: candidates.slice(0, 8), intent: query, ambiguous: true },
+      );
+    }
     throw metrics.tag(
       new Error(`"${query}" is not on this screen. Visible: ${sample || '(nothing readable)'}`),
       from === 'memory' ? 'ambiguous_intent' : 'unknown_screen',

package/src/input.js

@@ -623,6 +623,10 @@ export function shouldRebuildSession({ stale, bootedAt }, rebuiltFor) {
 const rebuiltForBoot = new Map();
 export async function ensureFreshSession(udid) {
   if (!udid) return null;
+  // Every input path calls this first, so it is the one place that knows the
+  // hands are about to move. `liveness` needs that timestamp to tell a calm
+  // screen from a dead surface — see `store.noteInput`.
+  store.noteInput(udid);
   const health = await sessionHealth(udid);
   if (!shouldRebuildSession(health, rebuiltForBoot.get(udid))) return null;
   rebuiltForBoot.set(udid, health.bootedAt);

package/src/matching.js

@@ -170,7 +170,23 @@ export function rank(targets, intent, { screen } = {}) {
     // An icon-only control has no readable name, so a synonym is the only way
     // to reach it — this is how "back" finds a bare chevron.
     if (group && base < 0.5 && !t.label && t.rawLabel) base = 0.55;
-    if (group && base < 0.5 && names.some((n) => group.words.includes(norm(n)))) base = 0.9;
+    // Genuine synonymy only: the name must be a *different* word in the group.
+    //
+    // This branch fires only when `base < 0.5`, which means its whole job is to
+    // overrule the coverage scaling the two branches in `nameScore` were taught
+    // — and when the query already contains the name literally, that scaling was
+    // the right answer and this flat 0.9 throws it away. Reported (138): a long
+    // descriptive phrase ending "…under Settings" scored a *heading* labelled
+    // "Settings" at 0.9 and the row the caller meant at 0.265, tapped the
+    // heading, and returned `ok [no visible change]` with an `or` list untried.
+    //
+    // A name the query spells out has already been scored on how much of the
+    // query it covers. What this branch is for is the case that scoring cannot
+    // see at all: "back" reaching a control labelled "Previous", "settings"
+    // reaching "Preferences". That is synonymy, and it is unaffected.
+    const spelledOut = (n) => bare.includes(norm(n)) || norm(intent).includes(norm(n));
+    if (group && base < 0.5
+      && names.some((n) => group.words.includes(norm(n)) && !spelledOut(n))) base = 0.9;
     if (base <= 0) continue;
 
     const reasons = [matched ? `label "${matched}"` : 'icon-only'];