simframe 0.8.0 → 0.10.0

package/src/index.js

@@ -9,6 +9,9 @@ import { decodePng, encodePng, scaleBitmap } from './png.js';
 import {
   REGION_COLS,
   hexToSignature,
+  isBlackFrame,
+  maxCellDelta,
+  CELL_CHANGE,
   regionDeltas,
   regionMap,
   signatureDiff,
@@ -19,6 +22,7 @@ import * as graph from './graph.js';
 import * as matching from './matching.js';
 import * as refs from './refs.js';
 import * as screenmap from './screenmap.js';
+import * as metrics from './metrics.js';
 import { capabilitiesFor, resolveDevice, resize, screenshot } from './platform/index.js';
 import * as store from './store.js';
 
@@ -314,10 +318,24 @@ function stallNote(health) {
   const parts = [`capture: stalled — the display surface has been unreadable for ${Math.round(forMs / 1000)}s`];
   if (health.reattaches) parts.push(`${health.reattaches} re-attach${health.reattaches === 1 ? '' : 'es'} did not help`);
   if (health.reason) parts.push(String(health.reason).slice(0, 120));
-  // The cure is the user's to apply. Saying so is the difference between an
-  // agent that reports "the simulator is wedged" and one that retries a tap
-  // twenty times because nothing appeared to change.
-  parts.push('only restarting the device is known to cure it');
+  // What to do about it, and this line has been wrong until now.
+  //
+  // It said "only restarting the device is known to cure it". Then the same
+  // daemon's log turned out to contain nine "capture recovered on its own"
+  // lines, and a wedge that had survived 223 port re-resolves and 6 device
+  // rebinds cleared by itself while nobody touched it. Meanwhile Apple's own
+  // `simctl io screenshot` on a wedged device returns a valid PNG whose every
+  // pixel is black, in 16 s — so the display pipeline has stopped rendering
+  // and simframe's read is an accurate report of that, not a bug in it.
+  //
+  // So: it is the simulator, it often comes back, and restarting the device
+  // also cures it. `simframe doctor` runs a screenshot probe when this is
+  // published and says which of the two faults it is.
+  parts.push(
+    "this is the simulator's display, not simframe's read of it: Apple's own screenshot path "
+    + 'returns an all-black image on a wedged device. It frequently recovers on its own; '
+    + 'restarting the device also cures it, and `simframe doctor` will confirm which fault this is',
+  );
   return parts.join('; ');
 }
 
@@ -383,6 +401,80 @@ export function resolveBaseline(state, since) {
   return { kind: 'unmatched', requested: key };
 }
 
+/**
+ * Is the baseline describing a screen that had already finished moving?
+ *
+ * Pure, so the rule can be argued with in a test rather than only observed on
+ * a device. The full reasoning is at the call site in `waitFor`; the short
+ * version is that stillness cannot accumulate in the milliseconds between a
+ * dispatch returning and a wait beginning, so a screen that already differs
+ * from the baseline *and* has already been at rest for the whole stillness
+ * window changed for some earlier reason.
+ */
+/** The newest frame's region signature, which the state already carries. */
+function currentSig(state) {
+  const h = state?.history;
+  if (!h?.length) return null;
+  const newest = h.find((x) => x.seq === state.seq) ?? h[h.length - 1];
+  return newest?.sig ?? null;
+}
+
+/**
+ * The longest pause inside a transition, measured after the transition is over.
+ *
+ * This is the unbiased half of the estimator that was reverted in Phase 11, and
+ * the difference is entirely about *when* the measurement stops.
+ *
+ * The biased version accumulated the statistic from inside the wait: the
+ * longest stretch of stillness the wait itself happened to observe. Feed that
+ * back into how long the next wait runs and it eats itself — a wait that ends
+ * early never sees the pauses that come later, so the gaps read as zero, the
+ * window ratchets down, the next wait ends earlier still, and eventually a
+ * settle returns mid-transition and the graph learns a screen it never reached.
+ * That is not a theory; it corrupted this device's graph in one afternoon.
+ *
+ * This one reads the frame history *after* the fact, over a window whose end is
+ * not decided by the wait. The frames are already on disk with their timestamps
+ * and their hashes, so the true profile of a transition is recoverable as long
+ * as the history still reaches back to it.
+ *
+ * That last condition is the whole reason this returns null rather than a
+ * number: the ring is bounded, and during fast motion it holds a second or two.
+ * A partial window would produce a *shorter* gap than really occurred, which is
+ * the exact direction of the bias being removed. Reporting nothing is the only
+ * honest answer to a question the evidence cannot reach.
+ */
+export function longestQuietGap(history, sinceMs, untilMs = Date.now()) {
+  if (!Array.isArray(history) || history.length < 2) return null;
+  const frames = history
+    .filter((f) => Number.isFinite(f?.at) && f.at >= sinceMs && f.at <= untilMs)
+    .sort((a, b) => a.at - b.at);
+  if (frames.length < 2) return null;
+  // The history has to reach back to the action itself. One frame interval of
+  // slack, because the frame that captures the moment of the action is not
+  // required to land exactly on it.
+  const span = frames[1].at - frames[0].at;
+  if (frames[0].at > sinceMs + Math.max(250, span)) return null;
+
+  let longest = 0;
+  let lastChangeAt = frames[0].at;
+  for (let i = 1; i < frames.length; i += 1) {
+    if (frames[i].hash !== frames[i - 1].hash) {
+      longest = Math.max(longest, frames[i].at - lastChangeAt);
+      lastChangeAt = frames[i].at;
+    }
+  }
+  // The quiet after the last change is not a pause *inside* the transition —
+  // it is the transition being over, which is what a settle already measures.
+  return longest;
+}
+
+export function baselineAlreadySettled({ mode, changedAtStart, stableForMs, stableMs } = {}) {
+  if (mode === 'stable' || !changedAtStart) return false;
+  if (!Number.isFinite(stableForMs) || !Number.isFinite(stableMs)) return false;
+  return stableForMs >= stableMs;
+}
+
 function compareToBaseline(state, baseline) {
   if (!baseline) return null;
   if (baseline.kind === 'history') {
@@ -473,7 +565,7 @@ function pngSize(png) {
   return { width: png.readUInt32BE(16), height: png.readUInt32BE(20) };
 }
 
-export async function getState(deviceQuery, { since, options } = {}) {
+export async function getState(deviceQuery, { since, options, inputHealth = false } = {}) {
   const { device, state } = await ensureDaemon(deviceQuery, options);
   return {
     device,
@@ -482,9 +574,58 @@ export async function getState(deviceQuery, { since, options } = {}) {
     map: regionMap(state.regions || [], REGION_COLS),
     since: compareToBaseline(state, resolveBaseline(state, since)),
     live: liveness(device.udid, state),
+    // Costs 32 integer comparisons on a signature already computed, and it is
+    // the difference between "the screen is calm" and "the display stopped
+    // rendering" — which looked identical to everything above this line.
+    black: isBlackFrame(currentSig(state)),
+    // Off by default and asked for by the state commands only. A flow step
+    // calls getState twice, and the fix for a stale session runs before every
+    // action anyway (input.ensureFreshSession) — this is the report, not the
+    // repair.
+    input: inputHealth ? await input.sessionHealth(device.udid) : undefined,
+    timing: inputHealth ? timingOfNow(device.udid, state) : undefined,
   };
 }
 
+/**
+ * How long this screen has been moving, against how long it usually takes.
+ *
+ * Research §7 asks `sim_state` for this, and the point is a sentence an agent
+ * can act on: a screen 400 ms into a transition that normally takes 350 ms is
+ * fine, and the same screen four seconds in is not. Elapsed comes from the
+ * frame store's own clock; the distribution comes from the edge that was most
+ * recently traversed into this screen.
+ *
+ * Reads no frames and takes no perception pass: the layout hash and the
+ * structural hash screen memory filed under it are both already on disk.
+ */
+function timingOfNow(udid, state) {
+  try {
+    const near = screenmap.recallNearest(udid, state.layoutHash);
+    const edge = graph.timingInto(udid, near?.entry?.structuralHash);
+    const elapsed = Number.isFinite(state.lastChangeAt) ? Date.now() - state.lastChangeAt : null;
+    const settled = state.stableForMs >= STRUCTURAL_SETTLE_MS;
+    const slower = graph.slowerThanUsual({
+      elapsedMs: elapsed,
+      p95: edge?.p95,
+      settled,
+      kind: state.transition?.kind,
+    });
+    return {
+      edge_p50: edge?.p50 ?? null,
+      edge_p95: edge?.p95 ?? null,
+      samples: edge?.samples ?? 0,
+      elapsed_ms: elapsed,
+      stable_for_ms: state.stableForMs ?? null,
+      slower_than_usual: Boolean(slower.slower),
+      note: slower.slower ? slower.note : null,
+    };
+  } catch {
+    // Timing is commentary. A screen with no history still has a state.
+    return null;
+  }
+}
+
 /**
  * Wait for the screen to settle (`mode: 'stable'`) or to move away from what it
  * shows right now (`mode: 'change'`). Removes the screenshot-retry loop.
@@ -508,7 +649,7 @@ export async function waitFor(
   const p = store.paths(device.udid);
   const requested = since ?? baselineHash;
   const resolved = resolveBaseline(first, requested);
-  const baselineHashValue =
+  let baselineHashValue =
     resolved?.kind === 'history' ? resolved.entry.hash : (requested ?? first.hash);
   const baselineResolved = resolved?.kind === 'history' || requested == null;
 
@@ -516,16 +657,93 @@ export async function waitFor(
   const deadline = startedAt + timeoutMs;
   const startSeq = first.seq;
   let last = first;
+  /**
+   * The longest pause *inside* this transition, in ms.
+   *
+   * This is the statistic a stillness window exists to defeat: a transition
+   * that pauses for 180 ms mid-flight will be mistaken for a finished screen
+   * by any window shorter than that. Nobody was measuring it, so the window
+   * was a constant — 500 ms, chosen once, paid by every step forever.
+   *
+   * Tracked as the highest `stableForMs` observed before the screen moved
+   * again. Reported so a caller can learn it per edge; a settle that ends
+   * without a second change has no gap to report and says 0.
+   */
+  let quietGapMs = 0;
+  let quietRun = 0;
+  /**
+   * The baseline's own signature, for changes the mean cannot see.
+   *
+   * Only when the baseline was found in the frame history — a hash we cannot
+   * place has no signature to compare against, and guessing one would be worse
+   * than not looking.
+   */
+  const baselineSig = resolved?.kind === 'history' && resolved.entry?.sig
+    ? hexToSignature(resolved.entry.sig)
+    : (requested == null ? hexToSignature(currentSig(first) ?? '') : null);
+  let smallChange = false;
+  /** Frames the display was not rendering at all. See `isBlackFrame`. */
+  let blackFrames = 0;
+  let blackSinceStart = null;
+  let lastHash = first.hash;
   let sawChange = mode === 'stable' || first.hash !== baselineHashValue;
   const changedAtStart = sawChange && mode !== 'stable';
 
+  /**
+   * The baseline describes a screen that has already finished moving.
+   *
+   * `since` means "the screen as it was before the action", and the whole
+   * reliability of these scripts rests on it being captured *before* rather
+   * than after — a baseline sampled afterwards is the commonest way to wait for
+   * a change that already happened. What was missing is the other end of it:
+   * time also passes between capturing the baseline and dispatching the action,
+   * and in a flow step that gap holds a `locate`, a perception pass and a
+   * settle wait — hundreds of milliseconds, not microseconds.
+   *
+   * So a transition can begin *and finish* in that gap, and then `sawChange` is
+   * true at wait start because of the previous action's animation. Measured:
+   * `tap Accessibility` returned `settled 124ms` against a 500 ms stillness
+   * window, the screen had never left the Settings root, and the graph recorded
+   * `root -> root` as a verified edge — count 11, changedOutcomes 5, flipping
+   * between the real destination and itself all day.
+   *
+   * The test is unambiguous rather than clever: the screen differs from the
+   * baseline *and has already been at rest for the full stillness window*.
+   * Stillness cannot have accumulated in the milliseconds between a dispatch
+   * returning and this call starting, so whatever changed, changed and settled
+   * before we looked, and it is not this action's doing. Re-baseline to what is
+   * actually on screen and wait for a further change — which is what the caller
+   * asked for and what a stale hash prevented.
+   *
+   * A screen that differs and is *still moving* is left alone: that is
+   * genuinely ambiguous, and after an action the usual reading is the right
+   * one.
+   */
+  const staleBaseline = baselineAlreadySettled({
+    mode, changedAtStart, stableForMs: first.stableForMs, stableMs,
+  });
+  if (staleBaseline) {
+    baselineHashValue = first.hash;
+    sawChange = false;
+  }
+
   const done = (satisfied, extra = {}) => ({
     device,
     state: last,
     satisfied,
     mode,
+    quietGapMs,
     sawChange,
     changedBeforeWait: changedAtStart,
+    // Something moved, but only in one region — a control changing state
+    // rather than a screen changing.
+    smallChange,
+    staleBaseline,
+    blackFrames,
+    // Said as an observation, never as a diagnosis: a screen can be black
+    // because the app drew black. What makes it the capture wedge is that it
+    // stays black while input is being delivered, and the caller knows that.
+    blackMs: blackSinceStart ? Date.now() - blackSinceStart : 0,
     baselineHash: baselineHashValue,
     baselineResolved,
     waitedMs: Date.now() - startedAt,
@@ -541,8 +759,55 @@ export async function waitFor(
       const live = liveness(device.udid, state);
       if (!live.ok) return done(false, { stalled: true });
 
+      // A black frame is not evidence, in either direction.
+      //
+      // The capture wedge leaves every frame black while the whole capture path
+      // reports success, so before this a settle read the black screen as a
+      // change (the hash differs from anything) and then as a calm one (nothing
+      // moves), and returned `ok` for an action nobody could see the result of.
+      // It self-recovers most times, so the useful behaviour is to keep waiting
+      // rather than to conclude.
+      const black = isBlackFrame(currentSig(state));
+      if (black) {
+        blackFrames += 1;
+        blackSinceStart = blackSinceStart ?? Date.now();
+        await sleep(60);
+        continue;
+      }
+      blackSinceStart = null;
+
       if (!sawChange && state.hash !== baselineHashValue) sawChange = true;
 
+      // A change too small for the whole-screen mean to see.
+      //
+      // A switch flipping moves one cell of thirty-two by 0.043 and the mean by
+      // 0.0013 — a third of the threshold — so every switch, radio dot,
+      // checkbox and segment highlight was an action that "changed nothing",
+      // and `no-visible-change` is a verdict that escalates. See
+      // analyze.CELL_CHANGE for the measured gap this sits in.
+      //
+      // Deliberately feeding `sawChange` and *not* stillness: `stableForMs`
+      // stays on the mean, because a blinking text caret is a small localised
+      // change and a screen with a cursor would otherwise never settle.
+      if (!sawChange && baselineSig) {
+        const sig = currentSig(state);
+        if (sig && maxCellDelta(hexToSignature(sig), baselineSig) > CELL_CHANGE) {
+          sawChange = true;
+          smallChange = true;
+        }
+      }
+
+      // A pause that turned out not to be the end of the transition. Only
+      // pauses followed by more movement count: the quiet at the end of a
+      // settle is the answer, not a gap.
+      if (state.hash !== lastHash) {
+        if (quietRun > quietGapMs) quietGapMs = quietRun;
+        quietRun = 0;
+        lastHash = state.hash;
+      } else if (Number.isFinite(state.stableForMs)) {
+        quietRun = Math.max(quietRun, state.stableForMs);
+      }
+
       // Some controls barely move the screen at all — a radio dot, a checkbox,
       // a button changing state. Waiting the full timeout for a change that
       // will never be visible turns a 100ms action into a 12s one, so give up
@@ -890,8 +1155,19 @@ export async function locate(
       const list = outcome.alternatives
         .map((a, i) => `[${i}] "${a.label}" (${a.x},${a.y}) ${a.region ?? 'content'} ${a.score}`)
         .join(', ');
-      throw new Error(
-        `"${query}" matches ${outcome.alternatives.length} things on this screen — say which, or pass index: ${list}`,
+      // Tagged, not just thrown: the reason an escalation happened is known
+      // here and nowhere above here. See metrics.tag — it adds a property and
+      // changes nothing else about the error.
+      throw metrics.tag(
+        new Error(
+          `"${query}" matches ${outcome.alternatives.length} things on this screen — say which, or pass index: ${list}`,
+        ),
+        'ambiguous_intent',
+        // Present, several times over — as opposed to absent, which also tags
+        // ambiguous_intent when the screen was one we thought we knew. A
+        // waiting caller needs the difference: more time cannot make a thing
+        // unique, and it can make an absent thing arrive.
+        { candidates: outcome.alternatives, ambiguous: true },
       );
     }
     if (outcome.status === 'ok') {
@@ -905,24 +1181,28 @@ export async function locate(
     // here undoes every guard above — it has no off-screen filter and no
     // coverage weighting, and it is what returned a scrolled-away list row for
     // "back". "Not found" is the correct answer.
-    const sample = entry.targets
-      .filter((t) => t.label && t.y >= 0 && t.y <= points.height)
-      .slice(0, 12)
-      .map((t) => t.label.slice(0, 24))
-      .join(', ');
-    throw new Error(`"${query}" is not on this screen. Visible: ${sample || '(nothing readable)'}`);
+    const visible = entry.targets.filter((t) => t.label && t.y >= 0 && t.y <= points.height);
+    const sample = visible.slice(0, 12).map((t) => t.label.slice(0, 24)).join(', ');
+    // Which escalation this is depends on whether the screen was recognised.
+    // Screen memory had nothing for it (`from` is one of the built values) and
+    // the target is missing: that is not knowing the screen. On a screen
+    // recalled from memory, the screen is known and the intent did not resolve.
+    throw metrics.tag(
+      new Error(`"${query}" is not on this screen. Visible: ${sample || '(nothing readable)'}`),
+      from === 'memory' ? 'ambiguous_intent' : 'unknown_screen',
+      { candidates: visible.slice(0, 8) },
+    );
   }
 
   const candidates = screenmap.rank(entry, query);
   const target = index != null ? candidates[index] : candidates[0];
   if (!target) {
-    const sample = entry.targets
-      .filter((t) => t.label)
-      .slice(0, 12)
-      .map((t) => t.label)
-      .join(', ');
-    throw new Error(
-      `"${query}" is not on this screen. Visible: ${sample || '(nothing readable)'}`,
+    const visible = entry.targets.filter((t) => t.label);
+    const sample = visible.slice(0, 12).map((t) => t.label).join(', ');
+    throw metrics.tag(
+      new Error(`"${query}" is not on this screen. Visible: ${sample || '(nothing readable)'}`),
+      from === 'memory' ? 'ambiguous_intent' : 'unknown_screen',
+      { candidates: visible.slice(0, 8) },
     );
   }
   return { device, state: current, entry, target, from, distance, settled, screens: screenmap.stats(udid).screens };
@@ -941,6 +1221,38 @@ export async function locate(
  */
 export const STRUCTURAL_SETTLE_MS = 300;
 
+/**
+ * How much of the structural window is still owed, given when the last sample's
+ * frame was captured.
+ *
+ * This was `sleep(300)` between the two readings, and unlike every other fixed
+ * wait in the engine it cannot be replaced by waiting for a signal — because
+ * there is no signal. The race it guards is a screen whose *pixels* have gone
+ * still while its structure has not: a list whose spinner has gone and whose
+ * rows have not landed is perfectly quiet and structurally wrong, so the settle
+ * detector, which watches pixels, has nothing to report. Only elapsed time
+ * separates the two readings.
+ *
+ * What can be fixed is that the wait was *additional*. The guarantee wanted is
+ * 300 ms between the frames the two samples read; the code slept 300 ms after a
+ * sample that had already spent an unbounded settle wait and a full perception
+ * pass getting there. On a screen that took a second to go quiet the separation
+ * was already there and the sleep bought nothing but a second of it. So credit
+ * what has passed and wait only for the remainder — the same guarantee, and
+ * usually none of the sleep.
+ *
+ * The window itself is per-screen learnable, and worth noting that its
+ * estimator has the *opposite* feedback sign to the one that corrupted the
+ * graph: a window too short produces disagreeing samples, which lengthens it.
+ * Self-correcting rather than self-reinforcing. It still waits on the
+ * perception eval harness, because "the samples agreed" is only evidence the
+ * window was long enough if the readings themselves are trustworthy.
+ */
+export function structuralSettleOwed(capturedAt, now = Date.now(), windowMs = STRUCTURAL_SETTLE_MS) {
+  if (!Number.isFinite(capturedAt)) return windowMs;
+  return Math.max(0, Math.min(windowMs, windowMs - (now - capturedAt)));
+}
+
 /**
  * How long identity will wait for pixels to go quiet.
  *
@@ -1024,7 +1336,8 @@ export async function screenIdentity(deviceQuery, { options, confirmNovel = true
   // Nothing recognises this, or the pixels have not gone quiet. Either way, make
   // it prove it is the same screen twice running before it becomes a node.
   for (let i = 1; i < STRUCTURAL_SETTLE_SAMPLES; i += 1) {
-    await sleep(STRUCTURAL_SETTLE_MS);
+    const owed = structuralSettleOwed(identity.state?.capturedAt);
+    if (owed > 0) await sleep(owed);
     const again = await read({ fresh: true });
     // Two readings agree if they are the same screen — the same test identity
     // itself uses. Demanding an identical hash is a stricter question than the