simframe 0.12.1 → 0.13.0

package/src/actions.js

@@ -260,6 +260,8 @@ export async function runScript(
         expect,
         failure,
         outcome,
+        supervisor: supervisor.requested(options),
+        situation: ruling?.context?.situation ?? null,
       });
     } catch {
       /* instrumentation must not be able to fail a flow it is only watching */
@@ -354,12 +356,19 @@ export async function runScript(
         }),
         observedMs: null,
       };
+      // Where the hands actually went, filled in by the step that moved them.
+      //
+      // Item 120 needs the *resolved* point, not the one in the script: an
+      // image-space `tapAt` is converted inside the step, and `scroll` invents
+      // its coordinates from the screen size. Reading them back off the step
+      // would diagnose a point nothing was ever aimed at.
+      const aim = { at: null };
       let detail;
       // A selector that did not resolve gets the step's own alternatives before
       // the batch is abandoned. Anything else propagates: retrying from a screen
       // we did not expect to be on is not a retry, it is a second guess.
       try {
-        detail = await runStep(deviceQuery, udid, step, { screen, options, frames, focus });
+        detail = await runStep(deviceQuery, udid, step, { screen, options, frames, focus, aim });
       } catch (thrown) {
         let err = thrown;
         // Ask the supervisor before anything is abandoned. It sits behind the
@@ -393,7 +402,7 @@ export async function runScript(
             options,
           }).catch(() => null);
           try {
-            detail = await runStep(deviceQuery, udid, step, { screen, options, frames, focus });
+            detail = await runStep(deviceQuery, udid, step, { screen, options, frames, focus, aim });
             detail += ` [the local supervisor said ${ruling.decision}; it worked on the second attempt]`;
             ruled('recovered');
             continue;
@@ -433,7 +442,7 @@ export async function runScript(
         let last = err;
         for (const label of allowed) {
           try {
-            detail = await runStep(deviceQuery, udid, stepWithTarget(step, label), { screen, options, frames, focus });
+            detail = await runStep(deviceQuery, udid, stepWithTarget(step, label), { screen, options, frames, focus, aim });
             detail += ` [after ${tried.map((t) => JSON.stringify(String(t))).join(', ')} did not resolve]`;
             last = null;
             break;
@@ -729,10 +738,40 @@ export async function runScript(
         ? ' [the screen did not change, so this app was already in front — or it did not come forward]'
         : '';
       const filling = stillFillingIn(afterReading?.entry);
+      // Item 120: when a gesture aimed at a coordinate does nothing, say what
+      // that coordinate resolved to. The element that swallowed it is normally
+      // already in the list printed under this very verdict — the gap was
+      // never the data, it was that nobody connected "started at y=750" to
+      // "there is a banner at y=753".
+      //
+      // Diagnosed against the screen as it was *before* the action, because
+      // that is the screen the finger landed on. Using the after-reading would
+      // describe the world the gesture failed to change.
+      // Both signals, because they are not the same one and only one of them
+      // fires in the reported case. `settled.noVisibleChange` is the pixel
+      // detector saying the frame never moved; the `no-visible-change` verdict
+      // is the fingerprint saying we are on the screen we started on. A swipe
+      // into a search bar settled in 62ms and produced the second without the
+      // first — and six swipes reporting "no visible change" is what item 120
+      // was reported against. Keying on one of them would have shipped a fix
+      // that did not fire on its own bug report.
+      const wentNowhere = Boolean(settled?.noVisibleChange) || verification?.verdict === 'no-visible-change';
+      const aimNote = wentNowhere && aim.at
+        ? screenmap.describePoint(
+          // `beforeScreen` is null with verification off, and that is exactly
+          // the mode someone falls back to when coordinates are misbehaving.
+          // A stored map keyed on the pre-action frame costs one file read.
+          beforeScreen?.entry ?? screenmap.recall(udid, before),
+          aim.at.point,
+          { what: aim.at.what },
+        )
+        : null;
       const note = launchNote
         + (filling ? ` [settled, but ${filling} — waitFor content, do not act on this yet]` : '')
         + (settled?.smallChange ? ' [a small change, in one region only]' : '')
         + (settled?.noVisibleChange ? ' [no visible change]' : '')
+        // After the symptom, because it is the explanation of it.
+        + (aimNote ? ` [${aimNote}]` : '')
         + (settled?.staleBaseline ? ' [baseline had already settled; re-taken from the live screen]' : '')
         + (settled?.blackFrames
           ? ` [${settled.blackFrames} black frame(s) waited through${settled.blackMs ? `, still black after ${settled.blackMs}ms` : ''}]`
@@ -780,6 +819,22 @@ export async function runScript(
           fingerprint: beforeScreen?.hash ?? null,
           reason: 'verification_failed',
           candidates: [],
+          // Assumed, not read. A verdict says the step did not do what was
+          // expected; it does not say which faculty would have prevented that.
+          //
+          // Measured in the field and it matters: on an app whose controls are
+          // largely unlabeled, `no-visible-change` came overwhelmingly from
+          // tapping an inert text label whose real hit target was an invisible
+          // chevron — icon semantics, Phase 15 — while this line filed every
+          // one as evidence about sense of time, Phase 11. The tester reached
+          // the right conclusion from their own notes and our instrument
+          // disagreed with them. It was wrong.
+          //
+          // Not reclassified here, because `no-visible-change` also covers a
+          // switch moving 0.1% of the screen, which is neither faculty. Saying
+          // "assumed" is the honest answer; guessing a better-sounding reason
+          // would be the same mistake in the other direction.
+          classified: false,
           // `verification_failed` is the largest reason class in the log and it
           // was the only one carrying no intent, which made most of the corpus
           // useless for asking what kind of decision costs us. The step knows
@@ -815,6 +870,7 @@ export async function runScript(
         // Carried from the throw site where it exists, and otherwise the step's
         // own target — which is what was asked for either way.
         intent: why.intent ?? goalOf(step),
+        classified: why.classified,
         outcome: 'failed',
         wallMs: Date.now() - stepStart,
         detail: err.message,
@@ -1215,17 +1271,25 @@ async function superviseFailure(deviceQuery, { goal, step, expected, err, option
     try {
       timing = udid && map.identity ? graph.timingFor(udid, map.identity, step) : null;
     } catch { /* an unknown edge is a fact about the graph, not a failure here */ }
-    const ruling = await supervisor.judge({
-      goal,
+    // The question, kept — not just the answer.
+    //
+    // Three items (101, 96, 106) want to ask "what would a different judge have
+    // said about these same situations", and until now the log held the verdict
+    // and threw away what was asked. So comparing two supervisors meant driving
+    // the device once per arm, which is thirty minutes an arm and introduces
+    // the device's own variance into a comparison that is supposed to be about
+    // the judges. With this, one device pass produces a population every arm
+    // can be asked, on identical inputs.
+    const situation = {
+      goal: goal ?? null,
       step: `${step.action} ${JSON.stringify(String(step.value ?? step.target ?? step.into ?? step.seek ?? '').slice(0, 60))}`,
-      expected,
+      expected: expected ?? null,
       failure: err.message,
       screen: (map.rows ?? []).filter((r) => r.label).map((r) => r.label),
       stillMs,
       note: stillFillingIn(map.identity?.entry),
-      options,
-      detail,
-    });
+    };
+    const ruling = await supervisor.judge({ ...situation, options, detail });
     if (!ruling) return null;
     return {
       ...ruling,
@@ -1237,6 +1301,7 @@ async function superviseFailure(deviceQuery, { goal, step, expected, err, option
         stillMs,
         p95: timing?.p95 ?? null,
         samples: timing?.samples ?? null,
+        situation,
       },
     };
   } catch {
@@ -2166,6 +2231,7 @@ async function runStep(deviceQuery, udid, step, ctx) {
           pointHeight: geo.pointHeight,
         }));
       }
+      if (ctx.aim) ctx.aim.at = { point: { x, y }, what: 'the tap point' };
       await input.tapPoint(udid, x, y, { durationMs: step.durationMs });
       return `tapped ${x},${y}`;
     }
@@ -2253,6 +2319,9 @@ async function runStep(deviceQuery, udid, step, ctx) {
     case 'swipe': {
       const from = { x: step.from?.[0] ?? step.from?.x, y: step.from?.[1] ?? step.from?.y };
       const to = { x: step.to?.[0] ?? step.to?.x, y: step.to?.[1] ?? step.to?.y };
+      // The start point only. A swipe is captured by whatever the finger goes
+      // down on; where it lifts never decides who received it.
+      if (ctx.aim) ctx.aim.at = { point: from, what: 'the swipe start point' };
       await input.swipe(udid, from, to, { durationMs: step.durationMs });
       return `swiped ${from.x},${from.y} -> ${to.x},${to.y}`;
     }
@@ -2269,6 +2338,7 @@ async function runStep(deviceQuery, udid, step, ctx) {
         right: [{ x: midX - span, y: midY }, { x: midX + span, y: midY }],
       };
       if (!moves[dir]) throw new Error(`unknown scroll direction "${dir}"`);
+      if (ctx.aim) ctx.aim.at = { point: moves[dir][0], what: 'the scroll start point' };
       await input.swipe(udid, moves[dir][0], moves[dir][1], { durationMs: step.durationMs ?? 250 });
       return `scrolled ${dir}`;
     }
@@ -2316,8 +2386,26 @@ async function runStep(deviceQuery, udid, step, ctx) {
         timeoutMs: step.timeoutMs ?? 8000,
         options: ctx.options,
       });
-      if (!w.satisfied) throw new Error(w.stalled ? w.live.note : `screen did not settle within ${w.waitedMs}ms`);
-      return `settled after ${w.waitedMs}ms`;
+      // Say where it is still moving — item 123.
+      //
+      // "did not settle within 25000ms" is true and unactionable, and a failed
+      // step aborts the rest of the batch, so a spinner nobody cares about can
+      // cost five steps that would have worked. The reporter who asked for this
+      // had landed on `pause` plus `continueOnError` and called it *"strictly
+      // worse than a settle that knows what to ignore"*. Knowing what to ignore
+      // is the expensive half and is not built. Saying where is nearly free —
+      // the signatures were already read to detect small changes — and it is
+      // the difference between re-planning a flow and ignoring a corner of it.
+      if (!w.satisfied) {
+        if (w.stalled) throw new Error(w.live.note);
+        throw new Error(`screen did not settle within ${w.waitedMs}ms — ${settleEvidence(w)}`);
+      }
+      // A settle that satisfied while part of the screen is still moving says
+      // so. The daemon has boxed the moving part all along and nothing read it;
+      // measured at 79s of claimed stillness on a screen animating at 85ms a
+      // frame. Not treated as a failure — see the note on `animatingNow`.
+      return `settled after ${w.waitedMs}ms`
+        + (w.animating ? ` (a ${w.animating.width}x${w.animating.height} region is still animating)` : '');
     }
     case 'waitText': {
       const target = step.value ?? step.text;
@@ -2332,10 +2420,12 @@ async function runStep(deviceQuery, udid, step, ctx) {
           // Same rule as `waitFor`, and `matchElement` says it in its own
           // words: a query that matched several elements has found them all
           // already.
-          if (/matched \d+ elements/.test(err.message)) {
-            throw new Error(
-              `${err.message}\n  (not waiting: it is already on screen, and waiting cannot make it unique)`,
-            );
+          // Satisfies the wait, for the same reason as `waitFor` above: several
+          // matches is an answer of "yes, it is here".
+          const many = err.message.match(/matched (\d+) elements/);
+          if (many) {
+            return `"${target}" is on screen (${many[1]} matches)`
+              + ' — the wait is satisfied; pass an index to act on one of them';
           }
         }
         await sleep(250);
@@ -2500,10 +2590,22 @@ async function runStep(deviceQuery, udid, step, ctx) {
           //
           // Distinguished by the tag at the throw site rather than by reading
           // the message, because "not on this screen" tags the same reason.
+          //
+          // And it **satisfies** the wait rather than failing it, which is the
+          // correction. The paragraph above had the reasoning right — ambiguous
+          // means the target is present, several times over — and then threw
+          // anyway. A `waitFor` asks one question, *has it arrived*, and two
+          // matches is a yes. Reported from the field: the batch was abandoned
+          // and three queued steps discarded on a screen that was exactly where
+          // the flow wanted to be, and the tester had to re-issue the lot with
+          // an index. The ambiguity is real and belongs in the next step's
+          // selector, not in this step's verdict.
           if (metrics.escalationOf(err)?.ambiguous) {
-            throw new Error(
-              `${err.message}\n  (not waiting: it is already on screen, and waiting cannot make it unique)`,
-            );
+            // The count comes from the message because the tag is a boolean —
+            // it marks *which kind* of ambiguity, not how many.
+            const n = err.message.match(/matches (\d+) things/)?.[1];
+            return `${query} is on screen${n ? ` (${n} matches)` : ' more than once'}`
+              + ' — the wait is satisfied; pass an index or a #ref to act on one of them';
           }
         }
         if (Date.now() >= limit) break;
@@ -2604,3 +2706,41 @@ async function runStep(deviceQuery, udid, step, ctx) {
       throw new Error(`unknown step "${step.action}"`);
   }
 }
+
+/**
+ * Why a settle gave up, in the numbers that decided it.
+ *
+ * `screen did not settle within 25008ms` names the one quantity that is never
+ * the reason, and two CI cycles went into guessing what was behind it. A wait
+ * turns on three things and the message carried none of them:
+ *
+ *   - how long the screen had actually been still when we gave up
+ *   - how much stillness was being asked for
+ *   - whether any frame arrived during the call at all
+ *
+ * *"Still for 1,200 ms of the 1,400 required"* and *"still for 135,000 ms and no
+ * frame ever arrived"* are opposite diagnoses — the first is a budget a hair too
+ * tight, the second is a stalled pipeline wearing a calm screen — and they had
+ * one sentence between them. So did *"something is animating in the top right"*.
+ *
+ * Ordered by what to do next: what is moving, then how close stillness got,
+ * then whether anything was observed, then black frames.
+ */
+export function settleEvidence(w) {
+  const parts = [];
+  const m = w.motion;
+  if (m) parts.push(`the movement is ${m.where}${m.localised ? ` (${m.share}% of it)` : ''}`);
+  if (Number.isFinite(w.stillForMs) && Number.isFinite(w.stableMsRequired)) {
+    parts.push(`still for ${w.stillForMs}ms of the ${w.stableMsRequired}ms required`);
+  }
+  // Zero frames is the loud one: a wait that observed nothing has not measured
+  // this screen, it has measured a file on disk.
+  if (Number.isFinite(w.framesSeen)) {
+    parts.push(w.framesSeen > 0
+      ? `${w.framesSeen} frame(s) arrived while waiting`
+      : 'NO frame arrived while waiting — capture, not the screen');
+  }
+  if (w.blackFrames > 0) parts.push(`${w.blackFrames} black frame(s) — see the capture wedge`);
+  return (parts.length ? parts.join('; ') : 'nothing observed')
+    + (m ? `\n${m.map}` : '');
+}

package/src/analyze.js

@@ -65,6 +65,25 @@ export function signatureDiff(a, b) {
  */
 export const CELL_CHANGE = 0.012;
 
+/**
+ * How far two *capture paths* may differ and still be looking at one screen.
+ *
+ * Not the change threshold, and the distinction matters. `signatureDiff > 0.004`
+ * asks whether a screen moved between two frames from the *same* path. This asks
+ * whether the daemon's frame and an independent `simctl` screenshot show the
+ * same thing — and they never match closely, because one is downscaled by the
+ * capture loop and the other is a full-resolution PNG scaled here.
+ *
+ * Measured on this device, which is the only reason a number appears:
+ *
+ *   same screen, two paths       0.00123  (three runs, identical to five places)
+ *   two different screens        0.68603  (before and after a home press)
+ *
+ * A separation of 550x, so the threshold is not delicate. 0.02 is sixteen times
+ * the scaling cost and thirty-four times under the signal.
+ */
+export const PATHS_AGREE = 0.02;
+
 /** The largest single-region change between two signatures, 0-1. */
 export function maxCellDelta(a, b) {
   const deltas = regionDeltas(a, b);
@@ -100,6 +119,43 @@ export function regionMap(deltas, cols = REGION_COLS) {
   return lines.join('\n');
 }
 
+/**
+ * Where the movement is, in words — item 123.
+ *
+ * A settle that gives up says only that it gave up, and the reporter who asked
+ * for this put the cost plainly: a streaming summary panel, a Lottie and a
+ * support widget never settle, so the settle fails, and because a failed step
+ * aborts the batch it takes the other five steps with it. Their workaround was
+ * `pause` plus `continueOnError`, which they called *"strictly worse than a
+ * settle that knows what to ignore"*. Knowing what to ignore is the expensive
+ * half. Saying **where** is nearly free, and it is what turns "did not settle"
+ * into "there is a spinner in the top-right and nobody cares about it".
+ *
+ * Deliberately coarse. Thirds of the screen in each axis, named the way a person
+ * would point at it, and a share so a caller can tell one spinner from a screen
+ * that is genuinely still in flight.
+ */
+export function describeMotion(deltas, cols = REGION_COLS) {
+  const total = deltas.reduce((a, b) => a + b, 0);
+  if (!deltas.length || total <= 0) return null;
+  const rows = Math.ceil(deltas.length / cols);
+  const bands = new Map();
+  const third = (i, n) => (i < n / 3 ? 0 : i < (2 * n) / 3 ? 1 : 2);
+  const DOWN = ['top', 'middle', 'bottom'];
+  const ACROSS = ['left', 'centre', 'right'];
+  deltas.forEach((d, i) => {
+    const key = `${DOWN[third(Math.floor(i / cols), rows)]}-${ACROSS[third(i % cols, cols)]}`;
+    bands.set(key, (bands.get(key) ?? 0) + d);
+  });
+  const ranked = [...bands.entries()].sort((a, b) => b[1] - a[1]);
+  const [where, amount] = ranked[0];
+  const share = Math.round((amount / total) * 100);
+  // A single band holding most of the movement is a localised animation; spread
+  // evenly, the whole screen is in flight and naming a corner would mislead.
+  if (share < 40) return { where: 'spread across the screen', share, localised: false };
+  return { where: where.replace('-', ' '), share, localised: true };
+}
+
 /** Signatures live in state.json, so they are stored as compact hex. */
 export function signatureToHex(sig) {
   return sig.map((v) => v.toString(16).padStart(2, '0')).join('');