simframe 0.12.2 → 0.14.0

package/src/actions.js

@@ -819,6 +819,22 @@ export async function runScript(
           fingerprint: beforeScreen?.hash ?? null,
           reason: 'verification_failed',
           candidates: [],
+          // Assumed, not read. A verdict says the step did not do what was
+          // expected; it does not say which faculty would have prevented that.
+          //
+          // Measured in the field and it matters: on an app whose controls are
+          // largely unlabeled, `no-visible-change` came overwhelmingly from
+          // tapping an inert text label whose real hit target was an invisible
+          // chevron — icon semantics, Phase 15 — while this line filed every
+          // one as evidence about sense of time, Phase 11. The tester reached
+          // the right conclusion from their own notes and our instrument
+          // disagreed with them. It was wrong.
+          //
+          // Not reclassified here, because `no-visible-change` also covers a
+          // switch moving 0.1% of the screen, which is neither faculty. Saying
+          // "assumed" is the honest answer; guessing a better-sounding reason
+          // would be the same mistake in the other direction.
+          classified: false,
           // `verification_failed` is the largest reason class in the log and it
           // was the only one carrying no intent, which made most of the corpus
           // useless for asking what kind of decision costs us. The step knows
@@ -854,6 +870,7 @@ export async function runScript(
         // Carried from the throw site where it exists, and otherwise the step's
         // own target — which is what was asked for either way.
         intent: why.intent ?? goalOf(step),
+        classified: why.classified,
         outcome: 'failed',
         wallMs: Date.now() - stepStart,
         detail: err.message,
@@ -2369,8 +2386,26 @@ async function runStep(deviceQuery, udid, step, ctx) {
         timeoutMs: step.timeoutMs ?? 8000,
         options: ctx.options,
       });
-      if (!w.satisfied) throw new Error(w.stalled ? w.live.note : `screen did not settle within ${w.waitedMs}ms`);
-      return `settled after ${w.waitedMs}ms`;
+      // Say where it is still moving — item 123.
+      //
+      // "did not settle within 25000ms" is true and unactionable, and a failed
+      // step aborts the rest of the batch, so a spinner nobody cares about can
+      // cost five steps that would have worked. The reporter who asked for this
+      // had landed on `pause` plus `continueOnError` and called it *"strictly
+      // worse than a settle that knows what to ignore"*. Knowing what to ignore
+      // is the expensive half and is not built. Saying where is nearly free —
+      // the signatures were already read to detect small changes — and it is
+      // the difference between re-planning a flow and ignoring a corner of it.
+      if (!w.satisfied) {
+        if (w.stalled) throw new Error(w.live.note);
+        throw new Error(`screen did not settle within ${w.waitedMs}ms — ${settleEvidence(w)}`);
+      }
+      // A settle that satisfied while part of the screen is still moving says
+      // so. The daemon has boxed the moving part all along and nothing read it;
+      // measured at 79s of claimed stillness on a screen animating at 85ms a
+      // frame. Not treated as a failure — see the note on `animatingNow`.
+      return `settled after ${w.waitedMs}ms`
+        + (w.animating ? ` (a ${w.animating.width}x${w.animating.height} region is still animating)` : '');
     }
     case 'waitText': {
       const target = step.value ?? step.text;
@@ -2385,10 +2420,12 @@ async function runStep(deviceQuery, udid, step, ctx) {
           // Same rule as `waitFor`, and `matchElement` says it in its own
           // words: a query that matched several elements has found them all
           // already.
-          if (/matched \d+ elements/.test(err.message)) {
-            throw new Error(
-              `${err.message}\n  (not waiting: it is already on screen, and waiting cannot make it unique)`,
-            );
+          // Satisfies the wait, for the same reason as `waitFor` above: several
+          // matches is an answer of "yes, it is here".
+          const many = err.message.match(/matched (\d+) elements/);
+          if (many) {
+            return `"${target}" is on screen (${many[1]} matches)`
+              + ' — the wait is satisfied; pass an index to act on one of them';
           }
         }
         await sleep(250);
@@ -2518,7 +2555,7 @@ async function runStep(deviceQuery, udid, step, ctx) {
         for (let attempt = 0; ; attempt += 1) {
           for (const one of alternatives) {
             try {
-              const found = await api.locate(deviceQuery, one, { refresh: attempt > 0 });
+              const found = await api.locate(deviceQuery, one, { refresh: step.refresh !== false, options: ctx.options });
               return `${JSON.stringify(one)} appeared at ${found.target.x},${found.target.y}`
                 + ` (first of ${alternatives.length} awaited)`;
             } catch (err) {
@@ -2526,10 +2563,12 @@ async function runStep(deviceQuery, udid, step, ctx) {
             }
           }
           if (Date.now() >= limit) {
+            const stillAny = await stillnessNote(deviceQuery, ctx);
             throw new Error(
               `none of ${alternatives.length} awaited strings appeared`
               + ` (${alternatives.map((a) => JSON.stringify(a)).join(', ')}) in ${step.timeoutMs ?? 8000}ms.`
-              + ` Last: ${lastError}`,
+              + ` Last: ${lastError}`
+              + (stillAny ? ` (${stillAny.note})` : ''),
             );
           }
           await api.waitFor(deviceQuery, { mode: 'stable', stableMs: 200, timeoutMs: 700, options: ctx.options })
@@ -2538,7 +2577,26 @@ async function runStep(deviceQuery, udid, step, ctx) {
       }
       for (let attempt = 0; ; attempt += 1) {
         try {
-          const found = await api.locate(deviceQuery, query, { index: step.index, refresh: attempt > 0 });
+          // Fresh, like `assert` — and for the same reason, found the same way.
+          //
+          // This read `refresh: attempt > 0`, so the FIRST look resolved against
+          // the remembered map. A wait that can be satisfied by memory is not a
+          // wait: if recall hands back the wrong screen's map — which it does
+          // when two screens collide on a layout hash — the target "appears"
+          // without ever having been on screen, instantly, and the caller
+          // proceeds against a screen it is not on.
+          //
+          // Caught by the fingerprint eval on CI, which is the only place it
+          // could show: `settings-general` read the **Settings root** and its
+          // `waitFor "About"` had passed. The signature is in the round numbers
+          // — r2 and r3, never r1 — because memory has to be warm before it can
+          // lie, and round 1 is always cold.
+          //
+          // `assert` was made fresh by default after this exact defect cost a
+          // reported session; `waitFor` was left as it was. One perception pass
+          // is the price, and it is the same trade: a read is cheaper than the
+          // round trip a wrong verdict causes.
+          const found = await api.locate(deviceQuery, query, { index: step.index, refresh: step.refresh !== false, options: ctx.options });
           return `"${found.target.label}" appeared at ${found.target.x},${found.target.y}`;
         } catch (err) {
           lastError = err.message;
@@ -2553,16 +2611,42 @@ async function runStep(deviceQuery, udid, step, ctx) {
           //
           // Distinguished by the tag at the throw site rather than by reading
           // the message, because "not on this screen" tags the same reason.
+          //
+          // And it **satisfies** the wait rather than failing it, which is the
+          // correction. The paragraph above had the reasoning right — ambiguous
+          // means the target is present, several times over — and then threw
+          // anyway. A `waitFor` asks one question, *has it arrived*, and two
+          // matches is a yes. Reported from the field: the batch was abandoned
+          // and three queued steps discarded on a screen that was exactly where
+          // the flow wanted to be, and the tester had to re-issue the lot with
+          // an index. The ambiguity is real and belongs in the next step's
+          // selector, not in this step's verdict.
           if (metrics.escalationOf(err)?.ambiguous) {
-            throw new Error(
-              `${err.message}\n  (not waiting: it is already on screen, and waiting cannot make it unique)`,
-            );
+            // The count comes from the message because the tag is a boolean —
+            // it marks *which kind* of ambiguity, not how many.
+            const n = err.message.match(/matches (\d+) things/)?.[1];
+            return `${query} is on screen${n ? ` (${n} matches)` : ' more than once'}`
+              + ' — the wait is satisfied; pass an index or a #ref to act on one of them';
           }
         }
         if (Date.now() >= limit) break;
+        // Opt-in: stop early on a screen that has plainly stopped changing.
+        if (Number.isFinite(step.failIfStillFor)) {
+          const still = await stillnessNote(deviceQuery, ctx);
+          if (still && still.ms >= step.failIfStillFor) {
+            throw new Error(`gave up on ${query} after ${Date.now() - (limit - (step.timeoutMs ?? 8000))}ms:`
+              + ` ${still.note}, which is past the ${step.failIfStillFor}ms you said to stop at.`
+              + ` Last: ${lastError}`);
+          }
+        }
         await sleep(POLL_MS);
       }
-      throw new Error(`waited ${step.timeoutMs ?? 8000}ms for ${query}: ${lastError}`);
+      // Say how long it had been still. A wait that burned three minutes on a
+      // screen static for the last twelve seconds should not make the reader
+      // work that out from a second command.
+      const still = await stillnessNote(deviceQuery, ctx);
+      throw new Error(`waited ${step.timeoutMs ?? 8000}ms for ${query}: ${lastError}`
+        + (still ? ` (${still.note} — pass failIfStillFor to stop early next time)` : ''));
     }
 
     // One assert step for every condition, because `assertText` could only ask
@@ -2590,6 +2674,32 @@ async function runStep(deviceQuery, udid, step, ctx) {
         // is the opposite of what you want learned."*
         found = await api.locate(deviceQuery, query, { index: step.index, refresh: step.refresh !== false, options: ctx.options });
       } catch (err) {
+        // Ambiguity answers "is it there", and it answers it *yes*.
+        //
+        // `locate` refuses a query that matches several elements, which is right
+        // for `tap` — picking the wrong one of two taps the wrong thing — and
+        // wrong for a question about presence. Reported from the field on
+        // 0.13.0: `assert Administrator is visible` failed, and aborted the rest
+        // of the batch, against a screen with **two** Administrators on it. The
+        // assertion's own semantics were satisfied twice over. The message was
+        // praised in the same breath — candidates, coordinates and scores — so
+        // what was wrong was the verdict, not the diagnosis.
+        //
+        // The `gone` direction had the mirror-image bug and nobody had hit it
+        // yet: any error at all returned "is gone", so a query matching two
+        // visible elements would have reported them absent. Ambiguity is the one
+        // error that is positive evidence of presence, and it was being read as
+        // proof of absence.
+        //
+        // Strict single-match resolution stays for `enabled`, `disabled` and
+        // `value`, where the question is about a *particular* element and
+        // answering it from the wrong one is how a confident wrong answer gets
+        // made.
+        if (err.ambiguous && err.candidates?.length) {
+          const n = err.candidates.length;
+          if (want === 'visible') return `${query} is visible (${n} things match it here — a presence check does not have to choose)`;
+          if (want === 'gone') throw new Error(`${query}: still here — ${n} things on this screen match it`);
+        }
         if (want === 'gone') return `${query} is gone`;
         throw new Error(`${query}: ${err.message}`);
       }
@@ -2657,3 +2767,93 @@ async function runStep(deviceQuery, udid, step, ctx) {
       throw new Error(`unknown step "${step.action}"`);
   }
 }
+
+/**
+ * Why a settle gave up, in the numbers that decided it.
+ *
+ * `screen did not settle within 25008ms` names the one quantity that is never
+ * the reason, and two CI cycles went into guessing what was behind it. A wait
+ * turns on three things and the message carried none of them:
+ *
+ *   - how long the screen had actually been still when we gave up
+ *   - how much stillness was being asked for
+ *   - whether any frame arrived during the call at all
+ *
+ * *"Still for 1,200 ms of the 1,400 required"* and *"still for 135,000 ms and no
+ * frame ever arrived"* are opposite diagnoses — the first is a budget a hair too
+ * tight, the second is a stalled pipeline wearing a calm screen — and they had
+ * one sentence between them. So did *"something is animating in the top right"*.
+ *
+ * Ordered by what to do next: what is moving, then how close stillness got,
+ * then whether anything was observed, then black frames.
+ */
+export function settleEvidence(w) {
+  const parts = [];
+  const m = w.motion;
+  if (m) parts.push(`the movement is ${m.where}${m.localised ? ` (${m.share}% of it)` : ''}`);
+  if (Number.isFinite(w.stillForMs) && Number.isFinite(w.stableMsRequired)) {
+    parts.push(`still for ${w.stillForMs}ms of the ${w.stableMsRequired}ms required`);
+  }
+  // Zero frames is the loud one: a wait that observed nothing has not measured
+  // this screen, it has measured a file on disk.
+  if (Number.isFinite(w.framesSeen)) {
+    parts.push(w.framesSeen > 0
+      ? `${w.framesSeen} frame(s) arrived while waiting`
+      : 'NO frame arrived while waiting — capture, not the screen');
+  }
+  if (w.blackFrames > 0) parts.push(`${w.blackFrames} black frame(s) — see the capture wedge`);
+  return (parts.length ? parts.join('; ') : 'nothing observed')
+    + (m ? `\n${m.map}` : '');
+}
+
+/**
+ * The one-line flow summary, so `ok` and `FAIL` each mean exactly one thing.
+ *
+ * `FLOW FAILED — 3/3 steps` was reported from the field as reading like a
+ * success, and the reporter had it exactly: the denominator means *attempted*
+ * on failure and *succeeded* on success, so the same shape carries opposite
+ * meanings. `flow completed — 5/5 steps` and `FLOW FAILED — 3/3 steps` differ
+ * only in a word, and the numbers argue against the word.
+ *
+ * On failure it says how many worked and how many did not, which is the thing a
+ * caller has to know to decide whether to resume or re-plan.
+ */
+export function flowSummary(res, { withTime = true } = {}) {
+  const time = withTime && Number.isFinite(res.totalMs) ? ` in ${res.totalMs}ms` : '';
+  if (res.ok) return `flow completed — ${res.ranSteps}/${res.totalSteps} steps${time}`;
+  const failed = (res.results ?? []).filter((r) => r.ok === false).length || 1;
+  const worked = Math.max(0, res.ranSteps - failed);
+  const unattempted = Math.max(0, res.totalSteps - res.ranSteps);
+  return `FLOW FAILED — ${worked} ok, ${failed} failed`
+    + (unattempted ? `, ${unattempted} not attempted` : '')
+    + ` (of ${res.totalSteps})${time}`;
+}
+
+/**
+ * How long the screen has been still, for a wait that is about to give up.
+ *
+ * A field report: a 180-second wait for a control that never appeared, on a
+ * screen that had been static for about twelve of those seconds — the app had
+ * logged itself out and was sitting on a login form. The failure message was
+ * praised for listing what *was* on screen; it arrived three minutes late.
+ *
+ * Stillness is already tracked and already printed by other commands, so the
+ * information existed and this wait simply never asked for it. Reported on
+ * every timeout, and — only when the caller opts in — allowed to end the wait
+ * early. Opt-in and not default, because a still screen is exactly what a
+ * pending network call looks like: the target may yet arrive, and a wait that
+ * gave up on stillness alone would break the case waits exist for.
+ *
+ * This is only trustworthy because of 134. Before the animation threshold was
+ * measured, a completely static screen claimed something was animating on 54%
+ * of its frames, and "nothing has moved" could not be said with a straight face.
+ */
+async function stillnessNote(deviceQuery, ctx) {
+  try {
+    const { state } = await api.getState(deviceQuery, { options: ctx.options });
+    const ms = state?.stableForMs;
+    return Number.isFinite(ms) ? { ms, note: `the screen has not moved for ${Math.round(ms)}ms` } : null;
+  } catch {
+    return null;
+  }
+}

package/src/analyze.js

@@ -65,6 +65,25 @@ export function signatureDiff(a, b) {
  */
 export const CELL_CHANGE = 0.012;
 
+/**
+ * How far two *capture paths* may differ and still be looking at one screen.
+ *
+ * Not the change threshold, and the distinction matters. `signatureDiff > 0.004`
+ * asks whether a screen moved between two frames from the *same* path. This asks
+ * whether the daemon's frame and an independent `simctl` screenshot show the
+ * same thing — and they never match closely, because one is downscaled by the
+ * capture loop and the other is a full-resolution PNG scaled here.
+ *
+ * Measured on this device, which is the only reason a number appears:
+ *
+ *   same screen, two paths       0.00123  (three runs, identical to five places)
+ *   two different screens        0.68603  (before and after a home press)
+ *
+ * A separation of 550x, so the threshold is not delicate. 0.02 is sixteen times
+ * the scaling cost and thirty-four times under the signal.
+ */
+export const PATHS_AGREE = 0.02;
+
 /** The largest single-region change between two signatures, 0-1. */
 export function maxCellDelta(a, b) {
   const deltas = regionDeltas(a, b);
@@ -100,6 +119,43 @@ export function regionMap(deltas, cols = REGION_COLS) {
   return lines.join('\n');
 }
 
+/**
+ * Where the movement is, in words — item 123.
+ *
+ * A settle that gives up says only that it gave up, and the reporter who asked
+ * for this put the cost plainly: a streaming summary panel, a Lottie and a
+ * support widget never settle, so the settle fails, and because a failed step
+ * aborts the batch it takes the other five steps with it. Their workaround was
+ * `pause` plus `continueOnError`, which they called *"strictly worse than a
+ * settle that knows what to ignore"*. Knowing what to ignore is the expensive
+ * half. Saying **where** is nearly free, and it is what turns "did not settle"
+ * into "there is a spinner in the top-right and nobody cares about it".
+ *
+ * Deliberately coarse. Thirds of the screen in each axis, named the way a person
+ * would point at it, and a share so a caller can tell one spinner from a screen
+ * that is genuinely still in flight.
+ */
+export function describeMotion(deltas, cols = REGION_COLS) {
+  const total = deltas.reduce((a, b) => a + b, 0);
+  if (!deltas.length || total <= 0) return null;
+  const rows = Math.ceil(deltas.length / cols);
+  const bands = new Map();
+  const third = (i, n) => (i < n / 3 ? 0 : i < (2 * n) / 3 ? 1 : 2);
+  const DOWN = ['top', 'middle', 'bottom'];
+  const ACROSS = ['left', 'centre', 'right'];
+  deltas.forEach((d, i) => {
+    const key = `${DOWN[third(Math.floor(i / cols), rows)]}-${ACROSS[third(i % cols, cols)]}`;
+    bands.set(key, (bands.get(key) ?? 0) + d);
+  });
+  const ranked = [...bands.entries()].sort((a, b) => b[1] - a[1]);
+  const [where, amount] = ranked[0];
+  const share = Math.round((amount / total) * 100);
+  // A single band holding most of the movement is a localised animation; spread
+  // evenly, the whole screen is in flight and naming a corner would mislead.
+  if (share < 40) return { where: 'spread across the screen', share, localised: false };
+  return { where: where.replace('-', ' '), share, localised: true };
+}
+
 /** Signatures live in state.json, so they are stored as compact hex. */
 export function signatureToHex(sig) {
   return sig.map((v) => v.toString(16).padStart(2, '0')).join('');