simframe 0.4.2 → 0.6.0-rc.1

package/src/actions.js

@@ -3,16 +3,28 @@
 // Waiting uses a baseline captured BEFORE each action, which is the whole
 // reason these scripts are reliable rather than racy.
 import * as api from './index.js';
+import * as graph from './graph.js';
 import * as input from './input.js';
 import * as intent from './intent.js';
-import { launchApp, openUrl, setPasteboard, terminateApp } from './simctl.js';
+import { launchApp, openUrl, setPasteboard, setPermission, terminateApp } from './simctl.js';
 
 const sleep = (ms) => new Promise((r) => setTimeout(r, ms));
 const MAX_PAUSE_MS = 5000;
+/** How long a text field needs after being tapped before it holds the keyboard focus. */
+const FOCUS_SETTLE_MS = 150;
+const POLL_MS = 250;
+/** A list that has not produced the target in this many screens does not contain it. */
+const MAX_SCROLLS = 20;
 
+/**
+ * Steps that change the device. Only these get a settle wait and a verified
+ * edge in the graph — asserting something is on screen does not move it.
+ * `scrollTo` is here because it scrolls; `permission` because a granted
+ * permission can change what the app shows.
+ */
 const ACTION_STEPS = new Set([
-  'tap', 'tapAt', 'type', 'paste', 'swipe', 'scroll', 'button', 'key',
-  'launch', 'terminate', 'openUrl', 'confirm', 'chooseAny',
+  'tap', 'tapAt', 'type', 'paste', 'swipe', 'scroll', 'scrollTo', 'button', 'key',
+  'launch', 'terminate', 'openUrl', 'confirm', 'chooseAny', 'permission',
 ]);
 
 /** Accept both `{tap: "Save"}` shorthand and `{action: "tap", target: "Save"}`. */
@@ -24,12 +36,63 @@ export function normalizeStep(raw) {
   // Siblings like timeoutMs sit alongside the shorthand key and must survive.
   const { [key]: value, ...rest } = raw;
   const inline = value && typeof value === 'object' && !Array.isArray(value) ? value : { value };
-  return { ...rest, ...inline, action: key };
+  const step = { ...rest, ...inline, action: key };
+  // Drop keys that are present but undefined. `simframe tap X` used to pass
+  // `index: undefined`, which survived here and then crashed the signature
+  // builder before the tap was ever sent.
+  for (const k of Object.keys(step)) if (step[k] === undefined) delete step[k];
+  return step;
+}
+
+/**
+ * Does this verdict mean the flow went somewhere nobody intended?
+ *
+ * Only an unexpected *screen* does. `unexpected-transition` is not a verdict at
+ * all any more — a noisy classifier disagreeing about whether a tab switch was
+ * a push or a pop is not a reason to call a correct navigation wrong, and a
+ * verdict that cries wolf trains you to ignore verdicts.
+ */
+export function wrongTurnFrom(verification) {
+  return verification?.verdict === 'unexpected-screen';
+}
+
+/**
+ * What a halted step does to the run as a whole.
+ *
+ * Both halves matter, and only one of them used to happen: the step is marked
+ * failed, AND so is the run. Without the second, `ok` meant no more than
+ * "nothing threw", so a flow stopped dead at step 0 by a wrong turn reported
+ * "flow completed" with no error — the exact shape of failure the verdict
+ * exists to make loud.
+ */
+export function haltDecision({ verification, stopOnUnexpected = true, continueOnError = false } = {}) {
+  if (!wrongTurnFrom(verification) || !stopOnUnexpected || continueOnError) {
+    return { halt: false, failRun: false, error: null };
+  }
+  return { halt: true, failRun: true, error: `${verification.verdict}: ${verification.detail}` };
 }
 
 export async function runScript(
   deviceQuery,
-  { steps, autoSettle = true, stableMs = 500, timeoutMs = 8000, continueOnError = false, options } = {},
+  {
+    steps,
+    autoSettle = true,
+    stableMs = 500,
+    timeoutMs = 8000,
+    confirmNovel = true,
+    continueOnError = false,
+    // Check each action against what it did last time, and remember what it
+    // does this time. On by default: a flow that cannot tell a wrong turn from
+    // a right one is worse than no flow.
+    verify = true,
+    // Stop when a verified step lands somewhere it should not have. A flow
+    // continuing past a wrong turn taps controls on a screen nobody intended.
+    stopOnUnexpected = true,
+    // Rebuild the HID session and retry once when a hardware button provably
+    // did nothing. Off only for a caller deliberately testing that path.
+    recoverInput = true,
+    options,
+  } = {},
 ) {
   if (!Array.isArray(steps) || !steps.length) throw new Error('a script needs at least one step');
   const { device } = await api.ensureDaemon(deviceQuery, options);
@@ -38,7 +101,11 @@ export async function runScript(
 
   const needsInput = steps.some((s) => ACTION_STEPS.has(normalizeStep(s).action));
   if (needsInput) {
-    const driver = await input.detectDriver();
+    // driverFor, not detectDriver: detectDriver asks specifically whether idb
+    // is installed, so every batch flow demanded idb even on a machine where
+    // the daemon was doing the input perfectly well. Single-step `simframe tap`
+    // already went through driverFor, so `tap` worked and `do` did not.
+    const driver = await input.driverFor(udid);
     if (!driver.available) throw new Error(driver.reason);
   }
 
@@ -48,16 +115,37 @@ export async function runScript(
   const results = [];
   const frames = [];
   let failed = false;
+  let carriedScreen = null;
+  // The last reading of where we ended up, confirmed or not. The compact map
+  // the caller returns to Claude is rendered from this, so describing the end
+  // state costs nothing beyond the verification pass the flow already ran.
+  let endScreen = null;
+  // At most one recovery per run. Pressing home while already on the springboard
+  // moves nothing and is not a failure, so an unbounded retry would rebuild the
+  // session and press again on every such step for no reason.
+  let inputRecovered = false;
 
   for (const [i, raw] of steps.entries()) {
     const step = normalizeStep(raw);
     const stepStart = Date.now();
     // The baseline for "did the screen react" must predate the action itself.
-    const before = (await api.getState(deviceQuery, { options })).state.hash;
+    const beforeState = (await api.getState(deviceQuery, { options })).state;
+    const before = beforeState.hash;
+    // Identity is structural: a list with new rows is the same screen, and the
+    // pixel hash cannot say so.
+    // The screen this step starts on is the screen the last one ended on —
+    // nothing happens in between. Recomputing it cost a full perception pass
+    // per step for an answer already in hand.
+    const beforeScreen = verify
+      ? (carriedScreen ?? await api.screenIdentity(deviceQuery, { options, settleMs: stableMs, timeoutMs, confirmNovel }))
+      : null;
+    carriedScreen = null;
+    // What this action did last time it was taken here, if ever.
+    const prediction = verify && beforeScreen?.hash ? graph.predict(udid, beforeScreen, step) : null;
     try {
-      const detail = await runStep(deviceQuery, udid, step, { screen, options, frames });
-      let settled = null;
-      if (autoSettle && ACTION_STEPS.has(step.action)) {
+      let detail = await runStep(deviceQuery, udid, step, { screen, options, frames });
+      const settleFor = async () => {
+        if (!autoSettle || !ACTION_STEPS.has(step.action)) return null;
         const w = await api.waitFor(deviceQuery, {
           mode: 'settle',
           since: before,
@@ -65,23 +153,82 @@ export async function runScript(
           timeoutMs: step.timeoutMs ?? timeoutMs,
           options,
         });
-        settled = {
+        return {
           ok: w.satisfied,
           waitedMs: w.waitedMs,
           sawChange: w.sawChange,
           stalled: Boolean(w.stalled),
           noVisibleChange: Boolean(w.noVisibleChange),
         };
+      };
+      let settled = await settleFor();
+
+      // A hardware button that moved nothing did not arrive.
+      //
+      // Input is the one path with no feedback, so a dispatched Indigo message
+      // reports success whether or not the device acted on it — measured, a
+      // long-running daemon returned `press in 66ms` with the screen frozen,
+      // and the same press worked on a fresh daemon. The frames are the only
+      // witness, and by here we have them.
+      //
+      // Only buttons, and only on no visible change. Home and lock always move
+      // the screen, so nothing moving is unambiguous; a tap that changes
+      // nothing is ordinary, and retrying one could act twice. Retrying an
+      // action that provably did nothing is not a repeat — it is the first
+      // attempt that counts.
+      if (recoverInput && !inputRecovered && step.action === 'button' && settled?.noVisibleChange) {
+        inputRecovered = true;
+        const reset = await input.resetSession(udid);
+        if (reset) {
+          detail += ' [input was not being delivered; HID session reset and retried]';
+          await runStep(deviceQuery, udid, step, { screen, options, frames });
+          settled = await settleFor();
+        }
+      }
+      // Verify against what was predicted, and remember what actually
+      // happened. Without this a step that moved the screen the wrong way
+      // reports success, and the flow carries on believing it worked.
+      let verification = null;
+      if (verify && ACTION_STEPS.has(step.action) && beforeScreen?.hash) {
+        const afterState = (await api.getState(deviceQuery, { options })).state;
+        const kind = afterState.transition?.kind;
+        const afterScreen = await api.screenIdentity(deviceQuery, { options, settleMs: stableMs, timeoutMs, confirmNovel });
+        verification = {
+          ...graph.verdict({ udid, prediction, before: beforeScreen.hash, after: afterScreen.hash, kind }),
+          predicted: prediction ? { to: prediction.to.slice(0, 10), kind: prediction.kind, seen: prediction.count } : null,
+          observed: { to: afterScreen.hash?.slice(0, 10), kind },
+        };
+        // Only remember what was seen on a settled screen: an edge recorded
+        // mid-transition points at a screen that never really existed.
+        // `confirmed` already means the fingerprint held still across two
+        // independent readings, which is the thing `settled` was standing in
+        // for. Requiring both meant a screen that settled slowly recorded
+        // nothing at all.
+        endScreen = afterScreen;
+        if (afterScreen.confirmed && afterScreen.hash) {
+          graph.record(udid, { from: beforeScreen, action: step, to: afterScreen, kind });
+          carriedScreen = afterScreen;
+        }
       }
+
+      const wrongTurn = wrongTurnFrom(verification);
       const note = settled?.noVisibleChange ? ' [no visible change]' : '';
       results.push({
         index: i,
         action: step.action,
+        verification,
         ok: true,
         ms: Date.now() - stepStart,
-        detail: `${detail}${note}`,
+        detail: `${detail}${note}${wrongTurn ? ` [${verification.verdict}: ${verification.detail}]` : ''}`,
         settled,
       });
+      const halt = haltDecision({ verification, stopOnUnexpected, continueOnError });
+      if (halt.halt) {
+        results[results.length - 1].ok = false;
+        results[results.length - 1].error = halt.error;
+        failed = halt.failRun;
+        break;
+      }
     } catch (err) {
       results.push({ index: i, action: step.action, ok: false, ms: Date.now() - stepStart, error: err.message });
       failed = true;
@@ -91,6 +238,10 @@ export async function runScript(
 
   return {
     device,
+    // Returned so a run that verified end to end can be handed straight to
+    // navigate.saveFlow without the caller reassembling what it just ran.
+    steps,
+    endScreen,
     results,
     ok: !failed,
     totalMs: Date.now() - startedAt,
@@ -126,10 +277,14 @@ async function runStep(deviceQuery, udid, step, ctx) {
     }
     case 'type': {
       if (step.into) {
-        const { node } = await input.tapLabel(udid, step.into, { index: step.index });
-        await sleep(150);
+        // locate, not tapLabel: tapLabel asks the accessibility tree directly,
+        // so a field that only OCR can see was untypeable, and a selector
+        // (`#4`, `@x,y`) meant nothing here.
+        const found = await api.locate(deviceQuery, step.into, { index: step.index, refresh: step.refresh });
+        await input.tapPoint(udid, found.target.x, found.target.y);
+        await sleep(FOCUS_SETTLE_MS);
         await input.typeText(udid, step.text ?? step.value);
-        return `typed into "${node.label ?? step.into}"`;
+        return `typed into "${found.target.label}" at ${found.target.x},${found.target.y}`;
       }
       await input.typeText(udid, step.text ?? step.value);
       return 'typed text';
@@ -168,9 +323,15 @@ async function runStep(deviceQuery, udid, step, ctx) {
     case 'key':
       await input.pressKey(udid, step.value ?? step.code);
       return `pressed key ${step.value ?? step.code}`;
-    case 'launch':
-      await launchApp(udid, step.value ?? step.bundleId);
-      return `launched ${step.value ?? step.bundleId}`;
+    case 'launch': {
+      const bundleId = step.value ?? step.bundleId;
+      await launchApp(udid, bundleId, {
+        args: step.args ?? [],
+        env: step.env ?? {},
+        terminateFirst: step.relaunch === true,
+      });
+      return `launched ${bundleId}${step.relaunch ? ' (relaunched)' : ''}`;
+    }
     case 'terminate':
       await terminateApp(udid, step.value ?? step.bundleId);
       return `terminated ${step.value ?? step.bundleId}`;
@@ -228,6 +389,91 @@ async function runStep(deviceQuery, udid, step, ctx) {
       }
       throw new Error(`"${target}" is still on screen`);
     }
+    // Bring something into view. A control that scrolled off the bottom of a
+    // list is not missing, and "not on this screen" is the wrong answer to give
+    // about it.
+    case 'scrollTo': {
+      const query = step.value ?? step.target ?? step.label;
+      const dir = String(step.direction ?? 'down').toLowerCase();
+      const max = Math.min(MAX_SCROLLS, step.maxScrolls ?? 6);
+      for (let i = 0; i <= max; i += 1) {
+        try {
+          const found = await api.locate(deviceQuery, query, { index: step.index, refresh: i > 0 });
+          return `"${found.target.label}" is in view at ${found.target.x},${found.target.y}` +
+            (i ? ` after ${i} scroll${i === 1 ? '' : 's'}` : ' already');
+        } catch (err) {
+          if (i === max) throw new Error(`scrolled ${dir} ${max}x without finding ${query}: ${err.message}`);
+        }
+        await runStep(deviceQuery, udid, { action: 'scroll', value: dir }, ctx);
+        await api.waitFor(deviceQuery, { mode: 'stable', stableMs: 250, timeoutMs: 2500, options: ctx.options });
+      }
+      throw new Error(`could not bring ${query} into view`);
+    }
+
+    // Wait for a selector rather than a label, so it works on screens the
+    // accessibility tree never described.
+    case 'waitFor': {
+      const query = step.value ?? step.target ?? step.text;
+      const limit = Date.now() + (step.timeoutMs ?? 8000);
+      let lastError = 'never appeared';
+      for (let attempt = 0; ; attempt += 1) {
+        try {
+          const found = await api.locate(deviceQuery, query, { index: step.index, refresh: attempt > 0 });
+          return `"${found.target.label}" appeared at ${found.target.x},${found.target.y}`;
+        } catch (err) {
+          lastError = err.message;
+        }
+        if (Date.now() >= limit) break;
+        await sleep(POLL_MS);
+      }
+      throw new Error(`waited ${step.timeoutMs ?? 8000}ms for ${query}: ${lastError}`);
+    }
+
+    // One assert step for every condition, because `assertText` could only ask
+    // one question and the interesting ones are about state: is Save enabled
+    // yet, does the field hold what was typed into it.
+    case 'assert': {
+      const query = step.value ?? step.target ?? step.text;
+      const want = String(step.is ?? (step.gone ? 'gone' : 'visible')).toLowerCase();
+      let found = null;
+      try {
+        found = await api.locate(deviceQuery, query, { index: step.index, refresh: step.refresh });
+      } catch (err) {
+        if (want === 'gone') return `${query} is gone`;
+        throw new Error(`${query}: ${err.message}`);
+      }
+      const t = found.target;
+      switch (want) {
+        case 'visible':
+          return `${query} is on screen at ${t.x},${t.y}`;
+        case 'gone':
+          throw new Error(`${query} is still on screen at ${t.x},${t.y}`);
+        case 'enabled':
+          if (t.enabled === false) throw new Error(`"${t.label}" is disabled`);
+          return `"${t.label}" is enabled`;
+        case 'disabled':
+          if (t.enabled !== false) throw new Error(`"${t.label}" is not disabled`);
+          return `"${t.label}" is disabled`;
+        case 'value': {
+          const expected = String(step.equals ?? step.text ?? '');
+          const actual = [t.label, t.value, ...(t.aliases ?? [])].filter(Boolean).join(' ');
+          if (!actual.toLowerCase().includes(expected.toLowerCase())) {
+            throw new Error(`expected "${expected}" but read "${actual}"`);
+          }
+          return `"${expected}" is what ${query} reads`;
+        }
+        default:
+          throw new Error(`unknown assert condition "${want}" — visible, gone, enabled, disabled or value`);
+      }
+    }
+
+    // Answering a system permission alert is not a test of the app. Setting the
+    // permission is.
+    case 'permission': {
+      const service = step.value ?? step.service;
+      return await setPermission(udid, step.grant ?? step.action ?? 'grant', service, step.bundleId);
+    }
+
     case 'look': {
       const frame = await api.getFrame(deviceQuery, { detail: step.detail ?? 'normal', options: ctx.options });
       ctx.frames.push({ label: step.label ?? `step frame`, png: frame.png });