simframe 0.6.2 → 0.7.2

package/scripts/eval-ax-tier.mjs

@@ -0,0 +1,191 @@
+#!/usr/bin/env node
+// What is the accessibility tier actually worth?
+//
+// Phase 2a moved the tree host-side and made it fast; the tree is also absent
+// on Android and will stay absent until an APK is worth shipping
+// (docs/DEFERRED.md, Phase 8b). Both of those decisions rest on the same
+// unmeasured quantity: how much of what simframe knows about a screen comes
+// from the tree rather than from OCR and CV.
+//
+// So this walks a tour and, on every screen, builds the map twice — once with
+// the tree and once without — from the same frame. Neither map is persisted, so
+// measuring does not teach the graph anything.
+//
+// Three questions, because they have different answers:
+//
+//   provenance   of the elements on a screen, and of the *interactive* ones,
+//                how many only the tree can see
+//   recognition  would a revisit be recognised as the same screen, per tier —
+//                the graph's own rule, `similarity >= SIMILARITY_THRESHOLD`
+//   intents      given a name a user would type, does the right element come
+//                back, per tier
+import fs from 'node:fs';
+import * as actions from '../src/actions.js';
+import * as api from '../src/index.js';
+import * as fingerprint from '../src/fingerprint.js';
+import * as graph from '../src/graph.js';
+import * as matching from '../src/matching.js';
+import * as screenmap from '../src/screenmap.js';
+
+const arg = (name, fallback) => {
+  const hit = process.argv.find((a) => a.startsWith(`--${name}=`));
+  return hit ? hit.slice(name.length + 3) : fallback;
+};
+const tourFile = arg('tour');
+const rounds = Number(arg('rounds', 2));
+const device = arg('device');
+const outFile = arg('out');
+const listLabels = process.argv.includes('--labels');
+
+if (!tourFile) {
+  console.error('usage: node scripts/eval-ax-tier.mjs --tour=<tour.json> [--rounds=2] [--device=<udid>] [--labels] [--out=<file>]');
+  process.exit(2);
+}
+const tour = JSON.parse(fs.readFileSync(tourFile, 'utf8'));
+
+const { device: dev } = await api.ensureDaemon(device);
+console.log(`device: ${dev.name} (${dev.runtime})`);
+console.log(`tour: ${tour.length} screens x ${rounds} rounds, each screen read twice — with the tree and without\n`);
+
+/** Roles a user can act on. Text is not one: it is what a screen says, not what it offers. */
+const INTERACTIVE = new Set(['button', 'field', 'switch', 'cell', 'link']);
+const isInteractive = (t) => INTERACTIVE.has(fingerprint.roleOf(t));
+const sourcesOf = (t) => String(t.source ?? '').split(/[+,]/).filter(Boolean);
+
+const readings = [];
+for (let round = 1; round <= rounds; round += 1) {
+  for (const screen of tour) {
+    if (screen.steps?.length) await actions.runScript(device, { steps: screen.steps, verify: false });
+    // One frame, two maps. `api.screenIdentity` is not used here because it
+    // decides for itself which layers to read; this has to pin them.
+    const withTree = await api.readScreenWith(device, { useAx: true });
+    const withoutTree = await api.readScreenWith(device, { useAx: false });
+    readings.push({ name: screen.name, round, withTree, withoutTree, intents: screen.intents ?? [] });
+    const t = withTree.entry.targets ?? [];
+    const o = withoutTree.entry.targets ?? [];
+    console.log(
+      `  round ${round}  ${screen.name.padEnd(22)} tree: ${String(t.length).padStart(3)} elements ` +
+      `(${t.filter(isInteractive).length} interactive)   no tree: ${String(o.length).padStart(3)} ` +
+      `(${o.filter(isInteractive).length} interactive)`,
+    );
+    if (listLabels) {
+      for (const e of t) {
+        console.log(`      [${String(e.source ?? '?').padEnd(6)}] ${fingerprint.roleOf(e).padEnd(7)} ${JSON.stringify(String(e.label ?? '').slice(0, 40))}`);
+      }
+    }
+  }
+}
+
+// --- provenance -------------------------------------------------------------
+// Agreement between the two sensors is not recorded in `source`. When OCR text
+// falls inside an accessibility element the map keeps the element and files the
+// text as an *alias* on it, so `source` stays 'ax' — which means counting
+// sources alone reports that the two sensors never see the same thing, and they
+// do. An alias is the evidence that they agreed.
+let all = 0;
+let axAndOcr = 0;
+let axAlone = 0;
+let ocrOnly = 0;
+let inter = 0;
+let interAxOnly = 0;
+let interAxInvisible = 0;
+for (const r of readings) {
+  for (const t of r.withTree.entry.targets ?? []) {
+    const src = sourcesOf(t);
+    all += 1;
+    const hasAx = src.includes('ax');
+    const alsoSeen = (t.aliases ?? []).length > 0;
+    if (hasAx && alsoSeen) axAndOcr += 1;
+    else if (hasAx) axAlone += 1;
+    else ocrOnly += 1;
+    if (isInteractive(t)) {
+      inter += 1;
+      if (hasAx) interAxOnly += 1;
+      // The interesting class: a control the tree knows about that OCR cannot
+      // see at all, because it has no text — an icon. This is what a platform
+      // without a tree simply cannot offer.
+      if (hasAx && !alsoSeen) interAxInvisible += 1;
+    }
+  }
+}
+const both = axAndOcr;
+const axOnly = axAlone;
+const pct = (n, of) => (of ? `${((n / of) * 100).toFixed(0)}%` : '—');
+
+console.log('\n--- provenance, over every element on every reading ---');
+console.log(`  elements                       ${all}`);
+console.log(`  from the tree, OCR saw nothing ${axOnly}  ${pct(axOnly, all)}`);
+console.log(`  from OCR/CV alone              ${ocrOnly}  ${pct(ocrOnly, all)}`);
+console.log(`  from the tree, OCR agreed      ${both}  ${pct(both, all)}  (text found inside the element)`);
+console.log(`  interactive elements           ${inter}`);
+console.log(`  interactive, from the tree      ${interAxOnly}  ${pct(interAxOnly, inter)}`);
+console.log(`  interactive with no text at all ${interAxInvisible}  ${pct(interAxInvisible, inter)}  (icons — OCR cannot see these)`);
+
+// --- recognition ------------------------------------------------------------
+function recognition(pick) {
+  const byName = new Map();
+  for (const r of readings) byName.set(r.name, [...(byName.get(r.name) ?? []), pick(r)]);
+  let pairs = 0;
+  let recognised = 0;
+  const worst = [];
+  for (const [name, list] of byName) {
+    for (let i = 0; i < list.length; i += 1) {
+      for (let j = i + 1; j < list.length; j += 1) {
+        const s = fingerprint.similarity(list[i].entry.structuralTokens ?? [], list[j].entry.structuralTokens ?? []);
+        pairs += 1;
+        if (s >= graph.SIMILARITY_THRESHOLD) recognised += 1;
+        worst.push({ name, s });
+      }
+    }
+  }
+  worst.sort((a, b) => a.s - b.s);
+  return { pairs, recognised, worst: worst[0] };
+}
+const treeRec = recognition((r) => r.withTree);
+const bareRec = recognition((r) => r.withoutTree);
+
+// --- intents ----------------------------------------------------------------
+function intents(pick) {
+  let asked = 0;
+  let right = 0;
+  const misses = [];
+  for (const r of readings) {
+    const entry = pick(r);
+    for (const want of r.intents) {
+      asked += 1;
+      const hit = matching.resolve(entry.entry.targets ?? [], want, { screen: entry.points });
+      // `ambiguous` is not a resolution: a caller gets an error and has to say
+      // which, so counting it as correct would flatter the tier that produced it.
+      const label = hit.status === 'ok' ? String(hit.target.label ?? '') : '';
+      if (label.toLowerCase().includes(String(want).toLowerCase())) right += 1;
+      else misses.push(`${r.name} r${r.round}: "${want}" → ${label ? JSON.stringify(label.slice(0, 30)) : hit.status}`);
+    }
+  }
+  return { asked, right, misses };
+}
+const treeInt = intents((r) => r.withTree);
+const bareInt = intents((r) => r.withoutTree);
+
+console.log('\n--- with the tree, and without ---');
+console.log(`${''.padEnd(30)} ${'tree + OCR'.padStart(12)} ${'OCR/CV only'.padStart(12)}`);
+console.log(`${'elements per reading'.padEnd(30)} ${(all / readings.length).toFixed(1).padStart(12)} ${((readings.reduce((n, r) => n + (r.withoutTree.entry.targets ?? []).length, 0)) / readings.length).toFixed(1).padStart(12)}`);
+console.log(`${'screens recognised on revisit'.padEnd(30)} ${`${treeRec.recognised}/${treeRec.pairs}`.padStart(12)} ${`${bareRec.recognised}/${bareRec.pairs}`.padStart(12)}`);
+console.log(`${'weakest revisit similarity'.padEnd(30)} ${treeRec.worst?.s.toFixed(2).padStart(12)} ${bareRec.worst?.s.toFixed(2).padStart(12)}`);
+console.log(`${'intents resolved correctly'.padEnd(30)} ${`${treeInt.right}/${treeInt.asked}`.padStart(12)} ${`${bareInt.right}/${bareInt.asked}`.padStart(12)}`);
+
+for (const [label, res] of [['tree + OCR', treeInt], ['OCR/CV only', bareInt]]) {
+  if (res.misses.length) {
+    console.log(`\nintents ${label} got wrong:`);
+    for (const m of res.misses.slice(0, 12)) console.log(`   ${m}`);
+  }
+}
+
+if (outFile) {
+  fs.writeFileSync(outFile, JSON.stringify({
+    device: dev.name, runtime: dev.runtime, rounds, at: Date.now(),
+    provenance: { all, axOnly, ocrOnly, both, interactive: inter, interactiveAxOnly: interAxOnly, interactiveIconOnly: interAxInvisible },
+    recognition: { tree: treeRec, bare: bareRec },
+    intents: { tree: treeInt, bare: bareInt },
+  }, null, 2));
+  console.log(`\nwrote ${outFile}`);
+}

package/scripts/eval-fingerprint.mjs

@@ -30,6 +30,24 @@ const arg = (name, fallback) => {
 
 const tourFile = arg('tour');
 const rounds = Number(arg('rounds', 3));
+/**
+ * How much room the threshold must have on each side.
+ *
+ * `gap > 0` was the only bar until the margin narrowed, and a gap can be wide
+ * while the threshold sits at the edge of it — which is the state that actually
+ * misclassifies a screen. So the bar is stated as clearance around the
+ * threshold itself: every same-screen revisit must score at least
+ * `threshold + CLEARANCE`, and every different-screen pair at most
+ * `threshold - CLEARANCE`.
+ *
+ * 0.10 is chosen against measurement, not taste. Clean runs on this machine
+ * put same-min at 0.67-0.75 and different-max at 0.05, so the clearance in
+ * hand is roughly 0.3 either way; requiring 0.10 fails well before a
+ * misclassification and does not fire on ordinary variation. Raise it when the
+ * recorded distributions say it can be raised.
+ */
+const CLEARANCE = 0.1;
+
 /**
  * Above this, two consecutive tour screens are the same screen and the
  * navigation between them failed. Deliberately well above the identity
@@ -103,6 +121,20 @@ for (let round = 1; round <= rounds; round += 1) {
       tokens: id.tokens ?? [],
       count: (id.tokens ?? []).length,
       settled: id.settled,
+      // The elements the tokens were computed from, and the screen they were
+      // measured in. Kept so a candidate change to the token rules can be
+      // simulated against recorded readings by re-running the real tokeniser,
+      // instead of by transforming its output and hoping that is equivalent.
+      targets: id.entry?.targets ?? [],
+      screen: id.points,
+      // Which sensors answered. Two readings of one screen taken with
+      // different sensors are *known* not to agree — the tree and OCR share
+      // 0.33-0.47 of a screen's structural tokens (docs/DEFERRED.md) — and
+      // absorbing that is the graph's job, through aliasing, not the
+      // fingerprint's. So the distributions below are split by sensor mix
+      // rather than averaged over it, which is what made a mixed pair look
+      // like fingerprint drift.
+      sources: id.entry?.sources ?? [],
     });
     process.stdout.write(
       `  round ${round}  ${screen.name.padEnd(14)} ${String(id.hash).slice(0, 10)}  ${String((id.tokens ?? []).length).padStart(3)} tokens${id.settled ? '' : '  (never settled)'}\n`,
@@ -120,14 +152,68 @@ if (arrivalFailures.length) {
   process.exit(1);
 }
 
+/**
+ * A reading taken somewhere other than where the tour meant to be.
+ *
+ * The arrival check above compares each reading with the one before it, which
+ * catches "the navigation did not happen" and misses "the navigation went
+ * somewhere else". It missed exactly that: a reading labelled
+ * settings-accessibility was in fact the Settings root list, and being unlike
+ * its own screen at one end and like a different screen at the other, it alone
+ * moved the same-screen minimum to 0.00 and the different-screen maximum to
+ * 0.40 across 30 pairs. Both distributions were then measuring the tour.
+ *
+ * So each reading is also checked against its own siblings: a reading that
+ * resembles no other reading of its own screen, while resembling some other
+ * screen at least as much, was not where it says it was. That needs at least
+ * two siblings to be an outlier test rather than a coin toss, so it only
+ * applies from three rounds up.
+ */
+function findStrays(all) {
+  const strays = [];
+  const byName = new Map();
+  for (const r of all) byName.set(r.name, [...(byName.get(r.name) ?? []), r]);
+  for (const r of all) {
+    const siblings = byName.get(r.name).filter((o) => o !== r);
+    if (siblings.length < 2) continue;
+    const bestSelf = Math.max(...siblings.map((o) => fingerprint.similarity(r.tokens, o.tokens)));
+    const others = all.filter((o) => o.name !== r.name);
+    const bestOther = others.length
+      ? Math.max(...others.map((o) => fingerprint.similarity(r.tokens, o.tokens)))
+      : 0;
+    if (bestSelf < graph.SIMILARITY_THRESHOLD && bestOther >= bestSelf) {
+      strays.push({ reading: r, bestSelf, bestOther });
+    }
+  }
+  return strays;
+}
+
+const strays = findStrays(readings);
+if (strays.length) {
+  console.error(`\nFAIL ${strays.length} reading(s) were taken on a screen other than the one named:`);
+  for (const { reading, bestSelf, bestOther } of strays) {
+    console.error(`       ${reading.name} r${reading.round}: resembles its own screen ${bestSelf.toFixed(2)}, `
+      + `another screen ${bestOther.toFixed(2)} (${reading.count} tokens, sources ${reading.sources.join('+') || 'none'})`);
+  }
+  console.error('\nThat is the tour going somewhere unintended, not the fingerprint drifting, and');
+  console.error('measuring it as either distribution poisons both ends. Fix the tour — a tap that');
+  console.error('missed, or a screen that needs longer than its pause — and re-run.');
+  process.exit(1);
+}
+
+/** Which sensors produced a reading, as a comparable key. */
+const mixOf = (r) => (r.sources ?? []).join('+') || 'unknown';
+
 const same = [];
+const mixed = [];
 const different = [];
 for (let i = 0; i < readings.length; i += 1) {
   for (let j = i + 1; j < readings.length; j += 1) {
     const s = fingerprint.similarity(readings[i].tokens, readings[j].tokens);
-    (readings[i].name === readings[j].name ? same : different).push({
-      a: readings[i], b: readings[j], similarity: s,
-    });
+    const pair = { a: readings[i], b: readings[j], similarity: s };
+    if (readings[i].name !== readings[j].name) different.push(pair);
+    else if (mixOf(readings[i]) === mixOf(readings[j])) same.push(pair);
+    else mixed.push(pair);
   }
 }
 
@@ -143,6 +229,7 @@ const stats = (rows) => {
 };
 
 const s = stats(same);
+const m = stats(mixed);
 const d = stats(different);
 const gap = s && d ? s.min - d.max : null;
 const threshold = graph.SIMILARITY_THRESHOLD;
@@ -150,7 +237,13 @@ const threshold = graph.SIMILARITY_THRESHOLD;
 const f = (x) => (x == null ? '—' : x.toFixed(2));
 console.log(`\n${'distribution'.padEnd(26)} ${'n'.padStart(4)} ${'min'.padStart(6)} ${'median'.padStart(7)} ${'max'.padStart(6)}`);
 console.log(`${'same screen, revisited'.padEnd(26)} ${String(s?.n ?? 0).padStart(4)} ${f(s?.min).padStart(6)} ${f(s?.median).padStart(7)} ${f(s?.max).padStart(6)}`);
+console.log(`${'same screen, mixed sensors'.padEnd(26)} ${String(m?.n ?? 0).padStart(4)} ${f(m?.min).padStart(6)} ${f(m?.median).padStart(7)} ${f(m?.max).padStart(6)}`);
 console.log(`${'different screens'.padEnd(26)} ${String(d?.n ?? 0).padStart(4)} ${f(d?.min).padStart(6)} ${f(d?.median).padStart(7)} ${f(d?.max).padStart(6)}`);
+if (m) {
+  console.log('\nthe mixed-sensor row is not a fingerprint failure: the tree and OCR see a screen');
+  console.log('differently by design, and the graph absorbs it by aliasing. It is here so that');
+  console.log('it cannot be mistaken for drift, which is what happened when the rows were one.');
+}
 console.log(`\ngap (same-min − different-max): ${f(gap)}`);
 console.log(`threshold in use: ${threshold}`);
 
@@ -169,6 +262,51 @@ if (worstDifferent) {
   console.log(`closest different-screen pair: ${worstDifferent.a.name} vs ${worstDifferent.b.name} = ${f(worstDifferent.similarity)}`);
 }
 
+/**
+ * Why a pair is as far apart as it is, token by token.
+ *
+ * A distribution is not actionable and neither is a similarity: 0.42 says the
+ * margin narrowed and nothing about what moved. The token grammar is
+ * `role:region[:@slot]:w:h["label"]:x:y#count`, so a diff of two token sets
+ * names the cause directly — a label that changed is a label token, a role that
+ * flipped is the same geometry under two roles, a bucket that straddled is the
+ * same key with `#1` against `#many`, and a shifted anchor is the same key at a
+ * different `x`/`y`.
+ */
+function explainPair(pair) {
+  const a = new Set(pair.a.tokens);
+  const b = new Set(pair.b.tokens);
+  const onlyA = [...a].filter((t) => !b.has(t)).sort();
+  const onlyB = [...b].filter((t) => !a.has(t)).sort();
+  const shared = [...a].filter((t) => b.has(t)).length;
+  console.log(`
+  shared ${shared}, only in r${pair.a.round} ${onlyA.length}, only in r${pair.b.round} ${onlyB.length}`);
+  // The same structural key under two different tails is a drift; a key present
+  // on one side only is an element that came or went. Telling those apart is
+  // the whole diagnosis, so they are printed apart.
+  const keyOf = (t) => t.replace(/:x-?\d+:y-?\d+#(1|many)$/, '');
+  const tailOf = (t) => t.slice(keyOf(t).length);
+  const keysA = new Map(onlyA.map((t) => [keyOf(t), tailOf(t)]));
+  const keysB = new Map(onlyB.map((t) => [keyOf(t), tailOf(t)]));
+  const drifted = [...keysA.keys()].filter((k) => keysB.has(k));
+  if (drifted.length) {
+    console.log('  same structure, moved or re-counted:');
+    for (const k of drifted) console.log(`    ${k}   r${pair.a.round}${keysA.get(k)}   r${pair.b.round}${keysB.get(k)}`);
+  }
+  const goneA = onlyA.filter((t) => !keysB.has(keyOf(t)));
+  const goneB = onlyB.filter((t) => !keysA.has(keyOf(t)));
+  if (goneA.length) {
+    console.log(`  only in r${pair.a.round}:`);
+    for (const t of goneA) console.log(`    ${t}`);
+  }
+  if (goneB.length) {
+    console.log(`  only in r${pair.b.round}:`);
+    for (const t of goneB) console.log(`    ${t}`);
+  }
+}
+
+if (worstSame) explainPair(worstSame);
+
 // Chrome labels are the only text in a fingerprint, so which of them got in is
 // the thing a band change actually moves.
 const labels = new Set();
@@ -182,11 +320,28 @@ console.log(`\n${labels.size} distinct chrome label(s) entered identity: ${[...l
 if (outFile) {
   fs.writeFileSync(outFile, JSON.stringify({
     label, device: dev.name, runtime: dev.runtime, rounds, at: Date.now(),
-    threshold, same: s, different: d, gap, separated, thresholdInGap,
+    threshold, same: s, mixed: m, different: d, gap, separated, thresholdInGap,
     labels: [...labels].sort(),
-    readings: readings.map(({ tokens, ...r }) => ({ ...r, tokenCount: tokens.length })),
+    // Tokens are kept. They were stripped here, and the first time the margin
+    // narrowed the run could not be diagnosed from its own output.
+    readings,
   }, null, 2));
   console.log(`\nwrote ${outFile}`);
 }
 
-process.exit(separated ? 0 : 1);
+// The stated margin, checked rather than eyeballed. A person noticing that a
+// number moved is not a test; this is the machine that re-measures it.
+const floor = threshold + CLEARANCE;
+const ceiling = threshold - CLEARANCE;
+const sameOk = s != null && s.min >= floor;
+const differentOk = d != null && d.max <= ceiling;
+console.log(`${sameOk ? 'ok  ' : 'FAIL'} every same-screen revisit scores at least ${floor.toFixed(2)} (worst ${f(s?.min)})`);
+console.log(`${differentOk ? 'ok  ' : 'FAIL'} every different-screen pair scores at most ${ceiling.toFixed(2)} (worst ${f(d?.max)})`);
+if (!sameOk || !differentOk) {
+  console.error('\nThe threshold no longer has the clearance this bar states. Diagnose before');
+  console.error('moving it: `node scripts/analyse-fingerprint.mjs <the --out file>` classifies every');
+  console.error('divergent token by cause, and the causes have different fixes. A threshold moved to');
+  console.error('make a run pass is a threshold that means nothing.');
+}
+
+process.exit(separated && sameOk && differentOk ? 0 : 1);

package/skills/simframe/SKILL.md

@@ -1,16 +1,23 @@
 ---
 name: simframe
-description: Drive and inspect the iOS Simulator with eyes, hands and memory. Use for any task that involves running, testing, navigating or verifying an iOS app on a simulator — "does this screen look right", "tap through the signup flow", "why is this button not working", "is the list loading". Reads screens as text rather than screenshots, batches whole flows into one command, and verifies each step against what it did last time.
+description: Drive and inspect the iOS Simulator or an Android emulator with eyes, hands and memory. Use for any task that involves running, testing, navigating or verifying an app on a simulator or emulator — "does this screen look right", "tap through the signup flow", "why is this button not working", "is the list loading". Reads screens as text rather than screenshots, batches whole flows into one command, and verifies each step against what it did last time.
 ---
 
 # simframe
 
-A background daemon keeps the simulator's framebuffer warm, reads the screen
+A background daemon keeps the device's framebuffer warm, reads the screen
 through the accessibility tree and on-device OCR, and remembers which action
 leads from which screen to which. So the three things that make simulator work
 expensive — waiting for screenshots, spending tokens on images, and re-deriving
 the same screen every time — are already paid for.
 
+Both an iOS simulator and an Android emulator are driven the same way, by the
+same commands, and `simframe devices` lists both. The one difference worth
+knowing: **Android has no accessibility tree**, so its screens are read by OCR
+and CV alone. Tapping by label works; screen recognition is thinner, so prefer
+naming a device explicitly and re-reading the screen after a step you are unsure
+about.
+
 ## Read the screen as text, not as an image
 
 ```bash
@@ -161,7 +168,7 @@ daemon.
 
 ```bash
 simframe start [device]     # capture starts on first use anyway
-simframe devices            # booted simulators
+simframe devices            # booted simulators and emulators
 simframe recall             # what happened in the last ~60s, as text
 simframe strip              # recent frames tiled into one image, for an animation
 simframe find "the save button"   # resolve an intent without acting on it

package/src/actions.js

@@ -6,12 +6,33 @@ import * as api from './index.js';
 import * as graph from './graph.js';
 import * as input from './input.js';
 import * as intent from './intent.js';
-import { launchApp, openUrl, setPasteboard, setPermission, terminateApp } from './simctl.js';
+import { launchApp, openUrl, setPermission, terminateApp } from './platform/index.js';
 
 const sleep = (ms) => new Promise((r) => setTimeout(r, ms));
 const MAX_PAUSE_MS = 5000;
-/** How long a text field needs after being tapped before it holds the keyboard focus. */
-const FOCUS_SETTLE_MS = 150;
+/**
+ * A tapped field is typed into once the screen has settled, not after a fixed
+ * wait.
+ *
+ * It was 150 ms, which is enough for a keyboard to rise over the screen you are
+ * already on and nowhere near enough for a tap that opens a whole activity.
+ * Measured on Android: tapping Settings' search box starts a separate search
+ * screen, the text went before its field had focus, and the step reported
+ * success while nothing had been typed — the worst shape a failure can take.
+ *
+ * `settle` needs a change before it will report stillness, so a tap that
+ * visibly does nothing (a field that already had focus) cannot satisfy it and
+ * falls out at `reaction` instead. That bounds the cost of the honest case
+ * rather than the broken one.
+ */
+const FOCUS_STABLE_MS = 250;
+/**
+ * Long enough for a slow capture loop to produce a frame or two. The screenshot
+ * engine idles at 1.5 fps — 667 ms between frames — so anything under that is a
+ * verdict reached before there was anything to look at.
+ */
+const FOCUS_REACTION_MS = 900;
+const FOCUS_TIMEOUT_MS = 3000;
 const POLL_MS = 250;
 /** A list that has not produced the target in this many screens does not contain it. */
 const MAX_SCROLLS = 20;
@@ -251,6 +272,33 @@ export async function runScript(
   };
 }
 
+/**
+ * Tap a field and wait for it to take focus, for the steps that then put text
+ * in it.
+ *
+ * `locate`, not `tapLabel`: tapLabel asks the accessibility tree directly, so a
+ * field only OCR can see was untypeable and a selector (`#4`, `@x,y`) meant
+ * nothing here. The returned `quiet` says when the field never visibly took
+ * focus — usually fine, since a field that already had focus does not move, but
+ * also exactly what a tap that missed looks like, so the caller should be told.
+ */
+async function focusField(deviceQuery, udid, step, ctx) {
+  const found = await api.locate(deviceQuery, step.into, { index: step.index, refresh: step.refresh });
+  await input.tapPoint(udid, found.target.x, found.target.y);
+  const focused = await api.waitFor(deviceQuery, {
+    mode: 'settle',
+    stableMs: FOCUS_STABLE_MS,
+    reactionMs: FOCUS_REACTION_MS,
+    timeoutMs: FOCUS_TIMEOUT_MS,
+    options: ctx.options,
+  });
+  return {
+    found,
+    where: `"${found.target.label}" at ${found.target.x},${found.target.y}`,
+    quiet: focused.satisfied ? '' : ' [the field did not visibly take focus]',
+  };
+}
+
 async function runStep(deviceQuery, udid, step, ctx) {
   switch (step.action) {
     case 'tap': {
@@ -277,23 +325,25 @@ async function runStep(deviceQuery, udid, step, ctx) {
     }
     case 'type': {
       if (step.into) {
-        // locate, not tapLabel: tapLabel asks the accessibility tree directly,
-        // so a field that only OCR can see was untypeable, and a selector
-        // (`#4`, `@x,y`) meant nothing here.
-        const found = await api.locate(deviceQuery, step.into, { index: step.index, refresh: step.refresh });
-        await input.tapPoint(udid, found.target.x, found.target.y);
-        await sleep(FOCUS_SETTLE_MS);
+        const field = await focusField(deviceQuery, udid, step, ctx);
         await input.typeText(udid, step.text ?? step.value);
-        return `typed into "${found.target.label}" at ${found.target.x},${found.target.y}`;
+        return `typed into ${field.where}${field.quiet}`;
       }
       await input.typeText(udid, step.text ?? step.value);
       return 'typed text';
     }
     case 'paste': {
-      // Long strings are much faster on the pasteboard than through the keyboard.
-      await setPasteboard(udid, step.text ?? step.value);
-      if (step.into) await input.tapLabel(udid, step.into, { index: step.index, durationMs: 900 });
-      return 'placed text on the pasteboard';
+      // Long strings are much faster on the pasteboard than through the
+      // keyboard. `pasteText` delivers the keystroke as well as setting the
+      // pasteboard, and throws if it cannot — this step used to do neither and
+      // report success anyway.
+      if (step.into) {
+        const field = await focusField(deviceQuery, udid, step, ctx);
+        await input.pasteText(udid, step.text ?? step.value);
+        return `pasted into ${field.where}${field.quiet}`;
+      }
+      await input.pasteText(udid, step.text ?? step.value);
+      return 'pasted into the focused field';
     }
     case 'swipe': {
       const from = { x: step.from?.[0] ?? step.from?.x, y: step.from?.[1] ?? step.from?.y };