simframe 0.9.0 → 0.10.0

package/src/navigate.js

@@ -41,12 +41,15 @@ export function stepFor(edge) {
  * count. The reason mapping lives in metrics.PLAN_REASONS so the five reasons
  * have one owner.
  */
-function refuse(udid, result, { detail = null } = {}) {
+function refuse(udid, result, { detail = null, flowName = null } = {}) {
   try {
     const reason = metrics.PLAN_REASONS[result.reason];
     if (reason) {
       metrics.recordEscalation(udid, {
         reason,
+        // A refusal by `goto` is about a destination and one by `flow run` is
+        // about a named flow. Either is what a breakdown wants to group by.
+        flowName,
         fingerprint: metrics.fingerprintNow(udid, screenmap),
         outcome: 'escalated_to_model',
         detail: detail ?? result.reason,
@@ -70,8 +73,8 @@ export async function goto(deviceQuery, target, { options, ...runOptions } = {})
   const udid = device.udid;
 
   const found = graph.findScreen(udid, target);
-  if (!found) return refuse(udid, { ok: false, reason: 'unknown-screen', known: knownScreens(udid) }, { detail: `no screen matches "${target}"` });
-  if (found.ambiguous) return refuse(udid, { ok: false, reason: 'ambiguous', candidates: found.ambiguous }, { detail: `"${target}" fits ${found.ambiguous.length} screens` });
+  if (!found) return refuse(udid, { ok: false, reason: 'unknown-screen', known: knownScreens(udid) }, { detail: `no screen matches "${target}"`, flowName: `goto:${target}` });
+  if (found.ambiguous) return refuse(udid, { ok: false, reason: 'ambiguous', candidates: found.ambiguous }, { detail: `"${target}" fits ${found.ambiguous.length} screens`, flowName: `goto:${target}` });
 
   const here = await api.screenIdentity(udid, {});
   if (here.hash === found.node.hash) {
@@ -84,12 +87,12 @@ export async function goto(deviceQuery, target, { options, ...runOptions } = {})
   // `Cannot read properties of null (reading 'slice')` instead of answering.
   // Not hypothetical on Android, where README's own table puts the launcher at
   // one token.
-  if (!here.hash) return refuse(udid, { ok: false, reason: 'no-identity', to: found.name });
+  if (!here.hash) return refuse(udid, { ok: false, reason: 'no-identity', to: found.name }, { flowName: `goto:${target}` });
   const path_ = graph.route(udid, { hash: here.hash, tokens: here.tokens }, found.node.hash);
-  if (!path_) return refuse(udid, { ok: false, reason: 'no-route', from: here.hash.slice(0, 8), to: found.name });
+  if (!path_) return refuse(udid, { ok: false, reason: 'no-route', from: here.hash.slice(0, 8), to: found.name }, { flowName: `goto:${target}` });
 
   const steps = path_.map(stepFor);
-  if (steps.some((s) => !s)) return refuse(udid, { ok: false, reason: 'unreplayable-edge', to: found.name });
+  if (steps.some((s) => !s)) return refuse(udid, { ok: false, reason: 'unreplayable-edge', to: found.name }, { flowName: `goto:${target}` });
 
   const result = await runScript(udid, { steps, stopOnUnexpected: true, ...runOptions });
   const arrived = await api.screenIdentity(udid, {});
@@ -148,7 +151,7 @@ export function listFlows(udid) {
 export async function runFlow(deviceQuery, name, { options, ...runOptions } = {}) {
   const { device } = await api.ensureDaemon(deviceQuery, options);
   const flow = loadFlow(device.udid, name);
-  if (!flow) return refuse(device.udid, { ok: false, reason: 'unknown-flow', known: listFlows(device.udid).map((f) => f.name) }, { detail: `no saved flow "${name}"` });
+  if (!flow) return refuse(device.udid, { ok: false, reason: 'unknown-flow', known: listFlows(device.udid).map((f) => f.name) }, { detail: `no saved flow "${name}"`, flowName: name });
   // A replayed flow knows its own name, so its record can be compared against
   // a human doing the same thing. `minSteps` comes from the flow definition or
   // stays null — the step count of a recorded route is not a claim about the

package/src/platform/android.js

@@ -827,7 +827,7 @@ async function launchApp(udid, bundleId, { args = [], env = {}, terminateFirst =
 async function terminateApp(udid, bundleId) {
   // `am` reports failure on stdout and still exits 0 — the same trap launchApp
   // and openUrl already check for. Without this, terminating a package that is
-  // not installed answered "terminated com.typo.app".
+  // not installed answered "terminated com.example.mistyped".
   const { stdout, stderr } = await adb(udid, ['shell', 'am', 'force-stop', bundleId]);
   const error = /^Error:.*$/m.exec(`${stdout}${stderr}`);
   if (error) {

package/src/screenmap.js

@@ -17,7 +17,7 @@ import * as regions from './regions.js';
 import { informative } from './refs.js';
 import * as store from './store.js';
 
-const MAP_VERSION = 8; // an OCR reading contained in a labelled ax element merges into it
+const MAP_VERSION = 9; // ax targets carry value, selected and focused
 
 function mapDir(udid) {
   return path.join(store.deviceDir(udid), 'screens');
@@ -185,11 +185,30 @@ export async function build(udid, {
         if (!n.frame || !n.label || isContainer(n)) continue;
         targets.push({
           label: n.label,
+          // What the control *contains*, whether it is on, and whether it has
+          // focus. All three come off the accessibility tree, the daemon has
+          // asked for all three since 0.6.0, and all three were dropped before
+          // this — `value` here and the other two one layer up in
+          // `normalizeNode` — so nothing above this line ever saw them.
+          //
+          // The cost was not theoretical. A real session could not verify the
+          // contents of a text field at all: they reached a row only as the OCR
+          // alias, which made them as old as the map and unauthoritative. An
+          // assert failed against a field that did contain the string, the
+          // operator retyped, and the field ended up with a doubled value and a
+          // validation error.
+          //
+          // `focused` is worth naming separately: it is a direct answer to "did
+          // this field take focus", which the focus wait in actions.js infers
+          // from elapsed time because it had nothing better to use.
+          value: n.value ?? undefined,
           x: input.centerOf(n).x,
           y: input.centerOf(n).y,
           frame: n.frame,
           type: n.type,
           enabled: n.enabled,
+          selected: n.selected ?? undefined,
+          focused: n.focused ?? undefined,
           source: 'ax',
         });
       }

package/src/view.js

@@ -202,9 +202,69 @@ export function rowsFor(entry, { screen, filter, interactive, all = false, limit
   return { rows: rows.slice(0, limit), truncated: Math.max(0, rows.length - limit), collapsed };
 }
 
+/**
+ * Say when the rows below were remembered rather than looked at.
+ *
+ * Screen memory is deliberately keyed on the pixel layout hash, because a list
+ * with new rows is the same screen and re-perceiving it per step is the cost
+ * Phase 13 exists to remove. That is right for *identity* and wrong for
+ * *contents*, and the map made no distinction: a field's text reaches a row as
+ * an OCR alias, so a recalled map reports the text the field held when the map
+ * was built. Reported from a real session — a picker described the previous
+ * sheet's options and did it in 23 ms, which is the giveaway, because 23 ms is
+ * not enough time to have looked.
+ *
+ * This does not fix that. It stops it being invisible, which is the part that
+ * cost two wrong conclusions about an app.
+ *
+ * Only past a second, because a map built by this very call is not a
+ * recollection and saying so on every screen is how a real warning gets
+ * skimmed.
+ */
+export const RECALL_NOTE_FLOOR_MS = 1000;
+
+export function recalledNote(identity, now = Date.now()) {
+  const at = identity?.entry?.at;
+  if (!Number.isFinite(at)) return null;
+  const age = now - at;
+  if (age < RECALL_NOTE_FLOOR_MS) return null;
+  const ago = age < 60_000 ? `${Math.round(age / 1000)}s` : `${Math.round(age / 60_000)}m`;
+  return `elements recalled from ${ago} ago — pass refresh for what is there now`;
+}
+
+/**
+ * What a control *contains*, from the sensor that actually knows.
+ *
+ * A field's text reached a row only as the OCR alias, which means it was as old
+ * as the map and had no authoritative source at all. Reported from a real
+ * session: an `assert` on a field's contents failed against a field that did
+ * contain the string, the operator retyped, and the field ended up with a
+ * doubled value and a validation error. A character counter read `0/1000` in
+ * the map and `56/1000` in a screenshot of the same frame.
+ *
+ * The accessibility tree carries `value` and always has —
+ * `input.elementToNode` sets it on every node — and the renderer simply never
+ * printed it. Printed as `= <value>` and *alongside* the OCR alias rather than
+ * instead of it, so when the two disagree that is visible instead of resolved
+ * by whichever one the renderer preferred. Disagreement is the signal.
+ *
+ * Skipped when the label already says it, which is most switches and rows: iOS
+ * labels a settings row "Larger Text, Off" and printing `= Off` after that is
+ * noise.
+ */
+function valueNote(r) {
+  if (r.value == null || r.value === '') return null;
+  const v = trim(String(r.value));
+  if (!v) return null;
+  const said = alnum(r.label);
+  if (said && alnum(v) && said.includes(alnum(v))) return null;
+  return `= ${v}`;
+}
+
 function renderRow(r) {
   const name = [
     trim(r.label) || (matching.isAxTarget(r) ? '(unlabelled)' : '(no text)'),
+    valueNote(r),
     aliasNote(r),
   ].filter(Boolean).join(' ');
   const state = [
@@ -320,6 +380,7 @@ export function render({ device, identity, rows, truncated, collapsed, screen, n
       : 'screen unidentified',
     identity?.keyboard ? 'keyboard up' : null,
     identity?.settled === false ? 'STILL MOVING' : null,
+    recalledNote(identity),
   ].filter(Boolean).join(' · ');
 
   const lines = [head];