simframe 0.8.0 → 0.10.0

package/src/platform/android.js

@@ -827,7 +827,7 @@ async function launchApp(udid, bundleId, { args = [], env = {}, terminateFirst =
 async function terminateApp(udid, bundleId) {
   // `am` reports failure on stdout and still exits 0 — the same trap launchApp
   // and openUrl already check for. Without this, terminating a package that is
-  // not installed answered "terminated com.typo.app".
+  // not installed answered "terminated com.example.mistyped".
   const { stdout, stderr } = await adb(udid, ['shell', 'am', 'force-stop', bundleId]);
   const error = /^Error:.*$/m.exec(`${stdout}${stderr}`);
   if (error) {
@@ -974,6 +974,16 @@ function toolchain() {
  * driver's name. `uiautomator dump` costs 2,012 ms a read, which is why it is
  * not the answer; see docs/DEFERRED.md for the shape of the one that would be.
  */
+async function bootedAt(serial) {
+  try {
+    const out = await adb(serial, ['shell', 'cat', '/proc/uptime']);
+    const seconds = Number(String(out.stdout ?? out).trim().split(/\s+/)[0]);
+    return Number.isFinite(seconds) ? Date.now() - seconds * 1000 : null;
+  } catch {
+    return null;
+  }
+}
+
 function capabilities() {
   return {
     captureEngines: ['screenshot'],
@@ -1004,6 +1014,11 @@ export const platform = {
   setPasteboard,
   getPasteboard,
   permissionServices: () => PERMISSION_SERVICES,
+  // Uptime, because there is no CoreSimulator directory to stat. `/proc/uptime`
+  // is seconds since boot, so boot is now minus that — and it is a real
+  // answer rather than the other platform's vocabulary, which is the rule a
+  // backend that cannot answer has to follow.
+  bootedAt,
   capabilities,
   toolchain,
 };

package/src/platform/index.js

@@ -63,6 +63,7 @@ export const PLATFORM_SURFACE = Object.freeze([
   'geometry', 'inputDriver',
   'screenshot', 'launchApp', 'terminateApp', 'openUrl',
   'setPermission', 'setPasteboard', 'permissionServices', 'capabilities', 'toolchain',
+  'bootedAt',
 ]);
 
 /** @type {Record<string, Platform>} */
@@ -237,6 +238,8 @@ export function permissionServices(udid) {
  * knows. Tap points computed from that are wrong, and nothing says so.
  */
 export const geometryFor = (udid) => platformFor(udid).geometry(udid);
+/** When the device last booted, epoch ms, or null. Both backends answer; neither guesses. */
+export const bootedAtFor = (udid) => platformFor(udid).bootedAt(udid);
 
 /**
  * The backend's own input path, or null when input comes from above the

package/src/platform/ios.js

@@ -5,6 +5,9 @@
 // and reachable only through the `platform` object at the bottom — the
 // JavaScript counterpart of the `SimulatorPlatform` protocol in Swift.
 import { execFile, execFileSync } from 'node:child_process';
+import fs from 'node:fs';
+import os from 'node:os';
+import path from 'node:path';
 import { promisify } from 'node:util';
 
 const run = promisify(execFile);
@@ -272,6 +275,42 @@ function geometry() {
  * *is* the platform's — the emulator console — which is why this is a question
  * a backend gets asked at all.
  */
+/**
+ * When this device last booted, in epoch ms, or null if it cannot be told.
+ *
+ * Why it matters: the HID session lives in the daemon, and a device restart
+ * kills it while leaving the daemon perfectly healthy. Every tap after that is
+ * dispatched successfully and moves nothing — measured, five runs in a row,
+ * on the correct coordinates for the correct element. Only hardware buttons
+ * recover on their own, deliberately, because retrying a tap can act twice.
+ *
+ * The signal is a stat, not a `simctl` call: CoreSimulator writes
+ * `data/var/run/syslog.pid` when the device's syslogd starts, and touches
+ * `device.plist` on every state change. Both read 21:21:26 on a device booted
+ * at 21:21:26. A stat costs microseconds, which matters because this is
+ * checked before input.
+ *
+ * A false positive costs one session rebuild and no action, so the ordering
+ * prefers the most boot-specific marker and falls back rather than guessing.
+ */
+function bootedAt(udid) {
+  const dir = path.join(
+    os.homedir(), 'Library', 'Developer', 'CoreSimulator', 'Devices', udid,
+  );
+  for (const marker of [
+    path.join(dir, 'data', 'var', 'run', 'syslog.pid'),
+    path.join(dir, 'data', 'var', 'run'),
+    path.join(dir, 'device.plist'),
+  ]) {
+    try {
+      return fs.statSync(marker).mtimeMs;
+    } catch {
+      /* try the next marker */
+    }
+  }
+  return null;
+}
+
 function inputDriver() {
   return null;
 }
@@ -301,6 +340,7 @@ export const platform = {
   isBootedSync,
   ownsUdid,
   geometry,
+  bootedAt,
   inputDriver,
   screenshot,
   launchApp,

package/src/screenmap.js

@@ -12,11 +12,12 @@ import * as control from './control.js';
 import * as fingerprint from './fingerprint.js';
 import * as input from './input.js';
 import * as ocr from './ocr.js';
+import * as matching from './matching.js';
 import * as regions from './regions.js';
 import { informative } from './refs.js';
 import * as store from './store.js';
 
-const MAP_VERSION = 7; // footprintless elements and containers no longer enter identity
+const MAP_VERSION = 9; // ax targets carry value, selected and focused
 
 function mapDir(udid) {
   return path.join(store.deviceDir(udid), 'screens');
@@ -184,11 +185,30 @@ export async function build(udid, {
         if (!n.frame || !n.label || isContainer(n)) continue;
         targets.push({
           label: n.label,
+          // What the control *contains*, whether it is on, and whether it has
+          // focus. All three come off the accessibility tree, the daemon has
+          // asked for all three since 0.6.0, and all three were dropped before
+          // this — `value` here and the other two one layer up in
+          // `normalizeNode` — so nothing above this line ever saw them.
+          //
+          // The cost was not theoretical. A real session could not verify the
+          // contents of a text field at all: they reached a row only as the OCR
+          // alias, which made them as old as the map and unauthoritative. An
+          // assert failed against a field that did contain the string, the
+          // operator retyped, and the field ended up with a doubled value and a
+          // validation error.
+          //
+          // `focused` is worth naming separately: it is a direct answer to "did
+          // this field take focus", which the focus wait in actions.js infers
+          // from elapsed time because it had nothing better to use.
+          value: n.value ?? undefined,
           x: input.centerOf(n).x,
           y: input.centerOf(n).y,
           frame: n.frame,
           type: n.type,
           enabled: n.enabled,
+          selected: n.selected ?? undefined,
+          focused: n.focused ?? undefined,
           source: 'ax',
         });
       }
@@ -232,22 +252,43 @@ export async function build(udid, {
         const point = { x: w.centerX, y: w.centerY };
         // If an accessibility element already covers this text, it is the same
         // control: keep the element and record the visible text as an alias.
-        // Containing text is not the same as being that control. A tab bar
-        // encloses all five tab labels but is not any of them, so only merge
-        // when the element is close to the text's own size.
+        //
+        // Two rules, and the second one cost 16 escalations and half of
+        // HPI_accuracy. The first is a size test: an element close to the
+        // text's own size, containing it, is that text. It exists to stop a
+        // tab bar from swallowing all five of its tab labels — containing text
+        // is not the same as being that control.
+        //
+        // But a full-width list row is 19× the area of the words printed in
+        // it, so the size test could never fire for the shape it matters most
+        // on: every Contacts and Settings row arrived as an ax element AND as
+        // an OCR text box, both scoring 1.00 for the same query, and `tap
+        // "Kate Bell"` refused as ambiguous on all five runs of the
+        // instrumented flow suite. The second rule is the one the size test
+        // was standing in for: near-total containment AND the same text. A tab
+        // bar contains "Assets" but is not labelled "Assets", so it is still
+        // refused; a row labelled "Kate Bell" containing OCR's "Kate Bell" is
+        // one element that two sensors saw.
         const textArea = Math.max(1, w.width * w.height);
+        const box = { x: w.x, y: w.y, width: w.width, height: w.height };
+        const eligible = (t) =>
+          matching.isAxTarget(t) &&
+          t.frame &&
+          !/^(Group|Application|ScrollView|Table|Collection)$/i.test(t.type || '');
         const covering = targets
-          .filter(
-            (t) =>
-              t.source === 'ax' &&
-              t.frame &&
-              inside(point, t.frame) &&
-              !/^(Group|Application|ScrollView|Table|Collection)$/i.test(t.type || '') &&
-              area(t.frame) <= textArea * 8,
-          )
-          .sort((a, b) => area(a.frame) - area(b.frame))[0];
+          .filter((t) => eligible(t) && inside(point, t.frame) && area(t.frame) <= textArea * 8)
+          .sort((a, b) => area(a.frame) - area(b.frame))[0]
+          ?? targets
+            .filter((t) => eligible(t) && matching.sameElementSeenTwice(t, { ...w, frame: box, label: w.text }))
+            .sort((a, b) => area(a.frame) - area(b.frame))[0];
         if (covering) {
           covering.aliases = [...(covering.aliases || []), w.text];
+          // Keep the ax role and frame — it is the hit target — and record that
+          // both sensors saw it. Anything asking "is this the tree's element?"
+          // must ask matching.isAxTarget, not `=== 'ax'`.
+          if (!String(covering.source ?? '').includes('ocr')) {
+            covering.source = `${covering.source ?? 'ax'}|ocr`;
+          }
           continue;
         }
         targets.push({
@@ -323,7 +364,7 @@ export function rank(entry, query) {
     ? exact
     : entry.targets.filter((t) => names(t).some((n) => n.includes(q)));
   return pool
-    .map((t) => ({ target: t, score: (isInteractive(t) ? 2 : 0) + (t.source === 'ax' ? 1 : 0) }))
+    .map((t) => ({ target: t, score: (isInteractive(t) ? 2 : 0) + (matching.isAxTarget(t) ? 1 : 0) }))
     .sort((a, b) => b.score - a.score)
     .map((r) => r.target);
 }

package/src/view.js

@@ -14,6 +14,7 @@
 import * as api from './index.js';
 import * as graph from './graph.js';
 import { writeRefs } from './refs.js';
+import * as matching from './matching.js';
 
 /** Reading order. Chrome frames the screen, so it reads first and last. */
 const REGION_ORDER = ['nav-bar', 'content', 'tab-bar', 'keyboard', 'status-bar'];
@@ -78,7 +79,7 @@ function isHost(t) {
   // An accessibility element the app gave a label to is a unit the app itself
   // considers one thing — a dashboard tile reading "WOs past ETA, 1910" is one
   // tap target whose parts OCR happens to read separately.
-  return t.source === 'ax' && Boolean(t.label);
+  return matching.isAxTarget(t) && Boolean(t.label);
 }
 
 /**
@@ -152,7 +153,7 @@ function dropContainers(targets, screen) {
  * an ellipsis menu, a chevron it decided was a period. Nothing can be tapped by
  * that name, so listing it is pure cost.
  */
-const isNoise = (t) => t.source === 'ocr' && !alnum(t.label);
+const isNoise = (t) => !matching.isAxTarget(t) && t.source === 'ocr' && !alnum(t.label);
 
 const trim = (text) => {
   const one = String(text ?? '').replace(/\s+/g, ' ').trim();
@@ -201,9 +202,69 @@ export function rowsFor(entry, { screen, filter, interactive, all = false, limit
   return { rows: rows.slice(0, limit), truncated: Math.max(0, rows.length - limit), collapsed };
 }
 
+/**
+ * Say when the rows below were remembered rather than looked at.
+ *
+ * Screen memory is deliberately keyed on the pixel layout hash, because a list
+ * with new rows is the same screen and re-perceiving it per step is the cost
+ * Phase 13 exists to remove. That is right for *identity* and wrong for
+ * *contents*, and the map made no distinction: a field's text reaches a row as
+ * an OCR alias, so a recalled map reports the text the field held when the map
+ * was built. Reported from a real session — a picker described the previous
+ * sheet's options and did it in 23 ms, which is the giveaway, because 23 ms is
+ * not enough time to have looked.
+ *
+ * This does not fix that. It stops it being invisible, which is the part that
+ * cost two wrong conclusions about an app.
+ *
+ * Only past a second, because a map built by this very call is not a
+ * recollection and saying so on every screen is how a real warning gets
+ * skimmed.
+ */
+export const RECALL_NOTE_FLOOR_MS = 1000;
+
+export function recalledNote(identity, now = Date.now()) {
+  const at = identity?.entry?.at;
+  if (!Number.isFinite(at)) return null;
+  const age = now - at;
+  if (age < RECALL_NOTE_FLOOR_MS) return null;
+  const ago = age < 60_000 ? `${Math.round(age / 1000)}s` : `${Math.round(age / 60_000)}m`;
+  return `elements recalled from ${ago} ago — pass refresh for what is there now`;
+}
+
+/**
+ * What a control *contains*, from the sensor that actually knows.
+ *
+ * A field's text reached a row only as the OCR alias, which means it was as old
+ * as the map and had no authoritative source at all. Reported from a real
+ * session: an `assert` on a field's contents failed against a field that did
+ * contain the string, the operator retyped, and the field ended up with a
+ * doubled value and a validation error. A character counter read `0/1000` in
+ * the map and `56/1000` in a screenshot of the same frame.
+ *
+ * The accessibility tree carries `value` and always has —
+ * `input.elementToNode` sets it on every node — and the renderer simply never
+ * printed it. Printed as `= <value>` and *alongside* the OCR alias rather than
+ * instead of it, so when the two disagree that is visible instead of resolved
+ * by whichever one the renderer preferred. Disagreement is the signal.
+ *
+ * Skipped when the label already says it, which is most switches and rows: iOS
+ * labels a settings row "Larger Text, Off" and printing `= Off` after that is
+ * noise.
+ */
+function valueNote(r) {
+  if (r.value == null || r.value === '') return null;
+  const v = trim(String(r.value));
+  if (!v) return null;
+  const said = alnum(r.label);
+  if (said && alnum(v) && said.includes(alnum(v))) return null;
+  return `= ${v}`;
+}
+
 function renderRow(r) {
   const name = [
-    trim(r.label) || (r.source === 'ax' ? '(unlabelled)' : '(no text)'),
+    trim(r.label) || (matching.isAxTarget(r) ? '(unlabelled)' : '(no text)'),
+    valueNote(r),
     aliasNote(r),
   ].filter(Boolean).join(' ');
   const state = [
@@ -319,6 +380,7 @@ export function render({ device, identity, rows, truncated, collapsed, screen, n
       : 'screen unidentified',
     identity?.keyboard ? 'keyboard up' : null,
     identity?.settled === false ? 'STILL MOVING' : null,
+    recalledNote(identity),
   ].filter(Boolean).join(' · ');
 
   const lines = [head];