simframe 0.7.2 → 0.9.0

package/src/platform/ios.js

@@ -5,6 +5,9 @@
 // and reachable only through the `platform` object at the bottom — the
 // JavaScript counterpart of the `SimulatorPlatform` protocol in Swift.
 import { execFile, execFileSync } from 'node:child_process';
+import fs from 'node:fs';
+import os from 'node:os';
+import path from 'node:path';
 import { promisify } from 'node:util';
 
 const run = promisify(execFile);
@@ -66,6 +69,29 @@ async function resolveDevice(query, opts) {
   const booted = all.filter((d) => d.state === 'Booted');
   if (!query) {
     if (booted.length === 0) throw new Error('no booted simulator (open Simulator.app or run `xcrun simctl boot <udid>`)');
+    // `booted[0]` was the wrong-device bug, and it was worse than it looked.
+    // simctl's order is not "yours" by any definition, so on a machine with
+    // more than one booted simulator a bare `simframe ui` read whichever came
+    // first — and a bare `simframe tap` would have *injected input* into it. A
+    // reviewer reproduced it deterministically against a colleague's simulator.
+    //
+    // `doctor` got a guard for its own fan-out and this default did not, which
+    // is how the same command set could pick two different devices in one
+    // moment. Refusing is the only safe answer here: the seam cannot see which
+    // device simframe is already driving (that is store state, above the
+    // boundary), and a backend must never guess when the cost of guessing wrong
+    // is a tap on somebody else's screen. `ambiguous` so that a second platform
+    // matching cleanly cannot override this — see resolveAcross.
+    if (booted.length > 1) {
+      throw Object.assign(
+        new Error(
+          `${booted.length} simulators are booted and none was named: ` +
+            `${booted.map((d) => `${d.name} (${d.udid})`).join(', ')} — name one with --device, ` +
+            'or set SIMFRAME_DEVICE to pick a default for this shell',
+        ),
+        { ambiguous: true },
+      );
+    }
     return booted[0];
   }
   const q = query.toLowerCase();
@@ -249,6 +275,42 @@ function geometry() {
  * *is* the platform's — the emulator console — which is why this is a question
  * a backend gets asked at all.
  */
+/**
+ * When this device last booted, in epoch ms, or null if it cannot be told.
+ *
+ * Why it matters: the HID session lives in the daemon, and a device restart
+ * kills it while leaving the daemon perfectly healthy. Every tap after that is
+ * dispatched successfully and moves nothing — measured, five runs in a row,
+ * on the correct coordinates for the correct element. Only hardware buttons
+ * recover on their own, deliberately, because retrying a tap can act twice.
+ *
+ * The signal is a stat, not a `simctl` call: CoreSimulator writes
+ * `data/var/run/syslog.pid` when the device's syslogd starts, and touches
+ * `device.plist` on every state change. Both read 21:21:26 on a device booted
+ * at 21:21:26. A stat costs microseconds, which matters because this is
+ * checked before input.
+ *
+ * A false positive costs one session rebuild and no action, so the ordering
+ * prefers the most boot-specific marker and falls back rather than guessing.
+ */
+function bootedAt(udid) {
+  const dir = path.join(
+    os.homedir(), 'Library', 'Developer', 'CoreSimulator', 'Devices', udid,
+  );
+  for (const marker of [
+    path.join(dir, 'data', 'var', 'run', 'syslog.pid'),
+    path.join(dir, 'data', 'var', 'run'),
+    path.join(dir, 'device.plist'),
+  ]) {
+    try {
+      return fs.statSync(marker).mtimeMs;
+    } catch {
+      /* try the next marker */
+    }
+  }
+  return null;
+}
+
 function inputDriver() {
   return null;
 }
@@ -278,6 +340,7 @@ export const platform = {
   isBootedSync,
   ownsUdid,
   geometry,
+  bootedAt,
   inputDriver,
   screenshot,
   launchApp,

package/src/screenmap.js

@@ -12,11 +12,12 @@ import * as control from './control.js';
 import * as fingerprint from './fingerprint.js';
 import * as input from './input.js';
 import * as ocr from './ocr.js';
+import * as matching from './matching.js';
 import * as regions from './regions.js';
 import { informative } from './refs.js';
 import * as store from './store.js';
 
-const MAP_VERSION = 7; // footprintless elements and containers no longer enter identity
+const MAP_VERSION = 8; // an OCR reading contained in a labelled ax element merges into it
 
 function mapDir(udid) {
   return path.join(store.deviceDir(udid), 'screens');
@@ -232,22 +233,43 @@ export async function build(udid, {
         const point = { x: w.centerX, y: w.centerY };
         // If an accessibility element already covers this text, it is the same
         // control: keep the element and record the visible text as an alias.
-        // Containing text is not the same as being that control. A tab bar
-        // encloses all five tab labels but is not any of them, so only merge
-        // when the element is close to the text's own size.
+        //
+        // Two rules, and the second one cost 16 escalations and half of
+        // HPI_accuracy. The first is a size test: an element close to the
+        // text's own size, containing it, is that text. It exists to stop a
+        // tab bar from swallowing all five of its tab labels — containing text
+        // is not the same as being that control.
+        //
+        // But a full-width list row is 19× the area of the words printed in
+        // it, so the size test could never fire for the shape it matters most
+        // on: every Contacts and Settings row arrived as an ax element AND as
+        // an OCR text box, both scoring 1.00 for the same query, and `tap
+        // "Kate Bell"` refused as ambiguous on all five runs of the
+        // instrumented flow suite. The second rule is the one the size test
+        // was standing in for: near-total containment AND the same text. A tab
+        // bar contains "Assets" but is not labelled "Assets", so it is still
+        // refused; a row labelled "Kate Bell" containing OCR's "Kate Bell" is
+        // one element that two sensors saw.
         const textArea = Math.max(1, w.width * w.height);
+        const box = { x: w.x, y: w.y, width: w.width, height: w.height };
+        const eligible = (t) =>
+          matching.isAxTarget(t) &&
+          t.frame &&
+          !/^(Group|Application|ScrollView|Table|Collection)$/i.test(t.type || '');
         const covering = targets
-          .filter(
-            (t) =>
-              t.source === 'ax' &&
-              t.frame &&
-              inside(point, t.frame) &&
-              !/^(Group|Application|ScrollView|Table|Collection)$/i.test(t.type || '') &&
-              area(t.frame) <= textArea * 8,
-          )
-          .sort((a, b) => area(a.frame) - area(b.frame))[0];
+          .filter((t) => eligible(t) && inside(point, t.frame) && area(t.frame) <= textArea * 8)
+          .sort((a, b) => area(a.frame) - area(b.frame))[0]
+          ?? targets
+            .filter((t) => eligible(t) && matching.sameElementSeenTwice(t, { ...w, frame: box, label: w.text }))
+            .sort((a, b) => area(a.frame) - area(b.frame))[0];
         if (covering) {
           covering.aliases = [...(covering.aliases || []), w.text];
+          // Keep the ax role and frame — it is the hit target — and record that
+          // both sensors saw it. Anything asking "is this the tree's element?"
+          // must ask matching.isAxTarget, not `=== 'ax'`.
+          if (!String(covering.source ?? '').includes('ocr')) {
+            covering.source = `${covering.source ?? 'ax'}|ocr`;
+          }
           continue;
         }
         targets.push({
@@ -323,7 +345,7 @@ export function rank(entry, query) {
     ? exact
     : entry.targets.filter((t) => names(t).some((n) => n.includes(q)));
   return pool
-    .map((t) => ({ target: t, score: (isInteractive(t) ? 2 : 0) + (t.source === 'ax' ? 1 : 0) }))
+    .map((t) => ({ target: t, score: (isInteractive(t) ? 2 : 0) + (matching.isAxTarget(t) ? 1 : 0) }))
     .sort((a, b) => b.score - a.score)
     .map((r) => r.target);
 }

package/src/view.js

@@ -14,6 +14,7 @@
 import * as api from './index.js';
 import * as graph from './graph.js';
 import { writeRefs } from './refs.js';
+import * as matching from './matching.js';
 
 /** Reading order. Chrome frames the screen, so it reads first and last. */
 const REGION_ORDER = ['nav-bar', 'content', 'tab-bar', 'keyboard', 'status-bar'];
@@ -78,7 +79,7 @@ function isHost(t) {
   // An accessibility element the app gave a label to is a unit the app itself
   // considers one thing — a dashboard tile reading "WOs past ETA, 1910" is one
   // tap target whose parts OCR happens to read separately.
-  return t.source === 'ax' && Boolean(t.label);
+  return matching.isAxTarget(t) && Boolean(t.label);
 }
 
 /**
@@ -152,7 +153,7 @@ function dropContainers(targets, screen) {
  * an ellipsis menu, a chevron it decided was a period. Nothing can be tapped by
  * that name, so listing it is pure cost.
  */
-const isNoise = (t) => t.source === 'ocr' && !alnum(t.label);
+const isNoise = (t) => !matching.isAxTarget(t) && t.source === 'ocr' && !alnum(t.label);
 
 const trim = (text) => {
   const one = String(text ?? '').replace(/\s+/g, ' ').trim();
@@ -203,7 +204,7 @@ export function rowsFor(entry, { screen, filter, interactive, all = false, limit
 
 function renderRow(r) {
   const name = [
-    trim(r.label) || (r.source === 'ax' ? '(unlabelled)' : '(no text)'),
+    trim(r.label) || (matching.isAxTarget(r) ? '(unlabelled)' : '(no text)'),
     aliasNote(r),
   ].filter(Boolean).join(' ');
   const state = [