simframe 0.12.2 → 0.13.0

package/src/platform/ios.js

@@ -178,17 +178,42 @@ function isBootedSync(udid) {
   return false;
 }
 
+/**
+ * What went wrong with a `simctl io screenshot`, in one sentence.
+ *
+ * Extracted so it can be *tested* rather than reasoned about, for the same
+ * reason `pickDevice` and `decisionOf` were: this is the line where a wrong
+ * answer was expensive, and it was wrong for a day.
+ */
+export function screenshotFailure(err) {
+  const killed = err.killed || err.signal === 'SIGTERM';
+  // `simctl` opens with `Note: No display specified …` on every run, success or
+  // failure. When the display surface is dead the command does not fail, it
+  // *hangs* — so at kill time that Note is the only thing on stderr, and the
+  // tool reported a benign informational line as the reason a capture failed.
+  // That is how this wedge stayed nameless through five CI failures.
+  const lines = String(err.stderr || '').trim().split('\n').map((l) => l.trim()).filter(Boolean);
+  const real = lines.filter((l) => !/^Note:/i.test(l)).pop();
+  if (killed && !real) {
+    // Run to completion the device names it exactly:
+    //   NSPOSIXErrorDomain code 60 — Timeout waiting for screen surfaces
+    // which is CoreSimulator saying the surface is gone, and the closest thing
+    // to a positive test for the wedge that exists.
+    return 'simctl screenshot did not return within 10s. The display surface is not answering'
+      + ' — run to completion it reports "Timeout waiting for screen surfaces" (NSPOSIXErrorDomain 60).'
+      + ' This is the device, not the capture loop: `simframe revive` restarts it.';
+  }
+  const detail = real ?? lines.pop();
+  return detail ? `simctl screenshot failed: ${detail}` : `simctl screenshot failed: ${err.message}`;
+}
+
 async function screenshot(udid, outFile, { mask = 'ignored' } = {}) {
   try {
     await run('xcrun', ['simctl', 'io', udid, 'screenshot', '--type=png', `--mask=${mask}`, outFile], {
       timeout: 10_000,
     });
   } catch (err) {
-    // Same reason as launchApp: execFile's message is "Command failed: <the
-    // whole command>" and simctl's actual complaint is in stderr. A CI failure
-    // here reported the command and nothing about why it did not work.
-    const detail = (err.stderr || '').trim().split('\n').filter(Boolean).pop();
-    throw new Error(detail ? `simctl screenshot failed: ${detail}` : `simctl screenshot failed: ${err.message}`);
+    throw new Error(screenshotFailure(err));
   }
 }
 

package/src/refs.js

@@ -149,7 +149,18 @@ export function resolveRef(udid, n, { structuralHash, layoutHash, screenKnown, s
   // numbers. Refusing costs a re-read; guessing taps whatever is at those
   // coordinates now.
   if (screenKnown === false) {
-    throw new Error(`#${n} cannot be trusted here — simframe does not recognise this screen. Read it again (sim_ui) to renumber.`);
+    // Flagged, like every other refusal in this function, and it was the one
+    // that was not.
+    //
+    // We tell callers to read `staleRef` rather than the sentence — the CI check
+    // for this very guard carries a comment saying it matched on prose twice and
+    // went red twice, so it reads the contract now. Then it went red a third
+    // time, on a refusal that was correct, well worded, and carried no field at
+    // all: `#1 cannot be trusted here — simframe does not recognise this
+    // screen`, reported by the harness as `no reason`. A refusal a human can
+    // read and a program cannot is the same defect as a failure that reads like
+    // a success, one level down.
+    throw staleError('simframe does not recognise this screen', 'unknown-screen');
   }
   // The pixel check stays, but only as a backstop, and only where it means
   // something. A dark or near-uniform screen produces a layout hash of almost

package/src/regions.js

@@ -470,6 +470,60 @@ export function offViewport(t, screen) {
   return false;
 }
 
+/**
+ * The smallest sliver of an element that is worth offering as a tap target.
+ *
+ * Apple's own minimum touch target is 44pt; this is deliberately smaller,
+ * because the question here is not "is this comfortable to tap" but "is this a
+ * real control a person can see and reach". A filter chip showing 29pt of
+ * itself at the edge of a horizontal strip is both. Below this, what is on
+ * screen is bleed rather than a control.
+ */
+export const MIN_VISIBLE_PT = 24;
+
+/**
+ * How much of an element is actually on screen, and where to tap what is.
+ *
+ * `offViewport` answers a yes/no question about the element's *centre*, which
+ * is right for "should I scroll to reach this" and wrong for "is this here at
+ * all". A chip at the end of a horizontal strip showed **29pt of itself** on a
+ * 402pt screen — real, visible, tappable — while its centre sat at x=416, so it
+ * was dropped from the map entirely. What the caller got in its place was OCR's
+ * reading of the visible sliver: a `text` element labelled **"Flc"** at x=392,
+ * which passes every filter because its own box is inside the viewport.
+ *
+ * So the map did not merely omit a control. It offered a different, meaningless
+ * name for it, at a coordinate that looks perfectly ordinary. That is the shape
+ * of item 121 — a caller who cannot trust what the map says about the edge of
+ * the screen — and the item's own words are "say so or clamp". This does both.
+ */
+export function clipping(t, screen) {
+  const f = t?.frame;
+  const w = screen?.width;
+  const h = screen?.height;
+  if (!f || !Number.isFinite(w) || !Number.isFinite(h)) return null;
+  const left = Math.max(0, f.x);
+  const right = Math.min(w, f.x + f.width);
+  const top = Math.max(0, f.y);
+  const bottom = Math.min(h, f.y + f.height);
+  const visibleWidth = right - left;
+  const visibleHeight = bottom - top;
+  if (visibleWidth <= 0 || visibleHeight <= 0) {
+    return { visibleWidth: 0, visibleHeight: 0, clipped: true, usable: false, point: null };
+  }
+  const clipped = f.x < 0 || f.y < 0 || f.x + f.width > w || f.y + f.height > h;
+  return {
+    visibleWidth,
+    visibleHeight,
+    clipped,
+    usable: visibleWidth >= MIN_VISIBLE_PT && visibleHeight >= MIN_VISIBLE_PT,
+    // The centre of what can be seen, not the centre of the element. Tapping
+    // the latter would aim off the screen, which is the clamp the item asked
+    // for and the reason a caller could not simply be handed the real centre.
+    point: { x: Math.round((left + right) / 2), y: Math.round((top + bottom) / 2) },
+  };
+}
+
 /** Annotate a target list with region and nav slot. Mutates and returns it. */
 export function annotate(targets, screen) {
   const band = bands(targets, screen);

package/src/screenmap.js

@@ -296,15 +296,64 @@ export async function build(udid, {
     try {
       // The tree is already in hand when the daemon answered; describeAll would
       // only ask for it a second time.
-      const nodes = daemonScreen?.sources?.includes('ax')
-        ? daemonScreen.elements.filter((e) => e.source?.includes('ax')).map(input.elementToNode)
-        : await input.describeAll(udid);
+      //
+      // And when the daemon answered *without* a tree, asking again is asking
+      // the thing that just failed. That path cost a red CI run and a wrong
+      // diagnosis: the daemon's ax read timed out, `describeAll` re-asked it
+      // (another 20s), got the same nothing, fell through to idb, and the map
+      // reported **"idb is not installed, so simframe can observe the screen but
+      // cannot touch it"**. A sentence about a tool this project deliberately
+      // does not use on CI, printed because a different tool had a bad read —
+      // and it points whoever reads it at installing idb, which would change
+      // nothing. The same read worked a minute later.
+      //
+      // The daemon has sent `axError` since it learned to time its own reads,
+      // and nothing here had ever read it. The OCR branch below reads
+      // `ocrError`, which is what makes the asymmetry visible: one sensor could
+      // say why it failed and the other borrowed a different tool's excuse.
+      let nodes;
+      if (daemonScreen?.sources?.includes('ax')) {
+        nodes = daemonScreen.elements.filter((e) => e.source?.includes('ax')).map(input.elementToNode);
+      } else if (daemonScreen) {
+        throw new Error(daemonScreen.axError ?? 'the daemon read the screen and the accessibility tree did not answer');
+      } else {
+        nodes = await input.describeAll(udid);
+      }
 
       sources.push('ax');
+      // A control the tree gives no name to is still a control — item 122.
+      //
+      // This line used to read `!n.label`, on the reasoning that a row nobody
+      // can name is a row nobody can tap. The whole of item 122 is that the
+      // opposite is true, and three field reports in a row said so
+      // independently: icon-only overflow menus on every card, a bottom-sheet
+      // drag handle, a back chevron. Real, tappable, on screen, and absent from
+      // the map — so every one of them needed a raw `@x,y` read off a
+      // screenshot, which is the exact round trip the text map exists to
+      // remove. Knowing that *a hit target is there* is most of the value even
+      // with no semantics attached to it.
+      //
+      // Measured on the testbed before it was written, because the premise
+      // could have been false: on the list screen, of 39 accessibility nodes
+      // exactly **one** is nameless — and it is the one control on that screen
+      // no caller could reach. This does not flood the map.
+      //
+      // It also settles a thing the reports could not: those controls were
+      // called "absent from the tree", and AXPTranslator had them all along.
+      // What is genuinely absent is the drag handle, a plain view holding a
+      // responder that UIKit is never told is accessible. No filter here can
+      // recover that one; see 122's second half.
       for (const n of nodes) {
-        if (!n.frame || !n.label || isContainer(n)) continue;
+        if (!n.frame || isContainer(n)) continue;
+        if (nameless(n) && !namelessHitTarget(n, nodes)) continue;
         targets.push({
-          label: n.label,
+          label: n.label ?? undefined,
+          // A `testID` arrives as AXIdentifier, and `tap` has matched on it
+          // since long before this — but only down the tree path, because the
+          // map never carried it. So the one name an unlabelled React Native
+          // control usually does have was invisible in the map and worked if
+          // you guessed it.
+          identifier: n.identifier ?? undefined,
           // What the control *contains*, whether it is on, and whether it has
           // focus. All three come off the accessibility tree, the daemon has
           // asked for all three since 0.6.0, and all three were dropped before
@@ -506,6 +555,36 @@ export function isInteractive(target) {
   return INTERACTIVE.test(target.type || '');
 }
 
+/** No name from any source: not a label, not an identifier. */
+export const nameless = (n) => !n?.label && !n?.identifier;
+
+/**
+ * Whether a nameless accessibility node is worth printing as a hit target.
+ *
+ * Two guards, because "has no name" is not the same as "is a control".
+ *
+ * The role has to be one the tree calls interactive. A nameless `Other` is a
+ * layout view and a real screen has hundreds of them; emitting those would bury
+ * the one node that matters under the scenery it sits in.
+ *
+ * And it must not enclose two or more other elements. That is the same test the
+ * fingerprint uses for scenery, reused deliberately rather than invented here:
+ * a table cell holding its own labels is not the target, its labels are.
+ */
+export function namelessHitTarget(node, nodes) {
+  const f = node?.frame;
+  if (!f || !(f.width > 0) || !(f.height > 0)) return false;
+  if (!INTERACTIVE.test(node.type || '')) return false;
+  const encloses = nodes.filter((o) => {
+    const g = o.frame;
+    if (!g || o === node) return false;
+    const cx = g.x + (g.width ?? 0) / 2;
+    const cy = g.y + (g.height ?? 0) / 2;
+    return cx > f.x && cx < f.x + f.width && cy > f.y && cy < f.y + f.height;
+  }).length;
+  return encloses < 2;
+}
+
 /**
  * Rank candidates for a label. Exact beats substring, and a real control beats
  * a caption that happens to read the same — a screen title and a tab are often
@@ -514,7 +593,7 @@ export function isInteractive(target) {
 export function rank(entry, query) {
   if (!entry) return [];
   const q = norm(query);
-  const names = (t) => [t.label, ...(t.aliases || [])].map(norm);
+  const names = (t) => [t.label, t.identifier, ...(t.aliases || [])].map(norm);
   const exact = entry.targets.filter((t) => names(t).includes(q));
   const pool = exact.length
     ? exact

package/src/store.js

@@ -34,10 +34,63 @@ export function paths(udid) {
     // wedged. It cannot ride in state.json: that is written when a frame is
     // recorded, and a stall is the absence of frames.
     captureHealth: path.join(dir, 'capture-health.json'),
+    lastInput: path.join(dir, 'last-input'),
   };
 }
 
 /** What the capture loop last said about its own health, or null if it has no complaint. */
+/**
+ * When input was last delivered to this device.
+ *
+ * Written by every input path and read by `liveness`, which needs it to catch
+ * the one wedge shape nothing else can see: a capture loop that is alive,
+ * incrementing its frame counter, and re-reading a **dead surface**. Field
+ * report, 0.12.2: `age=268ms` next to `stable=159753ms` while three screen
+ * transitions had just happened, and no warning fired because every signal we
+ * had was green. A screen that has not moved since before we last touched it,
+ * over and over, is a contradiction the daemon can notice locally.
+ */
+const INPUT_MEMORY = 8;
+
+export function noteInput(udid, at = Date.now()) {
+  try {
+    writeAtomic(paths(udid).lastInput, [...inputTimes(udid), at].slice(-INPUT_MEMORY).join(','));
+  } catch {
+    /* a timestamp nothing depends on for correctness must not fail an action */
+  }
+}
+
+/**
+ * The last few input timestamps, oldest first.
+ *
+ * A list rather than a single timestamp, and that is the whole correction.
+ * The first version kept only the latest, so the only question it could ask was
+ * "has the screen been still for a long time?" — which needed a duration
+ * threshold, and the threshold is what defeated it. A field report caught a
+ * three-hour-stale frame on a screen that had been still for **8.2 seconds**,
+ * under a 20-second gate, so the check could not fire on the case it was
+ * written for.
+ *
+ * What actually says "dead surface" is not duration. It is **several gestures
+ * delivered with no pixel moving at all** — one tap that changes nothing is
+ * ordinary, and three in a row are not.
+ */
+export function inputTimes(udid) {
+  try {
+    return fs.readFileSync(paths(udid).lastInput, 'utf8')
+      .split(',')
+      .map(Number)
+      .filter(Number.isFinite);
+  } catch {
+    return [];
+  }
+}
+
+export function lastInputAt(udid) {
+  const times = inputTimes(udid);
+  return times.length ? times[times.length - 1] : null;
+}
+
 export function captureHealth(udid) {
   return readJson(paths(udid).captureHealth);
 }

package/src/supervisor.js

@@ -84,6 +84,18 @@ function backendFor(want) {
  * text — so it broke when the branch grew an else, with nothing actually wrong.
  * A property this important deserves an assertion that runs it.
  */
+/**
+ * The fourth word, and why it is not in `DECISIONS`.
+ *
+ * `abstain` means "I cannot tell from what I was given", and the only correct
+ * thing to do with it is exactly what this module already does with every
+ * failure: return `null`, and behave as if there is no supervisor. So it is a
+ * recognised *answer* and not a recognised *decision*, and keeping those apart
+ * is the point — a caller must never be able to act on it, and a log must be
+ * able to tell it from a timeout, a refusal and a model that was never there.
+ */
+export const ABSTAIN = 'abstain';
+
 export function decisionOf(answer) {
   // A string, checked rather than coerced. `String(["wait"])` is `"wait"`, so a
   // `String(...)` coercion here let `{decision: ["wait"]}` through the one gate
@@ -113,7 +125,7 @@ export function requested(options) {
  */
 export async function judge({
   goal, step, expected, failure, screen, stillMs, note, options, timeoutMs = 2500,
-  detail,
+  detail, mayAbstain = false,
 } = {}) {
   const want = requested(options);
   if (!want) return null;
@@ -131,6 +143,7 @@ export async function judge({
     screen: (screen ?? []).filter(Boolean).map((s) => String(s).slice(0, 40)).slice(0, 25),
     stillMs: Number.isFinite(stillMs) ? Math.round(stillMs) : null,
     note: note ? String(note).slice(0, 200) : null,
+    mayAbstain: Boolean(mayAbstain),
   }, timeoutMs);
   const decision = decisionOf(answer);
   if (decision == null) {
@@ -144,7 +157,12 @@ export async function judge({
     // supervisor did not answer": a timeout, a guardrail refusal and a model
     // that was never installed were one indistinguishable line.
     if (detail && typeof detail === 'object') {
-      detail.kind = answer?.kind
+      // An abstention is an answer, and the least interesting thing a log can
+      // say about it is that the supervisor "did not answer". It is the model
+      // declining on purpose, which is the one failure mode worth encouraging.
+      detail.kind = answer?.decision === ABSTAIN
+        ? 'abstained'
+        : answer?.kind
         ?? (answer == null ? 'no answer' : answer.decision ? 'outside the vocabulary' : 'unparseable');
       if (answer?.error) detail.error = String(answer.error).slice(0, 200);
     }

package/src/view.js

@@ -247,8 +247,26 @@ export function actsInteractive(t) {
 export function rowsFor(entry, { screen, filter, interactive, all = false, limit = DEFAULT_LIMIT } = {}) {
   let kept = (entry?.targets ?? []).map((t) => ({ ...t })).filter((t) => {
     if (!isNum(t.x) || !isNum(t.y)) return false;
-    // Off-screen elements are real in the tree and untappable in fact.
-    if (regions.offViewport(t, screen)) return false;
+    // Off-screen elements are real in the tree and untappable in fact —
+    // *unless* enough of one is on screen to tap, which the centre test cannot
+    // see. A filter chip showing 29pt of itself at the right edge was dropped
+    // while OCR's reading of that same sliver, labelled "Flc", was kept and
+    // printed in its place. Item 121: say so or clamp. Both, here — the row is
+    // kept, its tap point is moved to the centre of the visible part, and it is
+    // marked so nobody reads a clamped coordinate as a whole element.
+    //
+    // Deliberately NOT also relaxing `locate`. A clipped element still answers
+    // "in the tree but not in view", so `scrollTo` keeps scrolling to it rather
+    // than declaring a sliver good enough — that conservatism was bought with a
+    // reported session lost to `"'Assigned to Me' is in view at 422,277
+    // already"`, and widening it here would buy the same bug back.
+    if (regions.offViewport(t, screen)) {
+      const clip = regions.clipping(t, screen);
+      if (!clip?.usable) return false;
+      t.clipped = true;
+      t.x = clip.point.x;
+      t.y = clip.point.y;
+    }
     if (!all && !regions.offerable(t.region)) return false;
     if (!all && isNoise(t)) return false;
     return true;
@@ -380,11 +398,42 @@ function valueNote(r) {
   return `= ${v}`;
 }
 
+/**
+ * What to call a row.
+ *
+ * A control with no accessibility label is not necessarily anonymous, and the
+ * order here is by how much the name can be trusted. Its own label first. Then
+ * its `testID`, which is a name a developer chose and which `tap` has always
+ * matched on. Then OCR's reading of it, which is how an icon-only control gets
+ * a name at all and which cannot contradict a label it does not have. Only when
+ * all three are missing is it really unaddressable, and that is the case item
+ * 122 is about: three field reports in a row spent their time on controls that
+ * existed, were tappable, and could be reached by no selector at all.
+ */
+export function displayName(r) {
+  return trim(r.label)
+    || trim(r.identifier)
+    // Only an alias with a word in it. OCR reads the three dots of an overflow
+    // menu as `...`, and on the first live run of this the button was named
+    // `#1 button 364,84 ...` — which looks like a name, cannot be typed into a
+    // selector, and made the count below say there was nothing unnamed here.
+    // `isNoise` already refuses such text as a row of its own; it must not get
+    // in through the alias door either.
+    || (r.aliases ?? []).find((a) => alnum(a))
+    || (matching.isAxTarget(r) ? '(unlabelled)' : '(no text)');
+}
+
+/** A row the map can print and no caller can name. */
+export const unaddressable = (r) =>
+  matching.isAxTarget(r) && !trim(r.label) && !trim(r.identifier)
+  && !(r.aliases ?? []).some((a) => alnum(a));
+
 function renderRow(r) {
+  const shown = displayName(r);
   const name = [
-    trim(r.label) || (matching.isAxTarget(r) ? '(unlabelled)' : '(no text)'),
+    shown,
     valueNote(r),
-    aliasNote(r),
+    aliasNote(r, shown),
   ].filter(Boolean).join(' ');
   const state = [
     r.enabled === false ? 'disabled' : null,
@@ -395,7 +444,9 @@ function renderRow(r) {
     shortType(r.type).padEnd(9),
     `${r.x},${r.y}`.padEnd(9),
     state ? `${state} ` : '',
-    name,
+    // After the name, and appended rather than added as a column: an extra
+    // element in this join puts an extra space on *every* row, clipped or not.
+    r.clipped ? `${name}  (partly off-screen — the coordinate is the middle of the visible part)` : name,
   ].join(' ');
 }
 
@@ -406,11 +457,14 @@ function renderRow(r) {
  * for, which is useful when they disagree and pure cost when they agree —
  * "WELCOME ~ WELCOME" was a third of some rows.
  */
-function aliasNote(r) {
+function aliasNote(r, shown = r.label) {
   const extra = (r.aliases ?? [])
     .filter((a) => {
       const t = alnum(a);
-      return t && !alnum(r.label).includes(t);
+      // Compared against the name actually printed, not against the label. An
+      // unlabelled control is named by its own alias now, and comparing against
+      // an absent label printed every one of them twice: `Sort ~ Sort`.
+      return t && !alnum(shown).includes(t);
     })
     .slice(0, 2);
   return extra.length ? `~ ${trim(extra.join(' '))}` : null;
@@ -511,6 +565,25 @@ export async function screenMap(deviceQuery, {
       + ' you did not expect to see as belonging to the layer underneath'
     : null;
 
+  // Controls that are on the screen and answer to no name — item 122.
+  //
+  // Counted from the rows about to be printed rather than from the map, so it
+  // is a statement about what the caller can see. The wording is the ask,
+  // near-verbatim from the report that made it: *"a line like 'N on-screen
+  // tappable views have no accessibility label — they cannot be addressed by
+  // selector' would push people toward instrumenting, which is the outcome
+  // everyone wants"*. It is also the honest answer to the second half of that
+  // item, which we cannot fix from here: a view UIKit was never told is
+  // accessible is invisible to the tree, so a screen whose controls are all
+  // undeclared shows this count as 0 and still needs a screenshot.
+  const anonymous = rows.filter(unaddressable).length;
+  const unnamed = anonymous
+    ? `${anonymous} on-screen control(s) have no accessibility label — they are listed with their`
+      + ' coordinates and can be tapped by point or by #ref, but not by name. If what you are'
+      + ' looking for is not in the list either, the app has views that were never declared'
+      + ' accessible and only a screenshot will find those.'
+    : null;
+
   return {
     device,
     identity,
@@ -529,7 +602,8 @@ export async function screenMap(deviceQuery, {
     staleExits,
     cleared,
     overlay,
-    text: render({ device, identity, rows, truncated, collapsed, screen, name, exits, exitList, staleExits, cleared, overlay }),
+    unnamed,
+    text: render({ device, identity, rows, truncated, collapsed, screen, name, exits, exitList, staleExits, cleared, overlay, unnamed }),
   };
 }
 
@@ -712,7 +786,7 @@ export function ambiguousLabels(rows) {
   return [...seen.values()].filter((n) => n > 1).length;
 }
 
-export function render({ device, identity, rows, truncated, collapsed, screen, name, exits, exitList, staleExits, verdictLine, ambiguities, cleared, overlay }) {
+export function render({ device, identity, rows, truncated, collapsed, screen, name, exits, exitList, staleExits, verdictLine, ambiguities, cleared, overlay, unnamed }) {
   const head = [
     device?.name,
     screen?.width ? `${screen.width}x${screen.height}pt` : null,
@@ -747,6 +821,7 @@ export function render({ device, identity, rows, truncated, collapsed, screen, n
   // already believes, which is the one kind of news that must not be scrolled to.
   if (cleared) lines.push(cleared);
   if (overlay) lines.push(overlay);
+  if (unnamed) lines.push(unnamed);
   const worked = exitsLine(exitList, { stale: staleExits });
   if (worked) lines.push(worked);