simframe 0.12.1 → 0.13.0

package/src/ollama.js

@@ -0,0 +1,232 @@
+/**
+ * A second supervisor arm, for measuring whether capacity matters.
+ *
+ * The owner's call, 2026-09-11: *"we can of course do our own test with a
+ * chosen model… and decide based on the numbers rather than speculations."*
+ * This exists to answer that and nothing else. It is **not** a recommendation,
+ * it is off unless asked for by name, and a model used to measure whether
+ * capacity matters is not a model we ship — conflating those is how a non-goal
+ * erodes.
+ *
+ * No dependency is added: Node has had a global `fetch` since 18, and Ollama
+ * is a process already on the machine, reached over the loopback interface.
+ *
+ * **The fairness condition, which is the whole reason this file is shaped the
+ * way it is.** Most small-model errors are invalid-output faults, and Apple's
+ * guided generation eliminates those at the sampling layer — constraints are
+ * enforced by logit masking, so a fourth word is unrepresentable rather than
+ * rejected afterwards (WWDC25 301; Tech Report §7). An unconstrained challenger
+ * would lose on formatting and we would read it as losing on judgement. So this
+ * arm passes a JSON schema whose `decision` is an enum of the same three words,
+ * and Ollama constrains sampling to it the same way.
+ *
+ * And the briefing is not a second copy. It is **read out of
+ * `native/supervise.swift`**, because two hand-maintained copies of a prompt is
+ * two arms answering different questions, and the difference would be invisible
+ * in the numbers. See `readBrief`.
+ */
+import fs from 'node:fs';
+import path from 'node:path';
+import { fileURLToPath } from 'node:url';
+
+const HERE = path.dirname(fileURLToPath(import.meta.url));
+const SWIFT = path.join(HERE, '..', 'native', 'supervise.swift');
+
+export const DEFAULT_MODEL = 'qwen3:8b';
+export const DEFAULT_HOST = 'http://127.0.0.1:11434';
+
+/**
+ * The Apple arm's own instructions, read from its source.
+ *
+ * Not duplicated. Two arms of a comparison that are briefed differently are
+ * measuring two different things, and nothing in the output would say so — the
+ * numbers would simply be wrong and look fine. The Swift file is the canonical
+ * copy because it is the shipped one; this parses the `let instructions = """
+ * … """` block out of it and un-escapes Swift's trailing-backslash line
+ * continuations, which is how that literal is wrapped.
+ *
+ * Throws rather than falling back to a built-in string. A silent fallback here
+ * would produce exactly the invisible unfairness this function exists to
+ * prevent.
+ */
+export function readBrief(file = SWIFT, { mayAbstain = false } = {}) {
+  const src = fs.readFileSync(file, 'utf8');
+  const base = block(src, 'let instructions = """', file);
+  if (!mayAbstain) return base;
+  // The addendum, from the same file and joined the way the Swift joins it.
+  // The fourth word has to reach both arms identically or the comparison is
+  // measuring two different briefs, which is the failure this whole function
+  // exists to prevent.
+  return `${base}\n\n${block(src, 'let abstainInstructions = instructions + "\\n\\n" + """', file)}`;
+}
+
+function block(src, opener, file) {
+  const open = src.indexOf(opener);
+  if (open === -1) throw new Error(`no ${JSON.stringify(opener)} block in ${file}`);
+  const bodyStart = src.indexOf('\n', open) + 1;
+  const close = src.indexOf('"""', bodyStart);
+  if (close === -1) throw new Error(`unterminated block in ${file}`);
+  return src
+    .slice(bodyStart, close)
+    .split('\n')
+    // A Swift multi-line literal strips the closing delimiter's indentation
+    // from every line; here that is four spaces.
+    .map((l) => l.replace(/^ {4}/, ''))
+    .join('\n')
+    // Swift's line continuation: a trailing backslash removes the newline and
+    // nothing else. Joining with a space instead would insert one after the
+    // space the line already ends with, and the two arms would be reading
+    // briefs that differ — invisibly, and in the one place this whole file
+    // exists to keep identical.
+    .replace(/\\\n/g, '')
+    .trim();
+}
+
+const clip = (t, n) => {
+  const s = String(t ?? '');
+  return s.length > n ? `${s.slice(0, n)}…` : s;
+};
+
+/**
+ * The situation, in the words the other arm gets.
+ *
+ * A line-for-line mirror of `main.swift`'s prompt assembly, including the clip
+ * lengths and the 14-label cap on the screen list. The field order matters as
+ * much as the content: the brief tells the model to weigh the plan's guidance
+ * first, and a prompt that presented it last would be testing a different
+ * instruction.
+ */
+export function promptFor(s = {}) {
+  let prompt = `Step: ${clip(s.step, 120)}\nIt failed with: ${clip(s.failure, 220)}`;
+  if (s.goal) prompt += `\nThe plan's guidance about this app: ${clip(s.goal, 300)}`;
+  if (s.expected) prompt += `\nExpected: ${clip(s.expected, 200)}`;
+  if (Number.isFinite(s.stillMs)) prompt += `\nThe screen has been still for ${s.stillMs}ms`;
+  if (s.note) prompt += `\nPerception note: ${clip(s.note, 160)}`;
+  if (s.screen?.length) {
+    prompt += `\nOn screen now: ${s.screen.slice(0, 14).map((l) => clip(l, 28)).join(', ')}`;
+  }
+  return prompt;
+}
+
+/**
+ * The schema. Three words and nothing else, enforced at sampling.
+ *
+ * Deliberately decision-only, matching `Judgement` in the Swift file: that
+ * struct has one field. Asking this arm for a rationale it would then be scored
+ * against would be a second difference between the arms, and the rationale is
+ * the one thing a supervisor is explicitly not trusted for.
+ */
+export const DECISION_WORDS = ['wait', 'retry', 'stop'];
+
+/**
+ * The fourth word, available on request.
+ *
+ * Not a new capability and it cannot become one: `abstain` means "behave as if
+ * there is no supervisor", which is the `null` every caller already handles on
+ * every failure path. It strictly *shrinks* what the model can cause to happen,
+ * which is why a component whose answer space is its safety property can afford
+ * to grow one.
+ */
+export const ABSTAIN = 'abstain';
+
+export const schemaFor = ({ mayAbstain = false } = {}) => ({
+  type: 'object',
+  properties: {
+    decision: { type: 'string', enum: mayAbstain ? [...DECISION_WORDS, ABSTAIN] : DECISION_WORDS },
+  },
+  required: ['decision'],
+});
+
+export const SCHEMA = schemaFor();
+
+/** Parse `ollama`, `ollama:qwen3:14b`, `ollama:qwen3:8b@http://host:port`. */
+export function parseTarget(raw) {
+  const rest = String(raw ?? '').replace(/^ollama:?/, '');
+  const [model, host] = rest.split('@');
+  return { model: model || DEFAULT_MODEL, host: host || DEFAULT_HOST };
+}
+
+async function post(host, route, body, timeoutMs) {
+  const control = new AbortController();
+  const timer = setTimeout(() => control.abort(), timeoutMs);
+  try {
+    const res = await fetch(`${host}${route}`, {
+      method: 'POST',
+      headers: { 'content-type': 'application/json' },
+      body: JSON.stringify(body),
+      signal: control.signal,
+    });
+    if (!res.ok) return { kind: 'http', error: `${res.status} ${await res.text().catch(() => '')}`.slice(0, 200) };
+    return { json: await res.json() };
+  } catch (err) {
+    return { kind: err.name === 'AbortError' ? 'timeout' : 'unreachable', error: String(err.message).slice(0, 200) };
+  } finally {
+    clearTimeout(timer);
+  }
+}
+
+/**
+ * One judgement. Same shape as the Apple helper's answer, so `judge` in
+ * `supervisor.js` cannot tell which arm it asked — including the failure
+ * shapes, because "the supervisor did not answer" covering a timeout, a
+ * refusal and a model that was never installed is a bug this project has
+ * already paid for once.
+ */
+export async function ask(target, situation, timeoutMs = 2500) {
+  const { model, host } = target;
+  const mayAbstain = Boolean(situation?.mayAbstain);
+  const started = Date.now();
+  const out = await post(host, '/api/chat', {
+    model,
+    messages: [
+      { role: 'system', content: readBrief(SWIFT, { mayAbstain }) },
+      { role: 'user', content: promptFor(situation) },
+    ],
+    stream: false,
+    format: schemaFor({ mayAbstain }),
+    // Qwen3 reasons out loud by default. Turned off for two reasons and both
+    // are about fairness rather than speed: the Apple arm does not deliberate
+    // either, and a supervisor that takes twenty seconds to answer has already
+    // lost the argument it is here to have — it sits in front of a 1.2s budget.
+    think: false,
+    options: { temperature: 0, num_predict: 32 },
+  }, timeoutMs);
+  if (!out.json) return { kind: out.kind, error: out.error };
+  const ms = Date.now() - started;
+  try {
+    return { ...JSON.parse(out.json.message?.content ?? ''), ms };
+  } catch {
+    return { kind: 'unparseable', error: clip(out.json.message?.content, 200), ms };
+  }
+}
+
+/**
+ * Load the weights before timing anything.
+ *
+ * The Apple arm calls `prewarm()` for exactly this reason, and it was worth
+ * ~280ms there. Here it is worth minutes: an 8B at 4-bit is 5.2 GB off disk,
+ * and the first `doctor` probe aborted at 20s and reported a working model as
+ * one that "did not answer" — a cold load reported as a fault. Ollama's
+ * documented preload is a chat with no messages.
+ */
+export async function preload(target, timeoutMs = 120_000) {
+  const { model, host } = target;
+  const started = Date.now();
+  const out = await post(host, '/api/chat', { model, messages: [], keep_alive: '10m' }, timeoutMs);
+  return out.json ? { ok: true, ms: Date.now() - started } : { ok: false, ...out };
+}
+
+/** For `doctor`: is this arm actually there, and does it answer? */
+export async function status(target, timeoutMs = 20_000) {
+  const { model, host } = target;
+  const tags = await post(host, '/api/show', { model }, 4000);
+  if (!tags.json) {
+    return {
+      ok: false,
+      reason: tags.kind === 'unreachable'
+        ? `no Ollama server at ${host} (start it, or pass a host with ollama:<model>@<url>)`
+        : `Ollama has no model "${model}" (${tags.error})`,
+    };
+  }
+  return { ok: true, model, host, timeoutMs };
+}

package/src/platform/ios.js

@@ -178,17 +178,42 @@ function isBootedSync(udid) {
   return false;
 }
 
+/**
+ * What went wrong with a `simctl io screenshot`, in one sentence.
+ *
+ * Extracted so it can be *tested* rather than reasoned about, for the same
+ * reason `pickDevice` and `decisionOf` were: this is the line where a wrong
+ * answer was expensive, and it was wrong for a day.
+ */
+export function screenshotFailure(err) {
+  const killed = err.killed || err.signal === 'SIGTERM';
+  // `simctl` opens with `Note: No display specified …` on every run, success or
+  // failure. When the display surface is dead the command does not fail, it
+  // *hangs* — so at kill time that Note is the only thing on stderr, and the
+  // tool reported a benign informational line as the reason a capture failed.
+  // That is how this wedge stayed nameless through five CI failures.
+  const lines = String(err.stderr || '').trim().split('\n').map((l) => l.trim()).filter(Boolean);
+  const real = lines.filter((l) => !/^Note:/i.test(l)).pop();
+  if (killed && !real) {
+    // Run to completion the device names it exactly:
+    //   NSPOSIXErrorDomain code 60 — Timeout waiting for screen surfaces
+    // which is CoreSimulator saying the surface is gone, and the closest thing
+    // to a positive test for the wedge that exists.
+    return 'simctl screenshot did not return within 10s. The display surface is not answering'
+      + ' — run to completion it reports "Timeout waiting for screen surfaces" (NSPOSIXErrorDomain 60).'
+      + ' This is the device, not the capture loop: `simframe revive` restarts it.';
+  }
+  const detail = real ?? lines.pop();
+  return detail ? `simctl screenshot failed: ${detail}` : `simctl screenshot failed: ${err.message}`;
+}
+
 async function screenshot(udid, outFile, { mask = 'ignored' } = {}) {
   try {
     await run('xcrun', ['simctl', 'io', udid, 'screenshot', '--type=png', `--mask=${mask}`, outFile], {
       timeout: 10_000,
     });
   } catch (err) {
-    // Same reason as launchApp: execFile's message is "Command failed: <the
-    // whole command>" and simctl's actual complaint is in stderr. A CI failure
-    // here reported the command and nothing about why it did not work.
-    const detail = (err.stderr || '').trim().split('\n').filter(Boolean).pop();
-    throw new Error(detail ? `simctl screenshot failed: ${detail}` : `simctl screenshot failed: ${err.message}`);
+    throw new Error(screenshotFailure(err));
   }
 }
 

package/src/refs.js

@@ -149,7 +149,18 @@ export function resolveRef(udid, n, { structuralHash, layoutHash, screenKnown, s
   // numbers. Refusing costs a re-read; guessing taps whatever is at those
   // coordinates now.
   if (screenKnown === false) {
-    throw new Error(`#${n} cannot be trusted here — simframe does not recognise this screen. Read it again (sim_ui) to renumber.`);
+    // Flagged, like every other refusal in this function, and it was the one
+    // that was not.
+    //
+    // We tell callers to read `staleRef` rather than the sentence — the CI check
+    // for this very guard carries a comment saying it matched on prose twice and
+    // went red twice, so it reads the contract now. Then it went red a third
+    // time, on a refusal that was correct, well worded, and carried no field at
+    // all: `#1 cannot be trusted here — simframe does not recognise this
+    // screen`, reported by the harness as `no reason`. A refusal a human can
+    // read and a program cannot is the same defect as a failure that reads like
+    // a success, one level down.
+    throw staleError('simframe does not recognise this screen', 'unknown-screen');
   }
   // The pixel check stays, but only as a backstop, and only where it means
   // something. A dark or near-uniform screen produces a layout hash of almost

package/src/regions.js

@@ -470,6 +470,60 @@ export function offViewport(t, screen) {
   return false;
 }
 
+/**
+ * The smallest sliver of an element that is worth offering as a tap target.
+ *
+ * Apple's own minimum touch target is 44pt; this is deliberately smaller,
+ * because the question here is not "is this comfortable to tap" but "is this a
+ * real control a person can see and reach". A filter chip showing 29pt of
+ * itself at the edge of a horizontal strip is both. Below this, what is on
+ * screen is bleed rather than a control.
+ */
+export const MIN_VISIBLE_PT = 24;
+
+/**
+ * How much of an element is actually on screen, and where to tap what is.
+ *
+ * `offViewport` answers a yes/no question about the element's *centre*, which
+ * is right for "should I scroll to reach this" and wrong for "is this here at
+ * all". A chip at the end of a horizontal strip showed **29pt of itself** on a
+ * 402pt screen — real, visible, tappable — while its centre sat at x=416, so it
+ * was dropped from the map entirely. What the caller got in its place was OCR's
+ * reading of the visible sliver: a `text` element labelled **"Flc"** at x=392,
+ * which passes every filter because its own box is inside the viewport.
+ *
+ * So the map did not merely omit a control. It offered a different, meaningless
+ * name for it, at a coordinate that looks perfectly ordinary. That is the shape
+ * of item 121 — a caller who cannot trust what the map says about the edge of
+ * the screen — and the item's own words are "say so or clamp". This does both.
+ */
+export function clipping(t, screen) {
+  const f = t?.frame;
+  const w = screen?.width;
+  const h = screen?.height;
+  if (!f || !Number.isFinite(w) || !Number.isFinite(h)) return null;
+  const left = Math.max(0, f.x);
+  const right = Math.min(w, f.x + f.width);
+  const top = Math.max(0, f.y);
+  const bottom = Math.min(h, f.y + f.height);
+  const visibleWidth = right - left;
+  const visibleHeight = bottom - top;
+  if (visibleWidth <= 0 || visibleHeight <= 0) {
+    return { visibleWidth: 0, visibleHeight: 0, clipped: true, usable: false, point: null };
+  }
+  const clipped = f.x < 0 || f.y < 0 || f.x + f.width > w || f.y + f.height > h;
+  return {
+    visibleWidth,
+    visibleHeight,
+    clipped,
+    usable: visibleWidth >= MIN_VISIBLE_PT && visibleHeight >= MIN_VISIBLE_PT,
+    // The centre of what can be seen, not the centre of the element. Tapping
+    // the latter would aim off the screen, which is the clamp the item asked
+    // for and the reason a caller could not simply be handed the real centre.
+    point: { x: Math.round((left + right) / 2), y: Math.round((top + bottom) / 2) },
+  };
+}
+
 /** Annotate a target list with region and nav slot. Mutates and returns it. */
 export function annotate(targets, screen) {
   const band = bands(targets, screen);

package/src/screenmap.js

@@ -151,6 +151,82 @@ const inside = (point, frame) =>
   point.y >= frame.y &&
   point.y <= frame.y + frame.height;
 
+const frameArea = (f) => (f ? Math.max(1, f.width) * Math.max(1, f.height) : Infinity);
+
+/**
+ * What the map says is at a point — everything containing it, smallest first.
+ *
+ * Item 120. Six consecutive `swipe [201,750] -> [201,250]` reported
+ * `no visible change` while a support banner sat at y≈753 swallowing every
+ * gesture. The banner was *in the element list the same call printed*; nothing
+ * connected "your swipe started at y=750" to "there is an element at y=753".
+ * The geometry was already in hand, so this is arithmetic over data we hold,
+ * not a new perception pass.
+ *
+ * Smallest first is a deliberately weaker claim than z-order. We do not know
+ * what is on top — the accessibility tree's order is not a paint order and OCR
+ * has none at all — and the honest statement is "these are the elements that
+ * cover that point", innermost first because the innermost is the one a
+ * gesture most often goes to. Saying "overlay" would be a guess wearing the
+ * clothes of a measurement.
+ */
+export function hitTest(entry, point) {
+  if (!entry?.targets || !Number.isFinite(point?.x) || !Number.isFinite(point?.y)) return [];
+  return entry.targets
+    .filter((t) => t.frame && inside(point, t.frame))
+    .sort((a, b) => frameArea(a.frame) - frameArea(b.frame));
+}
+
+const describeTarget = (t) => {
+  const name = t.label || t.text || (t.type ? `(unlabelled ${t.type})` : '(unlabelled)');
+  return t.region ? `"${name}" (${t.region})` : `"${name}"`;
+};
+
+/**
+ * One line saying what a coordinate resolved to, for a gesture that did
+ * nothing visible.
+ *
+ * Returns `null` when there is no map for the screen, because "we did not
+ * look" and "we looked and found nothing" are different answers and only one
+ * of them is worth printing.
+ */
+export function describePoint(entry, point, { what = 'the point' } = {}) {
+  if (!entry?.targets?.length) return null;
+  const at = `${Math.round(point.x)},${Math.round(point.y)}`;
+  const hits = hitTest(entry, point);
+  if (!hits.length) {
+    return `${what} ${at} is not inside any element on the map`
+      + ' — empty space, or a view with no label (nothing can be said about what caught it)';
+  }
+  // The contention clause only where there is contention. On one hit the
+  // element's name *is* the diagnosis and anything after it is noise — and a
+  // note that pads every case is how a real one stops being read.
+  const others = hits.length > 1
+    ? `, the smallest of ${hits.length} elements covering it — a gesture goes to whatever is on top there`
+    : '';
+  return `${what} ${at} is inside ${describeTarget(hits[0])}${others}`;
+}
+
+/**
+ * The point a step is aimed at, when it is aimed at a coordinate at all.
+ *
+ * A swipe is captured by whatever sits under where the finger goes *down*, so
+ * the start point is the one worth diagnosing; the end point never decides who
+ * receives the gesture.
+ */
+export function aimedAt(step) {
+  if (!step) return null;
+  if (step.action === 'tapAt' && Number.isFinite(step.x) && Number.isFinite(step.y)) {
+    return { point: { x: step.x, y: step.y }, what: 'the tap point' };
+  }
+  if (step.action === 'swipe') {
+    const x = step.from?.[0] ?? step.from?.x;
+    const y = step.from?.[1] ?? step.from?.y;
+    if (Number.isFinite(x) && Number.isFinite(y)) return { point: { x, y }, what: 'the swipe start point' };
+  }
+  return null;
+}
+
 /**
  * Build the map for the screen currently showing. Accessibility elements are the
  * real hit targets, so they win where they exist; OCR fills in everything the
@@ -220,15 +296,64 @@ export async function build(udid, {
     try {
       // The tree is already in hand when the daemon answered; describeAll would
       // only ask for it a second time.
-      const nodes = daemonScreen?.sources?.includes('ax')
-        ? daemonScreen.elements.filter((e) => e.source?.includes('ax')).map(input.elementToNode)
-        : await input.describeAll(udid);
+      //
+      // And when the daemon answered *without* a tree, asking again is asking
+      // the thing that just failed. That path cost a red CI run and a wrong
+      // diagnosis: the daemon's ax read timed out, `describeAll` re-asked it
+      // (another 20s), got the same nothing, fell through to idb, and the map
+      // reported **"idb is not installed, so simframe can observe the screen but
+      // cannot touch it"**. A sentence about a tool this project deliberately
+      // does not use on CI, printed because a different tool had a bad read —
+      // and it points whoever reads it at installing idb, which would change
+      // nothing. The same read worked a minute later.
+      //
+      // The daemon has sent `axError` since it learned to time its own reads,
+      // and nothing here had ever read it. The OCR branch below reads
+      // `ocrError`, which is what makes the asymmetry visible: one sensor could
+      // say why it failed and the other borrowed a different tool's excuse.
+      let nodes;
+      if (daemonScreen?.sources?.includes('ax')) {
+        nodes = daemonScreen.elements.filter((e) => e.source?.includes('ax')).map(input.elementToNode);
+      } else if (daemonScreen) {
+        throw new Error(daemonScreen.axError ?? 'the daemon read the screen and the accessibility tree did not answer');
+      } else {
+        nodes = await input.describeAll(udid);
+      }
 
       sources.push('ax');
+      // A control the tree gives no name to is still a control — item 122.
+      //
+      // This line used to read `!n.label`, on the reasoning that a row nobody
+      // can name is a row nobody can tap. The whole of item 122 is that the
+      // opposite is true, and three field reports in a row said so
+      // independently: icon-only overflow menus on every card, a bottom-sheet
+      // drag handle, a back chevron. Real, tappable, on screen, and absent from
+      // the map — so every one of them needed a raw `@x,y` read off a
+      // screenshot, which is the exact round trip the text map exists to
+      // remove. Knowing that *a hit target is there* is most of the value even
+      // with no semantics attached to it.
+      //
+      // Measured on the testbed before it was written, because the premise
+      // could have been false: on the list screen, of 39 accessibility nodes
+      // exactly **one** is nameless — and it is the one control on that screen
+      // no caller could reach. This does not flood the map.
+      //
+      // It also settles a thing the reports could not: those controls were
+      // called "absent from the tree", and AXPTranslator had them all along.
+      // What is genuinely absent is the drag handle, a plain view holding a
+      // responder that UIKit is never told is accessible. No filter here can
+      // recover that one; see 122's second half.
       for (const n of nodes) {
-        if (!n.frame || !n.label || isContainer(n)) continue;
+        if (!n.frame || isContainer(n)) continue;
+        if (nameless(n) && !namelessHitTarget(n, nodes)) continue;
         targets.push({
-          label: n.label,
+          label: n.label ?? undefined,
+          // A `testID` arrives as AXIdentifier, and `tap` has matched on it
+          // since long before this — but only down the tree path, because the
+          // map never carried it. So the one name an unlabelled React Native
+          // control usually does have was invisible in the map and worked if
+          // you guessed it.
+          identifier: n.identifier ?? undefined,
           // What the control *contains*, whether it is on, and whether it has
           // focus. All three come off the accessibility tree, the daemon has
           // asked for all three since 0.6.0, and all three were dropped before
@@ -430,6 +555,36 @@ export function isInteractive(target) {
   return INTERACTIVE.test(target.type || '');
 }
 
+/** No name from any source: not a label, not an identifier. */
+export const nameless = (n) => !n?.label && !n?.identifier;
+
+/**
+ * Whether a nameless accessibility node is worth printing as a hit target.
+ *
+ * Two guards, because "has no name" is not the same as "is a control".
+ *
+ * The role has to be one the tree calls interactive. A nameless `Other` is a
+ * layout view and a real screen has hundreds of them; emitting those would bury
+ * the one node that matters under the scenery it sits in.
+ *
+ * And it must not enclose two or more other elements. That is the same test the
+ * fingerprint uses for scenery, reused deliberately rather than invented here:
+ * a table cell holding its own labels is not the target, its labels are.
+ */
+export function namelessHitTarget(node, nodes) {
+  const f = node?.frame;
+  if (!f || !(f.width > 0) || !(f.height > 0)) return false;
+  if (!INTERACTIVE.test(node.type || '')) return false;
+  const encloses = nodes.filter((o) => {
+    const g = o.frame;
+    if (!g || o === node) return false;
+    const cx = g.x + (g.width ?? 0) / 2;
+    const cy = g.y + (g.height ?? 0) / 2;
+    return cx > f.x && cx < f.x + f.width && cy > f.y && cy < f.y + f.height;
+  }).length;
+  return encloses < 2;
+}
+
 /**
  * Rank candidates for a label. Exact beats substring, and a real control beats
  * a caption that happens to read the same — a screen title and a tab are often
@@ -438,7 +593,7 @@ export function isInteractive(target) {
 export function rank(entry, query) {
   if (!entry) return [];
   const q = norm(query);
-  const names = (t) => [t.label, ...(t.aliases || [])].map(norm);
+  const names = (t) => [t.label, t.identifier, ...(t.aliases || [])].map(norm);
   const exact = entry.targets.filter((t) => names(t).includes(q));
   const pool = exact.length
     ? exact

package/src/store.js

@@ -34,10 +34,63 @@ export function paths(udid) {
     // wedged. It cannot ride in state.json: that is written when a frame is
     // recorded, and a stall is the absence of frames.
     captureHealth: path.join(dir, 'capture-health.json'),
+    lastInput: path.join(dir, 'last-input'),
   };
 }
 
 /** What the capture loop last said about its own health, or null if it has no complaint. */
+/**
+ * When input was last delivered to this device.
+ *
+ * Written by every input path and read by `liveness`, which needs it to catch
+ * the one wedge shape nothing else can see: a capture loop that is alive,
+ * incrementing its frame counter, and re-reading a **dead surface**. Field
+ * report, 0.12.2: `age=268ms` next to `stable=159753ms` while three screen
+ * transitions had just happened, and no warning fired because every signal we
+ * had was green. A screen that has not moved since before we last touched it,
+ * over and over, is a contradiction the daemon can notice locally.
+ */
+const INPUT_MEMORY = 8;
+
+export function noteInput(udid, at = Date.now()) {
+  try {
+    writeAtomic(paths(udid).lastInput, [...inputTimes(udid), at].slice(-INPUT_MEMORY).join(','));
+  } catch {
+    /* a timestamp nothing depends on for correctness must not fail an action */
+  }
+}
+
+/**
+ * The last few input timestamps, oldest first.
+ *
+ * A list rather than a single timestamp, and that is the whole correction.
+ * The first version kept only the latest, so the only question it could ask was
+ * "has the screen been still for a long time?" — which needed a duration
+ * threshold, and the threshold is what defeated it. A field report caught a
+ * three-hour-stale frame on a screen that had been still for **8.2 seconds**,
+ * under a 20-second gate, so the check could not fire on the case it was
+ * written for.
+ *
+ * What actually says "dead surface" is not duration. It is **several gestures
+ * delivered with no pixel moving at all** — one tap that changes nothing is
+ * ordinary, and three in a row are not.
+ */
+export function inputTimes(udid) {
+  try {
+    return fs.readFileSync(paths(udid).lastInput, 'utf8')
+      .split(',')
+      .map(Number)
+      .filter(Number.isFinite);
+  } catch {
+    return [];
+  }
+}
+
+export function lastInputAt(udid) {
+  const times = inputTimes(udid);
+  return times.length ? times[times.length - 1] : null;
+}
+
 export function captureHealth(udid) {
   return readJson(paths(udid).captureHealth);
 }