simframe 0.9.0 → 0.11.0

package/src/regions.js

@@ -36,6 +36,8 @@ const STATUS_BAR_FRACTION = 0.065;
 
 /** Keyboards occupy the bottom of the screen and are unusually tall. */
 const KEYBOARD_MIN_FRACTION = 0.28;
+/** Most of a keyboard is keys. Below this it is a list that happens to be small. */
+const KEYBOARD_MIN_KEYISH = 0.6;
 
 /** Chrome is short. A 90pt list cell is not a tab item however low it sits. */
 const CHROME_MAX_HEIGHT_FRACTION = 0.075;
@@ -243,6 +245,50 @@ export function navSlot(frame, screen) {
  * common case and must stay cheap. This was the first band derived from the
  * elements rather than from a fraction, and it is the model the rest now follow.
  */
+/**
+ * Whether an element is shaped like a key rather than like content.
+ *
+ * The canonical version of this test, because two places need it and getting
+ * them out of step is what produced the bug below. A key is finger-sized and
+ * says almost nothing: a single character, a short named key, or nothing at
+ * all. A row of content is wider, or carries words.
+ */
+export const KEY_MAX_WIDTH = 120;
+
+const NAMED_KEY = /^(space|return|enter|shift|delete|backspace|done|globe|dictate|emoji|caps ?lock|number|numbers|symbols|letters|more|search|go|send|join|route|abc|123)$/i;
+
+export function looksLikeKey(t) {
+  if (/^key$/i.test(String(t?.type ?? ''))) return true;
+  const width = t?.frame?.width;
+  if (Number.isFinite(width) && width > KEY_MAX_WIDTH) return false;
+  const label = String(t?.label ?? '').trim();
+  if (!label) return true;
+  if (label.length <= 2) return true;
+  return NAMED_KEY.test(label);
+}
+
+/**
+ * Where the software keyboard starts, or null.
+ *
+ * Size and uniformity alone were not enough, and the failure was expensive. A
+ * read-only summary screen stacks a dozen short text rows of near-identical
+ * height in the bottom half — which satisfied every test here, so a keyboard
+ * was detected on a screen that had none.
+ *
+ * That mattered far beyond a mislabelled band, because `fingerprint.tokens`
+ * discards everything below `keyboardTop`. A phantom keyboard therefore
+ * deleted the screen's entire content from its own identity, leaving only
+ * chrome — so a wizard's form step and its read-only review screen, which
+ * share a nav title and a step indicator, **collapsed onto one hash**. From
+ * there: the graph offered one screen's remembered controls on the other (three
+ * absent controls, one of them beside a button that submits for real), and
+ * `locate` resolved against the wrong screen's stored element list, which is
+ * why `assert` insisted a string was absent while the map printed it four lines
+ * below. One phantom, three findings.
+ *
+ * So the test is now what a keyboard actually is: keys. A dozen small uniform
+ * boxes are a keyboard only if most of them are key-shaped.
+ */
 export function detectKeyboardTop(elements, screen) {
   if (!screen?.height || elements.length < 12) return null;
   const threshold = screen.height * (1 - KEYBOARD_MIN_FRACTION);
@@ -253,7 +299,70 @@ export function detectKeyboardTop(elements, screen) {
   // Keys are small and uniform; a list of cells down there is not.
   const uniform = heights.filter((h) => Math.abs(h - median) <= Math.max(3, median * 0.4)).length;
   if (uniform / low.length < 0.7 || median > screen.height * 0.07) return null;
-  return Math.min(...low.map((e) => e.frame.y));
+  // And they are keys. Uniformity says "a grid of something"; this says of what.
+  const keyish = low.filter(looksLikeKey).length;
+  if (keyish / low.length < KEYBOARD_MIN_KEYISH) return null;
+  return extendKeyboardUp(elements, Math.min(...low.map((e) => e.frame.y)), median);
+}
+
+/**
+ * Walk the boundary up through rows that are still keys.
+ *
+ * `KEYBOARD_MIN_FRACTION` is a **detection window**, not the keyboard's height,
+ * and using its edge as the boundary cut the keyboard's own top row off.
+ * Measured on a recorded iPhone 17 Pro screen with the software keyboard up: the
+ * window starts at y=629, the `q`–`p` row's frame top is **590**, so that entire
+ * row was excluded and the boundary landed on the `a` row at 644 — ten keys
+ * reported as page content, in the same map that said `keyboard up`.
+ *
+ * Widening the window instead would be the wrong fix: 0.28 of the screen is
+ * deliberately conservative so a list of short rows at the bottom of a page
+ * cannot be mistaken for a keyboard, and a real keyboard is nearer 0.38. So the
+ * window still *decides*, and this extends the boundary only while the rows
+ * above keep being key-shaped — which page content is not.
+ *
+ * The concrete cost of not having this: a sweep gesture aimed 8pt above the
+ * boundary still landed on the top row of keys and scrolled nothing.
+ */
+function extendKeyboardUp(elements, top, median) {
+  let boundary = top;
+  // Four rows is a full keyboard's worth; the loop stops on its own long before
+  // that on anything that is not one.
+  for (let i = 0; i < 4; i += 1) {
+    const row = (elements ?? []).filter((e) => e.frame
+      && looksLikeKey(e)
+      && Math.abs(heightOf(e.frame) - median) <= Math.max(3, median * 0.4)
+      // Sitting directly on the current boundary, within one row's height.
+      && e.frame.y + heightOf(e.frame) <= boundary + 4
+      && e.frame.y + heightOf(e.frame) >= boundary - median * 1.6);
+    if (row.length < 5) break;
+    const next = Math.min(...row.map((e) => e.frame.y));
+    if (!(next < boundary)) break;
+    boundary = next;
+  }
+  return boundary;
+}
+
+/**
+ * Is this element outside the viewport?
+ *
+ * **Both axes.** Every filter in this project checked `y` and ignored `x`,
+ * which is fine until a horizontal row: a filter chip reported at **x=422 on a
+ * 402pt-wide screen** counted as visible, and `scroll_to` then said *"'Assigned
+ * to Me' is in view at 422,277 already"* — confidently wrong about the one thing
+ * it exists to answer. Off-screen chips came back at **x=-247** the same way.
+ *
+ * Reported as the most expensive finding of an agent's session, and the cost was
+ * not the wrong answer itself: it was that the wrong answer was *confident*, so
+ * the recovery was hand-tuned swipes and two overshoots.
+ */
+export function offViewport(t, screen) {
+  if (!t) return false;
+  const w = screen?.width;
+  const h = screen?.height;
+  if (Number.isFinite(h) && (t.y < 0 || t.y > h)) return true;
+  if (Number.isFinite(w) && (t.x < 0 || t.x > w)) return true;
+  return false;
 }
 
 /** Annotate a target list with region and nav slot. Mutates and returns it. */

package/src/screenmap.js

@@ -17,7 +17,20 @@ import * as regions from './regions.js';
 import { informative } from './refs.js';
 import * as store from './store.js';
 
-const MAP_VERSION = 8; // an OCR reading contained in a labelled ax element merges into it
+const MAP_VERSION = 9; // ax targets carry value, selected and focused
+
+/**
+ * A stored map also holds a `structuralHash`, which the *fingerprint* rules
+ * produced. So a token-rule change invalidates every stored map, and relying on
+ * someone to remember to bump `MAP_VERSION` too is exactly how the phantom
+ * keyboard survived a fix: two copies of one dependency, one of them updated.
+ *
+ * Stating the dependency instead of remembering it. A map is only valid for the
+ * token rules that hashed it.
+ */
+const usable = (e) => Boolean(e)
+  && e.version === MAP_VERSION
+  && e.fingerprintVersion === fingerprint.TOKEN_RULES_VERSION;
 
 function mapDir(udid) {
   return path.join(store.deviceDir(udid), 'screens');
@@ -38,10 +51,38 @@ function mapDir(udid) {
  */
 export const DEFAULT_TOLERANCE = 20;
 
+/** Comparison that ignores what OCR adds — a caret, a stray glyph, spacing. */
+const alnum = (v) => String(v ?? '').toLowerCase().replace(/[^\p{L}\p{N}]+/gu, '');
+
+/**
+ * May an OCR word be recorded as an alias of the element enclosing it?
+ *
+ * Only when they are plausibly the same thing. A row labelled "Kate Bell"
+ * containing OCR's "Kate Bell" is one element two sensors saw; a sheet's
+ * "Area (Optional)" enclosing a dimmed page's "Exterior Building" is two things
+ * at one coordinate on different z-layers, and aliasing them reads as though
+ * the field contains that value.
+ *
+ * An element with **no label** takes the text outright — that is how an
+ * icon-only control gets a name, and it cannot contradict a label it does not
+ * have.
+ *
+ * A function rather than three lines inline, because the inline version read
+ * `covering.label` before anything checked that `covering` existed, and the
+ * TypeError that followed was swallowed by the OCR try/catch — silently
+ * disabling the sensor. A pure function can be tested with the value that broke
+ * it, and the source-shape assertion this replaces could not.
+ */
+export function aliasRelates(coveringLabel, text) {
+  const own = alnum(coveringLabel);
+  const seen = alnum(text);
+  return !own || !seen || own.includes(seen) || seen.includes(own);
+}
+
 export function recall(udid, hash) {
   if (!hash) return null;
   const entry = store.readJson(path.join(mapDir(udid), `${hash}.json`));
-  return entry && entry.version === MAP_VERSION ? entry : null;
+  return usable(entry) ? entry : null;
 }
 
 function loadAll(udid) {
@@ -53,7 +94,7 @@ function loadAll(udid) {
   }
   return files
     .map((f) => store.readJson(path.join(mapDir(udid), f)))
-    .filter((e) => e && e.version === MAP_VERSION);
+    .filter(usable);
 }
 
 /**
@@ -137,6 +178,9 @@ export async function build(udid, {
   // empty `sources` rethrew — so making a layer work turned a loud failure into
   // a quiet one.
   const degraded = [];
+  // Pairs where an ax element and an OCR word share a coordinate and disagree
+  // about what is there — the signature of one layer covering another.
+  const occluded = [];
   // One round trip for both, because the daemon runs the tree read and the
   // recognition pass concurrently against the same instant of the screen. Asked
   // separately they would queue: the control socket serves one request at a
@@ -185,11 +229,30 @@ export async function build(udid, {
         if (!n.frame || !n.label || isContainer(n)) continue;
         targets.push({
           label: n.label,
+          // What the control *contains*, whether it is on, and whether it has
+          // focus. All three come off the accessibility tree, the daemon has
+          // asked for all three since 0.6.0, and all three were dropped before
+          // this — `value` here and the other two one layer up in
+          // `normalizeNode` — so nothing above this line ever saw them.
+          //
+          // The cost was not theoretical. A real session could not verify the
+          // contents of a text field at all: they reached a row only as the OCR
+          // alias, which made them as old as the map and unauthoritative. An
+          // assert failed against a field that did contain the string, the
+          // operator retyped, and the field ended up with a doubled value and a
+          // validation error.
+          //
+          // `focused` is worth naming separately: it is a direct answer to "did
+          // this field take focus", which the focus wait in actions.js infers
+          // from elapsed time because it had nothing better to use.
+          value: n.value ?? undefined,
           x: input.centerOf(n).x,
           y: input.centerOf(n).y,
           frame: n.frame,
           type: n.type,
           enabled: n.enabled,
+          selected: n.selected ?? undefined,
+          focused: n.focused ?? undefined,
           source: 'ax',
         });
       }
@@ -262,15 +325,49 @@ export async function build(udid, {
           ?? targets
             .filter((t) => eligible(t) && matching.sameElementSeenTwice(t, { ...w, frame: box, label: w.text }))
             .sort((a, b) => area(a.frame) - area(b.frame))[0];
+        // An alias must be the *same thing*, read twice.
+        //
+        // The geometric branch above pairs an OCR word with whichever ax
+        // element encloses it, and across z-layers that is simply wrong.
+        // Reported on a modal-heavy screen, with a Select Area sheet open over a
+        // dimmed page: `#15 text 167,316 Area (Optional) ~ Exterior Building`,
+        // which reads as though the field "Area (Optional)" contains "Exterior
+        // Building". They are two unrelated things at one coordinate on
+        // different layers, and the reporter had to fall back to a screenshot to
+        // count five radio options — precisely the case the text map exists to
+        // remove.
+        //
+        // So a labelled ax element only takes an alias that relates to its own
+        // label. The justification for aliasing was always "a row labelled 'Kate
+        // Bell' containing OCR's 'Kate Bell' is one element two sensors saw" —
+        // that still holds. An *unlabelled* element still takes the text
+        // outright, because that is how an icon-only control gets a name at all,
+        // and it cannot contradict a label it does not have.
+        // NOTE the nesting, which is the whole point of this shape: `covering`
+        // is undefined whenever no ax element encloses this word, which is most
+        // words on most screens. Reading `covering.label` before checking that
+        // threw a TypeError inside the OCR try/catch — so the entire OCR pass
+        // was swallowed and reported as `degraded: text recognition`, silently
+        // disabling the sensor on every screen with one uncovered word. Neither
+        // the unit tests nor the perception harness caught it: the harness feeds
+        // *already fused* element lists, so it never runs this loop. The
+        // integration job caught it, which is what it is for.
         if (covering) {
-          covering.aliases = [...(covering.aliases || []), w.text];
-          // Keep the ax role and frame — it is the hit target — and record that
-          // both sensors saw it. Anything asking "is this the tree's element?"
-          // must ask matching.isAxTarget, not `=== 'ax'`.
-          if (!String(covering.source ?? '').includes('ocr')) {
-            covering.source = `${covering.source ?? 'ax'}|ocr`;
+          if (aliasRelates(covering.label, w.text)) {
+            covering.aliases = [...(covering.aliases || []), w.text];
+            // Keep the ax role and frame — it is the hit target — and record
+            // that both sensors saw it. Anything asking "is this the tree's
+            // element?" must ask matching.isAxTarget, not `=== 'ax'`.
+            if (!String(covering.source ?? '').includes('ocr')) {
+              covering.source = `${covering.source ?? 'ax'}|ocr`;
+            }
+            continue;
           }
-          continue;
+          // Rejected as an alias, so it falls through and becomes an element of
+          // its own — which is what it is. Marked, because "these two things
+          // overlap and disagree" is exactly the shape of an occluding layer,
+          // and a caller counting radio options needs to know it is there.
+          occluded.push({ over: covering.label, under: w.text });
         }
         targets.push({
           label: w.text,
@@ -304,6 +401,7 @@ export async function build(udid, {
       : { hash: null, tokens: [], keyboard: false };
     const entry = {
       version: MAP_VERSION,
+      fingerprintVersion: fingerprint.TOKEN_RULES_VERSION,
       hash,
       layoutHash,
       structuralHash: structure.hash,
@@ -317,6 +415,7 @@ export async function build(udid, {
     // map has to carry it, because this is what gets written into memory.
     if (daemonScreen?.axTruncated) degraded.push(`accessibility tree cut short: ${daemonScreen.axTruncated}`);
     if (degraded.length) entry.degraded = degraded;
+    if (occluded.length) entry.occluded = occluded;
     // Only a map of a settled screen is worth keeping; remembering a transition
     // fills the store with layouts that will never be seen again.
     return persist ? remember(udid, entry) : entry;

package/src/supervisor.js

@@ -0,0 +1,117 @@
+/**
+ * The local supervisor: three words, behind the hands, in front of Claude.
+ *
+ * The owner's design. Claude plans; the deterministic executor in `actions.js`
+ * runs the plan and verifies each step; and when a step fails, *this* decides
+ * whether the plan can proceed — before anything reaches Claude. It sits behind
+ * the hands and in front of the reasoner, and it is the first responder rather
+ * than the decision-maker.
+ *
+ * It may say **wait**, **retry** or **stop**. Nothing else. It cannot invent a
+ * step, skip one, substitute a target, or continue past an unexpected screen —
+ * not because a confidence threshold forbids it but because those are not words
+ * it can say. The answer space *is* the safety property. `seek` was given
+ * latitude over what to open and pressed "YES, THIS FIXED MY PROBLEM" in a live
+ * app; a component that can only choose among three words cannot do that,
+ * whatever it believes.
+ *
+ * **The plan briefs it**, which is the owner's second insight and the thing that
+ * made it work. Asked cold, it called a list that was plainly still arriving a
+ * dead end — because it does not know the app and Claude, by the time it writes
+ * the plan, does. So a batch may carry `supervise` and a step may carry
+ * `expect`, and both reach the supervisor as context. That costs a string and no
+ * round trips.
+ *
+ * One thing it is deliberately not trusted for: its own **prose**. In testing it
+ * returned a correct decision with a reason citing a rule that did not apply.
+ * The decision is used; the reason is logged and never shown as an explanation.
+ * Presenting a confabulated rationale as fact is the mistake `seek`'s
+ * documentation already made once.
+ *
+ * Off unless asked: `SIMFRAME_SUPERVISOR=apple`, or `supervisor` per call.
+ */
+import path from 'node:path';
+import { fileURLToPath } from 'node:url';
+import * as store from './store.js';
+import { compiler, lineServer } from './localhelper.js';
+
+const SOURCE = path.join(path.dirname(fileURLToPath(import.meta.url)), '..', 'native', 'supervise.swift');
+const BIN = path.join(store.ROOT, 'bin', 'supervise');
+
+const helper = lineServer({
+  ensureBinary: compiler({ source: SOURCE, binary: BIN, what: 'local supervisor' }),
+  what: 'local supervisor',
+});
+
+export const DECISIONS = new Set(['wait', 'retry', 'stop']);
+
+/** Which backend the caller asked for, per call first and environment second. */
+export function requested(options) {
+  const raw = String(options?.supervisor ?? process.env.SIMFRAME_SUPERVISOR ?? '').trim().toLowerCase();
+  if (!raw || raw === 'none' || raw === 'off' || raw === '0' || raw === 'false') return null;
+  return raw;
+}
+
+/**
+ * Can this plan proceed past the step that just failed?
+ *
+ * @returns {Promise<{decision: 'wait'|'retry'|'stop', reason: string, ms: number}|null>}
+ *   null on every failure mode — not asked, unavailable, timed out, unparseable,
+ *   or an answer outside the three words. A null means the executor behaves
+ *   exactly as it does without a supervisor, which is the only safe default.
+ */
+export async function judge({
+  goal, step, expected, failure, screen, stillMs, note, options, timeoutMs = 2500,
+} = {}) {
+  if (!requested(options)) return null;
+  if (!step || !failure) return null;
+  const answer = await helper.ask({
+    goal: goal ? String(goal).slice(0, 200) : null,
+    step: String(step).slice(0, 200),
+    expected: expected ? String(expected).slice(0, 300) : null,
+    failure: String(failure).slice(0, 300),
+    screen: (screen ?? []).filter(Boolean).map((s) => String(s).slice(0, 40)).slice(0, 25),
+    stillMs: Number.isFinite(stillMs) ? Math.round(stillMs) : null,
+    note: note ? String(note).slice(0, 200) : null,
+  }, timeoutMs);
+  const decision = String(answer?.decision ?? '').toLowerCase();
+  // An answer outside the vocabulary is not a decision. Refusing it here is
+  // what makes the three-word constraint real rather than merely documented.
+  if (!DECISIONS.has(decision)) return null;
+  return { decision, reason: String(answer.reason ?? '').slice(0, 120), ms: answer.ms ?? null };
+}
+
+/** For `doctor`: what the supervisor layer is, in one line. */
+export async function status(options) {
+  const want = requested(options);
+  if (!want) return { supervisor: 'none', detail: 'not requested (SIMFRAME_SUPERVISOR is unset)' };
+  if (want !== 'apple') return { supervisor: 'none', detail: `no such supervisor backend: "${want}"` };
+  const live = await helper.status();
+  if (!live.ok) return { supervisor: 'none', detail: live.reason };
+  // Prove a round trip, not a presence.
+  //
+  // Reporting "available" from a fresh process was true and useless: the model
+  // had stopped answering inside the long-lived MCP server, `doctor` opened its
+  // own process, got a healthy one, and said so — for twenty calls and six
+  // failures during which nothing was being judged. The reporter's fix, and it
+  // is the right one: make it answer something.
+  const probe = await helper.ask({
+    step: 'tap "Probe"',
+    failure: '"Probe" is not on this screen. Visible: Probe',
+    screen: ['Probe'],
+    stillMs: 5000,
+  }, 6000);
+  const decision = String(probe?.decision ?? '').toLowerCase();
+  if (!DECISIONS.has(decision)) {
+    return {
+      supervisor: 'none',
+      detail: 'the model loaded but did not answer a probe — it is present and not working',
+    };
+  }
+  return {
+    supervisor: 'apple',
+    detail: `Apple Foundation Models, on-device; answered a probe in ${probe.ms ?? '?'}ms; may only answer wait/retry/stop`,
+  };
+}
+
+export const close = helper.close;