simframe 0.10.0 → 0.11.0

package/src/screenmap.js

@@ -19,6 +19,19 @@ import * as store from './store.js';
 
 const MAP_VERSION = 9; // ax targets carry value, selected and focused
 
+/**
+ * A stored map also holds a `structuralHash`, which the *fingerprint* rules
+ * produced. So a token-rule change invalidates every stored map, and relying on
+ * someone to remember to bump `MAP_VERSION` too is exactly how the phantom
+ * keyboard survived a fix: two copies of one dependency, one of them updated.
+ *
+ * Stating the dependency instead of remembering it. A map is only valid for the
+ * token rules that hashed it.
+ */
+const usable = (e) => Boolean(e)
+  && e.version === MAP_VERSION
+  && e.fingerprintVersion === fingerprint.TOKEN_RULES_VERSION;
+
 function mapDir(udid) {
   return path.join(store.deviceDir(udid), 'screens');
 }
@@ -38,10 +51,38 @@ function mapDir(udid) {
  */
 export const DEFAULT_TOLERANCE = 20;
 
+/** Comparison that ignores what OCR adds — a caret, a stray glyph, spacing. */
+const alnum = (v) => String(v ?? '').toLowerCase().replace(/[^\p{L}\p{N}]+/gu, '');
+
+/**
+ * May an OCR word be recorded as an alias of the element enclosing it?
+ *
+ * Only when they are plausibly the same thing. A row labelled "Kate Bell"
+ * containing OCR's "Kate Bell" is one element two sensors saw; a sheet's
+ * "Area (Optional)" enclosing a dimmed page's "Exterior Building" is two things
+ * at one coordinate on different z-layers, and aliasing them reads as though
+ * the field contains that value.
+ *
+ * An element with **no label** takes the text outright — that is how an
+ * icon-only control gets a name, and it cannot contradict a label it does not
+ * have.
+ *
+ * A function rather than three lines inline, because the inline version read
+ * `covering.label` before anything checked that `covering` existed, and the
+ * TypeError that followed was swallowed by the OCR try/catch — silently
+ * disabling the sensor. A pure function can be tested with the value that broke
+ * it, and the source-shape assertion this replaces could not.
+ */
+export function aliasRelates(coveringLabel, text) {
+  const own = alnum(coveringLabel);
+  const seen = alnum(text);
+  return !own || !seen || own.includes(seen) || seen.includes(own);
+}
+
 export function recall(udid, hash) {
   if (!hash) return null;
   const entry = store.readJson(path.join(mapDir(udid), `${hash}.json`));
-  return entry && entry.version === MAP_VERSION ? entry : null;
+  return usable(entry) ? entry : null;
 }
 
 function loadAll(udid) {
@@ -53,7 +94,7 @@ function loadAll(udid) {
   }
   return files
     .map((f) => store.readJson(path.join(mapDir(udid), f)))
-    .filter((e) => e && e.version === MAP_VERSION);
+    .filter(usable);
 }
 
 /**
@@ -137,6 +178,9 @@ export async function build(udid, {
   // empty `sources` rethrew — so making a layer work turned a loud failure into
   // a quiet one.
   const degraded = [];
+  // Pairs where an ax element and an OCR word share a coordinate and disagree
+  // about what is there — the signature of one layer covering another.
+  const occluded = [];
   // One round trip for both, because the daemon runs the tree read and the
   // recognition pass concurrently against the same instant of the screen. Asked
   // separately they would queue: the control socket serves one request at a
@@ -281,15 +325,49 @@ export async function build(udid, {
           ?? targets
             .filter((t) => eligible(t) && matching.sameElementSeenTwice(t, { ...w, frame: box, label: w.text }))
             .sort((a, b) => area(a.frame) - area(b.frame))[0];
+        // An alias must be the *same thing*, read twice.
+        //
+        // The geometric branch above pairs an OCR word with whichever ax
+        // element encloses it, and across z-layers that is simply wrong.
+        // Reported on a modal-heavy screen, with a Select Area sheet open over a
+        // dimmed page: `#15 text 167,316 Area (Optional) ~ Exterior Building`,
+        // which reads as though the field "Area (Optional)" contains "Exterior
+        // Building". They are two unrelated things at one coordinate on
+        // different layers, and the reporter had to fall back to a screenshot to
+        // count five radio options — precisely the case the text map exists to
+        // remove.
+        //
+        // So a labelled ax element only takes an alias that relates to its own
+        // label. The justification for aliasing was always "a row labelled 'Kate
+        // Bell' containing OCR's 'Kate Bell' is one element two sensors saw" —
+        // that still holds. An *unlabelled* element still takes the text
+        // outright, because that is how an icon-only control gets a name at all,
+        // and it cannot contradict a label it does not have.
+        // NOTE the nesting, which is the whole point of this shape: `covering`
+        // is undefined whenever no ax element encloses this word, which is most
+        // words on most screens. Reading `covering.label` before checking that
+        // threw a TypeError inside the OCR try/catch — so the entire OCR pass
+        // was swallowed and reported as `degraded: text recognition`, silently
+        // disabling the sensor on every screen with one uncovered word. Neither
+        // the unit tests nor the perception harness caught it: the harness feeds
+        // *already fused* element lists, so it never runs this loop. The
+        // integration job caught it, which is what it is for.
         if (covering) {
-          covering.aliases = [...(covering.aliases || []), w.text];
-          // Keep the ax role and frame — it is the hit target — and record that
-          // both sensors saw it. Anything asking "is this the tree's element?"
-          // must ask matching.isAxTarget, not `=== 'ax'`.
-          if (!String(covering.source ?? '').includes('ocr')) {
-            covering.source = `${covering.source ?? 'ax'}|ocr`;
+          if (aliasRelates(covering.label, w.text)) {
+            covering.aliases = [...(covering.aliases || []), w.text];
+            // Keep the ax role and frame — it is the hit target — and record
+            // that both sensors saw it. Anything asking "is this the tree's
+            // element?" must ask matching.isAxTarget, not `=== 'ax'`.
+            if (!String(covering.source ?? '').includes('ocr')) {
+              covering.source = `${covering.source ?? 'ax'}|ocr`;
+            }
+            continue;
           }
-          continue;
+          // Rejected as an alias, so it falls through and becomes an element of
+          // its own — which is what it is. Marked, because "these two things
+          // overlap and disagree" is exactly the shape of an occluding layer,
+          // and a caller counting radio options needs to know it is there.
+          occluded.push({ over: covering.label, under: w.text });
         }
         targets.push({
           label: w.text,
@@ -323,6 +401,7 @@ export async function build(udid, {
       : { hash: null, tokens: [], keyboard: false };
     const entry = {
       version: MAP_VERSION,
+      fingerprintVersion: fingerprint.TOKEN_RULES_VERSION,
       hash,
       layoutHash,
       structuralHash: structure.hash,
@@ -336,6 +415,7 @@ export async function build(udid, {
     // map has to carry it, because this is what gets written into memory.
     if (daemonScreen?.axTruncated) degraded.push(`accessibility tree cut short: ${daemonScreen.axTruncated}`);
     if (degraded.length) entry.degraded = degraded;
+    if (occluded.length) entry.occluded = occluded;
     // Only a map of a settled screen is worth keeping; remembering a transition
     // fills the store with layouts that will never be seen again.
     return persist ? remember(udid, entry) : entry;

package/src/supervisor.js

@@ -0,0 +1,117 @@
+/**
+ * The local supervisor: three words, behind the hands, in front of Claude.
+ *
+ * The owner's design. Claude plans; the deterministic executor in `actions.js`
+ * runs the plan and verifies each step; and when a step fails, *this* decides
+ * whether the plan can proceed — before anything reaches Claude. It sits behind
+ * the hands and in front of the reasoner, and it is the first responder rather
+ * than the decision-maker.
+ *
+ * It may say **wait**, **retry** or **stop**. Nothing else. It cannot invent a
+ * step, skip one, substitute a target, or continue past an unexpected screen —
+ * not because a confidence threshold forbids it but because those are not words
+ * it can say. The answer space *is* the safety property. `seek` was given
+ * latitude over what to open and pressed "YES, THIS FIXED MY PROBLEM" in a live
+ * app; a component that can only choose among three words cannot do that,
+ * whatever it believes.
+ *
+ * **The plan briefs it**, which is the owner's second insight and the thing that
+ * made it work. Asked cold, it called a list that was plainly still arriving a
+ * dead end — because it does not know the app and Claude, by the time it writes
+ * the plan, does. So a batch may carry `supervise` and a step may carry
+ * `expect`, and both reach the supervisor as context. That costs a string and no
+ * round trips.
+ *
+ * One thing it is deliberately not trusted for: its own **prose**. In testing it
+ * returned a correct decision with a reason citing a rule that did not apply.
+ * The decision is used; the reason is logged and never shown as an explanation.
+ * Presenting a confabulated rationale as fact is the mistake `seek`'s
+ * documentation already made once.
+ *
+ * Off unless asked: `SIMFRAME_SUPERVISOR=apple`, or `supervisor` per call.
+ */
+import path from 'node:path';
+import { fileURLToPath } from 'node:url';
+import * as store from './store.js';
+import { compiler, lineServer } from './localhelper.js';
+
+const SOURCE = path.join(path.dirname(fileURLToPath(import.meta.url)), '..', 'native', 'supervise.swift');
+const BIN = path.join(store.ROOT, 'bin', 'supervise');
+
+const helper = lineServer({
+  ensureBinary: compiler({ source: SOURCE, binary: BIN, what: 'local supervisor' }),
+  what: 'local supervisor',
+});
+
+export const DECISIONS = new Set(['wait', 'retry', 'stop']);
+
+/** Which backend the caller asked for, per call first and environment second. */
+export function requested(options) {
+  const raw = String(options?.supervisor ?? process.env.SIMFRAME_SUPERVISOR ?? '').trim().toLowerCase();
+  if (!raw || raw === 'none' || raw === 'off' || raw === '0' || raw === 'false') return null;
+  return raw;
+}
+
+/**
+ * Can this plan proceed past the step that just failed?
+ *
+ * @returns {Promise<{decision: 'wait'|'retry'|'stop', reason: string, ms: number}|null>}
+ *   null on every failure mode — not asked, unavailable, timed out, unparseable,
+ *   or an answer outside the three words. A null means the executor behaves
+ *   exactly as it does without a supervisor, which is the only safe default.
+ */
+export async function judge({
+  goal, step, expected, failure, screen, stillMs, note, options, timeoutMs = 2500,
+} = {}) {
+  if (!requested(options)) return null;
+  if (!step || !failure) return null;
+  const answer = await helper.ask({
+    goal: goal ? String(goal).slice(0, 200) : null,
+    step: String(step).slice(0, 200),
+    expected: expected ? String(expected).slice(0, 300) : null,
+    failure: String(failure).slice(0, 300),
+    screen: (screen ?? []).filter(Boolean).map((s) => String(s).slice(0, 40)).slice(0, 25),
+    stillMs: Number.isFinite(stillMs) ? Math.round(stillMs) : null,
+    note: note ? String(note).slice(0, 200) : null,
+  }, timeoutMs);
+  const decision = String(answer?.decision ?? '').toLowerCase();
+  // An answer outside the vocabulary is not a decision. Refusing it here is
+  // what makes the three-word constraint real rather than merely documented.
+  if (!DECISIONS.has(decision)) return null;
+  return { decision, reason: String(answer.reason ?? '').slice(0, 120), ms: answer.ms ?? null };
+}
+
+/** For `doctor`: what the supervisor layer is, in one line. */
+export async function status(options) {
+  const want = requested(options);
+  if (!want) return { supervisor: 'none', detail: 'not requested (SIMFRAME_SUPERVISOR is unset)' };
+  if (want !== 'apple') return { supervisor: 'none', detail: `no such supervisor backend: "${want}"` };
+  const live = await helper.status();
+  if (!live.ok) return { supervisor: 'none', detail: live.reason };
+  // Prove a round trip, not a presence.
+  //
+  // Reporting "available" from a fresh process was true and useless: the model
+  // had stopped answering inside the long-lived MCP server, `doctor` opened its
+  // own process, got a healthy one, and said so — for twenty calls and six
+  // failures during which nothing was being judged. The reporter's fix, and it
+  // is the right one: make it answer something.
+  const probe = await helper.ask({
+    step: 'tap "Probe"',
+    failure: '"Probe" is not on this screen. Visible: Probe',
+    screen: ['Probe'],
+    stillMs: 5000,
+  }, 6000);
+  const decision = String(probe?.decision ?? '').toLowerCase();
+  if (!DECISIONS.has(decision)) {
+    return {
+      supervisor: 'none',
+      detail: 'the model loaded but did not answer a probe — it is present and not working',
+    };
+  }
+  return {
+    supervisor: 'apple',
+    detail: `Apple Foundation Models, on-device; answered a probe in ${probe.ms ?? '?'}ms; may only answer wait/retry/stop`,
+  };
+}
+
+export const close = helper.close;