simframe 0.12.1 → 0.13.0

package/src/supervisor.js

@@ -33,6 +33,7 @@
 import path from 'node:path';
 import { fileURLToPath } from 'node:url';
 import * as store from './store.js';
+import * as ollama from './ollama.js';
 import { compiler, lineServer } from './localhelper.js';
 
 const SOURCE = path.join(path.dirname(fileURLToPath(import.meta.url)), '..', 'native', 'supervise.swift');
@@ -45,6 +46,32 @@ const helper = lineServer({
 
 export const DECISIONS = new Set(['wait', 'retry', 'stop']);
 
+/**
+ * Which arm answers.
+ *
+ * Two, and the second one is an experiment rather than a recommendation. The
+ * owner asked for the capacity question to be settled with numbers instead of
+ * speculation, and a comparison needs something to compare against. `apple` is
+ * the shipped arm; `ollama:<model>` asks a local Ollama, off unless named.
+ *
+ * A backend is an object with `ask` and `status` and nothing else, which is the
+ * same shape the platform boundary uses one layer down — so `judge` below
+ * cannot tell which arm it asked, including in the failure shapes.
+ */
+function backendFor(want) {
+  if (want === 'apple') return { kind: 'apple', ask: helper.ask, status: appleStatus };
+  if (want === 'ollama' || want.startsWith('ollama:')) {
+    const target = ollama.parseTarget(want);
+    return {
+      kind: 'ollama',
+      target,
+      ask: (situation, timeoutMs) => ollama.ask(target, situation, timeoutMs),
+      status: () => ollamaStatus(target),
+    };
+  }
+  return null;
+}
+
 /**
  * The vocabulary gate, as a function so it can be *tested* rather than grepped.
  *
@@ -57,6 +84,18 @@ export const DECISIONS = new Set(['wait', 'retry', 'stop']);
  * text — so it broke when the branch grew an else, with nothing actually wrong.
  * A property this important deserves an assertion that runs it.
  */
+/**
+ * The fourth word, and why it is not in `DECISIONS`.
+ *
+ * `abstain` means "I cannot tell from what I was given", and the only correct
+ * thing to do with it is exactly what this module already does with every
+ * failure: return `null`, and behave as if there is no supervisor. So it is a
+ * recognised *answer* and not a recognised *decision*, and keeping those apart
+ * is the point — a caller must never be able to act on it, and a log must be
+ * able to tell it from a timeout, a refusal and a model that was never there.
+ */
+export const ABSTAIN = 'abstain';
+
 export function decisionOf(answer) {
   // A string, checked rather than coerced. `String(["wait"])` is `"wait"`, so a
   // `String(...)` coercion here let `{decision: ["wait"]}` through the one gate
@@ -86,11 +125,17 @@ export function requested(options) {
  */
 export async function judge({
   goal, step, expected, failure, screen, stillMs, note, options, timeoutMs = 2500,
-  detail,
+  detail, mayAbstain = false,
 } = {}) {
-  if (!requested(options)) return null;
+  const want = requested(options);
+  if (!want) return null;
   if (!step || !failure) return null;
-  const answer = await helper.ask({
+  const backend = backendFor(want);
+  if (!backend) {
+    if (detail && typeof detail === 'object') detail.kind = `no such supervisor backend: "${want}"`;
+    return null;
+  }
+  const answer = await backend.ask({
     goal: goal ? String(goal).slice(0, 200) : null,
     step: String(step).slice(0, 200),
     expected: expected ? String(expected).slice(0, 300) : null,
@@ -98,6 +143,7 @@ export async function judge({
     screen: (screen ?? []).filter(Boolean).map((s) => String(s).slice(0, 40)).slice(0, 25),
     stillMs: Number.isFinite(stillMs) ? Math.round(stillMs) : null,
     note: note ? String(note).slice(0, 200) : null,
+    mayAbstain: Boolean(mayAbstain),
   }, timeoutMs);
   const decision = decisionOf(answer);
   if (decision == null) {
@@ -111,7 +157,12 @@ export async function judge({
     // supervisor did not answer": a timeout, a guardrail refusal and a model
     // that was never installed were one indistinguishable line.
     if (detail && typeof detail === 'object') {
-      detail.kind = answer?.kind
+      // An abstention is an answer, and the least interesting thing a log can
+      // say about it is that the supervisor "did not answer". It is the model
+      // declining on purpose, which is the one failure mode worth encouraging.
+      detail.kind = answer?.decision === ABSTAIN
+        ? 'abstained'
+        : answer?.kind
         ?? (answer == null ? 'no answer' : answer.decision ? 'outside the vocabulary' : 'unparseable');
       if (answer?.error) detail.error = String(answer.error).slice(0, 200);
     }
@@ -124,7 +175,59 @@ export async function judge({
 export async function status(options) {
   const want = requested(options);
   if (!want) return { supervisor: 'none', detail: 'not requested (SIMFRAME_SUPERVISOR is unset)' };
-  if (want !== 'apple') return { supervisor: 'none', detail: `no such supervisor backend: "${want}"` };
+  const backend = backendFor(want);
+  if (!backend) return { supervisor: 'none', detail: `no such supervisor backend: "${want}"` };
+  return backend.status();
+}
+
+/**
+ * The experiment arm, reported exactly as honestly as the shipped one.
+ *
+ * Same probe, because "available" from a presence check is the failure this
+ * project has already paid for: the model had stopped answering inside the
+ * long-lived server while `doctor`, in its own process, said it was healthy —
+ * for twenty calls and six failures during which nothing was being judged.
+ *
+ * A wide probe budget on purpose. This arm loads several gigabytes on first
+ * ask, and a cold load timing out would report a working model as broken —
+ * which is a different lie from the one above and just as useless.
+ */
+async function ollamaStatus(target) {
+  const live = await ollama.status(target);
+  if (!live.ok) return { supervisor: 'none', detail: live.reason };
+  // Weights first, then the clock. Without this the probe measured a disk read
+  // and called a working model broken.
+  const warm = await ollama.preload(target);
+  if (!warm.ok) {
+    return {
+      supervisor: 'none',
+      detail: warm.kind === 'timeout'
+        ? `"${target.model}" is still loading after 120s — ask again once it is resident`
+        : `"${target.model}" would not load (${warm.kind}: ${warm.error})`,
+    };
+  }
+  const probe = await ollama.ask(target, {
+    step: 'tap "Probe"',
+    failure: '"Probe" is not on this screen. Visible: Probe',
+    screen: ['Probe'],
+    stillMs: 5000,
+  }, live.timeoutMs);
+  if (decisionOf(probe) == null) {
+    return {
+      supervisor: 'none',
+      detail: `Ollama has "${target.model}" and it did not answer a probe`
+        + `${probe?.kind ? ` (${probe.kind}${probe.error ? `: ${probe.error}` : ''})` : ''}`,
+    };
+  }
+  return {
+    supervisor: `ollama:${target.model}`,
+    detail: `${target.model} via Ollama at ${target.host}; answered a probe in ${probe.ms ?? '?'}ms;`
+      + ' schema-constrained to wait/retry/stop. An experiment arm, not a recommendation —'
+      + ' see docs/EXPERIMENTS.md for what it measured.',
+  };
+}
+
+async function appleStatus() {
   const live = await helper.status();
   if (!live.ok) return { supervisor: 'none', detail: live.reason };
   // Prove a round trip, not a presence.

package/src/view.js

@@ -247,8 +247,26 @@ export function actsInteractive(t) {
 export function rowsFor(entry, { screen, filter, interactive, all = false, limit = DEFAULT_LIMIT } = {}) {
   let kept = (entry?.targets ?? []).map((t) => ({ ...t })).filter((t) => {
     if (!isNum(t.x) || !isNum(t.y)) return false;
-    // Off-screen elements are real in the tree and untappable in fact.
-    if (regions.offViewport(t, screen)) return false;
+    // Off-screen elements are real in the tree and untappable in fact —
+    // *unless* enough of one is on screen to tap, which the centre test cannot
+    // see. A filter chip showing 29pt of itself at the right edge was dropped
+    // while OCR's reading of that same sliver, labelled "Flc", was kept and
+    // printed in its place. Item 121: say so or clamp. Both, here — the row is
+    // kept, its tap point is moved to the centre of the visible part, and it is
+    // marked so nobody reads a clamped coordinate as a whole element.
+    //
+    // Deliberately NOT also relaxing `locate`. A clipped element still answers
+    // "in the tree but not in view", so `scrollTo` keeps scrolling to it rather
+    // than declaring a sliver good enough — that conservatism was bought with a
+    // reported session lost to `"'Assigned to Me' is in view at 422,277
+    // already"`, and widening it here would buy the same bug back.
+    if (regions.offViewport(t, screen)) {
+      const clip = regions.clipping(t, screen);
+      if (!clip?.usable) return false;
+      t.clipped = true;
+      t.x = clip.point.x;
+      t.y = clip.point.y;
+    }
     if (!all && !regions.offerable(t.region)) return false;
     if (!all && isNoise(t)) return false;
     return true;
@@ -380,11 +398,42 @@ function valueNote(r) {
   return `= ${v}`;
 }
 
+/**
+ * What to call a row.
+ *
+ * A control with no accessibility label is not necessarily anonymous, and the
+ * order here is by how much the name can be trusted. Its own label first. Then
+ * its `testID`, which is a name a developer chose and which `tap` has always
+ * matched on. Then OCR's reading of it, which is how an icon-only control gets
+ * a name at all and which cannot contradict a label it does not have. Only when
+ * all three are missing is it really unaddressable, and that is the case item
+ * 122 is about: three field reports in a row spent their time on controls that
+ * existed, were tappable, and could be reached by no selector at all.
+ */
+export function displayName(r) {
+  return trim(r.label)
+    || trim(r.identifier)
+    // Only an alias with a word in it. OCR reads the three dots of an overflow
+    // menu as `...`, and on the first live run of this the button was named
+    // `#1 button 364,84 ...` — which looks like a name, cannot be typed into a
+    // selector, and made the count below say there was nothing unnamed here.
+    // `isNoise` already refuses such text as a row of its own; it must not get
+    // in through the alias door either.
+    || (r.aliases ?? []).find((a) => alnum(a))
+    || (matching.isAxTarget(r) ? '(unlabelled)' : '(no text)');
+}
+
+/** A row the map can print and no caller can name. */
+export const unaddressable = (r) =>
+  matching.isAxTarget(r) && !trim(r.label) && !trim(r.identifier)
+  && !(r.aliases ?? []).some((a) => alnum(a));
+
 function renderRow(r) {
+  const shown = displayName(r);
   const name = [
-    trim(r.label) || (matching.isAxTarget(r) ? '(unlabelled)' : '(no text)'),
+    shown,
     valueNote(r),
-    aliasNote(r),
+    aliasNote(r, shown),
   ].filter(Boolean).join(' ');
   const state = [
     r.enabled === false ? 'disabled' : null,
@@ -395,7 +444,9 @@ function renderRow(r) {
     shortType(r.type).padEnd(9),
     `${r.x},${r.y}`.padEnd(9),
     state ? `${state} ` : '',
-    name,
+    // After the name, and appended rather than added as a column: an extra
+    // element in this join puts an extra space on *every* row, clipped or not.
+    r.clipped ? `${name}  (partly off-screen — the coordinate is the middle of the visible part)` : name,
   ].join(' ');
 }
 
@@ -406,11 +457,14 @@ function renderRow(r) {
  * for, which is useful when they disagree and pure cost when they agree —
  * "WELCOME ~ WELCOME" was a third of some rows.
  */
-function aliasNote(r) {
+function aliasNote(r, shown = r.label) {
   const extra = (r.aliases ?? [])
     .filter((a) => {
       const t = alnum(a);
-      return t && !alnum(r.label).includes(t);
+      // Compared against the name actually printed, not against the label. An
+      // unlabelled control is named by its own alias now, and comparing against
+      // an absent label printed every one of them twice: `Sort ~ Sort`.
+      return t && !alnum(shown).includes(t);
     })
     .slice(0, 2);
   return extra.length ? `~ ${trim(extra.join(' '))}` : null;
@@ -511,6 +565,25 @@ export async function screenMap(deviceQuery, {
       + ' you did not expect to see as belonging to the layer underneath'
     : null;
 
+  // Controls that are on the screen and answer to no name — item 122.
+  //
+  // Counted from the rows about to be printed rather than from the map, so it
+  // is a statement about what the caller can see. The wording is the ask,
+  // near-verbatim from the report that made it: *"a line like 'N on-screen
+  // tappable views have no accessibility label — they cannot be addressed by
+  // selector' would push people toward instrumenting, which is the outcome
+  // everyone wants"*. It is also the honest answer to the second half of that
+  // item, which we cannot fix from here: a view UIKit was never told is
+  // accessible is invisible to the tree, so a screen whose controls are all
+  // undeclared shows this count as 0 and still needs a screenshot.
+  const anonymous = rows.filter(unaddressable).length;
+  const unnamed = anonymous
+    ? `${anonymous} on-screen control(s) have no accessibility label — they are listed with their`
+      + ' coordinates and can be tapped by point or by #ref, but not by name. If what you are'
+      + ' looking for is not in the list either, the app has views that were never declared'
+      + ' accessible and only a screenshot will find those.'
+    : null;
+
   return {
     device,
     identity,
@@ -529,7 +602,8 @@ export async function screenMap(deviceQuery, {
     staleExits,
     cleared,
     overlay,
-    text: render({ device, identity, rows, truncated, collapsed, screen, name, exits, exitList, staleExits, cleared, overlay }),
+    unnamed,
+    text: render({ device, identity, rows, truncated, collapsed, screen, name, exits, exitList, staleExits, cleared, overlay, unnamed }),
   };
 }
 
@@ -712,7 +786,7 @@ export function ambiguousLabels(rows) {
   return [...seen.values()].filter((n) => n > 1).length;
 }
 
-export function render({ device, identity, rows, truncated, collapsed, screen, name, exits, exitList, staleExits, verdictLine, ambiguities, cleared, overlay }) {
+export function render({ device, identity, rows, truncated, collapsed, screen, name, exits, exitList, staleExits, verdictLine, ambiguities, cleared, overlay, unnamed }) {
   const head = [
     device?.name,
     screen?.width ? `${screen.width}x${screen.height}pt` : null,
@@ -747,6 +821,7 @@ export function render({ device, identity, rows, truncated, collapsed, screen, n
   // already believes, which is the one kind of news that must not be scrolled to.
   if (cleared) lines.push(cleared);
   if (overlay) lines.push(overlay);
+  if (unnamed) lines.push(unnamed);
   const worked = exitsLine(exitList, { stale: staleExits });
   if (worked) lines.push(worked);