simframe 0.11.0 → 0.12.0

package/src/metrics.js

@@ -24,6 +24,21 @@ export const REASONS = [
 
 export const OUTCOMES = ['resolved_locally', 'escalated_to_model', 'failed'];
 
+/**
+ * What the executor observed after a supervisor ruling — the field that makes a
+ * ruling scoreable rather than merely recorded.
+ *
+ * A ruling on its own says what the supervisor thought. Three items on the
+ * deferred list (101's p95 replay, 106's cost of the steps a `stop` skipped,
+ * 96's response variable) need what happened *next*, and until now nothing
+ * persisted a ruling at all: they went into a `supervisions` array on the
+ * result and died with the process. Three rulings had ever existed anywhere.
+ *
+ * Closed, and mandatory, for the same reason `REASONS` is: a vocabulary that
+ * admits "other" collects a pile of "other".
+ */
+export const RULING_OUTCOMES = ['recovered', 'still_failed', 'stopped', 'no_ruling'];
+
 /**
  * Which faculty would have removed this escalation.
  *
@@ -57,6 +72,7 @@ function metricPaths(udid) {
     dir,
     escalations: path.join(dir, 'escalations.jsonl'),
     flows: path.join(dir, 'flows.jsonl'),
+    supervisions: path.join(dir, 'supervisions.jsonl'),
     baselines: path.join(dir, 'baselines'),
   };
 }
@@ -109,6 +125,106 @@ export function readJsonl(file, { limit } = {}) {
 
 export const readEscalations = (udid, opts) => readJsonl(metricPaths(udid).escalations, opts);
 export const readFlows = (udid, opts) => readJsonl(metricPaths(udid).flows, opts);
+export const readSupervisions = (udid, opts) => readJsonl(metricPaths(udid).supervisions, opts);
+
+/**
+ * Write down one supervisor ruling and what came of it.
+ *
+ * The fields are chosen so the three items waiting on this can be answered
+ * **offline, from the log**, rather than by another device run:
+ *
+ * - `screen`/`edge` — which `(screen_hash, action)` the ruling was about, so
+ *   rulings can be grouped by edge the way the graph groups timings.
+ * - `p95`/`samples` — what the graph knew about that edge **at the moment of
+ *   the ruling**. Recording it now rather than looking it up later is the
+ *   difference between 101 being arithmetic and 101 being archaeology: the
+ *   graph keeps learning, so a p95 read next week is not the p95 the
+ *   supervisor was implicitly competing with.
+ * - `stillMs` — the same input the supervisor got, so a replay sees what it saw.
+ * - `expect` — the plan's own note, because a ruling made with a briefing and
+ *   one made without are not the same measurement (96's critical arm).
+ * - `outcome` — what the executor observed afterwards. Mandatory.
+ *
+ * `from` separates a deterministic rule from a model answer. Both are rulings
+ * and both have outcomes, but a comparison that mixed them would credit the
+ * model for what a two-line rule decided.
+ */
+export function recordSupervision(udid, {
+  session, index, step, edge, screen, decision, from, reason, ms,
+  stillMs, p95, samples, expect, failure, outcome,
+}) {
+  // Swallowed rather than thrown, unlike `recordEscalation`'s guard, and the
+  // difference is deliberate: this is called from inside a flow's failure
+  // handler, where a throw would turn a recoverable step failure into a crash.
+  // It still lands in `lastWriteError`, which `simframe escalations` prints.
+  if (!RULING_OUTCOMES.includes(outcome)) {
+    lastWriteError = `supervision outcome must be one of ${RULING_OUTCOMES.join('/')}, got ${JSON.stringify(outcome)}`;
+    return false;
+  }
+  return appendJsonl(metricPaths(udid).supervisions, {
+    timestamp: new Date().toISOString(),
+    session_id: session ?? SESSION_ID,
+    client: CLIENT,
+    step_index: index ?? null,
+    step: step ?? null,
+    edge: edge ?? null,
+    screen_fingerprint: screen ?? null,
+    decision,
+    from: from ?? 'model',
+    // Recorded, never presented as the ground for what happened: the supervisor
+    // has returned a correct decision with a reason citing a rule that did not
+    // apply. Keeping it is how that stays measurable instead of anecdotal.
+    reason: reason ? String(reason).slice(0, 200) : null,
+    latency_ms: Number.isFinite(ms) ? Math.round(ms) : null,
+    still_ms: Number.isFinite(stillMs) ? Math.round(stillMs) : null,
+    edge_p95_ms: Number.isFinite(p95) ? Math.round(p95) : null,
+    edge_samples: Number.isFinite(samples) ? samples : null,
+    expect: expect ? String(expect).slice(0, 200) : null,
+    failure: failure ? String(failure).slice(0, 300) : null,
+    outcome,
+  });
+}
+
+/**
+ * What the ruling log currently says, and whether it can yet answer 101.
+ *
+ * Deliberately counts rather than scores. Item 101 asks which `wait`/`retry`
+ * rulings a p95-per-edge lookup would have got right, and that is a separate
+ * piece of work; what this answers is the question that comes first and was
+ * embarrassing to get wrong once already — **is there a population to measure
+ * at all, and do its rows carry the fields the measurement needs.** `p95_known`
+ * is that readiness check: a ruling recorded on an edge the graph had never
+ * timed cannot take part in the comparison, however many of them there are.
+ */
+export function supervisionBreakdown(records) {
+  const by = (key) => {
+    const out = {};
+    for (const r of records) {
+      const k = r[key] ?? 'unknown';
+      out[k] = (out[k] ?? 0) + 1;
+    }
+    return out;
+  };
+  const matrix = {};
+  for (const r of records) {
+    const k = `${r.decision ?? 'unknown'} -> ${r.outcome ?? 'unknown'}`;
+    matrix[k] = (matrix[k] ?? 0) + 1;
+  }
+  const timed = records.filter((r) => Number.isFinite(r.edge_p95_ms));
+  const latencies = records.map((r) => r.latency_ms).filter((n) => Number.isFinite(n)).sort((a, b) => a - b);
+  return {
+    total: records.length,
+    by_decision: by('decision'),
+    by_outcome: by('outcome'),
+    by_from: by('from'),
+    decision_to_outcome: matrix,
+    // Readiness for 101, not a result for it.
+    p95_known: timed.length,
+    p95_unknown: records.length - timed.length,
+    sessions: [...new Set(records.map((r) => r.session_id).filter(Boolean))],
+    median_latency_ms: latencies.length ? latencies[latencies.length >> 1] : null,
+  };
+}
 
 /**
  * Mark an error as an escalation with a reason, at the site that knows why.

package/src/platform/android.js

@@ -996,6 +996,28 @@ function capabilities() {
   };
 }
 
+/**
+ * Not implemented here, and it says so in its own vocabulary.
+ *
+ * `revive` exists for a display that has stopped rendering — a CoreSimulator
+ * pathology. An emulator's failure modes are its own (the console port going
+ * away, adb losing the device) and the remedy is not the same sequence, so
+ * borrowing the other platform's answer would be the mistake `doctor` made when
+ * it reported "input driver: idb" for an emulator: a claim about a tool that has
+ * never spoken to an Android device.
+ *
+ * When an emulator wedge is characterised rather than guessed at, this becomes
+ * a real implementation. Until then the honest answer is that this backend does
+ * not have the layer.
+ */
+async function restartDevice(serial) {
+  throw new Error(
+    `simframe cannot yet restart an emulator (${serial}) — that remedy is written for a`
+    + ' simulator display that stopped rendering, and an emulator fails differently.'
+    + ' Restart it from Android Studio, or `adb -s <serial> emu kill` and relaunch.',
+  );
+}
+
 /** @type {import('./index.js').Platform} */
 export const platform = {
   id: 'android',
@@ -1011,6 +1033,7 @@ export const platform = {
   launchApp,
   terminateApp,
   openUrl,
+  restartDevice,
   setPermission,
   setPasteboard,
   getPasteboard,

package/src/platform/index.js

@@ -61,7 +61,7 @@ export const PLATFORM_SURFACE = Object.freeze([
   'id', 'deviceNoun',
   'listDevices', 'bootedDevices', 'resolveDevice', 'isBootedSync', 'ownsUdid',
   'geometry', 'inputDriver',
-  'screenshot', 'launchApp', 'terminateApp', 'openUrl',
+  'screenshot', 'launchApp', 'terminateApp', 'openUrl', 'restartDevice',
   'setPermission', 'setPasteboard', 'permissionServices', 'capabilities', 'toolchain',
   'bootedAt',
 ]);
@@ -260,6 +260,16 @@ export const inputDriverFor = (udid) => platformFor(udid).inputDriver(udid);
  */
 export const capabilitiesFor = (udid) => platformFor(udid).capabilities();
 
+/**
+ * Power-cycle a device, on the backend that owns it.
+ *
+ * Reached only from `simframe revive`, never from the capture loop: the loop
+ * detects a stalled display and reports it, and restarting is the operator's
+ * call. A backend that does not have this remedy throws in its own terms rather
+ * than borrowing the other's.
+ */
+export const restartDevice = (udid) => platformFor(udid).restartDevice(udid);
+
 /** What `simframe doctor` should check: each registered backend's own toolchain. */
 export function toolchainChecks() {
   return backends().flatMap((backend) => backend.toolchain());

package/src/platform/ios.js

@@ -189,6 +189,28 @@ async function openUrl(udid, url) {
   await run('xcrun', ['simctl', 'openurl', udid, url], { timeout: 20_000 });
 }
 
+/**
+ * Shut a device down and bring it back, waiting for the boot to finish.
+ *
+ * The remedy for a display that has stopped rendering, which the capture loop
+ * can detect and must not perform: it reports `stalled` and stops, because a
+ * capture loop that rebooted the device it was watching would be a tool
+ * reaching for the mains when a reading looks wrong. This is the operator's
+ * decision, reached by `simframe revive`.
+ *
+ * `bootstatus -b` and not `boot`, for the reason it is used in CI: `boot`
+ * returns before the device is usable, and everything downstream then races the
+ * boot. Timeboxed generously — a cold boot on a busy machine is slow, and a
+ * boot that never finishes should fail here with a reason rather than as a
+ * puzzle further down.
+ */
+async function restartDevice(udid) {
+  // Tolerated: a device that is already off cannot be shut down, and that is
+  // the state this command is most often reached from.
+  await run('xcrun', ['simctl', 'shutdown', udid], { timeout: 60_000 }).catch(() => null);
+  await run('xcrun', ['simctl', 'bootstatus', udid, '-b'], { timeout: 240_000 });
+}
+
 const PERMISSION_SERVICES = [
   'all', 'calendar', 'contacts-limited', 'contacts', 'location', 'location-always',
   'photos-add', 'photos', 'media-library', 'microphone', 'motion', 'reminders', 'siri',
@@ -347,6 +369,7 @@ export const platform = {
   launchApp,
   terminateApp,
   openUrl,
+  restartDevice,
   setPermission,
   setPasteboard,
   permissionServices: () => PERMISSION_SERVICES,

package/src/regions.js

@@ -24,6 +24,27 @@ export const REGIONS = [
   'content',
 ];
 
+/**
+ * Regions the screen map will not offer as something to act on.
+ *
+ * The status bar says the time and the battery level. It is on every screen, it
+ * is never what anybody wants to tap, and it costs a row every time — so
+ * `sim_ui` hides it.
+ *
+ * It lives here rather than in `view.js`, where it started, because it is not
+ * only a presentation rule. A target the map refuses to *show* must also be a
+ * target nothing may resolve onto *behind the caller's back*, and the case that
+ * proved it was exactly that: a stale `#1` numbered "Reminders" in Reminders,
+ * re-resolved in Contacts onto the status-bar back-to-app breadcrumb
+ * "• Reminders", scored 0.64 and handed back a tap point at (47,40) — a place
+ * the map would never have put in front of anybody. One rule, one home, both
+ * readers.
+ */
+const UNOFFERED_REGIONS = new Set(['status-bar']);
+
+/** Would the screen map offer a target in this region? */
+export const offerable = (region) => !UNOFFERED_REGIONS.has(region);
+
 /**
  * The status bar stays positional, and deliberately.
  *
@@ -63,6 +84,46 @@ const BOTTOM_CHROME_LIMIT = 0.82;
  * screen sets its own scale.
  */
 const MIN_BOUNDARY_GAP_PT = 10;
+/**
+ * How wide a lone row may be and still be a screen's title rather than its
+ * first paragraph.
+ *
+ * 0.6 of the screen, measured: the Settings root's large title is 133 pt of
+ * 402 (0.33), while Reminders' empty-state heading "Welcome to Reminders" is
+ * 326 pt (0.81) and is content — it describes the screen instead of naming it.
+ * A title is a name and names are short.
+ */
+const LARGE_TITLE_MAX_WIDTH_FRACTION = 0.6;
+/**
+ * How far below the status bar a large title can start.
+ *
+ * iOS draws one at a **system** offset, not an app-chosen one, so this is a
+ * bound on a platform constant rather than a tuned threshold. Measured on the
+ * bench device: the Settings root's title starts 63-79 pt below the status bar
+ * depending on which sensor reports its box, while example.com's `<h1>` — page
+ * *content* that merely happens to be the first row, because Safari on iOS puts
+ * its chrome at the bottom — starts **122 pt** down.
+ *
+ * Without this bound the rule promoted that `<h1>` to chrome and "example
+ * domain" entered the screen's identity. Pulling page content into identity is
+ * the exact failure this module has been bitten by twice (a phantom keyboard,
+ * and content that merely fell into a band), so the bound is not optional.
+ */
+const LARGE_TITLE_MAX_INSET_PT = 96;
+/**
+ * And the least it can be, before it is just the next row.
+ *
+ * Absolute, like the maximum, and for the same reason: the inset is drawn by
+ * the system, so it is not a function of what the screen contains. The first
+ * version tested it against the screen's *median row gap* — and the testbed
+ * caught that on its first day, with two screens of the same app. A list of 24
+ * rows has a median gap of 0 and the rule fired; a list of 4 rows above a tab
+ * bar has a median gap of **414**, because the empty area counts as a gap, and
+ * the rule did not. Same title, same inset of 62.9pt, opposite answers — so one
+ * screen had a name in its identity and the other did not, and the graph then
+ * called them the same screen at 0.50 similarity.
+ */
+const LARGE_TITLE_MIN_INSET_PT = 24;
 const BOUNDARY_GAP_FACTOR = 1.9;
 
 /** A tab bar is several things spread across the width, not one thing at the bottom. */
@@ -177,6 +238,50 @@ export function bands(elements, screen) {
     }
   }
 
+  // --- a large title, which has no gap under it to be found by.
+  //
+  // The loop above identifies top chrome by the whitespace *beneath* it, and an
+  // iOS large title is drawn tight against the content it heads: measured on
+  // the Settings root, 79 pt of inset above it and **5.3 pt** below, against a
+  // bar of `max(10, typical * 1.9)` = 66.5. No threshold reaches that, so the
+  // title fell into `content` and was discarded as content — leaving the screen
+  // with **no name at all** in its fingerprint, in either sensor mode.
+  //
+  // That is not cosmetic. Chrome labels are the only text identity keeps, and
+  // `fingerprint.js` names the consequence: two list screens with identical
+  // structure differ by their title and nothing else says so. A nameless screen
+  // is pure geometry, and on a hosted runner two sparse nameless readings
+  // matched exactly — one screen's hash for two screens.
+  //
+  // So it is found by the inset *above* it instead, which is the half iOS does
+  // provide. A large title sits alone, narrow, high, under a generous gap; a
+  // compact bar is the mirror image of that (20-33 pt above, 118-134 below) and
+  // is already caught by the loop. Deliberately not keyed on the `Heading`
+  // role: OCR has no roles, and the reading that actually collided was
+  // OCR-only, so a role test would work only in the case that does not fail.
+  //
+  // Measured across all 17 perception fixtures before being written here: it
+  // changes exactly one of them, the Settings root.
+  if (!navBarBottom) {
+    const first = rows.findIndex((r) => r.top >= statusBarBottom - 1);
+    const row = first >= 0 ? rows[first] : null;
+    if (
+      row
+      && first + 1 < rows.length
+      && row.items.length === 1
+      && (row.items[0].frame?.width ?? 0) <= screen.width * LARGE_TITLE_MAX_WIDTH_FRACTION
+      && row.bottom <= screen.height * TOP_CHROME_LIMIT
+      // A real separation from the status bar, but a system-sized one: far
+      // enough to be an inset, near enough to still be the app's own title.
+      // Both bounds absolute — see LARGE_TITLE_MIN_INSET_PT for what keying the
+      // lower one on the screen's own row rhythm cost.
+      && row.top - statusBarBottom >= LARGE_TITLE_MIN_INSET_PT
+      && row.top - statusBarBottom <= LARGE_TITLE_MAX_INSET_PT
+    ) {
+      navBarBottom = row.bottom;
+    }
+  }
+
   // --- bottom chrome. One row: a tab bar is one row by construction, and the
   // gap above it is what separates it from the list it floats over.
   let tabBarTop = Infinity;

package/src/supervisor.js

@@ -45,6 +45,30 @@ const helper = lineServer({
 
 export const DECISIONS = new Set(['wait', 'retry', 'stop']);
 
+/**
+ * The vocabulary gate, as a function so it can be *tested* rather than grepped.
+ *
+ * This one line is the safety property: the supervisor cannot invent a step,
+ * skip one, substitute a target or continue past an unexpected screen, because
+ * those are not words it can say. Anything outside the three is not a decision
+ * and becomes `null`, which means "behave as if there is no supervisor".
+ *
+ * It was previously inline, and the test that guarded it matched the source
+ * text — so it broke when the branch grew an else, with nothing actually wrong.
+ * A property this important deserves an assertion that runs it.
+ */
+export function decisionOf(answer) {
+  // A string, checked rather than coerced. `String(["wait"])` is `"wait"`, so a
+  // `String(...)` coercion here let `{decision: ["wait"]}` through the one gate
+  // that defines this component's answer space. Found the first time this
+  // property was *run* instead of grepped for in the source — the old test
+  // matched the source text of the branch and could never have caught it.
+  const raw = answer?.decision;
+  if (typeof raw !== 'string') return null;
+  const decision = raw.toLowerCase();
+  return DECISIONS.has(decision) ? decision : null;
+}
+
 /** Which backend the caller asked for, per call first and environment second. */
 export function requested(options) {
   const raw = String(options?.supervisor ?? process.env.SIMFRAME_SUPERVISOR ?? '').trim().toLowerCase();
@@ -62,6 +86,7 @@ export function requested(options) {
  */
 export async function judge({
   goal, step, expected, failure, screen, stillMs, note, options, timeoutMs = 2500,
+  detail,
 } = {}) {
   if (!requested(options)) return null;
   if (!step || !failure) return null;
@@ -74,10 +99,24 @@ export async function judge({
     stillMs: Number.isFinite(stillMs) ? Math.round(stillMs) : null,
     note: note ? String(note).slice(0, 200) : null,
   }, timeoutMs);
-  const decision = String(answer?.decision ?? '').toLowerCase();
-  // An answer outside the vocabulary is not a decision. Refusing it here is
-  // what makes the three-word constraint real rather than merely documented.
-  if (!DECISIONS.has(decision)) return null;
+  const decision = decisionOf(answer);
+  if (decision == null) {
+    // Why it did not answer, for the caller's log — through an out-parameter
+    // rather than the return value, because returning anything truthy here
+    // would change what the executor does. `null` means "behave as if there is
+    // no supervisor" and that safety property is the one thing in this file
+    // that must not become conditional.
+    //
+    // Before this, every failure reached the supervision log as "the
+    // supervisor did not answer": a timeout, a guardrail refusal and a model
+    // that was never installed were one indistinguishable line.
+    if (detail && typeof detail === 'object') {
+      detail.kind = answer?.kind
+        ?? (answer == null ? 'no answer' : answer.decision ? 'outside the vocabulary' : 'unparseable');
+      if (answer?.error) detail.error = String(answer.error).slice(0, 200);
+    }
+    return null;
+  }
   return { decision, reason: String(answer.reason ?? '').slice(0, 120), ms: answer.ms ?? null };
 }
 
@@ -101,16 +140,21 @@ export async function status(options) {
     screen: ['Probe'],
     stillMs: 5000,
   }, 6000);
-  const decision = String(probe?.decision ?? '').toLowerCase();
-  if (!DECISIONS.has(decision)) {
+  if (decisionOf(probe) == null) {
     return {
       supervisor: 'none',
       detail: 'the model loaded but did not answer a probe — it is present and not working',
     };
   }
+  // The window, read from the model rather than repeated from documentation.
+  // Worth printing: the worst case our own clipping allows measures 1,918
+  // tokens against it, and that ratio is the reason there is no per-call token
+  // budget check — see DEFERRED 99.
+  const ctx = live.hello?.contextSize;
   return {
     supervisor: 'apple',
-    detail: `Apple Foundation Models, on-device; answered a probe in ${probe.ms ?? '?'}ms; may only answer wait/retry/stop`,
+    detail: `Apple Foundation Models, on-device; answered a probe in ${probe.ms ?? '?'}ms;`
+      + `${ctx ? ` ${ctx}-token window;` : ''} may only answer wait/retry/stop`,
   };
 }
 

package/src/view.js

@@ -22,10 +22,10 @@ import * as matching from './matching.js';
 const REGION_ORDER = ['nav-bar', 'content', 'tab-bar', 'keyboard', 'status-bar'];
 
 /**
- * The status bar says the time and the battery level. It is on every screen,
- * it is never what anybody wants to tap, and it costs a row every time.
+ * Which regions the map will not offer now lives in `regions.js`, because it is
+ * not only a presentation rule — see `regions.offerable`. A target this hides
+ * must also be one nothing resolves onto behind the caller's back.
  */
-const HIDDEN_REGIONS = new Set(['status-bar']);
 
 /** A keyboard is 30-odd keys nobody refers to by name. One line says it. */
 const COLLAPSE_REGIONS = new Set(['keyboard']);
@@ -249,7 +249,7 @@ export function rowsFor(entry, { screen, filter, interactive, all = false, limit
     if (!isNum(t.x) || !isNum(t.y)) return false;
     // Off-screen elements are real in the tree and untappable in fact.
     if (regions.offViewport(t, screen)) return false;
-    if (!all && HIDDEN_REGIONS.has(t.region)) return false;
+    if (!all && !regions.offerable(t.region)) return false;
     if (!all && isNoise(t)) return false;
     return true;
   });
@@ -329,6 +329,28 @@ export function recalledNote(identity, now = Date.now()) {
   return `elements recalled from ${ago} ago — pass refresh for what is there now`;
 }
 
+/**
+ * How old the frame behind this reading is, and whether that is a problem.
+ *
+ * Two thresholds, because "slightly behind" and "possibly a different screen"
+ * are different messages. Under `FRAME_FRESH_MS` nothing is said: a map that
+ * announced "42ms old" on every call would train a reader to skip the line that
+ * matters. Over `FRAME_STALE_MS` it shouts, because at that age the app may have
+ * moved on entirely and the whole element list is then a description of the past.
+ */
+export const FRAME_FRESH_MS = 1500;
+export const FRAME_STALE_MS = 4000;
+
+export function frameAgeNote(identity, now = Date.now()) {
+  const at = identity?.state?.capturedAt;
+  if (!Number.isFinite(at)) return null;
+  const age = Math.max(0, now - at);
+  if (age < FRAME_FRESH_MS) return null;
+  if (age < FRAME_STALE_MS) return `frame ${(age / 1000).toFixed(1)}s old`;
+  return `WARNING this frame is ${(age / 1000).toFixed(1)}s old — the screen may have moved on `
+    + 'since, so treat the elements below as a description of the past and pass refresh';
+}
+
 /**
  * What a control *contains*, from the sensor that actually knows.
  *
@@ -702,6 +724,20 @@ export function render({ device, identity, rows, truncated, collapsed, screen, n
     identity?.settled === false ? 'STILL MOVING' : null,
     // Still and finished are not the same thing.
     identity?.loading === true ? 'STILL LOADING' : null,
+    // How old the *frame* this map was read from is.
+    //
+    // `sim_look` and `sim_state` have printed this since they existed, and this
+    // map never has — so the one tool an agent is told to start with was the one
+    // that could not say how old its evidence was. Reported from the field: a
+    // complete 20-element map of a screen the app was not on, and *"a wrong
+    // answer is worse than an error here, because nothing downstream knows to
+    // doubt it"*. `ensureDaemon` will hand back a frame up to 30s old, so this
+    // was reachable without anything being broken.
+    //
+    // `recalledNote` below is a different claim — that the *element map* came
+    // from memory — and having one was what made the absence of the other easy
+    // to miss.
+    frameAgeNote(identity),
     recalledNote(identity),
   ].filter(Boolean).join(' · ');