simframe 0.10.0 → 0.11.0

package/src/metrics.js

@@ -119,7 +119,7 @@ export const readFlows = (udid, opts) => readJsonl(metricPaths(udid).flows, opts
  * matching error strings at the boundary — a regexed message is a reason that
  * silently becomes "unknown" the day somebody rewords it.
  */
-export function tag(err, reason, { candidates = [], tried = [], ambiguous = false } = {}) {
+export function tag(err, reason, { candidates = [], tried = [], ambiguous = false, intent = null } = {}) {
   if (!REASONS.includes(reason)) throw new Error(`not an escalation reason: ${reason}`);
   // `ambiguous` is narrower than the reason, and that is the point. Two very
   // different failures both tag `ambiguous_intent`: the target is on screen
@@ -127,7 +127,19 @@ export function tag(err, reason, { candidates = [], tried = [], ambiguous = fals
   // thought we knew. Only the first is resolvable by *choosing*, and only the
   // first tells a waiting caller that waiting is pointless — the thing it is
   // waiting for has already arrived.
-  err.escalation = { reason, candidates, tried, ambiguous };
+  // `intent` is the goal in the caller's own words, recorded as a field rather
+  // than left in the prose of `detail`.
+  //
+  // Phase 17's go/no-go asks whether an on-device model would pick the element
+  // Claude picked, given the goal and the element list. The element list is
+  // here as `candidates` and the eventual choice is recoverable from the
+  // graph — the tap that finally worked on this screen becomes a verified edge
+  // carrying its own step. The goal was the missing third, and it was sitting
+  // inside a sentence: `"X" matches 3 things on this screen — say which…`.
+  // Regexing it back out at export time is the exact habit this file exists to
+  // avoid, and it would silently return nothing the day that sentence is
+  // reworded.
+  err.escalation = { reason, candidates, tried, ambiguous, intent };
   return err;
 }
 
@@ -252,7 +264,20 @@ export function fingerprintNow(udid, screenmap) {
  * file is committed to a public repo in summary form, and the question it has
  * to answer is "was this all one agent", which needs no identity to answer.
  */
-const SESSION_ID = `${process.pid.toString(36)}-${Date.now().toString(36)}-${Math.random().toString(36).slice(2, 8)}`;
+const SESSION_ID = process.env.SIMFRAME_SESSION
+  ? String(process.env.SIMFRAME_SESSION).slice(0, 64)
+  : `${process.pid.toString(36)}-${Date.now().toString(36)}-${Math.random().toString(36).slice(2, 8)}`;
+
+/*
+ * Minting the id from the pid was right for the MCP server, which is one
+ * long-lived process, and wrong for everything else. A CLI-driven agent starts
+ * a process per command, so it got one "session" per command: on the benchmark
+ * device, 33 session ids for 46 records, 30 of them holding a single record.
+ * `escalations --session` was therefore unable to answer the one question it
+ * exists for, and Phase 17's go/no-go step 1 — "filter to one session id" —
+ * had nothing to filter. `SIMFRAME_SESSION` lets a caller that knows it is one
+ * session say so; the per-process id stays the default.
+ */
 
 /** How this process is being used, for reading a breakdown afterwards. */
 function clientKind() {
@@ -271,6 +296,7 @@ export const clientName = () => CLIENT;
 export function recordEscalation(udid, {
   flowId = null,
   flowName = null,
+  intent = null,
   stepIndex = null,
   fingerprint = null,
   reason,
@@ -292,6 +318,9 @@ export function recordEscalation(udid, {
     client: CLIENT,
     flow_id: flowId,
     flow_name: flowName,
+    // What was asked for, in the caller's words. Ground truth for Phase 17's
+    // go/no-go, and on its own it answers "what kind of decision is costing us".
+    intent: intent ? String(intent).slice(0, 120) : null,
     step_index: stepIndex,
     screen_fingerprint: fingerprint,
     reason,

package/src/ocr.js

@@ -19,6 +19,17 @@ const BIN = path.join(BIN_DIR, 'ocr');
 
 let ready = null;
 
+/**
+ * Which recognition level to ask Vision for.
+ *
+ * `accurate` is the default and what CLAUDE.md fixes; `fast` is the other thing
+ * Vision offers. Exposed so the pair can be scored against each other instead
+ * of one of them being a constant nobody measured.
+ */
+export function level() {
+  return String(process.env.SIMFRAME_OCR ?? '').toLowerCase() === 'fast' ? 'fast' : 'accurate';
+}
+
 /** Compile once, then reuse. Recompiles only if the source is newer than the binary. */
 export async function ensureBinary() {
   if (ready) return ready;
@@ -55,7 +66,13 @@ export async function ensureBinary() {
 export async function readText(pngFile, { density = 3 } = {}) {
   const built = await ensureBinary();
   if (!built.available) throw new Error(built.reason);
-  const { stdout } = await run(built.binary, [pngFile], { timeout: 30_000, maxBuffer: 16 << 20 });
+  // The recognition level rides in the environment rather than in argv, so the
+  // Swift side keeps its one-argument contract and an older binary still works.
+  const { stdout } = await run(built.binary, [pngFile], {
+    timeout: 30_000,
+    maxBuffer: 16 << 20,
+    env: { ...process.env, SIMFRAME_OCR: level() },
+  });
   const raw = JSON.parse(stdout || '[]');
   return raw.map((r) => ({
     text: r.text,

package/src/planner.js

@@ -0,0 +1,195 @@
+/**
+ * The local planner tier: a ranker, behind a flag, that may only reorder.
+ *
+ * **Why this exists at all, given Phase 17 was a no-go.** That phase asked a
+ * local model to *choose the next element*, and the answer was that the matcher
+ * already does — 37 of 40 real decisions. This is the complement and the one
+ * case a string matcher structurally cannot do: the goal matches **nothing** on
+ * screen, and something has to guess which container leads to it. "Change my
+ * username" shares no prefix, synonym or typo distance with "Account".
+ *
+ * Measured on this machine, six hand-written cases: 5 of 6 top-1, 6 of 6 top-3,
+ * median 564 ms warm. See `docs/BENCHMARKS.md`. Against a model round trip at
+ * 10–16 s that is roughly twenty times cheaper; against the honest baseline —
+ * breadth-first ordering, which needs no model — it won five of six.
+ *
+ * **What it is allowed to do, and it is deliberately almost nothing.** It
+ * reorders a list of candidates the caller has already permitted and will try
+ * in some order regardless. It cannot invent a label, cannot choose an action,
+ * cannot see pixels, and never runs on a destructive label because the caller
+ * filtered those out before asking (`src/vocabulary.js`). If it is wrong the
+ * exploration budget simply tries the next one. That is strictly weaker
+ * authority than Phase 17 proposed, which is what makes it safe to try.
+ *
+ * **Off unless asked.** `SIMFRAME_PLANNER=apple` turns it on; anything else,
+ * or any failure at all, degrades to `null` and the caller keeps its own order.
+ * `doctor` reports which. CI runs with it off.
+ */
+import { spawn, execFile } from 'node:child_process';
+import fs from 'node:fs';
+import path from 'node:path';
+import { fileURLToPath } from 'node:url';
+import { promisify } from 'node:util';
+import * as store from './store.js';
+
+const run = promisify(execFile);
+const SOURCE = path.join(path.dirname(fileURLToPath(import.meta.url)), '..', 'native', 'rank.swift');
+const BIN = path.join(store.ROOT, 'bin', 'rank');
+
+/** Which backend the caller asked for. Absent means no local planner. */
+export function requested(options) {
+  // Per call first, then the environment, for the reason in `sensorMode`: an
+  // MCP server's environment is fixed when it spawns, so a tester could not
+  // switch backends inside one session and a round came back with one arm of
+  // its A/B unrun.
+  const raw = String(options?.planner ?? process.env.SIMFRAME_PLANNER ?? '').trim().toLowerCase();
+  if (!raw || raw === 'none' || raw === 'off' || raw === '0' || raw === 'false') return null;
+  return raw;
+}
+
+let building = null;
+
+export async function ensureBinary() {
+  if (building) return building;
+  building = (async () => {
+    try {
+      const src = fs.statSync(SOURCE).mtimeMs;
+      const bin = fs.existsSync(BIN) ? fs.statSync(BIN).mtimeMs : 0;
+      if (bin > src) return { available: true, binary: BIN };
+    } catch {
+      return { available: false, reason: 'the ranker source is missing from this install' };
+    }
+    try {
+      fs.mkdirSync(path.dirname(BIN), { recursive: true });
+      // `swiftc`, not `xcrun swiftc`: nothing above the platform boundary may
+      // name a platform tool, and the boundary test catches it. `src/ocr.js`
+      // set this precedent — a compiler is not a device tool.
+      await run('swiftc', ['-O', SOURCE, '-o', BIN], { timeout: 180_000 });
+      return { available: true, binary: BIN };
+    } catch (err) {
+      building = null; // let a later call retry once a toolchain is present
+      return {
+        available: false,
+        reason: err.code === 'ENOENT'
+          ? 'swiftc is not installed, so the local planner cannot be built (install Xcode command line tools)'
+          : `could not build the local planner: ${String(err.message).split('\n')[0]}`,
+      };
+    }
+  })();
+  return building;
+}
+
+let session = null;
+
+/** Start the helper once and keep it, because the first answer pays model load. */
+async function open() {
+  if (session) return session;
+  const built = await ensureBinary();
+  if (!built.available) return { ok: false, reason: built.reason };
+  session = await new Promise((resolve) => {
+    const child = spawn(built.binary, [], { stdio: ['pipe', 'pipe', 'ignore'] });
+    // Deliberately NOT unref'd. Unreffing the child's stdout unreferences the
+    // very pipe every request waits on, so the process exited silently in the
+    // middle of an await — a flow that printed nothing and returned 0. The
+    // helper is closed explicitly instead, by whoever opened it.
+    let buffer = '';
+    const waiters = [];
+    let settled = false;
+    const fail = (reason) => {
+      if (!settled) { settled = true; resolve({ ok: false, reason }); }
+      while (waiters.length) waiters.shift()(null);
+    };
+    child.on('error', (err) => fail(`the local planner would not start: ${err.message}`));
+    child.on('exit', () => { session = null; fail('the local planner exited'); });
+    child.stdout.on('data', (chunk) => {
+      buffer += chunk;
+      let i = buffer.indexOf('\n');
+      while (i >= 0) {
+        const line = buffer.slice(0, i).trim();
+        buffer = buffer.slice(i + 1);
+        i = buffer.indexOf('\n');
+        if (!line) continue;
+        let msg;
+        try { msg = JSON.parse(line); } catch { continue; }
+        if (!settled) {
+          settled = true;
+          if (msg.ready) resolve({ ok: true, child, waiters });
+          else resolve({ ok: false, reason: msg.unavailable ?? 'the local planner did not become ready' });
+          continue;
+        }
+        const next = waiters.shift();
+        if (next) next(msg);
+      }
+    });
+  });
+  return session;
+}
+
+/**
+ * Reorder `options` by which is likeliest to lead to `goal`.
+ *
+ * @returns {Promise<string[]|null>} the caller's own order is correct when this
+ *   is null, which is every failure mode: flag off, no model, a timeout, a
+ *   parse problem, a paraphrasing answer. Never throws.
+ */
+export async function rank(goal, options, { timeoutMs = 3000, deviceOptions } = {}) {
+  if (!requested(deviceOptions)) return null;
+  if (!goal || !Array.isArray(options) || options.length < 2) return null;
+  let live;
+  try {
+    live = await open();
+  } catch {
+    return null;
+  }
+  if (!live?.ok) return null;
+  const answer = await new Promise((resolve) => {
+    // A timed-out waiter has to be *retired*, not merely resolved. Leaving it in
+    // the queue meant the next answer went to it instead of to the next asker,
+    // and every call after that was off by one — which showed up as an
+    // exploration run that never finished rather than as an error.
+    let done = false;
+    const waiter = (msg) => {
+      if (done) return;
+      done = true;
+      clearTimeout(timer);
+      resolve(msg);
+    };
+    const timer = setTimeout(() => {
+      if (done) return;
+      done = true;
+      const i = live.waiters.indexOf(waiter);
+      if (i >= 0) live.waiters.splice(i, 1);
+      resolve(null);
+    }, timeoutMs);
+    live.waiters.push(waiter);
+    try {
+      live.child.stdin.write(`${JSON.stringify({ goal: String(goal), options })}\n`);
+    } catch {
+      clearTimeout(timer);
+      resolve(null);
+    }
+  });
+  if (!answer?.order?.length) return null;
+  // It often returns a subset, so its order comes first and ours fills the tail.
+  // Trusting it to be exhaustive would silently drop candidates the budget was
+  // going to try.
+  const ranked = answer.order.filter((label) => options.includes(label));
+  const seen = new Set(ranked);
+  return [...ranked, ...options.filter((o) => !seen.has(o))];
+}
+
+/** For `doctor`: what the planner layer is, in one line. */
+export async function status(options) {
+  const want = requested(options);
+  if (!want) return { planner: 'none', detail: 'not requested (SIMFRAME_PLANNER is unset)' };
+  if (want !== 'apple') return { planner: 'none', detail: `no such planner backend: "${want}"` };
+  const live = await open();
+  if (!live?.ok) return { planner: 'none', detail: live?.reason ?? 'unavailable' };
+  return { planner: 'apple', detail: 'Apple Foundation Models, on-device, ranking only' };
+}
+
+/** Let a process exit without waiting on the helper. */
+export function close() {
+  try { session?.child?.kill(); } catch { /* already gone */ }
+  session = null;
+}

package/src/platform/android.js

@@ -183,7 +183,8 @@ async function resolveDevice(query, opts) {
         new Error(
           `${booted.length} emulators are running and none was named: ` +
             `${booted.map((d) => `${d.name} (${d.udid})`).join(', ')} — name one with --device, ` +
-            'or set SIMFRAME_DEVICE to pick a default for this shell',
+            'or set SIMFRAME_DEVICE to pick a default for this shell. Over MCP there is no shell: ' +
+            'pass "device" once on any call and the rest of the session remembers it',
         ),
         { ambiguous: true },
       );

package/src/platform/ios.js

@@ -87,7 +87,8 @@ async function resolveDevice(query, opts) {
         new Error(
           `${booted.length} simulators are booted and none was named: ` +
             `${booted.map((d) => `${d.name} (${d.udid})`).join(', ')} — name one with --device, ` +
-            'or set SIMFRAME_DEVICE to pick a default for this shell',
+            'or set SIMFRAME_DEVICE to pick a default for this shell. Over MCP there is no shell: ' +
+            'pass "device" once on any call and the rest of the session remembers it',
         ),
         { ambiguous: true },
       );

package/src/png.js

@@ -178,6 +178,32 @@ export function grayGrid(bmp, cols, rows) {
 }
 
 /** Nearest-neighbour scale. Only used for contact sheets, where speed beats quality. */
+/**
+ * A rectangle out of a bitmap, clamped to it.
+ *
+ * Exists because a whole screen at 1024px on the long edge cannot answer a
+ * question about one control. Reported from a real session: a selected filter
+ * chip and an unselected one are indistinguishable at that size, and selection
+ * state was the entire question the ticket turned on — so the agent shelled out
+ * to `simctl io` and PIL to crop and upscale the chip row, **for every single
+ * check**. Their estimate: six round trips.
+ *
+ * Coordinates are pixels; the caller converts from points, because only the
+ * caller knows the density it read them at.
+ */
+export function cropBitmap(bmp, x, y, width, height) {
+  const left = Math.max(0, Math.min(bmp.width - 1, Math.round(x)));
+  const top = Math.max(0, Math.min(bmp.height - 1, Math.round(y)));
+  const w = Math.max(1, Math.min(bmp.width - left, Math.round(width)));
+  const h = Math.max(1, Math.min(bmp.height - top, Math.round(height)));
+  const out = Buffer.alloc(w * h * 4);
+  for (let row = 0; row < h; row += 1) {
+    const from = ((top + row) * bmp.width + left) * 4;
+    bmp.data.copy(out, row * w * 4, from, from + w * 4);
+  }
+  return { width: w, height: h, data: out };
+}
+
 export function scaleBitmap(bmp, width, height) {
   const out = Buffer.allocUnsafe(width * height * 4);
   for (let y = 0; y < height; y++) {

package/src/refs.js

@@ -104,17 +104,46 @@ export function parseSelector(query) {
  * numbering introduces that labels do not have, and a ref resolved against the
  * wrong screen taps whatever now happens to sit at those coordinates.
  */
-export function resolveRef(udid, n, { structuralHash, layoutHash, screenKnown, tolerance = REF_TOLERANCE } = {}) {
+export function resolveRef(udid, n, { structuralHash, layoutHash, screenKnown, structuralDistance = 0, tolerance = REF_TOLERANCE } = {}) {
   const table = readRefs(udid);
   if (!table) throw new Error(`#${n} means nothing yet — read the screen first (sim_ui, or simframe ui)`);
-  const stale = (was, now) =>
-    `#${n} was numbered on a different screen (${was} → ${now}) — read the screen again before using refs`;
+  // The label this number was given to, when there is one. A stale ref is not
+  // nothing: the table records what it pointed at, which is enough for the
+  // caller to be offered the label instead of a bare refusal.
+  const labelFor = table.refs?.find((r) => r.ref === n)?.label ?? null;
+  // How long ago these numbers were handed out. Asked for by name: "refs
+  // expired (issued 4 calls ago) is actionable in a way this isn't".
+  const issued = Number.isFinite(table.at) ? ` refs were numbered ${Math.round((Date.now() - table.at) / 1000)}s ago;` : '';
+  // `staleKind` is the difference between "these numbers were drawn on a screen
+  // that has since shifted" and "you are somewhere else entirely", and only the
+  // first may be recovered by re-resolving the label the number stood for.
+  // Both wore the same flag once, and the caller re-resolved across an app
+  // switch: `#1` had been "Reminders" in Contacts, matched the status-bar
+  // back-to-app breadcrumb "• Reminders" at 0.64, and returned a tappable point
+  // in the status bar — a region the map itself refuses to offer. A refusal had
+  // become a confident wrong answer.
+  const staleError = (why, kind) => Object.assign(
+    new Error(`#${n} cannot be trusted here —${issued} ${why}. Read the screen again (sim_ui) to renumber`),
+    { staleRef: true, staleLabel: labelFor, staleKind: kind },
+  );
 
   // Structural identity first, because it is the question actually being asked:
   // is this the screen those numbers were assigned on? The caller gets it
   // cheaply — screen memory is a file read, not a perception pass.
-  if (table.structuralHash && structuralHash && table.structuralHash !== structuralHash) {
-    throw new Error(stale(table.structuralHash.slice(0, 8), structuralHash.slice(0, 8)));
+  //
+  // But only when the recall that produced it was exact. The identity arrives
+  // from `recallNearest`, which matches by layout within a tolerance so that a
+  // list with new rows stays one screen; above distance zero it is therefore a
+  // guess about *which* remembered screen this is, and a guess cannot be the
+  // sole reason to refuse. That mismatch was reported from the field as a
+  // refusal on unchanged state — the map had named the screen from the tolerant
+  // recall and printed the same header before and after, while this check read
+  // the same recall as exact and disagreed with it. Beyond distance zero the
+  // pixel backstop below is the one that decides, which is what it is for.
+  const exactRecall = structuralDistance === 0 || structuralDistance == null;
+  if (exactRecall && table.structuralHash && structuralHash && table.structuralHash !== structuralHash) {
+    throw staleError(`this is a different screen (${table.structuralHash.slice(0, 8)}`
+      + ` → ${structuralHash.slice(0, 8)})`, 'identity');
   }
   // Nothing recognises the screen we are on, so nothing can vouch for the
   // numbers. Refusing costs a re-read; guessing taps whatever is at those
@@ -128,9 +157,23 @@ export function resolveRef(udid, n, { structuralHash, layoutHash, screenKnown, t
   // other — measured: refs numbered on the springboard resolved happily on a
   // different screen because both hashes were degenerate. A hash with almost
   // no bits set is not evidence of anything.
-  if (layoutHash && table.layoutHash && informative(table.layoutHash) && informative(layoutHash)
-    && hashDistance(table.layoutHash, layoutHash) > tolerance) {
-    throw new Error(stale(table.layoutHash.slice(0, 8), layoutHash.slice(0, 8)));
+  // Reported three times in one session as `#4 was numbered on a different
+  // screen (03003714 → 03003714)` — a message that says the screen changed
+  // while showing that it did not, and left the reporter unable to tell a real
+  // move from a false positive. The cause was this branch printing eight
+  // characters of a **72-character** perceptual hash: its leading characters
+  // encode coarse structure, which is the very reason this comparison is a
+  // distance against a tolerance rather than an equality, so prefixes coincide
+  // routinely while the hashes differ. So this says the distance, and says that
+  // it is pixels rather than identity — a different thing from the branch above,
+  // which had been wearing the same sentence.
+  const drift = layoutHash && table.layoutHash && informative(table.layoutHash) && informative(layoutHash)
+    ? hashDistance(table.layoutHash, layoutHash)
+    : null;
+  if (drift != null && drift > tolerance) {
+    throw staleError(`the screen has moved too far from where these refs were numbered`
+      + ` (layout distance ${drift}, tolerance ${tolerance}) — the identity may be unchanged;`
+      + ' this is a pixel measurement, not a different screen', 'drift');
   }
   const hit = table.refs.find((r) => r.ref === n);
   if (!hit) {

package/src/regions.js

@@ -36,6 +36,8 @@ const STATUS_BAR_FRACTION = 0.065;
 
 /** Keyboards occupy the bottom of the screen and are unusually tall. */
 const KEYBOARD_MIN_FRACTION = 0.28;
+/** Most of a keyboard is keys. Below this it is a list that happens to be small. */
+const KEYBOARD_MIN_KEYISH = 0.6;
 
 /** Chrome is short. A 90pt list cell is not a tab item however low it sits. */
 const CHROME_MAX_HEIGHT_FRACTION = 0.075;
@@ -243,6 +245,50 @@ export function navSlot(frame, screen) {
  * common case and must stay cheap. This was the first band derived from the
  * elements rather than from a fraction, and it is the model the rest now follow.
  */
+/**
+ * Whether an element is shaped like a key rather than like content.
+ *
+ * The canonical version of this test, because two places need it and getting
+ * them out of step is what produced the bug below. A key is finger-sized and
+ * says almost nothing: a single character, a short named key, or nothing at
+ * all. A row of content is wider, or carries words.
+ */
+export const KEY_MAX_WIDTH = 120;
+
+const NAMED_KEY = /^(space|return|enter|shift|delete|backspace|done|globe|dictate|emoji|caps ?lock|number|numbers|symbols|letters|more|search|go|send|join|route|abc|123)$/i;
+
+export function looksLikeKey(t) {
+  if (/^key$/i.test(String(t?.type ?? ''))) return true;
+  const width = t?.frame?.width;
+  if (Number.isFinite(width) && width > KEY_MAX_WIDTH) return false;
+  const label = String(t?.label ?? '').trim();
+  if (!label) return true;
+  if (label.length <= 2) return true;
+  return NAMED_KEY.test(label);
+}
+
+/**
+ * Where the software keyboard starts, or null.
+ *
+ * Size and uniformity alone were not enough, and the failure was expensive. A
+ * read-only summary screen stacks a dozen short text rows of near-identical
+ * height in the bottom half — which satisfied every test here, so a keyboard
+ * was detected on a screen that had none.
+ *
+ * That mattered far beyond a mislabelled band, because `fingerprint.tokens`
+ * discards everything below `keyboardTop`. A phantom keyboard therefore
+ * deleted the screen's entire content from its own identity, leaving only
+ * chrome — so a wizard's form step and its read-only review screen, which
+ * share a nav title and a step indicator, **collapsed onto one hash**. From
+ * there: the graph offered one screen's remembered controls on the other (three
+ * absent controls, one of them beside a button that submits for real), and
+ * `locate` resolved against the wrong screen's stored element list, which is
+ * why `assert` insisted a string was absent while the map printed it four lines
+ * below. One phantom, three findings.
+ *
+ * So the test is now what a keyboard actually is: keys. A dozen small uniform
+ * boxes are a keyboard only if most of them are key-shaped.
+ */
 export function detectKeyboardTop(elements, screen) {
   if (!screen?.height || elements.length < 12) return null;
   const threshold = screen.height * (1 - KEYBOARD_MIN_FRACTION);
@@ -253,7 +299,70 @@ export function detectKeyboardTop(elements, screen) {
   // Keys are small and uniform; a list of cells down there is not.
   const uniform = heights.filter((h) => Math.abs(h - median) <= Math.max(3, median * 0.4)).length;
   if (uniform / low.length < 0.7 || median > screen.height * 0.07) return null;
-  return Math.min(...low.map((e) => e.frame.y));
+  // And they are keys. Uniformity says "a grid of something"; this says of what.
+  const keyish = low.filter(looksLikeKey).length;
+  if (keyish / low.length < KEYBOARD_MIN_KEYISH) return null;
+  return extendKeyboardUp(elements, Math.min(...low.map((e) => e.frame.y)), median);
+}
+
+/**
+ * Walk the boundary up through rows that are still keys.
+ *
+ * `KEYBOARD_MIN_FRACTION` is a **detection window**, not the keyboard's height,
+ * and using its edge as the boundary cut the keyboard's own top row off.
+ * Measured on a recorded iPhone 17 Pro screen with the software keyboard up: the
+ * window starts at y=629, the `q`–`p` row's frame top is **590**, so that entire
+ * row was excluded and the boundary landed on the `a` row at 644 — ten keys
+ * reported as page content, in the same map that said `keyboard up`.
+ *
+ * Widening the window instead would be the wrong fix: 0.28 of the screen is
+ * deliberately conservative so a list of short rows at the bottom of a page
+ * cannot be mistaken for a keyboard, and a real keyboard is nearer 0.38. So the
+ * window still *decides*, and this extends the boundary only while the rows
+ * above keep being key-shaped — which page content is not.
+ *
+ * The concrete cost of not having this: a sweep gesture aimed 8pt above the
+ * boundary still landed on the top row of keys and scrolled nothing.
+ */
+function extendKeyboardUp(elements, top, median) {
+  let boundary = top;
+  // Four rows is a full keyboard's worth; the loop stops on its own long before
+  // that on anything that is not one.
+  for (let i = 0; i < 4; i += 1) {
+    const row = (elements ?? []).filter((e) => e.frame
+      && looksLikeKey(e)
+      && Math.abs(heightOf(e.frame) - median) <= Math.max(3, median * 0.4)
+      // Sitting directly on the current boundary, within one row's height.
+      && e.frame.y + heightOf(e.frame) <= boundary + 4
+      && e.frame.y + heightOf(e.frame) >= boundary - median * 1.6);
+    if (row.length < 5) break;
+    const next = Math.min(...row.map((e) => e.frame.y));
+    if (!(next < boundary)) break;
+    boundary = next;
+  }
+  return boundary;
+}
+
+/**
+ * Is this element outside the viewport?
+ *
+ * **Both axes.** Every filter in this project checked `y` and ignored `x`,
+ * which is fine until a horizontal row: a filter chip reported at **x=422 on a
+ * 402pt-wide screen** counted as visible, and `scroll_to` then said *"'Assigned
+ * to Me' is in view at 422,277 already"* — confidently wrong about the one thing
+ * it exists to answer. Off-screen chips came back at **x=-247** the same way.
+ *
+ * Reported as the most expensive finding of an agent's session, and the cost was
+ * not the wrong answer itself: it was that the wrong answer was *confident*, so
+ * the recovery was hand-tuned swipes and two overshoots.
+ */
+export function offViewport(t, screen) {
+  if (!t) return false;
+  const w = screen?.width;
+  const h = screen?.height;
+  if (Number.isFinite(h) && (t.y < 0 || t.y > h)) return true;
+  if (Number.isFinite(w) && (t.x < 0 || t.x > w)) return true;
+  return false;
 }
 
 /** Annotate a target list with region and nav slot. Mutates and returns it. */