simframe 0.4.2 → 0.6.0-rc.1

package/src/index.js

@@ -4,6 +4,7 @@ import fs from 'node:fs';
 import path from 'node:path';
 import { fileURLToPath } from 'node:url';
 import { DEFAULTS, STATE_VERSION } from './daemon.js';
+import * as engine from './engine.js';
 import { decodePng, encodePng, scaleBitmap } from './png.js';
 import {
   REGION_COLS,
@@ -13,6 +14,10 @@ import {
   signatureDiff,
 } from './analyze.js';
 import * as input from './input.js';
+import * as fingerprint from './fingerprint.js';
+import * as graph from './graph.js';
+import * as matching from './matching.js';
+import * as refs from './refs.js';
 import * as screenmap from './screenmap.js';
 import { resolveDevice, resize, screenshot } from './simctl.js';
 import * as store from './store.js';
@@ -21,7 +26,24 @@ const HERE = path.dirname(fileURLToPath(import.meta.url));
 const CLI = path.join(HERE, 'cli.js');
 const sleep = (ms) => new Promise((r) => setTimeout(r, ms));
 
-export const DETAIL_LEVELS = { low: 420, normal: 700, high: 1100, full: 0 };
+export const DETAIL_LEVELS = { low: 420, normal: 700, high: 1024, full: 0 };
+
+/**
+ * The ceiling on any image handed to a model.
+ *
+ * An image costs ~1,600 tokens when Claude Code handles it as a native image
+ * block, and 15,000–25,000 when the base64 is treated as text (claude-code
+ * issue #31208) — enough to trip the 25,000-token tool-result limit on its own.
+ * A native-resolution frame buys nothing at either price: 1024 px on the long
+ * edge is already more than a 393-point screen has to say. Only the CLI, which
+ * writes to a file rather than into a context window, may exceed it.
+ */
+export const MODEL_MAX_IMAGE_DIM = 1024;
+
+export function modelDetail(detail) {
+  const dim = resolveMaxDim(detail);
+  return dim === 0 || dim > MODEL_MAX_IMAGE_DIM ? MODEL_MAX_IMAGE_DIM : dim;
+}
 
 export function resolveMaxDim(detail) {
   if (typeof detail === 'number') return detail;
@@ -37,6 +59,22 @@ export function resolveMaxDim(detail) {
 async function fullFrameFor(udid, state) {
   if (state.fullFile && fs.existsSync(state.fullFile)) return state.fullFile;
   const p = store.paths(udid);
+  // The pointer in state.json can name a frame retention has already thinned
+  // away, and it does so routinely — state named full/2475.png while the
+  // directory held 2470, 2869 and 2870. Falling straight through to simctl
+  // meant OCR quietly shelled out for a screenshot on a machine whose daemon
+  // was capturing full frames the whole time: slower, and it made the whole
+  // step fail on a runner where that shell-out did not work.
+  try {
+    const newest = fs.readdirSync(p.full)
+      .filter((f) => f.endsWith('.png'))
+      .map((f) => ({ f, seq: Number.parseInt(f, 10) }))
+      .filter((x) => Number.isFinite(x.seq))
+      .sort((a, b) => b.seq - a.seq)[0];
+    if (newest) return path.join(p.full, newest.f);
+  } catch {
+    /* no full directory yet; fall through */
+  }
   const file = path.join(p.dir, 'ocr-source.png');
   await screenshot(udid, file, { mask: 'ignored' });
   return file;
@@ -54,11 +92,16 @@ async function deviceGeometry(udid, state) {
   } catch {
     /* idb absent: fall through */
   }
+  // Last resort only. These numbers are an iPhone 17 Pro, so on anything else —
+  // an iPad especially — they are silently wrong, and every tap point derived
+  // from them lands in the wrong place. screenInfo asks the daemon first now,
+  // so reaching here means neither the daemon nor idb could answer.
   const density = 3;
   return {
     density,
     pointWidth: Math.round((state.width * (state.nativeScale ?? 1)) / 1) || 402,
     pointHeight: Math.round((state.height * (state.nativeScale ?? 1)) / 1) || 874,
+    guessed: true,
   };
 }
 
@@ -89,7 +132,7 @@ export async function ensureDaemon(deviceQuery, options = {}) {
   if (!existing.alive) {
     if (acquireSpawnLock(p.lock)) {
       try {
-        spawnDaemon(device.udid, options);
+        await startEngine(device.udid, options);
       } finally {
         // Hold the lock briefly so a burst of callers does not double-spawn.
         setTimeout(() => releaseSpawnLock(p.lock), 1500).unref?.();
@@ -97,7 +140,8 @@ export async function ensureDaemon(deviceQuery, options = {}) {
     }
   }
 
-  const deadline = Date.now() + (options.readyTimeoutMs ?? 8000);
+  // The daemon may need a first build, which is slower than a spawn.
+  const deadline = Date.now() + (options.readyTimeoutMs ?? 20_000);
   while (Date.now() < deadline) {
     const state = store.readJson(p.state);
     if (state && state.capturedAt >= minCapturedAt && Date.now() - state.capturedAt < 30_000) {
@@ -109,7 +153,63 @@ export async function ensureDaemon(deviceQuery, options = {}) {
   throw new Error(`simframe daemon did not produce a frame for ${device.name}${tail ? `\n${tail}` : ''}`);
 }
 
-function spawnDaemon(udid, options) {
+/** Why the daemon was not used, when it was not. Surfaced by doctor. */
+export let engineFallbackReason = null;
+
+/**
+ * Degrading has to announce itself.
+ *
+ * "Degrade rather than fail" is the right policy and it nearly sank the tool
+ * twice: a file missing from the published package made every install fall back
+ * to the simctl engine, and OCR ship disabled, both **silently**. Tests passed,
+ * CI passed, nothing printed. The failure was not the missing file; it was that
+ * the degradation was invisible.
+ *
+ * So the reason is written next to the device's state, not just held in this
+ * process's memory — otherwise a later `doctor` or `start` sees `engine=simctl`
+ * with no explanation, because the process that chose it has exited.
+ */
+function recordFallback(udid, reason) {
+  const file = path.join(store.deviceDir(udid), 'engine-fallback.json');
+  try {
+    if (reason) store.writeAtomic(file, JSON.stringify({ reason, at: Date.now() }));
+    else fs.rmSync(file, { force: true });
+  } catch {
+    // Never let bookkeeping stop capture from starting.
+  }
+}
+
+/** Why the running engine is not simframed, if it is not. Survives the process that chose it. */
+export function fallbackReason(udid) {
+  if (engineFallbackReason) return engineFallbackReason;
+  return store.readJson(path.join(store.deviceDir(udid), 'engine-fallback.json'))?.reason ?? null;
+}
+
+/**
+ * Start whichever engine was asked for.
+ *
+ * simframed unless told otherwise: it reads the framebuffer directly and is
+ * roughly thirty times faster per frame. The simctl loop stays reachable with
+ * `engine: 'simctl'`, and is used automatically when the daemon cannot be
+ * built — a machine with no Swift toolchain still has to work.
+ */
+async function startEngine(udid, options) {
+  if ((options.engine ?? 'simframed') === 'simframed') {
+    const built = await engine.ensureBuilt();
+    if (built.ok) {
+      engineFallbackReason = null;
+      recordFallback(udid, null);
+      engine.spawnDaemon(udid, options);
+      return 'simframed';
+    }
+    engineFallbackReason = built.reason ?? 'simframed unavailable';
+    recordFallback(udid, engineFallbackReason);
+  }
+  spawnNodeDaemon(udid, options);
+  return 'simctl';
+}
+
+function spawnNodeDaemon(udid, options) {
   const args = [CLI, 'daemon', udid];
   for (const key of ['fps', 'maxDim', 'ringSize', 'idleExitMs']) {
     if (options[key] != null) args.push(`--${key}=${options[key]}`);
@@ -164,6 +264,21 @@ function readLogTail(file, lines = 6) {
 /** A frame this old means the capture loop is wedged, not that the screen is calm. */
 export const STALE_FRAME_MS = 2500;
 
+/**
+ * How still the screen must be before a frame may be used to key screen memory.
+ *
+ * A screen map describes a screen, so it must be built from a frame that shows
+ * one — not from the middle of a transition, where the layout belongs to
+ * neither the screen you left nor the one you are arriving at. Without this,
+ * the capture rate leaks into the hit rate: a faster loop samples more
+ * transitional frames and remembers more layouts that will never recur.
+ *
+ * Phase 4 replaces this with the real settle detector, which can tell a
+ * spinner from a still screen. Until then, "nothing moved for a while" is
+ * enough to decouple memory from frame rate.
+ */
+export const MEMORY_SETTLE_MS = 250;
+
 /** Below this a "change" is a clock digit or a caret, not a new screen. */
 export const MINOR_CHANGE = 0.004;
 export const MAJOR_CHANGE = 0.03;
@@ -180,7 +295,17 @@ export function liveness(udid, state) {
   if (!running) {
     return { ok: false, ageMs, note: 'the capture loop has died; the frame you are looking at is the last one it wrote' };
   }
-  if (ageMs > STALE_FRAME_MS) {
+  // Frame age means "stalled" only for a fixed-rate loop.
+  //
+  // simframed captures on damage, so a screen that is genuinely still produces
+  // no frames at all — which is precisely the state `settle` exists to detect.
+  // Treating that as a stall made a flow fail on a static page with "capture
+  // loop is stalled: newest frame is 2984ms old" immediately after a step had
+  // succeeded. The pid check above is the honest liveness signal for this
+  // engine; the heartbeat file cannot help, because clients write it, not the
+  // daemon.
+  const damageDriven = engine.runningEngine(udid) === 'simframed';
+  if (!damageDriven && ageMs > STALE_FRAME_MS) {
     return { ok: false, ageMs, note: `capture loop is stalled: newest frame is ${ageMs}ms old` };
   }
   return { ok: true, ageMs, note: null };
@@ -577,15 +702,79 @@ export async function getFrameAt(deviceQuery, { msAgo = 0, options } = {}) {
  * A screen seen for the first time pays once to build its map, and every later
  * visit is a file read.
  */
-export async function locate(deviceQuery, query, { index, refresh = false, useAx = true, useOcr = true, options } = {}) {
-  const { device, state } = await ensureDaemon(deviceQuery, options);
+/**
+ * Wait, briefly, for a frame that is holding still. Returns whatever the newest
+ * frame is once the screen settles or the budget runs out, saying which.
+ */
+export async function settledState(udid, { settleMs = MEMORY_SETTLE_MS, timeoutMs = 1500 } = {}) {
+  const p = store.paths(udid);
+  const deadline = Date.now() + timeoutMs;
+  let state = store.readJson(p.state);
+  while (Date.now() < deadline) {
+    state = store.readJson(p.state) ?? state;
+    // The daemon runs a real settle detector that can tell a spinner from a
+    // still screen. Prefer it; the duration check is the fallback for the
+    // simctl engine, which has no such thing.
+    if (state?.settled === true) return { state, settled: true };
+    if (state && state.settled === undefined && state.stableForMs >= settleMs) {
+      return { state, settled: true };
+    }
+    await sleep(40);
+  }
+  return { state, settled: false };
+}
+
+export async function locate(
+  deviceQuery,
+  query,
+  { index, refresh = false, useAx = true, useOcr = true, settleMs = MEMORY_SETTLE_MS, options } = {},
+) {
+  const { device, state: firstState } = await ensureDaemon(deviceQuery, options);
   const udid = device.udid;
+
+  // Selectors resolve before any perception happens: `#3` is already an answer
+  // somebody numbered, and `@x,y` was never a question about the screen.
+  const selector = refs.parseSelector(query);
+  if (selector.kind === 'point') {
+    return {
+      device,
+      state: firstState,
+      target: { label: `(${selector.x},${selector.y})`, x: selector.x, y: selector.y, source: 'coordinates' },
+      from: 'selector',
+      distance: 0,
+      settled: true,
+    };
+  }
+  if (selector.kind === 'ref') {
+    // Screen memory answers "which screen is this?" from a file, so a ref can
+    // be checked against structural identity without paying for a perception
+    // pass — which is the whole reason a ref exists.
+    const near = screenmap.recallNearest(udid, firstState.layoutHash);
+    const hit = refs.resolveRef(udid, selector.ref, {
+      layoutHash: firstState.layoutHash,
+      structuralHash: near?.entry?.structuralHash ?? null,
+      screenKnown: Boolean(near),
+    });
+    return {
+      device,
+      state: firstState,
+      target: { ...hit, label: hit.label ?? `#${hit.ref}`, source: hit.source ?? 'ref' },
+      from: 'ref',
+      distance: 0,
+      settled: true,
+    };
+  }
+  if (selector.exact) query = selector.label;
+  // Key memory off a settled frame, never off whichever frame happened to be
+  // newest, so the capture rate cannot change what gets remembered.
+  const { state, settled } = await settledState(udid, { settleMs });
+  const current = state ?? firstState;
   let entry = null;
   let from = 'memory';
   let distance = 0;
 
   if (!refresh) {
-    const near = screenmap.recallNearest(udid, state.layoutHash);
+    const near = screenmap.recallNearest(udid, current.layoutHash);
     if (near) {
       entry = near.entry;
       distance = near.distance;
@@ -593,34 +782,59 @@ export async function locate(deviceQuery, query, { index, refresh = false, useAx
   }
 
   if (!entry) {
-    const geo = await deviceGeometry(udid, state);
+    const geo = await deviceGeometry(udid, current);
     entry = await screenmap.build(udid, {
-      hash: state.hash,
-      layoutHash: state.layoutHash,
-      fullFrame: await fullFrameFor(udid, state),
+      hash: current.hash,
+      layoutHash: current.layoutHash,
+      fullFrame: await fullFrameFor(udid, current),
       density: geo.density,
       screen: { width: geo.pointWidth, height: geo.pointHeight },
       useAx,
       useOcr,
+      // A map built while the screen was moving describes nothing that will
+      // recur, so it is used for this call and then thrown away.
+      persist: settled,
     });
-    from = 'built';
+    from = settled ? 'built' : 'built-unsettled';
   }
 
-  const candidates = screenmap.rank(entry, query);
-  if (candidates.length > 1 && index == null) {
-    const top = candidates[0];
-    const second = candidates[1];
-    const decisive = screenmap.isInteractive(top) && !screenmap.isInteractive(second);
-    if (!decisive) {
-      const list = candidates
-        .slice(0, 6)
-        .map((t, i) => `[${i}] "${t.label}" (${t.x},${t.y}) ${t.type}/${t.source}`)
+  const screenSize = { width: current.width, height: current.height };
+  const geo = await deviceGeometry(udid, current);
+  const points = { width: geo.pointWidth, height: geo.pointHeight };
+
+  // Intent resolution rather than string matching: it understands verbs
+  // ("tap Save"), typos, icon-only controls by synonym ("back"), and where on
+  // screen the caller meant ("Assets tab").
+  if (index == null) {
+    const outcome = matching.resolve(entry.targets, query, { screen: points });
+    if (outcome.status === 'ambiguous') {
+      const list = outcome.alternatives
+        .map((a, i) => `[${i}] "${a.label}" (${a.x},${a.y}) ${a.region ?? 'content'} ${a.score}`)
         .join(', ');
       throw new Error(
-        `"${query}" matches ${candidates.length} things on this screen — pass index to choose: ${list}`,
+        `"${query}" matches ${outcome.alternatives.length} things on this screen — say which, or pass index: ${list}`,
       );
     }
+    if (outcome.status === 'ok') {
+      return {
+        device, state: current, entry, target: outcome.target, from, distance, settled,
+        score: outcome.score, reasons: outcome.reasons, alternatives: outcome.alternatives,
+        screens: screenmap.stats(udid).screens,
+      };
+    }
+    // Nothing scored well enough. Falling through to plain substring matching
+    // here undoes every guard above — it has no off-screen filter and no
+    // coverage weighting, and it is what returned a scrolled-away list row for
+    // "back". "Not found" is the correct answer.
+    const sample = entry.targets
+      .filter((t) => t.label && t.y >= 0 && t.y <= points.height)
+      .slice(0, 12)
+      .map((t) => t.label.slice(0, 24))
+      .join(', ');
+    throw new Error(`"${query}" is not on this screen. Visible: ${sample || '(nothing readable)'}`);
   }
+
+  const candidates = screenmap.rank(entry, query);
   const target = index != null ? candidates[index] : candidates[0];
   if (!target) {
     const sample = entry.targets
@@ -632,7 +846,120 @@ export async function locate(deviceQuery, query, { index, refresh = false, useAx
       `"${query}" is not on this screen. Visible: ${sample || '(nothing readable)'}`,
     );
   }
-  return { device, state, entry, target, from, distance, screens: screenmap.stats(udid).screens };
+  return { device, state: current, entry, target, from, distance, settled, screens: screenmap.stats(udid).screens };
+}
+
+/**
+ * What screen this is, structurally.
+ *
+ * Uses the screen map — recalled when the pixel index finds one, built when it
+ * does not. The pixel hash is what makes the lookup cheap; the structural hash
+ * is what makes the answer right.
+ */
+/**
+ * Structural settle: how long to let a screen finish arriving before believing
+ * a fingerprint that nothing recognises.
+ */
+export const STRUCTURAL_SETTLE_MS = 300;
+
+/**
+ * How long identity will wait for pixels to go quiet.
+ *
+ * Not the caller's timeout. A flow allows twelve seconds for a screen to
+ * arrive, but some screens never report settled at all — live content, a
+ * looping animation — and spending the flow's whole budget waiting for a flag
+ * that will not come cost 53s on a tour that had taken 7s. Long enough to
+ * outlast a normal transition, short enough that a screen which never settles
+ * is cheap to give up on.
+ */
+export const IDENTITY_SETTLE_TIMEOUT_MS = 2500;
+const STRUCTURAL_SETTLE_SAMPLES = 3;
+
+/**
+ * What screen is this?
+ *
+ * `settled` is a question about pixels, and it is answered before a screen has
+ * necessarily finished arriving: a list whose spinner has gone but whose rows
+ * have not landed is perfectly still and structurally wrong. Measured, that put
+ * one screen at 17 tokens on one visit and 7 on the next, which is the whole
+ * reason the same-screen floor sits at 0.41 instead of somewhere comfortable.
+ *
+ * So structural identity gets a structural settle of its own — but only where
+ * it costs something worth paying for. A fingerprint that matches a screen we
+ * already know is taken at face value; the risk is not that we mislabel a known
+ * screen, it is that a half-drawn one becomes a new node nobody can navigate
+ * to. Novel fingerprints, and only those, are re-sampled until two consecutive
+ * readings agree.
+ */
+export async function screenIdentity(deviceQuery, { options, confirmNovel = true, settleMs, timeoutMs, fresh = false } = {}) {
+  const { device, state } = await ensureDaemon(deviceQuery, options);
+  const udid = device.udid;
+
+  const read = async ({ fresh = false } = {}) => {
+    // Settle with the caller's patience, not a default. settledState times out
+    // at 1.5s on its own, so inside a flow that allows twelve seconds this was
+    // calling a screen unsettled while the flow was still happily waiting for
+    // it — and an unsettled screen records no edge, so the graph learned
+    // nothing and every later step read `unverified`.
+    const { state: settledFrame, settled } = await settledState(udid, {
+      settleMs,
+      timeoutMs: Math.min(timeoutMs ?? IDENTITY_SETTLE_TIMEOUT_MS, IDENTITY_SETTLE_TIMEOUT_MS),
+    });
+    const current = settledFrame ?? state;
+    let entry = fresh ? null : screenmap.recallNearest(udid, current.layoutHash)?.entry;
+    const geo = await deviceGeometry(udid, current);
+    if (!entry) {
+      entry = await screenmap.build(udid, {
+        hash: current.hash,
+        layoutHash: current.layoutHash,
+        fullFrame: await fullFrameFor(udid, current),
+        density: geo.density,
+        screen: { width: geo.pointWidth, height: geo.pointHeight },
+        persist: settled,
+      });
+    }
+    return {
+      hash: entry.structuralHash,
+      tokens: entry.structuralTokens ?? [],
+      keyboard: Boolean(entry.keyboard),
+      layoutHash: current.layoutHash,
+      settled,
+      // Carried out so callers that want the elements as well as the identity
+      // do not pay for a second perception pass to get them. The compact
+      // screen map needs both, and reading twice was the whole cost of it.
+      entry,
+      state: current,
+      points: { width: geo.pointWidth, height: geo.pointHeight },
+    };
+  };
+
+  let identity = await read({ fresh });
+  if (!confirmNovel) return { ...identity, confirmed: identity.settled };
+  // Being recognised is stronger evidence than the pixel settle flag: a
+  // fingerprint that matches a screen already trusted has nothing left to
+  // prove, and some screens (live content, a looping animation) never report
+  // settled at all. The gate exists to stop a half-drawn screen becoming a new
+  // node — not to re-interrogate a known one.
+  if (graph.nearestScreen(udid, identity)) return { ...identity, confirmed: true, known: true };
+
+  // Nothing recognises this, or the pixels have not gone quiet. Either way, make
+  // it prove it is the same screen twice running before it becomes a node.
+  for (let i = 1; i < STRUCTURAL_SETTLE_SAMPLES; i += 1) {
+    await sleep(STRUCTURAL_SETTLE_MS);
+    const again = await read({ fresh: true });
+    // Two readings agree if they are the same screen — the same test identity
+    // itself uses. Demanding an identical hash is a stricter question than the
+    // one being asked, and a row a grid-unit wider fails it.
+    const agrees = again.hash === identity.hash
+      || fingerprint.similarity(again.tokens, identity.tokens) >= graph.SIMILARITY_THRESHOLD;
+    if (agrees) {
+      return { ...again, confirmed: true, known: Boolean(graph.nearestScreen(udid, again)) };
+    }
+    identity = again;
+  }
+  // Still moving structurally. Report the latest reading and say it is unproven,
+  // so callers can decline to record an edge to a screen that never held still.
+  return { ...identity, confirmed: false, known: false };
 }
 
 export { DEFAULTS, screenmap, store };