simframe 0.18.0 → 0.19.0

package/src/index.js

@@ -1397,22 +1397,76 @@ export async function getFrameAt(deviceQuery, { msAgo = 0, options } = {}) {
  * Wait, briefly, for a frame that is holding still. Returns whatever the newest
  * frame is once the screen settles or the budget runs out, saying which.
  */
-export async function settledState(udid, { settleMs = MEMORY_SETTLE_MS, timeoutMs = 1500 } = {}) {
+/**
+ * When the stillness a state reports began, or null when that cannot be told.
+ *
+ * `stableForMs` is measured as of the frame's capture, so the quiet period
+ * started `stableForMs` before `capturedAt`. Exported because the rule built on
+ * it below is the interesting part and deserves to be testable without a device.
+ */
+export function stillnessBegan(state) {
+  if (!Number.isFinite(state?.capturedAt) || !Number.isFinite(state?.stableForMs)) return null;
+  return state.capturedAt - state.stableForMs;
+}
+
+/**
+ * Has this screen been still *since we acted*, or was it still before we did?
+ *
+ * The distinction the settle detector was missing. It answers "how long has the
+ * screen been quiet" honestly and has no idea that the quiet it is describing
+ * belongs to the screen the caller has just left.
+ *
+ * Only a stillness we can **prove** predates the action is rejected. When the
+ * timestamps are missing this returns true, which is the previous behaviour —
+ * this may only ever add refusals it can demonstrate, never turn an unknown
+ * into a wait.
+ */
+export function stillSinceActing(state, actedAt) {
+  if (!Number.isFinite(actedAt)) return true;
+  const began = stillnessBegan(state);
+  if (began == null) return true;
+  return began >= actedAt;
+}
+
+export async function settledState(udid, { settleMs = MEMORY_SETTLE_MS, timeoutMs = 1500, since } = {}) {
   const p = store.paths(udid);
   const deadline = Date.now() + timeoutMs;
+  // What we are settling *after*. A caller that knows may say; otherwise it is
+  // the last launch, openUrl or gesture this device was given.
+  const actedAt = since ?? store.lastActionAt(udid);
   let state = store.readJson(p.state);
+  let stale = false;
   while (Date.now() < deadline) {
     state = store.readJson(p.state) ?? state;
-    // The daemon runs a real settle detector that can tell a spinner from a
-    // still screen. Prefer it; the duration check is the fallback for the
-    // simctl engine, which has no such thing.
-    if (state?.settled === true) return { state, settled: true };
-    if (state && state.settled === undefined && state.stableForMs >= settleMs) {
-      return { state, settled: true };
+    // **Stillness from before the action is not settlement.** Measured on the
+    // bench device: 283ms after a Settings launch the state said `settled` with
+    // `stableForMs: 4427` over a screen holding **zero** elements, which filled
+    // 2.3 seconds later; 408ms after `tap General` it said `settled` with
+    // `stableForMs: 7753` over the 23 elements of Settings root. Both readings
+    // were honest about the number and wrong about the screen.
+    //
+    // Field-reported independently as the highest-priority class here: a read
+    // that returns chrome with the content missing and no loading marker is
+    // indistinguishable from a screen that is genuinely empty, so an agent
+    // reports "this filter returns zero results" and means it.
+    const fresh = stillSinceActing(state, actedAt);
+    if (!fresh) stale = true;
+    else {
+      // The daemon runs a real settle detector that can tell a spinner from a
+      // still screen. Prefer it; the duration check is the fallback for the
+      // simctl engine, which has no such thing.
+      if (state?.settled === true) return { state, settled: true, waitedForAction: stale };
+      if (state && state.settled === undefined && state.stableForMs >= settleMs) {
+        return { state, settled: true, waitedForAction: stale };
+      }
     }
     await sleep(40);
   }
-  return { state, settled: false };
+  // `settled: false` is not a failure and never was — callers use the state and
+  // decline to persist a map built from it. What is new is that this can now be
+  // false because the screen has not been seen to move since the action, which
+  // is a different and more honest reason than "it is still moving".
+  return { state, settled: false, ...(stale ? { stillnessPredatesAction: true } : {}) };
 }
 
 /**
@@ -1459,6 +1513,33 @@ export function offScreenMatch(targets, query, points) {
   return hit.status === 'ambiguous' && hit.alternatives?.length ? hit.alternatives[0] : null;
 }
 
+/**
+ * Which way a target is outside the viewport, and by which measure.
+ *
+ * `offViewport` has checked both axes since it was written; the sentence
+ * reporting it only ever named `y` and the screen *height*. So a tab in a
+ * horizontally-scrolling strip was reported as *"it is at y=143 on a 874pt
+ * screen"* — a coordinate plainly inside the screen, next to a conclusion that
+ * it is not in view, with four of its siblings visible. Field-reported, and the
+ * reporter's summary is the right one: the honest half was right and only the
+ * axis was wrong.
+ *
+ * Vertical is checked first because it is overwhelmingly the common case, and
+ * because `scrollTo` reasons vertically — a caller told "below the fold" and a
+ * caller told "past the right edge" do different things next, which is the
+ * whole reason to say which.
+ */
+export function offScreenAxis(target, points) {
+  const { width, height } = points ?? {};
+  if (Number.isFinite(height) && (target.y < 0 || target.y > height)) {
+    return { axis: 'y', at: Math.round(target.y), extent: Math.round(height), edge: target.y < 0 ? 'above' : 'below' };
+  }
+  if (Number.isFinite(width) && (target.x < 0 || target.x > width)) {
+    return { axis: 'x', at: Math.round(target.x), extent: Math.round(width), edge: target.x < 0 ? 'left of' : 'right of' };
+  }
+  return null;
+}
+
 /**
  * Which sensors a read asks for by default.
  *
@@ -1500,9 +1581,52 @@ export function sensorMode(options) {
  */
 const nameFor = (t) => t.label || t.identifier || null;
 
+/**
+ * Did screen memory alone produce this miss?
+ *
+ * A recall that finds the screen and not the target tags `ambiguous_intent`
+ * without `ambiguous` — "I know this screen, and what you asked for is not on
+ * it". That is the only miss worth re-asking, because it is the only one whose
+ * answer came from a file rather than from the device. A miss off a map that
+ * was just built has already read both sensors, and a genuine ambiguity is not
+ * settled by reading again.
+ */
+export function memoryMiss(err) {
+  const why = metrics.escalationOf(err);
+  return Boolean(why && why.reason === 'ambiguous_intent' && !why.ambiguous);
+}
+
 export async function locate(deviceQuery, query, opts = {}) {
   if (sensorMode(opts.options) !== 'ax-first' || opts.useOcr === false || opts.escalated) {
-    return locateWith(deviceQuery, query, opts);
+    if (opts.refresh || opts.escalated) return locateWith(deviceQuery, query, opts);
+    try {
+      return await locateWith(deviceQuery, query, opts);
+    } catch (err) {
+      // **Memory may confirm, never deny.** Measured on the bench device, 8 of
+      // 8 and 4 of 4 in two separate runs: at the instant a flow's next step
+      // asks, a recall says "not on this screen" in 25 ms and a fresh read
+      // finds the target 1.8 s later, on the same device, without anything
+      // touching it in between.
+      //
+      // Why the recall is wrong, and it is not staleness in the usual sense:
+      // the frame it keys on was captured 47-124 ms *after* the tap — the first
+      // frame of the push animation, which still looks like the screen being
+      // left. Capture is damage-driven, so that frame then goes still,
+      // `settledState` calls it settled at 500-700 ms of stillness, and
+      // `recallNearest` — deliberately tolerant, because a list with new rows is
+      // still the same screen — matches it back to the previous screen and
+      // answers out of that screen's stored element list. The screen itself
+      // arrives about 750 ms later.
+      //
+      // The cost is paid only on a miss, which today aborts the batch and buys
+      // a ~20 s model round trip. 1.8 s to be sure is the cheaper mistake. The
+      // hit path — the one the speed argument rests on — is untouched.
+      //
+      // This recovery already existed for `ax-first` (below) and had never run
+      // in the default sensor mode, which is `full`.
+      if (!memoryMiss(err)) throw err;
+      return locateWith(deviceQuery, query, { ...opts, refresh: true, escalated: true });
+    }
   }
   const options = opts;
   try {
@@ -1723,8 +1847,12 @@ async function locateWith(
     if (offScreen) {
       throw metrics.tag(
         new Error(
-          `"${query}" is in the tree but not in view — it is at y=${Math.round(offScreen.y)}`
-          + ` on a ${Math.round(points.height)}pt screen. Scroll to it (sim_scroll_to) rather than waiting;`
+          `"${query}" is in the tree but not in view — ${(() => {
+            const off = offScreenAxis(offScreen, points);
+            if (!off) return `it is at ${Math.round(offScreen.x)},${Math.round(offScreen.y)}`;
+            return `it is ${off.edge} the viewport, at ${off.axis}=${off.at}`
+              + ` on a ${off.extent}pt ${off.axis === 'y' ? 'tall' : 'wide'} screen`;
+          })()}. Scroll to it (sim_scroll_to) rather than waiting;`
           + ' waiting cannot bring it into view.',
         ),
         from === 'memory' ? 'ambiguous_intent' : 'unknown_screen',

package/src/platform/cdp.js

@@ -0,0 +1,242 @@
+// A dependency-free Chrome DevTools Protocol client.
+//
+// Hand-rolled rather than `ws`, and not because of ideology: CLAUDE.md keeps
+// the runtime dependency list at exactly one (the MCP SDK), and the alternative
+// does not exist anyway. The global `WebSocket` is **absent on Node 18 and
+// flag-only on Node 20** — verified here on v20.20.0, where `typeof WebSocket`
+// is `undefined` — and this package supports 18, 20 and 22. So the choice is
+// between a dependency and forty lines of RFC 6455, and forty lines is smaller
+// than the surface a dependency brings.
+//
+// The other thing a probe on 2026-09-11 established, recorded because it costs
+// an afternoon to rediscover: Chrome refuses the upgrade with **401
+// Unauthorized** when an `Origin` header is present and does not match the
+// inspector's own host. A browser's own WebSocket API always sends one and
+// cannot change it, which is why a page cannot drive CDP — and why a
+// hand-rolled client, which simply omits the header, can.
+//
+// Scope: enough CDP to be the transport under `src/platform/web.js`. It speaks
+// one target at a time, it does not multiplex sessions, and it has no
+// reconnection policy. Anything cleverer belongs above the boundary or nowhere.
+import net from 'node:net';
+import crypto from 'node:crypto';
+import http from 'node:http';
+
+/** RFC 6455's fixed GUID, concatenated with the client key to prove the handshake. */
+export const WS_GUID = '258EAFA5-E914-47DA-95CA-C5AB0DC85B11';
+
+const OPCODE = { text: 0x1, binary: 0x2, close: 0x8, ping: 0x9, pong: 0xa };
+
+/** Ask a browser what it has open. `targetId` is the udid: a tab is a device. */
+export function listTargets(port, host = '127.0.0.1', { timeoutMs = 2000 } = {}) {
+  return new Promise((resolve, reject) => {
+    const req = http.get({ host, port, path: '/json/list', timeout: timeoutMs }, (res) => {
+      let body = '';
+      res.on('data', (d) => { body += d; });
+      res.on('end', () => {
+        if (res.statusCode !== 200) {
+          reject(new Error(`the browser's debugging endpoint answered ${res.statusCode} — is it running with --remote-debugging-port=${port}?`));
+          return;
+        }
+        try { resolve(JSON.parse(body)); } catch (err) { reject(new Error(`the debugging endpoint returned something that is not JSON: ${err.message}`)); }
+      });
+    });
+    req.on('timeout', () => { req.destroy(new Error(`no answer from ${host}:${port} within ${timeoutMs}ms`)); });
+    req.on('error', (err) => reject(new Error(
+      `cannot reach a browser on ${host}:${port} — ${err.message}.`
+      + ' Start one with --remote-debugging-port, or pass --port.',
+    )));
+  });
+}
+
+/**
+ * One frame, encoded.
+ *
+ * Client frames must be masked — an unmasked client frame is a protocol error
+ * and Chrome closes the socket rather than answering, which reads as a hang.
+ */
+function encodeFrame(payload, opcode = OPCODE.text) {
+  const data = Buffer.from(payload, 'utf8');
+  const mask = crypto.randomBytes(4);
+  const len = data.length;
+  const header = len < 126
+    ? Buffer.from([0x80 | opcode, 0x80 | len])
+    : len < 65536
+      ? Buffer.concat([Buffer.from([0x80 | opcode, 0x80 | 126]), (() => { const b = Buffer.alloc(2); b.writeUInt16BE(len); return b; })()])
+      : Buffer.concat([Buffer.from([0x80 | opcode, 0x80 | 127]), (() => { const b = Buffer.alloc(8); b.writeBigUInt64BE(BigInt(len)); return b; })()]);
+  const masked = Buffer.allocUnsafe(len);
+  for (let i = 0; i < len; i += 1) masked[i] = data[i] ^ mask[i % 4];
+  return Buffer.concat([header, mask, masked]);
+}
+
+/**
+ * Pull whole frames out of a growing buffer.
+ *
+ * Returns the frames it could complete and what is left over. CDP replies
+ * routinely exceed one TCP segment — a full accessibility tree is tens of
+ * kilobytes — so a reader that assumes one frame per `data` event works on a
+ * hello-world and fails on the first real payload.
+ */
+export function decodeFrames(buffer) {
+  const frames = [];
+  let rest = buffer;
+  for (;;) {
+    if (rest.length < 2) break;
+    const opcode = rest[0] & 0x0f;
+    const fin = Boolean(rest[0] & 0x80);
+    const masked = Boolean(rest[1] & 0x80);
+    let len = rest[1] & 0x7f;
+    let offset = 2;
+    if (len === 126) {
+      if (rest.length < 4) break;
+      len = rest.readUInt16BE(2);
+      offset = 4;
+    } else if (len === 127) {
+      if (rest.length < 10) break;
+      len = Number(rest.readBigUInt64BE(2));
+      offset = 10;
+    }
+    const maskKey = masked ? rest.subarray(offset, offset + 4) : null;
+    if (masked) offset += 4;
+    if (rest.length < offset + len) break;
+    let payload = rest.subarray(offset, offset + len);
+    if (maskKey) {
+      const out = Buffer.allocUnsafe(len);
+      for (let i = 0; i < len; i += 1) out[i] = payload[i] ^ maskKey[i % 4];
+      payload = out;
+    }
+    frames.push({ opcode, fin, payload });
+    rest = rest.subarray(offset + len);
+  }
+  return { frames, rest };
+}
+
+/**
+ * Open a CDP session against a websocket URL from `listTargets`.
+ *
+ * Resolves once the upgrade is accepted, so a caller that gets a connection has
+ * a usable one — the failure mode this avoids is a `send` that queues silently
+ * against a socket the browser refused.
+ */
+export function connect(wsUrl, { timeoutMs = 5000 } = {}) {
+  const url = new URL(wsUrl);
+  return new Promise((resolve, reject) => {
+    const key = crypto.randomBytes(16).toString('base64');
+    const socket = net.connect({ host: url.hostname, port: Number(url.port || 80) });
+    let settled = false;
+    const fail = (err) => {
+      if (settled) return;
+      settled = true;
+      socket.destroy();
+      reject(err);
+    };
+    const timer = setTimeout(() => fail(new Error(`the browser did not complete the websocket upgrade within ${timeoutMs}ms`)), timeoutMs);
+
+    socket.on('error', (err) => fail(new Error(`cannot open a debugging socket: ${err.message}`)));
+    socket.on('connect', () => {
+      // No `Origin` header, deliberately. Chrome answers 401 when one is
+      // present and does not match the inspector's host, and nothing here needs
+      // to claim an origin.
+      socket.write(
+        `GET ${url.pathname}${url.search} HTTP/1.1\r\n`
+        + `Host: ${url.host}\r\n`
+        + 'Upgrade: websocket\r\nConnection: Upgrade\r\n'
+        + `Sec-WebSocket-Key: ${key}\r\n`
+        + 'Sec-WebSocket-Version: 13\r\n\r\n',
+      );
+    });
+
+    let handshake = Buffer.alloc(0);
+    const onHandshake = (chunk) => {
+      handshake = Buffer.concat([handshake, chunk]);
+      const end = handshake.indexOf('\r\n\r\n');
+      if (end === -1) return;
+      const head = handshake.subarray(0, end).toString('latin1');
+      const status = /^HTTP\/1\.1 (\d+)/.exec(head)?.[1];
+      if (status !== '101') {
+        fail(new Error(
+          `the browser refused the debugging socket with HTTP ${status ?? '(no status)'}.`
+          + (status === '401'
+            ? ' That is the Origin check: Chrome rejects an upgrade whose Origin header does not'
+              + ' match the inspector host. This client sends none, so something else set one.'
+            : ' Check the target still exists — a closed tab keeps its id but not its socket.'),
+        ));
+        return;
+      }
+      const accept = /sec-websocket-accept:\s*(\S+)/i.exec(head)?.[1];
+      const expected = crypto.createHash('sha1').update(key + WS_GUID).digest('base64');
+      if (accept !== expected) {
+        fail(new Error('the browser accepted the upgrade with a key that does not match — this is not a websocket peer'));
+        return;
+      }
+      clearTimeout(timer);
+      settled = true;
+      socket.removeListener('data', onHandshake);
+      resolve(session(socket, handshake.subarray(end + 4)));
+    };
+    socket.on('data', onHandshake);
+  });
+}
+
+/** A connected session: one id space, one pending map, text frames only. */
+function session(socket, leftover) {
+  let buffer = leftover ?? Buffer.alloc(0);
+  let nextId = 1;
+  let closedWith = null;
+  const pending = new Map();
+  const listeners = new Set();
+
+  const rejectAll = (err) => {
+    for (const { reject } of pending.values()) reject(err);
+    pending.clear();
+  };
+
+  socket.on('data', (chunk) => {
+    buffer = Buffer.concat([buffer, chunk]);
+    const { frames, rest } = decodeFrames(buffer);
+    buffer = rest;
+    for (const frame of frames) {
+      if (frame.opcode === OPCODE.ping) { socket.write(encodeFrame(frame.payload.toString('utf8'), OPCODE.pong)); continue; }
+      if (frame.opcode === OPCODE.close) { closedWith = 'the browser closed the debugging socket'; socket.end(); continue; }
+      if (frame.opcode !== OPCODE.text) continue;
+      let message;
+      try { message = JSON.parse(frame.payload.toString('utf8')); } catch { continue; }
+      if (message.id != null && pending.has(message.id)) {
+        const { resolve, reject } = pending.get(message.id);
+        pending.delete(message.id);
+        // A CDP error is a reply, not a transport failure, and it carries the
+        // only sentence that says what was wrong with the call.
+        if (message.error) reject(new Error(`${message.error.message ?? 'CDP error'}${message.error.data ? ` — ${message.error.data}` : ''}`));
+        else resolve(message.result ?? {});
+      } else if (message.method) {
+        for (const fn of listeners) { try { fn(message); } catch { /* a listener must not break the socket */ } }
+      }
+    }
+  });
+
+  socket.on('close', () => { rejectAll(new Error(closedWith ?? 'the debugging socket closed')); });
+  socket.on('error', (err) => { rejectAll(new Error(`debugging socket error: ${err.message}`)); });
+
+  return {
+    /** One CDP call. Rejects with the browser's own message on a protocol error. */
+    send(method, params = {}, { timeoutMs = 10000 } = {}) {
+      if (socket.destroyed) return Promise.reject(new Error(closedWith ?? 'the debugging socket is closed'));
+      const id = nextId++;
+      return new Promise((resolve, reject) => {
+        const timer = setTimeout(() => {
+          pending.delete(id);
+          reject(new Error(`${method} did not answer within ${timeoutMs}ms`));
+        }, timeoutMs);
+        pending.set(id, {
+          resolve: (v) => { clearTimeout(timer); resolve(v); },
+          reject: (e) => { clearTimeout(timer); reject(e); },
+        });
+        socket.write(encodeFrame(JSON.stringify({ id, method, params })));
+      });
+    },
+    /** Subscribe to CDP events. Returns an unsubscribe. */
+    on(fn) { listeners.add(fn); return () => listeners.delete(fn); },
+    close() { try { socket.end(encodeFrame('', OPCODE.close)); } catch { socket.destroy(); } },
+    get closed() { return socket.destroyed; },
+  };
+}

package/src/platform/index.js

@@ -20,6 +20,7 @@
 // function that takes a udid routes on that udid — `ownsUdid` answers that from
 // the id's own shape, because the question is asked from inside a capture loop
 // in a process that never listed anything.
+import * as store from '../store.js';
 import { platform as android } from './android.js';
 import { platform as ios } from './ios.js';
 
@@ -204,9 +205,34 @@ export async function resolveAcross(query, opts, all) {
 // backend a call reaches, never what the call means.
 export const isBootedSync = (udid, ...args) => platformFor(udid).isBootedSync(udid, ...args);
 export const screenshot = (udid, ...args) => platformFor(udid).screenshot(udid, ...args);
-export const launchApp = (udid, ...args) => platformFor(udid).launchApp(udid, ...args);
+/**
+ * Launch and openUrl stamp the action clock; nothing else here does.
+ *
+ * These two change the screen without touching the digitizer, so they are the
+ * only actions a gesture log cannot see — and a settle that cannot see them
+ * measures how long the screen being *left* has been sitting still. Measured:
+ * 283ms after a Settings launch, `settled: true` with `stableForMs: 4427` over
+ * a screen holding zero elements.
+ *
+ * **At the seam rather than in the `launch` step, and that placement is the
+ * point.** It was in the step first, which covered flows and missed everything
+ * else that launches an app — `baseline.resetFor`, `wedge.revive`, the bench,
+ * and the probe that found this. A rule enforced where you noticed it covers a
+ * symptom; this project's handoff has three worked examples of exactly that and
+ * this was very nearly a fourth.
+ *
+ * `terminateApp` is deliberately not stamped: it removes an app rather than
+ * presenting one, and the screen it leaves behind is whatever was underneath.
+ */
+export const launchApp = (udid, ...args) => {
+  store.noteAction(udid);
+  return platformFor(udid).launchApp(udid, ...args);
+};
 export const terminateApp = (udid, ...args) => platformFor(udid).terminateApp(udid, ...args);
-export const openUrl = (udid, ...args) => platformFor(udid).openUrl(udid, ...args);
+export const openUrl = (udid, ...args) => {
+  store.noteAction(udid);
+  return platformFor(udid).openUrl(udid, ...args);
+};
 export const setPermission = (udid, ...args) => platformFor(udid).setPermission(udid, ...args);
 export const setPasteboard = (udid, ...args) => platformFor(udid).setPasteboard(udid, ...args);
 

package/src/store.js

@@ -35,6 +35,7 @@ export function paths(udid) {
     // recorded, and a stall is the absence of frames.
     captureHealth: path.join(dir, 'capture-health.json'),
     lastInput: path.join(dir, 'last-input'),
+    lastAction: path.join(dir, 'last-action'),
   };
 }
 
@@ -58,6 +59,44 @@ export function noteInput(udid, at = Date.now()) {
   } catch {
     /* a timestamp nothing depends on for correctness must not fail an action */
   }
+  noteAction(udid, at);
+}
+
+/**
+ * When we last did something that ought to change the screen.
+ *
+ * Deliberately separate from the input log above, which answers a different
+ * question — "have several *gestures* landed with no pixel moving" — and would
+ * be wrong to answer it about a launch, since a launch that paints nothing is
+ * not evidence of a dead digitizer.
+ *
+ * This one exists so a settle can tell its own stillness from the *previous*
+ * screen's. Measured on 2026-09-18: 283ms after a Settings launch the state
+ * read `settled: true` with `stableForMs: 4427` and **zero elements** — 4.4
+ * seconds of quiet that began before the launch was issued. 408ms after a
+ * `tap General`, `stableForMs: 7753` and the 23 elements of the screen being
+ * left. In both, the settle detector was honestly reporting how long the screen
+ * we had already abandoned had been sitting still.
+ *
+ * Every gesture writes it (via `noteInput`), and so does every launch and
+ * `openUrl`, because those change the screen without touching the digitizer.
+ */
+export function noteAction(udid, at = Date.now()) {
+  try {
+    writeAtomic(paths(udid).lastAction, String(at));
+  } catch {
+    /* as above: a timestamp may not fail the action it describes */
+  }
+}
+
+/** When the last screen-changing action was issued, or null if none is recorded. */
+export function lastActionAt(udid) {
+  try {
+    const n = Number(fs.readFileSync(paths(udid).lastAction, 'utf8').trim());
+    return Number.isFinite(n) ? n : null;
+  } catch {
+    return null;
+  }
 }
 
 /**