simframe 0.10.0 → 0.12.0

package/scripts/eval-perception.mjs

@@ -36,6 +36,7 @@ import { fileURLToPath } from 'node:url';
 import * as fingerprint from '../src/fingerprint.js';
 import * as matching from '../src/matching.js';
 import * as analyze from '../src/analyze.js';
+import * as view from '../src/view.js';
 
 const ROOT = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '..');
 const DIR = path.join(ROOT, 'test', 'perception', 'screens');
@@ -68,6 +69,7 @@ export function checkScreen(fx) {
   const authored = (expect.resolutions?.length ?? 0)
     + (expect.ambiguous?.length ?? 0)
     + (expect.none?.length ?? 0)
+    + (expect.discoverable?.length ?? 0)
     + (fx.frame_pairs?.length ?? 0)
     + (expect.identity ? 1 : 0);
 
@@ -118,6 +120,37 @@ export function checkScreen(fx) {
     }
   }
 
+  // Discovery: does the default map still SAY what is on this screen?
+  //
+  // This check exists because of a regression it would have caught. A rule that
+  // dropped long non-interactive rows left every React Native list card out of
+  // the map — the cards expose their children as one concatenated accessibility
+  // label, so they look exactly like a Settings caption. Nothing became
+  // untappable, because `locate` reads targets rather than rows, so every
+  // resolution expectation above still passed. What was lost was the map's
+  // account of what is there, and an agent that cannot see a row falls back to
+  // a ~1600-token screenshot.
+  //
+  // Resolution and discovery are different claims, and the harness could only
+  // make the first one.
+  if (expect.discoverable?.length && screen) {
+    const { rows } = view.rowsFor({ targets }, { screen });
+    const shown = rows.map((r) => String(r.label ?? ''));
+    for (const want of expect.discoverable) {
+      // Compared as a prefix, because a long label is legitimately truncated in
+      // a row — truncated is discoverable, absent is not.
+      const head = want.slice(0, 24);
+      if (!shown.some((got) => got.startsWith(head))) {
+        findings.push({
+          kind: 'discovery',
+          query: `${want.slice(0, 40)}${want.length > 40 ? '…' : ''}`,
+          want: 'listed in the default map',
+          got: `${rows.length} row(s), none starting with it`,
+        });
+      }
+    }
+  }
+
   // Identity drift. A token-rule change that silently alters a screen's
   // structural hash discards every stored map and graph for it, and the failure
   // is invisible — an old hash is a well-formed hash that matches nothing.

package/scripts/phase17-corpus.mjs

@@ -0,0 +1,176 @@
+#!/usr/bin/env node
+/**
+ * Phase 17, go/no-go step 1: build the corpus and ask whether there is a job.
+ *
+ * The plan in docs/PHASES-HUMAN-PARITY.md said to export 200 `ambiguous_intent`
+ * and `no_plan` escalations. That corpus cannot be assembled: `no_plan` has
+ * never been logged once, `ambiguous_intent` is the reason the ranking bug
+ * produced (so the pre-fix records encode a bug), and the session id is minted
+ * per process — so a CLI-driven agent gets one "session" per command and the
+ * "filter to one session id" step has nothing to filter.
+ *
+ * There is a better corpus and it was there all along. Every verified graph edge
+ * is a decision that *worked*: the goal is in `step`, the screen it was taken on
+ * is the node, and the element list for that screen is in the screen map. That
+ * is (goal, element list, action) for every successful step, not just the rare
+ * failures — and the ground truth is stronger, because the tap was verified.
+ *
+ * What this measures is the question the whole phase turns on: of the decisions
+ * an agent actually makes, how many does the local matcher already resolve? A
+ * local planner can only earn its keep on the remainder. If the remainder is
+ * small, Phase 17 is a no-go regardless of how good the model is.
+ *
+ * Output is aggregate by design. This reads a real device's memory of real
+ * third-party apps, so it prints counts and never a label, an app name or a
+ * screen's contents. See the standing rule in scripts/check-private.mjs.
+ *
+ * Usage: node scripts/phase17-corpus.mjs [--device <udid>] [--json]
+ */
+import { readdirSync, readFileSync, existsSync } from 'node:fs';
+import { join } from 'node:path';
+import { homedir } from 'node:os';
+import * as matching from '../src/matching.js';
+import { goalOf } from '../src/actions.js';
+
+const ROOT = process.env.SIMFRAME_HOME || join(homedir(), '.simframe');
+
+const readJson = (p) => {
+  try { return JSON.parse(readFileSync(p, 'utf8')); } catch { return null; }
+};
+const listDir = (p) => (existsSync(p) ? readdirSync(p).filter((f) => f.endsWith('.json')) : []);
+
+/** Screens are keyed by frame hash; graph nodes carry a layoutHash. Index both. */
+function screenIndex(dev) {
+  const idx = new Map();
+  for (const f of listDir(join(dev, 'screens'))) {
+    const s = readJson(join(dev, 'screens', f));
+    if (!s?.targets?.length) continue;
+    for (const k of ['layoutHash', 'structuralHash', 'hash']) {
+      if (s[k] && !idx.has(s[k])) idx.set(s[k], s);
+    }
+  }
+  return idx;
+}
+
+/**
+ * Only some steps involve choosing an element. `launch` names an app, `button`
+ * names hardware, `openUrl` names a URL, `scroll` names a direction — none of
+ * them is a decision a planner could help with, and counting them was inflating
+ * the "not found" bucket to 45% on the first run of this script.
+ */
+const CHOOSES_AN_ELEMENT = new Set(['tap', 'type', 'scroll_to', 'scrollTo', 'assert', 'swipe_from', 'longPress']);
+
+/** A "#3" is answered by the ref table and a "@x,y" by arithmetic. Neither is a decision. */
+const isSelectorNotIntent = (goal) => /^#\d+$/.test(goal) || /^@?-?\d+\s*,\s*-?\d+$/.test(goal);
+
+export function classify(dev) {
+  const idx = screenIndex(dev);
+  const out = {
+    screens_stored: idx.size,
+    edges: 0,
+    unjoinable: 0,
+    no_goal: 0,
+    not_an_element_step: 0,
+    selector_not_intent: 0,
+    cases: 0,
+    resolved: 0,
+    ambiguous: 0,
+    none: 0,
+  };
+  for (const f of listDir(join(dev, 'graph'))) {
+    const g = readJson(join(dev, 'graph', f));
+    const screen = idx.get(g?.layoutHash) ?? idx.get(g?.hash);
+    for (const e of g?.edges ?? []) {
+      out.edges += 1;
+      if (!screen) { out.unjoinable += 1; continue; }
+      const goal = goalOf(e.step);
+      if (!goal) { out.no_goal += 1; continue; }
+      if (!CHOOSES_AN_ELEMENT.has(e.step?.action)) { out.not_an_element_step += 1; continue; }
+      if (isSelectorNotIntent(String(goal).trim())) { out.selector_not_intent += 1; continue; }
+      out.cases += 1;
+      const r = matching.resolve(screen.targets, String(goal));
+      if (r.status === 'ok') out.resolved += 1;
+      else if (r.status === 'ambiguous') out.ambiguous += 1;
+      else out.none += 1;
+    }
+  }
+  return out;
+}
+
+/**
+ * The second instrument, with the opposite bias.
+ *
+ * A verified edge is a decision that *succeeded*, so the graph cannot see the
+ * ones the matcher fumbled — those became escalations, Claude fixed them, and
+ * the edge was written with the corrected goal. Measuring the matcher on its
+ * own successes is partly circular. So also count resolution failures from the
+ * log against total edge traversals: same question, denominator built from
+ * failures instead of successes.
+ */
+export function resolutionFailureRate(dev) {
+  let traversals = 0;
+  for (const f of listDir(join(dev, 'graph'))) {
+    for (const e of readJson(join(dev, 'graph', f))?.edges ?? []) traversals += e.count ?? 0;
+  }
+  let failures = 0;
+  let escalations = 0;
+  const log = join(dev, 'escalations.jsonl');
+  if (existsSync(log)) {
+    for (const line of readFileSync(log, 'utf8').split('\n')) {
+      if (!line.trim()) continue;
+      let r;
+      try { r = JSON.parse(line); } catch { continue; }
+      escalations += 1;
+      if (r.reason === 'ambiguous_intent' || r.reason === 'unknown_screen') failures += 1;
+    }
+  }
+  return { traversals, escalations, failures };
+}
+
+const flags = process.argv.slice(2);
+const only = flags.includes('--device') ? flags[flags.indexOf('--device') + 1] : null;
+const devices = (existsSync(ROOT) ? readdirSync(ROOT) : [])
+  .filter((d) => !d.startsWith('TEST-') && d !== 'bin')
+  .filter((d) => existsSync(join(ROOT, d, 'graph')))
+  .filter((d) => !only || d === only);
+
+const totals = { cases: 0, resolved: 0, ambiguous: 0, none: 0, edges: 0, unjoinable: 0, not_an_element_step: 0, selector_not_intent: 0 };
+const rows = [];
+for (const d of devices) {
+  const r = classify(join(ROOT, d));
+  if (!r.edges) continue;
+  rows.push({ device: `${d.slice(0, 8)}…`, ...r, ...resolutionFailureRate(join(ROOT, d)) });
+  for (const k of Object.keys(totals)) totals[k] += r[k];
+}
+
+if (flags.includes('--json')) {
+  console.log(JSON.stringify({ devices: rows, totals }, null, 2));
+} else {
+  console.log('Phase 17 corpus — verified graph edges as (goal, element list, action)\n');
+  for (const r of rows) {
+    console.log(`  ${r.device}  edges ${r.edges}  joinable ${r.edges - r.unjoinable}  decisions ${r.cases}` +
+      `  → matcher ok ${r.resolved}, ambiguous ${r.ambiguous}, not found ${r.none}`);
+  }
+  const { cases, resolved, ambiguous, none } = totals;
+  const pct = (n) => (cases ? `${Math.round((n / cases) * 1000) / 10}%` : '—');
+  console.log(`\n  ${totals.edges} edges: ${totals.unjoinable} unjoinable, ` +
+    `${totals.not_an_element_step} chose no element (launch/button/openUrl/scroll), ` +
+    `${totals.selector_not_intent} used a #ref or a coordinate.`);
+  console.log(`  TOTAL element decisions: ${cases}`);
+  console.log(`    already resolved locally  ${resolved}  ${pct(resolved)}   <- a planner adds nothing here`);
+  console.log(`    ambiguous                 ${ambiguous}  ${pct(ambiguous)}   <- a planner could pick`);
+  console.log(`    not found                 ${none}  ${pct(none)}   <- a planner cannot invent an element`);
+  const addressable = ambiguous;
+  console.log(`\n  Addressable by a local planner: ${addressable} of ${cases} (${pct(addressable)}).`);
+
+  console.log('\nSecond instrument — resolution failures from the log, per traversal.');
+  console.log('A rate over 100% means the graph was discarded while the log kept appending');
+  console.log('(a MAP_VERSION or GRAPH_VERSION bump), so it is not a rate. Read the device');
+  console.log('that was driven by an agent doing real work, not the bench device.\n');
+  for (const r of rows) {
+    const rate = r.traversals ? `${Math.round((r.failures / r.traversals) * 1000) / 10}%` : '—';
+    console.log(`  ${r.device}  traversals ${String(r.traversals).padStart(4)}` +
+      `  escalations ${String(r.escalations).padStart(4)}` +
+      `  resolution failures ${String(r.failures).padStart(3)}  ${rate.padStart(6)}`);
+  }
+}

package/scripts/probe-network.mjs

@@ -0,0 +1,118 @@
+// Minimal CDP client over a hand-rolled WebSocket — no dependencies, because
+// simframe has exactly one runtime dependency and this would not be it.
+//
+// Node 20 has no global WebSocket, so the upgrade and the framing are done by
+// hand. Item 80 records that the upgrade 401s without an `Origin` matching the
+// inspector's host; this sends one.
+import http from 'node:http';
+import crypto from 'node:crypto';
+
+const wsUrl = process.argv[2];
+const seconds = Number(process.argv[3] ?? 20);
+const u = new URL(wsUrl);
+
+const key = crypto.randomBytes(16).toString('base64');
+const req = http.request({
+  hostname: u.hostname,
+  port: u.port,
+  path: u.pathname + u.search,
+  headers: {
+    Connection: 'Upgrade',
+    Upgrade: 'websocket',
+    'Sec-WebSocket-Key': key,
+    'Sec-WebSocket-Version': '13',
+    Origin: `http://${u.host}`,
+  },
+});
+
+/** A client frame must be masked; the server's are not. */
+const frame = (text) => {
+  const payload = Buffer.from(text, 'utf8');
+  const mask = crypto.randomBytes(4);
+  const head = payload.length < 126
+    ? Buffer.from([0x81, 0x80 | payload.length])
+    : Buffer.concat([Buffer.from([0x81, 0xfe]), (() => { const b = Buffer.alloc(2); b.writeUInt16BE(payload.length); return b; })()]);
+  const masked = Buffer.from(payload);
+  for (let i = 0; i < masked.length; i += 1) masked[i] ^= mask[i % 4];
+  return Buffer.concat([head, mask, masked]);
+};
+
+req.on('upgrade', (res, socket) => {
+  console.log(`upgraded: ${res.statusCode} ${res.headers.upgrade ?? ''}`);
+  let buf = Buffer.alloc(0);
+  let id = 0;
+  const send = (method, params) => {
+    id += 1;
+    socket.write(frame(JSON.stringify({ id, method, params })));
+    console.log(`-> ${method}`);
+    return id;
+  };
+
+  const seen = new Map();
+  socket.on('data', (chunk) => {
+    buf = Buffer.concat([buf, chunk]);
+    // Enough framing to read small unfragmented text frames, which is all CDP
+    // sends here. Deliberately not a WebSocket implementation.
+    for (;;) {
+      if (buf.length < 2) return;
+      const len0 = buf[1] & 0x7f;
+      let offset = 2;
+      let len = len0;
+      if (len0 === 126) { if (buf.length < 4) return; len = buf.readUInt16BE(2); offset = 4; }
+      else if (len0 === 127) { if (buf.length < 10) return; len = Number(buf.readBigUInt64BE(2)); offset = 10; }
+      if (buf.length < offset + len) return;
+      const text = buf.slice(offset, offset + len).toString('utf8');
+      buf = buf.slice(offset + len);
+      try {
+        const msg = JSON.parse(text);
+        if (msg.method) {
+          seen.set(msg.method, (seen.get(msg.method) ?? 0) + 1);
+          // The contents are the whole question: "settled, and these three
+          // requests fired with these statuses" is only sayable if the status
+          // and the URL are actually in here.
+          if (msg.method.startsWith('Network.')) {
+            const p = msg.params ?? {};
+            const bits = [
+              p.requestId && `id=${p.requestId}`,
+              p.request?.method && `${p.request.method} ${p.request.url}`,
+              p.response && `status=${p.response.status} ${p.response.mimeType ?? ''}`,
+              p.response?.timing && 'timing=yes',
+              p.encodedDataLength != null && `bytes=${p.encodedDataLength}`,
+              p.type && `type=${p.type}`,
+            ].filter(Boolean);
+            console.log(`   ${msg.method}: ${bits.join('  ')}`);
+          }
+        }
+        else console.log(`<- reply #${msg.id} ${msg.error ? `ERROR ${JSON.stringify(msg.error)}` : JSON.stringify(msg.result).slice(0, 120)}`);
+      } catch { /* not JSON; ignore */ }
+    }
+  });
+
+  send('Network.enable', {});
+  send('Runtime.enable', {});
+  send('Log.enable', {});
+
+  // Trigger real traffic through the app's own JS runtime, rather than through
+  // its UI. This isolates the question — does React Native's networking layer
+  // report to CDP at all — from whether my app happens to call anything, and it
+  // needs no screen, so it does not fight whatever else is driving the device.
+  setTimeout(() => {
+    send('Runtime.evaluate', {
+      expression: "fetch('https://jsonplaceholder.typicode.com/todos/1')"
+        + ".then(r => r.json()).then(j => 'FETCHED ' + JSON.stringify(j))",
+      awaitPromise: true,
+      returnByValue: true,
+    });
+  }, 1500);
+
+  setTimeout(() => {
+    console.log('\nevents received, by method:');
+    if (!seen.size) console.log('  (none)');
+    for (const [m, n] of [...seen].sort((a, b) => b[1] - a[1])) console.log(`  ${String(n).padStart(4)}x ${m}`);
+    socket.destroy();
+    process.exit(0);
+  }, seconds * 1000);
+});
+req.on('response', (res) => { console.log(`NOT upgraded: ${res.statusCode}`); process.exit(1); });
+req.on('error', (e) => { console.log('error:', e.message); process.exit(1); });
+req.end();

package/scripts/soak-capture.mjs

@@ -0,0 +1,72 @@
+#!/usr/bin/env node
+// How long can this device be driven before capture wedges?
+//
+// The wedge is the thing standing between us and a ruling population: the
+// display stops rendering after a few minutes of hard driving, `simctl
+// screenshot` fails too, both of the daemon's recoveries fail, and only a
+// device restart cures it. Three times in one afternoon.
+//
+// A leak was found in our own code on 2026-09-12 — damage-callback registration
+// was not idempotent, and the recovery loop called it on every attempt, so one
+// log's 670 port re-resolves meant up to 670 live callbacks on one port, each
+// invoked per redraw. That is a real leak with a plausible path to saturating
+// the display service, and it is **not proof** that it is the cause. This is
+// how we find out: drive until it dies, and report how long that took.
+//
+//   node scripts/soak-capture.mjs --device=<udid> --minutes=25
+//
+// A number to compare against, not a pass/fail. Before the fix, the device
+// wedged roughly every 10-20 minutes of this kind of work.
+import * as actions from '../src/actions.js';
+import * as api from '../src/index.js';
+import * as store from '../src/store.js';
+
+const arg = (n, d) => {
+  const hit = process.argv.find((a) => a.startsWith(`--${n}=`));
+  return hit ? hit.slice(n.length + 3) : d;
+};
+const device = arg('device');
+const minutes = Number(arg('minutes', 20));
+const BUNDLE = 'com.example.simframetestbed';
+
+const { device: dev } = await api.ensureDaemon(device);
+console.log(`device: ${dev.name} (${dev.runtime})   budget: ${minutes} minutes`);
+
+const started = Date.now();
+const deadline = started + minutes * 60_000;
+let laps = 0;
+let reads = 0;
+const elapsed = () => ((Date.now() - started) / 60_000).toFixed(1);
+
+// A lap is deliberately the kind of work that provokes it: launching, moving
+// between screens, and a cold read on each — not idling with a poll.
+const LAP = [
+  [{ launch: { value: BUNDLE, relaunch: true } }, { pause: 2500 }],
+  [{ tap: 'Forms, tab, 2 of 3' }, { pause: 800 }],
+  [{ tap: 'Long form' }, { pause: 900 }],
+  [{ scroll: 'down' }, { pause: 500 }],
+  [{ scroll: 'down' }, { pause: 500 }],
+  [{ tap: 'Plants, tab, 1 of 3' }, { pause: 900 }],
+  [{ tap: 'Diagnostics, tab, 3 of 3' }, { pause: 900 }],
+];
+
+while (Date.now() < deadline) {
+  for (const steps of LAP) {
+    try {
+      await actions.runScript(device, { steps, verify: false, options: { supervisor: 'none' } });
+    } catch { /* a failed step is not the subject; a dead display is */ }
+    try {
+      await api.screenIdentity(device, { fresh: true, confirmNovel: false });
+      reads += 1;
+    } catch (err) {
+      console.log(`\nWEDGED after ${elapsed()} minutes, ${laps} laps, ${reads} cold reads`);
+      console.log(`  ${String(err.message).split('\n')[0]}`);
+      const health = store.captureHealth(dev.udid);
+      if (health) console.log(`  captureHealth: ${JSON.stringify(health)}`);
+      process.exit(1);
+    }
+  }
+  laps += 1;
+  if (laps % 5 === 0) process.stdout.write(`  ${elapsed()}m  ${laps} laps  ${reads} reads  still alive\n`);
+}
+console.log(`\nSURVIVED ${minutes} minutes: ${laps} laps, ${reads} cold reads, no wedge`);