simframe 0.9.0 → 0.11.0

package/scripts/eval-perception.mjs

@@ -0,0 +1,281 @@
+#!/usr/bin/env node
+// The perception eval harness. Deferred since Phase 5; four things wait on it.
+//
+// What it is, and the design decision behind it. The obvious harness replays
+// stored frames through the perception path and diffs the element lists, which
+// is what Phase 13 step 5 describes. That harness cannot exist: the element
+// list is the accessibility tree fused with OCR, the tree is not in the frame,
+// and OCR runs in the daemon against a live framebuffer. A frame on disk is
+// half the input.
+//
+// So the split is different and, for what actually needs gating, better. The
+// machine records the *input* — the fused element list for a screen, exactly as
+// perception produced it. A person authors the *expected output*. Everything
+// downstream of the element list is a pure function, so the check runs offline,
+// deterministically, with no simulator and no daemon:
+//
+//   resolution   matching.resolve(targets, query)  — which element a query picks
+//   identity     fingerprint.tokens(targets)       — which screen this is
+//   change       analyze.signatureDiff(a, b)       — whether a frame moved
+//
+// Those three are precisely the thresholds every blocked item wants to change.
+// What it does *not* cover is whether perception found the elements at all,
+// which is inherently live and stays with eval-fingerprint.mjs and the
+// integration job. Said plainly here rather than implied, because a harness
+// that is trusted for more than it measures is worse than no harness.
+//
+// Two rules from this repo:
+//   - The gate is an exit code. Nothing is verified through a pipe: this
+//     project has already shipped a check that could not fail because
+//     `node script.mjs | tail` reports tail's status.
+//   - A fixture with no authored expectations is reported as unauthored, never
+//     counted as a pass. An empty test suite is green.
+import fs from 'node:fs';
+import path from 'node:path';
+import { fileURLToPath } from 'node:url';
+import * as fingerprint from '../src/fingerprint.js';
+import * as matching from '../src/matching.js';
+import * as analyze from '../src/analyze.js';
+import * as view from '../src/view.js';
+
+const ROOT = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '..');
+const DIR = path.join(ROOT, 'test', 'perception', 'screens');
+const arg = (n, d = null) => {
+  const hit = process.argv.find((a) => a.startsWith(`--${n}=`));
+  return hit ? hit.slice(n.length + 3) : d;
+};
+const has = (n) => process.argv.includes(`--${n}`);
+
+export function fixtures(dir = DIR, only = null) {
+  if (!fs.existsSync(dir)) return [];
+  return fs.readdirSync(dir)
+    .filter((f) => f.endsWith('.json'))
+    .filter((f) => (only ? f.includes(only) : true))
+    .sort()
+    .map((f) => ({ file: f, ...JSON.parse(fs.readFileSync(path.join(dir, f), 'utf8')) }));
+}
+
+/**
+ * Check one screen's authored expectations against the pure layers.
+ *
+ * Returns findings rather than printing, so the same function backs the CLI and
+ * the unit tests — and so a finding can be counted without being formatted.
+ */
+export function checkScreen(fx) {
+  const findings = [];
+  const screen = fx.points ?? null;
+  const targets = fx.targets ?? [];
+  const expect = fx.expect ?? {};
+  const authored = (expect.resolutions?.length ?? 0)
+    + (expect.ambiguous?.length ?? 0)
+    + (expect.none?.length ?? 0)
+    + (expect.discoverable?.length ?? 0)
+    + (fx.frame_pairs?.length ?? 0)
+    + (expect.identity ? 1 : 0);
+
+  for (const r of expect.resolutions ?? []) {
+    const out = matching.resolve(targets, r.query, { screen });
+    if (out.status !== 'ok') {
+      findings.push({
+        kind: 'resolution',
+        query: r.query,
+        want: r.label,
+        got: out.status === 'ambiguous'
+          ? `ambiguous between ${out.alternatives.map((a) => a.label).join(', ')}`
+          : 'nothing resolved',
+      });
+      continue;
+    }
+    const got = out.target.label ?? '(icon-only)';
+    // Compared on the label because that is what the author can see and mean.
+    // Coordinates would make a fixture break every time a row moved a point.
+    if (got !== r.label) {
+      findings.push({ kind: 'resolution', query: r.query, want: r.label, got: `"${got}" at ${out.target.x},${out.target.y}` });
+    }
+  }
+
+  // The other half of the contract, and the half with the wrong-tap risk in it:
+  // "when two things answer equally well it says so rather than guessing".
+  for (const q of expect.ambiguous ?? []) {
+    const out = matching.resolve(targets, typeof q === 'string' ? q : q.query, { screen });
+    if (out.status !== 'ambiguous') {
+      findings.push({
+        kind: 'should-ask',
+        query: typeof q === 'string' ? q : q.query,
+        want: 'ambiguous',
+        got: out.status === 'ok' ? `picked "${out.target.label}" at score ${out.score}` : 'nothing resolved',
+      });
+    }
+  }
+
+  for (const q of expect.none ?? []) {
+    const out = matching.resolve(targets, q, { screen });
+    if (out.status !== 'none') {
+      findings.push({
+        kind: 'should-find-nothing',
+        query: q,
+        want: 'none',
+        got: out.status === 'ok' ? `picked "${out.target.label}"` : 'ambiguous',
+      });
+    }
+  }
+
+  // Discovery: does the default map still SAY what is on this screen?
+  //
+  // This check exists because of a regression it would have caught. A rule that
+  // dropped long non-interactive rows left every React Native list card out of
+  // the map — the cards expose their children as one concatenated accessibility
+  // label, so they look exactly like a Settings caption. Nothing became
+  // untappable, because `locate` reads targets rather than rows, so every
+  // resolution expectation above still passed. What was lost was the map's
+  // account of what is there, and an agent that cannot see a row falls back to
+  // a ~1600-token screenshot.
+  //
+  // Resolution and discovery are different claims, and the harness could only
+  // make the first one.
+  if (expect.discoverable?.length && screen) {
+    const { rows } = view.rowsFor({ targets }, { screen });
+    const shown = rows.map((r) => String(r.label ?? ''));
+    for (const want of expect.discoverable) {
+      // Compared as a prefix, because a long label is legitimately truncated in
+      // a row — truncated is discoverable, absent is not.
+      const head = want.slice(0, 24);
+      if (!shown.some((got) => got.startsWith(head))) {
+        findings.push({
+          kind: 'discovery',
+          query: `${want.slice(0, 40)}${want.length > 40 ? '…' : ''}`,
+          want: 'listed in the default map',
+          got: `${rows.length} row(s), none starting with it`,
+        });
+      }
+    }
+  }
+
+  // Identity drift. A token-rule change that silently alters a screen's
+  // structural hash discards every stored map and graph for it, and the failure
+  // is invisible — an old hash is a well-formed hash that matches nothing.
+  if (expect.identity && screen) {
+    const now = fingerprint.fingerprint(targets, screen);
+    if (now.hash !== expect.identity.hash) {
+      const similarity = fingerprint.similarity(now.tokens, expect.identity.tokens ?? []);
+      findings.push({
+        kind: 'identity',
+        query: '(structural hash)',
+        want: `${expect.identity.hash?.slice(0, 12)} (${expect.identity.tokens?.length ?? 0} tokens)`,
+        got: `${now.hash?.slice(0, 12)} (${now.tokens.length} tokens), similarity ${similarity.toFixed(2)}`,
+      });
+    }
+  }
+
+  // Frame pairs, for the change detector. `changed: true` means an action that
+  // a person would call visible — a switch flipping, a radio dot moving.
+  for (const pair of fx.frame_pairs ?? []) {
+    const a = analyze.hexToSignature(pair.after);
+    const b = analyze.hexToSignature(pair.before);
+    // Two different questions about the same pair of frames: has the *screen*
+    // changed (the mean, which drives stillness) and has a *control* changed
+    // (the largest single region, which drives the verdict). A switch flip
+    // answers no to the first and yes to the second, which is the whole reason
+    // both exist.
+    const diff = pair.per_cell ? analyze.maxCellDelta(a, b) : analyze.signatureDiff(a, b);
+    const seen = diff > (pair.threshold ?? 0.004);
+    if (seen !== Boolean(pair.changed)) {
+      findings.push({
+        kind: 'change',
+        query: pair.note ?? '(frame pair)',
+        want: pair.changed ? 'a visible change' : 'no change',
+        got: `${pair.per_cell ? 'max cell' : 'mean'} ${diff.toFixed(5)} against a threshold of ${pair.threshold ?? 0.004}`,
+      });
+    }
+  }
+
+  return { authored, findings };
+}
+
+async function record() {
+  const api = await import('../src/index.js');
+  const device = arg('device');
+  const name = arg('name');
+  if (!name) throw new Error('--record needs --name=<app>/<screen>');
+  const id = await api.screenIdentity(device, { fresh: true, confirmNovel: false });
+  const entry = id.entry ?? {};
+  const out = {
+    app: name.split('/')[0],
+    screen: name.split('/').slice(1).join('/') || name,
+    note: arg('note') ?? null,
+    recorded_at: new Date().toISOString(),
+    points: id.points,
+    // The recorded input. Trimmed to what the pure layers read, so a fixture
+    // is reviewable by a person rather than a wall of machine state.
+    targets: (entry.targets ?? []).map((t) => ({
+      label: t.label ?? null,
+      value: t.value ?? null,
+      type: t.type ?? null,
+      x: t.x, y: t.y,
+      frame: t.frame ?? null,
+      region: t.region ?? null,
+      source: t.source ?? null,
+      aliases: t.aliases ?? undefined,
+      navSlot: t.navSlot ?? undefined,
+      enabled: t.enabled ?? undefined,
+      selected: t.selected ?? undefined,
+    })),
+    expect: {
+      identity: { hash: entry.structuralHash, tokens: entry.structuralTokens ?? [] },
+      // Authored by hand. The machine records what perception saw; a person
+      // says what it should mean. Deriving these from current behaviour would
+      // bake today's bugs in as the specification.
+      resolutions: [],
+      ambiguous: [],
+      none: [],
+    },
+  };
+  fs.mkdirSync(DIR, { recursive: true });
+  const file = path.join(DIR, `${name.replace(/\//g, '__')}.json`);
+  fs.writeFileSync(file, `${JSON.stringify(out, null, 2)}\n`);
+  console.log(`recorded ${out.targets.length} element(s) from ${name} -> ${path.relative(ROOT, file)}`);
+  console.log('  now author expect.resolutions / expect.ambiguous / expect.none by hand');
+}
+
+function check() {
+  const all = fixtures(DIR, arg('only'));
+  if (!all.length) {
+    console.error('check-perception: no fixtures. Record some with --record --name=<app>/<screen>');
+    process.exitCode = 1;
+    return;
+  }
+  let failed = 0;
+  let unauthored = 0;
+  let checks = 0;
+  const apps = new Set();
+  for (const fx of all) {
+    apps.add(fx.app);
+    const { authored, findings } = checkScreen(fx);
+    checks += authored;
+    if (!authored) {
+      unauthored += 1;
+      console.log(`  ??  ${fx.app}/${fx.screen}  recorded, no expectations authored`);
+      continue;
+    }
+    if (!findings.length) {
+      console.log(`  ok  ${fx.app}/${fx.screen}  ${authored} expectation(s)`);
+      continue;
+    }
+    failed += findings.length;
+    console.log(`FAIL  ${fx.app}/${fx.screen}`);
+    for (const f of findings) {
+      console.log(`        ${f.kind}: ${f.query}\n          want ${f.want}\n          got  ${f.got}`);
+    }
+  }
+  console.log(`\n${all.length} screen(s) across ${apps.size} app(s), ${checks} expectation(s), ${failed} failure(s)`
+    + (unauthored ? `, ${unauthored} unauthored` : ''));
+  // An unauthored fixture is not a pass. A suite that counts recordings as
+  // successes is a suite that goes green by adding files.
+  if (failed || unauthored) process.exitCode = 1;
+}
+
+if (import.meta.url === `file://${process.argv[1]}`) {
+  if (has('record')) await record();
+  else if (has('list')) for (const f of fixtures()) console.log(`${f.app}/${f.screen}  ${f.targets?.length ?? 0} elements`);
+  else check();
+}

package/scripts/phase17-corpus.mjs

@@ -0,0 +1,176 @@
+#!/usr/bin/env node
+/**
+ * Phase 17, go/no-go step 1: build the corpus and ask whether there is a job.
+ *
+ * The plan in docs/PHASES-HUMAN-PARITY.md said to export 200 `ambiguous_intent`
+ * and `no_plan` escalations. That corpus cannot be assembled: `no_plan` has
+ * never been logged once, `ambiguous_intent` is the reason the ranking bug
+ * produced (so the pre-fix records encode a bug), and the session id is minted
+ * per process — so a CLI-driven agent gets one "session" per command and the
+ * "filter to one session id" step has nothing to filter.
+ *
+ * There is a better corpus and it was there all along. Every verified graph edge
+ * is a decision that *worked*: the goal is in `step`, the screen it was taken on
+ * is the node, and the element list for that screen is in the screen map. That
+ * is (goal, element list, action) for every successful step, not just the rare
+ * failures — and the ground truth is stronger, because the tap was verified.
+ *
+ * What this measures is the question the whole phase turns on: of the decisions
+ * an agent actually makes, how many does the local matcher already resolve? A
+ * local planner can only earn its keep on the remainder. If the remainder is
+ * small, Phase 17 is a no-go regardless of how good the model is.
+ *
+ * Output is aggregate by design. This reads a real device's memory of real
+ * third-party apps, so it prints counts and never a label, an app name or a
+ * screen's contents. See the standing rule in scripts/check-private.mjs.
+ *
+ * Usage: node scripts/phase17-corpus.mjs [--device <udid>] [--json]
+ */
+import { readdirSync, readFileSync, existsSync } from 'node:fs';
+import { join } from 'node:path';
+import { homedir } from 'node:os';
+import * as matching from '../src/matching.js';
+import { goalOf } from '../src/actions.js';
+
+const ROOT = process.env.SIMFRAME_HOME || join(homedir(), '.simframe');
+
+const readJson = (p) => {
+  try { return JSON.parse(readFileSync(p, 'utf8')); } catch { return null; }
+};
+const listDir = (p) => (existsSync(p) ? readdirSync(p).filter((f) => f.endsWith('.json')) : []);
+
+/** Screens are keyed by frame hash; graph nodes carry a layoutHash. Index both. */
+function screenIndex(dev) {
+  const idx = new Map();
+  for (const f of listDir(join(dev, 'screens'))) {
+    const s = readJson(join(dev, 'screens', f));
+    if (!s?.targets?.length) continue;
+    for (const k of ['layoutHash', 'structuralHash', 'hash']) {
+      if (s[k] && !idx.has(s[k])) idx.set(s[k], s);
+    }
+  }
+  return idx;
+}
+
+/**
+ * Only some steps involve choosing an element. `launch` names an app, `button`
+ * names hardware, `openUrl` names a URL, `scroll` names a direction — none of
+ * them is a decision a planner could help with, and counting them was inflating
+ * the "not found" bucket to 45% on the first run of this script.
+ */
+const CHOOSES_AN_ELEMENT = new Set(['tap', 'type', 'scroll_to', 'scrollTo', 'assert', 'swipe_from', 'longPress']);
+
+/** A "#3" is answered by the ref table and a "@x,y" by arithmetic. Neither is a decision. */
+const isSelectorNotIntent = (goal) => /^#\d+$/.test(goal) || /^@?-?\d+\s*,\s*-?\d+$/.test(goal);
+
+export function classify(dev) {
+  const idx = screenIndex(dev);
+  const out = {
+    screens_stored: idx.size,
+    edges: 0,
+    unjoinable: 0,
+    no_goal: 0,
+    not_an_element_step: 0,
+    selector_not_intent: 0,
+    cases: 0,
+    resolved: 0,
+    ambiguous: 0,
+    none: 0,
+  };
+  for (const f of listDir(join(dev, 'graph'))) {
+    const g = readJson(join(dev, 'graph', f));
+    const screen = idx.get(g?.layoutHash) ?? idx.get(g?.hash);
+    for (const e of g?.edges ?? []) {
+      out.edges += 1;
+      if (!screen) { out.unjoinable += 1; continue; }
+      const goal = goalOf(e.step);
+      if (!goal) { out.no_goal += 1; continue; }
+      if (!CHOOSES_AN_ELEMENT.has(e.step?.action)) { out.not_an_element_step += 1; continue; }
+      if (isSelectorNotIntent(String(goal).trim())) { out.selector_not_intent += 1; continue; }
+      out.cases += 1;
+      const r = matching.resolve(screen.targets, String(goal));
+      if (r.status === 'ok') out.resolved += 1;
+      else if (r.status === 'ambiguous') out.ambiguous += 1;
+      else out.none += 1;
+    }
+  }
+  return out;
+}
+
+/**
+ * The second instrument, with the opposite bias.
+ *
+ * A verified edge is a decision that *succeeded*, so the graph cannot see the
+ * ones the matcher fumbled — those became escalations, Claude fixed them, and
+ * the edge was written with the corrected goal. Measuring the matcher on its
+ * own successes is partly circular. So also count resolution failures from the
+ * log against total edge traversals: same question, denominator built from
+ * failures instead of successes.
+ */
+export function resolutionFailureRate(dev) {
+  let traversals = 0;
+  for (const f of listDir(join(dev, 'graph'))) {
+    for (const e of readJson(join(dev, 'graph', f))?.edges ?? []) traversals += e.count ?? 0;
+  }
+  let failures = 0;
+  let escalations = 0;
+  const log = join(dev, 'escalations.jsonl');
+  if (existsSync(log)) {
+    for (const line of readFileSync(log, 'utf8').split('\n')) {
+      if (!line.trim()) continue;
+      let r;
+      try { r = JSON.parse(line); } catch { continue; }
+      escalations += 1;
+      if (r.reason === 'ambiguous_intent' || r.reason === 'unknown_screen') failures += 1;
+    }
+  }
+  return { traversals, escalations, failures };
+}
+
+const flags = process.argv.slice(2);
+const only = flags.includes('--device') ? flags[flags.indexOf('--device') + 1] : null;
+const devices = (existsSync(ROOT) ? readdirSync(ROOT) : [])
+  .filter((d) => !d.startsWith('TEST-') && d !== 'bin')
+  .filter((d) => existsSync(join(ROOT, d, 'graph')))
+  .filter((d) => !only || d === only);
+
+const totals = { cases: 0, resolved: 0, ambiguous: 0, none: 0, edges: 0, unjoinable: 0, not_an_element_step: 0, selector_not_intent: 0 };
+const rows = [];
+for (const d of devices) {
+  const r = classify(join(ROOT, d));
+  if (!r.edges) continue;
+  rows.push({ device: `${d.slice(0, 8)}…`, ...r, ...resolutionFailureRate(join(ROOT, d)) });
+  for (const k of Object.keys(totals)) totals[k] += r[k];
+}
+
+if (flags.includes('--json')) {
+  console.log(JSON.stringify({ devices: rows, totals }, null, 2));
+} else {
+  console.log('Phase 17 corpus — verified graph edges as (goal, element list, action)\n');
+  for (const r of rows) {
+    console.log(`  ${r.device}  edges ${r.edges}  joinable ${r.edges - r.unjoinable}  decisions ${r.cases}` +
+      `  → matcher ok ${r.resolved}, ambiguous ${r.ambiguous}, not found ${r.none}`);
+  }
+  const { cases, resolved, ambiguous, none } = totals;
+  const pct = (n) => (cases ? `${Math.round((n / cases) * 1000) / 10}%` : '—');
+  console.log(`\n  ${totals.edges} edges: ${totals.unjoinable} unjoinable, ` +
+    `${totals.not_an_element_step} chose no element (launch/button/openUrl/scroll), ` +
+    `${totals.selector_not_intent} used a #ref or a coordinate.`);
+  console.log(`  TOTAL element decisions: ${cases}`);
+  console.log(`    already resolved locally  ${resolved}  ${pct(resolved)}   <- a planner adds nothing here`);
+  console.log(`    ambiguous                 ${ambiguous}  ${pct(ambiguous)}   <- a planner could pick`);
+  console.log(`    not found                 ${none}  ${pct(none)}   <- a planner cannot invent an element`);
+  const addressable = ambiguous;
+  console.log(`\n  Addressable by a local planner: ${addressable} of ${cases} (${pct(addressable)}).`);
+
+  console.log('\nSecond instrument — resolution failures from the log, per traversal.');
+  console.log('A rate over 100% means the graph was discarded while the log kept appending');
+  console.log('(a MAP_VERSION or GRAPH_VERSION bump), so it is not a rate. Read the device');
+  console.log('that was driven by an agent doing real work, not the bench device.\n');
+  for (const r of rows) {
+    const rate = r.traversals ? `${Math.round((r.failures / r.traversals) * 1000) / 10}%` : '—';
+    console.log(`  ${r.device}  traversals ${String(r.traversals).padStart(4)}` +
+      `  escalations ${String(r.escalations).padStart(4)}` +
+      `  resolution failures ${String(r.failures).padStart(3)}  ${rate.padStart(6)}`);
+  }
+}