simframe 0.10.0 → 0.11.0

package/native/supervise.swift

@@ -0,0 +1,181 @@
+// The local supervisor, and it may say exactly three things.
+//
+// Claude plans. The deterministic executor in src/actions.js runs the plan and
+// verifies each step — round 6's best result was 18 steps in one call, and code
+// is a better executor than a model: faster, exact, auditable, and it cannot
+// hallucinate a step. What the executor has never had is judgement at the
+// moment a step fails, so a batch died and a round trip was spent on a decision
+// that was usually obvious.
+//
+// This answers that moment and nothing else:
+//
+//   in   {"goal":"...","step":"tap REVIEW","expected":"...","failure":"...","screen":["…labels…"],"stillMs":120}
+//   out  {"decision":"wait","ms":410}
+//
+// `wait`, `retry`, `stop`. It cannot invent a step, skip one, substitute a
+// target, or continue past an unexpected screen — not because a threshold
+// forbids it but because those are not words it can say. That constraint is the
+// safety property. `seek` was given latitude over *what* to open and pressed
+// "YES, THIS FIXED MY PROBLEM" in a live app; a component whose whole answer
+// space is three words cannot do that whatever it believes.
+//
+// Unavailable is a normal answer. It says so and exits, `doctor` reports
+// `supervisor: none`, and the executor behaves exactly as it does today.
+import Foundation
+#if canImport(FoundationModels)
+import FoundationModels
+
+@available(macOS 26.0, *)
+@Generable
+enum Decision: String {
+    case wait
+    case retry
+    case stop
+}
+
+@available(macOS 26.0, *)
+@Generable
+struct Judgement {
+    @Guide(description: "wait if the screen is still arriving, retry if the same step should be attempted again, stop if nothing further can work")
+    var decision: Decision
+}
+
+struct Situation: Decodable {
+    let goal: String?
+    let step: String
+    let expected: String?
+    let failure: String
+    let screen: [String]?
+    let stillMs: Int?
+    let note: String?
+}
+
+func emit(_ object: [String: Any]) {
+    guard let data = try? JSONSerialization.data(withJSONObject: object),
+          let line = String(data: data, encoding: .utf8) else { return }
+    print(line)
+    fflush(stdout)
+}
+
+@available(macOS 26.0, *)
+func serve() async {
+    switch SystemLanguageModel.default.availability {
+    case .available: break
+    case .unavailable(let reason):
+        emit(["unavailable": "\(reason)"])
+        return
+    @unknown default:
+        emit(["unavailable": "unknown availability"])
+        return
+    }
+    // Instructions, reused; the *session* is not.
+    //
+    // One session reused across requests is what made the supervisor go silent
+    // in the field: `LanguageModelSession` accumulates its transcript, so after
+    // seven real failures it hit `exceededContextWindowSize` — 4,441 tokens
+    // against a 4,096 maximum — and every request after that errored. The tester
+    // saw three rulings and then nothing for twenty supervised calls, while
+    // `doctor` in a separate process kept reporting the model available. It
+     // degraded before it broke, too: latency climbed 987ms to 1,890ms as the
+    // transcript grew, and the reasons collapsed into boilerplate repeated
+    // verbatim across unrelated failures.
+    //
+    // Each judgement is independent by nature — a failed step, what was
+    // expected, what is on screen — so there is nothing for a transcript to
+    // carry, and carrying it was pure cost even before it was fatal. The process
+    // stays warm, which is where the model load is paid; only the conversation
+    // is fresh.
+    let instructions = """
+    You supervise a UI test that is running a plan someone else wrote. A step \
+    has just failed. Decide one of three things and nothing else.
+
+    wait  — the screen is still arriving: a spinner, a list that has not \
+    rendered, a count header with no rows, a transition in progress. Waiting \
+    would let the same step succeed.
+    retry — the step is sound and the moment was wrong: something moved under \
+    it, focus was lost, a stale reading. Repeating it would work.
+    stop  — nothing further in the plan can work: the app is somewhere else, a \
+    required control is absent or disabled for a reason, or the screen has not \
+    responded at all.
+
+    Weigh the evidence you are given, in this order.
+
+    The plan's own guidance about this app comes FIRST and outranks everything \
+    below. It is the only knowledge of this app you have, and it was written by \
+    someone who has seen it. If it says lists arrive late, then a row that is \
+    not there yet means wait — even on a screen that has gone completely still, \
+    because a screen waiting on a network call is perfectly still and perfectly \
+    empty.
+
+    If a note says the screen is still filling in — a spinner, a count header \
+    with too few rows, a transition — that is decisive: answer wait.
+    If the screen has been still for under about 1500ms, content may still be \
+    arriving: prefer wait.
+    If the visible labels belong to a different part of the app than the step \
+    implies, answer stop.
+    If a control the step names is present but refused, answer stop.
+    Otherwise, if the step looks like it simply caught a bad moment, answer \
+    retry.
+
+    Never suggest a different step, a different target, or skipping ahead. You \
+    are not being asked what to do, only whether this can proceed. Answer with \
+    the decision alone.
+    """
+    // A throwaway session up front so the first real judgement does not pay
+    // model load — measured at ~880ms against ~600ms warm.
+    _ = try? await LanguageModelSession(instructions: instructions)
+        .respond(to: "Step: warm\nIt failed with: warm", generating: Judgement.self)
+    emit(["ready": true])
+    while let line = readLine(strippingNewline: true) {
+        if line.isEmpty { continue }
+        guard let data = line.data(using: .utf8),
+              let s = try? JSONDecoder().decode(Situation.self, from: data) else {
+            emit(["error": "could not parse that line"])
+            continue
+        }
+        // Capped here as well as in the caller. A single realistic failure
+        // message carries the whole "Visible: …" list, and one request built
+        // from raw input measured 4,266 tokens against a 4,096 window — so a
+        // request can exceed the context on its own, with no transcript
+        // involved at all. Truncation belongs where the limit is.
+        let clip = { (t: String, n: Int) in t.count > n ? String(t.prefix(n)) + "…" : t }
+        var prompt = "Step: \(clip(s.step, 120))\nIt failed with: \(clip(s.failure, 220))"
+        if let goal = s.goal, !goal.isEmpty { prompt += "\nThe plan's guidance about this app: \(clip(goal, 300))" }
+        if let expected = s.expected, !expected.isEmpty { prompt += "\nExpected: \(clip(expected, 200))" }
+        if let ms = s.stillMs { prompt += "\nThe screen has been still for \(ms)ms" }
+        if let note = s.note, !note.isEmpty { prompt += "\nPerception note: \(clip(note, 160))" }
+        if let screen = s.screen, !screen.isEmpty {
+            prompt += "\nOn screen now: \(screen.prefix(14).map { clip($0, 28) }.joined(separator: ", "))"
+        }
+        let started = Date()
+        do {
+            let session = LanguageModelSession(instructions: instructions)
+            let out = try await session.respond(to: prompt, generating: Judgement.self)
+            // No reason field, deliberately. Asked for one it confabulated in
+            // every observed run: a correct `stop` justified as "screen is
+            // elsewhere" when the screen was exactly where the plan expected,
+            // and reasons repeated verbatim across unrelated failures. The
+            // reporter's judgement, and it is right — *"a right answer with a
+            // fabricated justification teaches me to distrust the
+            // justification, which is most of the value of it explaining
+            // itself. No reason at all would be better than a confident wrong
+            // one."* What the caller gets instead is which rule or which model
+            // answered, which is true by construction.
+            emit([
+                "decision": out.content.decision.rawValue,
+                "ms": Int(Date().timeIntervalSince(started) * 1000),
+            ])
+        } catch {
+            emit(["error": "\(error)"])
+        }
+    }
+}
+
+if #available(macOS 26.0, *) {
+    await serve()
+} else {
+    emit(["unavailable": "the on-device model needs macOS 26 or newer"])
+}
+#else
+print("{\"unavailable\":\"this toolchain cannot import FoundationModels\"}")
+#endif

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "simframe",
-  "version": "0.10.0",
+  "version": "0.11.0",
   "mcpName": "io.github.lvlrSajjad/simframe",
   "description": "Always-warm iOS Simulator and Android emulator frames: agents read the screen in ~20ms instead of waiting on screenshots. MCP server + CLI.",
   "keywords": [
@@ -42,8 +42,11 @@
   },
   "files": [
     "src",
+    "data",
     "flows",
     "native/ocr.swift",
+    "native/rank.swift",
+    "native/supervise.swift",
     "native/simframed/Package.swift",
     "native/simframed/Sources",
     "native/simframed/Tests",

package/scripts/check-package.mjs

@@ -60,8 +60,28 @@ function requiredFiles() {
     throw new Error('no Swift test sources found — has the layout moved? This check would silently pass.');
   }
 
-  // The standalone OCR helper, compiled on demand by src/ocr.js.
-  required.add(rel(path.join(ROOT, 'native', 'ocr.swift')));
+  // Every standalone Swift helper compiled on demand from `src/`.
+  //
+  // This listed `native/ocr.swift` by name, which is why it did not catch the
+  // next two: `rank.swift` and `supervise.swift` were absent from `files`, so
+  // the local planner and the local supervisor would have been silently
+  // unavailable for every installed user — the exact failure this whole check
+  // exists to prevent, reproduced because the check named one file instead of
+  // stating the rule.
+  //
+  // The rule is: if a module under `src/` compiles it, it has to ship.
+  const helpers = walk(path.join(ROOT, 'native'), (p) => p.endsWith('.swift'))
+    .filter((p) => !p.includes(`${path.sep}simframed${path.sep}`) && !p.includes(`${path.sep}spikes${path.sep}`));
+  if (!helpers.length) {
+    throw new Error('no standalone Swift helper found under native/ — has the layout moved? This check would silently pass.');
+  }
+  const sources = walk(path.join(ROOT, 'src'), (p) => p.endsWith('.js'))
+    .map((f) => fs.readFileSync(f, 'utf8')).join('\n');
+  for (const helper of helpers) {
+    const name = path.basename(helper);
+    if (!sources.includes(name)) continue; // not compiled by anything; not required
+    required.add(rel(helper));
+  }
 
   // Every runtime module. `files` ships "src" wholesale today; asserting each
   // one catches anybody narrowing that later.

package/scripts/ci-memory.mjs

@@ -56,6 +56,36 @@ function check(ok, label, detail = '') {
   return ok;
 }
 
+let skipped = 0;
+
+/**
+ * A check whose *setup* did not happen, reported as untested rather than failed.
+ *
+ * This exists because the harness did to itself what three peer reports spent a
+ * day telling us not to do to callers. A `simctl launch` timed out on a loaded
+ * runner, `allowFail` swallowed it, and the next check announced `FAIL the
+ * screen actually changed before testing the stale ref — 7070f77757 ->
+ * 7070f77757`. Every word of that is true and it names the wrong thing: the
+ * screen did not change because **the app never launched**, which the run knew
+ * and did not say. Two more checks failed downstream of the same cause.
+ *
+ * A skip does not fail the build, and that is deliberate. A red build caused by
+ * somebody else's build farm is the cry-wolf failure this project keeps writing
+ * down: it trains everyone to re-run rather than to read. But it is counted and
+ * printed, because a run that tested less than it claims must say so.
+ */
+function skip(label, why) {
+  skipped += 1;
+  console.log(`skip ${label} — NOT TESTED: ${why}`);
+  return false;
+}
+
+/** Did a setup flow actually do what it was there for? */
+function ran(res) {
+  if (!res || res.ok === false) return false;
+  return !(res.steps ?? []).some((st) => st.ok === false);
+}
+
 /**
  * `expectFail` asserts a non-zero exit; `allowFail` tolerates one.
  *
@@ -96,7 +126,21 @@ async function json(args, opts) {
  * rather than serving a stale frame, which is the right behaviour and a
  * transient condition. Retry those, and only those.
  */
+// Capture dropping out, and — since 2026-09-11 — simctl timing out.
+//
+// The second one is this repo's oldest CI complaint and it was never in this
+// pattern, so `jsonRetry` sailed past it: `simctl launch` takes 47-55s per
+// attempt on a loaded hosted runner and `simctl openurl` times out internally,
+// which means simframe is handed a failure it did not cause and cannot fix.
+// Three checks in one run failed downstream of exactly that.
+//
+// Retried HERE and deliberately not inside simframe, which is the rule DEFERRED
+// already wrote down for the bench script: a retried launch is an action that
+// fires twice, and the verify barrier exists to stop simframe doing that on its
+// own initiative. A test harness re-running its own setup is a different thing
+// from a driver silently repeating a user's action.
 const TRANSIENT = /did not produce a frame|display surface could not be read|no frames buffered/i;
+const SIMCTL_FLAKE = /simctl|Command failed: xcrun|timed out/i;
 
 async function jsonRetry(args, opts, attempts = 3) {
   let last;
@@ -105,9 +149,13 @@ async function jsonRetry(args, opts, attempts = 3) {
       return await json(args, opts);
     } catch (err) {
       last = err;
-      if (!TRANSIENT.test(err.message)) throw err;
-      console.log(`     (capture dropped out; retrying \`${args.join(' ')}\`)`);
-      await new Promise((r) => setTimeout(r, 1500));
+      const capture = TRANSIENT.test(err.message);
+      const simctl = SIMCTL_FLAKE.test(err.message);
+      if (!capture && !simctl) throw err;
+      console.log(`     (${capture ? 'capture dropped out' : 'simctl did not answer'}; retrying \`${args.join(' ')}\`)`);
+      // simctl's own timeouts are tens of seconds, so a 1.5s pause is not a
+      // wait, it is a formality. Give the runner room when that is the cause.
+      await new Promise((r) => setTimeout(r, simctl ? 5000 : 1500));
     }
   }
   // Out of attempts on a capture error: the device is not blinking, it is gone.
@@ -242,13 +290,15 @@ if (first) {
   // Now leave that screen WITHOUT re-reading it: `--json` skips the end-state
   // map, so the ref table still describes the screen we have left.
   const before = await markHash();
-  await jsonRetry(['do', AT_OTHER_SCREEN], { allowFail: true });
+  const left = await jsonRetry(['do', AT_OTHER_SCREEN], { allowFail: true });
   const after = await markHash();
   // Two different apps are two different screens by construction. The pixel
   // hashes only have to agree with that, and they only get a say when they are
   // informative enough to have one.
   const moved = !informativeHash(before) || !informativeHash(after) || before !== after;
-  check(moved, 'the screen actually changed before testing the stale ref',
+  if (!ran(left)) skip('the screen actually changed before testing the stale ref',
+    'the second app never launched, so there was no screen change to test against');
+  else check(moved, 'the screen actually changed before testing the stale ref',
     `${before.slice(0, 10)} -> ${after.slice(0, 10)}`
     + (informativeHash(before) && informativeHash(after) ? '' : ' (degenerate hash: not evidence either way)'));
   // Only assert the guard if the precondition actually held. Running it anyway
@@ -256,17 +306,28 @@ if (first) {
   // screen, which is a false accusation against the one layer this file exists
   // to defend — and it is how this check has failed twice.
   if (moved) {
-    const stale = await cli(['find', `#${first.ref}`], { expectFail: true });
-    // Matched on prose, which is this check's weakness: simframe refused
-    // correctly with "#1 cannot be trusted here — simframe does not recognise
-    // this screen. Read it again", and the check failed because that wording was
-    // not one of the two it knew. The refusal is what matters, so the
-    // alternation covers how a refusal is actually phrased; the durable fix is a
-    // machine-readable reason on the failure, which `find --json` does not yet
-    // carry.
-    check(/different screen|read the screen|read it again|does not recognise this screen|cannot be trusted/i.test(stale),
+    // This matched on prose twice and went red twice, both times for a refusal
+    // that was correct and better worded than the alternation knew — most
+    // recently `"Welcome to Reminders" is not on this screen`, which refuses
+    // *and* names what the number stood for. `find --json` now carries the
+    // reason as a field, so the check reads the contract instead of the
+    // sentence. What is under test is unchanged: the ref must not resolve to
+    // the coordinates it was numbered at on the screen we have left.
+    // A refusal that cannot be parsed is a failed check, not a dead script:
+    // `json` throws on anything non-JSON reaching the stream, and this is the
+    // one call site that expects a failure, so it is the one that would take
+    // the whole file down with it.
+    let stale;
+    try {
+      stale = await json(['find', `#${first.ref}`], { expectFail: true });
+    } catch (err) {
+      stale = { ok: null, error: err.message };
+    }
+    const refused = stale.ok === false
+      && (stale.staleRef === true || stale.reason === 'unknown_screen' || stale.reason === 'ambiguous_intent');
+    check(refused,
       'a ref numbered on another screen refuses instead of tapping those coordinates',
-      stale.trim().split('\n')[0]?.slice(0, 90));
+      `${stale.reason ?? 'no reason'}${stale.staleRef ? ' staleRef' : ''} — ${String(stale.error ?? '').split('\n')[0].slice(0, 70)}`);
   }
 } else {
   check(false, 'element refs', 'no elements to number');
@@ -343,7 +404,15 @@ const novelVerdicts = novelSteps.length
 // accusing the layer underneath it. A device that crashed SpringBoard mid-run
 // has not told us anything about the graph.
 const novelRan = novelSteps.length > 0 && novelSteps.every((r) => r.ok !== false);
-check(novelRan, 'the novel action ran at all', `[${novelVerdicts.join(', ')}]`);
+// The comment above states the principle and this line used to contradict it:
+// it called `check`, so a `simctl openurl` that timed out on the runner failed
+// the build and said "the novel action ran at all" as though the graph were at
+// fault. It is a precondition. Untested is not broken.
+if (!novelRan) {
+  skip('the novel action ran at all', `the action could not be dispatched — [${novelVerdicts.join(', ')}]`);
+} else {
+  check(true, 'the novel action ran at all', `[${novelVerdicts.join(', ')}]`);
+}
 // The other half of the precondition, which was written above as a comment and
 // then trusted. It is not trustworthy: the positioning run sends the device
 // home, and a simulator that has been driven hard stops delivering `home` while
@@ -361,9 +430,20 @@ if (novelRan && novelMoved) {
     'an action never taken here before is reported as unverified, not as verified',
     `[${novelVerdicts.join(', ')}]`);
 }
-check(passes.some((p) => p.verdicts.includes('ok')),
-  'and once the graph has seen it, the outcome is predicted',
-  `pass ${passes.findIndex((p) => p.verdicts.includes('ok')) + 1}`);
+// The same precondition rule as the two above. The graph can only predict an
+// outcome it has seen, and it can only have seen one if a pass actually ran —
+// so a run in which every pass failed to dispatch says nothing about
+// prediction. It failed the build as `pass 0` while the real cause was a
+// simctl launch timing out, three checks upstream.
+const anyPassRan = passes.some((p) => Array.isArray(p.run?.results) && p.run.results.some((r) => r.ok !== false));
+if (!anyPassRan) {
+  skip('and once the graph has seen it, the outcome is predicted',
+    'no pass dispatched a step, so the graph was never given anything to learn');
+} else {
+  check(passes.some((p) => p.verdicts.includes('ok')),
+    'and once the graph has seen it, the outcome is predicted',
+    `pass ${passes.findIndex((p) => p.verdicts.includes('ok')) + 1}`);
+}
 
 // One direction only: a wrong turn must fail the run. The converse does not
 // hold — a run can fail for reasons that are not wrong turns, such as a step
@@ -472,5 +552,9 @@ try {
   }
 } catch { /* nothing to clean up */ }
 
-console.log(`\n${failures ? `${failures} check(s) failed` : 'every check passed'}`);
+const summary = [
+  failures ? `${failures} check(s) failed` : 'every check passed',
+  skipped ? `${skipped} check(s) NOT TESTED — the runner could not set them up` : null,
+].filter(Boolean).join('; ');
+console.log(`\n${summary}`);
 process.exit(failures ? 1 : 0);

package/scripts/eval-perception.mjs

@@ -36,6 +36,7 @@ import { fileURLToPath } from 'node:url';
 import * as fingerprint from '../src/fingerprint.js';
 import * as matching from '../src/matching.js';
 import * as analyze from '../src/analyze.js';
+import * as view from '../src/view.js';
 
 const ROOT = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '..');
 const DIR = path.join(ROOT, 'test', 'perception', 'screens');
@@ -68,6 +69,7 @@ export function checkScreen(fx) {
   const authored = (expect.resolutions?.length ?? 0)
     + (expect.ambiguous?.length ?? 0)
     + (expect.none?.length ?? 0)
+    + (expect.discoverable?.length ?? 0)
     + (fx.frame_pairs?.length ?? 0)
     + (expect.identity ? 1 : 0);
 
@@ -118,6 +120,37 @@ export function checkScreen(fx) {
     }
   }
 
+  // Discovery: does the default map still SAY what is on this screen?
+  //
+  // This check exists because of a regression it would have caught. A rule that
+  // dropped long non-interactive rows left every React Native list card out of
+  // the map — the cards expose their children as one concatenated accessibility
+  // label, so they look exactly like a Settings caption. Nothing became
+  // untappable, because `locate` reads targets rather than rows, so every
+  // resolution expectation above still passed. What was lost was the map's
+  // account of what is there, and an agent that cannot see a row falls back to
+  // a ~1600-token screenshot.
+  //
+  // Resolution and discovery are different claims, and the harness could only
+  // make the first one.
+  if (expect.discoverable?.length && screen) {
+    const { rows } = view.rowsFor({ targets }, { screen });
+    const shown = rows.map((r) => String(r.label ?? ''));
+    for (const want of expect.discoverable) {
+      // Compared as a prefix, because a long label is legitimately truncated in
+      // a row — truncated is discoverable, absent is not.
+      const head = want.slice(0, 24);
+      if (!shown.some((got) => got.startsWith(head))) {
+        findings.push({
+          kind: 'discovery',
+          query: `${want.slice(0, 40)}${want.length > 40 ? '…' : ''}`,
+          want: 'listed in the default map',
+          got: `${rows.length} row(s), none starting with it`,
+        });
+      }
+    }
+  }
+
   // Identity drift. A token-rule change that silently alters a screen's
   // structural hash discards every stored map and graph for it, and the failure
   // is invisible — an old hash is a well-formed hash that matches nothing.

package/scripts/phase17-corpus.mjs

@@ -0,0 +1,176 @@
+#!/usr/bin/env node
+/**
+ * Phase 17, go/no-go step 1: build the corpus and ask whether there is a job.
+ *
+ * The plan in docs/PHASES-HUMAN-PARITY.md said to export 200 `ambiguous_intent`
+ * and `no_plan` escalations. That corpus cannot be assembled: `no_plan` has
+ * never been logged once, `ambiguous_intent` is the reason the ranking bug
+ * produced (so the pre-fix records encode a bug), and the session id is minted
+ * per process — so a CLI-driven agent gets one "session" per command and the
+ * "filter to one session id" step has nothing to filter.
+ *
+ * There is a better corpus and it was there all along. Every verified graph edge
+ * is a decision that *worked*: the goal is in `step`, the screen it was taken on
+ * is the node, and the element list for that screen is in the screen map. That
+ * is (goal, element list, action) for every successful step, not just the rare
+ * failures — and the ground truth is stronger, because the tap was verified.
+ *
+ * What this measures is the question the whole phase turns on: of the decisions
+ * an agent actually makes, how many does the local matcher already resolve? A
+ * local planner can only earn its keep on the remainder. If the remainder is
+ * small, Phase 17 is a no-go regardless of how good the model is.
+ *
+ * Output is aggregate by design. This reads a real device's memory of real
+ * third-party apps, so it prints counts and never a label, an app name or a
+ * screen's contents. See the standing rule in scripts/check-private.mjs.
+ *
+ * Usage: node scripts/phase17-corpus.mjs [--device <udid>] [--json]
+ */
+import { readdirSync, readFileSync, existsSync } from 'node:fs';
+import { join } from 'node:path';
+import { homedir } from 'node:os';
+import * as matching from '../src/matching.js';
+import { goalOf } from '../src/actions.js';
+
+const ROOT = process.env.SIMFRAME_HOME || join(homedir(), '.simframe');
+
+const readJson = (p) => {
+  try { return JSON.parse(readFileSync(p, 'utf8')); } catch { return null; }
+};
+const listDir = (p) => (existsSync(p) ? readdirSync(p).filter((f) => f.endsWith('.json')) : []);
+
+/** Screens are keyed by frame hash; graph nodes carry a layoutHash. Index both. */
+function screenIndex(dev) {
+  const idx = new Map();
+  for (const f of listDir(join(dev, 'screens'))) {
+    const s = readJson(join(dev, 'screens', f));
+    if (!s?.targets?.length) continue;
+    for (const k of ['layoutHash', 'structuralHash', 'hash']) {
+      if (s[k] && !idx.has(s[k])) idx.set(s[k], s);
+    }
+  }
+  return idx;
+}
+
+/**
+ * Only some steps involve choosing an element. `launch` names an app, `button`
+ * names hardware, `openUrl` names a URL, `scroll` names a direction — none of
+ * them is a decision a planner could help with, and counting them was inflating
+ * the "not found" bucket to 45% on the first run of this script.
+ */
+const CHOOSES_AN_ELEMENT = new Set(['tap', 'type', 'scroll_to', 'scrollTo', 'assert', 'swipe_from', 'longPress']);
+
+/** A "#3" is answered by the ref table and a "@x,y" by arithmetic. Neither is a decision. */
+const isSelectorNotIntent = (goal) => /^#\d+$/.test(goal) || /^@?-?\d+\s*,\s*-?\d+$/.test(goal);
+
+export function classify(dev) {
+  const idx = screenIndex(dev);
+  const out = {
+    screens_stored: idx.size,
+    edges: 0,
+    unjoinable: 0,
+    no_goal: 0,
+    not_an_element_step: 0,
+    selector_not_intent: 0,
+    cases: 0,
+    resolved: 0,
+    ambiguous: 0,
+    none: 0,
+  };
+  for (const f of listDir(join(dev, 'graph'))) {
+    const g = readJson(join(dev, 'graph', f));
+    const screen = idx.get(g?.layoutHash) ?? idx.get(g?.hash);
+    for (const e of g?.edges ?? []) {
+      out.edges += 1;
+      if (!screen) { out.unjoinable += 1; continue; }
+      const goal = goalOf(e.step);
+      if (!goal) { out.no_goal += 1; continue; }
+      if (!CHOOSES_AN_ELEMENT.has(e.step?.action)) { out.not_an_element_step += 1; continue; }
+      if (isSelectorNotIntent(String(goal).trim())) { out.selector_not_intent += 1; continue; }
+      out.cases += 1;
+      const r = matching.resolve(screen.targets, String(goal));
+      if (r.status === 'ok') out.resolved += 1;
+      else if (r.status === 'ambiguous') out.ambiguous += 1;
+      else out.none += 1;
+    }
+  }
+  return out;
+}
+
+/**
+ * The second instrument, with the opposite bias.
+ *
+ * A verified edge is a decision that *succeeded*, so the graph cannot see the
+ * ones the matcher fumbled — those became escalations, Claude fixed them, and
+ * the edge was written with the corrected goal. Measuring the matcher on its
+ * own successes is partly circular. So also count resolution failures from the
+ * log against total edge traversals: same question, denominator built from
+ * failures instead of successes.
+ */
+export function resolutionFailureRate(dev) {
+  let traversals = 0;
+  for (const f of listDir(join(dev, 'graph'))) {
+    for (const e of readJson(join(dev, 'graph', f))?.edges ?? []) traversals += e.count ?? 0;
+  }
+  let failures = 0;
+  let escalations = 0;
+  const log = join(dev, 'escalations.jsonl');
+  if (existsSync(log)) {
+    for (const line of readFileSync(log, 'utf8').split('\n')) {
+      if (!line.trim()) continue;
+      let r;
+      try { r = JSON.parse(line); } catch { continue; }
+      escalations += 1;
+      if (r.reason === 'ambiguous_intent' || r.reason === 'unknown_screen') failures += 1;
+    }
+  }
+  return { traversals, escalations, failures };
+}
+
+const flags = process.argv.slice(2);
+const only = flags.includes('--device') ? flags[flags.indexOf('--device') + 1] : null;
+const devices = (existsSync(ROOT) ? readdirSync(ROOT) : [])
+  .filter((d) => !d.startsWith('TEST-') && d !== 'bin')
+  .filter((d) => existsSync(join(ROOT, d, 'graph')))
+  .filter((d) => !only || d === only);
+
+const totals = { cases: 0, resolved: 0, ambiguous: 0, none: 0, edges: 0, unjoinable: 0, not_an_element_step: 0, selector_not_intent: 0 };
+const rows = [];
+for (const d of devices) {
+  const r = classify(join(ROOT, d));
+  if (!r.edges) continue;
+  rows.push({ device: `${d.slice(0, 8)}…`, ...r, ...resolutionFailureRate(join(ROOT, d)) });
+  for (const k of Object.keys(totals)) totals[k] += r[k];
+}
+
+if (flags.includes('--json')) {
+  console.log(JSON.stringify({ devices: rows, totals }, null, 2));
+} else {
+  console.log('Phase 17 corpus — verified graph edges as (goal, element list, action)\n');
+  for (const r of rows) {
+    console.log(`  ${r.device}  edges ${r.edges}  joinable ${r.edges - r.unjoinable}  decisions ${r.cases}` +
+      `  → matcher ok ${r.resolved}, ambiguous ${r.ambiguous}, not found ${r.none}`);
+  }
+  const { cases, resolved, ambiguous, none } = totals;
+  const pct = (n) => (cases ? `${Math.round((n / cases) * 1000) / 10}%` : '—');
+  console.log(`\n  ${totals.edges} edges: ${totals.unjoinable} unjoinable, ` +
+    `${totals.not_an_element_step} chose no element (launch/button/openUrl/scroll), ` +
+    `${totals.selector_not_intent} used a #ref or a coordinate.`);
+  console.log(`  TOTAL element decisions: ${cases}`);
+  console.log(`    already resolved locally  ${resolved}  ${pct(resolved)}   <- a planner adds nothing here`);
+  console.log(`    ambiguous                 ${ambiguous}  ${pct(ambiguous)}   <- a planner could pick`);
+  console.log(`    not found                 ${none}  ${pct(none)}   <- a planner cannot invent an element`);
+  const addressable = ambiguous;
+  console.log(`\n  Addressable by a local planner: ${addressable} of ${cases} (${pct(addressable)}).`);
+
+  console.log('\nSecond instrument — resolution failures from the log, per traversal.');
+  console.log('A rate over 100% means the graph was discarded while the log kept appending');
+  console.log('(a MAP_VERSION or GRAPH_VERSION bump), so it is not a rate. Read the device');
+  console.log('that was driven by an agent doing real work, not the bench device.\n');
+  for (const r of rows) {
+    const rate = r.traversals ? `${Math.round((r.failures / r.traversals) * 1000) / 10}%` : '—';
+    console.log(`  ${r.device}  traversals ${String(r.traversals).padStart(4)}` +
+      `  escalations ${String(r.escalations).padStart(4)}` +
+      `  resolution failures ${String(r.failures).padStart(3)}  ${rate.padStart(6)}`);
+  }
+}