simframe 0.12.1 → 0.13.0

package/scripts/ci-memory.mjs

@@ -50,6 +50,9 @@ let failures = 0;
  */
 let deviceDied = null;
 
+/** EX_TEMPFAIL: the device died under the checks, so nothing was tested. */
+const DEVICE_DIED_EXIT = 75;
+
 function check(ok, label, detail = '') {
   if (!ok) failures += 1;
   console.log(`${ok ? 'ok  ' : 'FAIL'} ${label}${detail ? ` — ${detail}` : ''}`);
@@ -171,9 +174,21 @@ async function jsonRetry(args, opts, attempts = 3) {
     console.error(`  ${String(last.message).split('\n')[0]}`);
     console.error('\nEverything after this point would be testing a dead simulator, so the run');
     console.error('stops here. This is not a memory-layer failure — it is the device-state');
-    console.error('problem in docs/DEFERRED.md. A device restart is the only known cure;');
-    console.error('on a hosted runner it means a retry.');
-    process.exit(1);
+    console.error('problem in docs/DEFERRED.md (126). A device restart is the only known cure,');
+    console.error('and `simframe revive` is that restart.');
+    // Exit 75, not 1, and the distinction is the whole point of this file.
+    //
+    // "A check about the memory layer failed" and "the simulator died under the
+    // checks" are different conditions with different responses, and for two CI
+    // rounds they were one exit code — so a caller could only retry everything
+    // or retry nothing. 75 is EX_TEMPFAIL, which is exactly what this is: the
+    // subject under test was never reached.
+    //
+    // The caller reviving and running again is not papering over a product bug.
+    // The wedge is a documented CoreSimulator condition, `frame --fresh` names
+    // it, and `revive` is the cure this project ships for it — CI simply had no
+    // way to say "use it".
+    process.exit(DEVICE_DIED_EXIT);
   }
   throw last;
 }
@@ -244,7 +259,36 @@ if (failures) {
 
 console.log('\n--- the screen map ---');
 await jsonRetry(['do', LOOP], { allowFail: true });
-const map = await jsonRetry(['ui']);
+const map = await readableMap();
+
+/**
+ * A map that is empty *and* says a sensor failed is a blink, not a result.
+ *
+ * `jsonRetry` retries a command that throws, and this one did not throw: `ui`
+ * returned 200 with zero elements and a degraded note, which is the shape of
+ * failure this whole repo keeps writing rules about. The run that made this
+ * necessary had the accessibility read time out once, report an empty map, and
+ * then resolve a ref correctly sixty seconds later on the same device — so the
+ * device was fine and the check had caught one bad read.
+ *
+ * Deliberately narrow. An empty map with **no** degraded sensor is a real
+ * answer — that is a blank screen and the check should fail on it. Only an
+ * empty map that admits a layer did not answer is worth asking again, and after
+ * three attempts it fails with what the sensor said, which is the diagnosis
+ * either way.
+ */
+async function readableMap(attempts = 3) {
+  let last;
+  for (let i = 0; i < attempts; i += 1) {
+    last = await jsonRetry(['ui']);
+    if (last.elements?.length || !(last.degraded ?? []).length) return last;
+    if (i < attempts - 1) {
+      console.log(`     (the map came back empty and a sensor said why — retrying \`ui\`: ${(last.degraded ?? []).join('; ')})`);
+      await new Promise((r) => setTimeout(r, 2000));
+    }
+  }
+  return last;
+}
 
 // Report what actually answered rather than asserting the runner's situation.
 // This line used to read "with no accessibility tree available" unconditionally,
@@ -263,14 +307,38 @@ check(Number.isFinite(map.points?.width) && Number.isFinite(map.points?.height),
   'the map knows the screen size in points', `${map.points?.width}x${map.points?.height}pt`);
 
 const refs = (map.elements ?? []).map((e) => e.ref);
-check(refs.every((r, i) => r === i + 1),
-  'refs are numbered 1..n with no gaps', `#1..#${refs.length}`);
-check((map.elements ?? []).every((e) =>
-  Number.isInteger(e.x) && Number.isInteger(e.y)
-  && e.y >= 0 && e.y <= map.points.height && e.x >= 0 && e.x <= map.points.width),
-  'every element has a tap point on the screen');
-check(!(map.elements ?? []).some((e) => e.region === 'status-bar'),
-  'the status bar is not offered as something to tap');
+// Three checks over a collection, and `every`/`!some` are all true of an empty
+// one. On a CI run whose map came back with **0 elements** they printed
+// `ok   refs are numbered 1..n with no gaps — #1..#0` and two more like it:
+// three lines of reassurance about nothing, directly under the failure that
+// said the map was empty.
+//
+// That is the exact defect three field reports spent a day describing — a
+// confident statement that verified nothing — and the harness was doing it to
+// itself in the same output. Untested is not passed.
+if (!map.elements?.length) {
+  for (const label of [
+    'refs are numbered 1..n with no gaps',
+    'every element has a tap point on the screen',
+    'the status bar is not offered as something to tap',
+  ]) skip(label, 'the map was empty, so there was nothing to check');
+  // And say what the device was showing, because an empty map on a live device
+  // is the shape this project has chased under four different symptoms. The
+  // liveness note carries the ignored-gesture check; `stableForMs` and the
+  // frame age are the two numbers whose disagreement names a dead surface.
+  const st = await jsonRetry(['state'], { allowFail: true });
+  console.log(`     the device at that moment: frame #${st?.seq ?? '?'}, `
+    + `${st?.stableForMs ?? '?'}ms still, ${st?.live?.note ?? 'liveness reported nothing'}`);
+} else {
+  check(refs.every((r, i) => r === i + 1),
+    'refs are numbered 1..n with no gaps', `#1..#${refs.length}`);
+  check(map.elements.every((e) =>
+    Number.isInteger(e.x) && Number.isInteger(e.y)
+    && e.y >= 0 && e.y <= map.points.height && e.x >= 0 && e.x <= map.points.width),
+    'every element has a tap point on the screen');
+  check(!map.elements.some((e) => e.region === 'status-bar'),
+    'the status bar is not offered as something to tap');
+}
 
 console.log('\n--- element refs ---');
 // Re-read on a screen we chose, rather than on whatever the device happened to
@@ -296,7 +364,8 @@ if (first) {
   // hashes only have to agree with that, and they only get a say when they are
   // informative enough to have one.
   const moved = !informativeHash(before) || !informativeHash(after) || before !== after;
-  if (!ran(left)) skip('the screen actually changed before testing the stale ref',
+  const launched = ran(left);
+  if (!launched) skip('the screen actually changed before testing the stale ref',
     'the second app never launched, so there was no screen change to test against');
   else check(moved, 'the screen actually changed before testing the stale ref',
     `${before.slice(0, 10)} -> ${after.slice(0, 10)}`
@@ -305,7 +374,16 @@ if (first) {
   // reports "the stale-ref guard failed" for a device that never left the
   // screen, which is a false accusation against the one layer this file exists
   // to defend — and it is how this check has failed twice.
-  if (moved) {
+  // A skip has to propagate. The precondition above reported NOT TESTED and this
+  // check ran anyway and failed — which is the harness doing to itself, one line
+  // later, exactly what `skip` was written to stop it doing. `moved` is true
+  // when a hash is too degenerate to have a say, and that is right for "did the
+  // screen change" and wrong as a licence to run a check whose setup is known
+  // not to have happened.
+  if (!launched) {
+    skip('a ref numbered on another screen refuses instead of tapping those coordinates',
+      'we never reached another screen, so there was nothing to refuse from');
+  } else if (moved) {
     // This matched on prose twice and went red twice, both times for a refusal
     // that was correct and better worded than the alternation knew — most
     // recently `"Welcome to Reminders" is not on this screen`, which refuses
@@ -491,7 +569,12 @@ const walked = await jsonRetry(['goto', target.hash], { allowFail: true });
 // neither walking there nor naming why it cannot, so this check failed with an
 // empty detail — the reason was `undefined` — and the check was right to fail.
 // Now there is a name for it, and the list has to know the name.
-const outcomes = ['no-route', 'unreplayable-edge', 'ambiguous', 'unknown-screen', 'no-identity'];
+// `route-halted` and `arrived-elsewhere` are new: a walk that ran and did not
+// land used to return `{ok: false}` with no reason at all, which failed this
+// very check with an empty detail. It was the one outcome here nobody had
+// named, and the check found it.
+const outcomes = ['no-route', 'unreplayable-edge', 'ambiguous', 'unknown-screen', 'no-identity',
+  'route-halted', 'arrived-elsewhere'];
 check(walked.ok === true || outcomes.includes(walked.reason),
   'and asked for a screen it knows, it either walks there or names why it cannot',
   walked.ok ? (walked.already ? 'already there' : `walked ${walked.ranSteps} step(s)`) : walked.reason);

package/scripts/collect-rulings.mjs

@@ -310,3 +310,17 @@ if (unattributed) console.log(`\n${unattributed} ruling(s) came from a step this
   }
 }
 console.log(`\nthe log now holds ${all.length} ruling(s) — simframe supervisions --device=${dev.udid}`);
+
+// Close the helper, or this script never exits.
+//
+// The supervisor keeps a warm child process and deliberately does not unref its
+// stdout — unreferencing it once unreferenced the pipe every request waits on,
+// and the process then exited silently mid-await. The consequence nobody had
+// noticed is at *this* end: the script finishes its work, prints this line, and
+// then sits at 0% CPU forever with the helper idle at `readLine`, because the
+// live child is holding the event loop open.
+//
+// That is the whole story of three "orphaned" collect-rulings processes killed
+// by PID the night before, which were attributed to a task runner not killing
+// its children. They had each finished their work and could not leave.
+supervisor.close();

package/scripts/eval-fingerprint.mjs

@@ -129,7 +129,24 @@ for (let round = 1; round <= rounds; round += 1) {
     if (screen.steps?.length) {
       // Verification is off: this measures fingerprints, and a wrong-turn
       // verdict computed from the very tokens under test would be circular.
-      await actions.runScript(device, { steps: screen.steps, verify: false });
+      //
+      // The tour asserts arrival rather than sleeping through it, so a step
+      // here can now fail — `openUrl` returns NSPOSIXErrorDomain 60 on a loaded
+      // hosted runner, and a page that never renders no longer passes silently
+      // as a six-token reading. That is the trade this harness wants: a loud
+      // failure naming the screen, over a quiet one that shows up thirty lines
+      // later as a threshold with no clearance. Everything read so far is
+      // written out first, because a failing run is the one whose evidence
+      // matters.
+      try {
+        await actions.runScript(device, { steps: screen.steps, verify: false });
+      } catch (err) {
+        save({ abandonedAt: { screen: screen.name, round, error: err.message } });
+        console.error(`\nFAIL round ${round}, "${screen.name}" never arrived: ${err.message}`);
+        console.error('The tour waits for something each screen actually shows. This is that wait');
+        console.error('giving up — not a fingerprint result. Check the app, the network, or the runner.');
+        process.exit(1);
+      }
     }
     const id = await api.screenIdentity(device, { fresh: true, confirmNovel: false });
     // Did we actually arrive? Two differently-named screens reading the same
@@ -147,8 +164,21 @@ for (let round = 1; round <= rounds; round += 1) {
       // means the navigation did not happen.
       const s = fingerprint.similarity(previous.tokens, id.tokens ?? []);
       if (s >= ARRIVAL_SUSPICION) {
+        // What was actually on screen, not just that it was the wrong thing.
+        //
+        // This check has fired twice on CI and both times the report was a
+        // similarity score and two screen names, which is enough to know the
+        // run is void and not enough to know why. The tour asserts arrival with
+        // a `waitFor` before this ever runs, so a failure here means the
+        // `waitFor` *passed* on a screen that was not the destination — and the
+        // labels are the only thing that can say what that screen was.
+        const labels = (id.entry?.targets ?? [])
+          .map((t) => t.label).filter(Boolean).slice(0, 12).map((l) => l.slice(0, 24));
         arrivalFailures.push(
-          `${previous.name} -> ${screen.name}: similarity ${s.toFixed(2)} — the screen did not change`);
+          `${previous.name} -> ${screen.name}: similarity ${s.toFixed(2)} — the screen did not change`
+          + `\n         sensors: ${(id.entry?.sources ?? []).join('+') || 'none'}`
+          + `, settled: ${id.settled}, frame ${Math.round(Date.now() - (id.state?.capturedAt ?? Date.now()))}ms old`
+          + `\n         on screen: ${labels.length ? labels.join(' · ') : '(nothing readable)'}`);
       }
     }
     readings.push({
@@ -199,6 +229,12 @@ if (arrivalFailures.length) {
   console.error('measure the tour rather than the fingerprint. Fix the tour and re-run.');
   console.error('A `settle` step defaults to mode "stable", which returns instantly in the');
   console.error('moment before an animation begins — action steps already settle on their own.');
+  console.error('');
+  console.error('The tour asserts arrival with a `waitFor` before any reading is taken, so a');
+  console.error('failure here means that wait PASSED on a screen that was not the destination.');
+  console.error('Read the labels above before changing the tour: either the wait matched');
+  console.error('something it should not have, or the reading came off a frame older than the');
+  console.error('navigation — and those have opposite fixes.');
   process.exit(1);
 }
 
@@ -339,7 +375,16 @@ console.log(`${thresholdInGap ? 'ok  ' : 'WARN'} the threshold ${thresholdInGap
 const worstSame = [...same].sort((a, b) => a.similarity - b.similarity)[0];
 const worstDifferent = [...different].sort((a, b) => b.similarity - a.similarity)[0];
 if (worstSame) {
-  console.log(`\nweakest same-screen pair:   ${worstSame.a.name} r${worstSame.a.round} vs r${worstSame.b.round} = ${f(worstSame.similarity)} (${worstSame.a.count} vs ${worstSame.b.count} tokens)`);
+  // Say whether either side was read off a screen that was still moving.
+  //
+  // The counts have always been printed and the settle flag never was, so a red
+  // run showing `11 vs 6 tokens` left it open whether the fingerprint had
+  // drifted or one reading had simply been taken too early. It is printed per
+  // reading above, thirty lines away and on a different row; here it is beside
+  // the number it explains.
+  const moving = [worstSame.a, worstSame.b].filter((r) => r.settled === false).length;
+  const movingNote = moving ? `, ${moving === 2 ? 'both readings' : 'one reading'} taken on a screen that never settled` : '';
+  console.log(`\nweakest same-screen pair:   ${worstSame.a.name} r${worstSame.a.round} vs r${worstSame.b.round} = ${f(worstSame.similarity)} (${worstSame.a.count} vs ${worstSame.b.count} tokens${movingNote})`);
 }
 if (worstDifferent) {
   console.log(`closest different-screen pair: ${worstDifferent.a.name} vs ${worstDifferent.b.name} = ${f(worstDifferent.similarity)}`);

package/scripts/replay-rulings.mjs

@@ -0,0 +1,206 @@
+#!/usr/bin/env node
+// Ask several supervisors the same questions.
+//
+//   node scripts/replay-rulings.mjs --device=<udid> --arms=apple,ollama:qwen3:8b,ollama:qwen3:14b
+//
+// The owner's call, 2026-09-11: settle the capacity question with numbers
+// rather than speculation. This is how, and the shape matters more than the
+// result.
+//
+// **Why replay rather than re-drive.** Collecting a population per arm means
+// driving the simulator once per arm — about half an hour each — and, worse, it
+// puts the *device's* variance inside a comparison that is supposed to be about
+// the judges. A list that happened to arrive faster on one pass than another is
+// not a fact about a model. So one device pass records the situations (see
+// `situation` in metrics.recordSupervision) and every arm answers the identical
+// set.
+//
+// **What replay cannot measure**, stated here rather than discovered later: the
+// `outcome` column. Whether acting on a ruling actually recovered the flow is a
+// fact about the device at that moment, and it belongs to the arm that was
+// live. Replay scores *decisions*. The distinction is load-bearing — the live
+// population scored 91% by decision and 69% by outcome, and almost all of that
+// gap was one fixture that took the right word eight times and recovered once.
+import * as metrics from '../src/metrics.js';
+import * as ollama from '../src/ollama.js';
+import * as supervisor from '../src/supervisor.js';
+import { resolveDevice } from '../src/platform/index.js';
+
+const arg = (n, d) => {
+  const hit = process.argv.find((a) => a.startsWith(`--${n}=`));
+  return hit ? hit.slice(n.length + 3) : d;
+};
+
+/** The fixtures and the word each situation actually calls for. Same table as the scorer. */
+const CORRECT = {
+  'the list is still loading; its rows arrive shortly after launch': 'wait',
+  'the detail screen is still fetching; its text arrives shortly': 'wait',
+  'the list arrives in waves and this row is in the last one': 'wait',
+  'Review is blocked until Species is filled in, and it is empty': 'stop',
+  'the first submit always fails and the second works, so waiting cannot help': 'stop',
+};
+
+/** `retry` and `wait` differ only in how long they settle, so both satisfy a wait. */
+const satisfies = (d, want) => (want === 'wait' ? d === 'wait' || d === 'retry' : d === want);
+
+const dev = await resolveDevice(arg('device'));
+const arms = String(arg('arms', 'apple')).split(',').map((a) => a.trim()).filter(Boolean);
+const lastN = Number(arg('last', 0));
+
+const all = metrics.readSupervisions(dev.udid)
+  .filter((r) => CORRECT[r.expect] && r.situation?.step && r.situation?.failure);
+const rows = lastN > 0 ? all.slice(-lastN) : all;
+
+if (rows.length < 8) {
+  console.error(`only ${rows.length} replayable ruling(s) — need situations recorded, which`);
+  console.error('means a population collected after the situation field was added.');
+  console.error('Run: SIMFRAME_SUPERVISOR=apple node scripts/collect-rulings.mjs --device=<udid>');
+  process.exit(2);
+}
+
+const want = (r) => CORRECT[r.expect];
+const counts = rows.reduce((a, r) => ({ ...a, [want(r)]: (a[want(r)] ?? 0) + 1 }), {});
+const commonest = Object.entries(counts).sort((a, b) => b[1] - a[1])[0];
+const baseline = commonest[1] / rows.length;
+const pct = (x) => `${(100 * x).toFixed(0)}%`;
+
+console.log(`${rows.length} situations from ${dev.name}`);
+console.log(`balance: ${JSON.stringify(counts)}`);
+console.log(`majority-class baseline: always "${commonest[0]}" scores ${pct(baseline)}`);
+if (baseline > 0.65) {
+  console.log('\nSKEWED — any accuracy below is mostly a fact about the fixture set.');
+}
+// The free comparison, computed here so it is in the same table as the models
+// rather than in a different report. It is not an arm; it is the thing every
+// arm has to beat to be worth its latency.
+const STILL_MS_THRESHOLD = 3000;
+console.log(`\nthe brief every model arm gets is ${ollama.readBrief().length} characters, read from native/supervise.swift`);
+
+const withAbstain = process.argv.includes('--abstain');
+
+const results = [];
+for (const arm of arms) {
+  const judged = [];
+  let unanswered = 0;
+  // Weights first, so the first situation is not timing a disk read.
+  if (arm.startsWith('ollama')) await ollama.preload(ollama.parseTarget(arm));
+  process.stdout.write(`\nasking ${arm} …`);
+  for (const r of rows) {
+    const detail = {};
+    const ruling = await supervisor.judge({
+      ...r.situation,
+      mayAbstain: withAbstain,
+      options: { supervisor: arm },
+      // Wide on purpose. The shipped budget is 2.5s and a 14B will exceed it;
+      // capping here would score the larger model on *latency* while calling it
+      // accuracy, and latency is reported separately below where it can be read
+      // for what it is.
+      timeoutMs: 30_000,
+      detail,
+    });
+    if (!ruling) unanswered += 1;
+    judged.push({ r, ruling, detail, abstained: detail.kind === 'abstained' });
+    process.stdout.write('.');
+  }
+  const answered = judged.filter((j) => j.ruling);
+  const right = answered.filter((j) => satisfies(j.ruling.decision, want(j.r))).length;
+  const lat = answered.map((j) => j.ruling.ms).filter(Number.isFinite).sort((a, b) => a - b);
+  results.push({
+    arm,
+    judged,
+    n: rows.length,
+    unanswered,
+    // Scored over every situation, not only the answered ones. A judge that
+    // declines half the questions and is right about the rest is not an 100%
+    // judge, and scoring only its answers would say it was.
+    accuracy: right / rows.length,
+    medianMs: lat.length ? lat[lat.length >> 1] : null,
+    errors: answered
+      .filter((j) => !satisfies(j.ruling.decision, want(j.r)))
+      .map((j) => `said "${j.ruling.decision}" where "${want(j.r)}" was right — ${j.r.expect.slice(0, 48)}`),
+  });
+  process.stdout.write(' done\n');
+}
+
+const stillRule = rows.filter((r) => {
+  const d = (r.situation.stillMs ?? 0) > STILL_MS_THRESHOLD ? 'stop' : 'wait';
+  return satisfies(d, want(r));
+}).length / rows.length;
+
+console.log(`\n${'arm'.padEnd(26)} ${'accuracy'.padStart(9)} ${'median'.padStart(9)} ${'no answer'.padStart(10)}`);
+console.log('-'.repeat(58));
+console.log(`${`always "${commonest[0]}"`.padEnd(26)} ${pct(baseline).padStart(9)} ${'—'.padStart(9)} ${'—'.padStart(10)}`);
+console.log(`${`stillMs > ${STILL_MS_THRESHOLD}ms`.padEnd(26)} ${pct(stillRule).padStart(9)} ${'0ms'.padStart(9)} ${'—'.padStart(10)}`);
+for (const r of results) {
+  console.log(`${r.arm.padEnd(26)} ${pct(r.accuracy).padStart(9)} ${`${r.medianMs ?? '—'}ms`.padStart(9)} ${`${r.unanswered}`.padStart(10)}`);
+}
+
+if (withAbstain) {
+  console.log(`\n--- item 100: the fourth word ---`);
+  console.log('The question is not whether it abstains. It is whether it abstains on');
+  console.log('the ones it would have got WRONG, or at random. A judge that declines');
+  console.log('uniformly has added latency and a round trip and bought nothing.\n');
+  console.log(`${'arm'.padEnd(22)} ${'answered'.padStart(9)} ${'of those'.padStart(9)} ${'abstained'.padStart(10)} ${'escalations'.padStart(12)}`);
+  console.log('-'.repeat(66));
+  for (const r of results) {
+    const answered = r.judged.filter((j) => j.ruling);
+    const right = answered.filter((j) => satisfies(j.ruling.decision, want(j.r))).length;
+    const abstained = r.judged.filter((j) => j.abstained).length;
+    console.log(`${r.arm.padEnd(22)} ${`${answered.length}/${rows.length}`.padStart(9)} ${pct(right / (answered.length || 1)).padStart(9)} ${`${abstained}`.padStart(10)} ${pct(abstained / rows.length).padStart(12)}`);
+  }
+  console.log('\n`of those` is accuracy on the questions it chose to answer. If the fourth');
+  console.log('word is working, that number is higher than the three-word accuracy above');
+  console.log('by more than the abstention rate would give by chance.');
+}
+
+console.log('\nerrors, by arm:');
+for (const r of results) {
+  console.log(`  ${r.arm}`);
+  if (!r.errors.length) console.log('    none');
+  const seen = new Map();
+  for (const e of r.errors) seen.set(e, (seen.get(e) ?? 0) + 1);
+  for (const [e, n] of [...seen].sort((a, b) => b[1] - a[1])) console.log(`    ${String(n).padStart(3)}x ${e}`);
+}
+
+// --- the cascade: threshold -> local model -> Claude -----------------------
+//
+// The owner's proposal, and the numbers are the only way to say whether it
+// helps: answer with the free rule where it is confident, fall through to the
+// on-device model where it is not, and only then pay a round trip.
+//
+// **A cascade needs a tier that can decline, and neither tier has one.** The
+// threshold is a comparison — it always answers. The supervisor's vocabulary is
+// three words and none of them is "I don't know". So the interesting number is
+// not "does a cascade help" but "how much would an abstain token be worth", and
+// that is item 100. This measures it directly, by letting the rule abstain in a
+// band around its own threshold and handing those to the next tier.
+const armDecisions = new Map(results.map((r) => [r.arm, r.judged]));
+console.log('\n--- cascade: the rule answers, the model covers where it abstains ---');
+console.log(`${'abstain band'.padEnd(22)} ${'rule'.padStart(6)} ${'->model'.padStart(8)} ${'cascade'.padStart(9)} ${'model calls'.padStart(12)}`);
+for (const band of [0, 250, 500, 750, 1000, 1500]) {
+  const lo = STILL_MS_THRESHOLD - band;
+  const hi = STILL_MS_THRESHOLD + band;
+  for (const arm of arms) {
+    const judged = armDecisions.get(arm) ?? [];
+    let right = 0;
+    let escalated = 0;
+    for (const [i, r] of rows.entries()) {
+      const still = r.situation.stillMs ?? 0;
+      if (band > 0 && still >= lo && still <= hi) {
+        escalated += 1;
+        const ruling = judged[i]?.ruling;
+        if (ruling && satisfies(ruling.decision, want(r))) right += 1;
+        continue;
+      }
+      if (satisfies(still > STILL_MS_THRESHOLD ? 'stop' : 'wait', want(r))) right += 1;
+    }
+    console.log(`${`+/-${band}ms -> ${arm}`.padEnd(22)} ${pct(stillRule).padStart(6)} ${`${escalated}`.padStart(8)} ${pct(right / rows.length).padStart(9)} ${`${escalated}/${rows.length}`.padStart(12)}`);
+  }
+  if (band === 0) console.log('  (band 0 = no abstention, the rule alone — every row below adds a tier)');
+}
+
+console.log('\nThis scores DECISIONS on identical inputs. It cannot score outcomes —');
+console.log('whether acting on a ruling recovered the flow is a fact about the device at');
+console.log('that moment, and belongs to whichever arm was live. See docs/EXPERIMENTS.md.');
+
+supervisor.close();

package/scripts/score-rulings.mjs

@@ -42,7 +42,25 @@ const all = metrics.readSupervisions(dev.udid).filter((r) => CORRECT[r.expect]);
 // test was 16/16. Mixing a known-bad population into the denominator is the
 // same error as before wearing a different hat.
 const lastN = Number(arg('last', 0));
-const rows = lastN > 0 ? all.slice(-lastN) : all;
+// Which judge. Every arm of the capacity comparison writes to one log, so
+// scoring without this would average a 3B, an 8B and a 14B into a single
+// meaningless number — and it would look like a result.
+const arm = arg('arm', null);
+const armOf = (r) => r.supervisor ?? 'unrecorded';
+const scoped = arm ? all.filter((r) => armOf(r) === arm) : all;
+const rows = lastN > 0 ? scoped.slice(-lastN) : scoped;
+
+// A population spanning several arms is not one population. Say so, and say
+// what to pass, rather than printing an average of things that were never
+// comparable.
+const arms = [...new Set(rows.map(armOf))];
+if (arms.length > 1) {
+  console.log(`this log holds rulings from ${arms.length} supervisor arms:`);
+  for (const a of arms) console.log(`  ${String(rows.filter((r) => armOf(r) === a).length).padStart(4)}  ${a}`);
+  console.log('\nScore one at a time — `--arm=<name>` — because averaging them is not a result.');
+  process.exit(2);
+}
+if (rows.length) console.log(`arm: ${arms[0]}\n`);
 if (rows.length < 8) {
   console.error(`only ${rows.length} labelled ruling(s) — run scripts/collect-rulings.mjs first`);
   process.exit(2);