simframe 0.12.0 → 0.12.1

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "simframe",
-  "version": "0.12.0",
+  "version": "0.12.1",
   "mcpName": "io.github.lvlrSajjad/simframe",
   "description": "Always-warm iOS Simulator and Android emulator frames: agents read the screen in ~20ms instead of waiting on screenshots. MCP server + CLI.",
   "keywords": [

package/scripts/score-rulings.mjs

@@ -0,0 +1,107 @@
+#!/usr/bin/env node
+// Score a ruling population against the baselines that could embarrass it.
+//
+// Written **before** the population it first scored was finished, deliberately.
+// The previous attempt computed a `stillMs` threshold after seeing the answers,
+// on 14 samples of which 12 shared one label, and produced "100%" — a number
+// that was fitted, not measured. Fixing that afterwards is not possible: you
+// cannot un-see the data. So the split rule, the candidate thresholds and the
+// baselines are all fixed here in advance.
+//
+//   node scripts/score-rulings.mjs --device=<udid>
+//
+// What it will not do: pick the best threshold and report its score. It fits on
+// the first half and scores on the second, and prints both numbers so a gap
+// between them is visible as overfitting rather than hidden as success.
+import * as metrics from '../src/metrics.js';
+import { resolveDevice } from '../src/platform/index.js';
+
+const arg = (n, d) => {
+  const hit = process.argv.find((a) => a.startsWith(`--${n}=`));
+  return hit ? hit.slice(n.length + 3) : d;
+};
+
+/** The fixtures, and the word each one's situation actually calls for. */
+const CORRECT = {
+  'the list is still loading; its rows arrive shortly after launch': 'wait',
+  'the detail screen is still fetching; its text arrives shortly': 'wait',
+  'the list arrives in waves and this row is in the last one': 'wait',
+  'Review is blocked until Species is filled in, and it is empty': 'stop',
+  'the first submit always fails and the second works, so waiting cannot help': 'stop',
+};
+
+/** `retry` and `wait` differ only in how long they settle, so both satisfy a wait. */
+const satisfies = (decision, want) =>
+  (want === 'wait' ? decision === 'wait' || decision === 'retry' : decision === want);
+
+const dev = await resolveDevice(arg('device'));
+const all = metrics.readSupervisions(dev.udid).filter((r) => CORRECT[r.expect]);
+// `--last=N` scores one batch rather than the whole log. Needed the first time
+// this ran: the log still held rulings from a deliberately skewed population,
+// so the balance read 19/35 and the baseline 65% when the batch actually under
+// test was 16/16. Mixing a known-bad population into the denominator is the
+// same error as before wearing a different hat.
+const lastN = Number(arg('last', 0));
+const rows = lastN > 0 ? all.slice(-lastN) : all;
+if (rows.length < 8) {
+  console.error(`only ${rows.length} labelled ruling(s) — run scripts/collect-rulings.mjs first`);
+  process.exit(2);
+}
+
+const want = (r) => CORRECT[r.expect];
+const counts = rows.reduce((a, r) => ({ ...a, [want(r)]: (a[want(r)] ?? 0) + 1 }), {});
+const commonest = Object.entries(counts).sort((a, b) => b[1] - a[1])[0];
+const baseline = commonest[1] / rows.length;
+
+console.log(`${rows.length} labelled rulings on ${dev.name}`);
+console.log(`balance: ${JSON.stringify(counts)}`);
+console.log(`majority-class baseline: always "${commonest[0]}" scores ${(100 * baseline).toFixed(0)}%`);
+if (baseline > 0.65) {
+  console.log('\nSKEWED — any accuracy below is mostly a fact about the fixture set.');
+}
+
+const acc = (rs, predict) => rs.filter((r) => satisfies(predict(r), want(r))).length / (rs.length || 1);
+const pct = (x) => `${(100 * x).toFixed(0)}%`;
+
+console.log(`\n${'the model'.padEnd(30)} ${pct(acc(rows, (r) => r.decision))}`);
+console.log(`${`always "${commonest[0]}"`.padEnd(30)} ${pct(baseline)}`);
+
+// --- the stillMs rule, fitted on one half and scored on the other.
+//
+// Split by *recording order* rather than at random or by fixture, because the
+// alternative is choosing a split, and choosing is the thing that went wrong.
+const half = Math.floor(rows.length / 2);
+const fit = rows.slice(0, half);
+const held = rows.slice(half);
+const CANDIDATES = [1000, 1500, 2000, 2500, 3000, 4000, 5000, 6000, 7000];
+const rule = (t) => (r) => ((r.still_ms ?? 0) > t ? 'stop' : 'wait');
+
+let best = null;
+for (const t of CANDIDATES) {
+  const a = acc(fit, rule(t));
+  if (!best || a > best.a) best = { t, a };
+}
+console.log(`\nstillMs threshold, fitted on the first ${fit.length} and scored on the last ${held.length}:`);
+console.log(`  chosen on the fit half:  > ${best.t}ms  (${pct(best.a)} there)`);
+console.log(`  ${'on the held-out half:'.padEnd(24)} ${pct(acc(held, rule(best.t)))}`);
+console.log(`  ${'the model, same half:'.padEnd(24)} ${pct(acc(held, (r) => r.decision))}`);
+console.log('\nA rule that scores far better on the fit half than the held-out half was');
+console.log('fitted to noise. That gap is the number this script exists to print.');
+
+// --- the direction of the errors, which does not depend on the balance at all.
+const wrong = rows.filter((r) => !satisfies(r.decision, want(r)));
+const dirs = wrong.reduce((a, r) => {
+  const k = `said "${r.decision}" where "${want(r)}" was right`;
+  return { ...a, [k]: (a[k] ?? 0) + 1 };
+}, {});
+console.log(`\n${wrong.length} error(s) of ${rows.length}:`);
+for (const [k, n] of Object.entries(dirs).sort((a, b) => b[1] - a[1])) console.log(`  ${String(n).padStart(3)}x ${k}`);
+if (Object.keys(dirs).length === 1 && wrong.length > 2) {
+  console.log('\nAll errors in one direction. A one-directional bias is what an abstain');
+  console.log('token addresses (DEFERRED 100), whatever the headline accuracy says.');
+}
+
+const lat = rows.map((r) => r.latency_ms).filter(Number.isFinite).sort((a, b) => a - b);
+if (lat.length) console.log(`\nmedian judgement latency: ${lat[lat.length >> 1]}ms`);
+const timed = rows.filter((r) => Number.isFinite(r.edge_p95_ms)).length;
+console.log(`edges the graph had timed: ${timed}/${rows.length}`);

package/src/platform/ios.js

@@ -66,6 +66,19 @@ async function bootedDevices(opts) {
  */
 async function resolveDevice(query, opts) {
   const all = await listDevices(opts);
+  return pickDevice(query, all);
+}
+
+/**
+ * Which device a query means, given the whole list.
+ *
+ * Separated from the listing so the decision can be **tested** rather than
+ * reasoned about, because two peer rounds in a row reported being handed the
+ * wrong device and both were decided here. The same move as `decisionOf` in the
+ * supervisor: the one line where a wrong answer is expensive should be a
+ * function somebody can call with adversarial input.
+ */
+export function pickDevice(query, all) {
   const booted = all.filter((d) => d.state === 'Booted');
   if (!query) {
     if (booted.length === 0) throw new Error('no booted simulator (open Simulator.app or run `xcrun simctl boot <udid>`)');
@@ -98,8 +111,40 @@ async function resolveDevice(query, opts) {
   const q = query.toLowerCase();
   const pools = [booted, all];
   for (const pool of pools) {
-    const exact = pool.find((d) => d.udid.toLowerCase() === q || d.name.toLowerCase() === q);
-    if (exact) return exact;
+    // A UDID is unique, so an exact UDID match needs no further thought.
+    const byUdid = pool.find((d) => d.udid.toLowerCase() === q);
+    if (byUdid) return byUdid;
+    // A **name is not unique**, and this branch used to treat it as though it
+    // were: one `find` over both fields returned whichever device the list
+    // happened to put first, and short-circuited past the ambiguity guard
+    // below. Item 83 recorded that two booted devices on this machine are both
+    // called "iPhone 17 Pro" and answered it by warning in `sim_devices` and
+    // printing a UDID prefix in headers — leaving the resolver, which is where
+    // the choice is actually made, untouched.
+    //
+    // What that cost, reported from a three-hour session on a real app: a
+    // caller passed the shared name, read a screen that was "38ms old" and an
+    // hour wrong, concluded the app had signed itself out, and abandoned a
+    // verification run that was fine. The frame was fresh — it was the *other*
+    // device's, idling on a login screen. `refresh: true` returned the matching
+    // tree because it refreshed the same wrong device. Two independent-looking
+    // sources agreeing with each other and both wrong.
+    //
+    // So a name that names two devices is an ambiguity, exactly like a partial
+    // match that hits two, and it refuses for the same reason: the cost of
+    // guessing wrong is reading somebody else's screen and believing it.
+    const byName = pool.filter((d) => d.name.toLowerCase() === q);
+    if (byName.length === 1) return byName[0];
+    if (byName.length > 1) {
+      throw Object.assign(
+        new Error(
+          `"${query}" is the name of ${byName.length} devices: `
+          + `${byName.map((d) => d.udid).join(', ')} — a name cannot say which one you mean, `
+          + 'so pass the UDID',
+        ),
+        { ambiguous: true },
+      );
+    }
     const partial = pool.filter((d) => d.name.toLowerCase().includes(q));
     if (partial.length === 1) return partial[0];
     if (partial.length > 1) {