ruvnet-brain 4.0.1 → 4.0.2

package/scripts/rerank-cap-eval.mjs

@@ -0,0 +1,265 @@
+#!/usr/bin/env node
+// rerank-cap-eval.mjs — does bounding the cross-encoder pool change the ANSWERS?
+//
+// The cross-encoder is 84.7% of a query's wall and reads 605 (query, passage) pairs for an
+// all-repos question. Capping that pool is the only lever that moves the number. But a cap
+// re-orders results, and a naive one was MEASURED to lose the right answer — so no cap ships
+// without a before/after on a real question set.
+//
+// TWO PHASES, because the measurement is expensive and the policy search is not:
+//
+//   --collect   Runs the FROZEN held-out set (evals/held-out.json — the same corpus
+//               scripts/eval-brain.mjs gates on) UNCAPPED, recording every scored candidate:
+//               repo, path, lane, within-lane depth, pool position, and its cross-encoder score.
+//               One ~8-minute query per question. This is the whole cost of the experiment.
+//
+//   --report    Replays capRerankPool + selectResults — the SHIPPING functions, imported, not
+//               re-implemented — against those recorded scores, for every candidate budget. The
+//               cross-encoder is deterministic per (query, passage) pair and pairs are scored
+//               independently, so a replay of a subset is EXACT: it is the same arithmetic on the
+//               same numbers, not a model of it.
+//
+// Reported metrics are the ones a wrong answer would move, graded by ground truth (never a model
+// judge — an LLM panel once scored a zero-citation answer 98/100 on this repo):
+//   pairs      — cross-encoder pairs actually scored. The load-independent primary evidence.
+//   top1-same  — the winning document is byte-identical to the uncapped winner.
+//   kept       — the uncapped top-k documents still present in the capped top-k.
+//   routed     — top-1 lands in an expected repo (eval-brain's own metric, same Wilson bound).
+//   abstain    — adversarial questions still decline (top ce < 0).
+//   banner     — a winning gist chunk still carries its provenance banner.
+//
+//   node scripts/rerank-cap-eval.mjs --collect [--conc 3] [--only ho-01,ho-02]
+//   node scripts/rerank-cap-eval.mjs --report [--budgets 605,272,136,69]
+
+import fs from 'node:fs';
+import os from 'node:os';
+import path from 'node:path';
+import { fileURLToPath, pathToFileURL } from 'node:url';
+import { execFile } from 'node:child_process';
+import { gradeQuestion, aggregate } from './eval-brain.mjs';
+
+const ROOT = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '..');
+const KB = process.env.RUVNET_BRAIN_KB || path.join(os.homedir(), '.cache', 'ruvnet-brain', 'kb');
+// The code under test is THIS checkout's, run against whatever corpus KB points at.
+const CODE = path.join(ROOT, 'kb');
+const argv = process.argv.slice(2);
+const arg = (f, d) => { const i = argv.indexOf(f); return i >= 0 && argv[i + 1] ? argv[i + 1] : d; };
+const TRACES = arg('--traces', path.join(os.tmpdir(), 'ruvnet-brain-ce-cap-traces'));
+
+// The frozen 120, plus probes for the paths a cap could silently break. These are NOT scored
+// against expectRepo (they are not part of the frozen set and must never contaminate it) — they
+// are watched for one thing only: does the cap change the winner?
+const PROBES = [
+  { id: 'px-mcp-policy', query: 'which file enforces the MCP tool policy in ruvector', why: 'the case a naive cap was measured to lose (mcp-policy.js -> an ADR)' },
+  { id: 'px-adr-085', query: 'ADR-085', why: 'bare ADR number — per-repo collision disclosure reads the whole pool' },
+  { id: 'px-pkg-rvf', query: 'what is @ruvector/rvf', why: 'exact @scope/name — exercises the RESCUE lane the cap must never drop' },
+  { id: 'px-meetings', query: 'how many commits and contributors does the project have', why: 'answer is BM25-only in a transcript store; dense buries it past rank 40' },
+];
+
+// Probes first, then the frozen set DEALT ROUND-ROBIN ACROSS STRATA. The held-out file is grouped
+// by stratum, and at ~8 minutes a question a run can be interrupted — grouped order would make any
+// prefix a biased sample (all 'described', no 'adversarial'), which is the kind of partial evidence
+// that reads as a result and is not one. Round-robin makes every prefix stratified.
+function loadQuestions() {
+  const { questions } = JSON.parse(fs.readFileSync(path.join(ROOT, 'evals', 'held-out.json'), 'utf8'));
+  const byStratum = new Map();
+  for (const q of questions) (byStratum.get(q.stratum) ?? byStratum.set(q.stratum, []).get(q.stratum)).push(q);
+  const lanes = [...byStratum.values()];
+  const dealt = [];
+  for (let i = 0; dealt.length < questions.length; i++) for (const lane of lanes) if (lane[i]) dealt.push(lane[i]);
+  return [...PROBES.map((p) => ({ ...p, stratum: 'probe' })), ...dealt];
+}
+
+// ── collect ─────────────────────────────────────────────────────────────────────────────────────
+async function collect() {
+  const only = arg('--only', '') ? new Set(arg('--only', '').split(',')) : null;
+  const questions = loadQuestions().filter((q) => !only || only.has(q.id));
+  const CONC = Math.max(1, parseInt(arg('--conc', '3'), 10) || 3);
+  fs.mkdirSync(TRACES, { recursive: true });
+
+  const run = (q) => new Promise((resolve) => {
+    const trace = path.join(TRACES, `${q.id}.jsonl`);
+    // A partial trace from a killed run must never be read as a complete one: write to .part and
+    // rename only on a clean exit, so --report can never score a truncated pool as a real pool.
+    const part = `${trace}.part`;
+    try { fs.rmSync(part, { force: true }); } catch { /* fresh anyway */ }
+    const t0 = Date.now();
+    execFile('node', ['forge-ask-all.mjs', '--dir', KB, '--q', q.query, '--k', '3'], {
+      cwd: CODE,
+      timeout: 1800000,
+      maxBuffer: 256 * 1024 * 1024,
+      // --cap 0 (the default) is the uncapped baseline. A NON-zero --cap re-collects the same
+      // questions with the cap actually engaged, which is the only way to check the replay against
+      // reality: capping changes which pairs share a batch, and batch composition was MEASURED to
+      // move scores by up to 0.26 logits, so a replayed cap is an approximation of a real one.
+      // --cascade K does the same for ADR-058's two-stage cascade. Both are collected FOR REAL for
+      // exactly that reason: the cascade re-batches its survivors too, so its scores are its own.
+      env: {
+        ...process.env,
+        KB_CE_TRACE: part,
+        KB_CE_MAX_PAIRS: arg('--cap', '0'),
+        KB_CE_CASCADE_K: arg('--cascade', '0'),
+        KB_CE_CASCADE_TOKENS: arg('--tokens', '192'),
+      },
+    }, (err) => {
+      const ms = Date.now() - t0;
+      if (!err && fs.existsSync(part) && fs.statSync(part).size > 0) {
+        const rec = JSON.parse(fs.readFileSync(part, 'utf8').trim().split('\n')[0]);
+        fs.writeFileSync(trace, JSON.stringify({ id: q.id, stratum: q.stratum, expectRepo: q.expectRepo ?? null, ms, ...rec }) + '\n');
+        fs.rmSync(part, { force: true });
+        resolve({ id: q.id, ok: true, ms, pooled: rec.pooledAll });
+      } else {
+        try { fs.rmSync(part, { force: true }); } catch { /* nothing to clean */ }
+        resolve({ id: q.id, ok: false, ms, err: String(err?.message || 'no trace written').slice(0, 120) });
+      }
+    });
+  });
+
+  let cursor = 0, done = 0;
+  const t0 = Date.now();
+  const worker = async () => {
+    while (cursor < questions.length) {
+      const q = questions[cursor++];
+      if (fs.existsSync(path.join(TRACES, `${q.id}.jsonl`))) { done++; continue; } // resumable: an 8h run must survive a restart
+      const r = await run(q);
+      done++;
+      const la = os.loadavg()[0].toFixed(0);
+      process.stderr.write(`[cap-eval] ${done}/${questions.length} ${r.ok ? 'ok' : 'FAIL'} ${r.id} ${(r.ms / 1000).toFixed(0)}s pooled=${r.pooled ?? '-'} load=${la} elapsed=${((Date.now() - t0) / 60000).toFixed(0)}m${r.ok ? '' : ` :: ${r.err}`}\n`);
+    }
+  };
+  await Promise.all(Array.from({ length: Math.min(CONC, questions.length) }, worker));
+  console.error(`[cap-eval] traces in ${TRACES}`);
+}
+
+// ── report ──────────────────────────────────────────────────────────────────────────────────────
+async function report() {
+  const { capRerankPool, selectResults } = await import(pathToFileURL(path.join(CODE, 'forge-ask-all.mjs')).href);
+  if (!fs.existsSync(TRACES)) { console.error(`no traces at ${TRACES} — run --collect first`); process.exit(2); }
+  const files = fs.readdirSync(TRACES).filter((f) => f.endsWith('.jsonl'));
+  if (!files.length) { console.error(`no traces at ${TRACES} — run --collect first`); process.exit(2); }
+  const traces = files.map((f) => JSON.parse(fs.readFileSync(path.join(TRACES, f), 'utf8').trim().split('\n')[0]));
+
+  const budgets = (arg('--budgets', '') ? arg('--budgets', '').split(',').map(Number)
+    : [0, 408, 272, 204, 170, 136, 102, 69, 48, 24]).sort((a, b) => b - a);
+
+  // Rebuild each trace's pool in its ORIGINAL fan-out order — the cap's tie-break depends on it.
+  const pools = traces.map((t) => ({
+    ...t,
+    pool: [...t.cands].sort((a, b) => a.poolIdx - b.poolIdx)
+      .map((c) => ({ ...c, ceScore: c.ce, fullText: '', text: '', _lane: c.lane, _srcRank: c.rank, _poolIdx: c.poolIdx })),
+  }));
+
+  // The exact-package boost reads a candidate's BODY, which a trace deliberately does not carry
+  // (605 whole documents x 120 questions). It fires only on an `@scope/name` query, and it fires
+  // identically on any candidate that survives the cap, so it cannot flip a comparison between two
+  // survivors — but say so out loud rather than let a reader assume the replay is total.
+  const bodyDependent = pools.filter((p) => /@[a-z0-9][a-z0-9._-]*\/[a-z0-9._-]+/i.test(p.query)).map((p) => p.id);
+
+  const idOf = (r) => `${r.repo}/${r.path}`;
+
+  // CASCADE — the alternative policy, kept because it was measured and lost. Score every store's
+  // best passage first (~1 pair per store), then spend the rest of the budget only on the R stores
+  // whose best passage scored highest. It is the obvious "let the cross-encoder decide where to
+  // dig" design; the table below is the reason it is not what ships.
+  const cascadeKeep = (pool, R) => {
+    const best = new Map();
+    for (const c of pool) if (c._srcRank === 0 && (!best.has(c.repo) || c.ceScore > best.get(c.repo))) best.set(c.repo, c.ceScore);
+    const top = new Set([...best.entries()].sort((a, b) => b[1] - a[1]).slice(0, R).map(([r]) => r));
+    return pool.filter((c) => c._srcRank === 0 || c._lane === 'rescue' || top.has(c.repo));
+  };
+
+  const runKept = (p, kept) => {
+    const ranked = [...kept].sort((a, b) => (b.ceScore ?? -Infinity) - (a.ceScore ?? -Infinity));
+    const { results } = selectResults({ query: p.query, ranked, k: p.k });
+    return { pairs: kept.length, results };
+  };
+  const runPolicy = (p, limit) => runKept(p, capRerankPool(p.pool, { limit }).kept);
+
+  const base = new Map(pools.map((p) => [p.id, runPolicy(p, 0)]));
+
+  // ── ROOT CAUSE ────────────────────────────────────────────────────────────────────────────────
+  // How deep in its OWN store's vector ranking does the winning document sit? This one histogram
+  // decides whether any pre-score cap can be safe. If winners clustered at depth 0, a tiny budget
+  // would be free. They do not: the cross-encoder routinely promotes a passage its own store
+  // ranked 4th or 7th, which means depth carries little information about who wins, which means
+  // every pair a depth cut removes is a real chance of removing the answer.
+  const depthTop1 = {}, depthTopK = {};
+  for (const p of pools) {
+    const r = base.get(p.id).results;
+    if (!r.length) continue;
+    depthTop1[r[0]._srcRank] = (depthTop1[r[0]._srcRank] || 0) + 1;
+    for (const x of r) depthTopK[x._srcRank] = (depthTopK[x._srcRank] || 0) + 1;
+  }
+  const hist = (h) => Object.keys(h).map(Number).sort((a, b) => a - b).map((d) => `depth ${d}: ${h[d]}`).join(' | ');
+
+  console.log(`\n# cross-encoder pool cap — quality vs pair count`);
+  console.log(`corpus: ${pools.length} questions (${pools.filter((p) => p.stratum !== 'probe').length} frozen held-out + ${pools.filter((p) => p.stratum === 'probe').length} probes)`);
+  console.log(`uncapped pool: min ${Math.min(...pools.map((p) => p.pooledAll))}, median ${median(pools.map((p) => p.pooledAll))}, max ${Math.max(...pools.map((p) => p.pooledAll))} pairs`);
+  console.log(`body-dependent boost not replayed for: ${bodyDependent.length ? bodyDependent.join(', ') : '(none — no @scope/name query in the set)'}`);
+  console.log(`\n## where the answer actually lives, in its own store's vector ranking`);
+  console.log(`  winning document : ${hist(depthTop1)}`);
+  console.log(`  every top-k hit  : ${hist(depthTopK)}`);
+  console.log(`  (a budget of B pairs across S stores reaches depth B/S. Winners spread across depth`);
+  console.log(`   means a depth cut drops answers roughly in proportion to what it saves.)\n`);
+  console.log('| policy | pairs (median) | top1-same | top-k kept | routed k/n (Wilson lo) | abstain | banner |');
+  console.log('|---|---|---|---|---|---|---|');
+
+  // Every policy the run compares: the shipping depth cap at a range of budgets, plus cascade.
+  const policies = [
+    ...budgets.map((limit) => ({ label: limit === 0 ? 'uncapped' : `depth B=${limit}`, key: `depth-${limit}`, keep: (p) => capRerankPool(p.pool, { limit }).kept })),
+    ...[4, 8, 12, 16, 24, 32].map((R) => ({ label: `cascade R=${R}`, key: `cascade-${R}`, keep: (p) => cascadeKeep(p.pool, R) })),
+  ];
+
+  const detail = {};
+  for (const policy of policies) {
+    const rows = [];
+    let top1Same = 0, keptNum = 0, keptDen = 0;
+    const changed = [];
+    for (const p of pools) {
+      const cur = runKept(p, policy.keep(p));
+      const b = base.get(p.id);
+      const same = idOf2(cur.results[0]) === idOf2(b.results[0]);
+      if (same) top1Same++; else changed.push(`${p.id}: ${idOf2(b.results[0])} -> ${idOf2(cur.results[0])}`);
+      const curSet = new Set(cur.results.map(idOf));
+      for (const r of b.results) { keptDen++; if (curSet.has(idOf(r))) keptNum++; }
+      if (p.stratum !== 'probe') {
+        const top = cur.results[0] ?? null;
+        // Same grading rule as scripts/eval-brain.mjs, fed from the replayed result set. `grounded`
+        // is structurally true here: every candidate is a passage the retriever actually returned.
+        rows.push({
+          id: p.id, stratum: p.stratum,
+          ...gradeQuestion({ stratum: p.stratum, expectRepo: p.expectRepo },
+            { grounded: cur.results.length > 0,
+              citations: top ? [{ repo: top.repo, fullPath: idOf(top), ce: top.ceScore }] : [],
+              bannerPresent: cur.results.some((r) => r.gist) }),
+        });
+      }
+      detail[policy.key] ??= {};
+      detail[policy.key][p.id] = { pairs: cur.pairs, top1: idOf2(cur.results[0]) };
+    }
+    const agg = aggregate(rows);
+    const pairs = median(pools.map((p) => policy.keep(p).length));
+    console.log(`| ${policy.label} | ${pairs} | ${top1Same}/${pools.length} (${pct(top1Same / pools.length)}) | ${keptNum}/${keptDen} (${pct(keptNum / keptDen)}) | ${agg.routed.k}/${agg.routed.n} (${pct(agg.routed.lo)}) | ${agg.abstain.k}/${agg.abstain.n} | ${agg.banner.k}/${agg.banner.n} |`);
+    if (changed.length && changed.length <= 12) detail[`changed@${policy.key}`] = changed;
+  }
+
+  console.log('\n## winners that changed, per policy (empty = the policy changed no answer)');
+  for (const policy of policies) {
+    if (policy.key === 'depth-0') continue;
+    const c = detail[`changed@${policy.key}`];
+    console.log(`\n### ${policy.label}`);
+    if (!c) console.log('  (too many to list — see the top1-same column)');
+    else if (!c.length) console.log('  none');
+    else for (const line of c) console.log(`  ${line}`);
+  }
+
+  if (argv.includes('--json')) fs.writeFileSync(path.join(TRACES, 'report.json'), JSON.stringify(detail, null, 2));
+}
+
+const idOf2 = (r) => (r ? `${r.repo}/${r.path}` : '(no result)');
+const pct = (x) => `${(x * 100).toFixed(1)}%`;
+const median = (a) => { const s = [...a].sort((x, y) => x - y); return s.length % 2 ? s[(s.length - 1) / 2] : Math.round((s[s.length / 2 - 1] + s[s.length / 2]) / 2); };
+
+if (argv.includes('--collect')) await collect();
+else if (argv.includes('--report')) await report();
+else { console.error('usage: rerank-cap-eval.mjs --collect | --report'); process.exit(2); }

package/scripts/rerank-cap-warm-ab.mjs

@@ -0,0 +1,129 @@
+#!/usr/bin/env node
+// rerank-cap-warm-ab.mjs — the paired, WARM before/after for the cross-encoder pool cap.
+//
+// Why this exists rather than timing the CLI: a cold `forge-ask-all.mjs` spends ~53s loading two
+// ONNX models before it scores anything, and the cap cannot touch that. Timing cold runs would
+// dilute the effect being measured by roughly 3x and would also compare runs taken hours apart on a
+// machine whose load moved underneath them. This harness loads the models ONCE and then runs each
+// question twice in the same process — uncapped and capped — so the only difference between the two
+// numbers is the thing under test.
+//
+// PAIRED AND ORDER-ALTERNATED: question i runs uncapped-then-capped on even i and capped-then-
+// uncapped on odd i, so any residual warm-up or thermal drift cannot systematically favour one arm.
+// Both arms' ANSWERS are recorded, not just their times: a cap that is fast and wrong is a failure,
+// and this is the file that would catch it.
+//
+// TWO POLICIES SHARE THIS HARNESS, because a number is only comparable to another number taken
+// the same way. --cap is ADR-057's flat pool cap (select by vector distance, one full read each).
+// --cascade is ADR-058's two-stage cascade (read every pooled pair at a truncated length, then
+// re-read the top K in full). Same questions, same pairing, same warm process, same table — so
+// "-30.4%" and whatever the cascade measures can be put side by side honestly.
+//
+//   node scripts/rerank-cap-warm-ab.mjs --cap 408      [--n 24] [--out result.json]
+//   node scripts/rerank-cap-warm-ab.mjs --cascade 64   [--n 24] [--tokens 192]
+
+import fs from 'node:fs';
+import os from 'node:os';
+import path from 'node:path';
+import { fileURLToPath, pathToFileURL } from 'node:url';
+
+const ROOT = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '..');
+const KB = process.env.RUVNET_BRAIN_KB || path.join(os.homedir(), '.cache', 'ruvnet-brain', 'kb');
+const argv = process.argv.slice(2);
+const arg = (f, d) => { const i = argv.indexOf(f); return i >= 0 && argv[i + 1] ? argv[i + 1] : d; };
+// Exactly one arm is under test. --cascade wins if both are given, and says so rather than
+// silently measuring a policy the operator did not ask for.
+const CASCADE = arg('--cascade', '');
+const CAP = arg('--cap', CASCADE ? '0' : '408');
+const TOKENS = arg('--tokens', '192');
+const MODE = CASCADE ? 'cascade' : 'cap';
+const LABEL = CASCADE ? `cascade K=${CASCADE} @${TOKENS}tok` : `capped B=${CAP}`;
+const N = parseInt(arg('--n', '24'), 10);
+const OUT = arg('--out', path.join(os.tmpdir(), `ce-${MODE}-warm-ab-${CASCADE || CAP}.json`));
+
+const { searchAll } = await import(pathToFileURL(path.join(ROOT, 'kb', 'forge-ask-all.mjs')).href);
+const { gradeQuestion, aggregate } = await import(pathToFileURL(path.join(ROOT, 'scripts', 'eval-brain.mjs')).href);
+
+// Stratified subset of the frozen held-out set, dealt round-robin so every stratum is represented
+// even at small n — a prefix of a grouped file would be all 'described' and no 'adversarial'.
+const { questions } = JSON.parse(fs.readFileSync(path.join(ROOT, 'evals', 'held-out.json'), 'utf8'));
+const byStratum = new Map();
+for (const q of questions) (byStratum.get(q.stratum) ?? byStratum.set(q.stratum, []).get(q.stratum)).push(q);
+const lanes = [...byStratum.values()];
+const dealt = [];
+for (let i = 0; dealt.length < questions.length; i++) for (const lane of lanes) if (lane[i]) dealt.push(lane[i]);
+const set = dealt.slice(0, N);
+
+const idOf = (r) => (r ? `${r.repo}/${r.path}` : '(none)');
+// `on` selects the arm under test; `off` is always the untouched uncapped, uncascaded path. Both
+// env knobs are set on EVERY call rather than only when engaged — a leftover value from the
+// previous call is exactly how an A/B measures the same arm twice and reports a 0% delta.
+async function once(query, on) {
+  process.env.KB_CE_MAX_PAIRS = String(on && MODE === 'cap' ? CAP : 0);
+  process.env.KB_CE_CASCADE_K = String(on && MODE === 'cascade' ? CASCADE : 0);
+  process.env.KB_CE_CASCADE_TOKENS = String(TOKENS);
+  const t0 = Date.now();
+  const out = await searchAll({ dir: KB, query, k: 3 });
+  return {
+    ms: Date.now() - t0, pairs: out.pooled, pooledAll: out.pooledAll,
+    prefiltered: out.prefiltered ?? 0, prefilterMs: out.prefilterMs ?? 0,
+    results: out.results.map((r) => ({ id: idOf(r), repo: r.repo, ce: r.ceScore, gist: !!r.gist })),
+  };
+}
+
+// Warm-up: the first query of a process pays both model loads. It is thrown away deliberately —
+// including it would credit the cap with a saving it did not produce.
+process.stderr.write(`[warm-ab] mode=${MODE} (${LABEL}) — loading models (first query is discarded)...\n`);
+const w0 = Date.now();
+await once('what is ruvector', false);
+process.stderr.write(`[warm-ab] warm after ${((Date.now() - w0) / 1000).toFixed(1)}s\n`);
+
+const rows = [];
+for (let i = 0; i < set.length; i++) {
+  const q = set[i];
+  const onFirst = i % 2 === 1;
+  const a = await once(q.query, onFirst);
+  const b = await once(q.query, !onFirst);
+  const [off, on] = onFirst ? [b, a] : [a, b];
+  rows.push({ id: q.id, stratum: q.stratum, expectRepo: q.expectRepo ?? null, onFirst, off, on });
+  process.stderr.write(`[warm-ab] ${i + 1}/${set.length} ${q.id} off=${(off.ms / 1000).toFixed(1)}s/${off.pairs}p on=${(on.ms / 1000).toFixed(1)}s/${on.pairs}p top1${idOf2(off) === idOf2(on) ? '=same' : ' CHANGED'}\n`);
+}
+function idOf2(x) { return x.results[0]?.id ?? '(none)'; }
+
+fs.writeFileSync(OUT, JSON.stringify({ mode: MODE, cap: CAP, cascade: CASCADE, tokens: TOKENS, kb: KB, n: set.length, load: os.loadavg(), rows }, null, 2));
+
+// ── the table ───────────────────────────────────────────────────────────────────────────────────
+const med = (a) => { const s = [...a].sort((x, y) => x - y); return s.length % 2 ? s[(s.length - 1) / 2] : Math.round((s[s.length / 2 - 1] + s[s.length / 2]) / 2); };
+const pct = (x) => `${(x * 100).toFixed(1)}%`;
+const grade = (arm) => aggregate(rows.map((r) => {
+  const top = r[arm].results[0] ?? null;
+  return { stratum: r.stratum, ...gradeQuestion({ stratum: r.stratum, expectRepo: r.expectRepo },
+    { grounded: r[arm].results.length > 0,
+      citations: top ? [{ repo: top.repo, fullPath: top.id, ce: top.ce }] : [],
+      bannerPresent: r[arm].results.some((x) => x.gist) }) };
+}));
+const top1Same = rows.filter((r) => idOf2(r.off) === idOf2(r.on)).length;
+let kn = 0, kd = 0;
+for (const r of rows) { const s = new Set(r.on.results.map((x) => x.id)); for (const x of r.off.results) { kd++; if (s.has(x.id)) kn++; } }
+const gOff = grade('off'), gOn = grade('on');
+
+console.log(`\n# warm A/B — ${MODE === 'cascade' ? `cross-encoder CASCADE KB_CE_CASCADE_K=${CASCADE} KB_CE_CASCADE_TOKENS=${TOKENS}` : `cross-encoder pool cap KB_CE_MAX_PAIRS=${CAP}`}`);
+console.log(`${rows.length} questions from the frozen held-out set, paired, order-alternated, one warm process. load1=${os.loadavg()[0].toFixed(1)} on ${os.cpus().length} cores.\n`);
+console.log('| | full reads (median) | warm wall median | warm wall mean | routed | abstain | banner |');
+console.log('|---|---|---|---|---|---|---|');
+const fmt = (arm, g) => `| ${med(rows.map((r) => r[arm].pairs))} | ${(med(rows.map((r) => r[arm].ms)) / 1000).toFixed(2)}s | ${(rows.reduce((a, r) => a + r[arm].ms, 0) / rows.length / 1000).toFixed(2)}s | ${g.routed.k}/${g.routed.n} | ${g.abstain.k}/${g.abstain.n} | ${g.banner.k}/${g.banner.n} |`;
+console.log(`| baseline, no policy (before) ${fmt('off', gOff)}`);
+console.log(`| ${LABEL} (after) ${fmt('on', gOn)}`);
+if (MODE === 'cascade') {
+  // Stage 1 is real work and must be visible, or the table reads as if 64 pairs were the whole cost.
+  console.log(`\nstage-1 prefilter: ${med(rows.map((r) => r.on.prefiltered))} pairs read at ${TOKENS} tokens, median ${(med(rows.map((r) => r.on.prefilterMs)) / 1000).toFixed(2)}s of the after-time above`);
+}
+const dMed = 1 - med(rows.map((r) => r.on.ms)) / med(rows.map((r) => r.off.ms));
+console.log(`\nwall-time change (median, paired): ${dMed >= 0 ? '-' : '+'}${pct(Math.abs(dMed))}`);
+console.log(`top-1 cited path identical : ${top1Same}/${rows.length} (${pct(top1Same / rows.length)})`);
+console.log(`top-3 cited paths retained : ${kn}/${kd} (${pct(kn / kd)})`);
+console.log('\n## every question whose top-1 changed');
+const changed = rows.filter((r) => idOf2(r.off) !== idOf2(r.on));
+if (!changed.length) console.log('  none');
+for (const r of changed) console.log(`  ${r.id} [${r.stratum}] expect=${(r.expectRepo || ['-']).join('|')}\n     before: ${idOf2(r.off)}  (ce ${r.off.results[0]?.ce?.toFixed(3)})\n     after : ${idOf2(r.on)}  (ce ${r.on.results[0]?.ce?.toFixed(3)})`);
+console.log(`\nraw: ${OUT}`);

package/scripts/route-cheap.mjs

@@ -27,6 +27,7 @@ import path from 'node:path';
 import os from 'node:os';
 import { spawnSync } from 'node:child_process';
 import { fileURLToPath, pathToFileURL } from 'node:url';
+import { loadRuntimePreferences, runtimeChildEnv } from '../plugin/scripts/runtime-preferences.mjs';
 
 const __dirname = path.dirname(fileURLToPath(import.meta.url));
 
@@ -90,20 +91,15 @@ export function receiptLine(model, costs) {
   return `\x1b[2m⚡ MetaHarness: routed to ${model} — ~${pct}% cheaper (est. ${fmt$(costs.cost)} vs ${fmt$(costs.frontier)} ${ref}, saved ~${fmt$(costs.saved)})\x1b[0m`;
 }
 
-// Load OPENROUTER_API_KEY from ruvnet-brain/.env if not already in env. Value never printed/logged.
-function ensureOpenRouterKey() {
-  if (process.env.OPENROUTER_API_KEY) return true;
+function agenticFlowExecutable(env) {
+  const home = env.HOME || os.homedir();
+  const global = path.join(home, '.npm-global', 'bin', process.platform === 'win32' ? 'agentic-flow.cmd' : 'agentic-flow');
   try {
-    const envFile = fs.readFileSync(path.join(__dirname, '..', '.env'), 'utf8');
-    const m = envFile.match(/^OPENROUTER_API_KEY=(.+)$/m);
-    if (m && m[1].trim()) {
-      process.env.OPENROUTER_API_KEY = m[1].trim().replace(/^["']|["']$/g, '');
-      return true;
-    }
+    fs.accessSync(global, fs.constants.X_OK);
+    return global;
   } catch {
-    /* no .env — fall through */
+    return 'agentic-flow';
   }
-  return false;
 }
 
 function parseArgs(argv) {
@@ -125,16 +121,25 @@ function main() {
     console.error(`Unknown model "${args.model}" — no verified pricing, refusing to invent savings. Known: ${Object.keys(PRICING).join(', ')}`);
     process.exit(2);
   }
-  if (!ensureOpenRouterKey()) {
-    console.error('OPENROUTER_API_KEY not found in env or ruvnet-brain/.env — cannot route. (Key value is never printed.)');
+  const policy = loadRuntimePreferences();
+  if (policy.values.routing !== 'auto') {
+    console.error(policy.values.routing === 'off'
+      ? 'Token-smart routing is off in RuvNet Brain Console — nothing was dispatched.'
+      : 'Token-smart routing has not been enabled in RuvNet Brain Console — nothing was dispatched.');
+    process.exit(1);
+  }
+  const childEnv = runtimeChildEnv();
+  if (!childEnv.OPENROUTER_API_KEY) {
+    console.error('OPENROUTER_API_KEY is not configured in the environment or encrypted Brain credential store — cannot route. (Key value is never printed.)');
     process.exit(1);
   }
 
   const started = Date.now();
-  const run = spawnSync('npx', ['agentic-flow@latest', '--agent', args.agent, '--model', args.model, '--task', args.task], {
+  const run = spawnSync(agenticFlowExecutable(childEnv), ['--agent', args.agent, '--model', args.model, '--task', args.task], {
     encoding: 'utf8',
     timeout: 180_000,
-    env: process.env,
+    env: childEnv,
+    shell: false,
   });
 
   if (run.error || run.status !== 0) {

package/scripts/router-utilization.mjs

@@ -0,0 +1,182 @@
+#!/usr/bin/env node
+// scripts/router-utilization.mjs — the ONGOING view. router-optimizer.mjs answers "what SHOULD each
+// bucket route to"; this answers "once you've actually been using it, how many tasks landed in each
+// bucket, and what did you save versus sending every one to the frontier model?"
+//
+// It implements the shape rUv specifies in ruflo ADR-149 §6 (Observability, Proposed):
+//   • modelDistribution   — per-model/per-band task counts ("how many are going to each bucket")
+//   • costOptimalitySaved — USD vs always paying the frontier ("what the savings are off the frontier")
+//
+// Pure/deterministic: it reads the real receipts ledger (route-cheap.mjs + subagent dispatch write it)
+// and makes NO model or network call. Every number is grounded:
+//   • realized cost      = the receipt's own recorded est_cost (what the routed model actually cost).
+//   • frontier baseline  = RECOMPUTED for every receipt against the CURRENT frontier model
+//     (route-cheap FRONTIER — now Fable 5) from that receipt's own est token counts. Recomputing (rather
+//     than trusting each receipt's stored est_frontier_cost) means switching the frontier re-prices the
+//     whole history consistently, so the "vs Fable 5" figure is never a stale Opus-era number.
+//   • a receipt whose model has no verified price is counted under `unpriced` and left OUT of the $ math
+//     — never assigned an invented cost.
+//
+// Usage: node scripts/router-utilization.mjs [--json]
+
+import fs from 'node:fs';
+import { FRONTIER, priceOf, receiptsPath } from './route-cheap.mjs';
+
+// Band = a human-legible grouping over the continuous complexity/cost axis (ADR-149: "tier_label is
+// metadata, not control flow"). The four bands mirror router-optimizer.mjs. Known models are mapped
+// explicitly; anything unknown is bucketed by its blended $/Mtok via bandOf()'s fallback.
+const BAND_BY_MODEL = {
+  'agent-booster': 'mechanical',
+  'inclusionai/ling-2.6-flash': 'cheap',
+  'claude-haiku-4.5': 'cheap',
+  'deepseek/deepseek-chat': 'cheap',
+  'deepseek/deepseek-v4-flash': 'cheap',
+  'meta-llama/llama-3.3-70b-instruct': 'mid',
+  'openai/gpt-4.1': 'mid',
+  'x-ai/grok-4.5': 'mid',
+  'claude-sonnet-5': 'mid',
+  'claude-opus-4.8': 'frontier',
+  'claude-fable-5': 'frontier',
+};
+export const BAND_ORDER = ['mechanical', 'cheap', 'mid', 'frontier'];
+const BAND_LABEL = { mechanical: 'Mechanical', cheap: 'Cheap', mid: 'Mid', frontier: 'Frontier' };
+
+/** Which band did this model's task land in? Known → explicit map; unknown → blended-price fallback. */
+export function bandOf(model) {
+  if (BAND_BY_MODEL[model]) return BAND_BY_MODEL[model];
+  const p = priceOf(model);
+  if (!p) return null; // unpriced → caller records it under `unpriced`, never invents a cost
+  const blended = (p.in + p.out) / 2;
+  if (blended === 0) return 'mechanical';
+  if (blended < 2) return 'cheap';
+  if (blended < 15) return 'mid';
+  return 'frontier';
+}
+
+function tokensOf(r) {
+  const i = Number(r.est_in_tokens);
+  const o = Number(r.est_out_tokens);
+  if (Number.isFinite(i) && Number.isFinite(o)) return { in: i, out: o };
+  return null;
+}
+
+/**
+ * Summarize the receipts ledger into a per-band distribution + realized-vs-frontier savings.
+ * @param {{ receiptsFile?: string }} [opts]
+ */
+export function utilization({ receiptsFile, frontier } = {}) {
+  const file = receiptsFile || receiptsPath();
+  // Frontier = the user's OWN house flagship (passed in by the console from the live-verified catalog);
+  // falls back to route-cheap's default. The counterfactual is recomputed vs THIS, so an OpenAI shop sees
+  // "vs GPT-5.6 Sol" and a Claude shop sees "vs Fable 5" — personalized, never one house for everyone.
+  const frModel = frontier?.model || FRONTIER.name;
+  const fr = (frontier && Number.isFinite(frontier.in) && Number.isFinite(frontier.out))
+    ? { in: frontier.in, out: frontier.out }
+    : priceOf(FRONTIER.name);
+
+  const bands = Object.fromEntries(
+    BAND_ORDER.map((b) => [b, { band: b, label: BAND_LABEL[b], tasks: 0, realizedUsd: 0, frontierUsd: 0, models: {} }])
+  );
+  let tasks = 0;
+  let unpriced = 0;
+  let realizedUsd = 0;
+  let frontierUsd = 0;
+  let since = null;
+  let until = null;
+
+  let lines = [];
+  try {
+    lines = fs.readFileSync(file, 'utf8').split('\n');
+  } catch {
+    /* no receipts yet — return the empty shape below */
+  }
+
+  for (const line of lines) {
+    if (!line.trim()) continue;
+    let r;
+    try { r = JSON.parse(line); } catch { continue; }
+    if (!r.model) continue;
+    tasks++;
+    if (r.ts) {
+      since = since && since < r.ts ? since : r.ts;
+      until = until && until > r.ts ? until : r.ts;
+    }
+    const band = bandOf(r.model);
+    const p = priceOf(r.model);
+    const isMech = band === 'mechanical';
+    // Mechanical ($0, no LLM — e.g. Agent Booster) has no per-token price but a real realized cost of $0.
+    // Everything else needs a verified price AND a priced frontier, or it's excluded (never invented).
+    if (!band || (!isMech && (!p || !fr))) { unpriced++; continue; }
+
+    const tok = tokensOf(r);
+    const realized = isMech ? 0
+      : Number.isFinite(Number(r.est_cost)) ? Number(r.est_cost)
+      : tok ? (tok.in * p.in + tok.out * p.out) / 1e6 : 0;
+    // Frontier counterfactual — ALWAYS recomputed vs the current frontier from the receipt's tokens.
+    const front = (tok && fr) ? (tok.in * fr.in + tok.out * fr.out) / 1e6 : realized;
+
+    const b = bands[band];
+    b.tasks++;
+    b.realizedUsd += realized;
+    b.frontierUsd += front;
+    b.models[r.model] = (b.models[r.model] || 0) + 1;
+    realizedUsd += realized;
+    frontierUsd += front;
+  }
+
+  const round = (n) => +Number(n).toFixed(4);
+  const distribution = BAND_ORDER.map((b) => {
+    const x = bands[b];
+    return {
+      band: b,
+      label: x.label,
+      tasks: x.tasks,
+      pctOfTasks: tasks ? Math.round((x.tasks / tasks) * 100) : 0,
+      realizedUsd: round(x.realizedUsd),
+      frontierUsd: round(x.frontierUsd),
+      savedUsd: round(x.frontierUsd - x.realizedUsd),
+      models: Object.entries(x.models)
+        .sort((a, c) => c[1] - a[1])
+        .map(([model, n]) => ({ model, tasks: n })),
+    };
+  });
+
+  const savedUsd = round(frontierUsd - realizedUsd);
+  return {
+    generatedAt: new Date().toISOString(),
+    frontierModel: frModel,
+    tasks,
+    unpriced,
+    since,
+    until,
+    realizedUsd: round(realizedUsd),
+    frontierUsd: round(frontierUsd),
+    costOptimalitySaved: savedUsd, // ADR-149 name
+    pctSaved: frontierUsd > 0 ? Math.round((savedUsd / frontierUsd) * 100) : null,
+    distribution, // ADR-149 modelDistribution, grouped into the four legible bands
+    note: `Recomputed live from ${tasks} receipt(s) against your frontier (${frModel}); token counts are the receipts’ own est. values, never projected.`,
+  };
+}
+
+export function printUtil(u) {
+  const money = (v) => (v == null ? '—' : v === 0 ? '$0' : '$' + v);
+  console.log(`\nRouting utilization — ${u.tasks} task(s), frontier = ${u.frontierModel}`);
+  if (u.since) console.log(`  window: ${u.since} → ${u.until}`);
+  console.log(`  ${'band'.padEnd(11)} ${'tasks'.padStart(6)} ${'%'.padStart(4)} ${'realized'.padStart(11)} ${'vs frontier'.padStart(12)} ${'saved'.padStart(11)}`);
+  for (const d of u.distribution) {
+    console.log(`  ${d.label.padEnd(11)} ${String(d.tasks).padStart(6)} ${String(d.pctOfTasks).padStart(3)}% ${money(d.realizedUsd).padStart(11)} ${money(d.frontierUsd).padStart(12)} ${money(d.savedUsd).padStart(11)}`);
+  }
+  console.log(`  ${'—'.repeat(11)}`);
+  console.log(`  TOTAL saved vs all-frontier: ${money(u.costOptimalitySaved)} (${u.pctSaved == null ? '—' : u.pctSaved + '%'}); realized ${money(u.realizedUsd)} vs ${money(u.frontierUsd)} on ${u.frontierModel}.`);
+  if (u.unpriced) console.log(`  (${u.unpriced} receipt(s) had an unpriced model and were excluded from the $ math.)`);
+}
+
+export function mainUtil() {
+  const u = utilization();
+  if (process.argv.includes('--json')) console.log(JSON.stringify(u, null, 2));
+  else printUtil(u);
+}
+
+import { fileURLToPath } from 'node:url';
+import path from 'node:path';
+if (process.argv[1] && path.resolve(process.argv[1]) === fileURLToPath(import.meta.url)) mainUtil();