ruvnet-brain 4.0.1 → 4.0.4

package/scripts/remedy-registry.mjs

@@ -0,0 +1,247 @@
+// remedy-registry.mjs — ONE object per recommendation id, owning detector ⇄ executor ⇄ inverse.
+//
+// WHY THIS EXISTS. Before this file, a recommendation's id, the code that ran it, and the code that
+// reversed it lived in three different places that nothing forced to agree. All three drifted, and
+// every drift was invisible until someone clicked the button:
+//
+//   1. `learning:enable-fleet` was constructed, validated, and offered — with NO executor at all.
+//      It fell through apply()'s if/else to `Unknown recommendation id`. The single most important
+//      recommendation in the product (ADR-027's North Star case) was a dead button.
+//   2. `repair:memory-index` journalled `kind:'restore-memory-backup'`, and undo() had no branch for
+//      it. It hit the default arm and reported "nothing to undo (the change reverses itself
+//      automatically)" — while the recommendation had promised "restore the backup taken immediately
+//      before the repair." The undo did not exist. The promise was a lie.
+//   3. `repair:memory-index` also satisfies `startsWith('repair:')`, so ONE reordering of an if/else
+//      chain silently routed a database repair into a global npm sync. That was caught by review,
+//      but only by review — nothing structural prevented it.
+//
+// The common shape: a chain of `if (id.startsWith(...))` cannot be audited, because the set of ids
+// it handles is not a value anything can inspect. So it becomes a value here. Each Remedy owns its
+// id, the executor as DATA (not a spawn), and a DECLARED inverse. `assertRegistryClosure()` then
+// proves, in a test, that every id the builders can construct resolves to exactly one remedy with a
+// real undo handler behind it — so a dead button fails CI instead of failing a user.
+//
+// PURITY: no I/O, no spawn, no fs — same discipline as console-engine.mjs (DDD context 4). A remedy
+// RETURNS a description of what to run; onboarding-console.mjs is the only thing that runs it. That
+// is what lets the closure test check every path without touching the machine.
+
+// ── Undo kinds ───────────────────────────────────────────────────────────────────────────────────
+// The set of inverses the console can actually perform. A remedy may not name a kind outside this
+// set, and every kind here MUST have a live branch in onboarding-console.undo(). Both directions are
+// enforced by test, because a missing branch does not throw — it silently returns "nothing to undo",
+// which is the most dangerous possible answer: it reads like success.
+//
+// NONE is a real, declared value, not an absence. "This genuinely has no inverse" and "nobody wrote
+// one" must never look the same, which is exactly the bug that made #2 above invisible.
+export const UNDO_KINDS = Object.freeze({
+  NONE: 'none',
+  REINSTALL_VERSION: 'reinstall-version',
+  RESTORE_BACKUP: 'restore-backup',
+  RESTORE_MEMORY_BACKUP: 'restore-memory-backup',
+  RESTORE_STORE_BACKUPS: 'restore-store-backups',
+  AUTO_REBUILD: 'auto-rebuild',
+  // distill-project.mjs's OWN `--restore` (see its header: a tested inverse, not a re-implementation
+  // of one). Distinct from RESTORE_MEMORY_BACKUP/RESTORE_STORE_BACKUPS because those restore backups
+  // *this server* located and named; this one hands the restore entirely to the same script that took
+  // the snapshot, which already knows where its own backups live.
+  RESTORE_PROJECT_DISTILL: 'restore-project-distill',
+});
+const K = UNDO_KINDS;
+
+// Ids that are exact, reserved words. A parameterized matcher (`repair:<pkg>`) must never capture
+// one of these — see the ambiguity throw in resolveRemedy().
+const RESERVED = new Set(['repair:memory-index', 'purge:shadows']);
+
+// ── The registry ─────────────────────────────────────────────────────────────────────────────────
+// match(id)   → params object if this remedy owns the id, else null.
+// plan(p)     → { script, args } — the executor, as data.
+// inverse(p)  → { kind, ...params } — journalled BEFORE the change is made.
+export const REMEDIES = [
+  {
+    key: 'memory-index',
+    autoEligible: true,
+    summary: 'REINDEX a corrupt AgentDB store',
+    match: (id) => (id === 'repair:memory-index' ? {} : null),
+    plan: () => ({ script: 'scripts/health-repair.mjs', args: ['--repair-memory'] }),
+    // health-repair.mjs takes an sqlite `.backup` of the store immediately before REINDEX (never a
+    // cp — that silently truncates a live WAL database, a standing lesson proven by experiment).
+    // The inverse is restoring it. This is the branch whose absence made the promise a lie.
+    inverse: () => ({ kind: K.RESTORE_MEMORY_BACKUP }),
+  },
+  {
+    key: 'learning-flush',
+    summary: 'drain the capture queue into the learner',
+    match: (id) => (id === 'learning:flush' ? {} : null),
+    plan: () => ({ script: 'scripts/health-repair.mjs', args: ['--flush-learning'] }),
+    // Genuinely additive: it moves already-captured local events into the learner. Declared NONE on
+    // purpose, and the human string says what a user would actually do instead.
+    inverse: () => ({ kind: K.NONE, human: 'nothing to reverse — this only adds observations the learner already had queued; learned state can be reset separately' }),
+  },
+  {
+    key: 'learning-train',
+    summary: 'run one training cycle',
+    match: (id) => (id === 'learning:train' ? {} : null),
+    plan: () => ({ script: 'scripts/health-repair.mjs', args: ['--train-learning'] }),
+    inverse: () => ({ kind: K.NONE, human: 'nothing to reverse here — learned state is reset with `ruflo hooks intelligence --reset`, which is a separate, deliberate action' }),
+  },
+  {
+    // THE ONE THAT HAD NO EXECUTOR. See ADR-027's North Star case: stores full of memories that
+    // teach nothing. The remedy is not ours to invent — memory-doctor.mjs has printed the exact fix
+    // since the day it was written ("embedded but never distilled — run: ruflo memory distill run"),
+    // and the console simply never said it out loud. This wires that sentence to a button.
+    key: 'distill-fleet',
+    summary: 'distill embedded-but-never-distilled stores into reusable patterns',
+    match: (id) => (id === 'learning:distill-fleet' ? {} : null),
+    // needsReceipt: this remedy touches a SET of stores discovered at run time, so the inverse
+    // cannot be described up front. The executor writes down exactly which stores it snapshotted
+    // and where; the inverse reads that receipt. Without it, "restore the backups" would be a hope
+    // rather than an instruction — and a hope is what made the memory-index undo a lie.
+    plan: () => ({ script: 'scripts/health-repair.mjs', args: ['--distill-fleet'], needsReceipt: true }),
+    // Distillation WRITES (reasoning_patterns, episodes, causal_edges), so it needs a real inverse.
+    // health-repair snapshots each store with `ruflo memory backup` (rUv's own WAL-safe, rotated
+    // snapshotter) before distilling it; the inverse restores those snapshots.
+    inverse: () => ({ kind: K.RESTORE_STORE_BACKUPS }),
+  },
+  {
+    // THE CAPABILITY BRIDGE'S ONLY CURRENT MEMBER. buildCapabilityRecommendations() in
+    // console-engine.mjs offers `enable:memory-distillation` only while the capability is OFF; this
+    // is the executor behind it. See distill-project.mjs's header for why THIS script and not bare
+    // `ruflo memory distill run`: the wrapper snapshots first, fails closed on a receipt-write
+    // failure, and its `--restore` is the tested inverse (proven 644→648→644→648, 2026-07-24).
+    key: 'enable-memory-distillation',
+    autoEligible: true,
+    summary: "mine this project's stored memories into reusable patterns (snapshots first; reversible)",
+    match: (id) => (id === 'enable:memory-distillation' ? {} : null),
+    // This console instance is always scoped to ONE project — the directory it was started in — the
+    // same assumption the memory-index/learning-flush/learning-train remedies above already make.
+    // `usesServerProject` asks onboarding-console.mjs (impure, process-aware) to supply that directory
+    // at call time; this file stays pure and never reads process.cwd() itself (see header).
+    plan: () => ({ script: 'scripts/distill-project.mjs', args: [], usesServerProject: true }),
+    // `--restore` with no path argument uses distill-project.mjs's OWN newestSnapshot() lookup inside
+    // this project's `.swarm/backups` — the exact mechanism its header proves end to end. Re-deriving
+    // which snapshot to restore here, instead of asking the tool that took it, is the kind of
+    // duplicate implementation this project has already been burned by once (ADR-047's rejected
+    // "offered command and promised undo live on different execution paths" bug).
+    inverse: () => ({ kind: K.RESTORE_PROJECT_DISTILL }),
+  },
+  {
+    key: 'stack-sync',
+    summary: 'install/repair a global package to its target version',
+    match: (id) => {
+      if (RESERVED.has(id)) return null; // `repair:memory-index` is NOT a package repair
+      const m = /^(?:sync|repair):(.+)$/.exec(id);
+      return m ? { pkg: m[1] } : null;
+    },
+    plan: () => ({ script: 'scripts/stack-sync.mjs', args: ['--sync'] }),
+    // The inverse of a version bump is the version that was on disk a moment ago — which only the
+    // caller can read, so it is filled in at journal time. Declaring it here is what makes the
+    // closure test able to check that undo() can honour it.
+    inverse: ({ pkg }) => ({ kind: K.REINSTALL_VERSION, pkg }),
+  },
+  {
+    key: 'purge-shadows',
+    summary: 'delete stale duplicate copies from the npx cache',
+    match: (id) => (id === 'purge:shadows' ? {} : null),
+    plan: () => ({ script: 'scripts/stack-sync.mjs', args: ['--sync'] }),
+    inverse: () => ({ kind: K.AUTO_REBUILD, human: 'the temporary cache re-fills itself on next use; no manual step needed' }),
+  },
+  {
+    key: 'reconcile-project',
+    autoEligible: true,
+    summary: 'rewire a project from npx to the global binary',
+    match: (id) => {
+      const m = /^reconcile:(.+)$/.exec(id);
+      return m ? { project: m[1] } : null;
+    },
+    plan: ({ project }) => ({ script: 'scripts/reconcile-project.mjs', args: ['--apply', '--project', project], resolveProject: true }),
+    inverse: ({ project }) => ({ kind: K.RESTORE_BACKUP, project }),
+  },
+];
+
+/**
+ * Resolve an id to EXACTLY ONE remedy.
+ *
+ * Ambiguity throws rather than picking a winner. Silently preferring the first match is precisely
+ * how `repair:memory-index` once routed into a global npm sync while telling the user their database
+ * had been repaired — a wrong action reported as the right one. A throw is a loud developer error;
+ * a silent misroute is a user's data.
+ */
+export function resolveRemedy(id) {
+  const hits = [];
+  for (const r of REMEDIES) {
+    const params = r.match(id);
+    if (params) hits.push({ remedy: r, params });
+  }
+  if (hits.length > 1) {
+    throw new Error(`Remedy id "${id}" is ambiguous — claimed by: ${hits.map((h) => h.remedy.key).join(', ')}. Exactly one remedy must own an id.`);
+  }
+  return hits[0] ?? null;
+}
+
+/** The full plan for an id: what to run, and what reverses it. Null when nothing owns the id. */
+export function planFor(id) {
+  const hit = resolveRemedy(id);
+  if (!hit) return null;
+  const { remedy, params } = hit;
+  return {
+    key: remedy.key,
+    summary: remedy.summary,
+    autoEligible: remedy.autoEligible === true,
+    exec: remedy.plan(params),
+    undo: remedy.inverse(params),
+    params,
+  };
+}
+
+/**
+ * THE CLOSURE PROOF. Every id that can be OFFERED must be runnable and reversible.
+ *
+ * Takes the ids the recommendation builders actually constructed (never a hand-typed list — a
+ * hand-typed list drifts from the builders, which is the whole failure mode) plus the undo kinds
+ * onboarding-console.undo() implements, and returns every gap it finds.
+ *
+ * @param {string[]} offeredIds  ids from buildHealth/Stack/WiringRecommendations
+ * @param {string[]} handledUndoKinds  kinds undo() has a real branch for
+ * @returns {{orphanIds:string[], ambiguousIds:string[], unhandledUndoKinds:string[], deadKinds:string[]}}
+ */
+export function assertRegistryClosure(offeredIds = [], handledUndoKinds = []) {
+  const orphanIds = [];
+  const ambiguousIds = [];
+  const unhandledUndoKinds = [];
+  const handled = new Set(handledUndoKinds);
+  const usedKinds = new Set();
+
+  for (const id of offeredIds) {
+    let plan = null;
+    try { plan = planFor(id); } catch { ambiguousIds.push(id); continue; }
+    if (!plan) { orphanIds.push(id); continue; } // offered with no executor — the dead-button bug
+    usedKinds.add(plan.undo.kind);
+    // NONE is self-handling by definition, but it must still be DECLARED (see UNDO_KINDS).
+    if (plan.undo.kind !== K.NONE && !handled.has(plan.undo.kind)) unhandledUndoKinds.push(`${id} → ${plan.undo.kind}`);
+  }
+
+  // The other direction: a kind the registry declares that undo() cannot perform is a broken promise
+  // waiting to happen, even if no builder currently emits that id.
+  const declared = new Set();
+  for (const r of REMEDIES) {
+    try { declared.add(r.inverse(r.match(sampleIdFor(r)) || {}).kind); } catch { /* sampling is best-effort */ }
+  }
+  const deadKinds = [...declared].filter((k) => k !== K.NONE && !handled.has(k));
+
+  return { orphanIds, ambiguousIds, unhandledUndoKinds, deadKinds };
+}
+
+/** A representative id for each remedy, so closure can sample parameterized matchers too. */
+export function sampleIdFor(remedy) {
+  switch (remedy.key) {
+    case 'memory-index': return 'repair:memory-index';
+    case 'learning-flush': return 'learning:flush';
+    case 'learning-train': return 'learning:train';
+    case 'distill-fleet': return 'learning:distill-fleet';
+    case 'enable-memory-distillation': return 'enable:memory-distillation';
+    case 'stack-sync': return 'sync:ruflo';
+    case 'purge-shadows': return 'purge:shadows';
+    case 'reconcile-project': return 'reconcile:example';
+    default: return `__unknown:${remedy.key}`;
+  }
+}

package/scripts/rerank-cap-eval.mjs

@@ -0,0 +1,265 @@
+#!/usr/bin/env node
+// rerank-cap-eval.mjs — does bounding the cross-encoder pool change the ANSWERS?
+//
+// The cross-encoder is 84.7% of a query's wall and reads 605 (query, passage) pairs for an
+// all-repos question. Capping that pool is the only lever that moves the number. But a cap
+// re-orders results, and a naive one was MEASURED to lose the right answer — so no cap ships
+// without a before/after on a real question set.
+//
+// TWO PHASES, because the measurement is expensive and the policy search is not:
+//
+//   --collect   Runs the FROZEN held-out set (evals/held-out.json — the same corpus
+//               scripts/eval-brain.mjs gates on) UNCAPPED, recording every scored candidate:
+//               repo, path, lane, within-lane depth, pool position, and its cross-encoder score.
+//               One ~8-minute query per question. This is the whole cost of the experiment.
+//
+//   --report    Replays capRerankPool + selectResults — the SHIPPING functions, imported, not
+//               re-implemented — against those recorded scores, for every candidate budget. The
+//               cross-encoder is deterministic per (query, passage) pair and pairs are scored
+//               independently, so a replay of a subset is EXACT: it is the same arithmetic on the
+//               same numbers, not a model of it.
+//
+// Reported metrics are the ones a wrong answer would move, graded by ground truth (never a model
+// judge — an LLM panel once scored a zero-citation answer 98/100 on this repo):
+//   pairs      — cross-encoder pairs actually scored. The load-independent primary evidence.
+//   top1-same  — the winning document is byte-identical to the uncapped winner.
+//   kept       — the uncapped top-k documents still present in the capped top-k.
+//   routed     — top-1 lands in an expected repo (eval-brain's own metric, same Wilson bound).
+//   abstain    — adversarial questions still decline (top ce < 0).
+//   banner     — a winning gist chunk still carries its provenance banner.
+//
+//   node scripts/rerank-cap-eval.mjs --collect [--conc 3] [--only ho-01,ho-02]
+//   node scripts/rerank-cap-eval.mjs --report [--budgets 605,272,136,69]
+
+import fs from 'node:fs';
+import os from 'node:os';
+import path from 'node:path';
+import { fileURLToPath, pathToFileURL } from 'node:url';
+import { execFile } from 'node:child_process';
+import { gradeQuestion, aggregate } from './eval-brain.mjs';
+
+const ROOT = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '..');
+const KB = process.env.RUVNET_BRAIN_KB || path.join(os.homedir(), '.cache', 'ruvnet-brain', 'kb');
+// The code under test is THIS checkout's, run against whatever corpus KB points at.
+const CODE = path.join(ROOT, 'kb');
+const argv = process.argv.slice(2);
+const arg = (f, d) => { const i = argv.indexOf(f); return i >= 0 && argv[i + 1] ? argv[i + 1] : d; };
+const TRACES = arg('--traces', path.join(os.tmpdir(), 'ruvnet-brain-ce-cap-traces'));
+
+// The frozen 120, plus probes for the paths a cap could silently break. These are NOT scored
+// against expectRepo (they are not part of the frozen set and must never contaminate it) — they
+// are watched for one thing only: does the cap change the winner?
+const PROBES = [
+  { id: 'px-mcp-policy', query: 'which file enforces the MCP tool policy in ruvector', why: 'the case a naive cap was measured to lose (mcp-policy.js -> an ADR)' },
+  { id: 'px-adr-085', query: 'ADR-085', why: 'bare ADR number — per-repo collision disclosure reads the whole pool' },
+  { id: 'px-pkg-rvf', query: 'what is @ruvector/rvf', why: 'exact @scope/name — exercises the RESCUE lane the cap must never drop' },
+  { id: 'px-meetings', query: 'how many commits and contributors does the project have', why: 'answer is BM25-only in a transcript store; dense buries it past rank 40' },
+];
+
+// Probes first, then the frozen set DEALT ROUND-ROBIN ACROSS STRATA. The held-out file is grouped
+// by stratum, and at ~8 minutes a question a run can be interrupted — grouped order would make any
+// prefix a biased sample (all 'described', no 'adversarial'), which is the kind of partial evidence
+// that reads as a result and is not one. Round-robin makes every prefix stratified.
+function loadQuestions() {
+  const { questions } = JSON.parse(fs.readFileSync(path.join(ROOT, 'evals', 'held-out.json'), 'utf8'));
+  const byStratum = new Map();
+  for (const q of questions) (byStratum.get(q.stratum) ?? byStratum.set(q.stratum, []).get(q.stratum)).push(q);
+  const lanes = [...byStratum.values()];
+  const dealt = [];
+  for (let i = 0; dealt.length < questions.length; i++) for (const lane of lanes) if (lane[i]) dealt.push(lane[i]);
+  return [...PROBES.map((p) => ({ ...p, stratum: 'probe' })), ...dealt];
+}
+
+// ── collect ─────────────────────────────────────────────────────────────────────────────────────
+async function collect() {
+  const only = arg('--only', '') ? new Set(arg('--only', '').split(',')) : null;
+  const questions = loadQuestions().filter((q) => !only || only.has(q.id));
+  const CONC = Math.max(1, parseInt(arg('--conc', '3'), 10) || 3);
+  fs.mkdirSync(TRACES, { recursive: true });
+
+  const run = (q) => new Promise((resolve) => {
+    const trace = path.join(TRACES, `${q.id}.jsonl`);
+    // A partial trace from a killed run must never be read as a complete one: write to .part and
+    // rename only on a clean exit, so --report can never score a truncated pool as a real pool.
+    const part = `${trace}.part`;
+    try { fs.rmSync(part, { force: true }); } catch { /* fresh anyway */ }
+    const t0 = Date.now();
+    execFile('node', ['forge-ask-all.mjs', '--dir', KB, '--q', q.query, '--k', '3'], {
+      cwd: CODE,
+      timeout: 1800000,
+      maxBuffer: 256 * 1024 * 1024,
+      // --cap 0 (the default) is the uncapped baseline. A NON-zero --cap re-collects the same
+      // questions with the cap actually engaged, which is the only way to check the replay against
+      // reality: capping changes which pairs share a batch, and batch composition was MEASURED to
+      // move scores by up to 0.26 logits, so a replayed cap is an approximation of a real one.
+      // --cascade K does the same for ADR-058's two-stage cascade. Both are collected FOR REAL for
+      // exactly that reason: the cascade re-batches its survivors too, so its scores are its own.
+      env: {
+        ...process.env,
+        KB_CE_TRACE: part,
+        KB_CE_MAX_PAIRS: arg('--cap', '0'),
+        KB_CE_CASCADE_K: arg('--cascade', '0'),
+        KB_CE_CASCADE_TOKENS: arg('--tokens', '192'),
+      },
+    }, (err) => {
+      const ms = Date.now() - t0;
+      if (!err && fs.existsSync(part) && fs.statSync(part).size > 0) {
+        const rec = JSON.parse(fs.readFileSync(part, 'utf8').trim().split('\n')[0]);
+        fs.writeFileSync(trace, JSON.stringify({ id: q.id, stratum: q.stratum, expectRepo: q.expectRepo ?? null, ms, ...rec }) + '\n');
+        fs.rmSync(part, { force: true });
+        resolve({ id: q.id, ok: true, ms, pooled: rec.pooledAll });
+      } else {
+        try { fs.rmSync(part, { force: true }); } catch { /* nothing to clean */ }
+        resolve({ id: q.id, ok: false, ms, err: String(err?.message || 'no trace written').slice(0, 120) });
+      }
+    });
+  });
+
+  let cursor = 0, done = 0;
+  const t0 = Date.now();
+  const worker = async () => {
+    while (cursor < questions.length) {
+      const q = questions[cursor++];
+      if (fs.existsSync(path.join(TRACES, `${q.id}.jsonl`))) { done++; continue; } // resumable: an 8h run must survive a restart
+      const r = await run(q);
+      done++;
+      const la = os.loadavg()[0].toFixed(0);
+      process.stderr.write(`[cap-eval] ${done}/${questions.length} ${r.ok ? 'ok' : 'FAIL'} ${r.id} ${(r.ms / 1000).toFixed(0)}s pooled=${r.pooled ?? '-'} load=${la} elapsed=${((Date.now() - t0) / 60000).toFixed(0)}m${r.ok ? '' : ` :: ${r.err}`}\n`);
+    }
+  };
+  await Promise.all(Array.from({ length: Math.min(CONC, questions.length) }, worker));
+  console.error(`[cap-eval] traces in ${TRACES}`);
+}
+
+// ── report ──────────────────────────────────────────────────────────────────────────────────────
+async function report() {
+  const { capRerankPool, selectResults } = await import(pathToFileURL(path.join(CODE, 'forge-ask-all.mjs')).href);
+  if (!fs.existsSync(TRACES)) { console.error(`no traces at ${TRACES} — run --collect first`); process.exit(2); }
+  const files = fs.readdirSync(TRACES).filter((f) => f.endsWith('.jsonl'));
+  if (!files.length) { console.error(`no traces at ${TRACES} — run --collect first`); process.exit(2); }
+  const traces = files.map((f) => JSON.parse(fs.readFileSync(path.join(TRACES, f), 'utf8').trim().split('\n')[0]));
+
+  const budgets = (arg('--budgets', '') ? arg('--budgets', '').split(',').map(Number)
+    : [0, 408, 272, 204, 170, 136, 102, 69, 48, 24]).sort((a, b) => b - a);
+
+  // Rebuild each trace's pool in its ORIGINAL fan-out order — the cap's tie-break depends on it.
+  const pools = traces.map((t) => ({
+    ...t,
+    pool: [...t.cands].sort((a, b) => a.poolIdx - b.poolIdx)
+      .map((c) => ({ ...c, ceScore: c.ce, fullText: '', text: '', _lane: c.lane, _srcRank: c.rank, _poolIdx: c.poolIdx })),
+  }));
+
+  // The exact-package boost reads a candidate's BODY, which a trace deliberately does not carry
+  // (605 whole documents x 120 questions). It fires only on an `@scope/name` query, and it fires
+  // identically on any candidate that survives the cap, so it cannot flip a comparison between two
+  // survivors — but say so out loud rather than let a reader assume the replay is total.
+  const bodyDependent = pools.filter((p) => /@[a-z0-9][a-z0-9._-]*\/[a-z0-9._-]+/i.test(p.query)).map((p) => p.id);
+
+  const idOf = (r) => `${r.repo}/${r.path}`;
+
+  // CASCADE — the alternative policy, kept because it was measured and lost. Score every store's
+  // best passage first (~1 pair per store), then spend the rest of the budget only on the R stores
+  // whose best passage scored highest. It is the obvious "let the cross-encoder decide where to
+  // dig" design; the table below is the reason it is not what ships.
+  const cascadeKeep = (pool, R) => {
+    const best = new Map();
+    for (const c of pool) if (c._srcRank === 0 && (!best.has(c.repo) || c.ceScore > best.get(c.repo))) best.set(c.repo, c.ceScore);
+    const top = new Set([...best.entries()].sort((a, b) => b[1] - a[1]).slice(0, R).map(([r]) => r));
+    return pool.filter((c) => c._srcRank === 0 || c._lane === 'rescue' || top.has(c.repo));
+  };
+
+  const runKept = (p, kept) => {
+    const ranked = [...kept].sort((a, b) => (b.ceScore ?? -Infinity) - (a.ceScore ?? -Infinity));
+    const { results } = selectResults({ query: p.query, ranked, k: p.k });
+    return { pairs: kept.length, results };
+  };
+  const runPolicy = (p, limit) => runKept(p, capRerankPool(p.pool, { limit }).kept);
+
+  const base = new Map(pools.map((p) => [p.id, runPolicy(p, 0)]));
+
+  // ── ROOT CAUSE ────────────────────────────────────────────────────────────────────────────────
+  // How deep in its OWN store's vector ranking does the winning document sit? This one histogram
+  // decides whether any pre-score cap can be safe. If winners clustered at depth 0, a tiny budget
+  // would be free. They do not: the cross-encoder routinely promotes a passage its own store
+  // ranked 4th or 7th, which means depth carries little information about who wins, which means
+  // every pair a depth cut removes is a real chance of removing the answer.
+  const depthTop1 = {}, depthTopK = {};
+  for (const p of pools) {
+    const r = base.get(p.id).results;
+    if (!r.length) continue;
+    depthTop1[r[0]._srcRank] = (depthTop1[r[0]._srcRank] || 0) + 1;
+    for (const x of r) depthTopK[x._srcRank] = (depthTopK[x._srcRank] || 0) + 1;
+  }
+  const hist = (h) => Object.keys(h).map(Number).sort((a, b) => a - b).map((d) => `depth ${d}: ${h[d]}`).join(' | ');
+
+  console.log(`\n# cross-encoder pool cap — quality vs pair count`);
+  console.log(`corpus: ${pools.length} questions (${pools.filter((p) => p.stratum !== 'probe').length} frozen held-out + ${pools.filter((p) => p.stratum === 'probe').length} probes)`);
+  console.log(`uncapped pool: min ${Math.min(...pools.map((p) => p.pooledAll))}, median ${median(pools.map((p) => p.pooledAll))}, max ${Math.max(...pools.map((p) => p.pooledAll))} pairs`);
+  console.log(`body-dependent boost not replayed for: ${bodyDependent.length ? bodyDependent.join(', ') : '(none — no @scope/name query in the set)'}`);
+  console.log(`\n## where the answer actually lives, in its own store's vector ranking`);
+  console.log(`  winning document : ${hist(depthTop1)}`);
+  console.log(`  every top-k hit  : ${hist(depthTopK)}`);
+  console.log(`  (a budget of B pairs across S stores reaches depth B/S. Winners spread across depth`);
+  console.log(`   means a depth cut drops answers roughly in proportion to what it saves.)\n`);
+  console.log('| policy | pairs (median) | top1-same | top-k kept | routed k/n (Wilson lo) | abstain | banner |');
+  console.log('|---|---|---|---|---|---|---|');
+
+  // Every policy the run compares: the shipping depth cap at a range of budgets, plus cascade.
+  const policies = [
+    ...budgets.map((limit) => ({ label: limit === 0 ? 'uncapped' : `depth B=${limit}`, key: `depth-${limit}`, keep: (p) => capRerankPool(p.pool, { limit }).kept })),
+    ...[4, 8, 12, 16, 24, 32].map((R) => ({ label: `cascade R=${R}`, key: `cascade-${R}`, keep: (p) => cascadeKeep(p.pool, R) })),
+  ];
+
+  const detail = {};
+  for (const policy of policies) {
+    const rows = [];
+    let top1Same = 0, keptNum = 0, keptDen = 0;
+    const changed = [];
+    for (const p of pools) {
+      const cur = runKept(p, policy.keep(p));
+      const b = base.get(p.id);
+      const same = idOf2(cur.results[0]) === idOf2(b.results[0]);
+      if (same) top1Same++; else changed.push(`${p.id}: ${idOf2(b.results[0])} -> ${idOf2(cur.results[0])}`);
+      const curSet = new Set(cur.results.map(idOf));
+      for (const r of b.results) { keptDen++; if (curSet.has(idOf(r))) keptNum++; }
+      if (p.stratum !== 'probe') {
+        const top = cur.results[0] ?? null;
+        // Same grading rule as scripts/eval-brain.mjs, fed from the replayed result set. `grounded`
+        // is structurally true here: every candidate is a passage the retriever actually returned.
+        rows.push({
+          id: p.id, stratum: p.stratum,
+          ...gradeQuestion({ stratum: p.stratum, expectRepo: p.expectRepo },
+            { grounded: cur.results.length > 0,
+              citations: top ? [{ repo: top.repo, fullPath: idOf(top), ce: top.ceScore }] : [],
+              bannerPresent: cur.results.some((r) => r.gist) }),
+        });
+      }
+      detail[policy.key] ??= {};
+      detail[policy.key][p.id] = { pairs: cur.pairs, top1: idOf2(cur.results[0]) };
+    }
+    const agg = aggregate(rows);
+    const pairs = median(pools.map((p) => policy.keep(p).length));
+    console.log(`| ${policy.label} | ${pairs} | ${top1Same}/${pools.length} (${pct(top1Same / pools.length)}) | ${keptNum}/${keptDen} (${pct(keptNum / keptDen)}) | ${agg.routed.k}/${agg.routed.n} (${pct(agg.routed.lo)}) | ${agg.abstain.k}/${agg.abstain.n} | ${agg.banner.k}/${agg.banner.n} |`);
+    if (changed.length && changed.length <= 12) detail[`changed@${policy.key}`] = changed;
+  }
+
+  console.log('\n## winners that changed, per policy (empty = the policy changed no answer)');
+  for (const policy of policies) {
+    if (policy.key === 'depth-0') continue;
+    const c = detail[`changed@${policy.key}`];
+    console.log(`\n### ${policy.label}`);
+    if (!c) console.log('  (too many to list — see the top1-same column)');
+    else if (!c.length) console.log('  none');
+    else for (const line of c) console.log(`  ${line}`);
+  }
+
+  if (argv.includes('--json')) fs.writeFileSync(path.join(TRACES, 'report.json'), JSON.stringify(detail, null, 2));
+}
+
+const idOf2 = (r) => (r ? `${r.repo}/${r.path}` : '(no result)');
+const pct = (x) => `${(x * 100).toFixed(1)}%`;
+const median = (a) => { const s = [...a].sort((x, y) => x - y); return s.length % 2 ? s[(s.length - 1) / 2] : Math.round((s[s.length / 2 - 1] + s[s.length / 2]) / 2); };
+
+if (argv.includes('--collect')) await collect();
+else if (argv.includes('--report')) await report();
+else { console.error('usage: rerank-cap-eval.mjs --collect | --report'); process.exit(2); }

package/scripts/rerank-cap-warm-ab.mjs

@@ -0,0 +1,129 @@
+#!/usr/bin/env node
+// rerank-cap-warm-ab.mjs — the paired, WARM before/after for the cross-encoder pool cap.
+//
+// Why this exists rather than timing the CLI: a cold `forge-ask-all.mjs` spends ~53s loading two
+// ONNX models before it scores anything, and the cap cannot touch that. Timing cold runs would
+// dilute the effect being measured by roughly 3x and would also compare runs taken hours apart on a
+// machine whose load moved underneath them. This harness loads the models ONCE and then runs each
+// question twice in the same process — uncapped and capped — so the only difference between the two
+// numbers is the thing under test.
+//
+// PAIRED AND ORDER-ALTERNATED: question i runs uncapped-then-capped on even i and capped-then-
+// uncapped on odd i, so any residual warm-up or thermal drift cannot systematically favour one arm.
+// Both arms' ANSWERS are recorded, not just their times: a cap that is fast and wrong is a failure,
+// and this is the file that would catch it.
+//
+// TWO POLICIES SHARE THIS HARNESS, because a number is only comparable to another number taken
+// the same way. --cap is ADR-057's flat pool cap (select by vector distance, one full read each).
+// --cascade is ADR-058's two-stage cascade (read every pooled pair at a truncated length, then
+// re-read the top K in full). Same questions, same pairing, same warm process, same table — so
+// "-30.4%" and whatever the cascade measures can be put side by side honestly.
+//
+//   node scripts/rerank-cap-warm-ab.mjs --cap 408      [--n 24] [--out result.json]
+//   node scripts/rerank-cap-warm-ab.mjs --cascade 64   [--n 24] [--tokens 192]
+
+import fs from 'node:fs';
+import os from 'node:os';
+import path from 'node:path';
+import { fileURLToPath, pathToFileURL } from 'node:url';
+
+const ROOT = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '..');
+const KB = process.env.RUVNET_BRAIN_KB || path.join(os.homedir(), '.cache', 'ruvnet-brain', 'kb');
+const argv = process.argv.slice(2);
+const arg = (f, d) => { const i = argv.indexOf(f); return i >= 0 && argv[i + 1] ? argv[i + 1] : d; };
+// Exactly one arm is under test. --cascade wins if both are given, and says so rather than
+// silently measuring a policy the operator did not ask for.
+const CASCADE = arg('--cascade', '');
+const CAP = arg('--cap', CASCADE ? '0' : '408');
+const TOKENS = arg('--tokens', '192');
+const MODE = CASCADE ? 'cascade' : 'cap';
+const LABEL = CASCADE ? `cascade K=${CASCADE} @${TOKENS}tok` : `capped B=${CAP}`;
+const N = parseInt(arg('--n', '24'), 10);
+const OUT = arg('--out', path.join(os.tmpdir(), `ce-${MODE}-warm-ab-${CASCADE || CAP}.json`));
+
+const { searchAll } = await import(pathToFileURL(path.join(ROOT, 'kb', 'forge-ask-all.mjs')).href);
+const { gradeQuestion, aggregate } = await import(pathToFileURL(path.join(ROOT, 'scripts', 'eval-brain.mjs')).href);
+
+// Stratified subset of the frozen held-out set, dealt round-robin so every stratum is represented
+// even at small n — a prefix of a grouped file would be all 'described' and no 'adversarial'.
+const { questions } = JSON.parse(fs.readFileSync(path.join(ROOT, 'evals', 'held-out.json'), 'utf8'));
+const byStratum = new Map();
+for (const q of questions) (byStratum.get(q.stratum) ?? byStratum.set(q.stratum, []).get(q.stratum)).push(q);
+const lanes = [...byStratum.values()];
+const dealt = [];
+for (let i = 0; dealt.length < questions.length; i++) for (const lane of lanes) if (lane[i]) dealt.push(lane[i]);
+const set = dealt.slice(0, N);
+
+const idOf = (r) => (r ? `${r.repo}/${r.path}` : '(none)');
+// `on` selects the arm under test; `off` is always the untouched uncapped, uncascaded path. Both
+// env knobs are set on EVERY call rather than only when engaged — a leftover value from the
+// previous call is exactly how an A/B measures the same arm twice and reports a 0% delta.
+async function once(query, on) {
+  process.env.KB_CE_MAX_PAIRS = String(on && MODE === 'cap' ? CAP : 0);
+  process.env.KB_CE_CASCADE_K = String(on && MODE === 'cascade' ? CASCADE : 0);
+  process.env.KB_CE_CASCADE_TOKENS = String(TOKENS);
+  const t0 = Date.now();
+  const out = await searchAll({ dir: KB, query, k: 3 });
+  return {
+    ms: Date.now() - t0, pairs: out.pooled, pooledAll: out.pooledAll,
+    prefiltered: out.prefiltered ?? 0, prefilterMs: out.prefilterMs ?? 0,
+    results: out.results.map((r) => ({ id: idOf(r), repo: r.repo, ce: r.ceScore, gist: !!r.gist })),
+  };
+}
+
+// Warm-up: the first query of a process pays both model loads. It is thrown away deliberately —
+// including it would credit the cap with a saving it did not produce.
+process.stderr.write(`[warm-ab] mode=${MODE} (${LABEL}) — loading models (first query is discarded)...\n`);
+const w0 = Date.now();
+await once('what is ruvector', false);
+process.stderr.write(`[warm-ab] warm after ${((Date.now() - w0) / 1000).toFixed(1)}s\n`);
+
+const rows = [];
+for (let i = 0; i < set.length; i++) {
+  const q = set[i];
+  const onFirst = i % 2 === 1;
+  const a = await once(q.query, onFirst);
+  const b = await once(q.query, !onFirst);
+  const [off, on] = onFirst ? [b, a] : [a, b];
+  rows.push({ id: q.id, stratum: q.stratum, expectRepo: q.expectRepo ?? null, onFirst, off, on });
+  process.stderr.write(`[warm-ab] ${i + 1}/${set.length} ${q.id} off=${(off.ms / 1000).toFixed(1)}s/${off.pairs}p on=${(on.ms / 1000).toFixed(1)}s/${on.pairs}p top1${idOf2(off) === idOf2(on) ? '=same' : ' CHANGED'}\n`);
+}
+function idOf2(x) { return x.results[0]?.id ?? '(none)'; }
+
+fs.writeFileSync(OUT, JSON.stringify({ mode: MODE, cap: CAP, cascade: CASCADE, tokens: TOKENS, kb: KB, n: set.length, load: os.loadavg(), rows }, null, 2));
+
+// ── the table ───────────────────────────────────────────────────────────────────────────────────
+const med = (a) => { const s = [...a].sort((x, y) => x - y); return s.length % 2 ? s[(s.length - 1) / 2] : Math.round((s[s.length / 2 - 1] + s[s.length / 2]) / 2); };
+const pct = (x) => `${(x * 100).toFixed(1)}%`;
+const grade = (arm) => aggregate(rows.map((r) => {
+  const top = r[arm].results[0] ?? null;
+  return { stratum: r.stratum, ...gradeQuestion({ stratum: r.stratum, expectRepo: r.expectRepo },
+    { grounded: r[arm].results.length > 0,
+      citations: top ? [{ repo: top.repo, fullPath: top.id, ce: top.ce }] : [],
+      bannerPresent: r[arm].results.some((x) => x.gist) }) };
+}));
+const top1Same = rows.filter((r) => idOf2(r.off) === idOf2(r.on)).length;
+let kn = 0, kd = 0;
+for (const r of rows) { const s = new Set(r.on.results.map((x) => x.id)); for (const x of r.off.results) { kd++; if (s.has(x.id)) kn++; } }
+const gOff = grade('off'), gOn = grade('on');
+
+console.log(`\n# warm A/B — ${MODE === 'cascade' ? `cross-encoder CASCADE KB_CE_CASCADE_K=${CASCADE} KB_CE_CASCADE_TOKENS=${TOKENS}` : `cross-encoder pool cap KB_CE_MAX_PAIRS=${CAP}`}`);
+console.log(`${rows.length} questions from the frozen held-out set, paired, order-alternated, one warm process. load1=${os.loadavg()[0].toFixed(1)} on ${os.cpus().length} cores.\n`);
+console.log('| | full reads (median) | warm wall median | warm wall mean | routed | abstain | banner |');
+console.log('|---|---|---|---|---|---|---|');
+const fmt = (arm, g) => `| ${med(rows.map((r) => r[arm].pairs))} | ${(med(rows.map((r) => r[arm].ms)) / 1000).toFixed(2)}s | ${(rows.reduce((a, r) => a + r[arm].ms, 0) / rows.length / 1000).toFixed(2)}s | ${g.routed.k}/${g.routed.n} | ${g.abstain.k}/${g.abstain.n} | ${g.banner.k}/${g.banner.n} |`;
+console.log(`| baseline, no policy (before) ${fmt('off', gOff)}`);
+console.log(`| ${LABEL} (after) ${fmt('on', gOn)}`);
+if (MODE === 'cascade') {
+  // Stage 1 is real work and must be visible, or the table reads as if 64 pairs were the whole cost.
+  console.log(`\nstage-1 prefilter: ${med(rows.map((r) => r.on.prefiltered))} pairs read at ${TOKENS} tokens, median ${(med(rows.map((r) => r.on.prefilterMs)) / 1000).toFixed(2)}s of the after-time above`);
+}
+const dMed = 1 - med(rows.map((r) => r.on.ms)) / med(rows.map((r) => r.off.ms));
+console.log(`\nwall-time change (median, paired): ${dMed >= 0 ? '-' : '+'}${pct(Math.abs(dMed))}`);
+console.log(`top-1 cited path identical : ${top1Same}/${rows.length} (${pct(top1Same / rows.length)})`);
+console.log(`top-3 cited paths retained : ${kn}/${kd} (${pct(kn / kd)})`);
+console.log('\n## every question whose top-1 changed');
+const changed = rows.filter((r) => idOf2(r.off) !== idOf2(r.on));
+if (!changed.length) console.log('  none');
+for (const r of changed) console.log(`  ${r.id} [${r.stratum}] expect=${(r.expectRepo || ['-']).join('|')}\n     before: ${idOf2(r.off)}  (ce ${r.off.results[0]?.ce?.toFixed(3)})\n     after : ${idOf2(r.on)}  (ce ${r.on.results[0]?.ce?.toFixed(3)})`);
+console.log(`\nraw: ${OUT}`);