ruvnet-brain 4.0.1 → 4.0.4

package/scripts/behavioral-l1-l4.mjs

@@ -0,0 +1,210 @@
+#!/usr/bin/env node
+// scripts/behavioral-l1-l4.mjs — the 4-level BEHAVIORAL test harness for the RuvNet brain.
+//
+// The existing eval scripts (prove.mjs, brain-capability-check.mjs, brain-grade-groundtruth.mjs)
+// test RETRIEVAL: does the right source come back, is it confident, is the answer graded high. This
+// harness tests the four BEHAVIORS a user actually depends on — including the one the brain is meant
+// to DRIVE (orchestration), which no prior script covered:
+//
+//   L1 ROUTE        — "which repo solves X?" → the brain routes the top hit to the correct repo.
+//   L2 DEEP-RECALL  — "how is X implemented?" → the brain returns CODE-level source (full bodies),
+//                     not just doc-comments — proof it walked the repo to the implementation.
+//   L3 IMPLEMENT    — "implement X using <repo>" → the #1 cited source actually contains the API you
+//                     would build against, so an implementation grounded in it is correct (mechanical
+//                     correctness proxy; full multi-vendor grading stays brain-grade-groundtruth.mjs).
+//   L4 ORCHESTRATE  — the newbie path. Run the real hooks — session-start.sh ONCE per harness run
+//                     (since ADR-0011 Phase 2 it carries THE PLAYBOOK: ground → SPARC → DDD/ADR →
+//                     parallel swarm → QA → score ≥98 → frontend-design → AI image-gen →
+//                     ask-for-API-key → prove) AND the per-turn UserPromptSubmit hook
+//                     (ground-ruvnet.sh) on "go make magic / here's a spec, build it" prompts — and
+//                     assert every directive appears at the layer that now owns it (see the ADR-0011
+//                     Phase 2 relocation note above the L4 table). Hook injection is the only
+//                     enforcement primitive that exists (ADR-0005), so testing the injected text IS
+//                     testing whether the brain takes the wheel.
+//
+// L1–L3 call searchAll() — the EXACT engine search_ruvnet wraps (forge-mcp-all.mjs:77). L4 shells the
+// real hooks. No mutation, read-only. Needs KB_MODEL_CACHE pointing at the ONNX model cache for L1–L3.
+//
+// Usage:
+//   KB_MODEL_CACHE=<cache> node scripts/behavioral-l1-l4.mjs [--dir kb] [--repos a,b,c] [--levels L1,L4]
+//   --repos  restrict the search pool (collision-safe while a build sweep is running)
+//   --levels comma list of L1,L2,L3,L4 to run (default all)
+// Exit 0 iff every selected check passes.
+
+import { execFileSync } from 'node:child_process';
+import fs from 'node:fs';
+import os from 'node:os';
+import path from 'node:path';
+import { fileURLToPath } from 'node:url';
+import { searchAll } from '../kb/forge-ask-all.mjs';
+
+const __dirname = path.dirname(fileURLToPath(import.meta.url));
+const ROOT = path.resolve(__dirname, '..');
+const arg = (f, d) => { const i = process.argv.indexOf(f); return i >= 0 ? process.argv[i + 1] : d; };
+const KB_DIR = path.resolve(arg('--dir', path.join(ROOT, 'kb')));
+const HOOK = path.resolve(arg('--hook', path.join(ROOT, 'plugin/scripts/ground-ruvnet.sh')));
+const SESSION_HOOK = path.resolve(arg('--session-hook', path.join(ROOT, 'plugin/scripts/session-start.sh')));
+const reposArg = arg('--repos', '');
+const REPOS = reposArg ? reposArg.split(',').map((s) => s.trim()).filter(Boolean) : undefined;
+const K = parseInt(arg('--k', '6'), 10);
+const LEVELS = new Set((arg('--levels', 'L1,L2,L3,L4')).split(',').map((s) => s.trim().toUpperCase()));
+
+const CODE_RX = /(\bfn\s|\bfunction\s|\bimpl\s|\bstruct\s|\bclass\s|\bexport\s|\basync\s|=>|\(full body\))/;
+const G = (s) => `\x1b[32m${s}\x1b[0m`, R = (s) => `\x1b[31m${s}\x1b[0m`, DIM = (s) => `\x1b[2m${s}\x1b[0m`;
+
+// ── L1 ROUTE: question → repo it should route to ──────────────────────────────────────────────
+const L1 = [
+  { q: 'single-file HNSW vector store with no server', expect: 'ruvector' },
+  { q: 'orchestrate a swarm of agents across multiple steps', expect: 'ruflo' },
+  { q: 'causal explainable agent memory — why did I recall that', expect: 'agentdb' },
+  { q: 'fork a million vectors cheaply, branch agent memory copy-on-write', expect: 'agenticow' },
+  { q: 'WiFi CSI sensing of vital signs on an edge device', expect: 'ruview' },
+];
+// ── L2 DEEP-RECALL: must return CODE (full bodies) from the right repo, not doc-comments ──────────
+const L2 = [
+  { q: 'how is the HNSW graph insert / neighbour selection implemented', expect: 'ruvector' },
+  { q: 'how does the swarm decide how many agents to spawn in code', expect: 'ruflo' },
+  { q: 'how is a copy-on-write branch created in code', expect: 'agenticow' },
+];
+// ── L3 IMPLEMENT: #1 cited source must contain the API you'd build against ────────────────────────
+const L3 = [
+  { q: 'implement a nearest-neighbour query against an .rvf store', expect: 'ruvector', apiRx: /query|search|knn|nearest/i },
+  { q: 'store and recall an agent memory entry with AgentDB', expect: 'agentdb', apiRx: /store|recall|memory|insert|search/i },
+];
+// ── L4 ORCHESTRATE: run the real hook; assert the injected directive is complete ──────────────────
+const L4 = [
+  {
+    name: 'newbie "go make magic"',
+    prompt: 'go make magic and build me an app that helps people learn',
+    // pure build prompt, no RuvNet keyword → Gate 3 (take-the-wheel) must fire with the full pipeline
+    must: ['take the wheel', 'SPARC', 'DDD', 'ADR', 'swarm', 'QA gate', '98', 'frontend-design', 'image generation', 'API key', 'PROVEN', 'PARALLEL'],
+  },
+  {
+    name: 'spec → build (RuvNet-named)',
+    prompt: 'here is a product spec, build a knowledge base feature using ruvector',
+    // names ruvector → Gate 1 (ground) AND Gate 3 (build) must both fire
+    must: ['search_ruvnet', 'ground', 'SPARC', 'DDD', 'frontend-design', 'image generation', 'API key', '98'],
+  },
+  {
+    name: 'classical-default drift',
+    prompt: 'set up pinecone and langchain to do rag over my docs',
+    // drift keywords → Gate 2 (hijack) must fire AND build keyword "set up" → Gate 3
+    must: ['classical default', 'RuVector', 'Ruflo', 'AgentDB', 'take the wheel'],
+  },
+  {
+    name: 'pure recall (no build)',
+    prompt: 'what can ruflo actually do',
+    must: ['search_ruvnet', 'ground'],
+    mustNot: ['take the wheel'], // no build verb → Gate 3 must stay silent
+  },
+];
+
+const results = { L1: [], L2: [], L3: [], L4: [] };
+
+async function runL1() {
+  for (const t of L1) {
+    const { results: r } = await searchAll({ dir: KB_DIR, query: t.q, k: K, repos: REPOS });
+    const top = r[0];
+    // Routing (breadth) is correct if the #1 hit (a) IS the expected repo, (b) is a `concepts`
+    // capability-card naming it (the by-description router that took routing 33%→96%), or (c) is a
+    // doc explicitly ABOUT it in a sibling repo (e.g. a cognitum-cogs page packaging ruview) — all
+    // three correctly point the user at the right building block. Annotated transparently below.
+    const named = !!top && new RegExp(`\\b${t.expect}\\b`, 'i').test(`${top.path} ${top.title} ${(top.fullText || '').slice(0, 400)}`);
+    const viaCard = !!top && top.repo === 'concepts' && named;
+    const viaTopic = !!top && top.repo !== t.expect && top.repo !== 'concepts' && named;
+    const pass = !!top && (top.repo === t.expect || viaCard || viaTopic);
+    const tag = !top ? '' : top.repo === t.expect ? '' : viaCard ? ' via card' : viaTopic ? ` via ${top.repo} doc` : '';
+    results.L1.push({ pass, info: `${t.q.slice(0, 38)}… → ${top ? top.repo + '/' + (top.path || '').split('/').pop() : 'none'} (want ${t.expect}${tag}; ce ${top?.ceScore?.toFixed(2) ?? 'n/a'})` });
+  }
+}
+async function runL2() {
+  // DEPTH is scoped to the target repo: "does repo X actually contain the implementation of Y?"
+  // (full-ecosystem routing is L1's job; here we ask the repo directly so cards/sibling docs can't
+  // mask whether the code is indexed). k bumped a bit — deep-recall may sit below overview chunks.
+  for (const t of L2) {
+    const { results: r } = await searchAll({ dir: KB_DIR, query: t.q, k: 10, repos: [t.expect] });
+    const hit = r.find((x) => CODE_RX.test(x.fullText || x.text || ''));
+    const pass = !!hit;
+    results.L2.push({ pass, info: `${t.q.slice(0, 42)}… → ${pass ? `CODE @ ${t.expect}/${hit.path}` : r[0] ? `doc-only @ ${t.expect}/${r[0].path}` : `no ${t.expect} hit`}` });
+  }
+}
+async function runL3() {
+  // Implementability scoped to the target repo: its #1 source for the task must contain the API.
+  for (const t of L3) {
+    const { results: r } = await searchAll({ dir: KB_DIR, query: t.q, k: K, repos: [t.expect] });
+    const top = r[0];
+    const apiOk = top && t.apiRx.test(top.fullText || top.text || '');
+    const pass = !!top && !!apiOk;
+    results.L3.push({ pass, info: `${t.q.slice(0, 40)}… → #1 ${top ? `${t.expect}/${top.path}` : 'none'} apiMatch=${!!apiOk}` });
+  }
+}
+function runL4() {
+  // Phase 2 (2026-07-09) relocated the full playbook into session-start.sh (once per session); the
+  // per-turn hook carries only a slim reminder. So `must` markers are asserted against the UNION of
+  // session-start output (captured once, exactly as a real session receives it) + the scenario's
+  // per-turn hook output. `mustNot` leak checks stay on the PER-TURN output alone — session-start
+  // legitimately always contains the playbook; the leak under test is a non-build turn wrongly
+  // triggering build directives.
+  const hookEnv = { ...process.env, CLAUDE_PLUGIN_ROOT: path.join(ROOT, 'plugin') };
+  let session = '';
+  try { session = execFileSync('bash', [SESSION_HOOK], { input: '', encoding: 'utf8', env: hookEnv }); } catch (e) { session = (e.stdout || ''); }
+  const sessionLc = session.toLowerCase();
+  for (const t of L4) {
+    let out = '';
+    try { out = execFileSync('sh', [HOOK], { input: JSON.stringify({ prompt: t.prompt }), encoding: 'utf8', env: hookEnv }); } catch (e) { out = (e.stdout || '') + (e.stderr || ''); }
+    const lc = out.toLowerCase();
+    const union = `${lc}\n${sessionLc}`;
+    const missing = (t.must || []).filter((m) => !union.includes(m.toLowerCase()));
+    const leaked = (t.mustNot || []).filter((m) => lc.includes(m.toLowerCase()));
+    const pass = missing.length === 0 && leaked.length === 0;
+    results.L4.push({ pass, info: `${t.name}${missing.length ? ` — MISSING: ${missing.join(', ')}` : ''}${leaked.length ? ` — LEAKED: ${leaked.join(', ')}` : ''}` });
+  }
+}
+
+(async () => {
+  console.log(`\n=== RuvNet Brain — L1–L4 behavioral harness ===`);
+  console.log(DIM(`kb=${KB_DIR}  pool=${REPOS ? REPOS.join(',') : 'ALL'}  hook=${path.relative(ROOT, HOOK)}\n`));
+  if (LEVELS.has('L1')) await runL1();
+  if (LEVELS.has('L2')) await runL2();
+  if (LEVELS.has('L3')) await runL3();
+  if (LEVELS.has('L4')) runL4();
+
+  // VACUOUS TRUTH IS NOT A PASS (2026-07-28). `allPass` started true and the loop below `continue`s
+  // over any level with no results, so `--levels L5` — a level that does not exist — ran ZERO checks
+  // and printed OVERALL: PASS, exit 0. GPT-5.6-Sol reproduced it during the Gen-2 QE grading and I
+  // reproduced it again here before fixing. A harness that certifies an empty run is worse than no
+  // harness: it is the mechanism by which README:484/526 could claim "L1-L4 behavioral harness -
+  // all pass" as evidence the hook "drives the full pipeline", while two independent graders scored
+  // the QE apparatus 38/100 and 53/100. Nothing could contradict the optimistic claim, because the
+  // thing meant to contradict it passed by running nothing.
+  //
+  // Two guards: an unknown level is an ERROR (not a silent no-op), and zero executed checks is a
+  // FAIL. "I checked nothing and found no problems" is the sentence this exists to make impossible.
+  const KNOWN = ['L1', 'L2', 'L3', 'L4'];
+  const unknown = [...LEVELS].filter((l) => !KNOWN.includes(l));
+  if (unknown.length) {
+    console.log(`${R('ERROR')}  unknown level(s): ${unknown.join(', ')} — known: ${KNOWN.join(', ')}`);
+    console.log(`=== OVERALL: ${R('FAIL')} (nothing was verified) ===\n`);
+    process.exit(2);
+  }
+  const executed = KNOWN.reduce((n, l) => n + (results[l]?.length || 0), 0);
+  if (executed === 0) {
+    console.log(`${R('ERROR')}  ZERO checks executed — an empty run is not a pass.`);
+    console.log(`=== OVERALL: ${R('FAIL')} (nothing was verified) ===\n`);
+    process.exit(2);
+  }
+
+  let allPass = true;
+  const titles = { L1: 'ROUTE', L2: 'DEEP-RECALL', L3: 'IMPLEMENT', L4: 'ORCHESTRATE' };
+  for (const lvl of ['L1', 'L2', 'L3', 'L4']) {
+    if (!LEVELS.has(lvl) || !results[lvl].length) continue;
+    const passN = results[lvl].filter((x) => x.pass).length;
+    const ok = passN === results[lvl].length;
+    allPass = allPass && ok;
+    console.log(`${ok ? G('PASS') : R('FAIL')}  ${lvl} ${titles[lvl]}  (${passN}/${results[lvl].length})`);
+    for (const x of results[lvl]) console.log(`   ${x.pass ? G('✓') : R('✗')} ${x.info}`);
+    console.log();
+  }
+  console.log(`=== OVERALL: ${allPass ? G('PASS') : R('FAIL')} ===\n`);
+  process.exit(allPass ? 0 : 1);
+})();

package/scripts/brain-capability-check.mjs

@@ -0,0 +1,72 @@
+#!/usr/bin/env node
+// brain-capability-check.mjs — the "never doubt a real capability" gate.
+//
+// Stuart's #1 ask: Claude Code must STOP guessing "I don't think RuvNet can do X" when X exists in
+// Ruv's real source. This measures exactly that. For each "Can <repo> do <Y>?" question — where Y is
+// a REAL capability — the bundle must return a CONFIDENTLY-relevant source document (positive cross-
+// encoder relevance) from the right repo whose text actually evidences the capability. A weak/negative
+// top hit is precisely where Claude would have doubted → a measured FAIL to drive to zero.
+//
+// It also supports CONTROL questions (truth:"no") — capabilities a repo genuinely does NOT have — to
+// confirm the bundle does not hallucinate capabilities either (it should NOT return a confident,
+// evidence-bearing hit for those).
+//
+//   node scripts/brain-capability-check.mjs --dir kb --questions kb/capability.agentdb.json [--tau 0] [--k 5] [--pool 12]
+import fs from 'node:fs';
+import path from 'node:path';
+import { fileURLToPath } from 'node:url';
+import { searchAll } from '../kb/forge-ask-all.mjs';
+
+const ROOT = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '..');
+const arg = (f, d) => { const i = process.argv.indexOf(f); return i >= 0 && process.argv[i + 1] ? process.argv[i + 1] : d; };
+const DIR = path.resolve(ROOT, arg('--dir', 'kb'));
+const QFILE = path.resolve(ROOT, arg('--questions', 'kb/capability.agentdb.json'));
+const TAU = parseFloat(arg('--tau', '0'));        // min cross-encoder relevance to count as "confident"
+const K = parseInt(arg('--k', '5'), 10);
+const POOL = parseInt(arg('--pool', '12'), 10);
+
+const spec = JSON.parse(fs.readFileSync(QFILE, 'utf8'));
+const questions = spec.questions || spec;
+
+// A capability is "evidenced + confident" if, within the top-k cross-repo hits, some hit is from the
+// expected repo (when specified), scores >= TAU, and its text contains >= half of the evidence terms.
+function evaluate(results, qq) {
+  const ev = (qq.evidence || []).map((s) => s.toLowerCase());
+  const need = Math.max(1, Math.ceil(ev.length / 2));
+  let best = null;
+  for (const r of results) {
+    // accept hits from the expected repo, OR a concepts-store hit attributed to it via its path prefix
+    if (qq.expectRepo && r.repo !== qq.expectRepo && !(r.repo === 'concepts' && (r.path || '').startsWith(qq.expectRepo + '/'))) continue;
+    const txt = (r.fullText || r.text || '').toLowerCase();
+    const hits = ev.filter((t) => txt.includes(t)).length;
+    const confident = (r.ceScore ?? -Infinity) >= TAU;
+    const cand = { repo: r.repo, path: r.path, ce: r.ceScore, evHits: hits, evNeed: need, confident, evidenced: hits >= need };
+    if (!best || (cand.confident && cand.evidenced && !(best.confident && best.evidenced)) || (r.ceScore ?? -Infinity) > (best.ce ?? -Infinity)) {
+      if (!best || (r.ceScore ?? -Infinity) > (best.ce ?? -Infinity) || (cand.confident && cand.evidenced)) best = cand;
+    }
+  }
+  return best;
+}
+
+const rows = [];
+for (let i = 0; i < questions.length; i++) {
+  const qq = questions[i];
+  const { results } = await searchAll({ dir: DIR, query: qq.q, k: K, pool: POOL });
+  const best = evaluate(results, qq);
+  const truthYes = (qq.truth || 'yes').toLowerCase() !== 'no';
+  // YES-capability: pass = confident + evidenced from expected repo. NO-capability (control):
+  // pass = the bundle does NOT return a confident, evidenced hit (it correctly has nothing to assert).
+  const evidencedConfident = !!(best && best.confident && best.evidenced);
+  const pass = truthYes ? evidencedConfident : !evidencedConfident;
+  rows.push({ i: i + 1, q: qq.q, truth: truthYes ? 'YES' : 'NO(control)', pass, best });
+}
+
+const passN = rows.filter((r) => r.pass).length;
+console.log(`\n=== capability-confidence gate — ${path.basename(QFILE)} (τ=${TAU}, k=${K}, pool=${POOL}) ===\n`);
+for (const r of rows) {
+  const b = r.best;
+  console.log(`${r.pass ? 'PASS' : 'FAIL'}  [${r.truth}]  ${r.q}`);
+  console.log(`      → top@repo=${b ? b.repo : '-'} ce=${b && b.ce != null ? b.ce.toFixed(3) : 'n/a'} evidence=${b ? `${b.evHits}/${b.evNeed}` : '-'} ${b ? (b.path) : ''}`);
+}
+console.log(`\nCAPABILITY-CONFIDENCE: ${passN}/${rows.length} pass (${(100 * passN / rows.length).toFixed(0)}%)  | gate target = 100% on YES + 100% on NO controls`);
+process.exit(passN === rows.length ? 0 : 1);

package/scripts/brain-grade-groundtruth.mjs

@@ -0,0 +1,100 @@
+#!/usr/bin/env node
+// brain-grade-groundtruth.mjs — the ADR-0002 gate of record.
+// For each question: (1) run the REAL consumer path (searchKb) → top-k cited source;
+// (2) MECHANICAL ground-truth: does the #1 cited path actually exist in the repo clone?
+// (3) MULTI-VENDOR panel: independent-vendor LLMs (GPT, Gemini — different families from the
+//     builder) grade STRICT (#1 doc alone) + REAL-USE (top-5) 1-100 against the RETURNED SOURCE,
+//     with the poison rule (incomplete-but-not-wrong < 50). No same-family LLM is the final word.
+// Emits a JSON report + console summary with REAL NUMBERS. Never claims PASS itself — prints the data.
+//
+//   node scripts/brain-grade-groundtruth.mjs --name ruflo --variant big \
+//        --questions kb/questions.ruflo.json --repo ../ruvnet-repos/ruflo   (or set RUVNET_REPO)
+import fs from 'node:fs';
+import path from 'node:path';
+import { fileURLToPath } from 'node:url';
+import { searchKb } from '../kb/forge-ask.mjs';
+import { rerankKb } from '../kb/forge-rerank.mjs';
+const RERANK = process.argv.includes('--rerank');
+
+const ROOT = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '..');
+const arg = (f, d) => { const i = process.argv.indexOf(f); return i >= 0 && process.argv[i + 1] ? process.argv[i + 1] : d; };
+const NAME = arg('--name', 'ruflo');
+const VARIANT = arg('--variant', 'big');
+const QFILE = arg('--questions', path.join(ROOT, 'kb/questions.ruflo.json'));
+const REPO = arg('--repo', process.env.RUVNET_REPO || path.join(ROOT, '..', 'ruvnet-repos', NAME));
+const MODELS = (arg('--models', 'openai/gpt-4o-mini,deepseek/deepseek-chat,meta-llama/llama-3.3-70b-instruct')).split(',').map(s => s.trim()).filter(Boolean);
+
+// OpenRouter key (presence already confirmed) — read from env or Ask-Ruvnet .env; never logged.
+function readKey() {
+  if (process.env.OPENROUTER_API_KEY) return process.env.OPENROUTER_API_KEY;
+  try {
+    const env = fs.readFileSync((process.env.RUVNET_ENV_FILE || '.env'), 'utf8');
+    const m = env.match(/^OPENROUTER_API_KEY=(.+)$/m); return m ? m[1].trim() : null;
+  } catch { return null; }
+}
+const KEY = readKey();
+const clip = (s, n = 2600) => (s && s.length > n ? s.slice(0, n) + `\n…[+${s.length - n} chars]` : (s || ''));
+
+async function gradeWith(model, q, strictEvidence, realUseEvidence) {
+  const sys = 'You are a STRICT senior code reviewer grading whether a knowledge-base answer is COMPLETE and CORRECT, judged ONLY against the retrieved source shown. Scale 1-100: 98=perfect/complete/actionable; an INCOMPLETE-but-not-wrong answer is POISON (<50). If the source does not actually answer the question, score low. Return ONLY compact JSON: {"strict":N,"realUse":N,"reason":"<=20 words"}.';
+  const usr = `QUESTION: ${q}\n\n--- STRICT: the #1 retrieved document ALONE ---\n${strictEvidence}\n\n--- REAL-USE: the top-5 documents a consumer would synthesize ---\n${realUseEvidence}\n\nGrade STRICT (can the #1 doc alone fully answer?) and REAL-USE (can the top-5 together?).`;
+  const res = await fetch('https://openrouter.ai/api/v1/chat/completions', {
+    method: 'POST',
+    headers: { 'Authorization': `Bearer ${KEY}`, 'Content-Type': 'application/json' },
+    body: JSON.stringify({ model, messages: [{ role: 'system', content: sys }, { role: 'user', content: usr }], temperature: 0, max_tokens: 200 }),
+  });
+  if (!res.ok) throw new Error(`${model} HTTP ${res.status}`);
+  const j = await res.json();
+  const txt = j.choices?.[0]?.message?.content || '';
+  const m = txt.match(/\{[\s\S]*\}/);
+  if (!m) throw new Error(`${model} no-json`);
+  const o = JSON.parse(m[0]);
+  return { strict: Number(o.strict), realUse: Number(o.realUse), reason: String(o.reason || '').slice(0, 120) };
+}
+
+const questions = JSON.parse(fs.readFileSync(QFILE, 'utf8'));
+if (!KEY) { console.error('No OPENROUTER_API_KEY — cannot run multi-vendor panel.'); process.exit(2); }
+console.log(`# Ground-truth + multi-vendor grade — ${NAME} [${VARIANT}] — ${questions.length} Q × ${MODELS.length} vendors\n`);
+
+const report = [];
+for (let i = 0; i < questions.length; i++) {
+  const q = questions[i].q;
+  const results = RERANK
+    ? await rerankKb({ dir: path.join(ROOT, 'kb'), name: NAME, query: q, k: 6, variant: VARIANT })
+    : await searchKb({ dir: path.join(ROOT, 'kb'), name: NAME, query: q, k: 6, variant: VARIANT });
+  const top = results[0];
+  const gt = top ? fs.existsSync(path.join(REPO, top.path)) : false;   // mechanical ground-truth: cited path real?
+  const strictEv = top ? `path: ${top.path}\n${clip(top.fullText)}` : '(no results)';
+  const realUseEv = results.slice(0, 5).map((r, j) => `${j + 1}. ${r.path}\n${clip(r.fullText, 1100)}`).join('\n\n');
+  const grades = [];
+  for (const model of MODELS) {
+    try { grades.push({ model, ...await gradeWith(model, q, strictEv, realUseEv) }); }
+    catch (e) { grades.push({ model, error: e.message }); }
+  }
+  const ok = grades.filter(g => Number.isFinite(g.strict));
+  const avgS = ok.length ? ok.reduce((s, g) => s + g.strict, 0) / ok.length : null;
+  const avgR = ok.length ? ok.reduce((s, g) => s + g.realUse, 0) / ok.length : null;
+  report.push({ q, topPath: top?.path || null, groundTruthPathExists: gt, grades, avgStrict: avgS, avgRealUse: avgR });
+  console.log(`Q${i + 1}. ${q}`);
+  console.log(`   #1: ${top?.path || '(none)'}  | path-exists: ${gt ? 'YES' : 'NO'}`);
+  for (const g of grades) console.log(`   ${g.model}: ${g.error ? 'ERR ' + g.error : `strict=${g.strict} realUse=${g.realUse} — ${g.reason}`}`);
+  console.log(`   → avg strict=${avgS?.toFixed(1) ?? '?'} realUse=${avgR?.toFixed(1) ?? '?'}\n`);
+}
+
+// Aggregate
+const valid = report.filter(r => Number.isFinite(r.avgStrict));
+const mean = (k) => valid.reduce((s, r) => s + r[k], 0) / valid.length;
+const minK = (k) => Math.min(...valid.map(r => r[k]));
+const gtFail = report.filter(r => !r.groundTruthPathExists).length;
+const summary = {
+  name: NAME, variant: VARIANT, questions: questions.length, models: MODELS,
+  avgStrict: +mean('avgStrict').toFixed(2), avgRealUse: +mean('avgRealUse').toFixed(2),
+  minStrict: minK('avgStrict'), minRealUse: minK('avgRealUse'),
+  poisonStrict: valid.filter(r => r.avgStrict < 50).length, poisonRealUse: valid.filter(r => r.avgRealUse < 50).length,
+  groundTruthCitationFailures: gtFail,
+};
+fs.writeFileSync(path.join(ROOT, `data/grade-${NAME}-${VARIANT}.json`), JSON.stringify({ summary, report }, null, 2));
+console.log('=== SUMMARY (real numbers) ===');
+console.log(JSON.stringify(summary, null, 2));
+console.log(`\nGate (ADR-0002): PASS needs avg>=98 AND min>=95 on BOTH metrics, 0 poison, 0 citation failures.`);
+console.log(`Report: data/grade-${NAME}-${VARIANT}.json`);

package/scripts/brain-latency-50.mjs

@@ -0,0 +1,227 @@
+#!/usr/bin/env node
+import { spawn } from 'node:child_process';
+import { createInterface } from 'node:readline';
+import fs from 'node:fs';
+import os from 'node:os';
+import path from 'node:path';
+import { fileURLToPath, pathToFileURL } from 'node:url';
+
+const ROOT = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '..');
+
+export const QUESTIONS = [
+  ['How does RuvNet Brain expose the rvbc console in Claude Code and Codex?', /repo=ruvnet-brain/i],
+  ['What command shows the RuvNet Brain 4.0 upgrade highlights?', /repo=ruvnet-brain/i],
+  ['How does RuvNet Brain build and include its own source RVF?', /repo=ruvnet-brain/i],
+  ['How does RuvNet Brain prevent a broad product query from searching every repository?', /repo=ruvnet-brain/i],
+  ['What release gates prevent RuvNet Brain from publishing an unverified artifact?', /repo=ruvnet-brain/i],
+  ['How does RuvNet Brain verify Claude Code plugin installation?', /repo=ruvnet-brain/i],
+  ['How does RuvNet Brain verify Codex skill discovery?', /repo=ruvnet-brain/i],
+  ['How does RuvNet Brain display what changed after a 4.0 upgrade?', /repo=ruvnet-brain/i],
+  ['How does RuvNet Brain detect stale project memory without interrupting the user?', /repo=ruvnet-brain/i],
+  ['How does RuvNet Brain bind release evidence to an exact Git SHA and artifact digest?', /repo=ruvnet-brain/i],
+  ['How does RuvNet Brain test narrow, broad, and concurrent source searches?', /repo=ruvnet-brain/i],
+  ['How does RuvNet Brain distinguish proposed ADR capabilities from shipped code?', /repo=ruvnet-brain/i],
+  ['How does RuvNet Brain prevent a zero-test QE run from passing?', /repo=ruvnet-brain/i],
+  ['How does RuvNet Brain verify the public npm and GitHub release surfaces agree?', /repo=ruvnet-brain/i],
+  ['How does RuvNet Brain update both Claude Code and Codex installations?', /repo=ruvnet-brain/i],
+  ['How does RuvNet Brain handle model-cache initialization across concurrent processes?', /repo=ruvnet-brain/i],
+  ['How does RuvNet Brain measure whether grounding evidence is thin or strong?', /repo=ruvnet-brain/i],
+  ['How does RuvNet Brain stop private stores from entering a public bundle?', /repo=ruvnet-brain/i],
+  ['How does RuvNet Brain sign and verify its downloadable RVF bundle?', /repo=ruvnet-brain/i],
+  ['What does the RuvNet Brain doctor command verify after installation?', /repo=ruvnet-brain/i],
+  ['How does RVF provide persistent HNSW vector search?', /repo=ruvector/i],
+  ['Which RuVector source implements RVF database creation and querying?', /repo=ruvector/i],
+  ['How does RuVector avoid hand-written cosine search for persisted knowledge?', /repo=ruvector/i],
+  ['How are witness chains represented in the RVF architecture?', /repo=ruvector/i],
+  ['How does RuVector support local embedding generation for vector search?', /repo=ruvector/i],
+  ['How does Ruflo initialize and coordinate a hierarchical agent swarm?', /repo=ruflo/i],
+  ['How does Ruflo route work to an architect agent?', /repo=ruflo/i],
+  ['How does Ruflo store and search project memory?', /repo=ruflo/i],
+  ['How does Ruflo track agent execution separately from agent registration?', /repo=ruflo/i],
+  ['How does Ruflo enforce policy before consequential actions?', /repo=ruflo/i],
+  ['How does AgentDB store structured agent memories?', /repo=agentdb/i],
+  ['How does AgentDB perform semantic memory search?', /repo=agentdb/i],
+  ['How does AgentDB handle SQLite WAL-backed persistence?', /repo=agentdb/i],
+  ['How does AgentDB represent graph relationships between memories?', /repo=agentdb/i],
+  ['How does AgentDB prevent unsafe memory input at its boundaries?', /repo=agentdb/i],
+  ['How does Agentic QE generate and execute a test fleet?', /repo=agentic-qe/i],
+  ['How does Agentic QE report test coverage gaps?', /repo=agentic-qe/i],
+  ['How does Agentic QE run security-focused quality checks?', /repo=agentic-qe/i],
+  ['How does Agentic QE prevent vacuous success when no tests execute?', /repo=(?:agentic-qe|ruvnet-brain)/i],
+  ['How does Agentic QE coordinate specialized testing agents?', /repo=agentic-qe/i],
+  ['What phases are defined by the SPARC methodology?', /repo=sparc/i],
+  ['How does RuvNet Brain package the managed CLI boundary users invoke?', /repo=ruvnet-brain/i],
+  ['How does RuvNet Brain validate that every advertised file exists in the npm tarball?', /repo=ruvnet-brain/i],
+  ['How does RuLake act as a read cache over vector collections?', /repo=rulake/i],
+  ['How does RuView process radar sensor data?', /repo=ruview/i],
+  ['Compare RuvNet Brain project memory with AgentDB structured storage.', /repo=(?:ruvnet-brain|agentdb)/i],
+  ['Compare RuvNet Brain source grounding with Ruflo orchestration.', /repo=(?:ruvnet-brain|ruflo)/i],
+  ['How should RuvNet Brain enforce an exact-artifact deployment process with clean lineage, GitHub checks, Agentic QE, independent graders, and post-publication verification?', /repo=ruvnet-brain/i],
+  ['Which shipped files implement the RuvNet Brain release-proof protocol?', /repo=ruvnet-brain/i],
+  ['What evidence proves RuvNet Brain answers its own product questions from RVF?', /repo=ruvnet-brain/i],
+];
+
+function percentile(values, fraction) {
+  if (!values.length) return null;
+  const sorted = [...values].sort((a, b) => a - b);
+  return sorted[Math.ceil(sorted.length * fraction) - 1];
+}
+
+function parseArgs(argv) {
+  const value = (flag, fallback) => {
+    const index = argv.indexOf(flag);
+    return index >= 0 ? argv[index + 1] : fallback;
+  };
+  return {
+    server: path.resolve(value('--server', path.join(ROOT, 'plugin/mcp/server.mjs'))),
+    kb: path.resolve(value('--kb', path.join(ROOT, 'kb'))),
+    timeoutMs: Number(value('--timeout-ms', '30000')),
+    modelCache: path.resolve(value('--model-cache', path.join(ROOT, 'kb', 'models-cache'))),
+    json: argv.includes('--json'),
+    from: Number(value('--from', '1')),
+    to: Number(value('--to', String(QUESTIONS.length))),
+  };
+}
+
+export async function benchmark(options) {
+  if (QUESTIONS.length !== 50) throw new Error(`benchmark must contain exactly 50 questions, found ${QUESTIONS.length}`);
+  const brainHome = fs.mkdtempSync(path.join(os.tmpdir(), 'ruvnet-brain-latency-'));
+  const child = spawn(process.execPath, [options.server], {
+    env: {
+      ...process.env,
+      KB_DIR: options.kb,
+      RUVNET_BRAIN_KB: options.kb,
+      RUVNET_BRAIN_CHILD_MCP: path.join(options.kb, 'forge-mcp-all.mjs'),
+      RUVNET_BRAIN_HOME: brainHome,
+      KB_MODEL_CACHE: options.modelCache,
+      RUVNET_BRAIN_CALL_TIMEOUT_MS: String(options.timeoutMs),
+      RUVNET_BRAIN_METER: '0',
+    },
+    stdio: ['pipe', 'pipe', 'pipe'],
+  });
+  const lines = createInterface({ input: child.stdout });
+  const pending = new Map();
+  let stderr = '';
+  let nextId = 1;
+  child.stderr.on('data', (chunk) => { stderr += chunk.toString(); });
+  lines.on('line', (line) => {
+    let message;
+    try { message = JSON.parse(line); } catch { return; }
+    const waiter = pending.get(message.id);
+    if (waiter) {
+      pending.delete(message.id);
+      waiter(message);
+    }
+  });
+
+  const request = (method, params, timeoutMs) => {
+    const id = nextId++;
+    return new Promise((resolve, reject) => {
+      const timer = setTimeout(() => {
+        pending.delete(id);
+        reject(new Error(`MCP ${method} timed out after ${timeoutMs}ms`));
+      }, timeoutMs);
+      pending.set(id, (message) => {
+        clearTimeout(timer);
+        resolve(message);
+      });
+      child.stdin.write(`${JSON.stringify({ jsonrpc: '2.0', id, method, params })}\n`);
+    });
+  };
+
+  const ask = (query, expected) => {
+    const id = nextId++;
+    const started = performance.now();
+    return new Promise((resolve) => {
+      const timer = setTimeout(() => {
+        pending.delete(id);
+        resolve({ query, elapsedMs: Math.round(performance.now() - started), status: 'TIMEOUT', cited: false });
+      }, options.timeoutMs + 5_000);
+      pending.set(id, (message) => {
+        clearTimeout(timer);
+        const text = message.result?.content?.[0]?.text || '';
+        const cited = expected.test(text);
+        const observedRepos = [...text.matchAll(/repo=([a-z0-9._-]+)/gi)].map((match) => match[1]);
+        resolve({
+          query,
+          elapsedMs: Math.round(performance.now() - started),
+          status: !message.result?.isError && cited ? 'PASS' : 'FAIL',
+          cited,
+          observedRepos: [...new Set(observedRepos)],
+          error: message.error?.message || (message.result?.isError ? text.slice(0, 500) : undefined),
+          preview: text.slice(0, 500),
+        });
+      });
+      child.stdin.write(`${JSON.stringify({
+        jsonrpc: '2.0',
+        id,
+        method: 'tools/call',
+        params: { name: 'search_ruvnet', arguments: { query, k: 5 } },
+      })}\n`);
+    });
+  };
+
+  const results = [];
+  const readinessStarted = performance.now();
+  const initialize = await request('initialize', {
+    protocolVersion: '2024-11-05',
+    capabilities: {},
+    clientInfo: { name: 'brain-latency-50', version: '1.0.0' },
+  }, 120_000);
+  if (initialize.error) throw new Error(`MCP initialize failed: ${initialize.error.message}`);
+  const listed = await request('tools/list', {}, 120_000);
+  if (!listed.result?.tools?.some((tool) => tool.name === 'search_ruvnet')) {
+    throw new Error('MCP readiness failed: search_ruvnet was not advertised');
+  }
+  const readinessMs = Math.round(performance.now() - readinessStarted);
+  try {
+    const selected = QUESTIONS.slice(Math.max(0, options.from - 1), Math.min(QUESTIONS.length, options.to));
+    for (const [query, expected] of selected) {
+      const result = await ask(query, expected);
+      results.push(result);
+      if (!options.json) {
+        process.stdout.write(`${String(results.length).padStart(2, '0')} ${result.status.padEnd(7)} ${String(result.elapsedMs).padStart(6)}ms  ${query}\n`);
+      }
+    }
+  } finally {
+    child.stdin.end();
+    await Promise.race([
+      new Promise((resolve) => child.once('exit', resolve)),
+      new Promise((resolve) => setTimeout(resolve, 5_000)),
+    ]);
+    if (child.exitCode === null) child.kill('SIGKILL');
+    lines.close();
+    fs.rmSync(brainHome, { recursive: true, force: true });
+  }
+
+  const times = results.map((result) => result.elapsedMs);
+  const summary = {
+    server: options.server,
+    kb: options.kb,
+    questions: results.length,
+    passed: results.filter((result) => result.status === 'PASS').length,
+    failed: results.filter((result) => result.status === 'FAIL').length,
+    timedOut: results.filter((result) => result.status === 'TIMEOUT').length,
+    cited: results.filter((result) => result.cited).length,
+    averageMs: Math.round(times.reduce((sum, value) => sum + value, 0) / times.length),
+    medianMs: percentile(times, 0.5),
+    p95Ms: percentile(times, 0.95),
+    longestMs: Math.max(...times),
+    deadlineMs: options.timeoutMs,
+    readinessMs,
+    stderr: stderr.trim(),
+    results,
+  };
+  if (options.json) console.log(JSON.stringify(summary, null, 2));
+  else console.log(`SUMMARY ${summary.passed}/${summary.questions} pass · avg ${summary.averageMs}ms · median ${summary.medianMs}ms · p95 ${summary.p95Ms}ms · max ${summary.longestMs}ms · timeouts ${summary.timedOut}`);
+  return summary;
+}
+
+export async function main(argv = process.argv.slice(2)) {
+  const summary = await benchmark(parseArgs(argv));
+  return summary.passed === summary.questions && summary.timedOut === 0 ? 0 : 1;
+}
+
+if (process.argv[1] && pathToFileURL(path.resolve(process.argv[1])).href === import.meta.url) {
+  process.exitCode = await main();
+}