ruvnet-brain 4.4.0 → 4.5.0

package/scripts/recommendation-e2e.mjs

@@ -0,0 +1,110 @@
+#!/usr/bin/env node
+/**
+ * recommendation-e2e.mjs — the package recommender AS SHIPPED, end to end (ADR-093 rev 2).
+ *
+ * 1. Spawns the real search worker (kb/forge-mcp-all.mjs) in a throwaway brain home with the flag on,
+ *    and runs its real `brain/warmup`, which starts the recommend endpoint.
+ * 2. For every eval prompt, runs the real hook producer (plugin/scripts/advocacy-route.mjs) as a
+ *    cold process — flag OFF then flag ON, interleaved — and records wall time and the exact hint
+ *    the host model would receive.
+ * 3. Writes a judge packet (qid, prompt, hint — no labels) and a separate key, so a model standing in
+ *    for the host can decide what it would say without seeing the answers.
+ *
+ * Nothing touches the real ~/.claude, ~/.codex or ~/.cache/ruvnet-brain: HOME and RUVNET_BRAIN_HOME
+ * are a temp dir. The embedder needs a model cache and @xenova/transformers:
+ *   --models <dir>   (KB_MODEL_CACHE)     --xenova <dir>  (XENOVA_PATH)
+ * Refuses to measure when the 1-minute load average exceeds --max-load (default 60).
+ *
+ *   node scripts/recommendation-e2e.mjs --models <d> --xenova <d> --out <dir> [--set f.json]... [--max-load 60] [--budget ms]
+ *
+ * --budget raises the hook's semantic budget for a QUALITY run (what the lane says when it answers);
+ * the default-budget run is the LATENCY run (how often, and how fast, it answers under its real limit).
+ */
+import { spawn, spawnSync } from 'node:child_process';
+import fs from 'node:fs';
+import os from 'node:os';
+import path from 'node:path';
+import readline from 'node:readline';
+import { fileURLToPath } from 'node:url';
+
+const ROOT = path.dirname(path.dirname(fileURLToPath(import.meta.url)));
+const arg = (n, d) => { const i = process.argv.indexOf(n); return i >= 0 && process.argv[i + 1] ? process.argv[i + 1] : d; };
+const sets = process.argv.flatMap((a, i) => (a === '--set' ? [process.argv[i + 1]] : []));
+const outDir = arg('--out', null);
+const maxLoad = Number(arg('--max-load', '60'));
+if (!outDir) { console.error('--out <dir> is required'); process.exit(2); }
+if (os.loadavg()[0] > maxLoad) { console.error(`load average ${os.loadavg()[0].toFixed(1)} > ${maxLoad}; refusing to measure`); process.exit(3); }
+
+const home = fs.mkdtempSync(path.join(os.tmpdir(), 'reco-e2e-'));
+const kb = path.join(home, 'kb');
+fs.mkdirSync(kb);
+const base = {
+  PATH: process.env.PATH, HOME: home, USERPROFILE: home, RUVNET_BRAIN_HOME: home, RUVNET_HOME_OVERRIDE: home,
+  KB_DIR: kb, KB_MODEL_CACHE: arg('--models', ''), XENOVA_PATH: arg('--xenova', ''),
+};
+const items = (sets.length ? sets : ['evals/recommendation-eval.v1.json'])
+  .flatMap((f) => JSON.parse(fs.readFileSync(path.resolve(ROOT, f), 'utf8')).items.map((i) => ({ ...i, set: path.basename(f) })));
+
+const worker = spawn(process.execPath, [path.join(ROOT, 'kb', 'forge-mcp-all.mjs')],
+  { env: { ...base, RUVNET_PACKAGE_RECOMMENDER: '1' }, stdio: ['pipe', 'pipe', 'ignore'] });
+const rl = readline.createInterface({ input: worker.stdout });
+const waiters = new Map();
+rl.on('line', (l) => { try { const m = JSON.parse(l); waiters.get(m.id)?.(m); } catch { /* not ours */ } });
+const call = (id, method) => new Promise((r) => { waiters.set(id, r); worker.stdin.write(`${JSON.stringify({ jsonrpc: '2.0', id, method, params: {} })}\n`); });
+
+function produce(prompt, flag, n) {
+  const env = {
+    ...base, RUVNET_EMIT_CANDIDATES: '1',
+    RUVNET_ADVOCACY_ROUTE_STATE: path.join(home, `state-${flag}-${n}.json`),
+    RUVNET_ADVOCACY_OUTCOMES: path.join(home, `outcomes-${flag}-${n}.jsonl`),
+    RUVNET_ADVOCACY_ROUTE_ROOTS: path.join(home, 'none'),
+    ...(flag === 'on' ? { RUVNET_PACKAGE_RECOMMENDER: '1' } : {}),
+    ...(arg('--budget', null) ? { RUVNET_PACKAGE_RECOMMENDER_BUDGET_MS: arg('--budget', null) } : {}),
+    // --floor 0 records every hint so a floor can be derived afterwards on the tuning set only.
+    ...(arg('--floor', null) !== null ? { RUVNET_PACKAGE_RECOMMENDER_MIN_SIMILARITY: arg('--floor', null) } : {}),
+  };
+  const t = process.hrtime.bigint();
+  const r = spawnSync(process.execPath, [path.join(ROOT, 'plugin', 'scripts', 'advocacy-route.mjs')],
+    { input: JSON.stringify({ session_id: `e2e-${flag}-${n}`, prompt }), env, encoding: 'utf8', timeout: 10000 });
+  const ms = Number(process.hrtime.bigint() - t) / 1e6;
+  let cand = null;
+  try { cand = r.stdout.trim() ? JSON.parse(r.stdout.trim()) : null; } catch { cand = null; }
+  return { ms, status: r.status, cand };
+}
+
+const q = (a, p) => { const s = [...a].sort((x, y) => x - y); return +s[Math.min(s.length - 1, Math.ceil(p * s.length) - 1)].toFixed(1); };
+try {
+  await call(1, 'initialize');
+  const warm = await call(2, 'brain/warmup');
+  if (!warm.result?.ready) throw new Error(`warmup failed: ${JSON.stringify(warm.error || warm.result)}`);
+  for (let i = 0; i < 50 && !fs.existsSync(path.join(home, 'run')); i++) await new Promise((r) => setTimeout(r, 100));
+  produce('warm the page cache once before measuring', 'on', 'warm');
+  const packet = []; const key = []; const off = []; const on = []; const added = [];
+  items.forEach((it, i) => {
+    const a = produce(it.prompt, 'off', i);
+    const b = produce(it.prompt, 'on', i);
+    off.push(a.ms); on.push(b.ms); added.push(b.ms - a.ms);
+    const qid = `Q${String(i + 1).padStart(3, '0')}`;
+    key.push({ qid, set: it.set, id: it.id, category: it.category, accept: it.accept || [], acceptStores: it.acceptStores || [],
+      lane: b.cand ? (b.cand.candidates ? 'semantic' : String(b.cand.findingId).startsWith('recommend:pkg:') ? 'lexical' : 'catalogue') : null,
+      offered: b.cand ? (b.cand.candidates || [b.cand.package || b.cand.capability]) : [], status: b.status,
+      topSimilarity: Array.isArray(b.cand?.similarities) ? b.cand.similarities[0] : null });
+    if (b.cand) packet.push({ qid, prompt: it.prompt, hint: b.cand.copy });
+  });
+  const timing = {
+    samples: items.length, loadAvg1m: +os.loadavg()[0].toFixed(1), host: `${os.platform()} ${os.arch()} ${os.cpus().length} vCPU node ${process.version}`,
+    flagOffMs: { p50: q(off, 0.5), p90: q(off, 0.9) }, flagOnMs: { p50: q(on, 0.5), p90: q(on, 0.9), max: q(on, 1) },
+    addedMs: { p50: q(added, 0.5), p90: q(added, 0.9), max: q(added, 1) },
+  };
+  fs.mkdirSync(outDir, { recursive: true });
+  fs.writeFileSync(path.join(outDir, 'judge-packet.json'), JSON.stringify(packet.sort(() => 0), null, 1));
+  fs.writeFileSync(path.join(outDir, 'judge-key.json'), JSON.stringify(key, null, 1));
+  fs.writeFileSync(path.join(outDir, 'timing.json'), JSON.stringify(timing, null, 1));
+  console.log(JSON.stringify(timing));
+  console.log(`hints emitted for ${packet.length}/${items.length} prompts; lanes: ${JSON.stringify(key.reduce((m, k) => ({ ...m, [k.lane]: (m[k.lane] || 0) + 1 }), {}))}`);
+} finally {
+  worker.kill('SIGTERM');
+  await new Promise((r) => setTimeout(r, 300));
+  fs.rmSync(home, { recursive: true, force: true });
+}
+process.exit(0);

package/scripts/recommendation-eval.mjs

@@ -0,0 +1,105 @@
+#!/usr/bin/env node
+/**
+ * recommendation-eval.mjs — score the package recommender against a need-prompt eval set (ADR-0093).
+ *
+ *   node scripts/recommendation-eval.mjs [--set evals/recommendation-eval.v1.json]... [--cards <file>] [--json] [--verbose]
+ *
+ * Reports, per split and overall: recall on positives (fired AND an accepted package), precision over
+ * all firings, false-firing rate on negatives (off-topic + no-fit) — each with a Wilson 95% interval,
+ * because 30 prompts is a small sample and a bare percentage would overstate what it shows.
+ * An `accept` id that is not in the card set is reported, never silently counted as a miss.
+ */
+import fs from 'node:fs';
+import path from 'node:path';
+import { fileURLToPath } from 'node:url';
+import { indexCards, rank, SNAPSHOT_FILE } from '../plugin/scripts/package-recommender.mjs';
+
+const ROOT = path.dirname(path.dirname(fileURLToPath(import.meta.url)));
+
+/** Wilson score interval, 95%. Returns [lo, hi] as fractions; [0, 0] for n = 0. */
+export function wilson(k, n, z = 1.96) {
+  if (!n) return [0, 0];
+  const p = k / n;
+  const den = 1 + (z * z) / n;
+  const centre = (p + (z * z) / (2 * n)) / den;
+  const half = (z * Math.sqrt((p * (1 - p)) / n + (z * z) / (4 * n * n))) / den;
+  return [Math.max(0, centre - half), Math.min(1, centre + half)];
+}
+
+const POSITIVE = new Set(['design', 'diagnosis']);
+
+export function scoreItem(item, decision) {
+  const fired = Boolean(decision);
+  const pkg = decision?.card?.id || null;
+  const store = decision?.card?.store || null;
+  if (!POSITIVE.has(item.category)) return { fired, pkg, correct: !fired, firedCorrect: false };
+  const accepted = fired && ((item.accept || []).includes(pkg) || (item.acceptStores || []).includes(store));
+  return { fired, pkg, correct: accepted, firedCorrect: accepted };
+}
+
+export function summarize(rows) {
+  const pos = rows.filter((r) => POSITIVE.has(r.category));
+  const neg = rows.filter((r) => !POSITIVE.has(r.category));
+  const fired = rows.filter((r) => r.fired);
+  const firedCorrect = rows.filter((r) => r.firedCorrect).length;
+  const hits = pos.filter((r) => r.correct).length;
+  const falseFires = neg.filter((r) => r.fired).length;
+  const pct = ([lo, hi]) => [+(lo * 100).toFixed(1), +(hi * 100).toFixed(1)];
+  return {
+    n: rows.length,
+    positives: pos.length,
+    negatives: neg.length,
+    recall: { k: hits, n: pos.length, pct: pos.length ? +((hits / pos.length) * 100).toFixed(1) : null, ci95: pct(wilson(hits, pos.length)) },
+    precision: { k: firedCorrect, n: fired.length, pct: fired.length ? +((firedCorrect / fired.length) * 100).toFixed(1) : null, ci95: pct(wilson(firedCorrect, fired.length)) },
+    falseFiring: { k: falseFires, n: neg.length, pct: neg.length ? +((falseFires / neg.length) * 100).toFixed(1) : null, ci95: pct(wilson(falseFires, neg.length)) },
+    wrongPackage: pos.filter((r) => r.fired && !r.correct).length,
+  };
+}
+
+export function evaluate(items, index) {
+  const ids = new Set(index.entries.map((e) => e.card.id));
+  const missing = [...new Set(items.flatMap((i) => i.accept || []).filter((id) => !ids.has(id)))];
+  const rows = items.map((item) => {
+    const { decision, reason, ranked } = rank(item.prompt, index);
+    return {
+      id: item.id, category: item.category, split: item.split, prompt: item.prompt, accept: item.accept,
+      ...scoreItem(item, decision),
+      reason, top: ranked.slice(0, 3).map((r) => `${r.card.id}:${r.score.toFixed(2)}[${r.matched.join(',')}]`),
+    };
+  });
+  const splits = [...new Set(rows.map((r) => r.split))];
+  return {
+    missingAcceptIds: missing,
+    overall: summarize(rows),
+    bySplit: Object.fromEntries(splits.map((s) => [s, summarize(rows.filter((r) => r.split === s))])),
+    rows,
+  };
+}
+
+const isMain = process.argv[1] && path.resolve(process.argv[1]) === fileURLToPath(import.meta.url);
+if (isMain) {
+  const sets = [];
+  let cards = SNAPSHOT_FILE;
+  for (let i = 2; i < process.argv.length; i++) {
+    if (process.argv[i] === '--set') sets.push(process.argv[++i]);
+    if (process.argv[i] === '--cards') cards = process.argv[++i];
+  }
+  if (!sets.length) sets.push(path.join(ROOT, 'evals', 'recommendation-eval.v1.json'));
+  // --split dev restricts BOTH scoring and output to one split, so tuning never shows held-out rows.
+  const splitArg = process.argv.includes('--split') ? process.argv[process.argv.indexOf('--split') + 1] : null;
+  const items = sets.flatMap((f) => JSON.parse(fs.readFileSync(f, 'utf8')).items)
+    .filter((i) => !splitArg || i.split === splitArg);
+  const index = indexCards(JSON.parse(fs.readFileSync(cards, 'utf8')));
+  const result = evaluate(items, index);
+  if (process.argv.includes('--json')) { process.stdout.write(`${JSON.stringify(result, null, 1)}\n`); process.exit(0); }
+  const line = (name, s) => console.log(`${name.padEnd(9)} n=${s.n}  recall ${s.recall.k}/${s.recall.n} ${s.recall.pct}% [${s.recall.ci95.join('–')}]  precision ${s.precision.k}/${s.precision.n} ${s.precision.pct}% [${s.precision.ci95.join('–')}]  false-firing ${s.falseFiring.k}/${s.falseFiring.n} ${s.falseFiring.pct}% [${s.falseFiring.ci95.join('–')}]  wrong-pkg ${s.wrongPackage}`);
+  if (result.missingAcceptIds.length) console.log(`accept ids not in card set: ${result.missingAcceptIds.join(', ')}`);
+  line('overall', result.overall);
+  for (const [s, v] of Object.entries(result.bySplit)) line(s, v);
+  if (process.argv.includes('--verbose')) {
+    for (const r of result.rows) {
+      const mark = r.correct ? 'ok ' : 'XX ';
+      console.log(`${mark}${r.id} ${r.category.padEnd(9)} -> ${r.fired ? r.pkg : '(silent:' + r.reason + ')'} | ${r.top.join('  ')}`);
+    }
+  }
+}

package/scripts/recommendation-floor.mjs

@@ -0,0 +1,56 @@
+#!/usr/bin/env node
+/**
+ * recommendation-floor.mjs — derive the semantic similarity floor on the TUNING set only, freeze it, and
+ * apply it to every set (ADR-093 rev 3). The rule is fixed in advance: the floor is the 5th percentile
+ * of the nearest card's similarity among the tuning set's correctly-judged recommendations (family-aware).
+ *
+ *   node scripts/recommendation-floor.mjs --dir <e2e out dir with judge-key.json + judge-picks.json>
+ *        [--tune recommendation-eval.v1.json] [--floor <value to apply instead of deriving>] [--json]
+ *
+ * The e2e run must be made with --floor 0 so every hint (and its top similarity) is recorded.
+ */
+import fs from 'node:fs';
+import path from 'node:path';
+import { fileURLToPath } from 'node:url';
+import { judgeScore } from './recommendation-judge-score.mjs';
+import { wilson } from './recommendation-eval.mjs';
+
+const ROOT = path.dirname(path.dirname(fileURLToPath(import.meta.url)));
+const POS = new Set(['design', 'diagnosis']);
+
+export function deriveFloor(key, picks, cards, tuneSet, pct = 0.05) {
+  const productOf = new Map(cards.map((c) => [c.id, c.product || c.id]));
+  const right = (k, p) => k.accept.includes(p) || k.accept.some((a) => productOf.has(p) && productOf.get(a) === productOf.get(p));
+  const sims = key.filter((k) => k.set === tuneSet && k.lane === 'semantic' && POS.has(k.category) && picks[k.qid] && right(k, picks[k.qid]))
+    .map((k) => k.topSimilarity).filter(Number.isFinite).sort((a, b) => a - b);
+  return sims.length ? +sims[Math.floor(sims.length * pct)].toFixed(3) : null;
+}
+
+/** Picks as the shipped hook would produce them under `floor`: below it, no hint, so no pick. */
+export function applyFloor(key, picks, floor) {
+  return Object.fromEntries(key.map((k) => [k.qid, k.lane === 'semantic' && !(k.topSimilarity >= floor) ? null : picks[k.qid] ?? null]));
+}
+
+const isMain = process.argv[1] && path.resolve(process.argv[1]) === fileURLToPath(import.meta.url);
+if (isMain) {
+  const arg = (n, d) => { const i = process.argv.indexOf(n); return i >= 0 && process.argv[i + 1] ? process.argv[i + 1] : d; };
+  const dir = arg('--dir', null);
+  const key = JSON.parse(fs.readFileSync(path.join(dir, 'judge-key.json'), 'utf8'));
+  const picks = JSON.parse(fs.readFileSync(path.join(dir, 'judge-picks.json'), 'utf8')).picks;
+  const cards = JSON.parse(fs.readFileSync(path.join(ROOT, 'plugin', 'scripts', 'package-cards.json'), 'utf8')).cards;
+  const floor = arg('--floor', null) !== null ? Number(arg('--floor')) : deriveFloor(key, picks, cards, arg('--tune', 'recommendation-eval.v1.json'));
+  const gated = applyFloor(key, picks, floor);
+  const strict = judgeScore(key, gated, cards);
+  const family = judgeScore(key, gated, cards, { familyAware: true });
+  const negHinted = (set) => key.filter((k) => set(k) && !POS.has(k.category) && k.lane === 'semantic' && k.topSimilarity >= floor).length;
+  const negs = key.filter((k) => /blind/.test(k.set) && !POS.has(k.category)).length;
+  const out = { floor, strict, family, hintsOnBlindNegatives: { k: negHinted((k) => /blind/.test(k.set)), n: negs, ci95: wilson(negHinted((k) => /blind/.test(k.set)), negs).map((x) => +(x * 100).toFixed(1)) } };
+  if (process.argv.includes('--json')) { process.stdout.write(`${JSON.stringify(out, null, 1)}\n`); process.exit(0); }
+  console.log(`floor ${floor}`);
+  const line = (name, s) => console.log(`${name.padEnd(40)} recall ${s.recall.k}/${s.recall.n} ${s.recall.pct}% [${s.recall.ci95.join('–')}]  precision ${s.precision.k}/${s.precision.n} ${s.precision.pct}% [${s.precision.ci95.join('–')}]  false-firing ${s.falseFiring.k}/${s.falseFiring.n} [${s.falseFiring.ci95.join('–')}]`);
+  for (const [mode, r] of [['strict', strict], ['family', family]]) {
+    line(`${mode} blinds`, r.blinds);
+    for (const [s, v] of Object.entries(r.bySet)) line(`${mode} ${s}`, v);
+  }
+  console.log(`hints injected on blind negatives: ${out.hintsOnBlindNegatives.k}/${out.hintsOnBlindNegatives.n} [${out.hintsOnBlindNegatives.ci95.join('–')}]`);
+}

package/scripts/recommendation-judge-score.mjs

@@ -0,0 +1,74 @@
+#!/usr/bin/env node
+/**
+ * recommendation-judge-score.mjs — score what the host model SAID, given the hints the shipped
+ * pipeline emitted (ADR-093 rev 2). Inputs come from scripts/recommendation-e2e.mjs (judge-key.json)
+ * and a judge that saw only prompts + hints (judge-picks.json: { picks: { qid: id|null } }).
+ *
+ *   node scripts/recommendation-judge-score.mjs --dir <e2e out dir> [--cards plugin/scripts/package-cards.json] [--family] [--json]
+ *
+ * A prompt the pipeline stayed silent on counts as silent. Precision is over what the model said.
+ * Wilson 95% intervals throughout.
+ */
+import fs from 'node:fs';
+import path from 'node:path';
+import { fileURLToPath } from 'node:url';
+import { wilson } from './recommendation-eval.mjs';
+
+const ROOT = path.dirname(path.dirname(fileURLToPath(import.meta.url)));
+const POS = new Set(['design', 'diagnosis']);
+// The closed catalogue speaks in building-block names; these are the packages each one names.
+export const CATALOGUE_PACKAGES = Object.freeze({
+  ruvector: ['ruvector', '@ruvector/rvf', '@ruvector/core', '@ruvector/node'],
+  agentdb: ['agentdb'],
+  ruflo: ['ruflo', 'claude-flow', '@claude-flow/swarm'],
+  aidefence: ['@claude-flow/aidefence', 'aidefence-core'],
+  'agentic-qe': ['agentic-qe'],
+  'agentic-flow': ['agentic-flow'],
+  rulake: ['rulake'],
+});
+
+export function judgeScore(key, picks, cards, { familyAware = false } = {}) {
+  const storeOf = new Map(cards.map((c) => [c.id, c.store]));
+  // FAMILY-AWARE (ADR-093 rev 3): a pick is right when it is the same PRODUCT as an accepted id, per the
+  // corpus-derived product map on the cards (scripts/package-cards.mjs deriveProducts). Off by default so
+  // the strict number is always the one reported first.
+  const productOf = new Map(cards.map((c) => [c.id, c.product || c.id]));
+  const sameProduct = (a, b) => familyAware && productOf.has(a) && productOf.get(a) === productOf.get(b);
+  const sets = [...new Set(key.map((k) => k.set))];
+  const pct = (k, n) => ({ k, n, pct: n ? +((100 * k) / n).toFixed(1) : null, ci95: wilson(k, n).map((x) => +(x * 100).toFixed(1)) });
+  const summarize = (rows) => {
+    let hit = 0; let said = 0; let correct = 0; let ff = 0; let wrong = 0;
+    const pos = rows.filter((r) => POS.has(r.category)).length;
+    for (const r of rows) {
+      const p = picks[r.qid] ?? null;
+      if (!p) continue;
+      said++;
+      const ids = [p, ...(CATALOGUE_PACKAGES[p] || [])];
+      const ok = ids.some((id) => r.accept.includes(id) || r.acceptStores.includes(storeOf.get(id))
+        || r.accept.some((a) => sameProduct(id, a)));
+      if (POS.has(r.category)) { if (ok) { hit++; correct++; } else wrong++; } else ff++;
+    }
+    return { n: rows.length, recall: pct(hit, pos), precision: pct(correct, said), falseFiring: pct(ff, rows.length - pos), wrongPackage: wrong };
+  };
+  return {
+    overall: summarize(key),
+    blinds: summarize(key.filter((k) => /blind/.test(k.set))),
+    bySet: Object.fromEntries(sets.map((s) => [s, summarize(key.filter((k) => k.set === s))])),
+  };
+}
+
+const isMain = process.argv[1] && path.resolve(process.argv[1]) === fileURLToPath(import.meta.url);
+if (isMain) {
+  const arg = (n, d) => { const i = process.argv.indexOf(n); return i >= 0 && process.argv[i + 1] ? process.argv[i + 1] : d; };
+  const dir = arg('--dir', null);
+  if (!dir) { console.error('--dir required'); process.exit(2); }
+  const key = JSON.parse(fs.readFileSync(path.join(dir, 'judge-key.json'), 'utf8'));
+  const picks = JSON.parse(fs.readFileSync(path.join(dir, 'judge-picks.json'), 'utf8')).picks || {};
+  const cards = JSON.parse(fs.readFileSync(arg('--cards', path.join(ROOT, 'plugin', 'scripts', 'package-cards.json')), 'utf8')).cards;
+  const r = judgeScore(key, picks, cards, { familyAware: process.argv.includes('--family') });
+  if (process.argv.includes('--json')) { process.stdout.write(`${JSON.stringify(r, null, 1)}\n`); process.exit(0); }
+  const line = (name, s) => console.log(`${name.padEnd(36)} recall ${s.recall.k}/${s.recall.n} ${s.recall.pct}% [${s.recall.ci95.join('–')}]  precision ${s.precision.k}/${s.precision.n} ${s.precision.pct}% [${s.precision.ci95.join('–')}]  false-firing ${s.falseFiring.k}/${s.falseFiring.n} ${s.falseFiring.pct}% [${s.falseFiring.ci95.join('–')}]  wrong ${s.wrongPackage}`);
+  line('overall', r.overall);
+  line('blinds (1+2)', r.blinds);
+  for (const [s, v] of Object.entries(r.bySet)) line(s, v);
+}

package/scripts/recommendation-latency.mjs

@@ -0,0 +1,95 @@
+#!/usr/bin/env node
+/**
+ * recommendation-latency.mjs — what the package recommender ADDS to every prompt (ADR-0093).
+ *
+ * Measured where the cost is paid: a COLD `node advocacy-route.mjs` process per prompt, exactly as
+ * unprompted-runtime.mjs spawns it on UserPromptSubmit. For each eval prompt the producer runs twice,
+ * flag OFF then flag ON, interleaved so machine drift lands on both arms; the per-prompt difference is
+ * the added latency. Every path the producer could write (state, ledger, HOME) is a fresh temp dir.
+ *
+ *   node scripts/recommendation-latency.mjs [--set <eval.json>]... [--rounds 1] [--json]
+ */
+import { spawnSync } from 'node:child_process';
+import fs from 'node:fs';
+import os from 'node:os';
+import path from 'node:path';
+import { fileURLToPath } from 'node:url';
+
+const ROOT = path.dirname(path.dirname(fileURLToPath(import.meta.url)));
+const ROUTE = path.join(ROOT, 'plugin', 'scripts', 'advocacy-route.mjs');
+const CARDS = path.join(ROOT, 'plugin', 'scripts', 'package-cards.json');
+
+const sets = [];
+let rounds = 1;
+for (let i = 2; i < process.argv.length; i++) {
+  if (process.argv[i] === '--set') sets.push(process.argv[++i]);
+  if (process.argv[i] === '--rounds') rounds = Math.max(1, Number(process.argv[++i]) || 1);
+}
+if (!sets.length) sets.push(path.join(ROOT, 'evals', 'recommendation-eval.v1.json'));
+const prompts = sets.flatMap((f) => JSON.parse(fs.readFileSync(f, 'utf8')).items.map((i) => i.prompt));
+
+const tmp = fs.mkdtempSync(path.join(os.tmpdir(), 'reco-latency-'));
+// Only what the producer needs; nothing inherited that could point it at a real store.
+const baseEnv = {
+  PATH: process.env.PATH,
+  HOME: tmp,
+  USERPROFILE: tmp,
+  RUVNET_HOME_OVERRIDE: tmp,
+  RUVNET_EMIT_CANDIDATES: '1',
+  RUVNET_PACKAGE_CARDS: CARDS,
+  RUVNET_ADVOCACY_ROUTE_ROOTS: path.join(tmp, 'no-modules'),
+};
+
+function run(prompt, flag, n) {
+  const env = {
+    ...baseEnv,
+    RUVNET_ADVOCACY_ROUTE_STATE: path.join(tmp, `state-${flag}-${n}.json`),
+    RUVNET_ADVOCACY_OUTCOMES: path.join(tmp, `outcomes-${flag}-${n}.jsonl`),
+    ...(flag === 'on' ? { RUVNET_PACKAGE_RECOMMENDER: '1' } : {}),
+  };
+  const input = JSON.stringify({ session_id: `lat-${flag}-${n}`, cwd: ROOT, hook_event_name: 'UserPromptSubmit', prompt });
+  const t0 = process.hrtime.bigint();
+  const r = spawnSync(process.execPath, [ROUTE], { input, env, encoding: 'utf8', timeout: 10000 });
+  const ms = Number(process.hrtime.bigint() - t0) / 1e6;
+  return { ms, status: r.status, emitted: Boolean((r.stdout || '').trim()) };
+}
+
+const q = (arr, p) => { const s = [...arr].sort((a, b) => a - b); return s[Math.min(s.length - 1, Math.ceil(p * s.length) - 1)]; };
+const off = []; const on = []; const added = []; let failures = 0;
+let n = 0;
+try {
+  run('warm up the filesystem cache once', 'off', 'warm');
+  for (let r = 0; r < rounds; r++) {
+    for (const prompt of prompts) {
+      n++;
+      const a = run(prompt, 'off', n);
+      const b = run(prompt, 'on', n);
+      if (a.status !== 0 || b.status !== 0) failures++;
+      off.push(a.ms); on.push(b.ms); added.push(b.ms - a.ms);
+    }
+  }
+} finally { fs.rmSync(tmp, { recursive: true, force: true }); }
+
+const fmt = (arr) => ({ p50: +q(arr, 0.5).toFixed(1), p90: +q(arr, 0.9).toFixed(1), max: +Math.max(...arr).toFixed(1) });
+// Percentile bootstrap (2000 resamples, fixed seed) for the ADDED p50/p90, so the report carries an
+// interval rather than one number from one noisy run.
+function bootstrap(arr, p, B = 2000) {
+  let seed = 0x9e3779b9;
+  const rnd = () => { seed ^= seed << 13; seed ^= seed >>> 17; seed ^= seed << 5; return ((seed >>> 0) % 1e9) / 1e9; };
+  const stats = [];
+  for (let b = 0; b < B; b++) stats.push(q(Array.from(arr, () => arr[Math.floor(rnd() * arr.length)]), p));
+  return [+q(stats, 0.025).toFixed(1), +q(stats, 0.975).toFixed(1)];
+}
+const out = {
+  samples: n, failures,
+  host: `${os.platform()} ${os.arch()} ${os.cpus().length} vCPU node ${process.version}`,
+  loadAvg1m: +os.loadavg()[0].toFixed(1),
+  flagOffMs: fmt(off), flagOnMs: fmt(on), addedMs: { ...fmt(added), p50ci95: bootstrap(added, 0.5), p90ci95: bootstrap(added, 0.9) },
+};
+if (process.argv.includes('--json')) process.stdout.write(`${JSON.stringify(out, null, 1)}\n`);
+else {
+  console.log(`[latency] ${out.samples} paired cold runs on ${out.host}, ${failures} non-zero exits`);
+  console.log(`[latency] flag off : p50 ${out.flagOffMs.p50} ms  p90 ${out.flagOffMs.p90} ms  max ${out.flagOffMs.max} ms`);
+  console.log(`[latency] flag on  : p50 ${out.flagOnMs.p50} ms  p90 ${out.flagOnMs.p90} ms  max ${out.flagOnMs.max} ms`);
+  console.log(`[latency] added    : p50 ${out.addedMs.p50} ms  p90 ${out.addedMs.p90} ms  max ${out.addedMs.max} ms`);
+}

package/scripts/recommendation-real-host-score.mjs

@@ -0,0 +1,76 @@
+#!/usr/bin/env node
+/**
+ * recommendation-real-host-score.mjs — score a real-host run (scripts/recommendation-real-host.mjs) and
+ * its agreement with the simulated host (ADR-093 rev 3).
+ *
+ *   node scripts/recommendation-real-host-score.mjs --real <real-host.json> --dir <e2e dir> [--floor 0.532] [--json]
+ *
+ * Correctness is family-aware (same product as an accepted id). Agreement compares, per prompt, what the
+ * real host named with what the simulated host would have named under the same floor: both silent, or
+ * the same product. A prompt the real hook did NOT inject a hint for is reported separately, because
+ * then the two hosts did not see the same input.
+ */
+import fs from 'node:fs';
+import path from 'node:path';
+import { fileURLToPath } from 'node:url';
+import { wilson } from './recommendation-eval.mjs';
+import { applyFloor } from './recommendation-floor.mjs';
+
+const ROOT = path.dirname(path.dirname(fileURLToPath(import.meta.url)));
+const POS = new Set(['design', 'diagnosis']);
+
+export function scoreRealHost(real, key, simPicks, cards) {
+  const productOf = new Map(cards.map((c) => [c.id, c.product || c.id]));
+  const same = (a, b) => a === b || (productOf.has(a) && productOf.get(a) === productOf.get(b));
+  const byQid = new Map(key.map((k) => [k.qid, k]));
+  const rows = real.rows.map((r) => {
+    const k = byQid.get(r.qid);
+    const right = r.said ? (k.accept || []).some((a) => same(r.said, a)) : null;
+    const sim = simPicks[r.qid] ?? null;
+    const agree = (!r.said && !sim) || (r.said && sim && same(r.said, sim));
+    return { ...r, accept: k.accept, sim, right, agree };
+  });
+  const pos = rows.filter((r) => POS.has(r.category));
+  const neg = rows.filter((r) => !POS.has(r.category));
+  const said = rows.filter((r) => r.said);
+  const pct = (a, n) => ({ k: a, n, pct: n ? +((100 * a) / n).toFixed(1) : null, ci95: wilson(a, n).map((x) => +(x * 100).toFixed(1)) });
+  const sameInput = rows.filter((r) => r.injected === Boolean(byQid.get(r.qid).lane));
+  // Hint delivery against the 1-minute load each prompt ran at: how the lane behaves on a busy laptop.
+  const buckets = [[0, 60], [60, 120], [120, 240], [240, Infinity]];
+  const deliveryByLoad = buckets.map(([lo, hi]) => {
+    const inB = rows.filter((r) => r.expectHint && Number.isFinite(r.load1m) && r.load1m >= lo && r.load1m < hi);
+    return { load: hi === Infinity ? `>=${lo}` : `${lo}-${hi}`, ...pct(inB.filter((r) => r.injected).length, inB.length) };
+  });
+  return {
+    n: rows.length,
+    deliveryByLoad,
+    recall: pct(pos.filter((r) => r.right).length, pos.length),
+    precision: pct(said.filter((r) => POS.has(r.category) && r.right).length, said.length),
+    falseFiring: pct(neg.filter((r) => r.said).length, neg.length),
+    hintDeliveredWhereExpected: pct(rows.filter((r) => r.expectHint && r.injected).length, rows.filter((r) => r.expectHint).length),
+    hintsNotExpected: rows.filter((r) => !r.expectHint && r.injected).length,   // catalogue/lexical lanes, which the floor does not govern
+    agreement: pct(rows.filter((r) => r.agree).length, rows.length),
+    agreementSameInput: pct(sameInput.filter((r) => r.agree).length, sameInput.length),
+    rows,
+  };
+}
+
+const isMain = process.argv[1] && path.resolve(process.argv[1]) === fileURLToPath(import.meta.url);
+if (isMain) {
+  const arg = (n, d) => { const i = process.argv.indexOf(n); return i >= 0 && process.argv[i + 1] ? process.argv[i + 1] : d; };
+  const real = JSON.parse(fs.readFileSync(arg('--real'), 'utf8'));
+  const dir = arg('--dir');
+  const key = JSON.parse(fs.readFileSync(path.join(dir, 'judge-key.json'), 'utf8'));
+  const floor = Number(arg('--floor', '0.532'));
+  const sim = applyFloor(key, JSON.parse(fs.readFileSync(path.join(dir, 'judge-picks.json'), 'utf8')).picks, floor);
+  const cards = JSON.parse(fs.readFileSync(path.join(ROOT, 'plugin', 'scripts', 'package-cards.json'), 'utf8')).cards;
+  const keyed = new Map(key.map((k) => [k.qid, k]));
+  real.rows.forEach((r) => { const k = keyed.get(r.qid); r.expectHint = Boolean(k.lane) && (k.lane !== 'semantic' || k.topSimilarity >= floor); });
+  const out = scoreRealHost(real, key, sim, cards);
+  if (process.argv.includes('--json')) { process.stdout.write(`${JSON.stringify(out, null, 1)}\n`); process.exit(0); }
+  const f = (n, s) => console.log(`${n.padEnd(30)} ${s.k}/${s.n} = ${s.pct}% [${s.ci95.join('–')}]`);
+  f('recall (family-aware)', out.recall); f('precision (family-aware)', out.precision); f('false firing', out.falseFiring);
+  f('hints delivered / expected', out.hintDeliveredWhereExpected);
+  for (const b of out.deliveryByLoad) f(`  delivered at load ${b.load}`, b); f('agreement with simulated host', out.agreement); f('  …where both saw a hint', out.agreementSameInput);
+  for (const r of out.rows) console.log(`${r.qid} ${r.category.padEnd(9)} hint=${r.injected ? 'y' : 'n'}${r.expectHint ? '' : '(not expected)'} real=${r.said || '-'} sim=${r.sim || '-'} ${r.said ? (r.right ? 'RIGHT' : 'WRONG') : ''} ${r.agree ? '' : 'DISAGREE'}`);
+}