ruvnet-brain 4.4.0 → 4.5.0

package/scripts/oracle/abstain-trace.mjs

@@ -0,0 +1,139 @@
+#!/usr/bin/env node
+/**
+ * scripts/oracle/abstain-trace.mjs — WHY does the reranker abstain on a novice need whose gold
+ * repository WAS searched?
+ *
+ * For each need it measures, in the gold store, with the production models:
+ *   poolRank       rank of the gold file among the store's dense pool (searchKb, depth --pool)
+ *   ceProduction   cross-encoder logit on the text production reranks (the assembled document, which
+ *                  the reranker reads only through its first 3000 chars / 512 tokens)
+ *   ceBestChunk    best logit over the gold file's own sidecar chunks, each read on its own
+ *   ceSpan         logit on the verbatim gold span the question was written from (an upper bound:
+ *                  the answer text itself, nothing else)
+ *   ceTopProd      production's top logit for this need (from the measured needs run)
+ * and classifies the abstention:
+ *   not-in-pool        dense retrieval never surfaced the gold file
+ *   window             gold scores >= 0 on some chunk but < 0 on the text production reads
+ *   calibration        even the verbatim span scores < 0: the model, not the window, says "irrelevant"
+ *   chunking           span >= 0 but every stored chunk < 0 (the answer is split across chunks)
+ *   outranked          production text >= 0, yet the top citation is another file
+ *
+ *   node scripts/oracle/abstain-trace.mjs --kb <kbDir> --set <need-set.json> --rows <needs.json>
+ *     [--runtime <dir>] [--keyword 8] [--sample 20] [--pool 64] [--out <file>]
+ *
+ * --rows is a measure-need-set output (it supplies reposSearched and the production top logit);
+ * only needs whose gold repository was searched are traced. Nothing written contains a local path.
+ */
+import fs from 'node:fs';
+import path from 'node:path';
+import readline from 'node:readline';
+import { fileURLToPath, pathToFileURL } from 'node:url';
+
+const args = process.argv.slice(2);
+const arg = (n, d) => { const i = args.indexOf(n); return i >= 0 ? args[i + 1] : d; };
+
+export function classify({ poolRank, ceProduction, ceBestChunk, ceSpan }) {
+  if (poolRank == null) return 'not-in-pool';
+  if (ceProduction >= 0) return 'outranked';
+  if (ceBestChunk >= 0) return 'window';
+  if (ceSpan < 0) return 'calibration';
+  return 'chunking';
+}
+
+/** Deterministic sample: every step-th need of the eligible list, spread over the whole list. */
+export function sampleEvenly(list, n) {
+  if (list.length <= n) return list;
+  const step = list.length / n;
+  return Array.from({ length: n }, (_, i) => list[Math.floor(i * step)]);
+}
+
+async function chunksOf(kb, store, file) {
+  const out = [];
+  for (const name of [`${store}.passages.jsonl`, `${store}.big.passages.jsonl`]) {
+    const p = path.join(kb, name);
+    if (!fs.existsSync(p)) continue;
+    const rl = readline.createInterface({ input: fs.createReadStream(p), crlfDelay: Infinity });
+    for await (const line of rl) {
+      if (!line.includes(file)) continue;
+      try { const r = JSON.parse(line); if (r.path === file) out.push(String(r.text || '')); } catch { /* skip */ }
+    }
+    if (out.length) break;
+  }
+  return out;
+}
+
+async function main() {
+  const kb = arg('--kb');
+  const set = JSON.parse(fs.readFileSync(arg('--set'), 'utf8'));
+  const rows = JSON.parse(fs.readFileSync(arg('--rows'), 'utf8')).rows;
+  const poolDepth = Number(arg('--pool', 64));
+  const want = Number(arg('--sample', 20));
+  const byId = new Map(set.questions.map((q) => [q.id, q]));
+  // The runtime is the KB directory's own copy of the reader (as in production), unless --runtime
+  // names another directory holding forge-ask.mjs / forge-rerank.mjs and their node_modules.
+  const runtime = path.resolve(arg('--runtime', kb));
+  const { searchKb } = await import(pathToFileURL(path.join(runtime, 'forge-ask.mjs')).href);
+  const { rerankPairs } = await import(pathToFileURL(path.join(runtime, 'forge-rerank.mjs')).href);
+  // --keyword N also counts the keyword lane (keyword-lane.mjs in the runtime) as part of the pool.
+  const keywordTopN = Number(arg('--keyword', 0));
+  const keywordCandidates = keywordTopN > 0
+    ? (await import(pathToFileURL(path.join(runtime, 'keyword-lane.mjs')).href)).keywordCandidates : null;
+  const ce = async (query, texts) => {
+    if (!texts.length) return [];
+    const scored = await rerankPairs(query, texts.map((t, i) => ({ fullText: t, i })));
+    const out = new Array(texts.length);
+    for (const s of scored) out[s.i] = s.ceScore;
+    return out;
+  };
+  const eligible = rows.filter((r) => (r.reposSearched || []).some((s) => s.toLowerCase() === r.repo.toLowerCase()));
+  const traced = [];
+  for (const r of eligible) {
+    const q = byId.get(r.id);
+    const store = r.repo.toLowerCase();
+    const hits = await searchKb({ dir: kb, name: store, query: q.need, k: poolDepth, n: poolDepth });
+    const idx = hits.findIndex((h) => h.path === q.path);
+    let lane = idx < 0 ? null : 'dense';
+    let gold = idx < 0 ? null : hits[idx];
+    if (!gold && keywordCandidates) {
+      const kw = keywordCandidates(kb, store, q.need, { topN: keywordTopN, exclude: new Set(hits.map((h) => h.path)) });
+      const k = kw.findIndex((c) => c.path === q.path);
+      if (k >= 0) { gold = kw[k]; lane = 'keyword'; }
+    }
+    traced.push({ r, q, store, gold, lane, poolRank: idx < 0 ? (gold ? poolDepth + 1 : null) : idx + 1 });
+    process.stderr.write(`\r[abstain-trace] pool ${traced.length}/${eligible.length}`);
+  }
+  const inPool = traced.filter((t) => t.poolRank != null);
+  const sample = sampleEvenly(inPool, want);
+  const out = [];
+  for (const t of sample) {
+    const { gold } = t;
+    const chunks = await chunksOf(kb, t.store, t.q.path);
+    const [ceProduction] = await ce(t.q.need, [gold.fullText || gold.text || '']);
+    const chunkScores = await ce(t.q.need, chunks);
+    const [ceSpan] = await ce(t.q.need, [t.q.span || '']);
+    const row = {
+      id: t.q.id, repo: t.r.repo, need: t.q.need, poolRank: t.poolRank, lane: t.lane,
+      ceProduction: +ceProduction.toFixed(3),
+      ceBestChunk: chunkScores.length ? +Math.max(...chunkScores).toFixed(3) : null,
+      chunks: chunks.length, goldDocChars: (gold.fullText || '').length,
+      ceSpan: +ceSpan.toFixed(3), ceTopProd: t.r.topCe, prodTop: t.r.topPath, prodAbstained: t.r.abstained,
+    };
+    row.cause = classify(row);
+    out.push(row);
+    process.stderr.write(`\r[abstain-trace] ce ${out.length}/${sample.length}   `);
+  }
+  const count = (k) => out.filter((x) => x.cause === k).length;
+  const report = {
+    kind: 'ruvnet-brain-abstain-trace', poolDepth,
+    eligible: eligible.length, goldInPool: inPool.length, goldViaKeyword: traced.filter((t) => t.lane === 'keyword').length, notInPool: traced.length - inPool.length, sampled: out.length,
+    causes: Object.fromEntries(['not-in-pool', 'window', 'calibration', 'chunking', 'outranked'].map((k) => [k, count(k)])),
+    rows: out,
+    poolRanks: traced.map((t) => ({ id: t.q.id, poolRank: t.poolRank })),
+  };
+  const file = arg('--out');
+  if (file) fs.writeFileSync(file, `${JSON.stringify(report, null, 1)}\n`);
+  console.log(JSON.stringify({ ...report, rows: undefined, poolRanks: undefined }, null, 1));
+  process.exit(0);
+}
+
+if (process.argv[1] && path.resolve(process.argv[1]) === fileURLToPath(import.meta.url)) await main();

package/scripts/oracle/doc2query-generate.mjs

@@ -0,0 +1,162 @@
+#!/usr/bin/env node
+/**
+ * scripts/oracle/doc2query-generate.mjs — ADR-099 arm A, generation: newcomer-style questions per
+ * documentation file ("doc2query"), produced ONCE where the corpus is built, never on a customer
+ * machine.
+ *
+ * The generator sees only the file's title, path and opening text, never the need set. Each question
+ * must describe a NEED in plain words. Deterministic checks drop any question that:
+ *   - names the product, the repository or a code identifier, or
+ *   - shares more than 3 consecutive words with the excerpt
+ * (the need-set producer's leak rules). Output is append-only JSONL
+ * {store, path, questions[], rejected}, so a run can be resumed and interrupted runs keep their work.
+ *
+ *   node scripts/oracle/doc2query-generate.mjs --kb <kbDir> --stores ruflo,ruvector,ruview --out <file.jsonl>
+ *     [--kind md] [--per-file 3] [--batch 25] [--model haiku] [--limit N] [--conc 2]
+ *
+ * Children run with scripts/subscription-hosts.mjs#subscriptionOnlyEnv (no API billing keys) and the
+ * same minimised-context claude flags as scripts/oracle/producer-hosts.mjs.
+ */
+import fs from 'node:fs';
+import path from 'node:path';
+import os from 'node:os';
+import { fileURLToPath } from 'node:url';
+import { claudeArgs, spawnHost, isQuotaRefusal } from './producer-hosts.mjs';
+import { subscriptionOnlyEnv } from '../subscription-hosts.mjs';
+
+export const EXCERPT_CHARS = 1500;
+export const D2Q_SCHEMA = Object.freeze({
+  type: 'object', additionalProperties: false,
+  properties: { items: { type: 'array', items: { type: 'object', additionalProperties: false,
+    properties: { docId: { type: 'string' }, questions: { type: 'array', items: { type: 'string' } } },
+    required: ['docId', 'questions'] } } },
+  required: ['items'],
+});
+export const D2Q_SYSTEM = 'You write search questions for documentation. You see only the documents in the message, '
+  + 'you have no tools, and you use no outside knowledge. Return only the structured output.';
+
+export function d2qPrompt(docs, perFile) {
+  return [
+    `For each DOC below write ${perFile} different questions that a newcomer might type into a search box, `
+      + 'where this document is what would help them.',
+    'Rules for every question:',
+    '- 12-35 words, plain everyday language, describing the person\'s NEED or problem, not the document.',
+    '- The person has never heard of this project: no product, project, repository, package, library or tool names, no code, no identifiers, no file names.',
+    '- Do not copy more than 3 consecutive words from the document.',
+    '- The three questions should cover different things the document helps with.',
+    'Return one item per DOC with its docId copied exactly.',
+    '',
+    ...docs.map((d) => `===DOC docId=${d.docId}\ntitle: ${d.title}\n${d.excerpt}\n===END`),
+  ].join('\n');
+}
+
+const PRODUCT_NAME = /^(?:ruv\w*|ruflo|rufl\w*|ruview|claudeflow|agentdb|agentic|sona|rvf|ruvllm|cognitum|densepose|metaharness|reasoningbank)$/;
+const words = (s) => String(s).toLowerCase().match(/[a-z0-9]+/g) || [];
+/** Longest run of consecutive words a question shares with the excerpt. */
+export function sharedRun(question, excerpt) {
+  const q = words(question);
+  const grams = new Set();
+  const e = words(excerpt);
+  let best = 0;
+  for (let n = 1; n <= q.length; n++) {
+    grams.clear();
+    for (let i = 0; i + n <= e.length; i++) grams.add(e.slice(i, i + n).join(' '));
+    let found = false;
+    for (let i = 0; i + n <= q.length; i++) if (grams.has(q.slice(i, i + n).join(' '))) { found = true; break; }
+    if (!found) break;
+    best = n;
+  }
+  return best;
+}
+
+/** Keep a question only if it obeys the leak rules; return the reason when it does not. */
+export function leakReason(question, { excerpt, store, path: p }) {
+  const q = String(question || '').trim();
+  const n = words(q).length;
+  if (n < 8 || n > 45) return 'length';
+  if (/[`{}<>=]|::|\w\(|\b[a-z]+[A-Z][A-Za-z]+\b|\b\w+\.(?:md|js|ts|rs|py|json|toml)\b|@[a-z0-9-]+\//.test(q)) return 'identifier';
+  // Product names: the store itself and the rUv family's own names (a newcomer has heard none of them).
+  if (words(q).some((w) => w === String(store).toLowerCase() || PRODUCT_NAME.test(w))) return 'names-source';
+  void p;
+  if (sharedRun(q, excerpt) > 3) return 'copies-source';
+  return null;
+}
+
+export function excerptOf(chunks) {
+  return chunks.join('\n\n').slice(0, EXCERPT_CHARS);
+}
+
+function readStoreDocs(kb, store, kind) {
+  const big = path.join(kb, `${store}.big.passages.jsonl`);
+  const file = fs.existsSync(big) ? big : path.join(kb, `${store}.passages.jsonl`);
+  const byPath = new Map();
+  for (const line of fs.readFileSync(file, 'utf8').split('\n')) {
+    if (!line) continue;
+    let r;
+    try { r = JSON.parse(line); } catch { continue; }
+    if (kind === 'md' && !/\.md$/i.test(r.path)) continue;
+    if (!byPath.has(r.path)) byPath.set(r.path, { title: r.title || path.basename(r.path), chunks: [] });
+    const d = byPath.get(r.path);
+    if (d.chunks.join('').length < EXCERPT_CHARS) d.chunks.push(String(r.text || ''));
+  }
+  return [...byPath.entries()].sort(([a], [b]) => a.localeCompare(b))
+    .map(([p, d]) => ({ store, path: p, title: d.title, excerpt: excerptOf(d.chunks) }));
+}
+
+async function main() {
+  const args = process.argv.slice(2);
+  const arg = (n, d) => { const i = args.indexOf(n); return i >= 0 ? args[i + 1] : d; };
+  const kb = arg('--kb');
+  const out = arg('--out');
+  const perFile = Number(arg('--per-file', 3));
+  const batchSize = Number(arg('--batch', 25));
+  const conc = Number(arg('--conc', 2));
+  const model = arg('--model', 'haiku');
+  const done = new Set();
+  if (fs.existsSync(out)) {
+    for (const l of fs.readFileSync(out, 'utf8').split('\n')) { try { const r = JSON.parse(l); done.add(`${r.store}|${r.path}`); } catch { /* partial */ } }
+  }
+  let docs = String(arg('--stores')).split(',').flatMap((s) => readStoreDocs(kb, s.trim(), arg('--kind', 'md')))
+    .filter((d) => !done.has(`${d.store}|${d.path}`));
+  if (arg('--limit')) docs = docs.slice(0, Number(arg('--limit')));
+  const batches = [];
+  for (let i = 0; i < docs.length; i += batchSize) batches.push(docs.slice(i, i + batchSize));
+  const env = { ...subscriptionOnlyEnv(), CLAUDE_HOOK: '/usr/bin/true' };
+  const cwd = fs.mkdtempSync(path.join(os.tmpdir(), 'd2q-'));
+  let next = 0;
+  let stop = false;
+  let written = 0;
+  const worker = async () => {
+    while (!stop && next < batches.length) {
+      const b = batches[next++];
+      const withIds = b.map((d, i) => ({ ...d, docId: `d${i}` }));
+      const res = await spawnHost('claude', claudeArgs({ model, effort: 'low', schema: D2Q_SCHEMA, systemPrompt: D2Q_SYSTEM }),
+        { cwd, env, timeoutMs: 600_000 }, d2qPrompt(withIds, perFile));
+      let items = null;
+      try {
+        const env2 = JSON.parse(res.stdout);
+        items = (env2.structured_output || JSON.parse(env2.result)).items;
+      } catch { items = null; }
+      if (!items) {
+        if (isQuotaRefusal(res.stdout + res.stderr)) { stop = true; process.stderr.write('\n[d2q] quota refusal: stopping\n'); }
+        else process.stderr.write(`\n[d2q] batch failed (status ${res.status}${res.timedOut ? ', timeout' : ''})\n`);
+        continue;
+      }
+      const byId = new Map(items.map((it) => [it.docId, it.questions || []]));
+      const lines = withIds.map((d) => {
+        const qs = byId.get(d.docId) || [];
+        const kept = [];
+        const rejected = [];
+        for (const q of qs) { const why = leakReason(q, d); if (why) rejected.push({ q, why }); else kept.push(q); }
+        return JSON.stringify({ store: d.store, path: d.path, questions: kept, rejected });
+      });
+      fs.appendFileSync(out, `${lines.join('\n')}\n`);
+      written += lines.length;
+      process.stderr.write(`\r[d2q] ${written}/${docs.length} files`);
+    }
+  };
+  await Promise.all(Array.from({ length: conc }, worker));
+  console.log(JSON.stringify({ files: docs.length, written, stopped: stop }));
+}
+
+if (process.argv[1] && path.resolve(process.argv[1]) === fileURLToPath(import.meta.url)) await main();

package/scripts/oracle/doc2query-reach.mjs

@@ -0,0 +1,110 @@
+#!/usr/bin/env node
+/**
+ * scripts/oracle/doc2query-reach.mjs — ADR-099 arm A, measurement: do generated entry points bring
+ * the gold file into the pool?
+ *
+ * Builds one RVF "entry" index per store from doc2query-generate output (every question embedded with
+ * the production query embedder, so a newcomer's question is matched question-to-question), then, for
+ * every need, asks the gold store's entry index for its top --k questions and collapses them to files.
+ * Pool reach is reported against the existing pool (dense 64 + keyword lane, from an abstain-trace
+ * run's poolRanks), on the TRAIN and HELD-OUT splits separately, with Wilson intervals:
+ *   baselineReach  gold already pooled by dense + keyword
+ *   entryReach     gold among the entry lane's files
+ *   unionReach     either
+ * Restricted to needs whose gold repository the router searched (the trace's eligible set); over all
+ * needs only entryReach is reported (store-level, independent of routing).
+ *
+ *   node scripts/oracle/doc2query-reach.mjs --runtime <kbDir with forge-ask.mjs + node_modules>
+ *     --d2q <generated.jsonl> --set <need-set.json> --split <split.json> --trace <abstain-trace.json>
+ *     [--k 24] [--files 8] [--index-dir <dir>] [--out <file>]
+ */
+import fs from 'node:fs';
+import path from 'node:path';
+import { fileURLToPath, pathToFileURL } from 'node:url';
+import { wilson } from '../eval-brain.mjs';
+
+/** Collapse ranked entry hits (each pointing at a file) to the first `files` distinct files. */
+export function filesFromEntries(hits, pathOfId, files) {
+  const out = [];
+  for (const h of hits) {
+    const p = pathOfId(h.id);
+    if (p && !out.includes(p)) out.push(p);
+    if (out.length >= files) break;
+  }
+  return out;
+}
+
+export function reachSummary(rows) {
+  const r = (pred) => { const k = rows.filter(pred).length; const w = wilson(k, rows.length); return { k, n: rows.length, lo: +w.lo.toFixed(4), hi: +w.hi.toFixed(4) }; };
+  return { baselineReach: r((x) => x.baseline), entryReach: r((x) => x.entry), unionReach: r((x) => x.baseline || x.entry),
+    gained: rows.filter((x) => x.entry && !x.baseline).length };
+}
+
+async function main() {
+  const args = process.argv.slice(2);
+  const arg = (n, d) => { const i = args.indexOf(n); return i >= 0 ? args[i + 1] : d; };
+  const runtime = path.resolve(arg('--runtime'));
+  const { __embedInternals } = await import(pathToFileURL(path.join(runtime, 'forge-ask.mjs')).href);
+  const { loadRvf } = await import(pathToFileURL(path.join(runtime, 'resolve-deps.mjs')).href);
+  const { RvfDatabase } = loadRvf().mod;
+  const embed = __embedInternals.embed;
+  const k = Number(arg('--k', 24));
+  const nFiles = Number(arg('--files', 8));
+  const indexDir = arg('--index-dir', fs.mkdtempSync(path.join(runtime, '..', 'd2q-index-')));
+  fs.mkdirSync(indexDir, { recursive: true });
+  const gen = fs.readFileSync(arg('--d2q'), 'utf8').split('\n').filter(Boolean).map((l) => JSON.parse(l));
+  const stores = [...new Set(gen.map((g) => g.store))];
+  const indexes = new Map();
+  // The entry index uses the store's own query embedder (big variant when present: bge + query prefix),
+  // exactly what searchKb computes for the question, so production needs one query embedding.
+  const cfgFor = (store) => {
+    for (const v of [`${store}.big.rvf.embed.json`, `${store}.rvf.embed.json`]) {
+      try { return JSON.parse(fs.readFileSync(path.join(runtime, v), 'utf8')); } catch { /* next */ }
+    }
+    return undefined;
+  };
+  for (const store of stores) {
+    const cfg = cfgFor(store);
+    const dims = cfg?.dimensions || 384;
+    const rows = gen.filter((g) => g.store === store).flatMap((g) => g.questions.map((q) => ({ q, path: g.path })));
+    const file = path.join(indexDir, `${store}.entry.rvf`);
+    fs.rmSync(file, { force: true });
+    const db = await RvfDatabase.create(file, { dimensions: dims, metric: 'cosine' });
+    const batch = [];
+    for (let i = 0; i < rows.length; i++) {
+      batch.push({ id: i + 1, vector: Array.from(await embed(rows[i].q, cfg)) });
+      if (batch.length === 64 || i === rows.length - 1) { await db.ingestBatch(batch.splice(0)); }
+      if (i % 500 === 0) process.stderr.write(`\r[d2q-reach] ${store} ${i}/${rows.length}`);
+    }
+    indexes.set(store, { db, cfg, paths: rows.map((r) => r.path), questions: rows.length, model: cfg?.model || 'default' });
+  }
+  const set = JSON.parse(fs.readFileSync(arg('--set'), 'utf8')).questions;
+  const split = JSON.parse(fs.readFileSync(arg('--split'), 'utf8'));
+  const trace = JSON.parse(fs.readFileSync(arg('--trace'), 'utf8'));
+  const pooled = new Map(trace.poolRanks.map((p) => [p.id, p.poolRank != null]));
+  const rows = [];
+  for (const q of set) {
+    const store = q.repo.toLowerCase();
+    const idx = indexes.get(store);
+    if (!idx) continue;
+    const hits = await idx.db.query(Array.from(await embed(q.need, idx.cfg)), k);
+    const files = filesFromEntries(hits, (id) => idx.paths[Number(id) - 1], nFiles);
+    rows.push({ id: q.id, split: split.train.includes(q.id) ? 'train' : 'heldout', routed: pooled.has(q.id),
+      baseline: pooled.get(q.id) === true, entry: files.includes(q.path), entryRank: files.indexOf(q.path) + 1 || null });
+  }
+  const report = { kind: 'ruvnet-brain-doc2query-reach', k, files: nFiles,
+    indexes: Object.fromEntries([...indexes].map(([s, v]) => [s, { questions: v.questions, model: v.model }])),
+    generatedFiles: gen.length, keptQuestions: gen.reduce((s, g) => s + g.questions.length, 0),
+    rejectedQuestions: gen.reduce((s, g) => s + (g.rejected?.length || 0), 0), results: {} };
+  for (const sp of ['train', 'heldout']) {
+    report.results[sp] = { routedNeeds: reachSummary(rows.filter((r) => r.split === sp && r.routed)),
+      // the trace covers only routed needs, so over all needs only the entry lane itself is known
+      allNeedsEntryReach: reachSummary(rows.filter((r) => r.split === sp)).entryReach };
+  }
+  report.rows = rows;
+  if (arg('--out')) fs.writeFileSync(arg('--out'), `${JSON.stringify(report, null, 1)}\n`);
+  console.log(JSON.stringify({ ...report, rows: undefined }, null, 1));
+  process.exit(0);
+}
+
+if (process.argv[1] && path.resolve(process.argv[1]) === fileURLToPath(import.meta.url)) await main();

package/scripts/oracle/judge-train.mjs

@@ -0,0 +1,158 @@
+#!/usr/bin/env node
+/**
+ * scripts/oracle/judge-train.mjs — fit the learned judge (kb/judge-rank.mjs, ADR-099 arm C) on the
+ * need-set TRAIN split and report it on the HELD-OUT split, offline.
+ *
+ * Input is the scored pool the search already recorded (KB_CE_TRACE: one line per question, every
+ * pooled candidate with its cross-encoder logit, dense distance, lane, title, path and length), so
+ * training and evaluation replay exactly what the cross-encoder saw; no model is run here.
+ *
+ * Model: logistic regression over JUDGE_FEATURES, standardised on train, L2 1.0, class-balanced,
+ * full-batch gradient descent (deterministic). Positive = the candidate is the gold file or a
+ * pre-registered alternative. Operating point: the smallest threshold on the judge's top score at
+ * which TRAIN precision of answered questions reaches --precision (default 0.8); it is folded into
+ * the bias so the product's existing "score < 0 abstains" rule applies unchanged.
+ *
+ *   node scripts/oracle/judge-train.mjs --set <need-set.json> --split <split.json> --trace <cetrace.jsonl>
+ *     [--precision 0.8] [--weights <out.json>] [--report <out.json>]
+ *     [--recall-trace <recall.cetrace.jsonl> --recall-fixture data/retrieval-query-evidence.json]
+ */
+import fs from 'node:fs';
+import path from 'node:path';
+import { fileURLToPath, pathToFileURL } from 'node:url';
+import { wilson } from '../eval-brain.mjs';
+
+const ROOT = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '../..');
+const { JUDGE_FEATURES, judgeFeatures, judgeLogit } = await import(pathToFileURL(path.join(ROOT, 'kb/judge-rank.mjs')).href);
+
+const norm = (p) => String(p || '').replace(/^\.\//, '');
+export function isTarget(q, cand) {
+  const targets = [{ repo: q.repo, path: q.path }, ...(q.alternatives || [])];
+  return targets.some((t) => String(t.repo).toLowerCase() === String(cand.repo).toLowerCase() && norm(t.path) === norm(cand.path));
+}
+
+/** Map need id -> its last recorded pool, from KB_CE_TRACE lines (unparsable lines are skipped). */
+export function poolsByNeed(questions, traceText) {
+  const byNeed = new Map(questions.map((q) => [q.need, q.id]));
+  const pools = new Map();
+  for (const line of traceText.split('\n')) {
+    if (!line.trim()) continue;
+    let row;
+    try { row = JSON.parse(line); } catch { continue; }
+    const id = byNeed.get(row.query);
+    if (id && Array.isArray(row.cands) && row.cands.length) pools.set(id, row.cands);
+  }
+  return pools;
+}
+
+export function fitLogistic(X, y, { l2 = 1, iters = 3000, lr = 0.1 } = {}) {
+  const d = X[0].length;
+  const mean = Array.from({ length: d }, (_, j) => X.reduce((s, x) => s + x[j], 0) / X.length);
+  const std = Array.from({ length: d }, (_, j) => Math.sqrt(X.reduce((s, x) => s + (x[j] - mean[j]) ** 2, 0) / X.length) || 1);
+  const Z = X.map((x) => x.map((v, j) => (v - mean[j]) / std[j]));
+  const pos = y.filter(Boolean).length;
+  const wPos = pos ? (y.length - pos) / pos : 1;
+  const w = new Array(d).fill(0);
+  let b = 0;
+  for (let it = 0; it < iters; it++) {
+    const g = new Array(d).fill(0);
+    let gb = 0;
+    let W = 0;
+    for (let i = 0; i < Z.length; i++) {
+      const s = b + Z[i].reduce((a, v, j) => a + v * w[j], 0);
+      const p = 1 / (1 + Math.exp(-s));
+      const cw = y[i] ? wPos : 1;
+      const e = cw * (p - (y[i] ? 1 : 0));
+      for (let j = 0; j < d; j++) g[j] += e * Z[i][j];
+      gb += e;
+      W += cw;
+    }
+    for (let j = 0; j < d; j++) w[j] -= lr * (g[j] / W + (l2 * w[j]) / Z.length);
+    b -= lr * (gb / W);
+  }
+  return { weights: w, bias: b, mean, std };
+}
+
+/** Per question: the top candidate under a scorer, whether a target is in the top 1 / top 5, top score. */
+export function decide(questions, pools, score) {
+  return questions.filter((q) => pools.has(q.id)).map((q) => {
+    const cands = pools.get(q.id);
+    const s = score(q, cands);
+    const order = s.map((v, i) => [v, i]).sort((a, b) => b[0] - a[0]).map(([, i]) => i);
+    return { id: q.id, top: s[order[0]], at1: isTarget(q, cands[order[0]]),
+      within5: order.slice(0, 5).some((i) => isTarget(q, cands[i])) };
+  });
+}
+
+export function metrics(rows, n, threshold = 0) {
+  const r = (k, m) => { const w = wilson(k, m); return { k, n: m, lo: +w.lo.toFixed(4), hi: +w.hi.toFixed(4) }; };
+  const answered = rows.filter((x) => x.top >= threshold);
+  return {
+    traced: rows.length,
+    goldWithin5: r(rows.filter((x) => x.within5).length, n),
+    goldAt1: r(rows.filter((x) => x.at1).length, n),
+    // measure-need-set's definitions (target within 5 AND not abstained), so baselines compare
+    confidentHit: r(answered.filter((x) => x.within5).length, n),
+    confidentWrong: r(answered.filter((x) => !x.within5).length, n),
+    // the stricter reading: the single top answer is the target
+    confidentHitTop1: r(answered.filter((x) => x.at1).length, n),
+    abstained: r(n - answered.length, n),
+    precision: r(answered.filter((x) => x.at1).length, answered.length),
+  };
+}
+
+export function chooseThreshold(rows, precision) {
+  const tops = [...new Set(rows.map((x) => x.top))].sort((a, b) => a - b);
+  for (const t of tops) {
+    const ans = rows.filter((x) => x.top >= t);
+    if (ans.length && ans.filter((x) => x.at1).length / ans.length >= precision) return t;
+  }
+  return Infinity;
+}
+
+async function main() {
+  const args = process.argv.slice(2);
+  const arg = (n, d) => { const i = args.indexOf(n); return i >= 0 ? args[i + 1] : d; };
+  const set = JSON.parse(fs.readFileSync(arg('--set'), 'utf8')).questions;
+  const split = JSON.parse(fs.readFileSync(arg('--split'), 'utf8'));
+  const pools = poolsByNeed(set, fs.readFileSync(arg('--trace'), 'utf8'));
+  const train = set.filter((q) => split.train.includes(q.id));
+  const held = set.filter((q) => split.heldout.includes(q.id));
+  const X = [];
+  const y = [];
+  for (const q of train) {
+    if (!pools.has(q.id)) continue;
+    const cands = pools.get(q.id);
+    judgeFeatures(q.need, cands).forEach((x, i) => { X.push(x); y.push(isTarget(q, cands[i])); });
+  }
+  const fit = fitLogistic(X, y);
+  const model0 = { ...fit, features: JUDGE_FEATURES };
+  const judgeScore = (m) => (q, cands) => judgeFeatures(q.need, cands).map((x) => judgeLogit(m, x));
+  const ceScore = (q, cands) => cands.map((c) => (typeof c.ce === 'number' ? c.ce : -Infinity));
+  const precision = Number(arg('--precision', 0.8));
+  const tau = chooseThreshold(decide(train, pools, judgeScore(model0)), precision);
+  const model = { ...model0, bias: Number.isFinite(tau) ? fit.bias - tau : -1e9, threshold: tau,
+    trainedOn: { split: split.salt, trainNeeds: train.length, tracedTrain: train.filter((q) => pools.has(q.id)).length,
+      candidates: X.length, positives: y.filter(Boolean).length, precisionTarget: precision } };
+  const report = {
+    kind: 'ruvnet-brain-judge-report', features: JUDGE_FEATURES, trainedOn: model.trainedOn, threshold: tau,
+    weights: Object.fromEntries(JUDGE_FEATURES.map((f, j) => [f, +model.weights[j].toFixed(4)])),
+    train: { crossEncoder: metrics(decide(train, pools, ceScore), train.length), judge: metrics(decide(train, pools, judgeScore(model)), train.length) },
+    heldout: { crossEncoder: metrics(decide(held, pools, ceScore), held.length), judge: metrics(decide(held, pools, judgeScore(model)), held.length) },
+  };
+  // Offline replay on the recall-gate fixture's recorded pools: does re-ranking hurt questions the
+  // cross-encoder already answers? (Approximate: selectResults' name boosts are not replayed here, so
+  // the full path is the authority; this only flags gross damage early.)
+  if (arg('--recall-trace') && arg('--recall-fixture')) {
+    const fx = JSON.parse(fs.readFileSync(arg('--recall-fixture'), 'utf8')).queries;
+    const items = Object.entries(fx).map(([store, v]) => ({ id: store, need: v.query, repo: store, path: v.expected.path }));
+    const rpools = poolsByNeed(items, fs.readFileSync(arg('--recall-trace'), 'utf8'));
+    report.recallReplay = { crossEncoder: metrics(decide(items, rpools, ceScore), items.length),
+      judge: metrics(decide(items, rpools, judgeScore(model)), items.length) };
+  }
+  if (arg('--weights')) fs.writeFileSync(arg('--weights'), `${JSON.stringify(model, null, 1)}\n`);
+  if (arg('--report')) fs.writeFileSync(arg('--report'), `${JSON.stringify(report, null, 1)}\n`);
+  console.log(JSON.stringify({ threshold: tau, trainedOn: model.trainedOn, heldout: report.heldout }, null, 1));
+}
+
+if (process.argv[1] && path.resolve(process.argv[1]) === fileURLToPath(import.meta.url)) await main();

package/scripts/oracle/need-set-split.mjs

@@ -0,0 +1,48 @@
+#!/usr/bin/env node
+/**
+ * scripts/oracle/need-set-split.mjs — a frozen, stratified train / held-out split of a need set.
+ *
+ * Anything that LEARNS from novice needs (a query adapter, a learned abstain model) may train only
+ * on `train`; every claim about it is measured on `heldout`. The split is a pure function of the
+ * question ids and a salt: within each repository the ids are ordered by sha256(salt + id) and the
+ * first round(n * trainFraction) go to train. Re-running it on the same set yields the same split.
+ *
+ *   node scripts/oracle/need-set-split.mjs --set <need-set.json> [--salt v1] [--train 0.5] [--out <file>]
+ */
+import { createHash } from 'node:crypto';
+import fs from 'node:fs';
+import path from 'node:path';
+import { fileURLToPath } from 'node:url';
+
+export function splitNeedSet(questions, { salt = 'v1', trainFraction = 0.5 } = {}) {
+  const byRepo = new Map();
+  for (const q of questions) {
+    if (!byRepo.has(q.repo)) byRepo.set(q.repo, []);
+    byRepo.get(q.repo).push(q.id);
+  }
+  const train = [];
+  const heldout = [];
+  for (const repo of [...byRepo.keys()].sort()) {
+    const ids = byRepo.get(repo)
+      .map((id) => ({ id, h: createHash('sha256').update(`${salt}\n${id}`).digest('hex') }))
+      .sort((a, b) => (a.h < b.h ? -1 : a.h > b.h ? 1 : 0))
+      .map((x) => x.id);
+    const cut = Math.round(ids.length * trainFraction);
+    train.push(...ids.slice(0, cut));
+    heldout.push(...ids.slice(cut));
+  }
+  return { salt, trainFraction, train, heldout };
+}
+
+async function main() {
+  const args = process.argv.slice(2);
+  const arg = (n, d) => { const i = args.indexOf(n); return i >= 0 ? args[i + 1] : d; };
+  const set = JSON.parse(fs.readFileSync(arg('--set'), 'utf8'));
+  const split = splitNeedSet(set.questions, { salt: arg('--salt', 'v1'), trainFraction: Number(arg('--train', 0.5)) });
+  const out = { kind: 'ruvnet-brain-need-set-split', set: { kind: set.kind, version: set.version, n: set.questions.length,
+    contentHash: set.contentHash }, ...split };
+  if (arg('--out')) fs.writeFileSync(arg('--out'), `${JSON.stringify(out, null, 1)}\n`);
+  console.log(JSON.stringify({ train: split.train.length, heldout: split.heldout.length, contentHash: set.contentHash }));
+}
+
+if (process.argv[1] && path.resolve(process.argv[1]) === fileURLToPath(import.meta.url)) await main();