ruvector-mragent 0.1.0

package/tools/genCorpus.mjs

@@ -0,0 +1,74 @@
+// Deterministic corpus generator for the MRAgent benchmark.
+//
+// Produces a large, class-balanced, DIFFICULTY-VARIED set of structured signal
+// specs (see agent/memory.mjs for how they become Cue-Tag-Content nodes). Variety
+// is the point: each class spans easy→hard instances (1-hop AND 2-hop bridges,
+// 1–3 ranking-distractors, etc.) so a train/test split constrains every gene and
+// the optimizer cannot overfit an under-constrained knob. No RNG — fully
+// reproducible. Run:  node tools/genCorpus.mjs  (writes data/eval-set.json)
+
+import fs from "node:fs";
+import path from "node:path";
+import { fileURLToPath } from "node:url";
+import { CONCEPT_NAMES } from "../agent/concepts.mjs";
+
+const __dirname = path.dirname(fileURLToPath(import.meta.url));
+const C = CONCEPT_NAMES;                 // 15 real concepts
+const c = (i) => C[i % C.length];
+const PER_CLASS = 10;                    // 6 classes × 10 = 60 tasks
+const OOD = ["weather", "tariff", "opera", "sprint", "recipe", "planet", "ballet", "comet", "cuisine", "monsoon", "harpsichord", "tundra"];
+
+const tasks = [];
+const fact = (cls, i) => `${cls}fact${i}`;
+
+for (let i = 0; i < PER_CLASS; i++) {
+  // semantic — answerable only via DENSE (paraphrase, no shared tokens)
+  tasks.push({
+    id: `s${i}`, class: "semantic", prompt: `semantic ${i}`, expected_fact: fact("s", i),
+    qConcepts: [c(i), c(i + 3)], qLex: [`s${i}tok`],
+    cue: { concepts: [c(i), c(i + 3)], lex: [] },
+    decoys: [{ concepts: [], lex: [`s${i}tok`] }],
+  });
+  // lexical — answerable only via SPARSE (rare id; concept is generic)
+  tasks.push({
+    id: `l${i}`, class: "lexical", prompt: `lexical ${i}`, expected_fact: fact("l", i),
+    qConcepts: [c(i + 1)], qLex: [`lx-${i}`],
+    cue: { concepts: [], lex: [`lx-${i}`] },
+    decoys: [{ concepts: [c(i + 1)], lex: [] }],
+  });
+  // hybrid — needs BOTH signals (RRF)
+  tasks.push({
+    id: `h${i}`, class: "hybrid", prompt: `hybrid ${i}`, expected_fact: fact("h", i),
+    qConcepts: [c(i), c(i + 5)], qLex: [`hy-${i}`],
+    cue: { concepts: [c(i)], lex: [`hy-${i}`] },
+    decoys: [{ concepts: [c(i), c(i + 5)], lex: [] }, { concepts: [], lex: [`hy-${i}`] }],
+  });
+  // bridge — ALTERNATE 1-hop and 2-hop so train+test both contain each difficulty
+  tasks.push({
+    id: `b${i}`, class: "bridge", prompt: `bridge ${i}`, expected_fact: fact("b", i),
+    qConcepts: [c(i + 2), c(i + 7)], qLex: [`b${i}tok`],
+    cue: { concepts: [c(i + 2), c(i + 7)], lex: [`b${i}tok`] },
+    bridges: 1 + (i % 2),
+  });
+  // distractor — VARY 1–3 ranking-distractors; corroborated correct content
+  tasks.push({
+    id: `d${i}`, class: "distractor", prompt: `distractor ${i}`, expected_fact: fact("d", i),
+    qConcepts: [c(i + 4), c(i + 9)], qLex: [`d${i}tok`],
+    cue: { concepts: [c(i + 4), c(i + 9)], lex: [`d${i}tok`] },
+    distractors: 1 + (i % 3), corroborate: true,
+  });
+  // unanswerable — out-of-distribution concepts, no correct content → abstain
+  tasks.push({
+    id: `u${i}`, class: "unanswerable", answerable: false, prompt: `unanswerable ${i}`, expected_fact: "N/A",
+    qConcepts: [OOD[i % OOD.length], OOD[(i + 6) % OOD.length]], qLex: [`u${i}tok`],
+    cue: { concepts: [OOD[i % OOD.length], OOD[(i + 6) % OOD.length]], lex: [`u${i}tok`] },
+    decoys: [{ concepts: [], lex: [`u${i}tok`] }],
+  });
+}
+
+const out = {
+  _comment: "GENERATED by tools/genCorpus.mjs — do not hand-edit. Class-balanced, difficulty-varied structured signal specs; agent/memory.mjs synthesizes Cue/Tag/Content node texts. 6 classes × 10 = 60 tasks. Bridges alternate 1-hop/2-hop; distractors vary 1–3. Regenerate: node tools/genCorpus.mjs",
+  tasks,
+};
+fs.writeFileSync(path.join(__dirname, "..", "data", "eval-set.json"), JSON.stringify(out, null, 2) + "\n");
+console.log(`wrote ${tasks.length} tasks to data/eval-set.json`);