qualia-framework 6.14.0 → 7.0.0

package/bin/eval-runner.js

@@ -0,0 +1,218 @@
+#!/usr/bin/env node
+// eval-runner.js — layered assertion engine for AI features (chat / RAG / voice).
+//
+// WHY: Qualia gates UI and code (contract-runner, slop-detect, verify-panel) but
+// has no equivalent for the AI artifacts it BUILDS. "The chatbot answers refund
+// questions" is not checkable by a grep. This runs a suite of cases against
+// captured AI outputs with LAYERED assertions: cheap deterministic checks first
+// (contains / regex / json_path / length / latency / cost), then llm_rubric for
+// the judgments only a model can make — same shape as contract-runner evidence.
+//
+// Deterministic-first by design: every assertion this module can settle WITHOUT
+// a model, it settles here. `llm_rubric` assertions carry a `verdict`
+// (pass|fail) that the /qualia-eval skill fills by spawning a judge BEFORE
+// calling this runner — exactly how verify-panel consumes skeptic votes. An
+// llm_rubric with no verdict is PENDING and fails the suite (never silently
+// passes an unjudged rubric).
+//
+// Suite (JSON — zero-dependency; no YAML parser pulled in):
+//   {
+//     "feature": "support-chat",
+//     "cases": [
+//       { "name": "refund window",
+//         "input": "what's your refund policy?",
+//         "output": "We refund within 30 days...",   // or "output_file": "path"
+//         "latency_ms": 1200, "cost_usd": 0.008,
+//         "assert": [
+//           { "type": "contains", "value": "30 days" },
+//           { "type": "not_contains", "value": "I cannot help" },
+//           { "type": "max_latency_ms", "value": 2000 },
+//           { "type": "llm_rubric", "rubric": "grounded in policy, no hallucination", "verdict": "pass" }
+//         ] } ] }
+//
+// Exit 0 = all cases pass, 1 = a failure or an unjudged rubric, 2 = bad input.
+// Zero npm dependencies. Library + CLI.
+
+const fs = require("fs");
+const path = require("path");
+
+function getPath(obj, dotted) {
+  const parts = String(dotted).split(".").filter(Boolean);
+  let cur = obj;
+  for (const p of parts) {
+    if (cur == null) return undefined;
+    const idx = /^\d+$/.test(p) ? Number(p) : p;
+    cur = cur[idx];
+  }
+  return cur;
+}
+
+function safeRegex(src) {
+  try { return new RegExp(src); } catch { return null; }
+}
+
+// Run one assertion against a case's captured output + metrics.
+// Returns { ok, detail }.
+function runAssertion(a, ctx) {
+  const out = ctx.output != null ? String(ctx.output) : "";
+  switch (a.type) {
+    case "contains":
+      return { ok: out.includes(String(a.value)), detail: `contains "${a.value}"` };
+    case "not_contains":
+      return { ok: !out.includes(String(a.value)), detail: `not_contains "${a.value}"` };
+    case "equals":
+      return { ok: out === String(a.value), detail: `equals "${a.value}"` };
+    case "regex": {
+      const re = safeRegex(a.value);
+      if (!re) return { ok: false, detail: `invalid regex: ${a.value}` };
+      return { ok: re.test(out), detail: `regex /${a.value}/` };
+    }
+    case "not_regex": {
+      const re = safeRegex(a.value);
+      if (!re) return { ok: false, detail: `invalid regex: ${a.value}` };
+      return { ok: !re.test(out), detail: `not_regex /${a.value}/` };
+    }
+    case "min_length":
+      return { ok: out.length >= Number(a.value), detail: `min_length ${a.value} (got ${out.length})` };
+    case "max_length":
+      return { ok: out.length <= Number(a.value), detail: `max_length ${a.value} (got ${out.length})` };
+    case "json_valid":
+      try { JSON.parse(out); return { ok: true, detail: "json_valid" }; }
+      catch { return { ok: false, detail: "output is not valid JSON" }; }
+    case "json_path": {
+      let parsed;
+      try { parsed = JSON.parse(out); } catch { return { ok: false, detail: "json_path: output not JSON" }; }
+      const val = getPath(parsed, a.path);
+      if (val === undefined) return { ok: false, detail: `json_path ${a.path}: missing` };
+      if (a.equals !== undefined) return { ok: String(val) === String(a.equals), detail: `json_path ${a.path} equals "${a.equals}" (got "${val}")` };
+      if (a.contains !== undefined) return { ok: String(val).includes(String(a.contains)), detail: `json_path ${a.path} contains "${a.contains}"` };
+      return { ok: true, detail: `json_path ${a.path} present` };
+    }
+    case "max_latency_ms":
+      if (ctx.latency_ms == null) return { ok: false, detail: "max_latency_ms: no latency_ms recorded on case" };
+      return { ok: Number(ctx.latency_ms) <= Number(a.value), detail: `max_latency_ms ${a.value} (got ${ctx.latency_ms})` };
+    case "max_cost_usd":
+      if (ctx.cost_usd == null) return { ok: false, detail: "max_cost_usd: no cost_usd recorded on case" };
+      return { ok: Number(ctx.cost_usd) <= Number(a.value), detail: `max_cost_usd ${a.value} (got ${ctx.cost_usd})` };
+    case "llm_rubric": {
+      const v = a.verdict != null ? String(a.verdict).toLowerCase() : null;
+      if (v === null) return { ok: false, pending: true, detail: `llm_rubric PENDING (no judge verdict): "${a.rubric}"` };
+      return { ok: v === "pass", detail: `llm_rubric "${a.rubric}" → ${v}` };
+    }
+    default:
+      return { ok: false, detail: `unknown assertion type: ${a.type}` };
+  }
+}
+
+function resolveOutput(root, c) {
+  if (c.output != null) return c.output;
+  if (c.output_file) {
+    try { return fs.readFileSync(path.resolve(root, c.output_file), "utf8"); }
+    catch (e) { return { __error: `output_file unreadable: ${e.message}` }; }
+  }
+  return "";
+}
+
+function run(suite, opts = {}) {
+  const root = path.resolve(opts.cwd || process.cwd());
+  const cases = [];
+  let pending = 0;
+  for (const c of suite.cases || []) {
+    const output = resolveOutput(root, c);
+    const ctx = { output: (output && output.__error) ? "" : output, latency_ms: c.latency_ms, cost_usd: c.cost_usd };
+    const assertions = [];
+    if (output && output.__error) {
+      assertions.push({ type: "output", ok: false, detail: output.__error });
+    }
+    for (const a of c.assert || []) {
+      const r = runAssertion(a, ctx);
+      if (r.pending) pending++;
+      assertions.push({ type: a.type, ok: !!r.ok, pending: !!r.pending, detail: r.detail });
+    }
+    const ok = assertions.length > 0 && assertions.every((x) => x.ok);
+    cases.push({ name: c.name || "(unnamed)", ok, assertions });
+  }
+  const passed = cases.filter((c) => c.ok).length;
+  return {
+    ok: cases.length > 0 && passed === cases.length,
+    feature: suite.feature || "(unnamed feature)",
+    total_cases: cases.length,
+    passed_cases: passed,
+    failed_cases: cases.length - passed,
+    pending,
+    cases,
+  };
+}
+
+// ── CLI ───────────────────────────────────────────────────────────────────
+function parseArgs(argv) {
+  const args = { _: [] };
+  for (let i = 2; i < argv.length; i++) {
+    const a = argv[i];
+    if (a === "--json") args.json = true;
+    else if (a === "--write") args.write = true;
+    else if (a === "--cwd") args.cwd = argv[++i];
+    else if (a.startsWith("--cwd=")) args.cwd = a.slice(6);
+    else args._.push(a);
+  }
+  return args;
+}
+
+function usage() {
+  console.error([
+    "Usage:",
+    "  eval-runner.js <suite.json> [--json] [--write] [--cwd DIR]",
+    "",
+    "Layered assertion runner for AI-feature eval suites.",
+    "Exit 0 = all cases pass, 1 = failure/unjudged rubric, 2 = invocation error.",
+  ].join("\n"));
+}
+
+function main(argv) {
+  const args = parseArgs(argv);
+  const suitePath = args._[0];
+  if (!suitePath || suitePath === "-h" || suitePath === "--help") { usage(); return 2; }
+  const root = path.resolve(args.cwd || process.cwd());
+
+  let suite;
+  try {
+    suite = JSON.parse(fs.readFileSync(path.resolve(root, suitePath), "utf8"));
+  } catch (e) {
+    console.error(`ERROR: cannot read eval suite: ${e.message}`);
+    return 2;
+  }
+
+  const result = run(suite, { cwd: root });
+
+  if (args.write) {
+    const dir = path.join(root, ".planning", "evals");
+    try {
+      fs.mkdirSync(dir, { recursive: true });
+      const slug = String(result.feature).toLowerCase().replace(/[^a-z0-9]+/g, "-").replace(/^-|-$/g, "");
+      fs.writeFileSync(path.join(dir, `eval-${slug || "feature"}.json`), JSON.stringify(result, null, 2) + "\n");
+    } catch (e) {
+      console.error(`WARN: could not write eval artifact: ${e.message}`);
+    }
+  }
+
+  if (args.json) {
+    console.log(JSON.stringify(result, null, 2));
+    return result.ok ? 0 : 1;
+  }
+
+  console.log(`EVAL ${result.ok ? "PASS" : "FAIL"} [${result.feature}]: ` +
+    `${result.passed_cases}/${result.total_cases} cases` +
+    (result.pending ? ` (${result.pending} rubric(s) unjudged)` : ""));
+  for (const c of result.cases) {
+    if (c.ok) continue;
+    console.log(`  ✗ ${c.name}`);
+    for (const a of c.assertions) if (!a.ok) console.log(`      - ${a.detail}`);
+  }
+  return result.ok ? 0 : 1;
+}
+
+module.exports = { run, runAssertion, getPath };
+
+if (require.main === module) {
+  process.exit(main(process.argv));
+}

package/bin/host-adapters.js

@@ -1,26 +1,66 @@
 #!/usr/bin/env node
-// Host adapter rendering for installed Qualia text surfaces.
+// host-adapters.js — the single internal contract for everything that differs
+// between the runtimes Qualia targets (Claude Code, OpenAI Codex).
+//
+// The rule (ACP shape): NOTHING else branches on runtime. Skills, install, and
+// hooks ask the adapter for a fact ("what's the instruction filename?", "how do
+// I render this text for this host?") instead of hardcoding an `if (codex)`.
+// When a third runtime arrives, it's one entry in HOSTS — not a grep-and-patch
+// across the tree.
 
 const path = require("path");
 const os = require("os");
 
+// Per-host facts. Each host declares — declaratively, in ONE place:
+//   home            install root
+//   name            display name (used in the naming map + UX copy)
+//   instructionFile the top-level agent-instructions file this runtime reads
+//   configFile      the runtime's settings file
+//   agentDir        where subagent definitions live (relative to home)
+//   agentExt        the subagent file extension the runtime consumes
+//   naming          source→host display-string swaps applied to rendered text
 const HOSTS = {
   claude: {
     name: "Claude Code",
     home: path.join(os.homedir(), ".claude"),
+    instructionFile: "CLAUDE.md",
+    configFile: "settings.json",
+    agentDir: "agents",
+    agentExt: ".md",
+    agentCli: "claude", // the CLI that runs a non-interactive turn for this host
+    agentExecPrefix: ["-p"], // `claude -p "<prompt>"`
+    naming: {}, // Claude is the canonical voice — no swaps.
   },
   codex: {
     name: "OpenAI Codex",
     home: path.join(os.homedir(), ".codex"),
+    instructionFile: "AGENTS.md",
+    configFile: "config.toml",
+    agentDir: "agents",
+    agentExt: ".toml",
+    agentCli: "codex",
+    agentExecPrefix: ["exec"], // `codex exec "<prompt>"`
+    naming: { "Claude Code": "Codex", "Claude's": "Codex's" },
   },
 };
 
+// Which host owns this install home? (".codex" → codex, else claude.) The one
+// place that maps a home dir back to a host name — callers ask, never re-derive.
+function hostForHome(home) {
+  return path.basename(home) === ".codex" ? "codex" : "claude";
+}
+
 function adapter(name) {
   const host = HOSTS[name];
   if (!host) throw new Error(`Unknown Qualia host adapter: ${name}`);
   const home = host.home;
   return {
     ...host,
+    instructionPath: path.join(home, host.instructionFile),
+    configPath: path.join(home, host.configFile),
+    agentPath: path.join(home, host.agentDir),
+    // Full argv (minus the binary) to run one non-interactive turn on this host.
+    agentExec: (prompt) => [...host.agentExecPrefix, prompt],
     tokens: {
       QUALIA_HOME: home,
       QUALIA_BIN: `${home}/bin`,
@@ -35,32 +75,64 @@ function adapter(name) {
   };
 }
 
-function renderText(content, hostName) {
+// Host-display naming swaps only (e.g. "Claude Code" → "Codex"). Split out from
+// path rendering so the canonical→artifact compile can swap names WITHOUT baking
+// in install-time paths (install resolves those per host afterwards).
+function applyNaming(content, hostName) {
+  const host = adapter(hostName);
+  let out = String(content);
+  for (const [from, to] of Object.entries(host.naming)) {
+    out = out.replaceAll(from, to);
+  }
+  return out;
+}
+
+// Path/token rendering: ${QUALIA_*} tokens + legacy `.claude/` literals → this
+// host's install paths. Idempotent (running twice is a no-op), so install can
+// safely re-render an already-compiled artifact.
+function applyPaths(content, hostName) {
   const host = adapter(hostName);
   let out = String(content);
   for (const [token, value] of Object.entries(host.tokens)) {
     out = out.replaceAll(`\${${token}}`, value);
   }
-
-  // Backward-compatible rendering while source files migrate from hardcoded
-  // Claude paths to explicit ${QUALIA_*} tokens.
-  out = out
+  return out
     .replaceAll("~/.claude/", `${host.home}/`)
     .replaceAll("$HOME/.claude/", `${host.home}/`)
     .replaceAll("${HOME}/.claude/", `${host.home}/`)
     .replaceAll("@~/.claude/", `@${host.home}/`)
     .replaceAll(".claude/", `${path.basename(host.home)}/`);
+}
 
-  if (hostName === "codex") {
-    out = out
-      .replaceAll("Claude Code", "Codex")
-      .replaceAll("Claude's", "Codex's");
-  }
-  return out;
+// Full install-time render: paths + naming. Unchanged public behavior.
+function renderText(content, hostName) {
+  return applyNaming(applyPaths(content, hostName), hostName);
+}
+
+// Host-conditional blocks in a canonical source:
+//   <!--QUALIA-HOST claude-->  …kept only for claude…  <!--/QUALIA-HOST-->
+// Keep the matching host's block (unwrapped), drop every other host's block.
+function stripHostBlocks(content, hostName) {
+  const re = /[ \t]*<!--QUALIA-HOST\s+(\w+)-->\n?([\s\S]*?)\n?[ \t]*<!--\/QUALIA-HOST-->\n?/g;
+  return String(content).replace(re, (_m, host, body) =>
+    host === hostName ? `${body}\n` : ""
+  );
+}
+
+// Compile a canonical instruction source into one host's artifact: resolve
+// conditional blocks + apply naming, but DO NOT resolve paths or {{ROLE}} —
+// those stay as tokens for install.js to render per concrete install.
+function compileInstructions(canonical, hostName) {
+  return applyNaming(stripHostBlocks(canonical, hostName), hostName);
 }
 
 module.exports = {
   HOSTS,
   adapter,
+  hostForHome,
+  applyNaming,
+  applyPaths,
   renderText,
+  stripHostBlocks,
+  compileInstructions,
 };

package/bin/install.js

@@ -6,7 +6,7 @@ const fs = require("fs");
 const ui = require("./qualia-ui.js");
 const { RUNTIME_BIN_SCRIPTS, binFiles } = require("./runtime-manifest.js");
 const { ACTIVE_SKILLS, RETIRED_SKILLS } = require("./command-surface.js");
-const { renderText } = require("./host-adapters.js");
+const { renderText, adapter } = require("./host-adapters.js");
 
 // ─── Colors (kept for legacy log lines; new sections route through qualia-ui) ─
 const TEAL = "\x1b[38;2;0;206;209m";
@@ -956,13 +956,16 @@ async function main() {
   // ─── CLAUDE.md with role ───────────────────────────────
   printSection("Configuration");
   try {
+    // Source + dest filenames come from the host adapter (single per-host
+    // contract). CLAUDE.md is a generated artifact of templates/instructions.md.
+    const claudeFile = adapter("claude").instructionFile;
     let claudeMd = fs.readFileSync(
-      path.join(FRAMEWORK_DIR, "CLAUDE.md"),
+      path.join(FRAMEWORK_DIR, claudeFile),
       "utf8"
     );
     claudeMd = claudeMd.replace("{{ROLE}}", member.role);
     claudeMd = claudeMd.replace("{{ROLE_DESCRIPTION}}", member.description);
-    const claudeDest = path.join(CLAUDE_DIR, "CLAUDE.md");
+    const claudeDest = path.join(CLAUDE_DIR, claudeFile);
     // v5.0: backup existing CLAUDE.md before overwrite. Users may have added
     // personal instructions; without a backup, re-install silently destroys
     // them. .bak files are harmless and easy to clean up.
@@ -1439,6 +1442,29 @@ Client-specific preferences, design choices, and requirements. Loaded by \`/qual
     ok("MCP: next-devtools (runtime error visibility for Next.js projects)");
   }
 
+  // ─── qualia-memory MCP ──────────────────────────────────
+  // Read-only access to the Karpathy LLM Wiki at ~/qualia-memory/. Three tools:
+  // memory.search, memory.read, memory.list. Zero deps — see docs/MEMORY-MCP.md.
+  // Registered globally so every Claude Code session has it without per-project
+  // .mcp.json. Users override the vault path via QUALIA_MEMORY_ROOT.
+  const memoryServerPath = path.join(FRAMEWORK_DIR, "mcp", "memory-mcp", "server.js");
+  if (!settings.mcpServers["qualia-memory"]) {
+    settings.mcpServers["qualia-memory"] = {
+      command: "node",
+      args: [memoryServerPath],
+      env: {
+        QUALIA_MEMORY_ROOT: path.join(require("os").homedir(), "qualia-memory"),
+      },
+      disabled: false,
+    };
+    ok("MCP: qualia-memory (read-only wiki at ~/qualia-memory/)");
+  } else {
+    // Always refresh the args path — the framework may have moved (npm i -g
+    // location, dev checkout vs published install). Env stays user-controlled.
+    settings.mcpServers["qualia-memory"].command = "node";
+    settings.mcpServers["qualia-memory"].args = [memoryServerPath];
+  }
+
   // v5.0: backup existing settings.json before overwrite. The merge logic above
   // preserves user fields, but a partial-write or merger bug could destroy MCP
   // configs / custom permissions. Atomic write (tmp + rename) avoids partial
@@ -1590,24 +1616,29 @@ async function installCodex(member, target, employeeMode = false) {
     }
   }
 
-  // Render AGENTS.md with role substitution. Source is the same AGENTS.md
-  // already shipped in the framework root (it's the cross-vendor mirror of
-  // CLAUDE.md, kept under 25 lines per Pocock's instruction-budget rule).
+  // AGENTS.md is the Codex artifact of templates/instructions.md (compiled by
+  // bin/compile-instructions.js). Source filename + render come from the host
+  // adapter, mirroring the CLAUDE.md path exactly. codexText() resolves
+  // ${QUALIA_*} tokens + .claude→.codex paths — CLAUDE.md got this rendering;
+  // AGENTS.md previously did not (latent drift the single-contract path fixes).
+  const codexFile = adapter("codex").instructionFile;
   let agentsContent;
   try {
-    agentsContent = fs.readFileSync(path.join(FRAMEWORK_DIR, "AGENTS.md"), "utf8");
-    agentsContent = agentsContent
-      .replace("{{ROLE}}", member.role)
-      .replace("{{ROLE_DESCRIPTION}}", member.description);
+    agentsContent = fs.readFileSync(path.join(FRAMEWORK_DIR, codexFile), "utf8");
+    agentsContent = codexText(
+      agentsContent
+        .replace("{{ROLE}}", member.role)
+        .replace("{{ROLE_DESCRIPTION}}", member.description)
+    );
   } catch (e) {
-    warn(`Codex — could not read framework AGENTS.md: ${e.message}`);
+    warn(`Codex — could not read framework ${codexFile}: ${e.message}`);
     return;
   }
 
-  const dest = path.join(CODEX_DIR, "AGENTS.md");
+  const dest = path.join(CODEX_DIR, codexFile);
 
   try {
-    backupIfDifferent(dest, agentsContent, "AGENTS.md");
+    backupIfDifferent(dest, agentsContent, codexFile);
     atomicWrite(dest, agentsContent);
     sectionCount++;
     ok(`AGENTS.md (configured as ${member.role})`);

package/bin/knowledge-flush.js

@@ -110,8 +110,11 @@ function dailyLogHasRecentEntries(windowDays) {
 }
 
 // ── Preflight ────────────────────────────────────────────
-const IS_CODEX_INSTALL = path.basename(QUALIA_HOME) === ".codex";
-const agentCli = IS_CODEX_INSTALL ? "codex" : "claude";
+// Per-host facts (which CLI, how to invoke it) come from the adapter — the one
+// place runtime differences live. See bin/host-adapters.js.
+const { adapter, hostForHome } = require("./host-adapters.js");
+const host = adapter(hostForHome(QUALIA_HOME));
+const agentCli = host.agentCli;
 const agentBin = which(agentCli);
 if (!agentBin) {
   logEvent({ event: "skipped", reason: `${agentCli}-cli-not-on-path` });
@@ -143,7 +146,7 @@ const prompt = [
   "Finish with one line starting exactly: ⬢ Flushed daily-log",
 ].join("\n");
 
-const cliArgs = IS_CODEX_INSTALL ? ["exec", prompt] : ["-p", prompt];
+const cliArgs = host.agentExec(prompt);
 const result = spawnSync(agentBin, cliArgs, {
   encoding: "utf8",
   timeout: 5 * 60 * 1000, // 5 min hard cap — flush should never take this long