sigmap 7.15.0 → 7.16.0

package/CHANGELOG.md

@@ -10,6 +10,15 @@ Format: [Semantic Versioning](https://semver.org/)
 
 ---
 
+## [7.16.0] — 2026-06-18
+
+Minor release — LLM A/B hallucination ablation harness (grounded codegen, IMPL §9).
+
+### Added
+- **LLM A/B hallucination ablation harness (#325):** the honest measurement behind the grounded-codegen plan (IMPL §9). Runs a model twice per task — (A) no SigMap context, (B) with SigMap grounding — pipes both outputs through the hallucination guard, and reports the measured delta in flagged codebase-fact errors. New zero-dependency, bundle-safe `src/eval/llm-ablation.js` (`buildGrounding`, `scoreAnswer`, `runAblation`) keeps the model call **injected**, so the harness is fully offline-testable; the live runner `scripts/run-llm-ablation.mjs` wires Anthropic via `ANTHROPIC_API_KEY` and prints the A/B table + delta (`npm run benchmark:llm-ablation`), degrading to a graceful skip (exit 0) when no key is set. The network fetch is confined to `scripts/`, never the published library surface. Starter corpus in `benchmarks/llm-ablation-tasks.json`. This turns §9 from an offline coverage proxy into a ready-to-run real A/B — the moment a key is present, it produces the measured hallucination delta.
+
+---
+
 ## [7.15.0] — 2026-06-18
 
 Minor release — `sigmap conventions --ci` (grounded codegen, Layer 3 polish).

package/gen-context.js

@@ -31,6 +31,123 @@ function __require(key) {
 // ── ./src/create/orchestrate ──
 // ── ./src/conventions/report ──
 // ── ./src/conventions/ci ──
+// ── ./src/eval/llm-ablation ──
+__factories["./src/eval/llm-ablation"] = function(module, exports) {
+  
+  /**
+   * LLM A/B hallucination ablation (IMPL.md §9) — the honest measurement.
+   *
+   * Runs a model twice per task — (A) no SigMap context, (B) with SigMap
+   * grounding — pipes both outputs through the hallucination guard, and reports
+   * the measured delta in flagged codebase-fact errors. The model call is
+   * INJECTED (`complete(prompt) → text`), so the harness itself is pure and
+   * offline-testable; the live model adapter lives in `scripts/run-llm-ablation.mjs`.
+   * Zero-dependency, bundle-safe (no network here).
+   */
+
+  const { verify } = __require('./src/verify/hallucination-guard');
+
+  /**
+   * Build the SigMap grounding block for a repo — what we prepend to a task
+   * prompt in arm B. Conventions (the house style) + the known-symbol list
+   * (so the model can reference real names instead of guessing).
+   * @param {string} cwd
+   * @param {object} [opts]
+   * @param {number} [opts.maxSymbols=80]
+   * @returns {string}
+   */
+  function buildGrounding(cwd, opts = {}) {
+    const maxSymbols = opts.maxSymbols != null ? opts.maxSymbols : 80;
+    const parts = [];
+
+    try {
+      const { extractConventions } = __require('./src/conventions/extract');
+      const { renderConventionsBlock } = __require('./src/conventions/inject');
+      const { loadConfig } = __require('./src/config/loader');
+      let files = [];
+      try {
+        const cfg = loadConfig(cwd);
+        const { buildSigIndex } = __require('./src/retrieval/ranker');
+        files = [...buildSigIndex(cwd).keys()];
+        void cfg;
+      } catch (_) {}
+      const conv = extractConventions(cwd, files);
+      parts.push(renderConventionsBlock(conv));
+    } catch (_) {}
+
+    try {
+      const { buildSymbolSet } = __require('./src/verify/hallucination-guard');
+      const { set } = buildSymbolSet(cwd);
+      const names = [...set].slice(0, maxSymbols);
+      if (names.length) parts.push(`## Known symbols (reference these exactly)\n${names.join(', ')}`);
+    } catch (_) {}
+
+    return parts.join('\n\n');
+  }
+
+  /**
+   * Count flagged codebase-fact errors in an answer (the §9 metric).
+   * @param {string} answerText
+   * @param {string} cwd
+   * @returns {number}
+   */
+  function scoreAnswer(answerText, cwd) {
+    try {
+      const { summary } = verify(String(answerText || ''), cwd);
+      return summary.total || 0;
+    } catch (_) {
+      return 0;
+    }
+  }
+
+  /**
+   * Run the A/B ablation over a task corpus.
+   * @param {Array<{id:string, prompt:string}>} tasks
+   * @param {string} cwd
+   * @param {(prompt:string, meta:object)=>string} complete injected model call
+   * @param {object} [opts]
+   * @param {string} [opts.grounding] precomputed grounding (else built from cwd)
+   * @returns {{ tasks: object[], aggregate: object }}
+   */
+  function runAblation(tasks, cwd, complete, opts = {}) {
+    const grounding = opts.grounding != null ? opts.grounding : buildGrounding(cwd);
+    const rows = [];
+    let sumA = 0;
+    let sumB = 0;
+
+    for (const task of tasks || []) {
+      const basePrompt = task.prompt || '';
+      const groundedPrompt = grounding ? `${grounding}\n\n---\n\n${basePrompt}` : basePrompt;
+
+      const outA = String(complete(basePrompt, { id: task.id, grounded: false }) || '');
+      const outB = String(complete(groundedPrompt, { id: task.id, grounded: true }) || '');
+
+      const aFlagged = scoreAnswer(outA, cwd);
+      const bFlagged = scoreAnswer(outB, cwd);
+      sumA += aFlagged;
+      sumB += bFlagged;
+      rows.push({ id: task.id, aFlagged, bFlagged });
+    }
+
+    const n = rows.length;
+    const per100 = (sum) => (n > 0 ? (sum / n) * 100 : 0);
+    return {
+      tasks: rows,
+      aggregate: {
+        n,
+        withoutFlagged: sumA,
+        withFlagged: sumB,
+        delta: sumA - sumB,
+        withoutPer100: per100(sumA),
+        withPer100: per100(sumB),
+      },
+    };
+  }
+
+  module.exports = { buildGrounding, scoreAnswer, runAblation };
+  
+};
+
 __factories["./src/conventions/ci"] = function(module, exports) {
   
   /**
@@ -7590,7 +7707,7 @@ __factories["./src/mcp/server"] = function(module, exports) {
 
   const SERVER_INFO = {
     name: 'sigmap',
-    version: '7.15.0',
+    version: '7.16.0',
     description: 'SigMap MCP server — code signatures on demand',
   };
 
@@ -13268,7 +13385,7 @@ function __tryGit(args, opts = {}) {
   catch (_) { return ''; }
 }
 
-const VERSION = '7.15.0';
+const VERSION = '7.16.0';
 const MARKER = '\n\n## Auto-generated signatures\n<!-- Updated by gen-context.js -->\n';
 
 function requireSourceOrBundled(key) {

package/llms-full.txt

@@ -9,7 +9,7 @@ the files relevant to the task — cutting tokens ~97% while keeping answers
 grounded. Deterministic, offline, no embeddings or vector database. Works with
 Claude, Cursor, GitHub Copilot, Aider, Windsurf, local LLMs, and MCP.
 
-# Version: 7.15.0 | Benchmark: sigmap-v7.0-main (2026-06-14)
+# Version: 7.16.0 | Benchmark: sigmap-v7.0-main (2026-06-14)
 # Source: auto-generated from package.json, version.json, src/mcp/tools.js, src/config/defaults.js
 # Regenerate: npm run generate:llms   |   Validate: npm run validate:llms
 

package/llms.txt

@@ -9,7 +9,7 @@ the files relevant to the task — cutting tokens ~97% while keeping answers
 grounded. Deterministic, offline, no embeddings or vector database. Works with
 Claude, Cursor, GitHub Copilot, Aider, Windsurf, local LLMs, and MCP.
 
-# Version: 7.15.0 | Benchmark: sigmap-v7.0-main (2026-06-14)
+# Version: 7.16.0 | Benchmark: sigmap-v7.0-main (2026-06-14)
 # Source: auto-generated from package.json, version.json, src/mcp/tools.js, src/config/defaults.js
 # Regenerate: npm run generate:llms   |   Validate: npm run validate:llms
 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "sigmap",
-  "version": "7.15.0",
+  "version": "7.16.0",
   "description": "97% token reduction for AI coding. Extracts function & class signatures with TF-IDF ranking to feed only the right files to Claude, Cursor, Copilot, Aider, Windsurf, local LLMs & MCP. Zero dependencies, runs offline via npx.",
   "main": "packages/core/index.js",
   "exports": {
@@ -39,7 +39,8 @@
     "generate:llms": "node scripts/generate-llms.mjs",
     "validate:llms": "node scripts/validate-llms.mjs",
     "prepublishOnly": "node scripts/check-bundle.mjs && node scripts/check-version-meta.mjs && node scripts/generate-llms.mjs",
-    "benchmark:grounding": "node scripts/run-hallucination-benchmark.mjs"
+    "benchmark:grounding": "node scripts/run-hallucination-benchmark.mjs",
+    "benchmark:llm-ablation": "node scripts/run-llm-ablation.mjs"
   },
   "files": [
     "gen-context.js",

package/packages/cli/package.json

@@ -1,6 +1,6 @@
 {
   "name": "sigmap-cli",
-  "version": "7.15.0",
+  "version": "7.16.0",
   "description": "SigMap CLI wrapper — thin adapter for programmatic CLI invocation",
   "main": "index.js",
   "keywords": [

package/packages/core/package.json

@@ -1,6 +1,6 @@
 {
   "name": "sigmap-core",
-  "version": "7.15.0",
+  "version": "7.16.0",
   "description": "SigMap core library — zero-dependency code signature extraction, retrieval, and security scanning",
   "main": "index.js",
   "keywords": [

package/src/eval/llm-ablation.js

@@ -0,0 +1,113 @@
+'use strict';
+
+/**
+ * LLM A/B hallucination ablation (IMPL.md §9) — the honest measurement.
+ *
+ * Runs a model twice per task — (A) no SigMap context, (B) with SigMap
+ * grounding — pipes both outputs through the hallucination guard, and reports
+ * the measured delta in flagged codebase-fact errors. The model call is
+ * INJECTED (`complete(prompt) → text`), so the harness itself is pure and
+ * offline-testable; the live model adapter lives in `scripts/run-llm-ablation.mjs`.
+ * Zero-dependency, bundle-safe (no network here).
+ */
+
+const { verify } = require('../verify/hallucination-guard');
+
+/**
+ * Build the SigMap grounding block for a repo — what we prepend to a task
+ * prompt in arm B. Conventions (the house style) + the known-symbol list
+ * (so the model can reference real names instead of guessing).
+ * @param {string} cwd
+ * @param {object} [opts]
+ * @param {number} [opts.maxSymbols=80]
+ * @returns {string}
+ */
+function buildGrounding(cwd, opts = {}) {
+  const maxSymbols = opts.maxSymbols != null ? opts.maxSymbols : 80;
+  const parts = [];
+
+  try {
+    const { extractConventions } = require('../conventions/extract');
+    const { renderConventionsBlock } = require('../conventions/inject');
+    const { loadConfig } = require('../config/loader');
+    let files = [];
+    try {
+      const cfg = loadConfig(cwd);
+      const { buildSigIndex } = require('../retrieval/ranker');
+      files = [...buildSigIndex(cwd).keys()];
+      void cfg;
+    } catch (_) {}
+    const conv = extractConventions(cwd, files);
+    parts.push(renderConventionsBlock(conv));
+  } catch (_) {}
+
+  try {
+    const { buildSymbolSet } = require('../verify/hallucination-guard');
+    const { set } = buildSymbolSet(cwd);
+    const names = [...set].slice(0, maxSymbols);
+    if (names.length) parts.push(`## Known symbols (reference these exactly)\n${names.join(', ')}`);
+  } catch (_) {}
+
+  return parts.join('\n\n');
+}
+
+/**
+ * Count flagged codebase-fact errors in an answer (the §9 metric).
+ * @param {string} answerText
+ * @param {string} cwd
+ * @returns {number}
+ */
+function scoreAnswer(answerText, cwd) {
+  try {
+    const { summary } = verify(String(answerText || ''), cwd);
+    return summary.total || 0;
+  } catch (_) {
+    return 0;
+  }
+}
+
+/**
+ * Run the A/B ablation over a task corpus.
+ * @param {Array<{id:string, prompt:string}>} tasks
+ * @param {string} cwd
+ * @param {(prompt:string, meta:object)=>string} complete injected model call
+ * @param {object} [opts]
+ * @param {string} [opts.grounding] precomputed grounding (else built from cwd)
+ * @returns {{ tasks: object[], aggregate: object }}
+ */
+function runAblation(tasks, cwd, complete, opts = {}) {
+  const grounding = opts.grounding != null ? opts.grounding : buildGrounding(cwd);
+  const rows = [];
+  let sumA = 0;
+  let sumB = 0;
+
+  for (const task of tasks || []) {
+    const basePrompt = task.prompt || '';
+    const groundedPrompt = grounding ? `${grounding}\n\n---\n\n${basePrompt}` : basePrompt;
+
+    const outA = String(complete(basePrompt, { id: task.id, grounded: false }) || '');
+    const outB = String(complete(groundedPrompt, { id: task.id, grounded: true }) || '');
+
+    const aFlagged = scoreAnswer(outA, cwd);
+    const bFlagged = scoreAnswer(outB, cwd);
+    sumA += aFlagged;
+    sumB += bFlagged;
+    rows.push({ id: task.id, aFlagged, bFlagged });
+  }
+
+  const n = rows.length;
+  const per100 = (sum) => (n > 0 ? (sum / n) * 100 : 0);
+  return {
+    tasks: rows,
+    aggregate: {
+      n,
+      withoutFlagged: sumA,
+      withFlagged: sumB,
+      delta: sumA - sumB,
+      withoutPer100: per100(sumA),
+      withPer100: per100(sumB),
+    },
+  };
+}
+
+module.exports = { buildGrounding, scoreAnswer, runAblation };

package/src/mcp/server.js

@@ -18,7 +18,7 @@ const { readContext, searchSignatures, getMap, createCheckpoint, getRouting, exp
 
 const SERVER_INFO = {
   name: 'sigmap',
-  version: '7.15.0',
+  version: '7.16.0',
   description: 'SigMap MCP server — code signatures on demand',
 };