sigmap 7.21.0 → 7.22.0

package/CHANGELOG.md

@@ -10,6 +10,18 @@ Format: [Semantic Versioning](https://semver.org/)
 
 ---
 
+## [7.22.0] — 2026-06-18
+
+Minor release — realistic §9 ablation (real-symbol corpus, exact-signature grounding, --verbose) + Gemini model fix.
+
+### Added
+- **Realistic §9 ablation — real-symbol corpus, exact-signature grounding, `--verbose` (#344):** the LLM A/B ablation now measures something meaningful. `buildGrounding` emits **exact signatures grouped by file** (what `get_callee_signatures` returns, bounded by `maxSignatures`) instead of a flat symbol-name dump — the real product behavior. New `scripts/gen-ablation-corpus.mjs` generates ~40 tasks from the repo's actual exported symbols/files (`benchmarks/llm-ablation-tasks.json`). `src/eval/llm-ablation.js` adds `scoreAnswerDetail` (count + issues) and `runAblation`'s `collectIssues`; the runner's `--verbose` prints every flagged item per arm. A 40-task Gemini run showed grounding reduced flagged errors 62.5 → 22.5 per 100 (directionally positive vs the earlier 4-task noise) — and `--verbose` revealed most flags are `verify-ai-output` file-path false-positives (e.g. "Node.js"), the next thing to harden before publishing a number.
+
+### Fixed
+- **Gemini default model (#343):** the ablation runner's default `gemini-2.0-flash` was retired by AI Studio (404 NOT_FOUND); the default is now the live `gemini-2.5-flash`. The `--model` flag selects any model.
+
+---
+
 ## [7.21.0] — 2026-06-18
 
 Minor release — LLM ablation runner gains a Gemini (AI Studio) provider.

package/gen-context.js

@@ -7931,7 +7931,7 @@ __factories["./src/mcp/server"] = function(module, exports) {
 
   const SERVER_INFO = {
     name: 'sigmap',
-    version: '7.21.0',
+    version: '7.22.0',
     description: 'SigMap MCP server — code signatures on demand',
   };
 
@@ -13609,7 +13609,7 @@ function __tryGit(args, opts = {}) {
   catch (_) { return ''; }
 }
 
-const VERSION = '7.21.0';
+const VERSION = '7.22.0';
 const MARKER = '\n\n## Auto-generated signatures\n<!-- Updated by gen-context.js -->\n';
 
 function requireSourceOrBundled(key) {

package/llms-full.txt

@@ -9,7 +9,7 @@ the files relevant to the task — cutting tokens ~97% while keeping answers
 grounded. Deterministic, offline, no embeddings or vector database. Works with
 Claude, Cursor, GitHub Copilot, Aider, Windsurf, local LLMs, and MCP.
 
-# Version: 7.21.0 | Benchmark: sigmap-v7.0-main (2026-06-14)
+# Version: 7.22.0 | Benchmark: sigmap-v7.0-main (2026-06-14)
 # Source: auto-generated from package.json, version.json, src/mcp/tools.js, src/config/defaults.js
 # Regenerate: npm run generate:llms   |   Validate: npm run validate:llms
 

package/llms.txt

@@ -9,7 +9,7 @@ the files relevant to the task — cutting tokens ~97% while keeping answers
 grounded. Deterministic, offline, no embeddings or vector database. Works with
 Claude, Cursor, GitHub Copilot, Aider, Windsurf, local LLMs, and MCP.
 
-# Version: 7.21.0 | Benchmark: sigmap-v7.0-main (2026-06-14)
+# Version: 7.22.0 | Benchmark: sigmap-v7.0-main (2026-06-14)
 # Source: auto-generated from package.json, version.json, src/mcp/tools.js, src/config/defaults.js
 # Regenerate: npm run generate:llms   |   Validate: npm run validate:llms
 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "sigmap",
-  "version": "7.21.0",
+  "version": "7.22.0",
   "description": "97% token reduction for AI coding. Extracts function & class signatures with TF-IDF ranking to feed only the right files to Claude, Cursor, Copilot, Aider, Windsurf, local LLMs & MCP. Zero dependencies, runs offline via npx.",
   "main": "packages/core/index.js",
   "exports": {

package/packages/cli/package.json

@@ -1,6 +1,6 @@
 {
   "name": "sigmap-cli",
-  "version": "7.21.0",
+  "version": "7.22.0",
   "description": "SigMap CLI wrapper — thin adapter for programmatic CLI invocation",
   "main": "index.js",
   "keywords": [

package/packages/core/package.json

@@ -1,6 +1,6 @@
 {
   "name": "sigmap-core",
-  "version": "7.21.0",
+  "version": "7.22.0",
   "description": "SigMap core library — zero-dependency code signature extraction, retrieval, and security scanning",
   "main": "index.js",
   "keywords": [

package/src/eval/llm-ablation.js

@@ -13,59 +13,82 @@
 
 const { verify } = require('../verify/hallucination-guard');
 
+const path = require('path');
+
+/** Strip a signature's trailing line anchor (` :12-20`) for prompt cleanliness. */
+function _cleanSig(sig) {
+  return String(sig).replace(/\s*:\d+(?:-\d+)?\s*$/, '').trim();
+}
+
 /**
  * Build the SigMap grounding block for a repo — what we prepend to a task
- * prompt in arm B. Conventions (the house style) + the known-symbol list
- * (so the model can reference real names instead of guessing).
+ * prompt in arm B. Conventions (the house style) + **exact signatures** grouped
+ * by file (what `get_callee_signatures` returns), so the model references the
+ * real surface instead of guessing — the actual product behavior, not a flat
+ * name dump.
  * @param {string} cwd
  * @param {object} [opts]
- * @param {number} [opts.maxSymbols=80]
+ * @param {number} [opts.maxSignatures=150] cap on signature lines (bounds prompt size)
  * @returns {string}
  */
 function buildGrounding(cwd, opts = {}) {
-  const maxSymbols = opts.maxSymbols != null ? opts.maxSymbols : 80;
+  const maxSignatures = opts.maxSignatures != null ? opts.maxSignatures : 150;
   const parts = [];
 
+  let index = null;
   try {
-    const { extractConventions } = require('../conventions/extract');
-    const { renderConventionsBlock } = require('../conventions/inject');
-    const { loadConfig } = require('../config/loader');
-    let files = [];
-    try {
-      const cfg = loadConfig(cwd);
-      const { buildSigIndex } = require('../retrieval/ranker');
-      files = [...buildSigIndex(cwd).keys()];
-      void cfg;
-    } catch (_) {}
-    const conv = extractConventions(cwd, files);
-    parts.push(renderConventionsBlock(conv));
+    const { buildSigIndex } = require('../retrieval/ranker');
+    index = buildSigIndex(cwd);
   } catch (_) {}
 
   try {
-    const { buildSymbolSet } = require('../verify/hallucination-guard');
-    const { set } = buildSymbolSet(cwd);
-    const names = [...set].slice(0, maxSymbols);
-    if (names.length) parts.push(`## Known symbols (reference these exactly)\n${names.join(', ')}`);
+    const { extractConventions } = require('../conventions/extract');
+    const { renderConventionsBlock } = require('../conventions/inject');
+    const files = index ? [...index.keys()] : [];
+    parts.push(renderConventionsBlock(extractConventions(cwd, files)));
   } catch (_) {}
 
+  if (index) {
+    const lines = ['## Exact signatures (use these — do not invent symbols or paths)'];
+    let count = 0;
+    for (const [file, sigs] of index) {
+      if (count >= maxSignatures) break;
+      const rel = path.relative(cwd, file).replace(/\\/g, '/');
+      const clean = (sigs || []).map(_cleanSig).filter(Boolean);
+      if (!clean.length) continue;
+      lines.push(`### ${rel}`);
+      for (const s of clean) {
+        if (count >= maxSignatures) break;
+        lines.push(s);
+        count++;
+      }
+    }
+    if (count > 0) parts.push(lines.join('\n'));
+  }
+
   return parts.join('\n\n');
 }
 
 /**
- * Count flagged codebase-fact errors in an answer (the §9 metric).
+ * Score an answer: flagged codebase-fact errors + the issue list (the §9 metric).
  * @param {string} answerText
  * @param {string} cwd
- * @returns {number}
+ * @returns {{ total: number, issues: object[] }}
  */
-function scoreAnswer(answerText, cwd) {
+function scoreAnswerDetail(answerText, cwd) {
   try {
-    const { summary } = verify(String(answerText || ''), cwd);
-    return summary.total || 0;
+    const { issues, summary } = verify(String(answerText || ''), cwd);
+    return { total: summary.total || 0, issues: issues || [] };
   } catch (_) {
-    return 0;
+    return { total: 0, issues: [] };
   }
 }
 
+/** Count flagged codebase-fact errors in an answer (the §9 metric). */
+function scoreAnswer(answerText, cwd) {
+  return scoreAnswerDetail(answerText, cwd).total;
+}
+
 /**
  * Run the A/B ablation over a task corpus.
  * @param {Array<{id:string, prompt:string}>} tasks
@@ -73,6 +96,7 @@ function scoreAnswer(answerText, cwd) {
  * @param {(prompt:string, meta:object)=>string} complete injected model call
  * @param {object} [opts]
  * @param {string} [opts.grounding] precomputed grounding (else built from cwd)
+ * @param {boolean} [opts.collectIssues] attach `aIssues`/`bIssues` per task
  * @returns {{ tasks: object[], aggregate: object }}
  */
 function runAblation(tasks, cwd, complete, opts = {}) {
@@ -88,11 +112,13 @@ function runAblation(tasks, cwd, complete, opts = {}) {
     const outA = String(complete(basePrompt, { id: task.id, grounded: false }) || '');
     const outB = String(complete(groundedPrompt, { id: task.id, grounded: true }) || '');
 
-    const aFlagged = scoreAnswer(outA, cwd);
-    const bFlagged = scoreAnswer(outB, cwd);
-    sumA += aFlagged;
-    sumB += bFlagged;
-    rows.push({ id: task.id, aFlagged, bFlagged });
+    const a = scoreAnswerDetail(outA, cwd);
+    const b = scoreAnswerDetail(outB, cwd);
+    sumA += a.total;
+    sumB += b.total;
+    const row = { id: task.id, aFlagged: a.total, bFlagged: b.total };
+    if (opts.collectIssues) { row.aIssues = a.issues; row.bIssues = b.issues; }
+    rows.push(row);
   }
 
   const n = rows.length;
@@ -110,4 +136,4 @@ function runAblation(tasks, cwd, complete, opts = {}) {
   };
 }
 
-module.exports = { buildGrounding, scoreAnswer, runAblation };
+module.exports = { buildGrounding, scoreAnswer, scoreAnswerDetail, runAblation };

package/src/mcp/server.js

@@ -18,7 +18,7 @@ const { readContext, searchSignatures, getMap, createCheckpoint, getRouting, exp
 
 const SERVER_INFO = {
   name: 'sigmap',
-  version: '7.21.0',
+  version: '7.22.0',
   description: 'SigMap MCP server — code signatures on demand',
 };