sigmap 2.0.1 → 2.1.0

package/CHANGELOG.md

@@ -6,6 +6,27 @@ Format: [Semantic Versioning](https://semver.org/)
 
 ---
 
+## [2.1.0] — 2026-04-05
+
+### Added
+- **Benchmark & evaluation system** — `src/eval/runner.js` and `src/eval/scorer.js`: zero-dependency retrieval quality measurement pipeline. Computes hit@5, MRR, and precision@5 against a JSONL task file.
+- **`benchmarks/` directory structure** — `benchmarks/tasks/retrieval.jsonl` (20 tasks against SigMap's own codebase), `benchmarks/results/` (gitignored run output), `benchmarks/reports/` (human-readable summaries).
+- **`--benchmark` CLI flag** — runs retrieval through all tasks in `benchmarks/tasks/retrieval.jsonl`, prints a markdown table (Task | Query | hit@5 | RR | Tokens) plus aggregate metrics; `--benchmark --json` for machine-readable output.
+- **`--eval` CLI flag** — alias for `--benchmark`.
+- **`src/eval/scorer.js`** — pure metric functions: `hitAtK(ranked, expected, k)`, `reciprocalRank(ranked, expected)`, `precisionAtK(ranked, expected, k)`, `aggregate(results)`. Never throws.
+- **`src/eval/runner.js`** — task loader (`loadTasks`), sig-index builder (`buildSigIndex`), keyword ranker (`rank`, `tokenize`), and main `run(tasksFile, cwd)` entry point. Reads generated context file from disk; no in-memory state.
+- **`test/integration/benchmark.test.js`** — 10 integration tests covering scorer unit tests, tokenizer, task loading, empty-file edge case, metrics shape, and `--benchmark --json` CLI output.
+
+### Validation gate
+- 21/21 extractor tests passed
+- All integration suites passed (includes 10 new benchmark tests)
+- `node gen-context.js --version` → `2.1.0`
+- `node gen-context.js --benchmark` runs without error on SigMap repo
+- `node gen-context.js --benchmark --json` → valid JSON with `metrics.hitAt5`, `metrics.mrr`, `tasks` array
+- `node gen-context.js --eval --json` → same output as `--benchmark --json`
+
+---
+
 ## [2.0.0] — 2026-04-04
 
 ### Added

package/README.md

@@ -26,6 +26,7 @@
 [![Changelog](https://img.shields.io/badge/changelog-CHANGELOG.md-blue)](CHANGELOG.md)
 [![PRs Welcome](https://img.shields.io/badge/PRs-welcome-brightgreen.svg)](CONTRIBUTING.md)
 [![VS Code](https://img.shields.io/badge/VS%20Code-extension-0078d4?logo=visual-studio-code)](https://marketplace.visualstudio.com/items?itemName=manojmallick.sigmap)
+[![Open VSX](https://img.shields.io/open-vsx/v/manojmallick/sigmap?color=a251e3&label=Open%20VSX&logo=vscodium)](https://open-vsx.org/extension/manojmallick/sigmap)
 
 </div>
 
@@ -69,6 +70,9 @@ gen-context.js ──► extracts signatures from 21 languages
 AI agent session starts with full context
 ```
 
+> **Dogfooding:** SigMap runs on itself — 40 JS files, 8,600 lines of code.
+> View the generated context: [`.github/copilot-instructions.md`](.github/copilot-instructions.md)
+
 ### Token reduction at every stage
 
 | Stage | Tokens | Reduction |
@@ -606,6 +610,12 @@ function extract(src) {   // src: string → string[]
 
 ---
 
+## ⭐ Support
+
+If SigMap saves you context or API spend, a ⭐ on [GitHub](https://github.com/manojmallick/sigmap) helps others find it.
+
+---
+
 ## 📄 License
 
 MIT © 2026 [Manoj Mallick](https://github.com/manojmallick) · Made in Amsterdam 🇳🇱

package/gen-context.js

@@ -3570,6 +3570,222 @@ __factories["./src/tracking/logger"] = function(module, exports) {
   
 };
 
+// ── ./src/eval/scorer ──
+__factories["./src/eval/scorer"] = function(module, exports) {
+  'use strict';
+  
+  function firstRank(ranked, expected) {
+    if (!Array.isArray(ranked) || !Array.isArray(expected)) return Infinity;
+    const expSet = new Set(expected.map((f) => normalizePath(f)));
+    for (let i = 0; i < ranked.length; i++) {
+      if (expSet.has(normalizePath(ranked[i]))) return i + 1;
+    }
+    return Infinity;
+  }
+  
+  function normalizePath(p) {
+    return String(p).replace(/^\.\//, '').replace(/\\/g, '/');
+  }
+  
+  function hitAtK(ranked, expected, k = 5) {
+    return firstRank(ranked, expected) <= k ? 1 : 0;
+  }
+  
+  function reciprocalRank(ranked, expected) {
+    const rank = firstRank(ranked, expected);
+    return rank === Infinity ? 0 : 1 / rank;
+  }
+  
+  function precisionAtK(ranked, expected, k = 5) {
+    if (!ranked || ranked.length === 0) return 0;
+    const expSet = new Set(expected.map((f) => normalizePath(f)));
+    const topK = ranked.slice(0, k);
+    const hits = topK.filter((f) => expSet.has(normalizePath(f))).length;
+    return hits / topK.length;
+  }
+  
+  function aggregate(results, k = 5) {
+    if (!Array.isArray(results) || results.length === 0) {
+      return { hitAt5: 0, mrr: 0, precisionAt5: 0, avgTokens: 0, tasks: 0 };
+    }
+    let totalHit = 0, totalRR = 0, totalPrec = 0, totalTokens = 0;
+    for (const r of results) {
+      const ranked = r.ranked || [];
+      const expected = r.expected || [];
+      totalHit += hitAtK(ranked, expected, k);
+      totalRR += reciprocalRank(ranked, expected);
+      totalPrec += precisionAtK(ranked, expected, k);
+      totalTokens += (typeof r.tokens === 'number' ? r.tokens : 0);
+    }
+    const n = results.length;
+    return {
+      hitAt5: Math.round(totalHit / n * 1000) / 1000,
+      mrr: Math.round(totalRR / n * 1000) / 1000,
+      precisionAt5: Math.round(totalPrec / n * 1000) / 1000,
+      avgTokens: Math.round(totalTokens / n),
+      tasks: n,
+    };
+  }
+  
+  module.exports = { hitAtK, reciprocalRank, precisionAtK, aggregate, firstRank };
+};
+
+// ── ./src/eval/runner ──
+__factories["./src/eval/runner"] = function(module, exports) {
+  'use strict';
+  
+  const fs = require('fs');
+  const path = require('path');
+  const { hitAtK, reciprocalRank, precisionAtK, aggregate } = __require('./src/eval/scorer');
+  
+  function buildSigIndex(cwd) {
+    const contextPath = path.join(cwd, '.github', 'copilot-instructions.md');
+    const index = new Map();
+    if (!fs.existsSync(contextPath)) return index;
+    const content = fs.readFileSync(contextPath, 'utf8');
+    const lines = content.split('\n');
+    let currentFile = null;
+    let inBlock = false;
+    let sigs = [];
+    for (const line of lines) {
+      const headerMatch = line.match(/^###\s+(\S+\.\w+)\s*$/);
+      if (headerMatch) {
+        if (currentFile !== null) index.set(currentFile, sigs);
+        currentFile = headerMatch[1];
+        sigs = [];
+        inBlock = false;
+        continue;
+      }
+      if (line.startsWith('```')) { inBlock = !inBlock; continue; }
+      if (inBlock && currentFile && line.trim()) sigs.push(line.trim());
+    }
+    if (currentFile !== null) index.set(currentFile, sigs);
+    return index;
+  }
+  
+  function tokenize(text) {
+    if (!text) return [];
+    return text
+      .replace(/([a-z])([A-Z])/g, '$1 $2')
+      .replace(/[_\-]/g, ' ')
+      .replace(/[^\w\s]/g, ' ')
+      .toLowerCase()
+      .split(/\s+/)
+      .filter((t) => t.length > 1);
+  }
+  
+  const STOP_WORDS = new Set([
+    'the', 'a', 'an', 'in', 'of', 'to', 'for', 'and', 'or', 'is', 'are',
+    'that', 'this', 'it', 'with', 'from', 'by', 'be', 'as', 'on', 'at',
+  ]);
+  
+  function scoreFile(sigs, queryTokens) {
+    if (!sigs || sigs.length === 0) return 0;
+    const sigText = sigs.join(' ');
+    const sigTokens = new Set(tokenize(sigText));
+    let score = 0;
+    for (const qt of queryTokens) {
+      if (STOP_WORDS.has(qt)) continue;
+      if (sigTokens.has(qt)) score += 1;
+      for (const st of sigTokens) {
+        if (st !== qt && st.startsWith(qt) && qt.length >= 4) score += 0.3;
+      }
+    }
+    return score;
+  }
+  
+  function rank(query, index, topK = 10) {
+    const queryTokens = tokenize(query);
+    const scored = [];
+    for (const [file, sigs] of index.entries()) {
+      scored.push({ file, score: scoreFile(sigs, queryTokens), sigs });
+    }
+    scored.sort((a, b) => b.score - a.score || a.file.localeCompare(b.file));
+    return scored.slice(0, topK);
+  }
+  
+  function estimateTokens(sigs) {
+    return Math.ceil((sigs || []).join('\n').length / 4);
+  }
+  
+  function loadTasks(tasksFile) {
+    if (!fs.existsSync(tasksFile)) return [];
+    const lines = fs.readFileSync(tasksFile, 'utf8').split('\n');
+    const tasks = [];
+    for (const line of lines) {
+      const trimmed = line.trim();
+      if (!trimmed) continue;
+      try {
+        const obj = JSON.parse(trimmed);
+        if (obj.query && Array.isArray(obj.expected_files)) {
+          tasks.push({
+            id: obj.id || String(tasks.length + 1),
+            query: obj.query,
+            expected: obj.expected_files,
+            repo: obj.repo || '.',
+          });
+        }
+      } catch (_) {}
+    }
+    return tasks;
+  }
+  
+  function run(tasksFile, cwd, opts = {}) {
+    const topK = opts.topK || 10;
+    const resolvedTasksFile = path.isAbsolute(tasksFile)
+      ? tasksFile
+      : path.resolve(cwd, tasksFile);
+    const tasks = loadTasks(resolvedTasksFile);
+    if (tasks.length === 0) {
+      return { tasks: [], metrics: { hitAt5: 0, mrr: 0, precisionAt5: 0, avgTokens: 0, tasks: 0 } };
+    }
+    const index = buildSigIndex(cwd);
+    const taskResults = [];
+    for (const task of tasks) {
+      const ranked = rank(task.query, index, topK).map((r) => r.file);
+      const topResults = rank(task.query, index, topK);
+      const tokens = topResults.reduce((sum, r) => sum + estimateTokens(r.sigs), 0);
+      taskResults.push({
+        id: task.id,
+        query: task.query,
+        expected: task.expected,
+        ranked,
+        hit5: hitAtK(ranked, task.expected, 5),
+        rr: reciprocalRank(ranked, task.expected),
+        precAt5: precisionAtK(ranked, task.expected, 5),
+        tokens,
+      });
+    }
+    const metrics = aggregate(
+      taskResults.map((r) => ({ ranked: r.ranked, expected: r.expected, tokens: r.tokens })),
+    );
+    return { tasks: taskResults, metrics };
+  }
+  
+  function formatTable(taskResults) {
+    const header = '| Task | Query | hit@5 | RR | Tokens |';
+    const divider = '|---|---|:---:|:---:|---:|';
+    const rows = taskResults.map((r) => {
+      const q = r.query.length > 40 ? r.query.slice(0, 37) + '...' : r.query;
+      return `| ${r.id} | ${q} | ${r.hit5 ? '✓' : '✗'} | ${r.rr.toFixed(2)} | ${r.tokens} |`;
+    });
+    return [header, divider, ...rows].join('\n');
+  }
+  
+  function formatMetrics(metrics) {
+    return [
+      `[sigmap] benchmark results:`,
+      `  tasks       : ${metrics.tasks}`,
+      `  hit@5       : ${(metrics.hitAt5 * 100).toFixed(1)}%`,
+      `  MRR         : ${metrics.mrr.toFixed(3)}`,
+      `  precision@5 : ${(metrics.precisionAt5 * 100).toFixed(1)}%`,
+      `  avg tokens  : ${metrics.avgTokens}`,
+    ].join('\n');
+  }
+  
+  module.exports = { run, rank, loadTasks, buildSigIndex, formatTable, formatMetrics, tokenize };
+};
+
 
 /**
  * SigMap — gen-context.js v1.2.0
@@ -3583,7 +3799,7 @@ const path = require('path');
 const os = require('os');
 const { execSync } = require('child_process');
 
-const VERSION = '2.0.0';
+const VERSION = '2.1.0';
 const MARKER = '\n\n## Auto-generated signatures\n<!-- Updated by gen-context.js -->\n';
 
 function requireSourceOrBundled(key) {
@@ -4789,6 +5005,9 @@ Usage:
   node gen-context.js --diff                            Generate context for git-changed files only
   node gen-context.js --diff <base-ref>                 Generate context + structural diff vs base ref (e.g. main)
   node gen-context.js --diff --staged                   Generate context for staged files only
+  node gen-context.js --benchmark                       Run retrieval benchmark (benchmarks/tasks/retrieval.jsonl)
+  node gen-context.js --benchmark --json                Benchmark results as JSON
+  node gen-context.js --eval                            Alias for --benchmark
   node gen-context.js --init                            Write example config + .contextignore scaffold
   node gen-context.js --help                            Show this message
   node gen-context.js --version                         Show version
@@ -4916,6 +5135,40 @@ function main() {
     process.exit(0);
   }
 
+  if (args.includes('--benchmark') || args.includes('--eval')) {
+    try {
+      const { run, formatTable, formatMetrics } = __require('./src/eval/runner');
+      const tasksFile = path.join(cwd, 'benchmarks', 'tasks', 'retrieval.jsonl');
+      const result = run(tasksFile, cwd, { topK: 10 });
+      if (args.includes('--json')) {
+        // Compact output: strip ranked arrays to keep payload small
+        const compact = {
+          metrics: result.metrics,
+          tasks: result.tasks.map((t) => ({
+            id: t.id,
+            query: t.query,
+            hit5: t.hit5,
+            rr: t.rr,
+            precAt5: t.precAt5,
+            tokens: t.tokens,
+            top1: t.ranked[0] || null,
+          })),
+        };
+        process.stdout.write(JSON.stringify(compact) + '\n');
+      } else {
+        console.log(formatMetrics(result.metrics));
+        if (result.tasks.length > 0) {
+          console.log('');
+          console.log(formatTable(result.tasks));
+        }
+      }
+    } catch (err) {
+      console.error(`[sigmap] benchmark error: ${err.message}`);
+      process.exit(1);
+    }
+    process.exit(0);
+  }
+
   if (args.includes('--report')) {
     if (args.includes('--history')) {
       try {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "sigmap",
-  "version": "2.0.1",
+  "version": "2.1.0",
   "description": "Zero-dependency AI context engine — 97% token reduction. No npm install. Runs on Node 18+.",
   "main": "gen-context.js",
   "bin": {

package/src/eval/runner.js

@@ -0,0 +1,308 @@
+'use strict';
+
+/**
+ * SigMap benchmark runner.
+ * Zero npm dependencies.
+ *
+ * Loads evaluation tasks from a JSONL file, runs signature-based retrieval
+ * against a target repo, and returns scored results.
+ *
+ * Usage (programmatic):
+ *   const { run } = require('./src/eval/runner');
+ *   const results = run('benchmarks/tasks/retrieval.jsonl', cwd);
+ *   // results: { tasks: [...], metrics: { hitAt5, mrr, precisionAt5, avgTokens, tasks } }
+ *
+ * Usage (CLI via gen-context.js --benchmark):
+ *   node gen-context.js --benchmark
+ *   node gen-context.js --benchmark --json
+ */
+
+const fs = require('fs');
+const path = require('path');
+const { aggregate } = require('./scorer');
+
+// ---------------------------------------------------------------------------
+// Context file reader
+// ---------------------------------------------------------------------------
+
+/**
+ * Read the generated context file and build a simple signature index:
+ *   Map<filePath, string[]>  — file → list of signature strings
+ *
+ * The context file uses section headers like:
+ *   ### src/extractors/python.js
+ * followed by ``` blocks containing signatures.
+ *
+ * @param {string} cwd
+ * @returns {Map<string, string[]>}
+ */
+function buildSigIndex(cwd) {
+  const contextPath = path.join(cwd, '.github', 'copilot-instructions.md');
+  const index = new Map();
+
+  if (!fs.existsSync(contextPath)) return index;
+
+  const content = fs.readFileSync(contextPath, 'utf8');
+  const lines = content.split('\n');
+
+  let currentFile = null;
+  let inBlock = false;
+  let sigs = [];
+
+  for (const line of lines) {
+    // Section header: ### path/to/file.js
+    const headerMatch = line.match(/^###\s+(\S+\.\w+)\s*$/);
+    if (headerMatch) {
+      if (currentFile !== null) {
+        index.set(currentFile, sigs);
+      }
+      currentFile = headerMatch[1];
+      sigs = [];
+      inBlock = false;
+      continue;
+    }
+
+    if (line.startsWith('```')) {
+      inBlock = !inBlock;
+      continue;
+    }
+
+    if (inBlock && currentFile && line.trim()) {
+      sigs.push(line.trim());
+    }
+  }
+
+  // Flush last file
+  if (currentFile !== null) {
+    index.set(currentFile, sigs);
+  }
+
+  return index;
+}
+
+// ---------------------------------------------------------------------------
+// Simple keyword-based ranking (pre-retrieval layer; v2.3 adds proper ranker)
+// ---------------------------------------------------------------------------
+
+/**
+ * Tokenize a query or signature into lower-case word tokens.
+ * Splits on whitespace, punctuation, camelCase, and snake_case.
+ * @param {string} text
+ * @returns {string[]}
+ */
+function tokenize(text) {
+  if (!text) return [];
+  return text
+    // split camelCase
+    .replace(/([a-z])([A-Z])/g, '$1 $2')
+    // split snake/kebab
+    .replace(/[_\-]/g, ' ')
+    // drop non-word chars
+    .replace(/[^\w\s]/g, ' ')
+    .toLowerCase()
+    .split(/\s+/)
+    .filter((t) => t.length > 1);
+}
+
+const STOP_WORDS = new Set([
+  'the', 'a', 'an', 'in', 'of', 'to', 'for', 'and', 'or', 'is', 'are',
+  'that', 'this', 'it', 'with', 'from', 'by', 'be', 'as', 'on', 'at',
+]);
+
+/**
+ * Score a single file's signatures against a query.
+ * Returns a non-negative number; higher = more relevant.
+ * @param {string[]} sigs  - array of signature strings for this file
+ * @param {string[]} queryTokens
+ * @returns {number}
+ */
+function scoreFile(sigs, queryTokens) {
+  if (!sigs || sigs.length === 0) return 0;
+
+  const sigText = sigs.join(' ');
+  const sigTokens = new Set(tokenize(sigText));
+
+  let score = 0;
+  for (const qt of queryTokens) {
+    if (STOP_WORDS.has(qt)) continue;
+    if (sigTokens.has(qt)) score += 1;
+    // Partial match (prefix)
+    for (const st of sigTokens) {
+      if (st !== qt && st.startsWith(qt) && qt.length >= 4) score += 0.3;
+    }
+  }
+
+  return score;
+}
+
+/**
+ * Rank all files in the index against a query. Returns file paths sorted
+ * by relevance score descending. Ties are broken by file path alphabetically.
+ * @param {string} query
+ * @param {Map<string, string[]>} index
+ * @param {number} topK
+ * @returns {{ file: string, score: number, sigs: string[] }[]}
+ */
+function rank(query, index, topK = 10) {
+  const queryTokens = tokenize(query);
+  const scored = [];
+
+  for (const [file, sigs] of index.entries()) {
+    const score = scoreFile(sigs, queryTokens);
+    scored.push({ file, score, sigs });
+  }
+
+  scored.sort((a, b) => b.score - a.score || a.file.localeCompare(b.file));
+  return scored.slice(0, topK);
+}
+
+// ---------------------------------------------------------------------------
+// Token estimation
+// ---------------------------------------------------------------------------
+
+/**
+ * Estimate token count from character count (chars/4, ±5%).
+ * @param {string[]} sigs
+ * @returns {number}
+ */
+function estimateTokens(sigs) {
+  const text = (sigs || []).join('\n');
+  return Math.ceil(text.length / 4);
+}
+
+// ---------------------------------------------------------------------------
+// Task loader
+// ---------------------------------------------------------------------------
+
+/**
+ * Load tasks from a JSONL file.
+ * Each line: { id, query, expected_files, repo }
+ * Invalid or blank lines are silently skipped.
+ * @param {string} tasksFile - absolute or relative path
+ * @returns {Array<{id:string, query:string, expected:string[], repo:string}>}
+ */
+function loadTasks(tasksFile) {
+  if (!fs.existsSync(tasksFile)) return [];
+  const lines = fs.readFileSync(tasksFile, 'utf8').split('\n');
+  const tasks = [];
+  for (const line of lines) {
+    const trimmed = line.trim();
+    if (!trimmed) continue;
+    try {
+      const obj = JSON.parse(trimmed);
+      if (obj.query && Array.isArray(obj.expected_files)) {
+        tasks.push({
+          id: obj.id || String(tasks.length + 1),
+          query: obj.query,
+          expected: obj.expected_files,
+          repo: obj.repo || '.',
+        });
+      }
+    } catch {
+      // skip invalid JSON lines
+    }
+  }
+  return tasks;
+}
+
+// ---------------------------------------------------------------------------
+// Main runner
+// ---------------------------------------------------------------------------
+
+/**
+ * Run all tasks in tasksFile against the repo at cwd.
+ *
+ * @param {string} tasksFile - path to JSONL task file (absolute or relative to cwd)
+ * @param {string} cwd       - project root
+ * @param {object} [opts]
+ * @param {number} [opts.topK=10] - how many results to rank per query
+ * @returns {{
+ *   tasks: Array<{id, query, expected, ranked, hit5, rr, precAt5, tokens}>,
+ *   metrics: { hitAt5, mrr, precisionAt5, avgTokens, tasks }
+ * }}
+ */
+function run(tasksFile, cwd, opts = {}) {
+  const topK = opts.topK || 10;
+
+  // Resolve paths
+  const resolvedTasksFile = path.isAbsolute(tasksFile)
+    ? tasksFile
+    : path.resolve(cwd, tasksFile);
+
+  const tasks = loadTasks(resolvedTasksFile);
+  if (tasks.length === 0) {
+    return {
+      tasks: [],
+      metrics: { hitAt5: 0, mrr: 0, precisionAt5: 0, avgTokens: 0, tasks: 0 },
+    };
+  }
+
+  // Build index once (re-used across all tasks in the same repo)
+  const index = buildSigIndex(cwd);
+
+  const taskResults = [];
+  for (const task of tasks) {
+    const ranked = rank(task.query, index, topK).map((r) => r.file);
+    const topResult = rank(task.query, index, topK);
+    const tokens = topResult.reduce((sum, r) => sum + estimateTokens(r.sigs), 0);
+
+    const { hitAtK, reciprocalRank, precisionAtK } = require('./scorer');
+    const hit5 = hitAtK(ranked, task.expected, 5);
+    const rr = reciprocalRank(ranked, task.expected);
+    const precAt5 = precisionAtK(ranked, task.expected, 5);
+
+    taskResults.push({
+      id: task.id,
+      query: task.query,
+      expected: task.expected,
+      ranked,
+      hit5,
+      rr,
+      precAt5,
+      tokens,
+    });
+  }
+
+  const metrics = aggregate(
+    taskResults.map((r) => ({ ranked: r.ranked, expected: r.expected, tokens: r.tokens })),
+  );
+
+  return { tasks: taskResults, metrics };
+}
+
+// ---------------------------------------------------------------------------
+// Table formatter
+// ---------------------------------------------------------------------------
+
+/**
+ * Format task results as a markdown table string.
+ * @param {Array} taskResults - from run()
+ * @returns {string}
+ */
+function formatTable(taskResults) {
+  const header = '| Task | Query | hit@5 | RR | Tokens |';
+  const divider = '|---|---|:---:|:---:|---:|';
+  const rows = taskResults.map((r) => {
+    const q = r.query.length > 40 ? r.query.slice(0, 37) + '...' : r.query;
+    return `| ${r.id} | ${q} | ${r.hit5 ? '✓' : '✗'} | ${r.rr.toFixed(2)} | ${r.tokens} |`;
+  });
+  return [header, divider, ...rows].join('\n');
+}
+
+/**
+ * Format aggregate metrics as a human-readable string.
+ * @param {object} metrics - from aggregate()
+ * @returns {string}
+ */
+function formatMetrics(metrics) {
+  return [
+    `[sigmap] benchmark results:`,
+    `  tasks       : ${metrics.tasks}`,
+    `  hit@5       : ${(metrics.hitAt5 * 100).toFixed(1)}%`,
+    `  MRR         : ${metrics.mrr.toFixed(3)}`,
+    `  precision@5 : ${(metrics.precisionAt5 * 100).toFixed(1)}%`,
+    `  avg tokens  : ${metrics.avgTokens}`,
+  ].join('\n');
+}
+
+module.exports = { run, rank, loadTasks, buildSigIndex, formatTable, formatMetrics, tokenize };

package/src/eval/scorer.js

@@ -0,0 +1,126 @@
+'use strict';
+
+/**
+ * SigMap evaluation scorer.
+ * Zero npm dependencies.
+ *
+ * Computes retrieval quality metrics:
+ *   hit@k     — fraction of tasks where the correct file appears in top-k results
+ *   MRR       — mean reciprocal rank (1/rank of first correct result)
+ *   precision@k — fraction of top-k results that are correct
+ *
+ * All functions are pure and never throw.
+ */
+
+/**
+ * Return the 1-based rank of the first expected file found in the ranked
+ * result list, or Infinity if none found.
+ * @param {string[]} ranked   - Ordered list of file paths returned by the system
+ * @param {string[]} expected - List of acceptable correct files (any match counts)
+ * @returns {number}
+ */
+function firstRank(ranked, expected) {
+  if (!Array.isArray(ranked) || !Array.isArray(expected)) return Infinity;
+  const expSet = new Set(expected.map((f) => normalizePath(f)));
+  for (let i = 0; i < ranked.length; i++) {
+    if (expSet.has(normalizePath(ranked[i]))) return i + 1;
+  }
+  return Infinity;
+}
+
+/**
+ * Normalize a file path for comparison (trim leading ./, lowercase on
+ * case-insensitive platforms is intentionally NOT done — keep paths as-is
+ * but trim leading directory separators and ./).
+ * @param {string} p
+ * @returns {string}
+ */
+function normalizePath(p) {
+  return String(p).replace(/^\.\//, '').replace(/\\/g, '/');
+}
+
+/**
+ * Compute hit@k for one task.
+ * @param {string[]} ranked   - Ordered results
+ * @param {string[]} expected - Correct files
+ * @param {number}   k        - Cut-off (default 5)
+ * @returns {0|1}
+ */
+function hitAtK(ranked, expected, k = 5) {
+  return firstRank(ranked, expected) <= k ? 1 : 0;
+}
+
+/**
+ * Compute reciprocal rank for one task.
+ * @param {string[]} ranked
+ * @param {string[]} expected
+ * @returns {number} value in (0, 1]
+ */
+function reciprocalRank(ranked, expected) {
+  const rank = firstRank(ranked, expected);
+  return rank === Infinity ? 0 : 1 / rank;
+}
+
+/**
+ * Compute precision@k for one task.
+ * Fraction of the top-k results that appear in expected.
+ * @param {string[]} ranked
+ * @param {string[]} expected
+ * @param {number}   k
+ * @returns {number} value in [0, 1]
+ */
+function precisionAtK(ranked, expected, k = 5) {
+  if (!ranked || ranked.length === 0) return 0;
+  const expSet = new Set(expected.map((f) => normalizePath(f)));
+  const topK = ranked.slice(0, k);
+  const hits = topK.filter((f) => expSet.has(normalizePath(f))).length;
+  return hits / topK.length;
+}
+
+/**
+ * Aggregate metrics across all task results.
+ *
+ * @param {Array<{ranked: string[], expected: string[], tokens: number}>} results
+ * @param {number} k - cut-off (default 5)
+ * @returns {{
+ *   hitAt5: number,    // fraction [0,1]
+ *   mrr: number,       // mean reciprocal rank [0,1]
+ *   precisionAt5: number,
+ *   avgTokens: number,
+ *   tasks: number
+ * }}
+ */
+function aggregate(results, k = 5) {
+  if (!Array.isArray(results) || results.length === 0) {
+    return { hitAt5: 0, mrr: 0, precisionAt5: 0, avgTokens: 0, tasks: 0 };
+  }
+
+  let totalHit = 0;
+  let totalRR = 0;
+  let totalPrec = 0;
+  let totalTokens = 0;
+
+  for (const r of results) {
+    const ranked = r.ranked || [];
+    const expected = r.expected || [];
+    totalHit += hitAtK(ranked, expected, k);
+    totalRR += reciprocalRank(ranked, expected);
+    totalPrec += precisionAtK(ranked, expected, k);
+    totalTokens += (typeof r.tokens === 'number' ? r.tokens : 0);
+  }
+
+  const n = results.length;
+  return {
+    hitAt5: round(totalHit / n),
+    mrr: round(totalRR / n),
+    precisionAt5: round(totalPrec / n),
+    avgTokens: Math.round(totalTokens / n),
+    tasks: n,
+  };
+}
+
+function round(x) {
+  return Math.round(x * 1000) / 1000;
+}
+
+module.exports = { hitAtK, reciprocalRank, precisionAtK, aggregate, firstRank };

package/src/mcp/server.js

@@ -18,7 +18,7 @@ const { readContext, searchSignatures, getMap, createCheckpoint, getRouting, exp
 
 const SERVER_INFO = {
   name: 'sigmap',
-  version: '2.0.0',
+  version: '2.1.0',
   description: 'SigMap MCP server — code signatures on demand',
 };