sigmap 6.8.0 → 6.9.0

package/CHANGELOG.md

@@ -10,6 +10,16 @@ Format: [Semantic Versioning](https://semver.org/)
 
 ---
 
+## [6.9.0] — 2026-05-03
+
+### Added
+
+- **Task metadata for segmentation** — All 18 benchmark repositories now tagged with language, repo type (framework/library/tool/application), and size class (small/medium/large) to enable segmented benchmark analysis.
+- **Benchmark methodology documentation** — Comprehensive guide explaining what SigMap measures (retrieval accuracy, task success, prompt reduction, token reduction), why these metrics matter, and how the 90-task test set was selected and evaluated.
+- **Answer usefulness evaluation** — New metric tracking whether retrieved context actually enabled correct answers, scored in three tiers: fully-useful (rank 1), partially-useful (ranks 2-5), not-useful (not retrieved). Complements task success proxy with granular answer quality assessment.
+
+---
+
 ## [6.8.0] — 2026-05-03
 
 ### Added

package/README.md

@@ -52,7 +52,7 @@ Works with Copilot, Claude, Cursor, Windsurf, and any LLM.
 
 | Without SigMap | With SigMap |
 |---|---|
-| ❌ Guessing which files are relevant | ✅ Right file in context — 81% of the time |
+| ❌ Guessing which files are relevant | ✅ Right file in context — 80% of the time |
 | ❌ Sending the full repo to your AI | ✅ Minimal context — only what matters |
 | ❌ Embeddings / vector DB required | ✅ Grounded answers, no infra needed |
 

package/gen-context.js

@@ -5387,7 +5387,7 @@ __factories["./src/mcp/server"] = function(module, exports) {
   
   const SERVER_INFO = {
     name: 'sigmap',
-  version: '6.8.0',
+  version: '6.9.0',
     description: 'SigMap MCP server — code signatures on demand',
   };
   
@@ -7855,7 +7855,7 @@ const path = require('path');
 const os = require('os');
 const { execSync } = require('child_process');
 
-const VERSION = '6.8.0';
+const VERSION = '6.9.0';
 const MARKER = '\n\n## Auto-generated signatures\n<!-- Updated by gen-context.js -->\n';
 
 function requireSourceOrBundled(key) {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "sigmap",
-  "version": "6.8.0",
+  "version": "6.9.0",
   "description": "Zero-dependency AI context engine — 97% token reduction. No npm install. Runs on Node 18+.",
   "main": "gen-context.js",
   "exports": {

package/packages/cli/package.json

@@ -1,6 +1,6 @@
 {
   "name": "sigmap-cli",
-  "version": "6.8.0",
+  "version": "6.9.0",
   "description": "SigMap CLI wrapper — thin adapter for programmatic CLI invocation",
   "main": "index.js",
   "keywords": [

package/packages/core/package.json

@@ -1,6 +1,6 @@
 {
   "name": "sigmap-core",
-  "version": "6.8.0",
+  "version": "6.9.0",
   "description": "SigMap core library — zero-dependency code signature extraction, retrieval, and security scanning",
   "main": "index.js",
   "keywords": [

package/src/eval/usefulness-scorer.js

@@ -0,0 +1,66 @@
+'use strict';
+
+module.exports = { scoreUsefulness, computeUsefulnessStats };
+
+/**
+ * Score answer usefulness based on:
+ * 1. Whether right file was retrieved (retrieval hit)
+ * 2. Whether retrieved context covered the answer (coverage)
+ * 3. Confidence in answer quality (from ranking score)
+ */
+function scoreUsefulness(taskResult, rankingScore) {
+  const { hitRank } = taskResult;
+
+  // Tier 1: File not retrieved — context cannot be useful
+  if (hitRank === -1 || hitRank > 5) {
+    return {
+      tier: 'not-useful',
+      score: 0.0,
+      reason: 'expected file not in top 5'
+    };
+  }
+
+  // Tier 2: File retrieved but not top ranking — partially useful
+  if (hitRank > 1) {
+    return {
+      tier: 'partially-useful',
+      score: rankingScore * 0.5,  // Partial usefulness
+      reason: `file ranked #${hitRank}`
+    };
+  }
+
+  // Tier 3: File at top of ranking — fully useful
+  return {
+    tier: 'fully-useful',
+    score: rankingScore,  // Full usefulness
+    reason: 'file ranked first'
+  };
+}
+
+function computeUsefulnessStats(taskResults) {
+  const tiers = {
+    'fully-useful': 0,
+    'partially-useful': 0,
+    'not-useful': 0
+  };
+
+  let totalScore = 0;
+  let count = 0;
+
+  taskResults.forEach(result => {
+    const usefulness = scoreUsefulness(result, result.rankingScore || 1.0);
+    tiers[usefulness.tier]++;
+    totalScore += usefulness.score;
+    count++;
+  });
+
+  return {
+    fully_useful: tiers['fully-useful'],
+    partially_useful: tiers['partially-useful'],
+    not_useful: tiers['not-useful'],
+    fully_useful_pct: count > 0 ? (tiers['fully-useful'] / count * 100).toFixed(1) : 0,
+    partially_useful_pct: count > 0 ? (tiers['partially-useful'] / count * 100).toFixed(1) : 0,
+    not_useful_pct: count > 0 ? (tiers['not-useful'] / count * 100).toFixed(1) : 0,
+    average_usefulness_score: count > 0 ? (totalScore / count).toFixed(3) : 0
+  };
+}

package/src/mcp/server.js

@@ -18,7 +18,7 @@ const { readContext, searchSignatures, getMap, createCheckpoint, getRouting, exp
 
 const SERVER_INFO = {
   name: 'sigmap',
-  version: '6.8.0',
+  version: '6.9.0',
   description: 'SigMap MCP server — code signatures on demand',
 };