sigmap 8.28.1 → 8.30.0

package/llms-full.txt

@@ -11,13 +11,13 @@ ranking keeps the relevant context in scope (cutting tokens ~97% as a side
 effect), with no LLM calls, embeddings, or vector database. Works with Claude,
 Cursor, GitHub Copilot, Aider, Windsurf, local LLMs, and MCP.
 
-# Version: 8.28.1 | Benchmark: sigmap-v8.28-main (2026-08-22)
+# Version: 8.30.0 | Benchmark: sigmap-v8.30-main (2026-09-07)
 # Source: auto-generated from package.json, version.json, benchmarks/latest.json, src/mcp/tools.js, src/config/defaults.js
 # Regenerate: npm run generate:llms   |   Validate: npm run validate:llms
 
 ---
 
-## Core metrics (benchmark: sigmap-v8.28-main, 2026-08-22)
+## Core metrics (benchmark: sigmap-v8.30-main, 2026-09-07)
 
 | Metric | Without SigMap | With SigMap |
 |--------|----------------|-------------|

package/llms.txt

@@ -11,7 +11,7 @@ ranking keeps the relevant context in scope (cutting tokens ~97% as a side
 effect), with no LLM calls, embeddings, or vector database. Works with Claude,
 Cursor, GitHub Copilot, Aider, Windsurf, local LLMs, and MCP.
 
-# Version: 8.28.1 | Benchmark: sigmap-v8.28-main (2026-08-22)
+# Version: 8.30.0 | Benchmark: sigmap-v8.30-main (2026-09-07)
 # Source: auto-generated from package.json, version.json, benchmarks/latest.json, src/mcp/tools.js, src/config/defaults.js
 # Regenerate: npm run generate:llms   |   Validate: npm run validate:llms
 
@@ -23,7 +23,7 @@ Cursor, GitHub Copilot, Aider, Windsurf, local LLMs, and MCP.
 - No blast-radius awareness before editing a hub file — `--impact` shows every file a change touches.
 - Pasted stack traces, CI logs, and JSON bloat the prompt — `squeeze` minimizes them and enriches the top frame from the symbol index.
 
-## Core metrics (benchmark: sigmap-v8.28-main, 2026-08-22)
+## Core metrics (benchmark: sigmap-v8.30-main, 2026-09-07)
 
 - hit@5 retrieval: 81.1% vs 44.0% single-shot grep baseline (1.73× lift)
 - Token reduction: 96.8% average across benchmark repos

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "sigmap",
-  "version": "8.28.1",
+  "version": "8.30.0",
   "description": "The deterministic, verifiable grounding layer for AI code work — a zero-dependency signature-and-evidence map that grounds Claude, Cursor, Copilot, Aider, Windsurf, local LLMs & MCP agents against your real code (repo + installed libraries) so they stop hallucinating files, imports & APIs. Runs offline via npx; byte-stable output; ~97% token reduction as proof.",
   "main": "packages/core/index.js",
   "exports": {
@@ -15,7 +15,7 @@
   },
   "scripts": {
     "test": "node test/run.js && node test/r-language.test.js",
-    "test:integration": "node test/integration/strategy.test.js && node test/integration/secret-scan.test.js && node test/integration/token-budget.test.js && node test/integration/auto-budget.test.js && node test/integration/mcp/server.test.js && node test/integration/verify-ai-output.test.js && node test/integration/memory-tools.test.js && node test/integration/squeeze.test.js && node test/integration/context-consistency.test.js && node test/integration/features/llms-current.test.js",
+    "test:integration": "node test/integration/all.js",
     "test:integration:all": "node test/integration/all.js",
     "test:python": "python3 test/test_python_ast_extractor.py",
     "test:all": "node test/run.js && node test/r-language.test.js && node test/integration/strategy.test.js && node test/integration/secret-scan.test.js",
@@ -53,7 +53,9 @@
     "prepublishOnly": "node scripts/check-bundle.mjs && node scripts/build-bundle.mjs --check && node scripts/gen-benchmark-latest.mjs --check && node scripts/check-version-meta.mjs && node scripts/sync-metrics.mjs --check && node scripts/generate-llms.mjs",
     "benchmark:grounding": "node scripts/run-hallucination-benchmark.mjs",
     "benchmark:honest": "node scripts/run-honest-benchmark.mjs --save",
-    "benchmark:llm-ablation": "node scripts/run-llm-ablation.mjs"
+    "benchmark:llm-ablation": "node scripts/run-llm-ablation.mjs",
+    "benchmark:retrieval": "node scripts/run-retrieval-gate.mjs",
+    "validate:retrieval": "node scripts/check-corpus.mjs && node scripts/check-corpus.mjs benchmarks/tasks/retrieval-mined.jsonl && node scripts/run-retrieval-gate.mjs --gate --no-regress"
   },
   "files": [
     "gen-context.js",

package/packages/cli/package.json

@@ -1,6 +1,6 @@
 {
   "name": "sigmap-cli",
-  "version": "8.28.1",
+  "version": "8.30.0",
   "description": "SigMap CLI wrapper — thin adapter for programmatic CLI invocation",
   "main": "index.js",
   "keywords": [

package/packages/core/package.json

@@ -1,6 +1,6 @@
 {
   "name": "sigmap-core",
-  "version": "8.28.1",
+  "version": "8.30.0",
   "description": "SigMap core library — zero-dependency code signature extraction, retrieval, and security scanning",
   "main": "index.js",
   "keywords": [

package/src/eval/runner.js

@@ -38,47 +38,12 @@ const { bm25rank } = require('../retrieval/bm25');
  * @returns {Map<string, string[]>}
  */
 function buildSigIndex(cwd) {
-  const contextPath = path.join(cwd, '.github', 'copilot-instructions.md');
-  const index = new Map();
-
-  if (!fs.existsSync(contextPath)) return index;
-
-  const content = fs.readFileSync(contextPath, 'utf8');
-  const lines = content.split('\n');
-
-  let currentFile = null;
-  let inBlock = false;
-  let sigs = [];
-
-  for (const line of lines) {
-    // Section header: ### path/to/file.js
-    const headerMatch = line.match(/^###\s+(\S+\.\w+)\s*$/);
-    if (headerMatch) {
-      if (currentFile !== null) {
-        index.set(currentFile, sigs);
-      }
-      currentFile = headerMatch[1];
-      sigs = [];
-      inBlock = false;
-      continue;
-    }
-
-    if (line.startsWith('```')) {
-      inBlock = !inBlock;
-      continue;
-    }
-
-    if (inBlock && currentFile && line.trim()) {
-      sigs.push(line.trim());
-    }
-  }
-
-  // Flush last file
-  if (currentFile !== null) {
-    index.set(currentFile, sigs);
-  }
-
-  return index;
+  // Delegate to the production index builder. This used to parse
+  // .github/copilot-instructions.md directly — a second parallel implementation
+  // that saw only the token-BUDGETED view, ignored the other adapters, the
+  // hot-cold/per-module strategy splits, and the complete retrieval index. The
+  // corpus therefore scored a smaller index than `sigmap ask` actually uses.
+  return require('../retrieval/ranker').buildSigIndex(cwd);
 }
 
 // ---------------------------------------------------------------------------
@@ -96,12 +61,13 @@ const { tokenize } = require('../retrieval/bm25');
  * @param {number} topK
  * @returns {{ file: string, score: number, sigs: string[] }[]}
  */
-function rank(query, index, topK = 10) {
-  const candidates = [];
-  for (const [file, sigs] of index.entries()) {
-    candidates.push({ file, sigs });
-  }
-  return bm25rank(query, candidates).slice(0, topK);
+function rank(query, index, topK = 10, opts = {}) {
+  // Measure the ranker users actually hit. This used to call bm25rank directly,
+  // which meant the corpus scored a parallel implementation with no penalties,
+  // graph boost, recency or learned weights — so no ranking regression in
+  // src/retrieval/ranker.js could ever show up in the benchmark numbers.
+  const { rank: prodRank } = require('../retrieval/ranker');
+  return prodRank(query, index, Object.assign({ topK }, opts)).slice(0, topK);
 }
 
 // ---------------------------------------------------------------------------
@@ -188,11 +154,14 @@ function run(tasksFile, cwd, opts = {}) {
 
   // Build index once (re-used across all tasks in the same repo)
   const index = buildSigIndex(cwd);
+  // Import graph built once too — the hop-1/hop-2 boost is part of what ships.
+  let graph = null;
+  try { graph = require('../graph/builder').buildFromCwd(cwd); } catch (_) {}
 
   const taskResults = [];
   for (const task of tasks) {
-    const ranked = rank(task.query, index, topK).map((r) => r.file);
-    const topResult = rank(task.query, index, topK);
+    const topResult = rank(task.query, index, topK, { cwd, graph, learned: opts.learned });
+    const ranked = topResult.map((r) => r.file);
     const tokens = topResult.reduce((sum, r) => sum + estimateTokens(r.sigs), 0);
 
     const { hitAtK, reciprocalRank, precisionAtK } = require('./scorer');

package/src/extractors/java.js

@@ -1,6 +1,19 @@
 'use strict';
 
 const { lineAt, withAnchor } = require('./line-anchor');
+const { capWithNotice, capMembersWithNotice } = require('../util/truncate');
+
+// Class bodies are scanned to this many characters. Generated JVM sources
+// (MyBatis/JPA entities) routinely run past 10KB, so the ceiling only guards
+// against pathological input rather than trimming ordinary classes.
+const MAX_CLASS_BODY_CHARS = 200000;
+
+// Per-class member ceiling. Sits above the default `maxSigsPerFile` so the
+// caller's configured budget governs the output rather than this file.
+const MAX_MEMBERS_PER_CLASS = 120;
+
+// Per-file signature ceiling, likewise above the configured default.
+const MAX_SIGS_PER_FILE = 200;
 
 /**
  * Extract signatures from Java source code.
@@ -28,17 +41,20 @@ function extract(src) {
     const block = extractBlock(stripped, bodyStart);
     sigs.push(hinted(withAnchor(`${m[1]} ${m[2]}`, lineAt(stripped, m.index), lineAt(stripped, bodyStart + block.length)), m[2]));
     for (const meth of extractMembers(block)) {
-      sigs.push(hinted(withAnchor(`  ${meth.text}`, lineAt(stripped, bodyStart + meth.declIdx), lineAt(stripped, bodyStart + meth.endIdx)), meth.name));
+      // The disclosure marker carries no offsets; anchor it at the class body.
+      const declIdx = meth.declIdx || 0;
+      const endIdx = meth.endIdx || 0;
+      sigs.push(hinted(withAnchor(`  ${meth.text}`, lineAt(stripped, bodyStart + declIdx), lineAt(stripped, bodyStart + endIdx)), meth.name));
     }
   }
 
-  return sigs.slice(0, 25);
+  return capWithNotice(sigs, MAX_SIGS_PER_FILE, 'signatures');
 }
 
 function extractBlock(src, startIndex) {
   let depth = 1;
   let i = startIndex;
-  const end = Math.min(src.length, startIndex + 5000);
+  const end = Math.min(src.length, startIndex + MAX_CLASS_BODY_CHARS);
   while (i < end && depth > 0) {
     if (src[i] === '{') depth++;
     else if (src[i] === '}') depth--;
@@ -60,7 +76,7 @@ function extractMembers(block) {
       endIdx: m.index + m[0].length,
     });
   }
-  return members.slice(0, 8);
+  return capMembersWithNotice(members, MAX_MEMBERS_PER_CLASS);
 }
 
 function normalizeParams(params) {

package/src/graph/builder.js

@@ -12,10 +12,11 @@
 const fs   = require('fs');
 const path = require('path');
 
-// Normalize paths for cross-platform consistency (Windows uses backslashes, Unix uses forward slashes)
-// Use lowercase to enable case-insensitive lookups on case-sensitive Windows filesystems
+// Cross-platform node key. Delegates to the ONE shared definition so this graph
+// and the call-graph cannot drift apart again (see src/graph/path-key.js).
+const { graphKey } = require('./path-key');
 function normalizePath(p) {
-  return path.normalize(p).toLowerCase();
+  return graphKey(p);
 }
 
 // ---------------------------------------------------------------------------

package/src/graph/call-graph.js

@@ -38,7 +38,8 @@ const NON_CALL = new Set([
   'synchronized',
 ]);
 
-function normalizePath(p) { return path.normalize(p).toLowerCase(); }
+const { graphKey } = require('./path-key');
+function normalizePath(p) { return graphKey(p); }
 function toRel(cwd, f) { return path.relative(cwd, f).replace(/\\/g, '/'); }
 function symId(cwd, absFile, name) { return `${toRel(cwd, absFile)}#${name}`; }
 
@@ -470,8 +471,11 @@ function buildCallFileGraph(cwd, opts = {}) {
     for (const calleeId of calleeIds) {
       const calleeDef = graph.defs.get(calleeId);
       if (!calleeDef || calleeDef.file === callerDef.file) continue;
-      const a = path.resolve(cwd, callerDef.file);
-      const b = path.resolve(cwd, calleeDef.file);
+      // Keyed through graphKey so the file-level call graph shares ONE key space
+      // with the import graph. Previously this kept case while builder.js
+      // lowercased, so a lookup correct for one silently missed on the other.
+      const a = graphKey(path.resolve(cwd, callerDef.file));
+      const b = graphKey(path.resolve(cwd, calleeDef.file));
       add(a, b);
       add(b, a);
     }

package/src/graph/path-key.js

@@ -0,0 +1,26 @@
+'use strict';
+
+/**
+ * The single definition of a graph node key.
+ *
+ * src/graph/builder.js and src/graph/call-graph.js each used to normalise paths
+ * their own way — builder lowercased, call-graph did not — so a lookup written
+ * for one silently missed on the other. That divergence disabled the import
+ * boost on every repo whose path contains an uppercase letter, and later caused
+ * a "fix" for one graph to break the other. Both now key through this function,
+ * so there is one convention rather than two conventions and a convention.
+ *
+ * Lowercasing keeps lookups stable across case-insensitive filesystems (macOS,
+ * Windows), where the same file legitimately arrives spelled two ways.
+ *
+ * Zero-dependency, pure, bundle-safe.
+ */
+
+const path = require('path');
+
+/** Canonical key for a filesystem path used as a graph node. */
+function graphKey(p) {
+  return path.normalize(String(p)).toLowerCase();
+}
+
+module.exports = { graphKey };

package/src/mcp/install.js

@@ -17,6 +17,7 @@ const os   = require('os');
 
 // Config shapes the supported clients use.
 //  - 'json'  → { mcpServers: { sigmap: { command, args } } }
+//  - 'vscode'→ { servers: { sigmap: { type: 'stdio', command, args } } }
 //  - 'zed'   → { context_servers: { sigmap: { command: { path, args } } } }
 //  - 'yaml'  → Codex CLI ~/.codex/config.yaml (mcpServers block, appended)
 const CLIENTS = {
@@ -25,7 +26,7 @@ const CLIENTS = {
   windsurf: { label: 'Windsurf',     format: 'json', scope: 'both',
               project: ['.windsurf', 'mcp.json'],
               global:  ['.codeium', 'windsurf', 'mcp_config.json'] },
-  vscode:   { label: 'VS Code',      format: 'json', scope: 'project', project: ['.vscode', 'mcp.json'] },
+  vscode:   { label: 'VS Code',      format: 'vscode', scope: 'project', project: ['.vscode', 'mcp.json'] },
   opencode: { label: 'OpenCode',     format: 'json', scope: 'both',
               project: ['opencode.json'],
               global:  ['.config', 'opencode', 'config.json'] },
@@ -80,6 +81,31 @@ function _installJson(filePath, scriptPath) {
   return 'installed';
 }
 
+/**
+ * Install into VS Code's `.vscode/mcp.json`, which keys servers under `servers`
+ * (not `mcpServers`) and expects an explicit transport `type`. A config written
+ * by an older SigMap under `mcpServers` is migrated rather than left in place,
+ * so re-running repairs it instead of leaving two entries VS Code cannot read.
+ */
+function _installVscode(filePath, scriptPath) {
+  let settings = {};
+  if (fs.existsSync(filePath)) {
+    try { settings = JSON.parse(fs.readFileSync(filePath, 'utf8')) || {}; }
+    catch (_) { settings = {}; }
+  }
+  const stale = settings.mcpServers && settings.mcpServers.sigmap;
+  if (stale) {
+    delete settings.mcpServers.sigmap;
+    if (Object.keys(settings.mcpServers).length === 0) delete settings.mcpServers;
+  }
+  if (!settings.servers) settings.servers = {};
+  if (settings.servers.sigmap && !stale) return 'already';
+  settings.servers.sigmap = { type: 'stdio', command: 'node', args: serverArgs(scriptPath) };
+  fs.mkdirSync(path.dirname(filePath), { recursive: true });
+  fs.writeFileSync(filePath, JSON.stringify(settings, null, 2) + '\n');
+  return stale ? 'updated' : 'installed';
+}
+
 /** Install into Zed's `context_servers` config (create file/dir if absent). */
 function _installZed(filePath, scriptPath) {
   let settings = {};
@@ -132,7 +158,8 @@ function installClient(client, opts = {}) {
   const filePath   = resolveTarget(spec, cwd, home, opts.global);
 
   let status;
-  if (spec.format === 'zed')       status = _installZed(filePath, scriptPath);
+  if (spec.format === 'vscode')    status = _installVscode(filePath, scriptPath);
+  else if (spec.format === 'zed')  status = _installZed(filePath, scriptPath);
   else if (spec.format === 'yaml') status = _installYaml(filePath, scriptPath);
   else                             status = _installJson(filePath, scriptPath);
 

package/src/mcp/server.js

@@ -18,7 +18,7 @@ const { readContext, searchSignatures, getMap, createCheckpoint, getRouting, exp
 
 const SERVER_INFO = {
   name: 'sigmap',
-  version: '8.28.1',
+  version: '8.30.0',
   description: 'SigMap MCP server — code signatures on demand',
 };
 

package/src/retrieval/bm25.js

@@ -108,6 +108,33 @@ const EXPANSION_GROUPS = [
 // the literal query token always outranks a synonym-only match.
 const EXPANSION_WEIGHT = 0.15;
 
+// Module-doc prose is indexed as a `# module: ...` pseudo-signature (index-only,
+// see src/retrieval/module-doc.js). Per token it is a weaker relevance signal
+// than a real signature — descriptive rather than definitional — so it is scored
+// as its own BM25F field rather than pooled with the code terms.
+const MODULE_DOC_RE = /^#\s*(module|docs):/;
+
+// Line anchors are metadata, not content, and this ranker is documented as
+// anchor-invariant. The previous strip was end-anchored, so it only fired when
+// the anchor was the last thing on the line — but extractors append a doc hint
+// AFTER it ("... :27-59  # Compute a normalized centrality score"). 27% of
+// signatures therefore leaked their line numbers into the term space as tokens
+// like "27" and "59": 840 junk terms, inflating document length for exactly the
+// well-documented files, which BM25 then penalised via length normalisation.
+const ANCHOR_RE = /\s*:\d+(?:-\d+)?(?=\s|$)/g;
+
+function stripAnchor(line) {
+  return String(line).replace(ANCHOR_RE, '');
+}
+// Tuned on the 30-task leak-free corpus. The 0.5-0.8 band is flat
+// (hit@5 63.3-66.7%, easy MRR steady at 0.825); adjacent values swing by up to
+// 6.7pp, which at 30 tasks is literally two tasks — noise, not signal. 0.6 is
+// chosen from the middle of that band rather than at its peak, because a
+// per-token weight below 1 is the principled position (prose is descriptive,
+// a signature is definitional) and picking the argmax of a 30-task sweep is
+// how you overfit a benchmark.
+const DOC_WEIGHT = 0.6;
+
 // Build a stemmed lookup: stem(member) → Set of the group's other stemmed members.
 const EXPANSIONS = (() => {
   const map = new Map();
@@ -151,22 +178,45 @@ function expandQuery(qToks) {
  * @param {{ file: string, sigs: string[] }[]} candidates
  * @returns {Array<object & { score: number }>}
  */
-function bm25rank(query, candidates) {
+function bm25rank(query, candidates, opts) {
   if (!Array.isArray(candidates) || candidates.length === 0) return [];
 
   const k1 = 1.5;
   const b = 0.75;
 
+  const docWeight = (opts && typeof opts.docWeight === 'number') ? opts.docWeight : DOC_WEIGHT;
+
   const docs = candidates.map((c) => {
     const pathToks = tokenize(c.file || '');
     // Ranking is anchor-invariant: `:start-end` line anchors are metadata,
     // not content — strip them before tokenizing so adding anchors to an
     // extractor never shifts BM25 length normalization or token counts.
-    const toks = tokenize((c.sigs || []).map((s) => String(s).replace(/\s*:\d+(?:-\d+)?\s*$/, '')).join(' '));
-    for (let i = 0; i < PATH_BOOST; i++) toks.push(...pathToks);
+    // BM25F-style fields. Module-doc prose and code signatures are different
+    // kinds of evidence and must not share one term-frequency pool: prose is
+    // ~30% of all indexed tokens, and a short file with a long header (few
+    // signatures, lots of description) otherwise wins unrelated queries purely
+    // through length normalisation.
+    const docLines = [];
+    const codeLines = [];
+    for (const line of (c.sigs || [])) (MODULE_DOC_RE.test(line) ? docLines : codeLines).push(line);
+    // TRIED AND REJECTED: splitting the declared symbol NAME into its own
+    // weighted BM25F field, on the IR prior that a name is a "title" and params
+    // are "body". Swept 1.0-4.0. hit@5 on the mined corpus rose 60.9% -> 65.2%,
+    // which is a single task crossing the rank-5 line — over 113 combined tasks
+    // hit@1 fell 52.2% -> 51.3%, hit@3 fell 66.4% -> 65.5%, hit@10 was identical
+    // and MRR dropped. It moves correct answers DOWN and happens to nudge one
+    // past a cutoff. A hit@5-only view would have shipped this.
+    const codeToks = tokenize(codeLines.map((x) => stripAnchor(x)).join(' '));
+    const docToks = tokenize(docLines.join(' '));
     const tf = new Map();
-    for (const t of toks) tf.set(t, (tf.get(t) || 0) + 1);
-    return { cand: c, tf, len: toks.length };
+    const addField = (toks, weight) => { for (const t of toks) tf.set(t, (tf.get(t) || 0) + weight); };
+    addField(codeToks, 1);
+    addField(pathToks, PATH_BOOST);
+    addField(docToks, docWeight);
+    // Length accumulates with the SAME weights, or a field's influence leaks
+    // back in through the normalisation term.
+    const len = codeToks.length + (PATH_BOOST * pathToks.length) + (docWeight * docToks.length);
+    return { cand: c, tf, len };
   });
 
   const N = docs.length || 1;
@@ -195,4 +245,4 @@ function bm25rank(query, candidates) {
     .sort((a, c) => c.score - a.score || String(a.file).localeCompare(String(c.file)));
 }
 
-module.exports = { tokenize, stem, bm25rank, PATH_BOOST, STOP, expandQuery, EXPANSIONS, EXPANSION_WEIGHT };
+module.exports = { tokenize, stem, bm25rank, PATH_BOOST, STOP, expandQuery, EXPANSIONS, EXPANSION_WEIGHT, DOC_WEIGHT, MODULE_DOC_RE, stripAnchor };

package/src/retrieval/module-doc.js

@@ -0,0 +1,120 @@
+'use strict';
+
+/**
+ * Module-level documentation extractor (retrieval only).
+ *
+ * Signatures describe a file's SHAPE — names, params, return types. A module's
+ * leading comment describes its PURPOSE, in prose, using the words a person
+ * would actually search with. That prose is the bridge between a behavioural
+ * query ("what fraction of the repo made it into the output") and the code
+ * that implements it (`coverageScore(cwd, fileEntries, config)`), which shares
+ * not one token with the query.
+ *
+ * This text is added to the RETRIEVAL INDEX ONLY — never to the generated
+ * context file. The prompt artifact is token-budgeted and prose is expensive
+ * there; the index is not injected into any prompt, so it can afford the words
+ * that make a file findable.
+ *
+ * Zero-dependency, pure, bundle-safe.
+ */
+
+// Enough to characterise a module without letting one verbose header dominate
+// BM25 length normalisation for the whole corpus.
+const MAX_CHARS = 400;
+const MAX_SCAN_LINES = 60;
+
+// Legal boilerplate is high-frequency noise: it appears in many files, shares no
+// vocabulary with real queries, and would flatten idf across the corpus.
+const BOILERPLATE = /\b(copyright|licensed under|SPDX-License|all rights reserved|permission is hereby granted)\b/i;
+
+const BLOCK_LANGS = new Set(['js', 'jsx', 'ts', 'tsx', 'java', 'go', 'rs', 'kt', 'swift', 'scala', 'cs', 'php', 'dart', 'c', 'cpp', 'h']);
+const HASH_LANGS = new Set(['py', 'rb', 'r', 'sh', 'yml', 'yaml', 'toml']);
+
+function _extOf(filePath) {
+  const m = String(filePath).match(/\.([A-Za-z0-9]+)$/);
+  return m ? m[1].toLowerCase() : '';
+}
+
+/** Strip comment furniture, JSDoc tags, and markup from one raw comment line. */
+function _cleanLine(line) {
+  return String(line)
+    .replace(/^\s*[/*#-]+\s?/, '')      // leading // /* * # ---
+    .replace(/\*+\/\s*$/, '')            // trailing */
+    .replace(/^\s*@\w+.*$/, '')          // @param / @returns tag lines
+    .replace(/[*_`]/g, '')               // markdown emphasis / code ticks
+    .trim();
+}
+
+/**
+ * Extract a module's leading documentation prose.
+ *
+ * @param {string} src       file contents
+ * @param {string} filePath  used only to pick a comment syntax
+ * @returns {string} collapsed prose, capped, or '' when there is none
+ */
+function extractModuleDoc(src, filePath) {
+  if (!src || typeof src !== 'string') return '';
+  const ext = _extOf(filePath);
+  const lines = src.split('\n', MAX_SCAN_LINES);
+
+  const collected = [];
+  let inBlock = false;
+  let started = false;
+
+  for (const raw of lines) {
+    const line = raw.trim();
+    if (!started) {
+      // Skip preamble that precedes the real header comment.
+      if (!line) continue;
+      if (line.startsWith('#!')) continue;                       // shebang
+      if (/^['"]use strict['"];?$/.test(line)) continue;
+      if (/^(package|import|from|using|#include)\b/.test(line)) continue;
+    }
+
+    if (BLOCK_LANGS.has(ext) || ext === '') {
+      if (!inBlock && line.startsWith('/*')) { inBlock = true; started = true; }
+      if (inBlock) {
+        const cleaned = _cleanLine(line);
+        if (cleaned) collected.push(cleaned);
+        if (line.includes('*/')) break;
+        continue;
+      }
+      if (line.startsWith('//')) {                                // run of // lines
+        started = true;
+        const cleaned = _cleanLine(line);
+        if (cleaned) collected.push(cleaned);
+        continue;
+      }
+      if (started || collected.length) break;
+      break;                                                      // first real code — no header
+    }
+
+    if (HASH_LANGS.has(ext)) {
+      if (/^("""|''')/.test(line)) {                              // python docstring
+        started = true; inBlock = !inBlock;
+        const cleaned = _cleanLine(line.replace(/^("""|''')/, '').replace(/("""|''')$/, ''));
+        if (cleaned) collected.push(cleaned);
+        if (!inBlock) break;
+        continue;
+      }
+      if (inBlock) { const c = _cleanLine(line); if (c) collected.push(c); continue; }
+      if (line.startsWith('#')) { started = true; const c = _cleanLine(line); if (c) collected.push(c); continue; }
+      if (collected.length) break;
+      break;
+    }
+    break;
+  }
+
+  const text = collected.join(' ').replace(/\s+/g, ' ').trim();
+  if (!text || BOILERPLATE.test(text)) return '';
+  if (text.length <= MAX_CHARS) return text;
+  return text.slice(0, MAX_CHARS).replace(/\s+\S*$/, '');        // never cut mid-word
+}
+
+/** Render as an index-only pseudo-signature, or '' when there is no doc. */
+function moduleDocSig(src, filePath) {
+  const doc = extractModuleDoc(src, filePath);
+  return doc ? `# module: ${doc}` : '';
+}
+
+module.exports = { extractModuleDoc, moduleDocSig, MAX_CHARS };