sigmap 2.1.0 → 2.3.0

package/CHANGELOG.md

@@ -6,6 +6,50 @@ Format: [Semantic Versioning](https://semver.org/)
 
 ---
 
+## [2.3.0] — 2026-04-07
+
+### Added
+- **Query-aware retrieval** — `src/retrieval/tokenizer.js` and `src/retrieval/ranker.js`: zero-dependency relevance ranker that scores every file against a free-text query by exact token, symbol, prefix, path, and recency signals.
+- **`--query "<text>"` CLI flag** — ranks all context files by relevance and prints a scored table (Rank | File | Score | Sigs | Tokens) plus the top-3 signature blocks; `--query "<text>" --json` for machine-readable output; `--query "<text>" --top <n>` to limit result set.
+- **`query_context` MCP tool** — 8th MCP tool; accepts `{ query: string, topK?: number }` and returns the same ranked table as the `--query` CLI flag; live within any running MCP session.
+- **Retrieval config** — `config.retrieval.topK` (default 10) and `config.retrieval.recencyBoost` (default 1.5×) added to `src/config/defaults.js`.
+- **`test/integration/retrieval.test.js`** — 23 integration tests covering tokenizer unit tests, ranker sorting/scoring/topK/empty-query, `formatRankTable`, `formatRankJSON`, CLI `--query` flags, and MCP `query_context`.
+
+### Changed
+- `src/mcp/server.js` version bumped to `2.3.0`.
+- `test/integration/mcp-server.test.js` and `mcp-v14.test.js` updated to assert 8 tools.
+- `test/integration/analyze.test.js` version assertion updated to `2.3.0`.
+
+### Validation gate
+- 21/21 extractor unit tests passed
+- 20/20 integration suites passed (0 failures)
+- `node gen-context.js --version` → `2.3.0`
+- `node gen-context.js --query "python extractor"` → `src/extractors/python.js` in top-3
+- `node gen-context.js --query "fix secret scanning" --json` → valid JSON
+- MCP `tools/list` → 8 tools including `query_context`
+
+---
+
+## [2.2.0] — 2026-04-06
+
+### Added
+- **Diagnostics & analyze command** — `src/eval/analyzer.js`: per-file breakdown of signature count, token cost, extractor used, and test coverage status.
+- **`--analyze` CLI flag** — prints a per-file table (File | Extractor | Sigs | Tokens | Covered) across all srcDirs; respects `exclude` config.
+- **`--analyze --json` flag** — outputs the same breakdown as structured JSON (`{ files, totalSigs, totalTokens, slowFiles, fileCount }`).
+- **`--analyze --slow` flag** — re-times each extractor and flags any file whose extraction takes >50ms in the table.
+- **`--diagnose-extractors` CLI flag** — runs all 21 language extractors against `test/fixtures/` and compares output to `test/expected/`; exits non-zero if any extractor diverges, shows first diff line per failure.
+- **`test/integration/analyze.test.js`** — 14 integration tests covering `analyzeFiles`, `formatAnalysisTable`, `formatAnalysisJSON`, and all four CLI flags.
+
+### Validation gate
+- 21/21 extractor tests passed
+- All integration suites passed (19 suites, 19 passed, 0 failed — includes 14 new analyze tests)
+- `node gen-context.js --version` → `2.2.0`
+- `node gen-context.js --analyze` runs without error on SigMap repo
+- `node gen-context.js --analyze --json` → valid JSON with required keys
+- `node gen-context.js --diagnose-extractors` → exits 0 on SigMap repo
+
+---
+
 ## [2.1.0] — 2026-04-05
 
 ### Added

package/gen-context.js

@@ -2879,7 +2879,23 @@ __factories["./src/mcp/handlers"] = function(module, exports) {
     ].join('\n');
   }
   
-  module.exports = { readContext, searchSignatures, getMap, createCheckpoint, getRouting, explainFile, listModules };
+  function queryContext(args, cwd) {
+    if (!args || !args.query) return 'Missing required argument: query';
+    const contextPath = path.join(cwd, CONTEXT_FILE);
+    if (!fs.existsSync(contextPath)) return 'No context file found. Run: node gen-context.js';
+    try {
+      const { rank, buildSigIndex, formatRankTable } = __require('./src/retrieval/ranker');
+      const index = buildSigIndex(cwd);
+      if (index.size === 0) return 'No signatures indexed. Run: node gen-context.js';
+      const topK = Math.min(Math.max(1, parseInt(args.topK, 10) || 10), 25);
+      const results = rank(args.query, index, { topK });
+      return formatRankTable(results, args.query);
+    } catch (err) {
+      return `_query_context failed: ${err.message}_`;
+    }
+  }
+  
+  module.exports = { readContext, searchSignatures, getMap, createCheckpoint, getRouting, explainFile, listModules, queryContext };
 };
 
 // ── ./src/mcp/server ──
@@ -2899,7 +2915,7 @@ __factories["./src/mcp/server"] = function(module, exports) {
   
   const readline = require('readline');
   const { TOOLS } = __require('./src/mcp/tools');
-  const { readContext, searchSignatures, getMap, createCheckpoint, getRouting, explainFile, listModules } = __require('./src/mcp/handlers');
+  const { readContext, searchSignatures, getMap, createCheckpoint, getRouting, explainFile, listModules, queryContext } = __require('./src/mcp/handlers');
   
   const SERVER_INFO = {
     name: 'sigmap',
@@ -2958,6 +2974,7 @@ __factories["./src/mcp/server"] = function(module, exports) {
         else if (name === 'get_routing') text = getRouting(args, cwd);
         else if (name === 'explain_file') text = explainFile(args, cwd);
         else if (name === 'list_modules') text = listModules(args, cwd);
+        else if (name === 'query_context') text = queryContext(args, cwd);
         else {
           respondError(id, -32601, `Unknown tool: ${name}`);
           return;
@@ -3137,6 +3154,30 @@ __factories["./src/mcp/tools"] = function(module, exports) {
         required: [],
       },
     },
+    {
+      name: 'query_context',
+      description:
+        'Rank and return the most relevant files for a specific task or question. ' +
+        'Uses keyword + symbol + path scoring to surface only the top-K files relevant ' +
+        'to the query — much cheaper than reading all context. ' +
+        'Returns ranked file list with signatures and relevance scores.',
+      inputSchema: {
+        type: 'object',
+        properties: {
+          query: {
+            type: 'string',
+            description:
+              'Natural language task description or keyword(s) to rank files against. ' +
+              'E.g. "add a new language extractor", "fix secret scanning", "auth module".',
+          },
+          topK: {
+            type: 'number',
+            description: 'Maximum number of files to return (default: 10, max: 25).',
+          },
+        },
+        required: ['query'],
+      },
+    },
   ];
   
   module.exports = { TOOLS };
@@ -3570,6 +3611,120 @@ __factories["./src/tracking/logger"] = function(module, exports) {
   
 };
 
+// ── ./src/retrieval/tokenizer ──
+__factories["./src/retrieval/tokenizer"] = function(module, exports) {
+  'use strict';
+  const STOP_WORDS = new Set([
+    'the', 'a', 'an', 'in', 'of', 'to', 'for', 'and', 'or', 'is', 'are',
+    'that', 'this', 'it', 'with', 'from', 'by', 'be', 'as', 'on', 'at',
+    'do', 'not', 'use', 'get', 'set', 'up', 'if', 'no', 'so', 'we',
+  ]);
+  function tokenize(text, opts) {
+    if (!text || typeof text !== 'string') return [];
+    const removeStop = opts && opts.removeStopWords === false ? false : true;
+    const minLen = (opts && opts.minLength) || 2;
+    const tokens = text
+      .replace(/\.\w{1,6}(?=\s|\/|$)/g, ' ')
+      .replace(/([a-z])([A-Z])/g, '$1 $2')
+      .replace(/([A-Z]+)([A-Z][a-z])/g, '$1 $2')
+      .replace(/[_\-\.\/]/g, ' ')
+      .replace(/[^\w\s]/g, ' ')
+      .toLowerCase()
+      .split(/\s+/)
+      .filter((t) => t.length >= minLen);
+    if (!removeStop) return [...new Set(tokens)];
+    return [...new Set(tokens.filter((t) => !STOP_WORDS.has(t)))];
+  }
+  module.exports = { tokenize, STOP_WORDS };
+};
+
+// ── ./src/retrieval/ranker ──
+__factories["./src/retrieval/ranker"] = function(module, exports) {
+  'use strict';
+  const { tokenize, STOP_WORDS } = __require('./src/retrieval/tokenizer');
+  const DEFAULT_WEIGHTS = {
+    exactToken: 1.0, symbolMatch: 0.5, prefixMatch: 0.3, pathMatch: 0.8, recencyBoost: 1.5,
+  };
+  function scoreFile(filePath, sigs, queryTokens, weights) {
+    if (!sigs || sigs.length === 0) return 0;
+    const w = weights || DEFAULT_WEIGHTS;
+    const sigTokenSet = new Set(tokenize(sigs.join(' ')));
+    const pathTokenSet = new Set(tokenize(filePath));
+    let score = 0;
+    for (const qt of queryTokens) {
+      if (STOP_WORDS.has(qt)) continue;
+      if (sigTokenSet.has(qt)) {
+        score += w.exactToken;
+        if (sigs.some((sig) => tokenize(sig.replace(/[^a-zA-Z0-9_\s]/g, ' ')).includes(qt))) score += w.symbolMatch;
+      }
+      if (qt.length >= 4) {
+        for (const st of sigTokenSet) {
+          if (st !== qt && st.startsWith(qt)) { score += w.prefixMatch; break; }
+        }
+      }
+      if (pathTokenSet.has(qt)) score += w.pathMatch;
+    }
+    return score;
+  }
+  function rank(query, sigIndex, opts) {
+    if (!query || typeof query !== 'string') return [];
+    if (!sigIndex || !(sigIndex instanceof Map) || sigIndex.size === 0) return [];
+    const topK = (opts && opts.topK) || 10;
+    const recencyMultiplier = (opts && opts.recencyBoost) || DEFAULT_WEIGHTS.recencyBoost;
+    const recencySet = (opts && opts.recencySet) || null;
+    const weights = (opts && opts.weights) ? Object.assign({}, DEFAULT_WEIGHTS, opts.weights) : DEFAULT_WEIGHTS;
+    const queryTokens = tokenize(query);
+    if (queryTokens.length === 0) {
+      const all = [];
+      for (const [file, sigs] of sigIndex.entries()) all.push({ file, score: sigs.length, sigs, tokens: Math.ceil(sigs.join('\n').length / 4) });
+      all.sort((a, b) => b.score - a.score || a.file.localeCompare(b.file));
+      return all.slice(0, topK);
+    }
+    const scored = [];
+    for (const [file, sigs] of sigIndex.entries()) {
+      let score = scoreFile(file, sigs, queryTokens, weights);
+      if (recencySet && recencySet.has(file) && score > 0) score *= recencyMultiplier;
+      scored.push({ file, score, sigs, tokens: Math.ceil(sigs.join('\n').length / 4) });
+    }
+    scored.sort((a, b) => b.score - a.score || a.file.localeCompare(b.file));
+    return scored.slice(0, topK);
+  }
+  function buildSigIndex(cwd) {
+    const fs = require('fs'); const path = require('path');
+    const contextPath = path.join(cwd, '.github', 'copilot-instructions.md');
+    const index = new Map();
+    if (!fs.existsSync(contextPath)) return index;
+    const content = fs.readFileSync(contextPath, 'utf8');
+    const lines = content.split('\n');
+    let currentFile = null; let inBlock = false; let sigs = [];
+    for (const line of lines) {
+      const hm = line.match(/^###\s+(\S+)\s*$/);
+      if (hm) { if (currentFile !== null) index.set(currentFile, sigs); currentFile = hm[1]; sigs = []; inBlock = false; continue; }
+      if (line.startsWith('```')) { inBlock = !inBlock; continue; }
+      if (inBlock && currentFile && line.trim()) sigs.push(line.trim());
+    }
+    if (currentFile !== null) index.set(currentFile, sigs);
+    return index;
+  }
+  function formatRankTable(results, query) {
+    if (!results || results.length === 0) return `No matching files found for query: "${query}"\n`;
+    const lines = [`## Query: ${query}`, '', '| Rank | File | Score | Sigs | Tokens |', '|------|------|-------|------|--------|',
+      ...results.map((r, i) => `| ${i + 1} | ${r.file} | ${r.score.toFixed(2)} | ${r.sigs.length} | ${r.tokens} |`), ''];
+    for (const r of results.slice(0, 3)) {
+      if (r.sigs.length > 0) {
+        lines.push(`### ${r.file}`, '```', ...r.sigs.slice(0, 10));
+        if (r.sigs.length > 10) lines.push(`... (${r.sigs.length - 10} more)`);
+        lines.push('```', '');
+      }
+    }
+    return lines.join('\n');
+  }
+  function formatRankJSON(results, query) {
+    return { query, results: (results || []).map((r, i) => ({ rank: i + 1, file: r.file, score: r.score, sigs: r.sigs, tokens: r.tokens })), totalResults: (results || []).length };
+  }
+  module.exports = { rank, buildSigIndex, scoreFile, formatRankTable, formatRankJSON, DEFAULT_WEIGHTS };
+};
+
 // ── ./src/eval/scorer ──
 __factories["./src/eval/scorer"] = function(module, exports) {
   'use strict';
@@ -3630,6 +3785,143 @@ __factories["./src/eval/scorer"] = function(module, exports) {
   module.exports = { hitAtK, reciprocalRank, precisionAtK, aggregate, firstRank };
 };
 
+// ── ./src/eval/analyzer ──
+__factories["./src/eval/analyzer"] = function(module, exports) {
+  'use strict';
+
+  const fs   = require('fs');
+  const path = require('path');
+
+  const EXT_MAP = {
+    '.ts': 'typescript', '.tsx': 'typescript',
+    '.js': 'javascript', '.jsx': 'javascript', '.mjs': 'javascript', '.cjs': 'javascript',
+    '.py': 'python',     '.pyw': 'python',
+    '.java': 'java',
+    '.kt': 'kotlin',     '.kts': 'kotlin',
+    '.go': 'go',
+    '.rs': 'rust',
+    '.cs': 'csharp',
+    '.cpp': 'cpp', '.c': 'cpp', '.h': 'cpp', '.hpp': 'cpp', '.cc': 'cpp',
+    '.rb': 'ruby',       '.rake': 'ruby',
+    '.php': 'php',
+    '.swift': 'swift',
+    '.dart': 'dart',
+    '.scala': 'scala',   '.sc': 'scala',
+    '.vue': 'vue',
+    '.svelte': 'svelte',
+    '.html': 'html',     '.htm': 'html',
+    '.css': 'css',       '.scss': 'css', '.sass': 'css', '.less': 'css',
+    '.yml': 'yaml',      '.yaml': 'yaml',
+    '.sh': 'shell',      '.bash': 'shell', '.zsh': 'shell', '.fish': 'shell',
+  };
+
+  function isDockerfile(name) { return name === 'Dockerfile' || name.startsWith('Dockerfile.'); }
+
+  function getExtractorName(filePath) {
+    const base = path.basename(filePath);
+    const ext  = path.extname(base).toLowerCase();
+    if (EXT_MAP[ext]) return EXT_MAP[ext];
+    if (isDockerfile(base)) return 'dockerfile';
+    return null;
+  }
+
+  function tokenCount(sigs) {
+    return Math.ceil(sigs.reduce((sum, s) => sum + s.length, 0) / 4);
+  }
+
+  function hasCoverage(filePath, cwd) {
+    const base = path.basename(filePath, path.extname(filePath));
+    const testDirs = ['test', 'tests', '__tests__', 'spec'];
+    for (const d of testDirs) {
+      const abs = path.join(cwd, d);
+      if (!fs.existsSync(abs)) continue;
+      let entries;
+      try { entries = fs.readdirSync(abs, { withFileTypes: true }); } catch (_) { continue; }
+      for (const e of entries) { if (e.name.includes(base)) return true; }
+    }
+    return false;
+  }
+
+  function analyzeFiles(files, cwd, opts) {
+    const slow    = (opts && opts.slow)    || false;
+    const slowMs  = (opts && opts.slowMs)  || 50;
+    const maxSigs = (opts && opts.maxSigs) || 25;
+    const stats   = [];
+    const cache   = {};
+
+    for (const filePath of files) {
+      const extractorName = getExtractorName(filePath);
+      if (!extractorName) continue;
+      if (!cache[extractorName]) {
+        try { cache[extractorName] = __require(`./src/extractors/${extractorName}`); } catch (_) { cache[extractorName] = null; }
+      }
+      const extractor = cache[extractorName];
+      if (!extractor || typeof extractor.extract !== 'function') continue;
+
+      let content;
+      try { content = fs.readFileSync(filePath, 'utf8'); } catch (_) { continue; }
+
+      let sigs; let elapsedMs = 0;
+      if (slow) {
+        const t0 = Date.now();
+        try { sigs = extractor.extract(content); } catch (_) { sigs = []; }
+        elapsedMs = Date.now() - t0;
+      } else {
+        try { sigs = extractor.extract(content); } catch (_) { sigs = []; }
+      }
+      sigs = (Array.isArray(sigs) ? sigs : []).slice(0, maxSigs);
+
+      stats.push({
+        file:      path.relative(cwd, filePath),
+        extractor: extractorName,
+        sigs:      sigs.length,
+        tokens:    tokenCount(sigs),
+        covered:   hasCoverage(filePath, cwd),
+        elapsedMs: slow ? elapsedMs : undefined,
+        slow:      slow ? (elapsedMs > slowMs) : undefined,
+      });
+    }
+    return stats;
+  }
+
+  function formatAnalysisTable(stats, showSlow) {
+    if (!stats || stats.length === 0) return '_(no files analyzed)_\n';
+    const maxFile = Math.max(4, ...stats.map((s) => s.file.length));
+    const header  = showSlow
+      ? `| ${'File'.padEnd(maxFile)} | Sigs | Tokens | Extractor   | Coverage   | Elapsed  |`
+      : `| ${'File'.padEnd(maxFile)} | Sigs | Tokens | Extractor   | Coverage   |`;
+    const sep = showSlow
+      ? `|${'-'.repeat(maxFile + 2)}|------|--------|-------------|------------|----------|`
+      : `|${'-'.repeat(maxFile + 2)}|------|--------|-------------|------------|`;
+    const rows = stats.map((s) => {
+      const cov  = s.covered ? '✓ tested  ' : '✗ untested';
+      const file = s.file.padEnd(maxFile);
+      const ext  = (s.extractor || '').padEnd(11);
+      const base = `| ${file} | ${String(s.sigs).padStart(4)} | ${String(s.tokens).padStart(6)} | ${ext} | ${cov} |`;
+      if (showSlow) {
+        const ms   = s.elapsedMs !== undefined ? `${s.elapsedMs}ms` : '';
+        return `${base} ${ms.padStart(6)}${s.slow ? ' ⚠️' : ''} |`;
+      }
+      return base;
+    });
+    const totalSigs   = stats.reduce((n, s) => n + s.sigs,   0);
+    const totalTokens = stats.reduce((n, s) => n + s.tokens, 0);
+    const slotFile    = ''.padEnd(maxFile);
+    const baseFoot    = `| ${slotFile} | ${String(totalSigs).padStart(4)} | ${String(totalTokens).padStart(6)} | **Total**   |            |`;
+    const footer      = showSlow ? `${baseFoot} ${' '.padStart(8)} |` : baseFoot;
+    return [header, sep, ...rows, sep, footer].join('\n') + '\n';
+  }
+
+  function formatAnalysisJSON(stats) {
+    const totalSigs   = stats.reduce((n, s) => n + s.sigs,   0);
+    const totalTokens = stats.reduce((n, s) => n + s.tokens, 0);
+    const slowFiles   = stats.filter((s) => s.slow).map((s) => ({ file: s.file, elapsedMs: s.elapsedMs }));
+    return { files: stats, totalSigs, totalTokens, slowFiles, fileCount: stats.length };
+  }
+
+  module.exports = { analyzeFiles, formatAnalysisTable, formatAnalysisJSON };
+};
+
 // ── ./src/eval/runner ──
 __factories["./src/eval/runner"] = function(module, exports) {
   'use strict';
@@ -3799,7 +4091,7 @@ const path = require('path');
 const os = require('os');
 const { execSync } = require('child_process');
 
-const VERSION = '2.1.0';
+const VERSION = '2.3.0';
 const MARKER = '\n\n## Auto-generated signatures\n<!-- Updated by gen-context.js -->\n';
 
 function requireSourceOrBundled(key) {
@@ -5008,6 +5300,13 @@ Usage:
   node gen-context.js --benchmark                       Run retrieval benchmark (benchmarks/tasks/retrieval.jsonl)
   node gen-context.js --benchmark --json                Benchmark results as JSON
   node gen-context.js --eval                            Alias for --benchmark
+  node gen-context.js --analyze                         Per-file breakdown: sigs, tokens, extractor, coverage
+  node gen-context.js --analyze --json                  Breakdown as JSON
+  node gen-context.js --analyze --slow                  Re-time each extractor; flag files >50ms
+  node gen-context.js --diagnose-extractors             Run all 21 extractors vs fixtures; show pass/fail + diff
+  node gen-context.js --query "<text>"                  Rank files by relevance to a query
+  node gen-context.js --query "<text>" --json           Ranked results as JSON
+  node gen-context.js --query "<text>" --top <n>        Limit results to top N files (default 10)
   node gen-context.js --init                            Write example config + .contextignore scaffold
   node gen-context.js --help                            Show this message
   node gen-context.js --version                         Show version
@@ -5169,6 +5468,163 @@ function main() {
     process.exit(0);
   }
 
+  if (args.includes('--analyze')) {
+    try {
+      const { analyzeFiles, formatAnalysisTable, formatAnalysisJSON } = requireSourceOrBundled('./src/eval/analyzer');
+      const cfg    = config || {};
+      const srcDirs = cfg.srcDirs || DEFAULTS.srcDirs;
+      const exclude = cfg.exclude || DEFAULTS.exclude;
+      const slow    = args.includes('--slow');
+
+      // Collect files (reuse existing file-walker if accessible, else inline)
+      const allFiles = [];
+      function walkForAnalyze(dir, depth) {
+        if (depth > (cfg.maxDepth || 6)) return;
+        let entries;
+        try { entries = fs.readdirSync(dir, { withFileTypes: true }); } catch (_) { return; }
+        for (const e of entries) {
+          if (exclude.some((x) => e.name === x || e.name.startsWith(x))) continue;
+          const full = path.join(dir, e.name);
+          if (e.isDirectory()) walkForAnalyze(full, depth + 1);
+          else if (e.isFile()) allFiles.push(full);
+        }
+      }
+      for (const sd of srcDirs) {
+        const abs = path.join(cwd, sd);
+        if (fs.existsSync(abs)) walkForAnalyze(abs, 0);
+      }
+
+      const stats = analyzeFiles(allFiles, cwd, { slow, maxSigs: cfg.maxSigsPerFile || 25 });
+
+      if (args.includes('--json')) {
+        process.stdout.write(JSON.stringify(formatAnalysisJSON(stats)) + '\n');
+      } else {
+        const table = formatAnalysisTable(stats, slow);
+        process.stdout.write(table);
+      }
+    } catch (err) {
+      console.error(`[sigmap] analyze error: ${err.message}`);
+      process.exit(1);
+    }
+    process.exit(0);
+  }
+
+  if (args.includes('--diagnose-extractors')) {
+    try {
+      const fixturesDir = path.join(cwd, 'test', 'fixtures');
+      const expectedDir = path.join(cwd, 'test', 'expected');
+      if (!fs.existsSync(fixturesDir) || !fs.existsSync(expectedDir)) {
+        console.error('[sigmap] test/fixtures or test/expected not found — run from the SigMap repo root');
+        process.exit(1);
+      }
+
+      const EXT_TO_LANG = {
+        '.ts': 'typescript', '.js': 'javascript', '.py': 'python',
+        '.java': 'java', '.kt': 'kotlin', '.go': 'go', '.rs': 'rust',
+        '.cs': 'csharp', '.cpp': 'cpp', '.rb': 'ruby', '.php': 'php',
+        '.swift': 'swift', '.dart': 'dart', '.scala': 'scala',
+        '.vue': 'vue', '.svelte': 'svelte', '.html': 'html',
+        '.css': 'css', '.yml': 'yaml', '.sh': 'shell',
+      };
+      const SPECIAL = { 'Dockerfile': 'dockerfile' };
+
+      let passed = 0; let failed = 0;
+      const entries = fs.readdirSync(fixturesDir).sort();
+
+      for (const filename of entries) {
+        const ext  = path.extname(filename).toLowerCase();
+        const lang = EXT_TO_LANG[ext] || SPECIAL[filename];
+        if (!lang) continue;
+
+        const fixturePath = path.join(fixturesDir, filename);
+        const expectedPath = path.join(expectedDir, `${lang}.txt`);
+        if (!fs.existsSync(expectedPath)) {
+          console.log(`  SKIP  ${lang.padEnd(12)} (no expected file)`);
+          continue;
+        }
+
+        const src      = fs.readFileSync(fixturePath, 'utf8');
+        const expected = fs.readFileSync(expectedPath, 'utf8').trim();
+
+        let mod;
+        try {
+          mod = requireSourceOrBundled(`./src/extractors/${lang}`);
+        } catch (e) {
+          console.log(`  ERROR ${lang.padEnd(12)} loader failed: ${e.message}`);
+          failed++;
+          continue;
+        }
+
+        let actual;
+        try {
+          const sigs = mod.extract(src);
+          actual = sigs.join('\n').trim();
+        } catch (e) {
+          console.log(`  ERROR ${lang.padEnd(12)} extract() threw: ${e.message}`);
+          failed++;
+          continue;
+        }
+
+        if (actual === expected) {
+          console.log(`  PASS  ${lang}`);
+          passed++;
+        } else {
+          console.log(`  FAIL  ${lang}`);
+          // Show first diff line
+          const aLines = actual.split('\n');
+          const eLines = expected.split('\n');
+          const maxLen = Math.max(aLines.length, eLines.length);
+          for (let i = 0; i < maxLen; i++) {
+            if (aLines[i] !== eLines[i]) {
+              console.log(`         expected: ${(eLines[i] || '(missing)').slice(0, 100)}`);
+              console.log(`         actual  : ${(aLines[i] || '(missing)').slice(0, 100)}`);
+              break;
+            }
+          }
+          failed++;
+        }
+      }
+
+      console.log(`\n${passed} passed, ${failed} failed`);
+      process.exit(failed > 0 ? 1 : 0);
+    } catch (err) {
+      console.error(`[sigmap] diagnose error: ${err.message}`);
+      process.exit(1);
+    }
+  }
+
+  if (args.includes('--query')) {
+    try {
+      const qIdx = args.indexOf('--query');
+      const query = (args[qIdx + 1] || '').trim();
+      if (!query || query.startsWith('--')) {
+        console.error('[sigmap] --query requires a search string');
+        console.error('  Example: node gen-context.js --query "add a new language extractor"');
+        process.exit(1);
+      }
+      const { rank, buildSigIndex, formatRankTable, formatRankJSON } = requireSourceOrBundled('./src/retrieval/ranker');
+      const index = buildSigIndex(cwd);
+      if (index.size === 0) {
+        console.error('[sigmap] no context file found. Run: node gen-context.js');
+        process.exit(1);
+      }
+      const topIdx = args.indexOf('--top');
+      const topK = topIdx >= 0 ? Math.min(Math.max(1, parseInt(args[topIdx + 1], 10) || 10), 25)
+                               : ((config && config.retrieval && config.retrieval.topK) || 10);
+      const recencyBoost = (config && config.retrieval && config.retrieval.recencyBoost) || 1.5;
+      const results = rank(query, index, { topK, recencyBoost });
+      if (args.includes('--json')) {
+        process.stdout.write(JSON.stringify(formatRankJSON(results, query)) + '\n');
+      } else {
+        process.stdout.write(formatRankTable(results, query));
+      }
+    } catch (err) {
+      console.error(`[sigmap] query error: ${err.message}`);
+      process.exit(1);
+    }
+    process.exit(0);
+  }
+
   if (args.includes('--report')) {
     if (args.includes('--history')) {
       try {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "sigmap",
-  "version": "2.1.0",
+  "version": "2.3.0",
   "description": "Zero-dependency AI context engine — 97% token reduction. No npm install. Runs on Node 18+.",
   "main": "gen-context.js",
   "bin": {

package/src/config/defaults.js

@@ -92,6 +92,14 @@ const DEFAULTS = {
 
   // Add reverse dependency usage hints on file headings (opt-in)
   impactRadius: false,
+
+  // Query-aware retrieval settings (v2.3)
+  retrieval: {
+    // Maximum number of files to return for --query
+    topK: 10,
+    // Multiplier applied to recently-changed files (>1 boosts them up)
+    recencyBoost: 1.5,
+  },
 };
 
 module.exports = { DEFAULTS };

package/src/eval/analyzer.js

@@ -0,0 +1,221 @@
+'use strict';
+
+/**
+ * SigMap file analyzer — per-file diagnostic statistics.
+ * Zero npm dependencies.
+ *
+ * Exports:
+ *   analyzeFiles(files, cwd, opts) → stats[]
+ *   formatAnalysisTable(stats)    → markdown table string
+ *   formatAnalysisJSON(stats)     → plain object suitable for JSON.stringify
+ */
+
+const fs   = require('fs');
+const path = require('path');
+
+// Extension → extractor name (mirrors EXT_MAP in gen-context.js)
+const EXT_MAP = {
+  '.ts': 'typescript', '.tsx': 'typescript',
+  '.js': 'javascript', '.jsx': 'javascript', '.mjs': 'javascript', '.cjs': 'javascript',
+  '.py': 'python',     '.pyw': 'python',
+  '.java': 'java',
+  '.kt': 'kotlin',     '.kts': 'kotlin',
+  '.go': 'go',
+  '.rs': 'rust',
+  '.cs': 'csharp',
+  '.cpp': 'cpp', '.c': 'cpp', '.h': 'cpp', '.hpp': 'cpp', '.cc': 'cpp',
+  '.rb': 'ruby',       '.rake': 'ruby',
+  '.php': 'php',
+  '.swift': 'swift',
+  '.dart': 'dart',
+  '.scala': 'scala',   '.sc': 'scala',
+  '.vue': 'vue',
+  '.svelte': 'svelte',
+  '.html': 'html',     '.htm': 'html',
+  '.css': 'css',       '.scss': 'css', '.sass': 'css', '.less': 'css',
+  '.yml': 'yaml',      '.yaml': 'yaml',
+  '.sh': 'shell',      '.bash': 'shell', '.zsh': 'shell', '.fish': 'shell',
+};
+
+function isDockerfile(name) {
+  return name === 'Dockerfile' || name.startsWith('Dockerfile.');
+}
+
+function getExtractorName(filePath) {
+  const base = path.basename(filePath);
+  const ext  = path.extname(base).toLowerCase();
+  if (EXT_MAP[ext]) return EXT_MAP[ext];
+  if (isDockerfile(base)) return 'dockerfile';
+  return null;
+}
+
+/** Rough token estimate: chars / 4 */
+function tokenCount(sigs) {
+  return Math.ceil(sigs.reduce((sum, s) => sum + s.length, 0) / 4);
+}
+
+/**
+ * Check whether a test file exists for this source file by looking for
+ * *.test.* / *.spec.* patterns in the test/ directory tree.
+ */
+function hasCoverage(filePath, cwd) {
+  const rel   = path.relative(cwd, filePath);
+  const base  = path.basename(rel, path.extname(rel));  // e.g. "python"
+  const testDirs = ['test', 'tests', '__tests__', 'spec'];
+  for (const d of testDirs) {
+    const abs = path.join(cwd, d);
+    if (!fs.existsSync(abs)) continue;
+    // Walk only one depth for speed
+    let entries;
+    try { entries = fs.readdirSync(abs, { withFileTypes: true }); } catch (_) { continue; }
+    for (const e of entries) {
+      if (e.name.includes(base)) return true;
+    }
+  }
+  return false;
+}
+
+/**
+ * Load an extractor module from src/extractors/ relative to cwd.
+ * Falls back to requiring from the module directory itself.
+ */
+function loadExtractor(name, cwd) {
+  // Try repo-local src/extractors first (for projects that embed sigmap)
+  const local = path.join(cwd, 'src', 'extractors', `${name}.js`);
+  if (fs.existsSync(local)) {
+    try { return require(local); } catch (_) {}
+  }
+  // Then standard node resolution from the current package
+  try { return require(path.join(__dirname, '..', 'extractors', `${name}.js`)); } catch (_) {}
+  return null;
+}
+
+/**
+ * Analyze a list of absolute file paths.
+ *
+ * @param {string[]} files   - absolute paths to analyze
+ * @param {string}   cwd     - project root
+ * @param {object}  [opts]
+ * @param {boolean} [opts.slow=false]  - if true, measure extraction time per file
+ * @param {number}  [opts.slowMs=50]   - threshold (ms) before a file is "slow"
+ * @param {number}  [opts.maxSigs=25]  - max sigs per file
+ * @returns {object[]} array of per-file stat objects
+ */
+function analyzeFiles(files, cwd, opts) {
+  const slow    = (opts && opts.slow)   || false;
+  const slowMs  = (opts && opts.slowMs) || 50;
+  const maxSigs = (opts && opts.maxSigs) || 25;
+
+  const stats = [];
+  const extractorCache = {};
+
+  for (const filePath of files) {
+    const extractorName = getExtractorName(filePath);
+    if (!extractorName) continue;
+
+    // Load extractor (cached)
+    if (!extractorCache[extractorName]) {
+      extractorCache[extractorName] = loadExtractor(extractorName, cwd);
+    }
+    const extractor = extractorCache[extractorName];
+    if (!extractor || typeof extractor.extract !== 'function') continue;
+
+    let content;
+    try { content = fs.readFileSync(filePath, 'utf8'); } catch (_) { continue; }
+
+    let sigs;
+    let elapsedMs = 0;
+
+    if (slow) {
+      const t0 = Date.now();
+      try { sigs = extractor.extract(content); } catch (_) { sigs = []; }
+      elapsedMs = Date.now() - t0;
+    } else {
+      try { sigs = extractor.extract(content); } catch (_) { sigs = []; }
+    }
+
+    sigs = (Array.isArray(sigs) ? sigs : []).slice(0, maxSigs);
+
+    const rel      = path.relative(cwd, filePath);
+    const tokens   = tokenCount(sigs);
+    const covered  = hasCoverage(filePath, cwd);
+    const isSlow   = slow && elapsedMs > slowMs;
+
+    stats.push({
+      file:          rel,
+      extractor:     extractorName,
+      sigs:          sigs.length,
+      tokens,
+      covered,
+      elapsedMs:     slow ? elapsedMs : undefined,
+      slow:          slow ? isSlow : undefined,
+    });
+  }
+
+  return stats;
+}
+
+/**
+ * Format stats as a markdown table.
+ *
+ * @param {object[]} stats - output of analyzeFiles()
+ * @param {boolean}  showSlow - whether to include the Elapsed column
+ * @returns {string}
+ */
+function formatAnalysisTable(stats, showSlow) {
+  if (!stats || stats.length === 0) return '_(no files analyzed)_\n';
+
+  // Column widths
+  const maxFile = Math.max(4, ...stats.map((s) => s.file.length));
+
+  const header = showSlow
+    ? `| ${'File'.padEnd(maxFile)} | Sigs | Tokens | Extractor   | Coverage   | Elapsed  |`
+    : `| ${'File'.padEnd(maxFile)} | Sigs | Tokens | Extractor   | Coverage   |`;
+
+  const sep = showSlow
+    ? `|${'-'.repeat(maxFile + 2)}|------|--------|-------------|------------|----------|`
+    : `|${'-'.repeat(maxFile + 2)}|------|--------|-------------|------------|`;
+
+  const rows = stats.map((s) => {
+    const cov  = s.covered ? '✓ tested  ' : '✗ untested';
+    const file = s.file.padEnd(maxFile);
+    const ext  = (s.extractor || '').padEnd(11);
+    const base = `| ${file} | ${String(s.sigs).padStart(4)} | ${String(s.tokens).padStart(6)} | ${ext} | ${cov} |`;
+    if (showSlow) {
+      const ms = s.elapsedMs !== undefined ? `${s.elapsedMs}ms` : '';
+      const flag = s.slow ? ' ⚠️' : '';
+      return `${base} ${ms.padStart(6)}${flag} |`;
+    }
+    return base;
+  });
+
+  const totalSigs   = stats.reduce((n, s) => n + s.sigs,   0);
+  const totalTokens = stats.reduce((n, s) => n + s.tokens, 0);
+  const slotFile    = ''.padEnd(maxFile);
+  const baseFoot    = `| ${slotFile} | ${String(totalSigs).padStart(4)} | ${String(totalTokens).padStart(6)} | **Total**   |            |`;
+  const footer = showSlow ? `${baseFoot} ${' '.padStart(8)} |` : baseFoot;
+
+  return [header, sep, ...rows, sep, footer].join('\n') + '\n';
+}
+
+/**
+ * Format stats as a plain-object suitable for JSON.stringify.
+ *
+ * @param {object[]} stats
+ * @returns {object}
+ */
+function formatAnalysisJSON(stats) {
+  const totalSigs   = stats.reduce((n, s) => n + s.sigs,   0);
+  const totalTokens = stats.reduce((n, s) => n + s.tokens, 0);
+  const slowFiles   = stats.filter((s) => s.slow);
+
+  return {
+    files:       stats,
+    totalSigs,
+    totalTokens,
+    slowFiles:   slowFiles.map((s) => ({ file: s.file, elapsedMs: s.elapsedMs })),
+    fileCount:   stats.length,
+  };
+}
+
+module.exports = { analyzeFiles, formatAnalysisTable, formatAnalysisJSON };

package/src/mcp/handlers.js

@@ -430,4 +430,31 @@ function listModules(args, cwd) {
   ].join('\n');
 }
 
-module.exports = { readContext, searchSignatures, getMap, createCheckpoint, getRouting, explainFile, listModules };
+/**
+ * query_context({ query, topK? }) → string
+ *
+ * Ranks context-file entries by relevance to the query and returns the
+ * top-K most relevant files with their signatures and scores.
+ */
+function queryContext(args, cwd) {
+  if (!args || !args.query) return 'Missing required argument: query';
+
+  const contextPath = path.join(cwd, CONTEXT_FILE);
+  if (!fs.existsSync(contextPath)) {
+    return 'No context file found. Run: node gen-context.js';
+  }
+
+  try {
+    const { rank, buildSigIndex, formatRankTable } = require('../retrieval/ranker');
+    const index = buildSigIndex(cwd);
+    if (index.size === 0) return 'No signatures indexed. Run: node gen-context.js';
+
+    const topK = Math.min(Math.max(1, parseInt(args.topK, 10) || 10), 25);
+    const results = rank(args.query, index, { topK });
+    return formatRankTable(results, args.query);
+  } catch (err) {
+    return `_query_context failed: ${err.message}_`;
+  }
+}
+
+module.exports = { readContext, searchSignatures, getMap, createCheckpoint, getRouting, explainFile, listModules, queryContext };

package/src/mcp/server.js

@@ -14,11 +14,11 @@
 
 const readline = require('readline');
 const { TOOLS } = require('./tools');
-const { readContext, searchSignatures, getMap, createCheckpoint, getRouting, explainFile, listModules } = require('./handlers');
+const { readContext, searchSignatures, getMap, createCheckpoint, getRouting, explainFile, listModules, queryContext } = require('./handlers');
 
 const SERVER_INFO = {
   name: 'sigmap',
-  version: '2.1.0',
+  version: '2.3.0',
   description: 'SigMap MCP server — code signatures on demand',
 };
 
@@ -73,6 +73,7 @@ function dispatch(msg, cwd) {
       else if (name === 'get_routing') text = getRouting(args, cwd);
       else if (name === 'explain_file') text = explainFile(args, cwd);
       else if (name === 'list_modules') text = listModules(args, cwd);
+      else if (name === 'query_context') text = queryContext(args, cwd);
       else {
         respondError(id, -32601, `Unknown tool: ${name}`);
         return;

package/src/mcp/tools.js

@@ -120,6 +120,30 @@ const TOOLS = [
       required: [],
     },
   },
+  {
+    name: 'query_context',
+    description:
+      'Rank and return the most relevant files for a specific task or question. ' +
+      'Uses keyword + symbol + path scoring to surface only the top-K files relevant ' +
+      'to the query — much cheaper than reading all context. ' +
+      'Returns ranked file list with signatures and relevance scores.',
+    inputSchema: {
+      type: 'object',
+      properties: {
+        query: {
+          type: 'string',
+          description:
+            'Natural language task description or keyword(s) to rank files against. ' +
+            'E.g. "add a new language extractor", "fix secret scanning", "auth module".',
+        },
+        topK: {
+          type: 'number',
+          description: 'Maximum number of files to return (default: 10, max: 25).',
+        },
+      },
+      required: ['query'],
+    },
+  },
 ];
 
 module.exports = { TOOLS };

package/src/retrieval/ranker.js

@@ -0,0 +1,242 @@
+'use strict';
+
+/**
+ * SigMap zero-dependency relevance ranker.
+ *
+ * Ranks all files in a signature index against a natural-language query.
+ * Scoring weights:
+ *   - keyword overlap (exact token match against sigs)
+ *   - symbol match (token appears in a top-level identifier / function name)
+ *   - partial prefix match (token is prefix of a sig token, length ≥ 4)
+ *   - path relevance (query token appears in the file path)
+ *   - recency boost (applied externally via recency map)
+ *
+ * Usage:
+ *   const { rank } = require('./src/retrieval/ranker');
+ *   const results = rank(query, sigIndex, { topK: 10 });
+ *   // results: [{ file, score, sigs, tokens }]
+ */
+
+const { tokenize, STOP_WORDS } = require('./tokenizer');
+
+// ---------------------------------------------------------------------------
+// Default weights
+// ---------------------------------------------------------------------------
+const DEFAULT_WEIGHTS = {
+  exactToken: 1.0,       // query token exactly in sig tokens
+  symbolMatch: 0.5,      // bonus if token appears in a function/class name line
+  prefixMatch: 0.3,      // partial prefix hit (query token ≥ 4 chars)
+  pathMatch: 0.8,        // query token appears in the file path
+  recencyBoost: 1.5,     // multiplier applied when file is in recencySet
+};
+
+/**
+ * Score a single file against a query.
+ *
+ * @param {string}   filePath   - relative file path (e.g. 'src/extractors/python.js')
+ * @param {string[]} sigs       - signature strings for this file
+ * @param {string[]} queryTokens - pre-tokenized query
+ * @param {object}   weights
+ * @returns {number}
+ */
+function scoreFile(filePath, sigs, queryTokens, weights) {
+  if (!sigs || sigs.length === 0) return 0;
+
+  const w = weights || DEFAULT_WEIGHTS;
+
+  // Build token set from all signatures
+  const sigText = sigs.join(' ');
+  const sigTokenSet = new Set(tokenize(sigText));
+
+  // Build token set from the file path
+  const pathTokenSet = new Set(tokenize(filePath));
+
+  let score = 0;
+
+  for (const qt of queryTokens) {
+    if (STOP_WORDS.has(qt)) continue;
+
+    // Exact token match in sigs
+    if (sigTokenSet.has(qt)) {
+      score += w.exactToken;
+
+      // Bonus: appears directly in a function/class/method name line
+      const nameLineMatch = sigs.some((sig) => {
+        const nt = tokenize(sig.replace(/[^a-zA-Z0-9_\s]/g, ' '));
+        return nt.includes(qt);
+      });
+      if (nameLineMatch) score += w.symbolMatch;
+    }
+
+    // Prefix match (e.g. query "python" matches "pythonDeps")
+    if (qt.length >= 4) {
+      for (const st of sigTokenSet) {
+        if (st !== qt && st.startsWith(qt)) {
+          score += w.prefixMatch;
+          break; // one bonus per query token
+        }
+      }
+    }
+
+    // Path token match
+    if (pathTokenSet.has(qt)) {
+      score += w.pathMatch;
+    }
+  }
+
+  return score;
+}
+
+/**
+ * Rank all files in a signature index against a query.
+ *
+ * @param {string}              query     - natural language query
+ * @param {Map<string,string[]>} sigIndex - Map<file, sigs[]>
+ * @param {object}  [opts]
+ * @param {number}  [opts.topK=10]               - max results to return
+ * @param {number}  [opts.recencyBoost=1.5]       - multiplier for recent files
+ * @param {Set<string>} [opts.recencySet]         - set of recently-changed file paths
+ * @param {object}  [opts.weights]               - override scoring weights
+ * @returns {{ file: string, score: number, sigs: string[], tokens: number }[]}
+ */
+function rank(query, sigIndex, opts) {
+  if (!query || typeof query !== 'string') return [];
+  if (!sigIndex || !(sigIndex instanceof Map) || sigIndex.size === 0) return [];
+
+  const topK = (opts && opts.topK) || 10;
+  const recencyMultiplier = (opts && opts.recencyBoost) || DEFAULT_WEIGHTS.recencyBoost;
+  const recencySet = (opts && opts.recencySet) || null;
+  const weights = (opts && opts.weights) ? Object.assign({}, DEFAULT_WEIGHTS, opts.weights) : DEFAULT_WEIGHTS;
+
+  const queryTokens = tokenize(query);
+  if (queryTokens.length === 0) {
+    // Empty query: return top-K by file count (most signatures = most useful)
+    const all = [];
+    for (const [file, sigs] of sigIndex.entries()) {
+      all.push({ file, score: sigs.length, sigs, tokens: Math.ceil(sigs.join('\n').length / 4) });
+    }
+    all.sort((a, b) => b.score - a.score || a.file.localeCompare(b.file));
+    return all.slice(0, topK);
+  }
+
+  const scored = [];
+  for (const [file, sigs] of sigIndex.entries()) {
+    let score = scoreFile(file, sigs, queryTokens, weights);
+
+    // Recency boost
+    if (recencySet && recencySet.has(file) && score > 0) {
+      score *= recencyMultiplier;
+    }
+
+    scored.push({
+      file,
+      score,
+      sigs,
+      tokens: Math.ceil(sigs.join('\n').length / 4),
+    });
+  }
+
+  scored.sort((a, b) => b.score - a.score || a.file.localeCompare(b.file));
+  return scored.slice(0, topK);
+}
+
+/**
+ * Build a signature index from the generated context file.
+ * Returns Map<filePath, string[]> where filePath is the relative path
+ * as it appears in the ### headers of copilot-instructions.md.
+ *
+ * @param {string} cwd
+ * @returns {Map<string, string[]>}
+ */
+function buildSigIndex(cwd) {
+  const fs   = require('fs');
+  const path = require('path');
+  const contextPath = path.join(cwd, '.github', 'copilot-instructions.md');
+  const index = new Map();
+
+  if (!fs.existsSync(contextPath)) return index;
+
+  const content = fs.readFileSync(contextPath, 'utf8');
+  const lines = content.split('\n');
+
+  let currentFile = null;
+  let inBlock = false;
+  let sigs = [];
+
+  for (const line of lines) {
+    const headerMatch = line.match(/^###\s+(\S+)\s*$/);
+    if (headerMatch) {
+      if (currentFile !== null) index.set(currentFile, sigs);
+      currentFile = headerMatch[1];
+      sigs = [];
+      inBlock = false;
+      continue;
+    }
+    if (line.startsWith('```')) { inBlock = !inBlock; continue; }
+    if (inBlock && currentFile && line.trim()) sigs.push(line.trim());
+  }
+  if (currentFile !== null) index.set(currentFile, sigs);
+
+  return index;
+}
+
+/**
+ * Format ranked results as a markdown table string.
+ *
+ * @param {{ file: string, score: number, sigs: string[], tokens: number }[]} results
+ * @param {string} query
+ * @returns {string}
+ */
+function formatRankTable(results, query) {
+  if (!results || results.length === 0) {
+    return `No matching files found for query: "${query}"\n`;
+  }
+
+  const lines = [
+    `## Query: ${query}`,
+    '',
+    '| Rank | File | Score | Sigs | Tokens |',
+    '|------|------|-------|------|--------|',
+    ...results.map((r, i) =>
+      `| ${i + 1} | ${r.file} | ${r.score.toFixed(2)} | ${r.sigs.length} | ${r.tokens} |`
+    ),
+    '',
+  ];
+
+  // Add signature details for top results
+  for (const r of results.slice(0, 3)) {
+    if (r.sigs.length > 0) {
+      lines.push(`### ${r.file}`);
+      lines.push('```');
+      lines.push(...r.sigs.slice(0, 10));
+      if (r.sigs.length > 10) lines.push(`... (${r.sigs.length - 10} more)`);
+      lines.push('```');
+      lines.push('');
+    }
+  }
+
+  return lines.join('\n');
+}
+
+/**
+ * Format ranked results as a structured JSON-serialisable object.
+ *
+ * @param {{ file: string, score: number, sigs: string[], tokens: number }[]} results
+ * @param {string} query
+ * @returns {object}
+ */
+function formatRankJSON(results, query) {
+  return {
+    query,
+    results: (results || []).map((r, i) => ({
+      rank: i + 1,
+      file: r.file,
+      score: r.score,
+      sigs: r.sigs,
+      tokens: r.tokens,
+    })),
+    totalResults: (results || []).length,
+  };
+}
+
+module.exports = { rank, buildSigIndex, scoreFile, formatRankTable, formatRankJSON, DEFAULT_WEIGHTS };

package/src/retrieval/tokenizer.js

@@ -0,0 +1,54 @@
+'use strict';
+
+/**
+ * SigMap zero-dependency tokenizer.
+ * Splits code identifiers: camelCase, snake_case, kebab-case, PascalCase,
+ * removes stop words, and returns lower-case tokens.
+ */
+
+const STOP_WORDS = new Set([
+  'the', 'a', 'an', 'in', 'of', 'to', 'for', 'and', 'or', 'is', 'are',
+  'that', 'this', 'it', 'with', 'from', 'by', 'be', 'as', 'on', 'at',
+  'do', 'not', 'use', 'get', 'set', 'up', 'if', 'no', 'so', 'we',
+]);
+
+/**
+ * Tokenize any text (query or code signature) into unique lower-case tokens.
+ * Handles:
+ *   - camelCase  → ['camel', 'case']
+ *   - PascalCase → ['pascal', 'case']
+ *   - snake_case → ['snake', 'case']
+ *   - kebab-case → ['kebab', 'case']
+ *   - dot.notation → ['dot', 'notation']
+ *   - File paths  → individual path components (no extension)
+ *
+ * @param {string} text
+ * @param {object} [opts]
+ * @param {boolean} [opts.removeStopWords=true]
+ * @param {number}  [opts.minLength=2]
+ * @returns {string[]}
+ */
+function tokenize(text, opts) {
+  if (!text || typeof text !== 'string') return [];
+  const removeStop = opts && opts.removeStopWords === false ? false : true;
+  const minLen = (opts && opts.minLength) || 2;
+
+  const tokens = text
+    // strip file extension (e.g. .js, .ts, .py)
+    .replace(/\.\w{1,6}(?=\s|\/|$)/g, ' ')
+    // camelCase / PascalCase split
+    .replace(/([a-z])([A-Z])/g, '$1 $2')
+    .replace(/([A-Z]+)([A-Z][a-z])/g, '$1 $2')
+    // snake_case / kebab-case / dot.notation
+    .replace(/[_\-\.\/]/g, ' ')
+    // drop remaining non-word characters
+    .replace(/[^\w\s]/g, ' ')
+    .toLowerCase()
+    .split(/\s+/)
+    .filter((t) => t.length >= minLen);
+
+  if (!removeStop) return [...new Set(tokens)];
+  return [...new Set(tokens.filter((t) => !STOP_WORDS.has(t)))];
+}
+
+module.exports = { tokenize, STOP_WORDS };