sigmap 7.30.0 → 8.0.0

package/src/map/migrations.js

@@ -0,0 +1,84 @@
+'use strict';
+
+/**
+ * Database-migration extractor (v8.5 C1).
+ *
+ * Detects schema-migration files across the common frameworks — Rails
+ * (db/migrate), Django/Alembic, Prisma, Flyway (`V1__name.sql`), knex/Sequelize,
+ * and timestamped SQL — and surfaces them with a parsed version + name. Pure,
+ * zero-dependency, deterministic.
+ *
+ * @param {string[]} files — absolute file paths (unused; the tree is walked)
+ * @param {string}   cwd   — project root
+ * @returns {string} formatted markdown table (empty string if none found)
+ */
+
+const fs = require('fs');
+const path = require('path');
+
+const MAX_DEPTH = 6;
+const MAX_ROWS = 200;
+const SKIP_DIR = new Set(['.git', 'node_modules', 'vendor', 'dist', 'build', 'target', '.venv', 'venv', '__pycache__']);
+const MIG_EXT = new Set(['.sql', '.rb', '.py', '.js', '.ts']);
+
+// A directory whose path marks its children as migrations.
+const MIG_DIR_RE = /(^|\/)(db\/migrate|migrations?|alembic\/versions|prisma\/migrations)$/i;
+// A filename that is itself a migration regardless of directory.
+const FLYWAY_RE = /^V\d+(?:[._]\d+)*__(.+)\.(sql|java)$/;
+const TIMESTAMP_RE = /^(\d{8,})[_-](.+)\.(sql|rb|py|js|ts)$/;
+const NAMED_RE = /[._-]migrations?[._-]/i;
+
+function walk(dir, cwd, depth, out) {
+  if (depth > MAX_DEPTH) return;
+  let entries;
+  try { entries = fs.readdirSync(dir, { withFileTypes: true }); } catch (_) { return; }
+  entries.sort((a, b) => (a.name < b.name ? -1 : a.name > b.name ? 1 : 0));
+
+  const relDir = path.relative(cwd, dir).replace(/\\/g, '/');
+  const dirIsMigration = MIG_DIR_RE.test(relDir);
+
+  for (const e of entries) {
+    if (e.isDirectory()) {
+      if (SKIP_DIR.has(e.name)) continue;
+      walk(path.join(dir, e.name), cwd, depth + 1, out);
+      continue;
+    }
+    const ext = path.extname(e.name).toLowerCase();
+    if (!MIG_EXT.has(ext)) continue;
+
+    const rel = path.relative(cwd, path.join(dir, e.name)).replace(/\\/g, '/');
+    let version = null;
+    let name = null;
+
+    let m;
+    if ((m = e.name.match(FLYWAY_RE))) { version = e.name.split('__')[0]; name = m[1].replace(/_/g, ' '); }
+    else if ((m = e.name.match(TIMESTAMP_RE))) { version = m[1]; name = m[2].replace(/[_-]/g, ' '); }
+    else if (dirIsMigration) { version = '—'; name = e.name.replace(ext, ''); }
+    else if (NAMED_RE.test(e.name)) { version = '—'; name = e.name.replace(ext, ''); }
+    else continue;
+
+    out.push({ version, name, file: rel });
+  }
+}
+
+function analyze(files, cwd) {
+  const found = [];
+  walk(cwd, cwd, 0, found);
+  if (found.length === 0) return '';
+
+  found.sort((a, b) => (a.file < b.file ? -1 : a.file > b.file ? 1 : 0));
+
+  const lines = [
+    '| Version | Migration | File |',
+    '|---------|-----------|------|',
+  ];
+  for (const r of found.slice(0, MAX_ROWS)) {
+    lines.push(`| ${r.version} | ${r.name} | ${r.file} |`);
+  }
+  if (found.length > MAX_ROWS) {
+    lines.push(`| … | +${found.length - MAX_ROWS} more | |`);
+  }
+  return lines.join('\n');
+}
+
+module.exports = { analyze };

package/src/mcp/handlers.js

@@ -21,6 +21,10 @@ const MAP_SECTIONS = {
   imports: '### Import graph',
   classes: '### Class hierarchy',
   routes: '### Route table',
+  env: '### Environment variables',
+  buildci: '### Build & CI',
+  manifests: '### Config & manifests',
+  migrations: '### Database migrations',
 };
 
 /**
@@ -106,7 +110,7 @@ function getMap(args, cwd) {
 
   const header = MAP_SECTIONS[args.type];
   if (!header) {
-    return `Unknown map type: "${args.type}". Use: imports, classes, routes`;
+    return `Unknown map type: "${args.type}". Use: ${Object.keys(MAP_SECTIONS).join(', ')}`;
   }
 
   const mapPath = path.join(cwd, 'PROJECT_MAP.md');

package/src/mcp/server.js

@@ -18,7 +18,7 @@ const { readContext, searchSignatures, getMap, createCheckpoint, getRouting, exp
 
 const SERVER_INFO = {
   name: 'sigmap',
-  version: '7.30.0',
+  version: '8.0.0',
   description: 'SigMap MCP server — code signatures on demand',
 };
 

package/src/retrieval/bm25.js

@@ -0,0 +1,122 @@
+'use strict';
+
+/**
+ * SigMap identifier-aware BM25 re-ranker (zero dependencies, deterministic).
+ *
+ * Plain exact-token TF-IDF misses queries whose terms live *inside* code
+ * identifiers — e.g. `component emit` never surfaces `componentEmits.ts`,
+ * because "componentEmits" is one token that shares no exact term with the
+ * query. This module fixes that with four small additions:
+ *
+ *   1. Identifier-aware tokenization — split camelCase and snake_case.
+ *   2. Light stemming — plurals / common suffixes (`emits` → `emit`).
+ *   3. Path-token boost — file path / basename tokens weigh PATH_BOOST× more.
+ *   4. BM25 scoring instead of raw TF-IDF (length-normalized).
+ *
+ * On 85 curated tasks across 17 repos this lifted hit@5 from 75.3% → 82.4%
+ * (MRR +16% relative). See issue #395.
+ */
+
+// Stop words: common English + low-signal code verbs/nouns that appear in
+// nearly every signature and so carry little retrieval signal.
+const STOP = new Set(
+  ('a an the of to in on for and or is are be by with as at from that this it its ' +
+   'into get set add new return value test')
+    .split(' ')
+);
+
+/**
+ * Light suffix stemmer — conservative, tuned for code identifiers rather than
+ * prose. Words of 3 chars or fewer pass through unchanged; a result shorter
+ * than 3 chars reverts to the original token.
+ *
+ * @param {string} w
+ * @returns {string}
+ */
+function stem(w) {
+  if (w.length <= 3) return w;
+  let s = w;
+  s = s.replace(/ies$/, 'y');
+  s = s.replace(/(sses|shes|ches|xes|zes)$/, (m) => m.slice(0, -2));
+  s = s.replace(/([^s])s$/, '$1');
+  s = s.replace(/(ization|izations)$/, 'ize');
+  s = s.replace(/(ing|edly|ed|er|ers|ation|ations|ment|ness|ity|ive|able|ible|ize|ise|al)$/, '');
+  return s.length >= 3 ? s : w;
+}
+
+/**
+ * Split on non-alphanumeric characters AND camelCase / snake_case boundaries,
+ * lowercase, drop stop words and single characters, then stem.
+ *
+ * @param {string} text
+ * @returns {string[]}
+ */
+function tokenize(text) {
+  if (!text || typeof text !== 'string') return [];
+  return text
+    .replace(/[^A-Za-z0-9]+/g, ' ')
+    .replace(/([a-z0-9])([A-Z])/g, '$1 $2')
+    .replace(/([A-Z]+)([A-Z][a-z])/g, '$1 $2')
+    .toLowerCase()
+    .split(/\s+/)
+    .filter((t) => t.length > 1 && !STOP.has(t))
+    .map(stem)
+    .filter(Boolean);
+}
+
+// The file path / basename is highly indicative of relevance, so its tokens
+// are counted PATH_BOOST times when building the document term-frequency map.
+const PATH_BOOST = 3;
+
+/**
+ * BM25 re-rank of candidates against a query. Each candidate is
+ * `{ file, sigs }`; the returned objects preserve all original candidate
+ * fields and add a numeric `score` (higher = more relevant), sorted best-first
+ * with a deterministic path tie-break. A `score` of 0 means no query token
+ * matched — callers typically drop those.
+ *
+ * @param {string} query
+ * @param {{ file: string, sigs: string[] }[]} candidates
+ * @returns {Array<object & { score: number }>}
+ */
+function bm25rank(query, candidates) {
+  if (!Array.isArray(candidates) || candidates.length === 0) return [];
+
+  const k1 = 1.5;
+  const b = 0.75;
+
+  const docs = candidates.map((c) => {
+    const pathToks = tokenize(c.file || '');
+    const toks = tokenize((c.sigs || []).join(' '));
+    for (let i = 0; i < PATH_BOOST; i++) toks.push(...pathToks);
+    const tf = new Map();
+    for (const t of toks) tf.set(t, (tf.get(t) || 0) + 1);
+    return { cand: c, tf, len: toks.length };
+  });
+
+  const N = docs.length || 1;
+  const avgdl = docs.reduce((s, d) => s + d.len, 0) / N || 1;
+
+  const df = new Map();
+  for (const d of docs) {
+    for (const t of d.tf.keys()) df.set(t, (df.get(t) || 0) + 1);
+  }
+
+  const qToks = [...new Set(tokenize(query))];
+
+  return docs
+    .map((d) => {
+      let score = 0;
+      for (const t of qToks) {
+        const f = d.tf.get(t);
+        if (!f) continue;
+        const dfT = df.get(t);
+        const idf = Math.log(1 + (N - dfT + 0.5) / (dfT + 0.5));
+        score += (idf * (f * (k1 + 1))) / (f + k1 * (1 - b + (b * d.len) / avgdl));
+      }
+      return Object.assign({}, d.cand, { score });
+    })
+    .sort((a, c) => c.score - a.score || String(a.file).localeCompare(String(c.file)));
+}
+
+module.exports = { tokenize, stem, bm25rank, PATH_BOOST, STOP };

package/src/retrieval/ranker.js

@@ -19,6 +19,7 @@
 
 const { loadWeights } = require('../learning/weights');
 const { tokenize, STOP_WORDS } = require('./tokenizer');
+const { bm25rank } = require('./bm25');
 
 // ---------------------------------------------------------------------------
 // Default weights
@@ -197,11 +198,24 @@ function rank(query, sigIndex, opts) {
     return all.slice(0, topK);
   }
 
+  // Identifier-aware BM25 base relevance over the whole index (#395). BM25
+  // splits camelCase/snake_case, stems, and boosts path tokens, so queries
+  // whose terms live inside identifiers (e.g. "component emit" → componentEmits)
+  // are matched. The existing negative-signal penalty and recency/graph/learned
+  // boosts are layered on top; the per-token signals stay for the explain table.
+  const bm25Scores = new Map();
+  for (const c of bm25rank(query, [...sigIndex.entries()].map(([file, sigs]) => ({ file, sigs })))) {
+    bm25Scores.set(c.file, c.score);
+  }
+
   const scored = [];
   for (const [file, sigs] of sigIndex.entries()) {
     const result = scoreFile(file, sigs, queryTokens, weights);
-    let score = result.score;
+    const penalty = result.signals.penalty;
+    const base = bm25Scores.get(file) || 0;
+    let score = base * penalty;
     const signals = result.signals;
+    signals.bm25 = base;
 
     // Recency boost
     if (recencySet && recencySet.has(file) && score > 0) {