sigmap 2.3.0 → 2.5.0

package/packages/core/README.md

@@ -0,0 +1,141 @@
+# sigmap-core
+
+Programmatic API for [SigMap](https://manojmallick.github.io/sigmap/) — zero-dependency code signature extraction, ranked retrieval, secret scanning, and project health scoring.
+
+## Installation
+
+```bash
+npm install sigmap        # installs the full package (CLI + core)
+```
+
+`require('sigmap')` resolves to this library via the root `exports` field.
+
+## Quick start
+
+```js
+const { extract, rank, buildSigIndex, scan, score } = require('sigmap');
+
+// 1. Extract signatures from any source file
+const sigs = extract('function hello() { return "world"; }', 'javascript');
+// → ['function hello()']
+
+// 2. Scan for secrets before storing signatures
+const { safe, redacted } = scan(sigs, 'src/utils.js');
+
+// 3. Build an index from the generated context file
+const index = buildSigIndex('/path/to/your/project');
+
+// 4. Rank files against a query
+const results = rank('add a new language extractor', index, { topK: 5 });
+// → [{ file: 'src/extractors/python.js', score: 3.5, sigs: [...], tokens: 42 }, ...]
+
+// 5. Check project health
+const health = score('/path/to/your/project');
+// → { score: 92, grade: 'A', strategy: 'full', ... }
+```
+
+## API reference
+
+### `extract(src, language)` → `string[]`
+
+Extract code signatures from source text.
+
+| Param | Type | Description |
+|---|---|---|
+| `src` | `string` | Raw file content |
+| `language` | `string` | Language name (`'typescript'`, `'python'`, etc.) **or** a file path/name with a recognised extension |
+
+Returns an array of signature strings. Never throws — returns `[]` on any error.
+
+**Supported languages:** typescript, javascript, python, java, kotlin, go, rust, csharp, cpp, ruby, php, swift, dart, scala, vue, svelte, html, css, yaml, shell, dockerfile (21 total)
+
+```js
+// By language name
+extract(src, 'python');
+
+// By file path (extension is used to detect language)
+extract(src, 'src/server.ts');
+extract(src, 'Dockerfile');
+```
+
+---
+
+### `rank(query, sigIndex, opts?)` → `Result[]`
+
+Rank all files in a signature index against a natural-language query.
+
+| Param | Type | Description |
+|---|---|---|
+| `query` | `string` | Natural language or keyword query |
+| `sigIndex` | `Map<string, string[]>` | File → signatures map (from `buildSigIndex`) |
+| `opts.topK` | `number` | Max files to return (default: `10`) |
+| `opts.weights` | `object` | Override default scoring weights |
+| `opts.recencySet` | `Set<string>` | Files to boost with `recencyBoost` multiplier |
+
+Each result: `{ file: string, score: number, sigs: string[], tokens: number }`
+
+---
+
+### `buildSigIndex(cwd)` → `Map<string, string[]>`
+
+Build a file→signatures map from the generated `.github/copilot-instructions.md`.
+Requires `node gen-context.js` to have been run first.
+
+```js
+const index = buildSigIndex('/path/to/project');
+// → Map { 'src/extractors/python.js' => ['class Extractor', '  def extract(src)'], ... }
+```
+
+---
+
+### `scan(sigs, filePath)` → `{ safe: string[], redacted: boolean }`
+
+Scan signature strings for secrets (AWS keys, GitHub tokens, DB connection strings, etc.) and redact any matches.
+
+```js
+const { safe, redacted } = scan(
+  ['const SECRET = "ghp_abc123xyz..."'],
+  'src/config.ts'
+);
+// safe → ['[REDACTED — GitHub Token detected in src/config.ts]']
+// redacted → true
+```
+
+**Detected patterns:** AWS Access Key, AWS Secret Key, GCP API Key, GitHub Token, JWT Token, DB Connection String, SSH Private Key, Stripe Key, Twilio Key, Generic Secret
+
+---
+
+### `score(cwd)` → `HealthResult`
+
+Compute a composite health score for the SigMap installation in a project.
+
+```js
+const health = score('/path/to/project');
+// {
+//   score: 92,
+//   grade: 'A',           // A ≥90 | B ≥75 | C ≥60 | D <60
+//   strategy: 'full',
+//   tokenReductionPct: 97.2,
+//   daysSinceRegen: 0.1,
+//   totalRuns: 48,
+//   overBudgetRuns: 0,
+// }
+```
+
+## Migration from v2.3 and earlier
+
+`require('sigmap')` was not available before v2.4. The programmatic API is new — no migration needed for CLI usage.
+
+All existing CLI flags (`--generate`, `--watch`, `--mcp`, `--query`, `--analyze`, `--benchmark`, `--health`, …) are unchanged.
+
+## What's next — v2.5-v2.6
+
+v2.5 adds `analyzeImpact(changedFiles, cwd)` to `packages/core` — given a list of changed files, it returns every file that transitively imports them. See [issue #14](https://github.com/manojmallick/sigmap/issues/14).
+
+v2.6 adds benchmark and paper reporting capabilities — run evaluations against external repos and export metrics in LaTeX format for academic papers. See [issue #16](https://github.com/manojmallick/sigmap/issues/16).
+
+See the full [roadmap](https://manojmallick.github.io/sigmap/roadmap.html).
+
+## Zero dependencies
+
+This package has zero runtime npm dependencies. It uses only Node.js built-ins: `fs`, `path`.

package/packages/core/index.js

@@ -0,0 +1,215 @@
+'use strict';
+
+/**
+ * sigmap-core — public programmatic API
+ *
+ * Usage:
+ *   const { extract, rank, scan, score } = require('sigmap');
+ *
+ * All functions are zero-dependency and never throw.
+ */
+
+const path = require('path');
+
+// ---------------------------------------------------------------------------
+// Language extractor registry
+// ---------------------------------------------------------------------------
+const EXT_MAP = {
+  '.ts': 'typescript', '.tsx': 'typescript',
+  '.js': 'javascript', '.jsx': 'javascript', '.mjs': 'javascript', '.cjs': 'javascript',
+  '.py': 'python',     '.pyw': 'python',
+  '.java': 'java',
+  '.kt': 'kotlin',     '.kts': 'kotlin',
+  '.go': 'go',
+  '.rs': 'rust',
+  '.cs': 'csharp',
+  '.cpp': 'cpp', '.c': 'cpp', '.h': 'cpp', '.hpp': 'cpp', '.cc': 'cpp',
+  '.rb': 'ruby',       '.rake': 'ruby',
+  '.php': 'php',
+  '.swift': 'swift',
+  '.dart': 'dart',
+  '.scala': 'scala',   '.sc': 'scala',
+  '.vue': 'vue',
+  '.svelte': 'svelte',
+  '.html': 'html',     '.htm': 'html',
+  '.css': 'css',       '.scss': 'css', '.sass': 'css', '.less': 'css',
+  '.yml': 'yaml',      '.yaml': 'yaml',
+  '.sh': 'shell',      '.bash': 'shell', '.zsh': 'shell', '.fish': 'shell',
+};
+
+const SRC_ROOT = path.resolve(__dirname, '..', '..', 'src');
+
+function _resolveExtractor(language) {
+  const extPath = path.join(SRC_ROOT, 'extractors', language + '.js');
+  try {
+    return require(extPath);
+  } catch (_) {
+    return null;
+  }
+}
+
+// ---------------------------------------------------------------------------
+// extract(src, language) → string[]
+// ---------------------------------------------------------------------------
+/**
+ * Extract code signatures from source text for the given language.
+ *
+ * @param {string} src       - Raw source file content
+ * @param {string} language  - Language name (e.g. 'typescript', 'python')
+ *                             OR a file path/name with a recognised extension
+ * @returns {string[]} Array of signature strings (never throws)
+ *
+ * @example
+ *   const sigs = extract('function hello() {}', 'javascript');
+ *   // → ['function hello()']
+ *
+ *   const sigs2 = extract(src, 'src/server.ts');
+ *   // → detected as typescript via extension
+ */
+function extract(src, language) {
+  if (!src || typeof src !== 'string') return [];
+  if (!language || typeof language !== 'string') return [];
+
+  // If language looks like a file path, derive language from extension
+  let lang = language;
+  if (language.includes('.') || language.includes('/') || language.includes('\\')) {
+    const ext = path.extname(language).toLowerCase();
+    const base = path.basename(language);
+    if (base === 'Dockerfile' || base.startsWith('Dockerfile.')) {
+      lang = 'dockerfile';
+    } else {
+      lang = EXT_MAP[ext] || null;
+    }
+    if (!lang) return [];
+  } else {
+    // Normalise e.g. 'JavaScript' → 'javascript'
+    lang = language.toLowerCase();
+  }
+
+  const mod = _resolveExtractor(lang);
+  if (!mod || typeof mod.extract !== 'function') return [];
+
+  try {
+    const result = mod.extract(src);
+    return Array.isArray(result) ? result : [];
+  } catch (_) {
+    return [];
+  }
+}
+
+// ---------------------------------------------------------------------------
+// rank(query, sigIndex, opts?) → { file, score, sigs, tokens }[]
+// ---------------------------------------------------------------------------
+/**
+ * Rank files in a signature index against a natural-language query.
+ *
+ * @param {string} query   - Natural language or keyword query
+ * @param {Map<string, string[]>} sigIndex - File → signatures map
+ * @param {object} [opts]
+ * @param {number} [opts.topK=10]        - Maximum results to return
+ * @param {number} [opts.recencyBoost]   - Score multiplier for recent files
+ * @param {Set<string>} [opts.recencySet] - Set of file paths considered recent
+ * @param {object} [opts.weights]        - Override default scoring weights
+ * @returns {{ file: string, score: number, sigs: string[], tokens: number }[]}
+ *
+ * @example
+ *   const { rank, buildSigIndex } = require('sigmap');
+ *   const index = buildSigIndex('/path/to/project');
+ *   const results = rank('add a new language extractor', index, { topK: 5 });
+ */
+function rank(query, sigIndex, opts) {
+  try {
+    const { rank: _rank } = require(path.join(SRC_ROOT, 'retrieval', 'ranker.js'));
+    return _rank(query, sigIndex, opts);
+  } catch (_) {
+    return [];
+  }
+}
+
+// ---------------------------------------------------------------------------
+// buildSigIndex(cwd) → Map<string, string[]>
+// ---------------------------------------------------------------------------
+/**
+ * Build a file→signatures index from the generated context file.
+ * Requires gen-context.js to have been run first.
+ *
+ * @param {string} cwd - Project root directory
+ * @returns {Map<string, string[]>}
+ */
+function buildSigIndex(cwd) {
+  try {
+    const { buildSigIndex: _build } = require(path.join(SRC_ROOT, 'retrieval', 'ranker.js'));
+    return _build(cwd);
+  } catch (_) {
+    return new Map();
+  }
+}
+
+// ---------------------------------------------------------------------------
+// scan(sigs, filePath) → { safe: string[], redacted: boolean }
+// ---------------------------------------------------------------------------
+/**
+ * Scan an array of signature strings for secrets and redact any matches.
+ *
+ * @param {string[]} sigs      - Signature strings to scan
+ * @param {string}   filePath  - Source file path (used in redaction message)
+ * @returns {{ safe: string[], redacted: boolean }}
+ *
+ * @example
+ *   const { safe, redacted } = scan(['const KEY = "AKIAEXAMPLE123..."'], 'config.js');
+ *   // redacted === true — key was replaced with [REDACTED — AWS Access Key ...]
+ */
+function scan(sigs, filePath) {
+  try {
+    const { scan: _scan } = require(path.join(SRC_ROOT, 'security', 'scanner.js'));
+    return _scan(sigs, filePath);
+  } catch (_) {
+    return { safe: Array.isArray(sigs) ? sigs : [], redacted: false };
+  }
+}
+
+// ---------------------------------------------------------------------------
+// score(cwd) → { score, grade, ... }
+// ---------------------------------------------------------------------------
+/**
+ * Compute a composite health score for the project at cwd.
+ *
+ * @param {string} cwd - Project root directory
+ * @returns {{
+ *   score: number,
+ *   grade: 'A'|'B'|'C'|'D',
+ *   strategy: string,
+ *   tokenReductionPct: number|null,
+ *   daysSinceRegen: number|null,
+ *   totalRuns: number,
+ *   overBudgetRuns: number,
+ * }}
+ *
+ * @example
+ *   const health = score('/path/to/project');
+ *   console.log(health.grade); // 'A'
+ */
+function score(cwd) {
+  try {
+    const { score: _score } = require(path.join(SRC_ROOT, 'health', 'scorer.js'));
+    return _score(cwd);
+  } catch (_) {
+    return { score: 0, grade: 'D', strategy: 'full', tokenReductionPct: null, daysSinceRegen: null, totalRuns: 0, overBudgetRuns: 0 };
+  }
+}
+
+// ---------------------------------------------------------------------------
+// Exports
+// ---------------------------------------------------------------------------
+module.exports = {
+  /** Extract signatures from source text */
+  extract,
+  /** Rank project files against a query */
+  rank,
+  /** Build a signature index from the generated context file */
+  buildSigIndex,
+  /** Scan signatures for secrets (redacts matches) */
+  scan,
+  /** Compute project health score */
+  score,
+};

package/packages/core/package.json

@@ -0,0 +1,28 @@
+{
+  "name": "sigmap-core",
+  "version": "2.4.0",
+  "description": "SigMap core library — zero-dependency code signature extraction, retrieval, and security scanning",
+  "main": "index.js",
+  "keywords": [
+    "sigmap",
+    "ai-context",
+    "code-signatures",
+    "extraction",
+    "retrieval",
+    "zero-dependency"
+  ],
+  "author": {
+    "name": "Manoj Mallick",
+    "url": "https://github.com/manojmallick"
+  },
+  "repository": {
+    "type": "git",
+    "url": "https://github.com/manojmallick/sigmap.git",
+    "directory": "packages/core"
+  },
+  "homepage": "https://manojmallick.github.io/sigmap/",
+  "license": "MIT",
+  "engines": {
+    "node": ">=18.0.0"
+  }
+}

package/src/config/defaults.js

@@ -100,6 +100,14 @@ const DEFAULTS = {
     // Multiplier applied to recently-changed files (>1 boosts them up)
     recencyBoost: 1.5,
   },
+
+  // Impact layer settings (v2.5)
+  impact: {
+    // BFS traversal depth limit for --impact (0 = unlimited)
+    depth: 3,
+    // Include signatures of impacted files in --impact output
+    includeSigs: true,
+  },
 };
 
 module.exports = { DEFAULTS };

package/src/graph/builder.js

@@ -0,0 +1,259 @@
+'use strict';
+
+/**
+ * Dependency graph builder (v2.5).
+ *
+ * Builds a forward and reverse dependency graph by resolving import/require
+ * statements across JS/TS, Python, Go, Rust, Java, Kotlin, and Ruby files.
+ *
+ * @module src/graph/builder
+ */
+
+const fs   = require('fs');
+const path = require('path');
+
+// ---------------------------------------------------------------------------
+// Language-specific import extractors
+// ---------------------------------------------------------------------------
+
+const JS_EXTS  = new Set(['.ts', '.tsx', '.js', '.jsx', '.mjs', '.cjs']);
+const PY_EXTS  = new Set(['.py', '.pyw']);
+const GO_EXTS  = new Set(['.go']);
+const RS_EXTS  = new Set(['.rs']);
+const JVM_EXTS = new Set(['.java', '.kt', '.kts', '.scala', '.sc']);
+const RB_EXTS  = new Set(['.rb', '.rake']);
+
+/**
+ * Resolve a JS/TS relative import string to an absolute path in fileSet.
+ * @param {string} dir - directory of the importing file
+ * @param {string} importStr - raw import string (e.g. './utils', '../store')
+ * @param {Set<string>} fileSet
+ * @returns {string|null}
+ */
+function resolveJsPath(dir, importStr, fileSet) {
+  const base = path.resolve(dir, importStr);
+  const candidates = [
+    base,
+    base + '.ts', base + '.tsx',
+    base + '.js', base + '.jsx', base + '.mjs', base + '.cjs',
+    path.join(base, 'index.ts'),
+    path.join(base, 'index.js'),
+  ];
+  for (const c of candidates) {
+    if (fileSet.has(c)) return c;
+  }
+  return null;
+}
+
+/**
+ * Extract absolute dependency paths from a single file.
+ * @param {string} filePath - absolute path to the file
+ * @param {string} content  - file source content
+ * @param {Set<string>} fileSet - set of all known absolute file paths
+ * @returns {string[]} resolved absolute paths this file imports
+ */
+function extractFileDeps(filePath, content, fileSet) {
+  const ext = path.extname(filePath).toLowerCase();
+  const dir = path.dirname(filePath);
+  const found = [];
+
+  // ── JS / TS ───────────────────────────────────────────────────────────────
+  if (JS_EXTS.has(ext)) {
+    const stripped = content
+      .replace(/\/\/.*$/gm, '')
+      .replace(/\/\*[\s\S]*?\*\//g, '');
+
+    // ES imports:   import ... from './foo'  or  import './side-effect'
+    const reEs = /(?:^|[\r\n])\s*import\s+(?:[^'";\r\n]*?\s+from\s+)?['"](\.[^'"]+)['"]/g;
+    let m;
+    while ((m = reEs.exec(stripped)) !== null) {
+      const r = resolveJsPath(dir, m[1], fileSet);
+      if (r) found.push(r);
+    }
+    // CommonJS: require('./foo')
+    const reCjs = /\brequire\s*\(\s*['"](\.[^'"]+)['"]\s*\)/g;
+    while ((m = reCjs.exec(stripped)) !== null) {
+      const r = resolveJsPath(dir, m[1], fileSet);
+      if (r) found.push(r);
+    }
+  }
+
+  // ── Python ────────────────────────────────────────────────────────────────
+  if (PY_EXTS.has(ext)) {
+    // from .module import ...  /  from ..pkg import ...
+    const re = /^[ \t]*from\s+(\.+[\w.]*)\s+import/gm;
+    let m;
+    while ((m = re.exec(content)) !== null) {
+      const dotCount = (m[1].match(/^\.+/) || [''])[0].length;
+      const modPart  = m[1].slice(dotCount).replace(/\./g, '/');
+      let base = dir;
+      for (let i = 1; i < dotCount; i++) base = path.dirname(base);
+      const candidate = modPart
+        ? path.join(base, modPart + '.py')
+        : null;
+      if (candidate && fileSet.has(candidate)) found.push(candidate);
+    }
+  }
+
+  // ── Go ────────────────────────────────────────────────────────────────────
+  // Go uses module paths, not relative file paths — we match same-module paths
+  // by checking if any known file's relative path matches the imported suffix.
+  if (GO_EXTS.has(ext)) {
+    const re = /import\s*\(\s*([\s\S]*?)\s*\)/g;
+    const reInline = /import\s+"([^"]+)"/g;
+    const imports = [];
+    let m;
+    while ((m = re.exec(content)) !== null) {
+      for (const imp of m[1].matchAll(/"([^"]+)"/g)) imports.push(imp[1]);
+    }
+    while ((m = reInline.exec(content)) !== null) imports.push(m[1]);
+
+    for (const imp of imports) {
+      const suffix = imp.split('/').pop();
+      for (const f of fileSet) {
+        if (f.endsWith(path.sep + suffix + '.go') ||
+            f.includes(path.sep + suffix + path.sep)) {
+          found.push(f);
+          break;
+        }
+      }
+    }
+  }
+
+  // ── Rust ──────────────────────────────────────────────────────────────────
+  // Match `mod foo;` and `use crate::foo::bar` — resolve to sibling .rs files
+  if (RS_EXTS.has(ext)) {
+    const reMod = /^\s*(?:pub\s+)?mod\s+(\w+)\s*;/gm;
+    let m;
+    while ((m = reMod.exec(content)) !== null) {
+      const candidate = path.join(dir, m[1] + '.rs');
+      if (fileSet.has(candidate)) found.push(candidate);
+      // Also try mod/mod.rs
+      const candidate2 = path.join(dir, m[1], 'mod.rs');
+      if (fileSet.has(candidate2)) found.push(candidate2);
+    }
+  }
+
+  // ── Java / Kotlin / Scala ─────────────────────────────────────────────────
+  // Match same-project import statements by matching package-relative paths
+  if (JVM_EXTS.has(ext)) {
+    const re = /^\s*import\s+([\w.]+)\s*;?/gm;
+    let m;
+    while ((m = re.exec(content)) !== null) {
+      // Convert com.example.utils.StringHelper → com/example/utils/StringHelper.java
+      const asPath = m[1].replace(/\./g, path.sep);
+      for (const jvmExt of ['.java', '.kt', '.kts', '.scala', '.sc']) {
+        for (const f of fileSet) {
+          if (f.endsWith(asPath + jvmExt)) { found.push(f); break; }
+        }
+      }
+    }
+  }
+
+  // ── Ruby ──────────────────────────────────────────────────────────────────
+  if (RB_EXTS.has(ext)) {
+    const re = /^\s*require_relative\s+['"]([^'"]+)['"]/gm;
+    let m;
+    while ((m = re.exec(content)) !== null) {
+      const base = path.resolve(dir, m[1]);
+      const candidate  = base.endsWith('.rb') ? base : base + '.rb';
+      if (fileSet.has(candidate)) found.push(candidate);
+    }
+  }
+
+  return [...new Set(found)];
+}
+
+// ---------------------------------------------------------------------------
+// Public API
+// ---------------------------------------------------------------------------
+
+/**
+ * Build a forward and reverse dependency graph for all given files.
+ *
+ * @param {string[]} files - absolute file paths to analyze
+ * @param {string}   cwd   - project root (used only for error reporting)
+ * @returns {{ forward: Map<string,string[]>, reverse: Map<string,string[]> }}
+ */
+function build(files, cwd) {
+  const fileSet = new Set(files.map((f) => path.resolve(f)));
+  const forward = new Map();
+  const reverse = new Map();
+
+  // Initialise every known file in both maps (ensures isolated files appear)
+  for (const f of fileSet) {
+    if (!forward.has(f)) forward.set(f, []);
+    if (!reverse.has(f)) reverse.set(f, []);
+  }
+
+  for (const filePath of fileSet) {
+    let content;
+    try {
+      content = fs.readFileSync(filePath, 'utf8');
+    } catch (_) {
+      continue;
+    }
+
+    const deps = extractFileDeps(filePath, content, fileSet);
+    if (deps.length > 0) {
+      forward.set(filePath, deps);
+      for (const dep of deps) {
+        if (!reverse.has(dep)) reverse.set(dep, []);
+        reverse.get(dep).push(filePath);
+      }
+    }
+  }
+
+  return { forward, reverse };
+}
+
+/**
+ * Build a dependency graph scoped to a single cwd by walking all JS/TS/Py/Go
+ * files under srcDirs. Useful for the MCP tool handler.
+ *
+ * @param {string} cwd
+ * @param {object} [opts]
+ * @param {string[]} [opts.srcDirs]
+ * @param {string[]} [opts.exclude]
+ * @returns {{ forward: Map<string,string[]>, reverse: Map<string,string[]> }}
+ */
+function buildFromCwd(cwd, opts) {
+  const { srcDirs = ['src', 'app', 'lib'], exclude = ['node_modules', '.git', 'dist', 'build'] } = opts || {};
+  const excludeSet = new Set(exclude);
+
+  function walkDir(dir, depth) {
+    if (depth > 8) return [];
+    let entries;
+    try { entries = fs.readdirSync(dir, { withFileTypes: true }); } catch (_) { return []; }
+    const out = [];
+    for (const e of entries) {
+      if (excludeSet.has(e.name) || e.name.startsWith('.')) continue;
+      const full = path.join(dir, e.name);
+      if (e.isDirectory()) {
+        out.push(...walkDir(full, depth + 1));
+      } else if (e.isFile()) {
+        const ext = path.extname(e.name).toLowerCase();
+        if (JS_EXTS.has(ext) || PY_EXTS.has(ext) || GO_EXTS.has(ext) ||
+            RS_EXTS.has(ext) || JVM_EXTS.has(ext) || RB_EXTS.has(ext)) {
+          out.push(full);
+        }
+      }
+    }
+    return out;
+  }
+
+  const files = [];
+  for (const sd of srcDirs) {
+    const absDir = path.resolve(cwd, sd);
+    if (fs.existsSync(absDir)) files.push(...walkDir(absDir, 0));
+  }
+  // Also include root-level entry files
+  for (const rootFile of ['gen-context.js', 'index.js', 'main.js', 'app.js']) {
+    const abs = path.resolve(cwd, rootFile);
+    if (fs.existsSync(abs)) files.push(abs);
+  }
+
+  return build(files, cwd);
+}
+
+module.exports = { build, buildFromCwd, extractFileDeps };