sigmap 6.13.0 → 6.15.0

package/src/verify/hallucination-guard.js

@@ -0,0 +1,310 @@
+'use strict';
+
+/**
+ * Hallucination Guard — deterministic core (Reliable MVP, v6.15.0).
+ *
+ * Given the text of an AI answer, flag claims that do not match the repo:
+ *   - fake-file      : a referenced path is not on disk
+ *   - fake-test-file : a referenced *test* path is not on disk (sub-type)
+ *   - fake-import    : a relative import does not resolve; a bare import is
+ *                      absent from package.json deps (builtins allow-listed)
+ *   - fake-symbol    : a called function/class is absent from the symbol index
+ *   - fake-npm-script: `npm run X` where X is not a package.json script
+ *
+ * Each issue carries a `confidence` (detection certainty) and, where a near
+ * match exists, a heuristic `suggestion` ("Did you mean …?"). No network, no
+ * LLM. Reuses SigMap primitives (buildSigIndex) but every external dependency
+ * is injectable via `opts` so the core stays unit-testable.
+ */
+
+const fs = require('fs');
+const path = require('path');
+const parsers = require('./parsers');
+const { closestMatch, buildSymbolCandidates, formatSuggestion } = require('./closest-match');
+
+// A path that looks like a test file (JS/TS spec/test, Python test_/_test, or
+// a tests/__tests__ directory). Used to flag fake-test-file separately.
+const TEST_PATH_RE = /(?:\.(?:test|spec)\.[mc]?[jt]sx?$)|(?:(?:^|\/)__tests__\/)|(?:(?:^|\/)test_[^/]+\.py$)|(?:_test\.py$)|(?:(?:^|\/)tests?\/)/i;
+function isTestPath(p) { return TEST_PATH_RE.test(p); }
+
+const NODE_BUILTINS = new Set([
+  'fs', 'path', 'os', 'util', 'events', 'stream', 'http', 'https', 'crypto',
+  'child_process', 'url', 'querystring', 'assert', 'zlib', 'readline', 'net',
+  'tls', 'dns', 'buffer', 'process', 'vm', 'module', 'console', 'timers',
+  'string_decoder', 'perf_hooks', 'worker_threads', 'cluster', 'dgram', 'v8',
+  'tty', 'repl', 'async_hooks', 'inspector', 'fs/promises', 'path/posix',
+]);
+
+const PY_BUILTINS = new Set([
+  'os', 'sys', 're', 'json', 'math', 'typing', 'collections', 'itertools',
+  'functools', 'datetime', 'pathlib', 'subprocess', 'abc', 'dataclasses',
+  'enum', 'io', 'time', 'random', 'logging', 'argparse', 'unittest', 'asyncio',
+  'copy', 'hashlib', 'threading', 'string', 'csv', 'glob', 'shutil', 'tempfile',
+]);
+
+const LANG_GLOBALS = new Set([
+  // JS
+  'console', 'require', 'module', 'exports', 'process', 'Object', 'Array',
+  'String', 'Number', 'Boolean', 'Math', 'JSON', 'Date', 'Promise', 'Map',
+  'Set', 'WeakMap', 'WeakSet', 'RegExp', 'Error', 'Symbol', 'parseInt',
+  'parseFloat', 'isNaN', 'setTimeout', 'setInterval', 'clearTimeout', 'fetch',
+  'Buffer', 'Function', 'eval', 'encodeURIComponent', 'decodeURIComponent',
+  // Python
+  'print', 'len', 'range', 'str', 'int', 'float', 'dict', 'list', 'tuple',
+  'set', 'bool', 'open', 'enumerate', 'zip', 'map', 'filter', 'sorted',
+  'sum', 'min', 'max', 'abs', 'isinstance', 'super', 'type', 'getattr',
+  'setattr', 'hasattr',
+]);
+
+const REL_EXTS = ['', '.js', '.ts', '.tsx', '.jsx', '.mjs', '.cjs', '.json', '.py', '.r', '.R', '.vue'];
+const REL_INDEX = ['index.js', 'index.ts', 'index.tsx', 'index.jsx', '__init__.py'];
+
+/**
+ * Build the set of known symbol identifiers from the SigMap signature index,
+ * plus `{ name, file, line }` candidates (for closest-match suggestions).
+ */
+function buildSymbolSet(cwd) {
+  const set = new Set();
+  let fileKeys = [];
+  let symbolCandidates = [];
+  try {
+    const { buildSigIndex } = require('../retrieval/ranker');
+    const idx = buildSigIndex(cwd);
+    fileKeys = [...idx.keys()];
+    for (const sigs of idx.values()) {
+      for (const sig of sigs) {
+        const cleaned = String(sig).replace(/\s*:\d+(?:-\d+)?\s*$/, '');
+        const ids = cleaned.match(/[A-Za-z_$][\w$]*/g) || [];
+        for (const id of ids) set.add(id);
+      }
+    }
+    symbolCandidates = buildSymbolCandidates(idx);
+  } catch (_) {}
+  return { set, fileKeys, symbolCandidates };
+}
+
+/** Load declared dependency names from package.json. */
+function loadDeps(cwd) {
+  const deps = new Set();
+  let hasPkg = false;
+  try {
+    const pkg = JSON.parse(fs.readFileSync(path.join(cwd, 'package.json'), 'utf8'));
+    hasPkg = true;
+    for (const k of ['dependencies', 'devDependencies', 'peerDependencies', 'optionalDependencies']) {
+      if (pkg[k] && typeof pkg[k] === 'object') {
+        for (const name of Object.keys(pkg[k])) deps.add(name);
+      }
+    }
+  } catch (_) {}
+  return { deps, hasPkg };
+}
+
+/** Load the set of npm script names declared in package.json. */
+function loadScripts(cwd) {
+  const scripts = new Set();
+  try {
+    const pkg = JSON.parse(fs.readFileSync(path.join(cwd, 'package.json'), 'utf8'));
+    if (pkg.scripts && typeof pkg.scripts === 'object') {
+      for (const name of Object.keys(pkg.scripts)) scripts.add(name);
+    }
+  } catch (_) {}
+  return scripts;
+}
+
+/** Default file-existence check: resolve a referenced path against cwd. */
+function defaultFileExists(cwd, ref) {
+  const clean = ref.replace(/^\.\//, '');
+  for (const c of [path.resolve(cwd, clean), path.resolve(cwd, ref)]) {
+    try {
+      if (fs.existsSync(c)) return true;
+    } catch (_) {}
+  }
+  return false;
+}
+
+/** Default relative-import resolver: fs candidates + basename match in index. */
+function defaultRelativeResolvable(cwd, mod, fileBasenames) {
+  const base = path.resolve(cwd, mod);
+  for (const e of REL_EXTS) {
+    try {
+      if (fs.existsSync(base + e)) return true;
+    } catch (_) {}
+  }
+  for (const idx of REL_INDEX) {
+    try {
+      if (fs.existsSync(path.join(base, idx))) return true;
+    } catch (_) {}
+  }
+  // Fall back to basename match against the indexed file set (the answer's
+  // import is relative to a file we cannot know, so a name match is enough
+  // to avoid false positives).
+  const wantBase = path.basename(mod).replace(/\.[^.]+$/, '').toLowerCase();
+  return fileBasenames.has(wantBase);
+}
+
+/**
+ * Verify an AI answer against the repository.
+ *
+ * Each issue has the shape:
+ *   { type, value, line, location, message, confidence, suggestion }
+ * where `confidence` is the *detection* certainty ('high' for path/dep/script
+ * checks, 'medium' for symbol checks) and `suggestion` is a heuristic
+ * closest-match hint (or null).
+ *
+ * @param {string} answerText
+ * @param {string} cwd
+ * @param {object} [opts]
+ * @param {Set<string>} [opts.symbolSet]      override known symbols
+ * @param {Array}       [opts.symbolCandidates] override { name, file, line } list
+ * @param {Array<string>} [opts.fileCandidates]  override repo file paths (suggestions)
+ * @param {Set<string>} [opts.deps]           override package deps
+ * @param {Set<string>} [opts.scripts]        override package.json script names
+ * @param {boolean}     [opts.hasPkg]         whether a package.json exists
+ * @param {(ref: string) => boolean} [opts.fileExists]          override file check
+ * @param {(mod: string) => boolean} [opts.relativeResolvable]  override rel-import check
+ * @returns {{ issues: object[], summary: object }}
+ */
+function verify(answerText, cwd, opts = {}) {
+  let symbolSet = opts.symbolSet;
+  let fileBasenames = opts.fileBasenames;
+  let symbolCandidates = opts.symbolCandidates || [];
+  let fileCandidates = opts.fileCandidates || [];
+  if (!symbolSet) {
+    const built = buildSymbolSet(cwd);
+    symbolSet = built.set;
+    fileBasenames = new Set(built.fileKeys.map(
+      (k) => path.basename(k).replace(/\.[^.]+$/, '').toLowerCase()
+    ));
+    symbolCandidates = built.symbolCandidates;
+    fileCandidates = built.fileKeys;
+  }
+  if (!fileBasenames) fileBasenames = new Set();
+
+  let deps = opts.deps;
+  let hasPkg = opts.hasPkg;
+  if (!deps) {
+    const loaded = loadDeps(cwd);
+    deps = loaded.deps;
+    if (hasPkg === undefined) hasPkg = loaded.hasPkg;
+  }
+  const scripts = opts.scripts || (hasPkg ? loadScripts(cwd) : new Set());
+
+  const fileExists = opts.fileExists || ((ref) => defaultFileExists(cwd, ref));
+  const relativeResolvable = opts.relativeResolvable
+    || ((mod) => defaultRelativeResolvable(cwd, mod, fileBasenames));
+
+  // Pre-derive basename candidates for file suggestions (compare on basename so
+  // a wrong directory still surfaces the right file).
+  const fileBasenameCandidates = fileCandidates.map((f) => ({ name: path.basename(f), file: f }));
+
+  const issues = [];
+  const dedupe = new Set();
+  const add = (issue) => {
+    const key = `${issue.type}::${issue.value}`;
+    if (dedupe.has(key)) return;
+    dedupe.add(key);
+    if (!('suggestion' in issue)) issue.suggestion = null;
+    issue.location = `L${issue.line}`;
+    issues.push(issue);
+  };
+
+  // 1. fake-file / fake-test-file
+  for (const { path: p, line } of parsers.extractFilePaths(answerText)) {
+    if (fileExists(p)) continue;
+    const isTest = isTestPath(p);
+    const match = closestMatch(path.basename(p), fileBasenameCandidates, { minLen: 4 });
+    add({
+      type: isTest ? 'fake-test-file' : 'fake-file',
+      value: p,
+      line,
+      message: `${isTest ? 'Test file' : 'File'} not found on disk: ${p}`,
+      confidence: 'high',
+      suggestion: match ? formatSuggestion(match, false) : null,
+    });
+  }
+
+  // 2. fake-import
+  for (const imp of parsers.extractImports(answerText)) {
+    if (imp.relative) {
+      if (!relativeResolvable(imp.module)) {
+        add({ type: 'fake-import', value: imp.module, line: imp.line, message: `Import does not resolve: ${imp.module}`, confidence: 'high' });
+      }
+      continue;
+    }
+    // Bare module — only verifiable for JS when a package.json exists.
+    const top = imp.module.split('/')[0];
+    if (imp.kind === 'js') {
+      if (!hasPkg) continue;
+      if (NODE_BUILTINS.has(imp.module) || NODE_BUILTINS.has(top)) continue;
+      if (top.startsWith('@')) {
+        const scoped = imp.module.split('/').slice(0, 2).join('/');
+        if (deps.has(scoped) || deps.has(imp.module)) continue;
+      } else if (deps.has(top) || deps.has(imp.module)) {
+        continue;
+      }
+      const match = closestMatch(top, [...deps], { minLen: 3 });
+      add({
+        type: 'fake-import',
+        value: imp.module,
+        line: imp.line,
+        message: `Package not in dependencies: ${imp.module}`,
+        confidence: 'high',
+        suggestion: match ? formatSuggestion({ name: match.name }, false) : null,
+      });
+    }
+    // Python bare imports: stdlib is unbounded offline — skip to keep precision.
+  }
+
+  // 3. fake-symbol
+  if (symbolSet.size > 0) {
+    for (const { name, line } of parsers.extractSymbols(answerText)) {
+      if (symbolSet.has(name)) continue;
+      if (LANG_GLOBALS.has(name) || NODE_BUILTINS.has(name) || PY_BUILTINS.has(name)) continue;
+      const match = closestMatch(name, symbolCandidates, { minLen: 4 });
+      add({
+        type: 'fake-symbol',
+        value: name,
+        line,
+        message: `Symbol not found in repo index: ${name}()`,
+        confidence: 'medium',
+        suggestion: match ? formatSuggestion(match, true) : null,
+      });
+    }
+  }
+
+  // 4. fake-npm-script
+  if (hasPkg && scripts.size > 0) {
+    for (const { name, line } of parsers.extractNpmScripts(answerText)) {
+      if (scripts.has(name)) continue;
+      const match = closestMatch(name, [...scripts], { minLen: 2 });
+      add({
+        type: 'fake-npm-script',
+        value: name,
+        line,
+        message: `npm script not in package.json: ${name}`,
+        confidence: 'high',
+        suggestion: match ? formatSuggestion({ name: match.name }, false) : null,
+      });
+    }
+  }
+
+  issues.sort((a, b) => a.line - b.line);
+
+  const byType = {
+    'fake-file': 0, 'fake-test-file': 0, 'fake-import': 0,
+    'fake-symbol': 0, 'fake-npm-script': 0,
+  };
+  for (const i of issues) byType[i.type] = (byType[i.type] || 0) + 1;
+
+  const summary = {
+    total: issues.length,
+    byType,
+    clean: issues.length === 0,
+    symbolsIndexed: symbolSet.size,
+    withSuggestion: issues.filter((i) => i.suggestion).length,
+  };
+
+  return { issues, summary };
+}
+
+module.exports = { verify, buildSymbolSet, loadDeps, loadScripts, isTestPath };

package/src/verify/parsers.js

@@ -0,0 +1,200 @@
+'use strict';
+
+/**
+ * Parsers for the Hallucination Guard (verify-ai-output).
+ *
+ * Extract the verifiable claims an AI answer makes about a codebase:
+ *   - file paths it references
+ *   - import / require statements it shows
+ *   - function / class symbols it calls
+ *   - fenced code blocks (so callers can scope checks to code vs prose)
+ *
+ * Everything here is deterministic and offline — pure string analysis.
+ */
+
+// Extensions we are confident name a source/code/config file (no slash required).
+const KNOWN_CODE_EXT = new Set([
+  'js', 'jsx', 'mjs', 'cjs', 'ts', 'tsx', 'py', 'pyw', 'rb', 'go', 'rs',
+  'java', 'kt', 'swift', 'c', 'h', 'cpp', 'hpp', 'cs', 'php', 'r',
+  'vue', 'svelte', 'css', 'scss', 'less', 'html', 'json', 'yml', 'yaml',
+  'toml', 'xml', 'sql', 'graphql', 'gql', 'proto', 'tf', 'md', 'sh',
+  'gd', 'gdscript',
+]);
+
+/**
+ * Extract fenced code blocks.
+ * @param {string} text
+ * @returns {{ lang: string, content: string, line: number }[]}
+ */
+function extractCodeBlocks(text) {
+  const blocks = [];
+  const lines = text.split('\n');
+  let inBlock = false;
+  let lang = '';
+  let buf = [];
+  let startLine = 0;
+  for (let i = 0; i < lines.length; i++) {
+    const m = lines[i].match(/^```(\w*)/);
+    if (m) {
+      if (!inBlock) {
+        inBlock = true;
+        lang = m[1] || '';
+        buf = [];
+        startLine = i + 2; // first content line (1-based)
+      } else {
+        blocks.push({ lang, content: buf.join('\n'), line: startLine });
+        inBlock = false;
+      }
+      continue;
+    }
+    if (inBlock) buf.push(lines[i]);
+  }
+  return blocks;
+}
+
+/**
+ * Extract file-path references (deduped, first-seen line kept).
+ * A token counts as a path when it has a `.<letter…>` extension AND
+ * either contains a `/` or carries a known code/config extension.
+ * @param {string} text
+ * @returns {{ path: string, line: number }[]}
+ */
+function extractFilePaths(text) {
+  const lines = text.split('\n');
+  const seen = new Map();
+  const re = /(?:^|[\s`"'(\[<])([A-Za-z0-9_][\w./-]*\.[A-Za-z][A-Za-z0-9]*)/g;
+  for (let i = 0; i < lines.length; i++) {
+    const line = lines[i];
+    let m;
+    re.lastIndex = 0;
+    while ((m = re.exec(line)) !== null) {
+      const p = m[1];
+      if (/^https?:/i.test(p)) continue;
+      const ext = (p.split('.').pop() || '').toLowerCase();
+      const hasSlash = p.includes('/');
+      if (!hasSlash && !KNOWN_CODE_EXT.has(ext)) continue;
+      if (!seen.has(p)) seen.set(p, i + 1);
+    }
+  }
+  return [...seen.entries()].map(([p, line]) => ({ path: p, line }));
+}
+
+/**
+ * Extract import / require statements.
+ * @param {string} text
+ * @returns {{ module: string, kind: 'js'|'py', relative: boolean, line: number, raw: string }[]}
+ */
+function extractImports(text) {
+  const lines = text.split('\n');
+  const out = [];
+  const push = (module, kind, line, raw) => {
+    if (!module) return;
+    out.push({ module, kind, relative: /^[./]/.test(module), line, raw: raw.trim() });
+  };
+  for (let i = 0; i < lines.length; i++) {
+    const line = lines[i];
+    let m;
+    // JS/TS: import ... from 'x'  |  export ... from 'x'
+    if ((m = line.match(/\b(?:import|export)\b[^'"]*\bfrom\s*['"]([^'"]+)['"]/))) {
+      push(m[1], 'js', i + 1, line);
+    } else if ((m = line.match(/\bimport\s*['"]([^'"]+)['"]/))) {
+      // side-effect import 'x'
+      push(m[1], 'js', i + 1, line);
+    }
+    // require('x') / dynamic import('x') — may co-occur, scan separately
+    const reqRe = /\b(?:require|import)\s*\(\s*['"]([^'"]+)['"]\s*\)/g;
+    let r;
+    while ((r = reqRe.exec(line)) !== null) push(r[1], 'js', i + 1, line);
+
+    // TS: import X = require('mod')
+    if ((m = line.match(/\bimport\s+[A-Za-z_$][\w$]*\s*=\s*require\s*\(\s*['"]([^'"]+)['"]\s*\)/))) {
+      push(m[1], 'js', i + 1, line);
+    }
+
+    // Python: from x import y  |  import x
+    if ((m = line.match(/^\s*from\s+([.\w]+)\s+import\b/))) {
+      push(m[1], 'py', i + 1, line);
+    } else if ((m = line.match(/^\s*import\s+([A-Za-z_][\w.]*)/))) {
+      push(m[1], 'py', i + 1, line);
+    }
+  }
+
+  // Multi-line JS/TS imports, e.g.
+  //   import {
+  //     A as B,
+  //   } from './mod';
+  // The per-line pass above misses these because `from '…'` sits on a later
+  // line. Trigger only when the opening line has no quote and no `from` yet,
+  // then gather forward until the source string appears.
+  for (let i = 0; i < lines.length; i++) {
+    const start = lines[i];
+    if (!/^\s*(?:import|export)\b/.test(start)) continue;
+    if (/['"]/.test(start) || /\bfrom\b/.test(start)) continue; // single-line, already handled
+    let joined = start;
+    for (let j = i + 1; j < Math.min(lines.length, i + 12); j++) {
+      joined += ' ' + lines[j];
+      const fm = joined.match(/\bfrom\s*['"]([^'"]+)['"]/);
+      if (fm) { push(fm[1], 'js', i + 1, start.trim()); break; }
+      if (/['"]/.test(lines[j]) && !/\bfrom\b/.test(joined)) break; // a string that isn't a source — bail
+    }
+  }
+  return out;
+}
+
+/**
+ * Extract npm/pnpm/yarn script invocations (`npm run <name>`).
+ * Only the explicit `run` form is matched, to avoid confusing package-manager
+ * subcommands (`yarn add`, `pnpm install`) with script names.
+ * @param {string} text
+ * @returns {{ name: string, line: number }[]}
+ */
+function extractNpmScripts(text) {
+  const lines = text.split('\n');
+  const out = [];
+  const seen = new Set();
+  const re = /\b(?:npm|pnpm|yarn)\s+run(?:-script)?\s+([A-Za-z0-9:_-]+)/g;
+  for (let i = 0; i < lines.length; i++) {
+    let m;
+    re.lastIndex = 0;
+    while ((m = re.exec(lines[i])) !== null) {
+      const name = m[1];
+      if (seen.has(name)) continue;
+      seen.add(name);
+      out.push({ name, line: i + 1 });
+    }
+  }
+  return out;
+}
+
+/**
+ * Extract function/class symbol references that look like calls.
+ * Restricted to backtick-wrapped calls (`foo(...)`) for high precision.
+ * @param {string} text
+ * @returns {{ name: string, line: number }[]}
+ */
+function extractSymbols(text) {
+  const lines = text.split('\n');
+  const out = [];
+  const seen = new Set();
+  const re = /`([A-Za-z_$][\w$]*)\s*\([^`]*\)`/g;
+  for (let i = 0; i < lines.length; i++) {
+    let m;
+    re.lastIndex = 0;
+    while ((m = re.exec(lines[i])) !== null) {
+      const name = m[1];
+      const key = name + '@' + (i + 1);
+      if (seen.has(key)) continue;
+      seen.add(key);
+      out.push({ name, line: i + 1 });
+    }
+  }
+  return out;
+}
+
+module.exports = {
+  extractCodeBlocks,
+  extractFilePaths,
+  extractImports,
+  extractSymbols,
+  extractNpmScripts,
+};