pi-supernova 0.0.15 → 0.2.0

package/snap.js

@@ -1,7 +1,8 @@
-
 import * as path from "node:path";
 import { isString } from "./decode.js";
+import { truncateChars } from "./format.js";
 import { WorkspaceIndex } from "./repo-index.js";
+import { extractStructuralSurface } from "./surface.js";
 import { isTestPath } from "./workspace.js";
 
 const STOP_WORDS = new Set([
@@ -9,240 +10,208 @@ const STOP_WORDS = new Set([
   "by", "from", "is", "it", "this", "that", "where", "how", "what", "which",
   "file", "code", "function", "class", "method", "find", "get", "look",
 ]);
+const SOURCE_EXT = new Set([".ts", ".tsx", ".js", ".jsx", ".mjs", ".cjs", ".rs", ".py", ".go"]);
+const TYPED_EXT = new Set([".ts", ".tsx", ".rs", ".go"]);
+const MAX_SEARCH_CHARS = 2 * 1024 * 1024;
+const MAX_ALTERNATIVES = 3;
 
 export function tokenizeQuery(query) {
-  if (!isString(query) || !query.trim()) {
-    return { tokens: [], wantsTest: false, wantsType: false, wantsDoc: false };
-  }
-
-  const raw = query
-    .replace(/([a-z])([A-Z])/g, "$1 $2")
-    .toLowerCase()
-    .split(/[^a-zA-Z0-9_]+/);
-
-  const tokens = raw.filter((t) => t.length > 1 && !STOP_WORDS.has(t));
-  const queryLower = query.toLowerCase();
-
+  if (!isString(query) || !query.trim()) return { tokens: [], wantsTest: false, wantsType: false, wantsDoc: false };
+  const words = query.replace(/([a-z])([A-Z])/g, "$1 $2").toLowerCase().split(/[^a-zA-Z0-9_]+/);
   return {
-    tokens: [...new Set(tokens)],
-    wantsTest: queryLower.includes("test") || queryLower.includes("spec"),
-    wantsType: queryLower.includes("type") || queryLower.includes("interface") || queryLower.includes("schema"),
-    wantsDoc: queryLower.includes("doc") || queryLower.includes("readme"),
+    tokens: [...new Set(words.filter(word => word.length > 1 && !STOP_WORDS.has(word)))],
+    wantsTest: words.some(word => ["test", "tests", "testing", "spec", "specs"].includes(word)),
+    wantsType: words.some(word => ["type", "types", "interface", "interfaces", "schema", "schemas"].includes(word)),
+    wantsDoc: words.some(word => ["doc", "docs", "documentation", "readme"].includes(word)),
   };
 }
 
-const SOURCE_EXT = new Set([".ts", ".js", ".mjs", ".rs", ".py", ".go"]);
-const TYPED_EXT = new Set([".ts", ".d.ts", ".rs", ".go"]);
-const VENDOR_SEGMENTS = ["node_modules/", "dist/", "target/"];
-
-function tokenPathScore(token, basename, pathParts, norm) {
-  if (basename === token || basename.startsWith(token + ".")) return 60;
-  if (basename.includes(token)) return 30;
-  if (pathParts.includes(token)) return 15;
-  if (norm.includes(token)) return 5;
-  return 0;
-}
-
-function extensionBonus(ext, { wantsDoc, wantsType }) {
-  let bonus = 0;
-  if (SOURCE_EXT.has(ext) && !wantsDoc) bonus += 5;
-  if (wantsType && TYPED_EXT.has(ext)) bonus += 10;
-  return bonus;
-}
-
 export function scorePathTopology(filePath, tokens, flags) {
-  const norm = filePath.replaceAll("\\", "/").toLowerCase();
-  const isTest = norm.includes("test") || norm.includes("spec") || norm.includes("__tests__");
-  if (isTest && !flags.wantsTest) return -50;
-  if (!isTest && flags.wantsTest) return -20;
-  if (VENDOR_SEGMENTS.some((segment) => norm.includes(segment))) return -100;
-
-  const basename = path.basename(norm);
-  const pathParts = norm.split(/[^a-zA-Z0-9]+/);
-  let score = extensionBonus(path.extname(norm), flags);
-  for (const token of tokens) score += tokenPathScore(token, basename, pathParts, norm);
-  return score;
-}
-
-function isSkippableLine(lower) {
-  return !lower || lower.startsWith("//") || lower.startsWith("#") || lower.startsWith("*");
-}
-
-/** A line defines a token only when the declared name contains it; `const x = foo(token)` is a mention. */
-function lineScoreFor(lower, tokens, definedName) {
-  let lineScore = 0;
+  const normalized = filePath.replaceAll("\\", "/").toLowerCase();
+  const parts = normalized.split("/");
+  if (parts.some(part => ["node_modules", "dist", "target"].includes(part))) return -100;
+  const test = isTestPath(normalized);
+  if (test && !flags.wantsTest) return -50;
+  if (!test && flags.wantsTest) return -20;
+  const base = path.basename(normalized);
+  const words = normalized.split(/[^a-zA-Z0-9]+/);
+  const ext = path.extname(normalized);
+  let score = SOURCE_EXT.has(ext) && !flags.wantsDoc ? 5 : 0;
+  if (flags.wantsType && TYPED_EXT.has(ext)) score += 10;
   for (const token of tokens) {
-    if (!lower.includes(token)) continue;
-    lineScore += definedName.includes(token) ? 40 : 5;
+    if (base === token || base.startsWith(token + ".")) score += 60;
+    else if (base.includes(token)) score += 30;
+    else if (words.includes(token)) score += 15;
+    else if (normalized.includes(token)) score += 5;
   }
-  return lineScore;
+  return score;
 }
 
-// Mentions are capped so a file that calls a symbol many times cannot outrank the file that defines it.
-const MAX_MENTION_SCORE = 60;
-
-function scoreContentDefinitions(entry, tokens) {
-  const { lower, defNames } = WorkspaceIndex.linesOf(entry);
-  let defScore = 0;
-  let mentionScore = 0;
-  let bestLine = 1;
-  let bestLineScore = 0;
-  for (let i = 0; i < lower.length; i++) {
-    if (isSkippableLine(lower[i])) continue;
-    const lineScore = lineScoreFor(lower[i], tokens, defNames[i]);
-    if (lineScore > bestLineScore) {
-      bestLineScore = lineScore;
-      bestLine = i + 1;
+function inScope(filePath, dir, includeHidden) {
+  const relative = path.relative(dir, filePath);
+  if (relative === ".." || relative.startsWith(".." + path.sep) || path.isAbsolute(relative)) return false;
+  const parts = relative.split(path.sep);
+  return !parts.includes(".git") && (includeHidden || !parts.some(part => part.startsWith(".") && part.length > 1));
+}
+
+function makeCandidate(filePath, dir, query, tokens, flags) {
+  const relative = path.relative(dir, filePath);
+  const lower = relative.toLowerCase();
+  const base = path.basename(lower);
+  const exactPath = lower === query.toLowerCase() || base === query.toLowerCase()
+    || base.slice(0, -path.extname(base).length) === query.toLowerCase();
+  return { path: filePath, pathScore: scorePathTopology(relative, tokens, flags), exactPath,
+    pathCoverage: tokens.filter(token => lower.includes(token)).length,
+    matched: new Set(), exactDefinition: false, definitionCoverage: 0, lineCoverage: 0,
+    line: 1, signature: "", context: new Map(), recent: [], anchorScore: -1 };
+}
+
+function inspectLine(candidate, lineNumber, raw, query, tokens, isMatch) {
+  const text = raw.replace(/\r?\n$/, "");
+  const lower = text.toLowerCase();
+  if (isMatch) {
+    const matches = tokens.filter(token => lower.includes(token));
+    for (const token of matches) candidate.matched.add(token);
+    const ext = path.extname(candidate.path).toLowerCase();
+    const items = SOURCE_EXT.has(ext) ? extractStructuralSurface(text, ext).items : [];
+    let declaration;
+    let definitionCoverage = 0;
+    let exact = false;
+    for (const item of items) {
+      const name = item.name.toLowerCase();
+      const itemExact = name === query.toLowerCase();
+      const coverage = tokens.filter(token => name.includes(token)).length;
+      if (itemExact || coverage > definitionCoverage) { declaration = item; definitionCoverage = coverage; exact = itemExact; }
+      if (exact) break;
+    }
+    const score = (exact ? 10000 : 0) + definitionCoverage * 40 + matches.length;
+    if (score > candidate.anchorScore) {
+      candidate.anchorScore = score;
+      candidate.line = lineNumber;
+      candidate.signature = truncateChars(declaration?.signature ?? "", 240, "signature").text;
+      candidate.exactDefinition = exact;
+      candidate.definitionCoverage = definitionCoverage;
+      candidate.lineCoverage = matches.length;
+      candidate.context.clear();
+      for (const [number, line] of candidate.recent) if (number >= lineNumber - 2) candidate.context.set(number, line);
     }
-    if (defNames[i]) defScore += lineScore;
-    else mentionScore += lineScore;
-  }
-  return { totalScore: defScore + Math.min(mentionScore, MAX_MENTION_SCORE), bestLine, bestLineScore };
-}
-
-function relativeHasSegment(relativePath, segmentName) {
-  return relativePath.split(path.sep).includes(segmentName);
-}
-
-function relativeHasHiddenSegment(relativePath) {
-  return relativePath.split(path.sep).some((segment) => segment.startsWith(".") && segment.length > 1);
-}
-
-async function listCandidateFiles(dir, includeHidden, index) {
-  return (await index.files(dir, includeHidden)).slice();
-}
-
-function mergePendingPaths(fileList, pendingPaths, dir, includeHidden = false) {
-  const resolvedDir = path.resolve(dir);
-  const seenPaths = new Set(fileList.map((filePath) => path.resolve(filePath)));
-  for (const pendingPath of pendingPaths) {
-    const absolutePath = path.resolve(pendingPath);
-    const relativePath = path.relative(resolvedDir, absolutePath);
-    const escapesDir = relativePath === ".." || relativePath.startsWith(`..${path.sep}`) || path.isAbsolute(relativePath);
-    const hiddenRelativePath = relativeHasHiddenSegment(relativePath);
-    if (escapesDir || relativeHasSegment(relativePath, ".git") || (!includeHidden && hiddenRelativePath) || seenPaths.has(absolutePath)) continue;
-    seenPaths.add(absolutePath);
-    fileList.push(absolutePath);
-  }
-  return fileList;
-}
-
-function mergeGrepHits(candidates, grepHits) {
-  const seen = new Set(candidates);
-  for (const h of grepHits) {
-    if (seen.has(h)) continue;
-    seen.add(h);
-    candidates.push(h);
-    if (candidates.length >= 15) break;
   }
-  return candidates;
-}
-
-function expandCandidatesWithGrep(candidates, fileList, tokens, flags, index) {
-  if (candidates.length >= 5) return candidates;
-  const salient = tokens.filter((t) => t.length > 2).slice(0, 4);
-  const scope = flags.wantsTest ? fileList : fileList.filter((f) => !isTestPath(f));
-  const hits = index.filesContaining(scope, salient, true);
-  mergeGrepHits(candidates, hits);
-  if (candidates.length === 0) return fileList.slice(0, 5);
-  return candidates;
-}
-
-function scoreSurfaceItems(items, tokens, fallbackLine) {
-  let bonus = 0;
-  let best = null;
-  let bestMatches = 0;
-  for (const item of items) {
-    const nameLower = item.name.toLowerCase();
-    const matches = tokens.filter((token) => nameLower.includes(token)).length;
-    if (matches === 0) continue;
-    bonus += matches * (item.isExport ? 80 : 50);
-    if (matches > bestMatches) {
-      bestMatches = matches;
-      best = item;
+  const excerpt = truncateChars(text, 240, "source line").text;
+  if (lineNumber >= candidate.line - 2 && lineNumber <= candidate.line + 4) candidate.context.set(lineNumber, excerpt);
+  candidate.recent.push([lineNumber, excerpt]);
+  if (candidate.recent.length > 2) candidate.recent.shift();
+}
+
+function inspectOverlay(candidate, text, needles, query, tokens) {
+  const lines = text.split("\n");
+  const matches = [];
+  for (let i = 0; i < lines.length; i++) if (needles.some(needle => lines[i].toLowerCase().includes(needle))) matches.push(i);
+  for (const i of matches) inspectLine(candidate, i + 1, lines[i], query, tokens, true);
+  candidate.context.clear();
+  for (let i = Math.max(0, candidate.line - 3); i < Math.min(lines.length, candidate.line + 4); i++) candidate.context.set(i + 1, truncateChars(lines[i], 240, "source line").text);
+}
+
+async function contentCandidates({ dir, includeHidden, query, tokens, flags, fileSet, index, overlayText, signal, exact, diskFiles }) {
+  const needles = exact ? [query.toLowerCase()] : tokens;
+  const args = ["rg", "--json", "--fixed-strings", "--ignore-case", "--before-context", "2", "--after-context", "4"];
+  if (includeHidden) args.push("--hidden");
+  args.push("-g", "!.git/**", "-g", "!**/.git/**");
+  for (const needle of needles) args.push("-e", needle);
+  args.push("--", dir);
+  const response = diskFiles ? await index.runCommand(args, { cwd: dir, timeoutMs: 15000, maxOutputChars: MAX_SEARCH_CHARS, signal })
+    : { stdout: "", stderr: "", exitCode: 1 };
+  if (response.exitCode !== 0 && response.exitCode !== 1) throw new Error("source search failed: " + response.stderr.trim());
+  const candidates = new Map();
+  const records = response.stdout.split("\n");
+  for (let i = 0; i < records.length; i++) {
+    if (!records[i]) continue;
+    let record;
+    try { record = JSON.parse(records[i]); } catch (error) {
+      if (response.outputTruncated && i === records.length - 1) break;
+      throw error;
     }
+    if (record.type !== "match" && record.type !== "context") continue;
+    const data = record.data;
+    if (!data?.path?.text || !isString(data.lines?.text)) continue;
+    const filePath = path.resolve(dir, data.path.text);
+    if (!fileSet.has(filePath) || overlayText(filePath) !== undefined) continue;
+    let candidate = candidates.get(filePath);
+    if (!candidate) {
+      candidate = makeCandidate(filePath, dir, query, tokens, flags);
+      candidates.set(filePath, candidate);
+    }
+    inspectLine(candidate, data.line_number, data.lines.text, query, tokens, record.type === "match");
   }
-  return { bonus, signature: best?.signature ?? "", anchorLine: best?.line ?? fallbackLine };
-}
-
-function scoreCandidateContents(candidates, tokens, flags, index, overlayText) {
-  const candidateScores = [];
-  for (const filePath of candidates) {
+  for (const filePath of fileSet) {
     const pending = overlayText(filePath);
-    const entry = pending === undefined ? index.entry(filePath) : WorkspaceIndex.fromText(filePath, pending);
-    if (!entry) continue;
-    const content = entry.text;
-    const { totalScore, bestLine, bestLineScore } = scoreContentDefinitions(entry, tokens);
-    const surface = WorkspaceIndex.surfaceOf(entry);
-    const { bonus: surfaceBonus, signature, anchorLine } = scoreSurfaceItems(surface.items, tokens, bestLine);
-    const lowerPath = filePath.toLowerCase();
-    const isTestFile = lowerPath.includes("test") || lowerPath.includes("spec");
-    const testAdjustment = !isTestFile ? 0 : (flags.wantsTest ? 100 : -200);
-    candidateScores.push({
-      path: filePath,
-      score: totalScore + surfaceBonus + (bestLineScore * 2) + testAdjustment,
-      anchorLine,
-      signature,
-      content,
-    });
+    if (pending === undefined) continue;
+    const candidate = makeCandidate(filePath, dir, query, tokens, flags);
+    inspectOverlay(candidate, pending, needles, query, tokens);
+    if (candidate.matched.size) candidates.set(filePath, candidate);
   }
-  candidateScores.sort((a, b) => b.score - a.score);
-  return candidateScores;
+  return { candidates, truncated: response.outputTruncated === true };
 }
 
-function rankCandidates(fileList, tokens, flags, index, overlayText) {
-  const scoredPaths = [];
-  for (const f of fileList) {
-    const score = scorePathTopology(f, tokens, flags);
-    if (score > 0) scoredPaths.push({ path: f, score });
-  }
-  scoredPaths.sort((a, b) => b.score - a.score);
-  const selected = scoredPaths.filter((p) => p.score >= 25).slice(0, 10).map((p) => p.path);
-  const candidates = expandCandidatesWithGrep(selected, fileList, tokens, flags, index);
-  const candidateScores = scoreCandidateContents(candidates, tokens, flags, index, overlayText);
-  return { candidates, candidateScores };
+function rankScore(candidate, tokenCount) {
+  return (candidate.exactDefinition ? 10000 : 0) + (candidate.exactPath ? 500 : 0)
+    + candidate.definitionCoverage / tokenCount * 100 + candidate.matched.size / tokenCount * 30
+    + candidate.pathCoverage / tokenCount * 20 + candidate.lineCoverage / tokenCount * 10
+    + Math.max(-40, Math.min(20, candidate.pathScore / 5));
 }
 
-function buildSnapResult(candidates, candidateScores, fileList, root) {
-  const relative = (p) => path.relative(root, p) || p;
-  if (candidateScores.length === 0 || candidateScores[0].score <= 0) {
-    return { path: relative(candidates[0] || fileList[0]), line: 1, signature: "", confidence: 0.3, context: [] };
-  }
-  const best = candidateScores[0];
-  const lines = best.content.split("\n");
-  // Two lines before and four after: enough to confirm the hit; read() is the tool for more.
-  const startLine = Math.max(1, best.anchorLine - 2);
-  const endLine = Math.min(lines.length, best.anchorLine + 4);
-  const context = [];
-  for (let l = startLine; l <= endLine; l++) {
-    const marker = l === best.anchorLine ? "►" : " ";
-    context.push(marker + l + " " + lines[l - 1]);
-  }
-  const confidence = Math.min(0.98, Math.max(0.65, best.score / 150));
-  return {
-    path: relative(best.path),
-    line: best.anchorLine,
-    signature: best.signature,
-    confidence: Number(confidence.toFixed(2)),
-    context,
-  };
-}
-
-export async function executeSnap({ query, searchDir, root, includeHidden = false, index, overlayText = () => undefined, pendingPaths = [] }) {
-  const { tokens, wantsTest, wantsType, wantsDoc } = tokenizeQuery(query);
-  if (tokens.length === 0) {
-    throw new Error("snap requires at least one searchable concept keyword");
-  }
-  const dir = searchDir || process.cwd();
-  if (path.resolve(dir).split(path.sep).includes(".git")) {
-    throw new Error("snap cannot search Git metadata");
+function location(candidate, root, index, overlayText) {
+  let context = candidate.context;
+  if (context.size === 0) {
+    const pending = overlayText(candidate.path);
+    const entry = pending === undefined ? index.entry(candidate.path) : WorkspaceIndex.fromText(candidate.path, pending);
+    const lines = entry?.text.split("\n") ?? [];
+    context = new Map(lines.slice(0, 7).map((line, i) => [i + 1, truncateChars(line, 240, "source line").text]));
   }
-  const fileList = await listCandidateFiles(dir, includeHidden, index);
-  mergePendingPaths(fileList, pendingPaths, dir, includeHidden);
-  if (fileList.length === 0) {
-    throw new Error(`no files found to search in ${dir}`);
+  return { path: path.relative(root, candidate.path), line: candidate.line, signature: candidate.signature,
+    context: [...context].sort((a, b) => a[0] - b[0]).map(([line, text]) => (line === candidate.line ? "►" : " ") + line + " " + text) };
+}
+
+export async function executeSnap({ query, searchDir, root, includeHidden = false, index, overlayText = () => undefined, pendingPaths = [], signal }) {
+  const flags = tokenizeQuery(query);
+  const { tokens } = flags;
+  if (tokens.length === 0) throw new Error("read requires a file path or a searchable source question");
+  if (tokens.length > 16) throw new Error("source question is too broad; use at most 16 keywords");
+  query = query.trim();
+  const dir = path.resolve(searchDir || process.cwd());
+  if (dir.split(path.sep).includes(".git")) throw new Error("cannot search Git metadata");
+  signal?.throwIfAborted();
+  flags.wantsTest ||= isTestPath(path.relative(root ?? dir, dir));
+  const files = await index.files(dir, includeHidden, signal);
+  const fileSet = new Set(files.filter(file => inScope(file, dir, includeHidden)));
+  for (const pending of pendingPaths) if (inScope(pending, dir, includeHidden)) fileSet.add(path.resolve(pending));
+  const listing = index.lists.get(dir + "\0" + (includeHidden ? "h" : ""));
+  if (listing?.error && !(listing.missing && fileSet.size)) throw new Error("source file listing failed: " + listing.error);
+  const empty = { path: null, line: null, signature: "", confidence: 0, context: [] };
+  if (fileSet.size === 0 && !listing?.truncated) return { ...empty, status: "not_found" };
+  const exact = /^[a-zA-Z_$][\w$]*$/.test(query);
+  const search = await contentCandidates({ dir, includeHidden, query, tokens, flags, fileSet, index, overlayText, signal, exact, diskFiles: files.length });
+  for (const filePath of fileSet) {
+    if (search.candidates.has(filePath)) continue;
+    const relative = path.relative(dir, filePath).toLowerCase();
+    if (!tokens.some(token => relative.includes(token))) continue;
+    if (exact && tokens.length > 1 && !relative.includes(query.toLowerCase())) continue;
+    const candidate = makeCandidate(filePath, dir, query, tokens, flags);
+    if (candidate.pathScore > 0) search.candidates.set(filePath, candidate);
   }
-  const flags = { wantsTest, wantsDoc, wantsType };
-  const { candidates, candidateScores } = rankCandidates(fileList, tokens, flags, index, overlayText);
-  return buildSnapResult(candidates, candidateScores, fileList, root ?? dir);
+  const ranked = [...search.candidates.values()].filter(candidate => candidate.pathScore > -50)
+    .map(candidate => ({ ...candidate, score: rankScore(candidate, tokens.length) }))
+    .sort((a, b) => b.score - a.score || a.path.localeCompare(b.path));
+  const incomplete = search.truncated || listing?.truncated === true;
+  const relativeRoot = root ?? dir;
+  const candidates = ranked.slice(0, MAX_ALTERNATIVES).map(candidate => location(candidate, relativeRoot, index, overlayText));
+  if (incomplete) return { ...empty, status: "incomplete", candidates, message: "Search output exceeded its budget. Narrow the directory with read(path, {about: question})." };
+  if (!ranked.length) return { ...empty, status: "not_found" };
+  const best = ranked[0];
+  const second = ranked[1];
+  const margin = second ? (best.score - second.score) / Math.max(1, best.score) : 1;
+  const coverage = Math.max(best.matched.size, best.pathCoverage) / tokens.length;
+  const uniqueExact = best.exactDefinition && !second?.exactDefinition || best.exactPath && !second?.exactPath && !second?.exactDefinition;
+  if (!uniqueExact && (coverage < 0.6 || margin < 0.15 || best.definitionCoverage / tokens.length < 0.5)) return { ...empty, status: "ambiguous", candidates };
+  const confidence = uniqueExact ? 0.95 : Math.min(0.85, 0.5 + coverage * 0.2 + margin * 0.15);
+  return { ...candidates[0], status: "found", confidence: Number(confidence.toFixed(2)) };
 }