pi-supernova 0.0.8 → 0.0.11

package/snap.js

@@ -1,7 +1,7 @@
 
 import * as path from "node:path";
 import { isString } from "./decode.js";
-import { extractStructuralSurface } from "./surface.js";
+import { WorkspaceIndex } from "./repo-index.js";
 
 const STOP_WORDS = new Set([
   "the", "a", "an", "and", "or", "in", "on", "at", "to", "for", "of", "with",
@@ -30,72 +30,73 @@ export function tokenizeQuery(query) {
   };
 }
 
-export function scorePathTopology(filePath, tokens, { wantsTest, wantsDoc, wantsType }) {
-  const norm = filePath.replace(/\\/g, "/").toLowerCase();
-  const basename = path.basename(norm);
-  const ext = path.extname(norm);
-
-  const isTest = norm.includes("test") || norm.includes("spec") || norm.includes("__tests__");
-  if (isTest && !wantsTest) return -50;
-  if (!isTest && wantsTest) return -20;
+const SOURCE_EXT = new Set([".ts", ".js", ".mjs", ".rs", ".py", ".go"]);
+const TYPED_EXT = new Set([".ts", ".d.ts", ".rs", ".go"]);
+const VENDOR_SEGMENTS = ["node_modules/", "dist/", "target/"];
 
-  if (norm.includes("node_modules/") || norm.includes("dist/") || norm.includes("target/")) {
-    return -100;
-  }
-
-  let score = 0;
-  const pathParts = norm.split(/[^a-zA-Z0-9]+/);
+function tokenPathScore(token, basename, pathParts, norm) {
+  if (basename === token || basename.startsWith(token + ".")) return 60;
+  if (basename.includes(token)) return 30;
+  if (pathParts.includes(token)) return 15;
+  if (norm.includes(token)) return 5;
+  return 0;
+}
 
-  for (const token of tokens) {
-    if (basename === token || basename.startsWith(token + ".")) score += 60;
-    else if (basename.includes(token)) score += 30;
-    else if (pathParts.includes(token)) score += 15;
-    else if (norm.includes(token)) score += 5;
-  }
+function extensionBonus(ext, { wantsDoc, wantsType }) {
+  let bonus = 0;
+  if (SOURCE_EXT.has(ext) && !wantsDoc) bonus += 5;
+  if (wantsType && TYPED_EXT.has(ext)) bonus += 10;
+  return bonus;
+}
 
-  if ([".ts", ".js", ".mjs", ".rs", ".py", ".go"].includes(ext) && !wantsDoc) {
-    score += 5;
-  }
-  if (wantsType && [".ts", ".d.ts", ".rs", ".go"].includes(ext)) {
-    score += 10;
-  }
+export function scorePathTopology(filePath, tokens, flags) {
+  const norm = filePath.replaceAll("\\", "/").toLowerCase();
+  const isTest = norm.includes("test") || norm.includes("spec") || norm.includes("__tests__");
+  if (isTest && !flags.wantsTest) return -50;
+  if (!isTest && flags.wantsTest) return -20;
+  if (VENDOR_SEGMENTS.some((segment) => norm.includes(segment))) return -100;
 
+  const basename = path.basename(norm);
+  const pathParts = norm.split(/[^a-zA-Z0-9]+/);
+  let score = extensionBonus(path.extname(norm), flags);
+  for (const token of tokens) score += tokenPathScore(token, basename, pathParts, norm);
   return score;
 }
 
-function isSkippableLine(line) {
-  return !line || line.startsWith("//") || line.startsWith("#") || line.startsWith("*");
+function isSkippableLine(lower) {
+  return !lower || lower.startsWith("//") || lower.startsWith("#") || lower.startsWith("*");
 }
 
-function lineScoreFor(line, tokens, isDef) {
-  const lower = line.toLowerCase();
+/** A line defines a token only when the declared name contains it; `const x = foo(token)` is a mention. */
+function lineScoreFor(lower, tokens, definedName) {
   let lineScore = 0;
   for (const token of tokens) {
-    if (lower.includes(token)) lineScore += isDef ? 40 : 5;
+    if (!lower.includes(token)) continue;
+    lineScore += definedName.includes(token) ? 40 : 5;
   }
   return lineScore;
 }
 
-function accumulateContentScore(lines, tokens, defPattern) {
-  let score = 0;
+// Mentions are capped so a file that calls a symbol many times cannot outrank the file that defines it.
+const MAX_MENTION_SCORE = 60;
+
+function scoreContentDefinitions(entry, tokens) {
+  const { lower, defNames } = WorkspaceIndex.linesOf(entry);
+  let defScore = 0;
+  let mentionScore = 0;
   let bestLine = 1;
   let bestLineScore = 0;
-  for (let i = 0; i < lines.length; i++) {
-    const line = lines[i].trim();
-    if (isSkippableLine(line)) continue;
-    const lineScore = lineScoreFor(line, tokens, defPattern.test(line));
+  for (let i = 0; i < lower.length; i++) {
+    if (isSkippableLine(lower[i])) continue;
+    const lineScore = lineScoreFor(lower[i], tokens, defNames[i]);
     if (lineScore > bestLineScore) {
       bestLineScore = lineScore;
       bestLine = i + 1;
     }
-    score += lineScore;
+    if (defNames[i]) defScore += lineScore;
+    else mentionScore += lineScore;
   }
-  return { totalScore: score, bestLine, bestLineScore };
-}
-
-function scoreContentDefinitions(content, tokens) {
-  const defRegex = /^(?:pub\s+)?(?:export\s+)?(?:async\s+)?(?:default\s+)?(function|class|def|fn|const|let|interface|type|struct|enum)\s+([a-zA-Z0-9_$]+)/;
-  return accumulateContentScore(content.split("\n"), tokens, defRegex);
+  return { totalScore: defScore + Math.min(mentionScore, MAX_MENTION_SCORE), bestLine, bestLineScore };
 }
 
 function relativeHasSegment(relativePath, segmentName) {
@@ -106,17 +107,8 @@ function relativeHasHiddenSegment(relativePath) {
   return relativePath.split(path.sep).some((segment) => segment.startsWith(".") && segment.length > 1);
 }
 
-async function listCandidateFiles(dir, includeHidden, runCommand) {
-  const rgArgs = ["rg", "--files"];
-  if (includeHidden) rgArgs.push("--hidden");
-  rgArgs.push("-g", "!.git/**", "-g", "!**/.git/**", dir);
-  try {
-    const res = await runCommand(rgArgs, { timeoutMs: 15_000 });
-    // rg --files is multithreaded and emits in nondeterministic order; ties must rank stably.
-    return res.stdout.split("\n").map((f) => f.trim()).filter(Boolean).sort();
-  } catch {
-    return [];
-  }
+async function listCandidateFiles(dir, includeHidden, index) {
+  return (await index.files(dir, includeHidden)).slice();
 }
 
 function mergePendingPaths(fileList, pendingPaths, dir, includeHidden = false) {
@@ -145,53 +137,48 @@ function mergeGrepHits(candidates, grepHits) {
   return candidates;
 }
 
-async function expandCandidatesWithGrep(candidates, fileList, tokens, flags, dir, includeHidden, runCommand) {
+function isTestPath(filePath) {
+  const segments = filePath.split(path.sep);
+  const base = segments[segments.length - 1];
+  return segments.includes("test") || segments.includes("tests") || base.includes(".test.") || base.includes(".spec.");
+}
+
+function expandCandidatesWithGrep(candidates, fileList, tokens, flags, index) {
   if (candidates.length >= 5) return candidates;
-  try {
-    // Tokens are lowercased; identifiers are not.
-    const grepArgs = ["-l", "-i", "--max-count=1", "-g", "!.git/**", "-g", "!**/.git/**"];
-    if (includeHidden) grepArgs.push("--hidden");
-    if (!flags.wantsTest) {
-      grepArgs.push("-g", "!test/**", "-g", "!tests/**", "-g", "!*.test.*", "-g", "!*.spec.*");
-    }
-    const salient = tokens.filter((t) => t.length > 2).slice(0, 4);
-    for (const t of salient) grepArgs.push("-e", t);
-    const res = await runCommand(["rg", ...grepArgs, dir], { timeoutMs: 15_000 });
-    mergeGrepHits(candidates, res.stdout.split("\n").map((f) => f.trim()).filter(Boolean).sort());
-  } catch {
-    if (candidates.length === 0) candidates = fileList.slice(0, 5);
-  }
+  const salient = tokens.filter((t) => t.length > 2).slice(0, 4);
+  const scope = flags.wantsTest ? fileList : fileList.filter((f) => !isTestPath(f));
+  const hits = index.filesContaining(scope, salient, true);
+  mergeGrepHits(candidates, hits);
+  if (candidates.length === 0) return fileList.slice(0, 5);
   return candidates;
 }
 
 function scoreSurfaceItems(items, tokens, fallbackLine) {
   let bonus = 0;
-  let signature = "";
-  let anchorLine = fallbackLine;
+  let best = null;
+  let bestMatches = 0;
   for (const item of items) {
     const nameLower = item.name.toLowerCase();
-    for (const token of tokens) {
-      if (!nameLower.includes(token)) continue;
-      bonus += item.isExport ? 80 : 50;
-      if (signature) continue;
-      signature = item.signature;
-      anchorLine = item.line;
+    const matches = tokens.filter((token) => nameLower.includes(token)).length;
+    if (matches === 0) continue;
+    bonus += matches * (item.isExport ? 80 : 50);
+    if (matches > bestMatches) {
+      bestMatches = matches;
+      best = item;
     }
   }
-  return { bonus, signature, anchorLine };
+  return { bonus, signature: best?.signature ?? "", anchorLine: best?.line ?? fallbackLine };
 }
 
-async function scoreCandidateContents(candidates, tokens, flags, vfs) {
+function scoreCandidateContents(candidates, tokens, flags, index, overlayText) {
   const candidateScores = [];
   for (const filePath of candidates) {
-    let content = "";
-    try {
-      content = await vfs.read(filePath);
-    } catch {
-      continue;
-    }
-    const { totalScore, bestLine, bestLineScore } = scoreContentDefinitions(content, tokens);
-    const surface = extractStructuralSurface(content, path.extname(filePath));
+    const pending = overlayText(filePath);
+    const entry = pending === undefined ? index.entry(filePath) : WorkspaceIndex.fromText(filePath, pending);
+    if (!entry) continue;
+    const content = entry.text;
+    const { totalScore, bestLine, bestLineScore } = scoreContentDefinitions(entry, tokens);
+    const surface = WorkspaceIndex.surfaceOf(entry);
     const { bonus: surfaceBonus, signature, anchorLine } = scoreSurfaceItems(surface.items, tokens, bestLine);
     const lowerPath = filePath.toLowerCase();
     const isTestFile = lowerPath.includes("test") || lowerPath.includes("spec");
@@ -208,7 +195,7 @@ async function scoreCandidateContents(candidates, tokens, flags, vfs) {
   return candidateScores;
 }
 
-async function rankCandidates(fileList, tokens, flags, dir, includeHidden, runCommand, vfs) {
+function rankCandidates(fileList, tokens, flags, index, overlayText) {
   const scoredPaths = [];
   for (const f of fileList) {
     const score = scorePathTopology(f, tokens, flags);
@@ -216,33 +203,29 @@ async function rankCandidates(fileList, tokens, flags, dir, includeHidden, runCo
   }
   scoredPaths.sort((a, b) => b.score - a.score);
   const selected = scoredPaths.filter((p) => p.score >= 25).slice(0, 10).map((p) => p.path);
-  const candidates = await expandCandidatesWithGrep(selected, fileList, tokens, flags, dir, includeHidden, runCommand);
-  const candidateScores = await scoreCandidateContents(candidates, tokens, flags, vfs);
+  const candidates = expandCandidatesWithGrep(selected, fileList, tokens, flags, index);
+  const candidateScores = scoreCandidateContents(candidates, tokens, flags, index, overlayText);
   return { candidates, candidateScores };
 }
 
-function buildSnapResult(candidates, candidateScores, fileList) {
+function buildSnapResult(candidates, candidateScores, fileList, root) {
+  const relative = (p) => path.relative(root, p) || p;
   if (candidateScores.length === 0 || candidateScores[0].score <= 0) {
-    return {
-      path: candidates[0] || fileList[0],
-      line: 1,
-      signature: "",
-      confidence: 0.3,
-      context: [],
-    };
+    return { path: relative(candidates[0] || fileList[0]), line: 1, signature: "", confidence: 0.3, context: [] };
   }
   const best = candidateScores[0];
   const lines = best.content.split("\n");
-  const startLine = Math.max(1, best.anchorLine - 3);
-  const endLine = Math.min(lines.length, best.anchorLine + 8);
+  // Two lines before and four after: enough to confirm the hit; read() is the tool for more.
+  const startLine = Math.max(1, best.anchorLine - 2);
+  const endLine = Math.min(lines.length, best.anchorLine + 4);
   const context = [];
   for (let l = startLine; l <= endLine; l++) {
     const marker = l === best.anchorLine ? "►" : " ";
-    context.push(`${marker} ${String(l).padStart(4)} │ ${lines[l - 1]}`);
+    context.push(marker + l + " " + lines[l - 1]);
   }
   const confidence = Math.min(0.98, Math.max(0.65, best.score / 150));
   return {
-    path: best.path,
+    path: relative(best.path),
     line: best.anchorLine,
     signature: best.signature,
     confidence: Number(confidence.toFixed(2)),
@@ -250,7 +233,7 @@ function buildSnapResult(candidates, candidateScores, fileList) {
   };
 }
 
-export async function executeSnap({ query, searchDir, includeHidden = false, vfs, runCommand, pendingPaths = [] }) {
+export async function executeSnap({ query, searchDir, root, includeHidden = false, index, overlayText = () => undefined, pendingPaths = [] }) {
   const { tokens, wantsTest, wantsType, wantsDoc } = tokenizeQuery(query);
   if (tokens.length === 0) {
     throw new Error("snap requires at least one searchable concept keyword");
@@ -259,12 +242,12 @@ export async function executeSnap({ query, searchDir, includeHidden = false, vfs
   if (path.resolve(dir).split(path.sep).includes(".git")) {
     throw new Error("snap cannot search Git metadata");
   }
-  const fileList = await listCandidateFiles(dir, includeHidden, runCommand);
+  const fileList = await listCandidateFiles(dir, includeHidden, index);
   mergePendingPaths(fileList, pendingPaths, dir, includeHidden);
   if (fileList.length === 0) {
     throw new Error(`no files found to search in ${dir}`);
   }
   const flags = { wantsTest, wantsDoc, wantsType };
-  const { candidates, candidateScores } = await rankCandidates(fileList, tokens, flags, dir, includeHidden, runCommand, vfs);
-  return buildSnapResult(candidates, candidateScores, fileList);
+  const { candidates, candidateScores } = rankCandidates(fileList, tokens, flags, index, overlayText);
+  return buildSnapResult(candidates, candidateScores, fileList, root ?? dir);
 }

package/surface.js

@@ -66,13 +66,16 @@ const JS_DECL_PATTERNS = [
   [/^(?:async\s+)?(function\*?|class)\s+([a-zA-Z0-9_$]+)/, false],
   [/^(interface|type)\s+([a-zA-Z0-9_$]+)/, false],
 ];
+// Module-level tables/constants (column 0 only): without them the previous declaration's span swallows them.
+const JS_TOP_LEVEL_BINDING = /^(const|let|var)\s+([a-zA-Z0-9_$]+)\s*=/;
 
 function scanJavaScript(lines) {
   const items = [];
   for (let i = 0; i < lines.length; i++) {
     const line = lines[i].trim();
     if (!line || line.startsWith("//") || line.startsWith("/*") || line.startsWith("*")) continue;
-    for (const [pattern, isExport] of JS_DECL_PATTERNS) {
+    const patterns = /^\S/.test(lines[i]) ? [...JS_DECL_PATTERNS, [JS_TOP_LEVEL_BINDING, false]] : JS_DECL_PATTERNS;
+    for (const [pattern, isExport] of patterns) {
       const match = pattern.exec(line);
       if (!match) continue;
       items.push({

package/vfs.js

@@ -4,9 +4,10 @@ import * as path from "node:path";
 const VFS_CACHE_MAX = 1024;
 
 export class CausalVfs {
-  constructor() {
+  constructor(onNewFile) {
     this.cache = new Map();
     this.overlays = [];
+    this.onNewFile = onNewFile;
   }
 
   setCache(target, content) {
@@ -62,6 +63,7 @@ export class CausalVfs {
       return { speculative: true };
     }
 
+    let existed = true;
     try {
       const stat = await fs.stat(target);
       if (stat.isDirectory()) {
@@ -69,11 +71,13 @@ export class CausalVfs {
       }
     } catch (err) {
       if (err.code !== "ENOENT") throw err;
+      existed = false;
     }
 
     await fs.mkdir(path.dirname(target), { recursive: true });
     await fs.writeFile(target, content, "utf8");
     this.setCache(target, content);
+    if (!existed) this.onNewFile?.(target);
     return { speculative: false };
   }
 
@@ -95,6 +99,7 @@ export class CausalVfs {
       await fs.writeFile(filePath, fileContent, "utf8");
       this.setCache(filePath, fileContent);
     }
+    if (top.size > 0) this.onNewFile?.();
     return { committed: top.size, depth: 0 };
   }
 
@@ -115,6 +120,7 @@ export class CausalVfs {
       await fs.writeFile(filePath, fileContent, "utf8");
       this.setCache(filePath, fileContent);
     }
+    if (pending.size > 0) this.onNewFile?.();
     this.overlays[0] = new Map();
     return pending.size > 0;
   }

package/workspace.js

@@ -5,6 +5,14 @@ import { isString } from "./decode.js";
 
 let cachedCwd = null;
 let cachedResolvedCwd = null;
+// realpath results per program: two syscalls per call otherwise dominate a cached read.
+const realRoots = new Map();
+const realNearest = new Map();
+const PATH_CACHE_MAX = 2048;
+
+export function clearPathCache() {
+  realNearest.clear();
+}
 
 function getResolvedCwd(cwd) {
   if (cwd === cachedCwd && cachedResolvedCwd) return cachedResolvedCwd;
@@ -43,8 +51,17 @@ export async function resolveWorkspacePath(cwd, inputPath, opName, allowRoot = f
   if (!allowRoot && target === resolvedCwd) {
     throw new Error(`${opName} path cannot be the workspace root directory`);
   }
-  const realRoot = await fs.realpath(resolvedCwd);
-  const probe = await realpathNearest(target);
+  let realRoot = realRoots.get(resolvedCwd);
+  if (!realRoot) {
+    realRoot = await fs.realpath(resolvedCwd);
+    realRoots.set(resolvedCwd, realRoot);
+  }
+  let probe = realNearest.get(target);
+  if (!probe) {
+    probe = await realpathNearest(target);
+    if (realNearest.size >= PATH_CACHE_MAX) realNearest.clear();
+    realNearest.set(target, probe);
+  }
   assertInside(path.relative(realRoot, probe), `${opName} path escapes workspace through symlink`);
   return target;
 }