pi-supernova 0.0.7 → 0.0.11

package/snap.js

@@ -1,7 +1,7 @@
 
 import * as path from "node:path";
 import { isString } from "./decode.js";
-import { extractStructuralSurface } from "./surface.js";
+import { WorkspaceIndex } from "./repo-index.js";
 
 const STOP_WORDS = new Set([
   "the", "a", "an", "and", "or", "in", "on", "at", "to", "for", "of", "with",
@@ -30,68 +30,73 @@ export function tokenizeQuery(query) {
   };
 }
 
-export function scorePathTopology(filePath, tokens, { wantsTest, wantsDoc, wantsType }) {
-  const norm = filePath.replace(/\\/g, "/").toLowerCase();
-  const basename = path.basename(norm);
-  const ext = path.extname(norm);
+const SOURCE_EXT = new Set([".ts", ".js", ".mjs", ".rs", ".py", ".go"]);
+const TYPED_EXT = new Set([".ts", ".d.ts", ".rs", ".go"]);
+const VENDOR_SEGMENTS = ["node_modules/", "dist/", "target/"];
 
-  const isTest = norm.includes("test") || norm.includes("spec") || norm.includes("__tests__");
-  if (isTest && !wantsTest) return -50;
-  if (!isTest && wantsTest) return -20;
+function tokenPathScore(token, basename, pathParts, norm) {
+  if (basename === token || basename.startsWith(token + ".")) return 60;
+  if (basename.includes(token)) return 30;
+  if (pathParts.includes(token)) return 15;
+  if (norm.includes(token)) return 5;
+  return 0;
+}
 
-  if (norm.includes("node_modules/") || norm.includes("dist/") || norm.includes("target/")) {
-    return -100;
-  }
+function extensionBonus(ext, { wantsDoc, wantsType }) {
+  let bonus = 0;
+  if (SOURCE_EXT.has(ext) && !wantsDoc) bonus += 5;
+  if (wantsType && TYPED_EXT.has(ext)) bonus += 10;
+  return bonus;
+}
 
-  let score = 0;
+export function scorePathTopology(filePath, tokens, flags) {
+  const norm = filePath.replaceAll("\\", "/").toLowerCase();
+  const isTest = norm.includes("test") || norm.includes("spec") || norm.includes("__tests__");
+  if (isTest && !flags.wantsTest) return -50;
+  if (!isTest && flags.wantsTest) return -20;
+  if (VENDOR_SEGMENTS.some((segment) => norm.includes(segment))) return -100;
+
+  const basename = path.basename(norm);
   const pathParts = norm.split(/[^a-zA-Z0-9]+/);
+  let score = extensionBonus(path.extname(norm), flags);
+  for (const token of tokens) score += tokenPathScore(token, basename, pathParts, norm);
+  return score;
+}
 
-  for (const token of tokens) {
-    if (basename === token || basename.startsWith(token + ".")) score += 60;
-    else if (basename.includes(token)) score += 30;
-    else if (pathParts.includes(token)) score += 15;
-    else if (norm.includes(token)) score += 5;
-  }
+function isSkippableLine(lower) {
+  return !lower || lower.startsWith("//") || lower.startsWith("#") || lower.startsWith("*");
+}
 
-  if ([".ts", ".js", ".mjs", ".rs", ".py", ".go"].includes(ext) && !wantsDoc) {
-    score += 5;
-  }
-  if (wantsType && [".ts", ".d.ts", ".rs", ".go"].includes(ext)) {
-    score += 10;
+/** A line defines a token only when the declared name contains it; `const x = foo(token)` is a mention. */
+function lineScoreFor(lower, tokens, definedName) {
+  let lineScore = 0;
+  for (const token of tokens) {
+    if (!lower.includes(token)) continue;
+    lineScore += definedName.includes(token) ? 40 : 5;
   }
-
-  return score;
+  return lineScore;
 }
 
-function scoreContentDefinitions(content, tokens) {
-  const lines = content.split("\n");
-  let score = 0;
+// Mentions are capped so a file that calls a symbol many times cannot outrank the file that defines it.
+const MAX_MENTION_SCORE = 60;
+
+function scoreContentDefinitions(entry, tokens) {
+  const { lower, defNames } = WorkspaceIndex.linesOf(entry);
+  let defScore = 0;
+  let mentionScore = 0;
   let bestLine = 1;
   let bestLineScore = 0;
-
-  const defRegex = /^(?:pub\s+)?(?:export\s+)?(?:async\s+)?(?:default\s+)?(function|class|def|fn|const|let|interface|type|struct|enum)\s+([a-zA-Z0-9_$]+)/;
-
-  for (let i = 0; i < lines.length; i++) {
-    const line = lines[i].trim();
-    if (!line || line.startsWith("//") || line.startsWith("#") || line.startsWith("*")) continue;
-
-    let lineScore = 0;
-    const isDef = defRegex.test(line);
-
-    for (const token of tokens) {
-      if (line.toLowerCase().includes(token)) {
-        lineScore += isDef ? 40 : 5;
-      }
-    }
-
+  for (let i = 0; i < lower.length; i++) {
+    if (isSkippableLine(lower[i])) continue;
+    const lineScore = lineScoreFor(lower[i], tokens, defNames[i]);
     if (lineScore > bestLineScore) {
       bestLineScore = lineScore;
       bestLine = i + 1;
     }
-    score += lineScore;
+    if (defNames[i]) defScore += lineScore;
+    else mentionScore += lineScore;
   }
-
-  return { totalScore: score, bestLine, bestLineScore };
+  return { totalScore: defScore + Math.min(mentionScore, MAX_MENTION_SCORE), bestLine, bestLineScore };
 }
 
 function relativeHasSegment(relativePath, segmentName) {
@@ -102,150 +107,147 @@ function relativeHasHiddenSegment(relativePath) {
   return relativePath.split(path.sep).some((segment) => segment.startsWith(".") && segment.length > 1);
 }
 
-export async function executeSnap({ query, searchDir, includeHidden = false, vfs, runCommand, pendingPaths = [] }) {
-  const { tokens, wantsTest, wantsType, wantsDoc } = tokenizeQuery(query);
-  if (tokens.length === 0) {
-    throw new Error("snap requires at least one searchable concept keyword");
-  }
-
-  const dir = searchDir || process.cwd();
-  if (path.resolve(dir).split(path.sep).includes(".git")) {
-    throw new Error("snap cannot search Git metadata");
-  }
-  let fileList = [];
-  try {
-    const rgArgs = ["rg", "--files"];
-    if (includeHidden) rgArgs.push("--hidden");
-    rgArgs.push("-g", "!.git/**", "-g", "!**/.git/**", dir);
-    const res = await runCommand(rgArgs, { timeoutMs: 15_000 });
-    fileList = res.stdout.split("\n").map((f) => f.trim()).filter(Boolean);
-  } catch {
-    fileList = [];
-  }
+async function listCandidateFiles(dir, includeHidden, index) {
+  return (await index.files(dir, includeHidden)).slice();
+}
 
+function mergePendingPaths(fileList, pendingPaths, dir, includeHidden = false) {
+  const resolvedDir = path.resolve(dir);
   const seenPaths = new Set(fileList.map((filePath) => path.resolve(filePath)));
   for (const pendingPath of pendingPaths) {
     const absolutePath = path.resolve(pendingPath);
-    const relativePath = path.relative(path.resolve(dir), absolutePath);
+    const relativePath = path.relative(resolvedDir, absolutePath);
     const escapesDir = relativePath === ".." || relativePath.startsWith(`..${path.sep}`) || path.isAbsolute(relativePath);
     const hiddenRelativePath = relativeHasHiddenSegment(relativePath);
     if (escapesDir || relativeHasSegment(relativePath, ".git") || (!includeHidden && hiddenRelativePath) || seenPaths.has(absolutePath)) continue;
     seenPaths.add(absolutePath);
     fileList.push(absolutePath);
   }
+  return fileList;
+}
 
-  if (fileList.length === 0) {
-    throw new Error(`no files found to search in ${dir}`);
+function mergeGrepHits(candidates, grepHits) {
+  const seen = new Set(candidates);
+  for (const h of grepHits) {
+    if (seen.has(h)) continue;
+    seen.add(h);
+    candidates.push(h);
+    if (candidates.length >= 15) break;
   }
+  return candidates;
+}
 
-  const scoredPaths = [];
-  for (const f of fileList) {
-    const score = scorePathTopology(f, tokens, { wantsTest, wantsDoc, wantsType });
-    if (score > 0) scoredPaths.push({ path: f, score });
-  }
+function isTestPath(filePath) {
+  const segments = filePath.split(path.sep);
+  const base = segments[segments.length - 1];
+  return segments.includes("test") || segments.includes("tests") || base.includes(".test.") || base.includes(".spec.");
+}
 
-  scoredPaths.sort((a, b) => b.score - a.score);
+function expandCandidatesWithGrep(candidates, fileList, tokens, flags, index) {
+  if (candidates.length >= 5) return candidates;
+  const salient = tokens.filter((t) => t.length > 2).slice(0, 4);
+  const scope = flags.wantsTest ? fileList : fileList.filter((f) => !isTestPath(f));
+  const hits = index.filesContaining(scope, salient, true);
+  mergeGrepHits(candidates, hits);
+  if (candidates.length === 0) return fileList.slice(0, 5);
+  return candidates;
+}
 
-  let candidates = scoredPaths.filter((p) => p.score >= 25).slice(0, 10).map((p) => p.path);
-
-  if (candidates.length < 5) {
-    try {
-      const grepArgs = ["-l", "--max-count=1", "-g", "!.git/**", "-g", "!**/.git/**"];
-      if (includeHidden) grepArgs.push("--hidden");
-      if (!wantsTest) {
-        grepArgs.push("-g", "!test/**", "-g", "!tests/**", "-g", "!*.test.*", "-g", "!*.spec.*");
-      }
-      const salient = tokens.filter((t) => t.length > 2).slice(0, 4);
-      for (const t of salient) grepArgs.push("-e", t);
-      const res = await runCommand(["rg", ...grepArgs, dir], { timeoutMs: 15_000 });
-      const grepHits = res.stdout.split("\n").map((f) => f.trim()).filter(Boolean);
-      const seen = new Set(candidates);
-      for (const h of grepHits) {
-        if (!seen.has(h)) {
-          seen.add(h);
-          candidates.push(h);
-          if (candidates.length >= 15) break;
-        }
-      }
-    } catch {
-      if (candidates.length === 0) candidates = fileList.slice(0, 5);
+function scoreSurfaceItems(items, tokens, fallbackLine) {
+  let bonus = 0;
+  let best = null;
+  let bestMatches = 0;
+  for (const item of items) {
+    const nameLower = item.name.toLowerCase();
+    const matches = tokens.filter((token) => nameLower.includes(token)).length;
+    if (matches === 0) continue;
+    bonus += matches * (item.isExport ? 80 : 50);
+    if (matches > bestMatches) {
+      bestMatches = matches;
+      best = item;
     }
   }
+  return { bonus, signature: best?.signature ?? "", anchorLine: best?.line ?? fallbackLine };
+}
 
+function scoreCandidateContents(candidates, tokens, flags, index, overlayText) {
   const candidateScores = [];
-
   for (const filePath of candidates) {
-    let content = "";
-    try {
-      content = await vfs.read(filePath);
-    } catch {
-      continue;
-    }
-
-    const { totalScore, bestLine, bestLineScore } = scoreContentDefinitions(content, tokens);
-    const ext = path.extname(filePath);
-    const surface = extractStructuralSurface(content, ext);
-
-    let surfaceBonus = 0;
-    let signature = "";
-    let anchorLine = bestLine;
-
-    for (const item of surface.items) {
-      const nameLower = item.name.toLowerCase();
-      for (const token of tokens) {
-        if (nameLower.includes(token)) {
-          surfaceBonus += item.isExport ? 80 : 50;
-          if (!signature) {
-            signature = item.signature;
-            anchorLine = item.line;
-          }
-        }
-      }
-    }
-
-    const isTestFile = filePath.toLowerCase().includes("test") || filePath.toLowerCase().includes("spec");
-    const testAdjustment = isTestFile && !wantsTest ? -200 : (isTestFile && wantsTest ? 100 : 0);
-    const finalScore = totalScore + surfaceBonus + (bestLineScore * 2) + testAdjustment;
+    const pending = overlayText(filePath);
+    const entry = pending === undefined ? index.entry(filePath) : WorkspaceIndex.fromText(filePath, pending);
+    if (!entry) continue;
+    const content = entry.text;
+    const { totalScore, bestLine, bestLineScore } = scoreContentDefinitions(entry, tokens);
+    const surface = WorkspaceIndex.surfaceOf(entry);
+    const { bonus: surfaceBonus, signature, anchorLine } = scoreSurfaceItems(surface.items, tokens, bestLine);
+    const lowerPath = filePath.toLowerCase();
+    const isTestFile = lowerPath.includes("test") || lowerPath.includes("spec");
+    const testAdjustment = !isTestFile ? 0 : (flags.wantsTest ? 100 : -200);
     candidateScores.push({
       path: filePath,
-      score: finalScore,
+      score: totalScore + surfaceBonus + (bestLineScore * 2) + testAdjustment,
       anchorLine,
       signature,
       content,
     });
   }
-
   candidateScores.sort((a, b) => b.score - a.score);
+  return candidateScores;
+}
 
-  if (candidateScores.length === 0 || candidateScores[0].score <= 0) {
-    const fallbackPath = candidates[0] || fileList[0];
-    return {
-      path: fallbackPath,
-      line: 1,
-      signature: "",
-      confidence: 0.3,
-      context: [],
-    };
+function rankCandidates(fileList, tokens, flags, index, overlayText) {
+  const scoredPaths = [];
+  for (const f of fileList) {
+    const score = scorePathTopology(f, tokens, flags);
+    if (score > 0) scoredPaths.push({ path: f, score });
   }
+  scoredPaths.sort((a, b) => b.score - a.score);
+  const selected = scoredPaths.filter((p) => p.score >= 25).slice(0, 10).map((p) => p.path);
+  const candidates = expandCandidatesWithGrep(selected, fileList, tokens, flags, index);
+  const candidateScores = scoreCandidateContents(candidates, tokens, flags, index, overlayText);
+  return { candidates, candidateScores };
+}
 
+function buildSnapResult(candidates, candidateScores, fileList, root) {
+  const relative = (p) => path.relative(root, p) || p;
+  if (candidateScores.length === 0 || candidateScores[0].score <= 0) {
+    return { path: relative(candidates[0] || fileList[0]), line: 1, signature: "", confidence: 0.3, context: [] };
+  }
   const best = candidateScores[0];
   const lines = best.content.split("\n");
-  const startLine = Math.max(1, best.anchorLine - 3);
-  const endLine = Math.min(lines.length, best.anchorLine + 8);
-
-  const contextLines = [];
+  // Two lines before and four after: enough to confirm the hit; read() is the tool for more.
+  const startLine = Math.max(1, best.anchorLine - 2);
+  const endLine = Math.min(lines.length, best.anchorLine + 4);
+  const context = [];
   for (let l = startLine; l <= endLine; l++) {
     const marker = l === best.anchorLine ? "►" : " ";
-    contextLines.push(`${marker} ${String(l).padStart(4)} │ ${lines[l - 1]}`);
+    context.push(marker + l + " " + lines[l - 1]);
   }
-
   const confidence = Math.min(0.98, Math.max(0.65, best.score / 150));
-
   return {
-    path: best.path,
+    path: relative(best.path),
     line: best.anchorLine,
     signature: best.signature,
     confidence: Number(confidence.toFixed(2)),
-    context: contextLines,
+    context,
   };
 }
+
+export async function executeSnap({ query, searchDir, root, includeHidden = false, index, overlayText = () => undefined, pendingPaths = [] }) {
+  const { tokens, wantsTest, wantsType, wantsDoc } = tokenizeQuery(query);
+  if (tokens.length === 0) {
+    throw new Error("snap requires at least one searchable concept keyword");
+  }
+  const dir = searchDir || process.cwd();
+  if (path.resolve(dir).split(path.sep).includes(".git")) {
+    throw new Error("snap cannot search Git metadata");
+  }
+  const fileList = await listCandidateFiles(dir, includeHidden, index);
+  mergePendingPaths(fileList, pendingPaths, dir, includeHidden);
+  if (fileList.length === 0) {
+    throw new Error(`no files found to search in ${dir}`);
+  }
+  const flags = { wantsTest, wantsDoc, wantsType };
+  const { candidates, candidateScores } = rankCandidates(fileList, tokens, flags, index, overlayText);
+  return buildSnapResult(candidates, candidateScores, fileList, root ?? dir);
+}

package/surface.js

@@ -61,45 +61,31 @@ function scanGo(lines) {
   return items;
 }
 
+const JS_DECL_PATTERNS = [
+  [/^export\s+(?:default\s+)?(?:async\s+)?(function\*?|class|const|let|var|type|interface|enum)\s+([a-zA-Z0-9_$]+)/, true],
+  [/^(?:async\s+)?(function\*?|class)\s+([a-zA-Z0-9_$]+)/, false],
+  [/^(interface|type)\s+([a-zA-Z0-9_$]+)/, false],
+];
+// Module-level tables/constants (column 0 only): without them the previous declaration's span swallows them.
+const JS_TOP_LEVEL_BINDING = /^(const|let|var)\s+([a-zA-Z0-9_$]+)\s*=/;
+
 function scanJavaScript(lines) {
   const items = [];
   for (let i = 0; i < lines.length; i++) {
     const line = lines[i].trim();
     if (!line || line.startsWith("//") || line.startsWith("/*") || line.startsWith("*")) continue;
-
-    const expMatch = /^export\s+(?:default\s+)?(?:async\s+)?(function\*?|class|const|let|var|type|interface|enum)\s+([a-zA-Z0-9_$]+)/.exec(line);
-    if (expMatch) {
-      items.push({
-        kind: expMatch[1],
-        name: expMatch[2],
-        isExport: true,
-        signature: line.replace(/\{.*$/, "").trim(),
-        line: i + 1,
-      });
-      continue;
-    }
-
-    const declMatch = /^(?:async\s+)?(function\*?|class)\s+([a-zA-Z0-9_$]+)/.exec(line);
-    if (declMatch) {
-      items.push({
-        kind: declMatch[1],
-        name: declMatch[2],
-        isExport: false,
-        signature: line.replace(/\{.*$/, "").trim(),
-        line: i + 1,
-      });
-      continue;
-    }
-
-    const tsMatch = /^(interface|type)\s+([a-zA-Z0-9_$]+)/.exec(line);
-    if (tsMatch) {
+    const patterns = /^\S/.test(lines[i]) ? [...JS_DECL_PATTERNS, [JS_TOP_LEVEL_BINDING, false]] : JS_DECL_PATTERNS;
+    for (const [pattern, isExport] of patterns) {
+      const match = pattern.exec(line);
+      if (!match) continue;
       items.push({
-        kind: tsMatch[1],
-        name: tsMatch[2],
-        isExport: false,
+        kind: match[1],
+        name: match[2],
+        isExport,
         signature: line.replace(/\{.*$/, "").trim(),
         line: i + 1,
       });
+      break;
     }
   }
   return items;

package/vfs.js

@@ -0,0 +1,144 @@
+import * as fs from "node:fs/promises";
+import * as path from "node:path";
+
+const VFS_CACHE_MAX = 1024;
+
+export class CausalVfs {
+  constructor(onNewFile) {
+    this.cache = new Map();
+    this.overlays = [];
+    this.onNewFile = onNewFile;
+  }
+
+  setCache(target, content) {
+    if (this.cache.size >= VFS_CACHE_MAX && !this.cache.has(target)) {
+      const oldest = this.cache.keys().next().value;
+      if (oldest !== undefined) this.cache.delete(oldest);
+    }
+    this.cache.set(target, content);
+  }
+
+  getOverlay(target) {
+    for (let i = this.overlays.length - 1; i >= 0; i--) {
+      if (this.overlays[i].has(target)) return this.overlays[i].get(target);
+    }
+    return undefined;
+  }
+
+  getOverlayPaths() {
+    const paths = new Set();
+    for (const overlay of this.overlays) {
+      for (const target of overlay.keys()) paths.add(target);
+    }
+    return [...paths];
+  }
+
+  async read(target) {
+    const overlay = this.getOverlay(target);
+    if (overlay !== undefined) return overlay;
+
+    const cached = this.cache.get(target);
+    if (cached !== undefined) return cached;
+
+    try {
+      const text = await fs.readFile(target, "utf8");
+      this.setCache(target, text);
+      return text;
+    } catch (err) {
+      if (err.code === "EISDIR") {
+        throw new Error(`read path is a directory, not a file: ${target}`);
+      }
+      if (err.code === "ENOENT") {
+        const missing = new Error(`no such file: ${target} (locate it with nova.call("glob", {pattern}) or snap(query))`);
+        missing.code = "ENOENT";
+        throw missing;
+      }
+      throw err;
+    }
+  }
+
+  async write(target, content) {
+    if (this.overlays.length > 0) {
+      this.overlays[this.overlays.length - 1].set(target, content);
+      return { speculative: true };
+    }
+
+    let existed = true;
+    try {
+      const stat = await fs.stat(target);
+      if (stat.isDirectory()) {
+        throw new Error(`cannot write to a directory: ${target}`);
+      }
+    } catch (err) {
+      if (err.code !== "ENOENT") throw err;
+      existed = false;
+    }
+
+    await fs.mkdir(path.dirname(target), { recursive: true });
+    await fs.writeFile(target, content, "utf8");
+    this.setCache(target, content);
+    if (!existed) this.onNewFile?.(target);
+    return { speculative: false };
+  }
+
+  begin() {
+    this.overlays.push(new Map());
+    return this.overlays.length;
+  }
+
+  async commit() {
+    if (this.overlays.length === 0) return { committed: 0, depth: 0 };
+    const top = this.overlays.pop();
+    if (this.overlays.length > 0) {
+      const parent = this.overlays[this.overlays.length - 1];
+      for (const [k, v] of top.entries()) parent.set(k, v);
+      return { committed: top.size, depth: this.overlays.length };
+    }
+    for (const [filePath, fileContent] of top.entries()) {
+      await fs.mkdir(path.dirname(filePath), { recursive: true });
+      await fs.writeFile(filePath, fileContent, "utf8");
+      this.setCache(filePath, fileContent);
+    }
+    if (top.size > 0) this.onNewFile?.();
+    return { committed: top.size, depth: 0 };
+  }
+
+  rollback() {
+    if (this.overlays.length === 0) return { rolledBack: 0, depth: 0 };
+    const top = this.overlays.pop();
+    return { rolledBack: top.size, depth: this.overlays.length };
+  }
+
+  async prepareExternalMutation(name) {
+    if (this.overlays.length > 1) {
+      throw new Error(`${name} cannot run inside nova.speculate because external mutations cannot be rolled back`);
+    }
+    if (this.overlays.length === 0) return false;
+    const pending = this.overlays[0];
+    for (const [filePath, fileContent] of pending.entries()) {
+      await fs.mkdir(path.dirname(filePath), { recursive: true });
+      await fs.writeFile(filePath, fileContent, "utf8");
+      this.setCache(filePath, fileContent);
+    }
+    if (pending.size > 0) this.onNewFile?.();
+    this.overlays[0] = new Map();
+    return pending.size > 0;
+  }
+
+  invalidateCache() {
+    this.cache.clear();
+  }
+
+  clear() {
+    this.invalidateCache();
+    this.overlays.length = 0;
+  }
+
+  getCacheSize() {
+    return this.cache.size;
+  }
+
+  getOverlayDepth() {
+    return this.overlays.length;
+  }
+}