pi-supernova 0.5.0 → 0.7.0

package/src/context/search.js

@@ -1,3 +1,4 @@
+import * as fs from "node:fs/promises";
 import * as path from "node:path";
 import { isString } from "../shared/decode.js";
 import { WorkspaceIndex, globToRegExp } from "./repo-index.js";
@@ -12,6 +13,63 @@ function textResult(text, details) {
   return { content: [{ type: "text", text: String(text ?? "") }], details: details || {} };
 }
 
+async function candidateFileList(index, root, includeHidden = false, signal) {
+  const stat = await fs.stat(root).catch(() => null);
+
+  if (stat?.isFile()) return [root];
+
+  return index.files(root, includeHidden, signal);
+}
+
+function pendingInScope(root, pendingPaths) {
+  return pendingPaths.filter(file => {
+    const relative = path.relative(root, file);
+
+    return relative === "" || (relative !== ".." && !relative.startsWith(".." + path.sep) && !path.isAbsolute(relative));
+  });
+}
+
+function parseMatchRecord(line, truncated, isLast) {
+  if (!line) return { skip: true };
+
+  try { return { record: JSON.parse(line) }; }
+  catch (error) {
+    if (truncated && isLast) return { stop: true };
+    throw error;
+  }
+}
+
+function isRgMatch(record) {
+  return record.type === "match" && isString(record.data?.path?.text) && isString(record.data.lines?.text);
+}
+
+function applyMatchRecord(record, root, overlayText, add) {
+  if (!isRgMatch(record)) return;
+  const file = path.resolve(root, record.data.path.text);
+
+  if (overlayText(file) === undefined) add(file, record.data.line_number, record.data.lines.text);
+}
+
+function ingestRgMatches(records, result, signal, root, overlayText, add) {
+  for (let i = 0; i < records.length; i++) {
+    signal?.throwIfAborted();
+    const parsed = parseMatchRecord(records[i], result.outputTruncated, i === records.length - 1);
+
+    if (parsed.stop) break;
+
+    if (parsed.skip) continue;
+    applyMatchRecord(parsed.record, root, overlayText, add);
+  }
+}
+
+function ingestPendingRefs(root, pendingPaths, overlayText, add) {
+  for (const file of pendingInScope(root, pendingPaths)) {
+    const text = overlayText(file);
+
+    if (text !== undefined && Buffer.byteLength(text, "utf8") <= 512 * 1024) text.split("\n").forEach((line, i) => add(file, i + 1, line));
+  }
+}
+
 /** One bounded direct search for all changed names; no repository index or per-name spawn. */
 export async function referencesForNames({ root, names, excludePath, overlayText, pendingPaths, signal, run = runCommand }) {
   const references = new Map(names.map(name => [name, []]));
@@ -31,57 +89,90 @@ export async function referencesForNames({ root, names, excludePath, overlayText
     { cwd: root, signal, timeoutMs: 5000, maxOutputChars: 65536 });
 
   if (result.exitCode !== 0 && result.exitCode !== 1) throw new Error(result.stderr.trim() || "reference search failed");
-  const records = result.stdout.split("\n");
+  ingestRgMatches(result.stdout.split("\n"), result, signal, root, overlayText, add);
+  ingestPendingRefs(root, pendingPaths, overlayText, add);
 
-  for (let i = 0; i < records.length; i++) {
-    signal?.throwIfAborted();
+  return { references, incomplete: result.outputTruncated === true };
+}
 
-    if (!records[i]) continue;
-    let record;
+function grepCaseSensitive(pattern, params) {
+  return params?.caseSensitive === true || (params?.caseSensitive !== false && smartCase(pattern));
+}
 
-    try { record = JSON.parse(records[i]); }
-    catch (error) { if (result.outputTruncated && i === records.length - 1) break; throw error; }
+function pushGrepFlags(args, pattern, params) {
+  if (!grepCaseSensitive(pattern, params)) args.push("--ignore-case");
 
-    if (record.type !== "match" || !isString(record.data?.path?.text) || !isString(record.data.lines?.text)) continue;
-    const file = path.resolve(root, record.data.path.text);
+  if (params?.glob) args.push("--glob", String(params.glob));
+  const limit = params?.limit;
 
-    if (overlayText(file) === undefined) add(file, record.data.line_number, record.data.lines.text);
-  }
+  if (Number.isInteger(limit) && limit > 0) args.push("--max-count", String(Math.min(limit, 2000)));
+}
 
-  for (const file of pendingPaths) {
-    const text = overlayText(file);
+export function rgGrepArgs(pattern, params, searchPath) {
+  const args = ["--line-number", "--no-heading", "--color", "never"];
+  pushGrepFlags(args, pattern, params);
+  args.push("--", pattern, searchPath);
 
-    if (text !== undefined) text.split("\n").forEach((line, i) => add(file, i + 1, line));
-  }
+  return args;
+}
 
-  return { references, incomplete: result.outputTruncated === true };
+function globMatcher(pattern) {
+  if (!pattern) return null;
+
+  try { return globToRegExp(pattern); }
+  catch { return /^$/; }
 }
 
-export function rgGrepArgs(pattern, params, searchPath) {
-  const args = ["--line-number", "--no-heading", "--color", "never"];
+function isDirectList(stat, pendingLength) {
+  return !!(stat?.isFile() || (!stat?.isDirectory() && pendingLength));
+}
 
-  if (params?.caseSensitive !== true) args.push("--ignore-case");
+function listDirectRows(stat, searchDir, cwd, pending, matcher) {
+  const rel = stat?.isFile() ? relativeSlash(cwd, searchDir) : null;
 
-  if (params?.glob) args.push("--glob", String(params.glob));
-  args.push("--", pattern, searchPath);
+  return [...new Set([...(rel ? [rel] : []), ...pending])].filter(file => !matcher || matcher.test(file));
+}
 
-  return args;
+function mergeListStdout(stdout, cwd, pendingMerged) {
+  const diskRows = String(stdout || "").split("\n").filter(Boolean)
+    .map(row => relativeSlash(cwd, path.isAbsolute(row) ? row : path.resolve(cwd, row)));
+  const rows = [...new Set([...diskRows, ...pendingMerged])];
+
+  return rows.length ? rows.join("\n") + "\n" : "";
 }
 
-/** rg --files, then find(1) when rg is unavailable; both accept an optional glob/name pattern. */
-export async function listWithTools(searchDir, pattern, cwd, signal) {
+async function listDisk(searchDir, pattern, cwd, signal) {
   const args = ["--files"];
 
   if (pattern) args.push("-g", pattern);
   const res = await runCommand(["rg", ...args, searchDir], { cwd, timeoutMs: 30_000, signal }).catch(() => null);
 
-  if (res && (res.exitCode === 0 || res.exitCode === 1)) return textResult(res.stdout, { via: "rg" });
+  if (res && (res.exitCode === 0 || res.exitCode === 1)) return { stdout: res.stdout, via: "rg", outputTruncated: res.outputTruncated === true };
   const findArgs = [searchDir];
 
   if (pattern) findArgs.push("-name", pattern);
   const findRes = await runCommand(["find", ...findArgs], { cwd, timeoutMs: 30_000, signal });
 
-  return textResult(findRes.stdout, { via: "find" });
+  return { stdout: findRes.stdout, via: "find", outputTruncated: findRes.outputTruncated === true };
+}
+
+/** rg --files, then find(1) when rg is unavailable; both accept an optional glob/name pattern. */
+export async function listWithTools(searchDir, pattern, cwd, signal, pendingPaths = []) {
+  const stat = await fs.stat(searchDir).catch(() => null);
+  const pendingAbs = pendingInScope(searchDir, pendingPaths);
+  const pending = pendingAbs.map(file => relativeSlash(cwd, file));
+  const matcher = globMatcher(pattern);
+
+  if (isDirectList(stat, pending.length)) {
+    const rows = listDirectRows(stat, searchDir, cwd, pending, matcher);
+
+    return textResult(rows.length ? rows.join("\n") + "\n" : "", { via: pending.length ? "vfs" : "file" });
+  }
+
+  const pendingMerged = pendingAbs.filter((_, i) => !matcher || matcher.test(pending[i]));
+  const listed = await listDisk(searchDir, pattern, cwd, signal);
+
+  return textResult(mergeListStdout(listed.stdout, cwd, pendingMerged), { via: listed.via, outputTruncated: listed.outputTruncated });
 }
 
 const GLOB_CHARS = /[*?[\]{}]/;
@@ -90,9 +181,9 @@ const GLOB_CHARS = /[*?[\]{}]/;
  * fffind: a pattern without glob characters is a fuzzy, typo-tolerant, frecency-ranked path query.
  * Returns "path" rows (best first) or null when the pattern is a real glob.
  */
-export async function fuzzyFind(index, root, cwd, pattern, limit = 20) {
+export async function fuzzyFind(index, root, cwd, pattern, limit = 20, pendingPaths = []) {
   if (!pattern || GLOB_CHARS.test(pattern)) return null;
-  const files = await index.files(root);
+  const files = [...new Set([...await candidateFileList(index, root), ...pendingInScope(root, pendingPaths)])];
 
   if (!index.canScan(files)) return null;
   const rel = files.map((f) => relativeSlash(cwd, f));
@@ -108,29 +199,43 @@ export async function fuzzyFind(index, root, cwd, pattern, limit = 20) {
   return rows.map((r) => r.path).join("\n") + "\n";
 }
 
+function applyGlob(files, params, cwd) {
+  if (!params?.glob) return files;
+  const matcher = globToRegExp(String(params.glob));
+
+  return files.filter((f) => matcher.test(relativeSlash(cwd, f)));
+}
+
+function filesReadable(index, files, overlayText) {
+  for (const file of files) {
+    const overlay = overlayText(file);
+
+    if (overlay === undefined && index.entry(file) === null) return false;
+  }
+
+  return true;
+}
+
 /** fff-style grep: smart-case, definition lines first, fuzzy fallback when the literal has no hits. */
-export async function grepIndexed(index, pattern, params, searchPath, cwd) {
+export async function grepIndexed(index, pattern, params, searchPath, cwd, overlayText = () => undefined, pendingPaths = []) {
   const compiled = grepRegex(pattern, params);
 
   if (!compiled) return null;
   const { regex, caseSensitive } = compiled;
-  let files = await index.files(searchPath);
+  let files = [...new Set([...await candidateFileList(index, searchPath), ...pendingInScope(searchPath, pendingPaths)])];
 
   if (!index.canScan(files)) return null;
+  files = applyGlob(files, params, cwd);
 
-  if (params?.glob) {
-    const matcher = globToRegExp(String(params.glob));
-    files = files.filter((f) => matcher.test(relativeSlash(cwd, f)));
-  }
-
-  const rows = index.grepRows(files, regex, cwd);
-  const fallback = rows.length === 0 && /^[\w$.-]{4,}$/.test(pattern) ? fuzzyGrepRows(index, files, pattern, cwd, caseSensitive) : rows;
+  if (!filesReadable(index, files, overlayText)) return null;
+  const rows = index.grepRows(files, regex, cwd, overlayText);
+  const fallback = rows.length === 0 && /^[\w$.-]{4,}$/.test(pattern) ? fuzzyGrepRows(index, files, pattern, cwd, caseSensitive, overlayText) : rows;
 
   return formatGrepRows(fallback, grepLimit(params));
 }
 
 function grepLimit(params) {
-  return Number.isInteger(params?.limit) && params.limit > 0 ? params.limit : 200;
+  return Number.isInteger(params?.limit) && params.limit > 0 ? Math.min(params.limit, 2000) : 200;
 }
 
 function grepRegex(pattern, params) {
@@ -143,32 +248,45 @@ function grepRegex(pattern, params) {
   }
 }
 
+function overlaySearchEntry(index, filePath, overlayText) {
+  const pending = overlayText(filePath);
+
+  return pending === undefined
+    ? index.entry(filePath)
+    : Buffer.byteLength(pending, "utf8") <= 512 * 1024 ? WorkspaceIndex.fromText(filePath, pending) : null;
+}
+
+function fuzzyLineRow(pattern, rawLine, defName, rel, line, maxTypos, caseSensitive) {
+  const m = fuzzyMatch(pattern, rawLine, { maxTypos, caseSensitive });
+
+  if (!m || m.end - m.start > pattern.length + 2) return null;
+
+  return { rel, line, text: rawLine, def: defName !== "" && fuzzyMatch(pattern, defName, { maxTypos }) !== null };
+}
+
 /** Zero literal hits: retry each line fuzzily (1 typo, 2 for long names) within a tight span, so IsOffTheRecord finds is_off_the_record. */
-function fuzzyGrepRows(index, files, pattern, cwd, caseSensitive) {
+function fuzzyGrepRows(index, files, pattern, cwd, caseSensitive, overlayText = () => undefined) {
   const maxTypos = pattern.length >= 8 ? 2 : 1;
   const rows = [];
 
   for (const filePath of files) {
-    const e = index.entry(filePath);
+    const e = overlaySearchEntry(index, filePath, overlayText);
 
     if (!e) continue;
     const { raw, defNames } = WorkspaceIndex.linesOf(e);
     const rel = relativeSlash(cwd, filePath);
 
     for (let i = 0; i < raw.length && rows.length <= 400; i++) {
-      const m = fuzzyMatch(pattern, raw[i], { maxTypos, caseSensitive });
+      const row = fuzzyLineRow(pattern, raw[i], defNames[i], rel, i + 1, maxTypos, caseSensitive);
 
-      if (!m || m.end - m.start > pattern.length + 2) continue;
-      rows.push({ rel, line: i + 1, text: raw[i], def: defNames[i] !== "" && fuzzyMatch(pattern, defNames[i], { maxTypos }) !== null });
+      if (row) rows.push(row);
     }
   }
 
   return rows;
 }
 
-/** fff definition-first hinting: files that declare the name come first, declarations first within a file; one header per file. */
-function formatGrepRows(rows, limit) {
-  if (rows.length === 0) return "";
+function groupGrepRows(rows) {
   const groups = new Map();
 
   for (const r of rows) {
@@ -176,19 +294,34 @@ function formatGrepRows(rows, limit) {
     groups.get(r.rel).push(r);
   }
 
-  const files = [...groups.values()].sort((a, b) => Number(b.some((r) => r.def)) - Number(a.some((r) => r.def)));
+  return [...groups.values()].sort((a, b) => Number(b.some((r) => r.def)) - Number(a.some((r) => r.def)));
+}
+
+function formatGroup(group, limit, shown) {
+  let out = group[0].rel + "\n";
+  group.sort((a, b) => Number(b.def) - Number(a.def) || a.line - b.line);
+  let n = shown;
+
+  for (const r of group) {
+    if (n++ >= limit) break;
+    out += "  " + r.line + (r.def ? "*" : ":") + " " + r.text.trim() + "\n";
+  }
+
+  return { out, shown: n };
+}
+
+/** fff definition-first hinting: files that declare the name come first, declarations first within a file; one header per file. */
+function formatGrepRows(rows, limit) {
+  if (rows.length === 0) return "";
+  const files = groupGrepRows(rows);
   let out = "";
   let shown = 0;
 
   for (const group of files) {
     if (shown >= limit) break;
-    out += group[0].rel + "\n";
-    group.sort((a, b) => Number(b.def) - Number(a.def) || a.line - b.line);
-
-    for (const r of group) {
-      if (shown++ >= limit) break;
-      out += "  " + r.line + (r.def ? "*" : ":") + " " + r.text.trim() + "\n";
-    }
+    const next = formatGroup(group, limit, shown);
+    out += next.out;
+    shown = next.shown;
   }
 
   if (rows.length > limit) out += "… " + (rows.length - limit) + " more matches (pass limit or narrow the pattern)\n";
@@ -197,8 +330,8 @@ function formatGrepRows(rows, limit) {
 }
 
 /** rg --files [-g pattern] served from the index; null when the tree is too large. */
-export async function listIndexed(index, root, cwd, pattern) {
-  const files = await index.files(root);
+export async function listIndexed(index, root, cwd, pattern, pendingPaths = []) {
+  const files = [...new Set([...await candidateFileList(index, root), ...pendingInScope(root, pendingPaths)])];
 
   if (!index.canScan(files)) return null;
   const rel = files.map((f) => path.relative(cwd, f).split(path.sep).join("/"));