pi-supernova 0.8.2 → 0.9.0

package/src/context/repo-index.js

@@ -1,6 +1,9 @@
+export {globToRegExp} from '../fs/glob.js';
+export {declaredName} from './source-entry.js';
+import {fromText,linesOf,spansOf,surfaceOf} from './source-entry.js';
 import * as fs from "node:fs";
 import * as path from "node:path";
-import { extractStructuralSurface } from "./surface.js";
+
 import { isFunction } from "../shared/decode.js";
 import { Frecency } from "./fuzzy.js";
 import { relativeSlash } from "../fs/workspace.js";
@@ -29,126 +32,10 @@ const BINARY_EXT = new Set([
   ".jar", ".so", ".dylib", ".dll", ".exe", ".bin", ".o", ".a", ".node", ".lock", ".sqlite", ".sqlite3", ".db",
 ]);
 
-const REGEX_SPECIAL = /[.+^${}()|\\]/g;
-
-const IDENT_TOKEN = /[A-Za-z_$][\w$]*/g;
-
-const EMPTY = Object.freeze([]);
-
-const DEF_PATTERN = /^(?:pub\s+)?(?:export\s+)?(?:async\s+)?(?:default\s+)?(?:(function|class|def|fn|const|let|interface|type|struct|enum)\s+([a-zA-Z0-9_$]+)|([A-Z][A-Z0-9_$]*)\s*(?::[^=\n]+)?=)/;
-
-/** Declared identifier on a line (function/class/UPPER_CASE constant/…), or ""; the same rule snap and grep use. */
-export function declaredName(line) {
-  const match = DEF_PATTERN.exec(String(line).trim());
-
-  return match?.[2] ?? match?.[3] ?? "";
-}
-
 function isTextCandidate(filePath) {
   return !BINARY_EXT.has(path.extname(filePath).toLowerCase());
 }
 
-/** Translate one glob token at index i → [regexSource, nextIndex]. */
-function globToken(glob, i) {
-  const ch = glob[i];
-
-  if (ch === "*" && glob[i + 1] === "*") {
-    const slashAfter = glob[i + 2] === "/";
-
-    return [slashAfter ? "(?:.*/)?" : ".*", i + (slashAfter ? 3 : 2)];
-  }
-
-  if (ch === "*") return ["[^/]*", i + 1];
-
-  if (ch === "?") return ["[^/]", i + 1];
-
-  if (ch === "{" || ch === "[") return globGroup(glob, i, ch);
-
-  return [ch.replace(REGEX_SPECIAL, "\\$&"), i + 1];
-}
-
-/** {a,b} alternation or [..] class starting at i. */
-function globGroup(glob, i, open) {
-  const close = open === "{" ? "}" : "]";
-  const end = glob.indexOf(close, i);
-
-  if (end < 0) throw new SyntaxError("unclosed " + open + " in glob");
-  const inner = glob.slice(i + 1, end);
-  const source = open === "{"
-    ? "(?:" + inner.split(",").map(globBody).join("|") + ")"
-    : "[" + (inner.startsWith("!") ? "^" + inner.slice(1) : inner) + "]";
-
-  return [source, end + 1];
-}
-
-function globBody(glob) {
-  let source = "";
-  let i = 0;
-
-  while (i < glob.length) {
-    const [piece, next] = globToken(glob, i);
-    source += piece;
-    i = next;
-  }
-
-  return source;
-}
-
-/** gitignore-style glob (rg -g) → RegExp over a "/"-separated relative path. No slash ⇒ basename match anywhere. */
-export function globToRegExp(glob) {
-  const body = globBody(glob);
-
-  return new RegExp(glob.includes("/") ? "^" + body + "$" : "(?:^|/)" + body + "$");
-}
-
-function lineIndent(raw, i) {
-  return raw[i].length - raw[i].trimStart().length;
-}
-
-function pythonDeclarationEnd(raw, lower, start, lineCount) {
-  const base = lineIndent(raw, start - 1);
-  let end = start;
-
-  for (let i = start; i < lineCount; i++) {
-    if (lower[i] === "") { end = i + 1; continue; }
-    if (lineIndent(raw, i) <= base) break;
-    end = i + 1;
-  }
-
-  return Math.min(end, lineCount);
-}
-
-function braceDelta(text) {
-  let depth = 0;
-
-  for (const ch of text) {
-    if (ch === "{") depth++;
-    else if (ch === "}") depth--;
-  }
-
-  return depth;
-}
-
-function braceDeclarationEnd(raw, start, lineCount) {
-  let depth = braceDelta(raw[start - 1] ?? "");
-
-  if (depth <= 0) return start;
-
-  for (let i = start; i < raw.length; i++) {
-    depth += braceDelta(raw[i]);
-
-    if (depth <= 0) return i + 1;
-  }
-
-  return lineCount;
-}
-
-function declarationEnd(raw, lower, start, lineCount, ext) {
-  if (ext === ".py") return pythonDeclarationEnd(raw, lower, start, lineCount);
-
-  return braceDeclarationEnd(raw, start, lineCount);
-}
-
 function readFdBuffer(fd, buffer) {
   let offset = 0;
 
@@ -450,59 +337,18 @@ export class WorkspaceIndex {
     return this.storeCreated(filePath, loaded);
   }
 
-  static fromText(filePath, text) {
-    return { text, lower: text.toLowerCase(), ext: path.extname(filePath), surface: undefined, lines: undefined, spans: undefined };
-  }
+  static fromText(filePath, text) { return fromText(filePath, text); }
 
   /** Per-line raw text, lowercase text, declared identifier (or ""), and identifier tokens, computed once per entry. */
-  static linesOf(entry) {
-    if (entry.lines) return entry.lines;
-    const raw = entry.text.split("\n");
-    const lower = [];
-    const defNames = [];
-    const idents = [];
-
-    for (let i = 0; i < raw.length; i++) {
-      const trimmed = raw[i].trim();
-      lower[i] = trimmed.toLowerCase();
-      const declared = DEF_PATTERN.exec(trimmed);
-      defNames[i] = (declared?.[2] ?? declared?.[3] ?? "").toLowerCase();
-      idents[i] = trimmed.match(IDENT_TOKEN) || EMPTY;
-    }
-
-    entry.lines = { raw, lower, defNames, idents };
-
-    return entry.lines;
-  }
+  static linesOf(entry) { return linesOf(entry); }
 
   /**
    * Declaration spans [start, end] (1-based, inclusive). Nested bodies stay inside the parent
    * (brace-matched for JS-like, indent for Python). The file's leading header is not a span.
    */
-  static spansOf(entry) {
-    if (entry.spans) return entry.spans;
-    const { items, lineCount } = WorkspaceIndex.surfaceOf(entry);
-    const { lower, raw } = WorkspaceIndex.linesOf(entry);
-    const spans = [];
-
-    for (let i = 0; i < items.length; i++) {
-      const start = items[i].line;
-      let end = declarationEnd(raw, lower, start, lineCount, entry.ext);
-
-      while (end > start && lower[end - 1] === "") end--;
-      spans.push({ start, end, name: items[i].name, kind: items[i].kind, isExport: items[i].isExport === true });
-    }
+  static spansOf(entry) { return spansOf(entry); }
 
-    entry.spans = spans;
-
-    return spans;
-  }
-
-  static surfaceOf(entry) {
-    if (!entry.surface) entry.surface = extractStructuralSurface(entry.text, entry.ext);
-
-    return entry.surface;
-  }
+  static surfaceOf(entry) { return surfaceOf(entry); }
 
   /** True when the list is small enough to scan in-process instead of spawning rg. */
   canScan(files) {

package/src/context/search-files.js

@@ -0,0 +1,19 @@
+import * as path from 'node:path';
+import {WorkspaceIndex} from './repo-index.js';
+
+function pendingInScope(root, pendingPaths) {
+  return pendingPaths.filter(file => {
+    const relative = path.relative(root, file);
+
+    return relative === "" || (relative !== ".." && !relative.startsWith(".." + path.sep) && !path.isAbsolute(relative));
+  });
+}
+
+function overlaySearchEntry(index, filePath, overlayText) {
+  const pending = overlayText(filePath);
+
+  return pending === undefined
+    ? index.entry(filePath)
+    : Buffer.byteLength(pending, "utf8") <= 512 * 1024 ? WorkspaceIndex.fromText(filePath, pending) : null;
+}
+export { pendingInScope, overlaySearchEntry };

package/src/context/search.js

@@ -1,3 +1,5 @@
+import {textResult} from '../shared/result.js';
+import {pendingInScope,overlaySearchEntry} from './search-files.js';
 import * as fs from "node:fs/promises";
 import * as path from "node:path";
 import { isString } from "../shared/decode.js";
@@ -5,14 +7,6 @@ import { WorkspaceIndex, globToRegExp } from "./repo-index.js";
 import { rankPaths, smartCase, fuzzyMatch } from "./fuzzy.js";
 import { runCommand, relativeSlash } from "../fs/workspace.js";
 
-// Search served from the in-process index: fuzzy path find (fff port), smart-case grep with
-// definition-first rows and fuzzy fallback, glob listing. rg is spawned only for trees too
-// large to scan in-process.
-
-function textResult(text, details) {
-  return { content: [{ type: "text", text: String(text ?? "") }], details: details || {} };
-}
-
 async function candidateFileList(index, root, includeHidden = false, signal) {
   const stat = await fs.stat(root).catch(() => null);
 
@@ -21,14 +15,6 @@ async function candidateFileList(index, root, includeHidden = false, signal) {
   return index.files(root, includeHidden, signal);
 }
 
-function pendingInScope(root, pendingPaths) {
-  return pendingPaths.filter(file => {
-    const relative = path.relative(root, file);
-
-    return relative === "" || (relative !== ".." && !relative.startsWith(".." + path.sep) && !path.isAbsolute(relative));
-  });
-}
-
 function parseMatchRecord(line, truncated, isLast) {
   if (!line) return { skip: true };
 
@@ -248,14 +234,6 @@ function grepRegex(pattern, params) {
   }
 }
 
-function overlaySearchEntry(index, filePath, overlayText) {
-  const pending = overlayText(filePath);
-
-  return pending === undefined
-    ? index.entry(filePath)
-    : Buffer.byteLength(pending, "utf8") <= 512 * 1024 ? WorkspaceIndex.fromText(filePath, pending) : null;
-}
-
 function fuzzyLineRow(pattern, rawLine, defName, rel, line, maxTypos, caseSensitive) {
   const m = fuzzyMatch(pattern, rawLine, { maxTypos, caseSensitive });
 

package/src/context/snap-search.js

@@ -0,0 +1,202 @@
+import * as path from 'node:path';
+import {isString} from '../shared/decode.js';
+import {truncateChars} from '../output/format.js';
+import {extractStructuralSurface} from './surface.js';
+import {scorePathTopology,stem,SOURCE_EXT,MAX_NEEDLE_CHARS} from './query.js';
+
+const MAX_SEARCH_CHARS = 2 * 1024 * 1024;
+
+function inScope(filePath, dir, includeHidden) {
+  const relative = path.relative(dir, filePath);
+
+  if (relative === ".." || relative.startsWith(".." + path.sep) || path.isAbsolute(relative)) return false;
+  const parts = relative.split(path.sep);
+
+  return !parts.includes(".git") && (includeHidden || !parts.some(part => part.startsWith(".") && part.length > 1));
+}
+
+function makeCandidate(filePath, dir, query, tokens, flags) {
+  const relative = path.relative(dir, filePath);
+  const lower = relative.toLowerCase();
+  const base = path.basename(lower);
+
+  const extension = path.extname(base);
+  const stemBase = extension ? base.slice(0, -extension.length) : base;
+  const exactPath = lower === query.toLowerCase() || base === query.toLowerCase()
+    || stemBase === query.toLowerCase();
+
+  const needles = tokens.map(token => stem(token).slice(0, MAX_NEEDLE_CHARS));
+
+  return { path: filePath, pathScore: scorePathTopology(relative, tokens, flags), exactPath,
+    pathCoverage: tokens.filter((token, index) => lower.includes(needles[index] ?? token)).length,
+    matched: new Set(), exactDefinition: false, definitionCoverage: 0, lineCoverage: 0,
+    line: 1, signature: "", context: new Map(), recent: [], anchorScore: -1, exactLines: new Set() };
+}
+
+function bestDeclaration(items, query, tokens, needles) {
+  let declaration;
+  let definitionCoverage = 0;
+  let exact = false;
+  const queryLower = query.toLowerCase();
+
+  for (const item of items) {
+    const name = item.name.toLowerCase();
+    const itemExact = name === queryLower;
+    const coverage = tokens.filter((token, index) => name.includes(needles[index] ?? token)).length;
+
+    if (itemExact || coverage > definitionCoverage) { declaration = item; definitionCoverage = coverage; exact = itemExact; }
+    if (exact) break;
+  }
+
+  return { declaration, definitionCoverage, exact };
+}
+
+function applyMatch(candidate, lineNumber, text, query, tokens, needles, lower) {
+  const matches = tokens.filter((token, index) => lower.includes(needles[index] ?? token));
+
+  for (const token of matches) candidate.matched.add(token);
+  const ext = path.extname(candidate.path).toLowerCase();
+  const items = SOURCE_EXT.has(ext) ? extractStructuralSurface(text, ext).items : [];
+  const { declaration, definitionCoverage, exact } = bestDeclaration(items, query, tokens, needles);
+  const score = (exact ? 10000 : 0) + definitionCoverage * 40 + matches.length;
+
+  if (exact) candidate.exactLines.add(lineNumber);
+
+  if (score > candidate.anchorScore) {
+    candidate.anchorScore = score;
+    candidate.line = lineNumber;
+    candidate.signature = truncateChars(declaration?.signature ?? "", 240, "signature").text;
+    candidate.exactDefinition = exact;
+    candidate.definitionCoverage = definitionCoverage;
+    candidate.lineCoverage = matches.length;
+    candidate.context.clear();
+
+    for (const [number, line] of candidate.recent) if (number >= lineNumber - 2) candidate.context.set(number, line);
+  }
+}
+
+function inspectLine(candidate, lineNumber, raw, query, tokens, needles, isMatch) {
+  const text = raw.replace(/\r?\n$/, "");
+  const lower = text.toLowerCase();
+
+  if (isMatch) applyMatch(candidate, lineNumber, text, query, tokens, needles, lower);
+  const excerpt = truncateChars(text, 240, "source line").text;
+
+  if (lineNumber >= candidate.line - 2 && lineNumber <= candidate.line + 4) candidate.context.set(lineNumber, excerpt);
+  candidate.recent.push([lineNumber, excerpt]);
+
+  if (candidate.recent.length > 2) candidate.recent.shift();
+}
+
+function inspectOverlay(candidate, text, needles, query, tokens, signal) {
+  let start = 0, line = 1, truncated = false;
+
+  // Keep only the candidate and its short context, not another copy of every
+  // line in a staged document. Oversized individual lines disclose uncertainty.
+  while (start < text.length) {
+    if ((line & 127) === 0) signal?.throwIfAborted();
+    const newline = text.indexOf("\n", start);
+    const end = newline < 0 ? text.length : newline + 1;
+
+    if (end - start > MAX_SEARCH_CHARS) truncated = true;
+    else {
+      const row = text.slice(start, end);
+      const lower = row.toLowerCase();
+      inspectLine(candidate, line, row, query, tokens, needles, needles.some(needle => lower.includes(needle)));
+    }
+    start = end;
+    line++;
+  }
+
+  return truncated;
+}
+
+function parseRgRecord(line, truncated, isLast) {
+  if (!line) return null;
+
+  try { return JSON.parse(line); } catch (error) {
+    if (truncated && isLast) return undefined;
+    throw error;
+  }
+}
+
+function absorbRgHit(candidates, record, dir, includeHidden, overlayText, candidateRoot, query, tokens, flags, needles) {
+  if (record.type !== "match" && record.type !== "context") return;
+  const data = record.data;
+
+  if (!data?.path?.text || !isString(data.lines?.text)) return;
+  const filePath = path.resolve(dir, data.path.text);
+
+  if (!inScope(filePath, dir, includeHidden) || overlayText(filePath) !== undefined) return;
+  let candidate = candidates.get(filePath);
+
+  if (!candidate) {
+    candidate = makeCandidate(filePath, candidateRoot, query, tokens, flags);
+    candidates.set(filePath, candidate);
+  }
+
+  inspectLine(candidate, data.line_number, data.lines.text, query, tokens, needles, record.type === "match");
+}
+
+function overlayCandidates(candidates, pendingPaths, overlayText, candidateRoot, query, tokens, flags, needles, signal) {
+  let overlayTruncated = false;
+
+  for (const filePath of pendingPaths) {
+    const pending = overlayText(filePath);
+
+    if (pending === undefined) continue;
+    const candidate = makeCandidate(filePath, candidateRoot, query, tokens, flags);
+    overlayTruncated = inspectOverlay(candidate, pending, needles, query, tokens, signal) || overlayTruncated;
+
+    if (candidate.matched.size) candidates.set(filePath, candidate);
+  }
+
+  return overlayTruncated;
+}
+
+function rgSearchArgs(includeHidden, searchNeedles, focusFile, dir) {
+  const args = ["rg", "--json", "--fixed-strings", "--ignore-case", "--before-context", "2", "--after-context", "4"];
+
+  if (includeHidden) args.push("--hidden");
+  args.push("-g", "!.git/**", "-g", "!**/.git/**");
+  for (const needle of searchNeedles) args.push("-e", needle);
+  args.push("--", focusFile ?? dir);
+
+  return args;
+}
+
+async function runContentSearch({ dir, includeHidden, searchNeedles, run, overlayText, signal, diskFiles, focusFile }) {
+  const args = rgSearchArgs(includeHidden, searchNeedles, focusFile, dir);
+  const response = diskFiles || (focusFile && overlayText(focusFile) === undefined)
+    ? await run(args, { cwd: focusFile ? path.dirname(focusFile) : dir, timeoutMs: 15000, maxOutputChars: MAX_SEARCH_CHARS, signal })
+    : { stdout: "", stderr: "", exitCode: 1 };
+
+  if (response.exitCode !== 0 && response.exitCode !== 1) throw new Error("source search failed: " + response.stderr.trim());
+
+  return response;
+}
+
+function absorbRgRecords(candidates, response, dir, includeHidden, overlayText, candidateRoot, query, tokens, flags, needles, signal) {
+  const records = response.stdout.split("\n");
+
+  for (let i = 0; i < records.length; i++) {
+    if ((i & 127) === 0) signal?.throwIfAborted();
+    const record = parseRgRecord(records[i], response.outputTruncated, i === records.length - 1);
+
+    if (record === undefined) break;
+    if (!record) continue;
+    absorbRgHit(candidates, record, dir, includeHidden, overlayText, candidateRoot, query, tokens, flags, needles);
+  }
+}
+
+async function contentCandidates({ dir, includeHidden, query, tokens, flags, pendingPaths, run, overlayText, signal, exact, diskFiles, focusFile }) {
+  const needles = exact ? [query.toLowerCase().slice(0, MAX_NEEDLE_CHARS)] : tokens.map(token => stem(token).slice(0, MAX_NEEDLE_CHARS));
+  const candidateRoot = focusFile ? path.dirname(focusFile) : dir;
+  const candidates = new Map();
+  const response = await runContentSearch({ dir, includeHidden, searchNeedles: [...new Set(needles)], run, overlayText, signal, diskFiles, focusFile });
+  absorbRgRecords(candidates, response, dir, includeHidden, overlayText, candidateRoot, query, tokens, flags, needles, signal);
+  const overlayTruncated = overlayCandidates(candidates, pendingPaths, overlayText, candidateRoot, query, tokens, flags, needles, signal);
+
+  return { candidates, truncated: response.outputTruncated === true || overlayTruncated };
+}
+export { inScope, makeCandidate, contentCandidates, MAX_SEARCH_CHARS };