pi-supernova 0.0.8 → 0.0.15

package/repo-index.js

@@ -0,0 +1,273 @@
+import * as fs from "node:fs";
+import * as path from "node:path";
+import { extractStructuralSurface } from "./surface.js";
+import { Frecency } from "./fuzzy.js";
+import { relativeSlash } from "./workspace.js";
+
+// In-process workspace index: the gitignore-aware file list comes from one
+// \`rg --files\` spawn and is then reused; file text, lowercase text, and the
+// structural surface are cached per path and validated by mtime. snap/grep/glob
+// read from here instead of spawning, so a warm call is sub-millisecond.
+
+// With a working fs.watch the list only refreshes on change; the TTL is the fallback when watching fails.
+const LIST_TTL_MS = 10_000;
+const WATCHED_TTL_MS = 5 * 60_000;
+const WATCH_DEBOUNCE_MS = 150;
+const MAX_INDEXED_FILES = 4000;
+const MAX_FILE_BYTES = 512 * 1024;
+const BINARY_EXT = new Set([
+  ".png", ".jpg", ".jpeg", ".gif", ".webp", ".ico", ".pdf", ".zip", ".gz", ".tgz", ".tar", ".bz2", ".xz", ".7z",
+  ".woff", ".woff2", ".ttf", ".otf", ".eot", ".mp3", ".mp4", ".mov", ".wav", ".ogg", ".webm", ".wasm", ".class",
+  ".jar", ".so", ".dylib", ".dll", ".exe", ".bin", ".o", ".a", ".node", ".lock", ".sqlite", ".sqlite3", ".db",
+]);
+const REGEX_SPECIAL = /[.+^${}()|\\]/g;
+const IDENT_TOKEN = /[A-Za-z_$][\w$]*/g;
+const EMPTY = Object.freeze([]);
+const DEF_PATTERN = /^(?:pub\s+)?(?:export\s+)?(?:async\s+)?(?:default\s+)?(function|class|def|fn|const|let|interface|type|struct|enum)\s+([a-zA-Z0-9_$]+)/;
+
+/** Declared identifier on a line (function/class/const/…), or ""; the same rule snap and grep use. */
+export function declaredName(line) {
+  return DEF_PATTERN.exec(String(line).trim())?.[2] ?? "";
+}
+
+function isTextCandidate(filePath) {
+  return !BINARY_EXT.has(path.extname(filePath).toLowerCase());
+}
+
+/** Translate one glob token at index i → [regexSource, nextIndex]. */
+function globToken(glob, i) {
+  const ch = glob[i];
+  if (ch === "*" && glob[i + 1] === "*") {
+    const slashAfter = glob[i + 2] === "/";
+    return [slashAfter ? "(?:.*/)?" : ".*", i + (slashAfter ? 3 : 2)];
+  }
+  if (ch === "*") return ["[^/]*", i + 1];
+  if (ch === "?") return ["[^/]", i + 1];
+  if (ch === "{" || ch === "[") return globGroup(glob, i, ch);
+  return [ch.replace(REGEX_SPECIAL, "\\$&"), i + 1];
+}
+
+/** {a,b} alternation or [..] class starting at i. */
+function globGroup(glob, i, open) {
+  const close = open === "{" ? "}" : "]";
+  const end = glob.indexOf(close, i);
+  if (end < 0) throw new SyntaxError("unclosed " + open + " in glob");
+  const inner = glob.slice(i + 1, end);
+  const source = open === "{" ? "(?:" + inner.split(",").map(globBody).join("|") + ")" : "[" + inner + "]";
+  return [source, end + 1];
+}
+
+function globBody(glob) {
+  let source = "";
+  let i = 0;
+  while (i < glob.length) {
+    const [piece, next] = globToken(glob, i);
+    source += piece;
+    i = next;
+  }
+  return source;
+}
+
+/** gitignore-style glob (rg -g) → RegExp over a "/"-separated relative path. No slash ⇒ basename match anywhere. */
+export function globToRegExp(glob) {
+  const body = globBody(glob);
+  return new RegExp(glob.includes("/") ? "^" + body + "$" : "(?:^|/)" + body + "$");
+}
+
+export class WorkspaceIndex {
+  constructor(runCommand) {
+    this.runCommand = runCommand;
+    this.lists = new Map();
+    this.entries = new Map();
+    this.watchers = new Map();
+    this.frecency = new Frecency();
+    this.gitModified = new Map(); // root → Set(relative "/"-joined paths)
+    this.lastTouched = null;
+  }
+
+  invalidate() {
+    this.lists.clear();
+  }
+
+  /** fff frecency: every read/edit is an access; the newest one is the "current file" for distance penalties. */
+  touch(relPath) {
+    this.frecency.record(relPath);
+    this.lastTouched = relPath;
+  }
+
+  watch(root) {
+    if (this.watchers.has(root)) return this.watchers.get(root);
+    let ok = false;
+    try {
+      let timer = null;
+      const watcher = fs.watch(root, { recursive: true }, () => {
+        if (timer) return;
+        timer = setTimeout(() => {
+          timer = null;
+          this.lists.clear();
+          this.gitModified.delete(root);
+        }, WATCH_DEBOUNCE_MS);
+      });
+      watcher.on("error", () => {
+        this.watchers.set(root, false);
+        this.lists.clear();
+      });
+      if (typeof watcher.unref === "function") watcher.unref();
+      ok = true;
+    } catch {
+      ok = false;
+    }
+    this.watchers.set(root, ok);
+    return ok;
+  }
+
+  /** Paths git reports as modified/added/untracked (fff's git-status boost); one spawn per list refresh. */
+  async modifiedFiles(root) {
+    const cached = this.gitModified.get(root);
+    if (cached) return cached;
+    const set = new Set();
+    try {
+      const res = await this.runCommand(["git", "status", "--porcelain", "-z", "--untracked-files=all"], { cwd: root, timeoutMs: 5_000 });
+      if (res.exitCode === 0) {
+        for (const row of res.stdout.split("\0")) {
+          if (row.length > 3) set.add(row.slice(3));
+        }
+      }
+    } catch {}
+    this.gitModified.set(root, set);
+    return set;
+  }
+
+  mtimeSeconds(filePath) {
+    const e = this.entries.get(filePath);
+    if (e) return e.mtimeMs / 1000;
+    try {
+      return fs.statSync(filePath).mtimeMs / 1000;
+    } catch {
+      return 0;
+    }
+  }
+
+  /** Absolute, sorted file list for a root; gitignore-aware via rg; cached for LIST_TTL_MS. */
+  async files(root, includeHidden = false) {
+    const key = root + "\0" + (includeHidden ? "h" : "");
+    const cached = this.lists.get(key);
+    const ttl = this.watch(root) ? WATCHED_TTL_MS : LIST_TTL_MS;
+    if (cached && Date.now() - cached.at < ttl) return cached.files;
+    const args = ["rg", "--files"];
+    if (includeHidden) args.push("--hidden");
+    args.push("-g", "!.git/**", "-g", "!**/.git/**", root);
+    let files = [];
+    try {
+      const res = await this.runCommand(args, { cwd: root, timeoutMs: 15_000 });
+      files = res.stdout.split("\n").map((f) => f.trim()).filter(Boolean).map((f) => path.resolve(root, f)).sort();
+    } catch {
+      files = [];
+    }
+    this.lists.set(key, { files, at: Date.now() });
+    return files;
+  }
+
+  /** Cached {text, lower, ext, surface?} for a file, re-read when mtime/size changed. Null for unreadable, binary, or huge files. */
+  entry(filePath) {
+    if (!isTextCandidate(filePath)) return null;
+    let stat;
+    try {
+      stat = fs.statSync(filePath);
+    } catch {
+      this.entries.delete(filePath);
+      return null;
+    }
+    if (!stat.isFile() || stat.size > MAX_FILE_BYTES) return null;
+    const cached = this.entries.get(filePath);
+    if (cached && cached.mtimeMs === stat.mtimeMs && cached.size === stat.size) return cached;
+    let text;
+    try {
+      text = fs.readFileSync(filePath, "utf8");
+    } catch {
+      return null;
+    }
+    if (text.includes("\0")) return null;
+    const created = { text, lower: text.toLowerCase(), mtimeMs: stat.mtimeMs, size: stat.size, ext: path.extname(filePath), surface: undefined, lines: undefined, spans: undefined };
+    this.entries.set(filePath, created);
+    return created;
+  }
+
+  static fromText(filePath, text) {
+    return { text, lower: text.toLowerCase(), ext: path.extname(filePath), surface: undefined, lines: undefined, spans: undefined };
+  }
+
+  /** Per-line raw text, lowercase text, declared identifier (or ""), and identifier tokens, computed once per entry. */
+  static linesOf(entry) {
+    if (entry.lines) return entry.lines;
+    const raw = entry.text.split("\n");
+    const lower = new Array(raw.length);
+    const defNames = new Array(raw.length);
+    const idents = new Array(raw.length);
+    for (let i = 0; i < raw.length; i++) {
+      const trimmed = raw[i].trim();
+      lower[i] = trimmed.toLowerCase();
+      defNames[i] = DEF_PATTERN.exec(trimmed)?.[2].toLowerCase() ?? "";
+      idents[i] = trimmed.match(IDENT_TOKEN) || EMPTY;
+    }
+    entry.lines = { raw, lower, defNames, idents };
+    return entry.lines;
+  }
+
+  /**
+   * Declaration spans [start, end] (1-based, inclusive) in file order, trailing blank lines trimmed.
+   * A span runs to the line before the next declaration; the file's leading header is not a span.
+   */
+  static spansOf(entry) {
+    if (entry.spans) return entry.spans;
+    const { items, lineCount } = WorkspaceIndex.surfaceOf(entry);
+    const { lower } = WorkspaceIndex.linesOf(entry);
+    const spans = [];
+    for (let i = 0; i < items.length; i++) {
+      const start = items[i].line;
+      let end = Math.min(i + 1 < items.length ? items[i + 1].line - 1 : lineCount, lineCount);
+      while (end > start && lower[end - 1] === "") end--;
+      spans.push({ start, end, name: items[i].name, kind: items[i].kind, isExport: items[i].isExport === true });
+    }
+    entry.spans = spans;
+    return spans;
+  }
+
+  static surfaceOf(entry) {
+    if (!entry.surface) entry.surface = extractStructuralSurface(entry.text, entry.ext);
+    return entry.surface;
+  }
+
+  /** True when the list is small enough to scan in-process instead of spawning rg. */
+  canScan(files) {
+    return files.length <= MAX_INDEXED_FILES;
+  }
+
+  /** Files whose lowercase text contains any (or every) needle; needles are lowercase. */
+  filesContaining(files, needles, anyOf) {
+    const hits = [];
+    for (const filePath of files) {
+      const e = this.entry(filePath);
+      if (!e) continue;
+      const found = anyOf ? needles.some((n) => e.lower.includes(n)) : needles.every((n) => e.lower.includes(n));
+      if (found) hits.push(filePath);
+    }
+    return hits;
+  }
+
+  /** Structured grep rows {rel, line, text, def}; def marks lines whose declared name itself matches. */
+  grepRows(files, regex, root) {
+    const out = [];
+    const nameRegex = new RegExp(regex.source, "i");
+    for (const filePath of files) {
+      const e = this.entry(filePath);
+      if (!e || !regex.test(e.text)) continue;
+      const { raw, defNames } = WorkspaceIndex.linesOf(e);
+      const rel = relativeSlash(root, filePath);
+      for (let i = 0; i < raw.length; i++) {
+        if (regex.test(raw[i])) out.push({ rel, line: i + 1, text: raw[i], def: defNames[i] !== "" && nameRegex.test(defNames[i]) });
+      }
+    }
+    return out;
+  }
+}

package/runtime.js

@@ -94,15 +94,6 @@ export function warmGuestWorker(config) {
   return handle.ready;
 }
 
-/** Terminate every guest worker (tests, shutdown). */
-export async function shutdownGuestWorkers() {
-  if (!idleWorker) return;
-  const handle = idleWorker;
-  idleWorker = null;
-  handle.dead = true;
-  await handle.worker.terminate();
-}
-
 const RPC_METHODS = {
   call: (nova, args) => {
     if (!isFunction(nova?.call)) throw new Error("nova.call unavailable");
@@ -122,17 +113,9 @@ const RPC_METHODS = {
     if (!isFunction(nova?.describe)) throw new Error("nova.describe unavailable");
     return nova.describe(args[0]);
   },
-  surface: (nova, args) => {
-    if (isFunction(nova?.surface)) return nova.surface(args[0]);
-    return nova.call("surface", { path: args[0] });
-  },
-  snap: (nova, args) => {
-    if (isFunction(nova?.snap)) return nova.snap(args[0], args[1]);
-    return nova.call("snap", { query: args[0], path: args[1] });
-  },
-  speculateBegin: (nova) => (isFunction(nova?.speculateBegin) ? nova.speculateBegin() : undefined),
-  speculateCommit: (nova) => (isFunction(nova?.speculateCommit) ? nova.speculateCommit() : undefined),
-  speculateRollback: (nova) => (isFunction(nova?.speculateRollback) ? nova.speculateRollback() : undefined),
+  speculateBegin: (nova) => nova?.speculateBegin?.(),
+  speculateCommit: (nova) => nova?.speculateCommit?.(),
+  speculateRollback: (nova) => nova?.speculateRollback?.(),
 };
 
 async function dispatchRpc(nova, method, args) {

package/search.js

@@ -0,0 +1,141 @@
+import * as path from "node:path";
+import { WorkspaceIndex, globToRegExp } from "./repo-index.js";
+import { rankPaths, smartCase, fuzzyMatch } from "./fuzzy.js";
+import { runCommand, relativeSlash } from "./workspace.js";
+
+// Search served from the in-process index: fuzzy path find (fff port), smart-case grep with
+// definition-first rows and fuzzy fallback, glob listing. rg is spawned only for trees too
+// large to scan in-process.
+
+function textResult(text, details) {
+  return { content: [{ type: "text", text: String(text ?? "") }], details: details || {} };
+}
+
+export function rgGrepArgs(pattern, params, searchPath) {
+  const args = ["--line-number", "--no-heading", "--color", "never"];
+  if (params?.caseSensitive !== true) args.push("--ignore-case");
+  if (params?.glob) args.push("--glob", String(params.glob));
+  args.push("--", pattern, searchPath);
+  return args;
+}
+
+/** rg --files, then find(1) when rg is unavailable; both accept an optional glob/name pattern. */
+export async function listWithTools(searchDir, pattern, cwd, signal) {
+  const args = ["--files"];
+  if (pattern) args.push("-g", pattern);
+  const res = await runCommand(["rg", ...args, searchDir], { cwd, timeoutMs: 30_000, signal }).catch(() => null);
+  if (res && (res.exitCode === 0 || res.exitCode === 1)) return textResult(res.stdout, { via: "rg" });
+  const findArgs = [searchDir];
+  if (pattern) findArgs.push("-name", pattern);
+  const findRes = await runCommand(["find", ...findArgs], { cwd, timeoutMs: 30_000, signal });
+  return textResult(findRes.stdout, { via: "find" });
+}
+
+const GLOB_CHARS = /[*?[\]{}]/;
+
+/**
+ * fffind: a pattern without glob characters is a fuzzy, typo-tolerant, frecency-ranked path query.
+ * Returns "path" rows (best first) or null when the pattern is a real glob.
+ */
+export async function fuzzyFind(index, root, cwd, pattern, limit = 20) {
+  if (!pattern || GLOB_CHARS.test(pattern)) return null;
+  const files = await index.files(root);
+  if (!index.canScan(files)) return null;
+  const rel = files.map((f) => relativeSlash(cwd, f));
+  const absolute = new Map(rel.map((r, i) => [r, files[i]]));
+  // mtime is only consulted for paths that matched; never stat the whole tree.
+  const mtimeOf = (r) => index.mtimeSeconds(absolute.get(r));
+  const ranked = rankPaths(pattern, rel, { frecency: index.frecency, mtimeOf, modified: await index.modifiedFiles(cwd), currentFile: index.lastTouched });
+  // fff weak-match detector: when nothing matches exactly and the best is mostly typos, say so instead of flooding.
+  const rows = ranked.slice(0, limit);
+  if (rows.length === 0) return "";
+  return rows.map((r) => r.path).join("\n") + "\n";
+}
+
+/** fff-style grep: smart-case, definition lines first, fuzzy fallback when the literal has no hits. */
+export async function grepIndexed(index, pattern, params, searchPath, cwd) {
+  const compiled = grepRegex(pattern, params);
+  if (!compiled) return null;
+  const { regex, caseSensitive } = compiled;
+  let files = await index.files(searchPath);
+  if (!index.canScan(files)) return null;
+  if (params?.glob) {
+    const matcher = globToRegExp(String(params.glob));
+    files = files.filter((f) => matcher.test(relativeSlash(cwd, f)));
+  }
+  const rows = index.grepRows(files, regex, cwd);
+  const fallback = rows.length === 0 && /^[\w$.-]{4,}$/.test(pattern) ? fuzzyGrepRows(index, files, pattern, cwd, caseSensitive) : rows;
+  return formatGrepRows(fallback, grepLimit(params));
+}
+
+function grepLimit(params) {
+  return Number.isInteger(params?.limit) && params.limit > 0 ? params.limit : 200;
+}
+
+function grepRegex(pattern, params) {
+  const caseSensitive = params?.caseSensitive === true || (params?.caseSensitive !== false && smartCase(pattern));
+  try {
+    return { regex: new RegExp(pattern, caseSensitive ? "" : "i"), caseSensitive };
+  } catch {
+    return null;
+  }
+}
+
+/** Zero literal hits: retry each line fuzzily (1 typo, 2 for long names) within a tight span, so IsOffTheRecord finds is_off_the_record. */
+function fuzzyGrepRows(index, files, pattern, cwd, caseSensitive) {
+  const maxTypos = pattern.length >= 8 ? 2 : 1;
+  const rows = [];
+  for (const filePath of files) {
+    const e = index.entry(filePath);
+    if (!e) continue;
+    const { raw, defNames } = WorkspaceIndex.linesOf(e);
+    const rel = relativeSlash(cwd, filePath);
+    for (let i = 0; i < raw.length && rows.length <= 400; i++) {
+      const m = fuzzyMatch(pattern, raw[i], { maxTypos, caseSensitive });
+      if (!m || m.end - m.start > pattern.length + 2) continue;
+      rows.push({ rel, line: i + 1, text: raw[i], def: defNames[i] !== "" && fuzzyMatch(pattern, defNames[i], { maxTypos }) !== null });
+    }
+  }
+  return rows;
+}
+
+/** fff definition-first hinting: files that declare the name come first, declarations first within a file; one header per file. */
+function formatGrepRows(rows, limit) {
+  if (rows.length === 0) return "";
+  const groups = new Map();
+  for (const r of rows) {
+    if (!groups.has(r.rel)) groups.set(r.rel, []);
+    groups.get(r.rel).push(r);
+  }
+  const files = [...groups.values()].sort((a, b) => Number(b.some((r) => r.def)) - Number(a.some((r) => r.def)));
+  let out = "";
+  let shown = 0;
+  for (const group of files) {
+    if (shown >= limit) break;
+    out += group[0].rel + "\n";
+    group.sort((a, b) => Number(b.def) - Number(a.def) || a.line - b.line);
+    for (const r of group) {
+      if (shown++ >= limit) break;
+      out += "  " + r.line + (r.def ? "*" : ":") + " " + r.text.trim() + "\n";
+    }
+  }
+  if (rows.length > limit) out += "… " + (rows.length - limit) + " more matches (pass limit or narrow the pattern)\n";
+  return out;
+}
+
+/** rg --files [-g pattern] served from the index; null when the tree is too large. */
+export async function listIndexed(index, root, cwd, pattern) {
+  const files = await index.files(root);
+  if (!index.canScan(files)) return null;
+  const rel = files.map((f) => path.relative(cwd, f).split(path.sep).join("/"));
+  if (!pattern) return rel.length ? rel.join("\n") + "\n" : "";
+  let matcher;
+  try {
+    matcher = globToRegExp(pattern);
+  } catch {
+    return null;
+  }
+  const hits = rel.filter((f) => matcher.test(f));
+  return hits.length ? hits.join("\n") + "\n" : "";
+}
+