pi-diet-ripgrep 0.1.3 → 0.1.5

package/README.md

@@ -1,6 +1,6 @@
 # pi-diet-ripgrep
 
-> One of my diet context engineering and workflow extensions. Add pi-diet-LSP, pi-diet-Ripgrep, pi-repo-move and others from [npm](https://www.npmjs.com/~probabilityengineer).
+> One of my diet context engineering and workflow extensions. Explore the complete collection: <https://www.npmjs.com/~probabilityengineer>
 
 A minimal Pi extension that exposes ripgrep as a first-class exact-search tool.
 
@@ -37,6 +37,28 @@ mkdir -p ~/.pi/agent/extensions/pi-diet-ripgrep
 ln -sf /path/to/pi-diet-ripgrep/index.ts ~/.pi/agent/extensions/pi-diet-ripgrep/index.ts
 ```
 
+## Development and validation
+
+Install a current Node.js release and `rg` (ripgrep), then run:
+
+```bash
+npm ci
+npm run lint
+npm run build
+npm test
+npm pack --dry-run
+```
+
+`lint` performs strict TypeScript checking of the extension and tests. `build`
+optionally compiles the extension to ignored `dist/` output; Pi and the published
+package continue to load `index.ts` directly.
+
+Tests require a POSIX environment (macOS or Linux). They exercise the registered
+tool with controlled subprocess fixtures plus a real `rg` integration test.
+Coverage includes arguments and leading-dash patterns, formatting and context,
+no matches, invalid regexes, missing executables, match/byte/line limits, complete
+saved output, and self-contained tool guidance.
+
 ## Publishing
 
 First publish manually once:
@@ -67,4 +89,4 @@ git push && git push origin v0.1.1
 
 ## Tool
 
-`rg` is intended for exact text and regex search only.
+`rg` is intended for exact text and regex search: literal verification, logs, config keys, TODOs, and message lookup. Text matches alone do not establish symbol identity, code structure, or behavior. Its guidance does not assume other extensions are installed.

package/index.ts

@@ -1,8 +1,9 @@
-import { execFile as execFileCallback } from "node:child_process";
-import { mkdtemp, writeFile } from "node:fs/promises";
+import { spawn } from "node:child_process";
+import { mkdtemp, open, rm } from "node:fs/promises";
+import { createReadStream } from "node:fs";
+import { createInterface } from "node:readline";
 import { tmpdir } from "node:os";
 import { join } from "node:path";
-import { promisify } from "node:util";
 import { Type } from "@earendil-works/pi-ai";
 import {
   DEFAULT_MAX_BYTES,
@@ -11,16 +12,16 @@ import {
   formatSize,
   truncateHead,
   type TruncationResult,
-  withFileMutationQueue,
   type ExtensionAPI,
 } from "@earendil-works/pi-coding-agent";
 
-const execFile = promisify(execFileCallback);
+const MAX_METADATA_MATCH_BYTES = 20_000;
+const MAX_LINE_BYTES = 1_024;
 
 const rgTool = defineTool({
   name: "rg",
   label: "ripgrep",
-  description: "Exact text/regex search with ripgrep. Use for literal verification, logs, config keys, TODOs, and message lookup. Prefer LSP/AST/Semble for symbol, structural, or semantic discovery.",
+  description: "Exact text/regex search with ripgrep. Use for literal verification, logs, config keys, TODOs, and message lookup. Text matches alone do not establish symbol identity, code structure, or behavior.",
   parameters: Type.Object({
     query: Type.String({ description: "Pattern" }),
     paths: Type.Optional(Type.Array(Type.String({ description: "Path" }))),
@@ -33,72 +34,58 @@ const rgTool = defineTool({
     fixedStrings: Type.Optional(Type.Boolean({ description: "Literal strings" })),
   }),
 
-  async execute(_toolCallId, params, _signal, _onUpdate, ctx) {
+  async execute(_toolCallId, params, signal, _onUpdate, ctx) {
     const args = buildArgs(params);
     const searchPaths = params.paths?.length ? params.paths : ["."];
     const maxMatches = params.maxMatches ?? 50;
 
-    let stdout = "";
-    try {
-      const result = await execFile("rg", [...args, params.query, ...searchPaths], {
-        cwd: ctx.cwd,
-        encoding: "utf8",
-        maxBuffer: 100 * 1024 * 1024,
-      });
-      stdout = result.stdout;
-    } catch (error: any) {
-      stdout = error?.stdout ?? "";
-      if (error?.code !== 1) {
-        throw new Error(error?.stderr?.trim() || error?.message || "ripgrep failed");
-      }
-    }
-
-    const parsed = parseRgJson(stdout, maxMatches);
+    const capture = await captureRg([...args, "--", params.query, ...searchPaths], ctx.cwd, signal);
+    const parsed = await parseRgJson(capture.rawPath, capture.fullPath, maxMatches);
+    const truncation = truncateHead(parsed.lines.join("\n"), {
+      maxLines: DEFAULT_MAX_LINES,
+      maxBytes: DEFAULT_MAX_BYTES,
+    });
     const details: RgDetails = {
-      query: params.query,
-      searchedPaths: searchPaths,
+      query: shortText(params.query, 512),
+      searchedPaths: searchPaths.slice(0, 20).map(value => shortText(value, 256)),
       matchCount: parsed.totalMatchCount,
-      truncated: parsed.totalMatchCount > parsed.matches.length,
+      truncated: parsed.omittedMatches > 0 || parsed.metadataOmitted > 0 ||
+        parsed.shortenedLines > 0 || parsed.previewOverflow || truncation.truncated ||
+        parsed.invalidRecords > 0,
       matches: parsed.matches,
+      metadataOmitted: parsed.metadataOmitted,
+      shortenedLines: parsed.shortenedLines,
+      incomplete: parsed.invalidRecords > 0,
     };
-
-    if (!parsed.totalMatchCount) {
-      return {
-        content: [{ type: "text", text: "No matches found" }],
-        details,
-      };
-    }
-
-    const fullText = parsed.lines.join("\n");
-    const truncation = truncateHead(fullText, {
-      maxLines: DEFAULT_MAX_LINES,
-      maxBytes: DEFAULT_MAX_BYTES,
-    });
-
-    let text = truncation.content;
-    if (details.truncated || truncation.truncated) {
-      details.truncated = true;
-      details.truncation = truncation;
-      const tempDir = await mkdtemp(join(tmpdir(), "pi-diet-ripgrep-"));
-      const tempFile = join(tempDir, "output.txt");
-      await withFileMutationQueue(tempFile, async () => {
-        await writeFile(tempFile, fullText, "utf8");
-      });
-      details.fullOutputPath = tempFile;
-      const omittedMatches = Math.max(0, parsed.totalMatchCount - parsed.matches.length);
+    if (details.query !== params.query || searchPaths.length > 20 ||
+        searchPaths.some(value => Buffer.byteLength(value) > 256)) details.truncated = true;
+    let text = truncation.content || (parsed.invalidRecords ? "No complete matching records parsed." : "No matches found");
+    if (details.truncated) {
+      // Do not duplicate the display content inside transcript metadata.
+      const { content: _content, ...stats } = truncation;
+      details.truncation = stats;
+      details.fullOutputPath = capture.fullPath;
+      details.rawOutputPath = capture.rawPath;
       const notices: string[] = [];
-      if (omittedMatches) notices.push(`${omittedMatches} matches omitted by maxMatches cap`);
-      if (truncation.truncated) {
+      if (parsed.omittedMatches) notices.push(`${parsed.omittedMatches} matches omitted by maxMatches cap`);
+      if (parsed.metadataOmitted) notices.push(`${parsed.metadataOmitted} selected matches omitted from metadata by its byte budget`);
+      if (parsed.shortenedLines) notices.push(`${parsed.shortenedLines} long lines shortened; match windows may omit other occurrences`);
+      if (parsed.invalidRecords) notices.push(`${parsed.invalidRecords} invalid JSON records; search evidence is incomplete`);
+      if (truncation.truncated || parsed.previewOverflow) {
         notices.push(
           `output truncated to ${truncation.outputLines}/${truncation.totalLines} lines and ${formatSize(truncation.outputBytes)}/${formatSize(truncation.totalBytes)}`,
         );
       }
-      text += `\n\n[${notices.join("; ")}. Full output: ${tempFile}]`;
+      if (!notices.length) notices.push("query/path metadata shortened");
+      text += `\n\n[${notices.join("; ")}. Full output: ${capture.fullPath}; raw rg JSON: ${capture.rawPath}. Temporary files; not a durable archive.]`;
+    } else {
+      await rm(capture.directory, { recursive: true, force: true });
     }
 
     return {
       content: [{ type: "text", text }],
       details,
+      isError: parsed.invalidRecords > 0,
     };
   },
 });
@@ -142,21 +129,94 @@ type RgDetails = {
   matchCount: number;
   truncated: boolean;
   matches: RgMatch[];
-  truncation?: TruncationResult;
+  truncation?: Omit<TruncationResult, "content">;
   fullOutputPath?: string;
+  rawOutputPath?: string;
+  metadataOmitted?: number;
+  shortenedLines?: number;
+  incomplete?: boolean;
 };
 
-function parseRgJson(stdout: string, maxMatches: number) {
+function shortText(value: string, maxBytes: number) {
+  const buffer = Buffer.from(value);
+  if (buffer.length <= maxBytes) return value;
+  return windowText(buffer, 0, maxBytes - 3) + "...";
+}
+
+function windowText(buffer: Buffer, start: number, end: number) {
+  while (start < end && (buffer[start] & 0xc0) === 0x80) start++;
+  while (end < buffer.length && end > start && (buffer[end] & 0xc0) === 0x80) end--;
+  return buffer.subarray(start, end).toString("utf8");
+}
+
+function linePreview(value: string, submatches: { start: number }[] = []) {
+  const buffer = Buffer.from(value);
+  if (buffer.length <= MAX_LINE_BYTES) return value;
+  if (!submatches.length) return shortText(value, MAX_LINE_BYTES);
+  return submatches.slice(0, 3).map(match => {
+    const start = Math.max(0, match.start - 120);
+    const end = Math.min(buffer.length, match.start + 200);
+    return `[match byte column ${match.start + 1}] ${start > 0 ? "..." : ""}${windowText(buffer, start, end)}${end < buffer.length ? "..." : ""}`;
+  }).join(" | ");
+}
+
+async function captureRg(args: string[], cwd: string, signal?: AbortSignal) {
+  const directory = await mkdtemp(join(tmpdir(), "pi-diet-ripgrep-"));
+  const rawPath = join(directory, "rg.jsonl");
+  const stderrPath = join(directory, "stderr.txt");
+  const out = await open(rawPath, "w", 0o600);
+  const err = await open(stderrPath, "w", 0o600);
+  let failure: Error | undefined;
+  let code: number | null;
+  try {
+    code = await new Promise<number | null>(resolve => {
+      const child = spawn("rg", args, { cwd, signal, stdio: ["ignore", out.fd, err.fd] });
+      child.on("error", error => { failure = error; });
+      child.on("close", resolve);
+    });
+  } finally {
+    await Promise.all([out.close(), err.close()]);
+  }
+  if (failure || (code !== 0 && code !== 1)) {
+    const file = await open(stderrPath, "r");
+    let message: string;
+    try {
+      const bytes = Buffer.alloc(4_000);
+      const { bytesRead } = await file.read(bytes, 0, bytes.length, 0);
+      message = bytes.subarray(0, bytesRead).toString("utf8").trim();
+    } finally { await file.close(); }
+    throw new Error(`${message || failure?.message || `ripgrep exited with code ${code}`}\nCaptured stdout: ${rawPath}; stderr: ${stderrPath}`);
+  }
+  return { directory, rawPath, fullPath: join(directory, "output.txt") };
+}
+
+async function parseRgJson(rawPath: string, fullPath: string, maxMatches: number) {
   const lines: string[] = [];
   const matches: RgMatch[] = [];
   let totalMatchCount = 0;
-
-  for (const rawLine of stdout.split(/\r?\n/)) {
+  let metadataBytes = 0, metadataOmitted = 0, shortenedLines = 0, invalidRecords = 0;
+  let previewBytes = 0, previewOverflow = false, first = true;
+  const output = await open(fullPath, "w", 0o600);
+  const input = createReadStream(rawPath);
+  const records = createInterface({ input, crlfDelay: Infinity });
+  const addPreview = (line: string) => {
+    if (previewOverflow) return;
+    lines.push(line);
+    previewBytes += Buffer.byteLength(line) + 1;
+    if (lines.length > DEFAULT_MAX_LINES || previewBytes > DEFAULT_MAX_BYTES) previewOverflow = true;
+  };
+  try {
+  for await (const rawLine of records) {
     if (!rawLine.trim()) continue;
     let record: any;
     try {
       record = JSON.parse(rawLine);
     } catch {
+      invalidRecords++;
+      continue;
+    }
+    if (!record || typeof record !== "object") {
+      invalidRecords++;
       continue;
     }
 
@@ -166,20 +226,41 @@ function parseRgJson(stdout: string, maxMatches: number) {
       const line = record.data?.line_number ?? 0;
       const text = String(record.data?.lines?.text ?? "").replace(/\n$/, "");
       const column = (record.data?.submatches?.[0]?.start ?? 0) + 1;
-      if (matches.length < maxMatches) {
-        matches.push({ path, line, column, text });
-        lines.push(`${path}:${line}:${column}:${text}`);
+      await output.writeFile(`${first ? "" : "\n"}${path}:${line}:${column}:${text}`);
+      first = false;
+      if (totalMatchCount <= maxMatches) {
+        const preview = linePreview(text, record.data?.submatches);
+        if (preview !== text || Buffer.byteLength(path) > 512) shortenedLines++;
+        const match = { path: shortText(path, 512), line, column, text: preview };
+        const bytes = Buffer.byteLength(JSON.stringify(match));
+        if (metadataBytes + bytes <= MAX_METADATA_MATCH_BYTES) {
+          matches.push(match);
+          metadataBytes += bytes;
+        } else metadataOmitted++;
+        addPreview(`${shortText(path, 512)}:${line}:${column}:${preview}`);
       }
       continue;
     }
 
-    if (record.type === "context" && matches.length > 0) {
+    if (record.type === "context") {
       const path = record.data?.path?.text ?? "<unknown>";
       const line = record.data?.line_number ?? 0;
       const text = String(record.data?.lines?.text ?? "").replace(/\n$/, "");
-      lines.push(`${path}-${line}- ${text}`);
+      const formatted = `${path}-${line}- ${text}`;
+      await output.writeFile(`${first ? "" : "\n"}${formatted}`);
+      first = false;
+      if (totalMatchCount <= maxMatches) {
+        const preview = linePreview(text);
+        if (preview !== text || Buffer.byteLength(path) > 512) shortenedLines++;
+        addPreview(`${shortText(path, 512)}-${line}- ${preview}`);
+      }
     }
   }
-
-  return { lines, matches, totalMatchCount };
+  } finally {
+    records.close();
+    input.destroy();
+    await output.close();
+  }
+  return { lines, matches, totalMatchCount, metadataOmitted, shortenedLines, invalidRecords,
+    previewOverflow, omittedMatches: Math.max(0, totalMatchCount - maxMatches) };
 }

package/package.json

@@ -1,9 +1,14 @@
 {
   "name": "pi-diet-ripgrep",
-  "version": "0.1.3",
+  "version": "0.1.5",
   "description": "Minimal Pi extension exposing ripgrep as a first-class exact-search tool.",
   "type": "module",
   "main": "index.ts",
+  "scripts": {
+    "lint": "tsc --noEmit",
+    "build": "tsc --project tsconfig.build.json",
+    "test": "tsx --test tests/*.test.ts"
+  },
   "files": [
     "index.ts",
     "README.md",
@@ -38,5 +43,12 @@
     "extensions": [
       "./index.ts"
     ]
+  },
+  "devDependencies": {
+    "@earendil-works/pi-ai": "^1.4.0",
+    "@earendil-works/pi-coding-agent": "^1.4.0",
+    "@types/node": "^26.6.5",
+    "tsx": "^4.23.15",
+    "typescript": "^7.0.2"
   }
 }

package/spec.md

@@ -8,11 +8,7 @@ The extension is intentionally "diet": it should add as little system-prompt ove
 
 ## Problem
 
-Pi already supports:
-- LSP for symbol-aware navigation
-- AST tools for structural search
-- Semble for semantic discovery
-- `bash` for ad hoc `rg`
+Pi supports `bash` for ad hoc `rg`, but additional code-intelligence extensions are optional and must not be assumed to be installed.
 
 What is missing is a compact, structured, exact-text search tool that:
 - is cheaper and safer than freeform shell use
@@ -26,11 +22,11 @@ What is missing is a compact, structured, exact-text search tool that:
 2. Keep added prompt footprint to roughly 100-200 tokens, ideally under 200.
 3. Return concise structured output suitable for agent use.
 4. Bound output size and support truncation safely.
-5. Avoid duplicating guidance already covered by Pi's built-in search/routing instructions.
+5. Keep guidance self-contained: describe this tool's purpose and limits without recommending other extensions.
 
 ## Non-goals
 
-- Replacing LSP, AST, or semantic tools.
+- Providing symbol-aware navigation, structural matching, or semantic discovery.
 - Becoming a full grep UI.
 - Supporting every ripgrep flag.
 - Adding large prompt instructions, examples, or verbose routing prose.
@@ -41,7 +37,7 @@ Use `rg` when the task needs:
 - exact literals
 - regex search
 - finding strings, log messages, config keys, TODOs, identifiers by text
-- quick scoped verification after semantic/AST discovery
+- quick scoped verification of candidate source locations
 
 Do not prefer `rg` for:
 - definitions/references/types
@@ -166,7 +162,7 @@ That usage is common enough to justify a small first-class tool, but not a verbo
 Build the richer-but-still-diet version:
 - `rg` tool only
 - compact schema with `hidden` and `fixedStrings`
-- routing-aware description that defers to LSP/AST/Semble
+- self-contained description covering exact-search use cases and the limits of text matches
 - `rg --json`
 - structured matches
 - small default result cap