pi-diet-ripgrep 0.1.0

package/README.md

@@ -0,0 +1,59 @@
+# pi-diet-ripgrep
+
+A minimal Pi extension that exposes ripgrep as a first-class exact-search tool.
+
+## Files
+
+- `index.ts` — extension entrypoint
+- `spec.md` — design spec
+- `scripts/estimate_prompt_footprint.py` — rough prompt-cost estimator
+
+## Installation
+
+After the package is published to npm:
+
+```bash
+pi install npm:pi-diet-ripgrep
+```
+
+For a pinned GitHub install:
+
+```bash
+pi install git:github.com/ProbabilityEngineer/pi-diet-ripgrep@v0.1.0
+```
+
+For local development:
+
+```bash
+pi -e /path/to/pi-diet-ripgrep/index.ts
+```
+
+Or symlink/copy into an auto-discovered extension location:
+
+```bash
+mkdir -p ~/.pi/agent/extensions/pi-diet-ripgrep
+ln -sf /path/to/pi-diet-ripgrep/index.ts ~/.pi/agent/extensions/pi-diet-ripgrep/index.ts
+```
+
+## Publishing
+
+First publish manually once:
+
+```bash
+npm publish --access public
+```
+
+Then configure npm Trusted Publishing for GitHub Actions on npmjs.com using this repository and `.github/workflows/npm-publish.yml`.
+
+After trusted publishing is configured, publish future versions by bumping `package.json` and pushing a matching version tag:
+
+```bash
+npm version patch --no-git-tag-version
+git commit -am "Release v0.1.1"
+git tag v0.1.1
+git push && git push origin v0.1.1
+```
+
+## Tool
+
+`rg` is intended for exact text and regex search only.

package/index.ts

@@ -0,0 +1,185 @@
+import { execFile as execFileCallback } from "node:child_process";
+import { mkdtemp, writeFile } from "node:fs/promises";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import { promisify } from "node:util";
+import { Type } from "@earendil-works/pi-ai";
+import {
+  DEFAULT_MAX_BYTES,
+  DEFAULT_MAX_LINES,
+  defineTool,
+  formatSize,
+  truncateHead,
+  type TruncationResult,
+  withFileMutationQueue,
+  type ExtensionAPI,
+} from "@earendil-works/pi-coding-agent";
+
+const execFile = promisify(execFileCallback);
+
+const rgTool = defineTool({
+  name: "rg",
+  label: "ripgrep",
+  description: "Exact text/regex search with ripgrep. Use for literal verification, logs, config keys, TODOs, and message lookup. Prefer LSP/AST/Semble for symbol, structural, or semantic discovery.",
+  parameters: Type.Object({
+    query: Type.String({ description: "Pattern" }),
+    paths: Type.Optional(Type.Array(Type.String({ description: "Path" }))),
+    glob: Type.Optional(Type.Array(Type.String({ description: "Glob" }))),
+    caseSensitive: Type.Optional(Type.Boolean({ description: "Case-sensitive" })),
+    word: Type.Optional(Type.Boolean({ description: "Word match" })),
+    context: Type.Optional(Type.Integer({ minimum: 0, description: "Context lines" })),
+    maxMatches: Type.Optional(Type.Integer({ minimum: 1, maximum: 200, description: "Match cap" })),
+    hidden: Type.Optional(Type.Boolean({ description: "Search hidden files" })),
+    fixedStrings: Type.Optional(Type.Boolean({ description: "Literal strings" })),
+  }),
+
+  async execute(_toolCallId, params, _signal, _onUpdate, ctx) {
+    const args = buildArgs(params);
+    const searchPaths = params.paths?.length ? params.paths : ["."];
+    const maxMatches = params.maxMatches ?? 50;
+
+    let stdout = "";
+    try {
+      const result = await execFile("rg", [...args, params.query, ...searchPaths], {
+        cwd: ctx.cwd,
+        encoding: "utf8",
+        maxBuffer: 100 * 1024 * 1024,
+      });
+      stdout = result.stdout;
+    } catch (error: any) {
+      stdout = error?.stdout ?? "";
+      if (error?.code !== 1) {
+        throw new Error(error?.stderr?.trim() || error?.message || "ripgrep failed");
+      }
+    }
+
+    const parsed = parseRgJson(stdout, maxMatches);
+    const details: RgDetails = {
+      query: params.query,
+      searchedPaths: searchPaths,
+      matchCount: parsed.totalMatchCount,
+      truncated: parsed.totalMatchCount > parsed.matches.length,
+      matches: parsed.matches,
+    };
+
+    if (!parsed.totalMatchCount) {
+      return {
+        content: [{ type: "text", text: "No matches found" }],
+        details,
+      };
+    }
+
+    const fullText = parsed.lines.join("\n");
+    const truncation = truncateHead(fullText, {
+      maxLines: DEFAULT_MAX_LINES,
+      maxBytes: DEFAULT_MAX_BYTES,
+    });
+
+    let text = truncation.content;
+    if (details.truncated || truncation.truncated) {
+      details.truncated = true;
+      details.truncation = truncation;
+      const tempDir = await mkdtemp(join(tmpdir(), "pi-diet-ripgrep-"));
+      const tempFile = join(tempDir, "output.txt");
+      await withFileMutationQueue(tempFile, async () => {
+        await writeFile(tempFile, fullText, "utf8");
+      });
+      details.fullOutputPath = tempFile;
+      const omittedMatches = Math.max(0, parsed.totalMatchCount - parsed.matches.length);
+      const notices: string[] = [];
+      if (omittedMatches) notices.push(`${omittedMatches} matches omitted by maxMatches cap`);
+      if (truncation.truncated) {
+        notices.push(
+          `output truncated to ${truncation.outputLines}/${truncation.totalLines} lines and ${formatSize(truncation.outputBytes)}/${formatSize(truncation.totalBytes)}`,
+        );
+      }
+      text += `\n\n[${notices.join("; ")}. Full output: ${tempFile}]`;
+    }
+
+    return {
+      content: [{ type: "text", text }],
+      details,
+    };
+  },
+});
+
+export default function (pi: ExtensionAPI) {
+  pi.registerTool(rgTool);
+}
+
+function buildArgs(params: {
+  glob?: string[];
+  caseSensitive?: boolean;
+  word?: boolean;
+  context?: number;
+  hidden?: boolean;
+  fixedStrings?: boolean;
+}) {
+  const args = ["--json", "--color=never", "--line-number", "--column"];
+  if (params.caseSensitive) {
+    args.push("--case-sensitive");
+  } else {
+    args.push("--smart-case");
+  }
+  if (params.word) args.push("-w");
+  if (params.hidden) args.push("--hidden");
+  if (params.fixedStrings) args.push("--fixed-strings");
+  if (typeof params.context === "number") args.push("-C", String(params.context));
+  for (const glob of params.glob ?? []) args.push("--glob", glob);
+  return args;
+}
+
+type RgMatch = {
+  path: string;
+  line: number;
+  column: number;
+  text: string;
+};
+
+type RgDetails = {
+  query: string;
+  searchedPaths: string[];
+  matchCount: number;
+  truncated: boolean;
+  matches: RgMatch[];
+  truncation?: TruncationResult;
+  fullOutputPath?: string;
+};
+
+function parseRgJson(stdout: string, maxMatches: number) {
+  const lines: string[] = [];
+  const matches: RgMatch[] = [];
+  let totalMatchCount = 0;
+
+  for (const rawLine of stdout.split(/\r?\n/)) {
+    if (!rawLine.trim()) continue;
+    let record: any;
+    try {
+      record = JSON.parse(rawLine);
+    } catch {
+      continue;
+    }
+
+    if (record.type === "match") {
+      totalMatchCount += 1;
+      const path = record.data?.path?.text ?? "<unknown>";
+      const line = record.data?.line_number ?? 0;
+      const text = String(record.data?.lines?.text ?? "").replace(/\n$/, "");
+      const column = (record.data?.submatches?.[0]?.start ?? 0) + 1;
+      if (matches.length < maxMatches) {
+        matches.push({ path, line, column, text });
+        lines.push(`${path}:${line}:${column}:${text}`);
+      }
+      continue;
+    }
+
+    if (record.type === "context" && matches.length > 0) {
+      const path = record.data?.path?.text ?? "<unknown>";
+      const line = record.data?.line_number ?? 0;
+      const text = String(record.data?.lines?.text ?? "").replace(/\n$/, "");
+      lines.push(`${path}-${line}- ${text}`);
+    }
+  }
+
+  return { lines, matches, totalMatchCount };
+}

package/package.json

@@ -0,0 +1,42 @@
+{
+  "name": "pi-diet-ripgrep",
+  "version": "0.1.0",
+  "description": "Minimal Pi extension exposing ripgrep as a first-class exact-search tool.",
+  "type": "module",
+  "main": "index.ts",
+  "files": [
+    "index.ts",
+    "README.md",
+    "spec.md"
+  ],
+  "keywords": [
+    "pi-package",
+    "pi",
+    "pi-extension",
+    "ripgrep",
+    "rg",
+    "search"
+  ],
+  "author": "ProbabilityEngineer",
+  "license": "MIT",
+  "repository": {
+    "type": "git",
+    "url": "git+ssh://git@github.com/ProbabilityEngineer/pi-diet-ripgrep.git"
+  },
+  "bugs": {
+    "url": "https://github.com/ProbabilityEngineer/pi-diet-ripgrep/issues"
+  },
+  "homepage": "https://github.com/ProbabilityEngineer/pi-diet-ripgrep#readme",
+  "publishConfig": {
+    "access": "public"
+  },
+  "peerDependencies": {
+    "@earendil-works/pi-ai": "*",
+    "@earendil-works/pi-coding-agent": "*"
+  },
+  "pi": {
+    "extensions": [
+      "./index.ts"
+    ]
+  }
+}

package/spec.md

@@ -0,0 +1,174 @@
+# pi-diet-ripgrep spec
+
+## Summary
+
+`pi-diet-ripgrep` is a minimal Pi extension that exposes ripgrep as a first-class tool for exact text and regex search with a small prompt footprint.
+
+The extension is intentionally "diet": it should add as little system-prompt overhead as possible while still being useful enough to prefer over raw `bash "rg ..."` for common literal-search workflows.
+
+## Problem
+
+Pi already supports:
+- LSP for symbol-aware navigation
+- AST tools for structural search
+- Semble for semantic discovery
+- `bash` for ad hoc `rg`
+
+What is missing is a compact, structured, exact-text search tool that:
+- is cheaper and safer than freeform shell use
+- has consistent defaults
+- returns structured results
+- preserves a clear routing boundary: use `rg` for exact text, not semantic or structural discovery
+
+## Goals
+
+1. Expose a first-class ripgrep tool for exact text/regex search.
+2. Keep added prompt footprint to roughly 100-200 tokens, ideally under 200.
+3. Return concise structured output suitable for agent use.
+4. Bound output size and support truncation safely.
+5. Avoid duplicating guidance already covered by Pi's built-in search/routing instructions.
+
+## Non-goals
+
+- Replacing LSP, AST, or semantic tools.
+- Becoming a full grep UI.
+- Supporting every ripgrep flag.
+- Adding large prompt instructions, examples, or verbose routing prose.
+
+## Intended use
+
+Use `rg` when the task needs:
+- exact literals
+- regex search
+- finding strings, log messages, config keys, TODOs, identifiers by text
+- quick scoped verification after semantic/AST discovery
+
+Do not prefer `rg` for:
+- definitions/references/types
+- structural code patterns
+- semantic behavior discovery
+
+## Tool shape
+
+### Name
+
+`rg`
+
+### Description
+
+Short form only, e.g.:
+
+> Exact text/regex search with ripgrep. Use for literal search, not semantic or structural discovery.
+
+### Parameters
+
+Keep the schema compact. Initial proposal:
+
+- `query: string` — pattern to search for
+- `paths?: string[]` — files/directories to search
+- `glob?: string[]` — include globs
+- `caseSensitive?: boolean`
+- `word?: boolean` — word-boundary match
+- `context?: number` — surrounding lines
+- `maxMatches?: number` — hard cap on returned matches
+
+Included in v1 because the prompt cost is acceptable:
+- `hidden?: boolean` — include hidden files when explicitly requested
+- `fixedStrings?: boolean` — treat query as a literal string instead of regex
+
+## Output
+
+Return:
+- human-readable concise text summary
+- structured details with matches
+
+Suggested details shape:
+
+```json
+{
+  "query": "TODO",
+  "searchedPaths": ["."],
+  "matchCount": 3,
+  "truncated": false,
+  "matches": [
+    {
+      "path": "src/a.ts",
+      "line": 12,
+      "column": 4,
+      "text": "// TODO: tighten validation"
+    }
+  ]
+}
+```
+
+## Execution behavior
+
+Use `rg --json` under the hood so results are structured and easier to parse safely.
+
+Default behavior:
+- no color
+- line numbers included
+- column included when available
+- cwd defaults to current Pi cwd
+- if `paths` omitted, search `.`
+
+## Truncation and limits
+
+The tool must hard-bound output.
+
+Initial rules:
+- truncate text output at Pi's normal custom-tool limits
+- cap returned structured matches with `maxMatches` defaulting to a small value, e.g. 50
+- when truncated, say so explicitly
+- if full output is preserved, write it to a temp file and report the path
+
+## Prompt budget
+
+Budget target:
+- ideal: <= 150 tokens
+- acceptable: <= 200 tokens
+- avoid: > 250 tokens
+
+Ways to stay within budget:
+- one-sentence description
+- short parameter descriptions
+- no embedded examples in tool description
+- no duplicated search-routing rules beyond one brief sentence
+
+## Why this is justified
+
+This tool should be used frequently enough for:
+- exact literal search
+- regex search
+- log/error/message lookup
+- config-key discovery
+- post-discovery verification
+
+That usage is common enough to justify a small first-class tool, but not a verbose extension.
+
+## Success criteria
+
+1. Tool prompt footprint stays within the budget.
+2. Agent can use `rg` instead of raw `bash` for common exact-search tasks.
+3. Results are safer and easier to parse than shell output.
+4. The extension does not encourage misuse for semantic or structural search.
+5. Implementation remains small and maintainable.
+
+## Open questions
+
+1. Should `fixedStrings` be included in v1 or omitted to save prompt/schema budget?
+2. Should `hidden` be included in v1 or left to future expansion?
+3. Should the tool expose only summary + first N matches, or always store full JSON in a temp file on truncation?
+4. Should the extension also add a tiny prompt append, or rely entirely on the tool description?
+
+## Recommended v1
+
+Build the richer-but-still-diet version:
+- `rg` tool only
+- compact schema with `hidden` and `fixedStrings`
+- routing-aware description that defers to LSP/AST/Semble
+- `rg --json`
+- structured matches
+- small default result cap
+- explicit truncation handling
+- no extra prompt append unless measurement shows it is necessary