tinker-agent 2.9.0 → 2.10.0

package/src/tools/grep.ts

@@ -4,6 +4,9 @@ import { throwIfTurnCancelled } from "../agent/turn-cancellation";
 import { isWorkspaceLocalCwd, type CwdState } from "./cwd-state";
 import { resolveWorkspacePath, toDisplayPath } from "./path-safety";
 import { ripGrep } from "./ripgrep";
+import { parseGrepOutput } from "./grep-output";
+import { applyContentHeadLimit, applyHeadLimit } from "./grep-pagination";
+import { formatGrepPath } from "./grep-path";
 import { defineToolExecutor } from "./types";
 import type {
   GrepOutputMode,
@@ -39,7 +42,7 @@ export type GrepToolOptions = {
   };
 };
 
-const ignoredDirectories = [
+const defaultExcludedDirectories = [
   ".git",
   ".svn",
   ".hg",
@@ -50,35 +53,37 @@ const ignoredDirectories = [
   ".tinker",
 ];
 
-const defaultHeadLimit = 250;
-
 export function createGrepToolExecutor(options: GrepToolOptions): ToolExecutor {
   return defineToolExecutor("grep", {
     definition: {
       name: "Grep",
-      description: "Search file contents with ripgrep.",
+      description:
+        "Search file contents with ripgrep. Directory searches include hidden files, respect ignore rules (including .gitignore and .ignore), and exclude .git, .svn, .hg, .bzr, .jj, .sl, node_modules, and .tinker by default. Directory traversal does not follow symlinks; binary detection can stop searching a file. No matches means no matches within the effective search scope. Results use a consistent file-path order. Pagination is not a snapshot: file changes between calls may cause skipped or repeated results. Quoted paths use JSON escaping; pass the decoded path to file tools.",
       parameters: {
         type: "object",
         additionalProperties: false,
         properties: {
           pattern: {
             type: "string",
+            minLength: 1,
             description:
               "The regular expression pattern to search for in file contents.",
           },
           path: {
             type: "string",
             description:
-              "Optional workspace-relative or absolute file or directory to search in. Defaults to the current workspace-local cwd.",
+              "Optional workspace-relative or absolute file or directory to search in. Defaults to the current workspace-local cwd. For directories, ripgrep runs in that directory with . as its search path; explicitly selecting an excluded directory allows searching inside it. Files are passed as absolute paths: explicit files bypass ignore/glob/type filtering, and explicit symlink files are followed. Explicit binary files may yield matches but are not guaranteed to be searched completely.",
           },
           glob: {
             type: "string",
-            description: 'Glob pattern to filter files, such as "*.js" or "**/*.tsx".',
+            description:
+              "Passed unchanged to ripgrep's --glob option after default exclusions. A glob that matches files inside an excluded directory does not necessarily allow traversal into that directory: **/node_modules/** can return no matches because node_modules itself is still excluded. To search an excluded directory, set path to that directory (for example, node_modules/pkg), then use glob to filter files within it. Broad globs such as **/* can also match excluded directory entries and allow traversal. Does not enable symlink traversal or disable binary detection.",
           },
           output_mode: {
             type: "string",
-            enum: ["content", "files_with_matches", "count"],
-            description: 'Output mode. Defaults to "files_with_matches".',
+            enum: ["content", "files_with_matches", "count", "count-matches"],
+            description:
+              'Defaults to "files_with_matches" (matching file paths). "content" returns matching lines. Lines longer than 500 Unicode code points are excerpted: match windows include up to 100 code points on each side, with at most 500 source code points retained per line; later matches or long matches may be omitted. Long context lines retain their first 500 code points. Omission markers report skipped code points; use Read for full lines. "count" returns matching lines per file, counting a line once even with multiple matches; cannot be combined with multiline=true. "count-matches" returns non-overlapping matches per file, including multiple matches on one line. Count summaries cover only the displayed files when paginated.',
           },
           "-B": {
             type: "integer",
@@ -113,24 +118,24 @@ export function createGrepToolExecutor(options: GrepToolOptions): ToolExecutor {
           },
           type: {
             type: "string",
-            description: "File type to search, such as js, ts, py, rust, go, or java.",
+            description: "Passed unchanged to ripgrep's --type option.",
           },
           head_limit: {
             type: "integer",
             minimum: 0,
             description:
-              "Limit output to first N lines or entries. Defaults to 250. Pass 0 for unlimited.",
+              "Limit selected results. In content mode, counts matching lines (one per line even with multiple matches), or complete ripgrep match events with multiline=true. Each selected result includes its requested context when the search completes; context does not consume the limit. Nearby matches shown within that context are not consumed or expanded and may reappear on later pages. Overlapping context is deduplicated within a page. Other modes count file entries. Defaults to 250. Pass 0 for unlimited.",
           },
           offset: {
             type: "integer",
             minimum: 0,
             description:
-              "Skip first N lines or entries before applying head_limit. Defaults to 0.",
+              "Skip N results in the same units as head_limit, excluding context. Defaults to 0. To continue, pass nextOffset with the same search parameters. Pages may repeat context lines.",
           },
           multiline: {
             type: "boolean",
             description:
-              "Enable multiline mode where dot matches newlines. Defaults to false.",
+              'Enable multiline mode where dot matches newlines. Defaults to false. For counting, use output_mode="count-matches"; "count" rejects multiline=true.',
           },
         },
         required: ["pattern"],
@@ -178,10 +183,16 @@ export function createGrepToolExecutor(options: GrepToolOptions): ToolExecutor {
         });
       }
 
-      const rgArgs = buildRipgrepArgs(input, mode, absoluteSearchPath);
+      const searchCwd = pathCheck.isDirectory ? absoluteSearchPath : undefined;
+      const rgArgs = buildRipgrepArgs(
+        input,
+        mode,
+        pathCheck.isDirectory ? "." : absoluteSearchPath,
+      );
       const rg = await ripGrep(rgArgs, {
         signal: context.signal,
         ...options.ripgrep,
+        cwd: searchCwd,
       });
 
       if (!rg.ok) {
@@ -201,67 +212,111 @@ export function createGrepToolExecutor(options: GrepToolOptions): ToolExecutor {
         searchPath,
         absoluteSearchPath,
         mode,
-        ignored: ignoredDirectories,
       };
       const partialWarning = rg.truncated ? rg.error : undefined;
 
-      if (mode === "files_with_matches") {
-        const sorted = await sortFilesForOutput([...new Set(rg.lines)]);
-        const page = applyHeadLimit(sorted, input.head_limit, input.offset ?? 0);
-        const filenames = page.items.map((file) =>
-          toDisplayPath(options.workspaceRoot, file),
+      let records;
+      try {
+        records = parseGrepOutput(
+          rg.stdout,
+          mode,
+          options.workspaceRoot,
+          rg.truncated,
+          searchCwd,
         );
-
-        return {
-          ...base,
-          filenames,
-          numFiles: filenames.length,
-          appliedLimit: page.appliedLimit,
-          appliedOffset: appliedOffset(input.offset),
-          truncated: rg.truncated || page.appliedLimit !== undefined || undefined,
-          error: partialWarning,
-        };
+      } catch (error) {
+        return grepFailure({
+          pattern: input.pattern,
+          searchPath,
+          absoluteSearchPath,
+          mode,
+          error: `Invalid ripgrep output: ${errorMessage(error)}`,
+        });
+      }
+      if (rg.truncated && records.length === 0) {
+        return grepFailure({
+          pattern: input.pattern,
+          searchPath,
+          absoluteSearchPath,
+          mode,
+          truncated: true,
+          error:
+            partialWarning ??
+            "Incomplete ripgrep output contained no complete records.",
+        });
       }
 
-      if (mode === "count") {
-        const sorted = [...rg.lines].sort((left, right) => left.localeCompare(right));
-        const page = applyHeadLimit(sorted, input.head_limit, input.offset ?? 0);
-        const entries = page.items.map((line) =>
-          parseCountLine(line, options.workspaceRoot),
+      const requestedContext = resolveGrepContext(input);
+      const page =
+        mode === "content"
+          ? applyContentHeadLimit(
+              records,
+              input.head_limit,
+              input.offset ?? 0,
+              requestedContext,
+            )
+          : applyHeadLimit(records, input.head_limit, input.offset ?? 0);
+      const filenames = [...new Set(page.items.map((record) => record.filePath))];
+      const pagination = {
+        totalResults: page.totalResults,
+        returnedResults: page.returnedResults,
+        paginationUnit:
+          mode === "content"
+            ? input.multiline === true
+              ? ("match_events" as const)
+              : ("matching_lines" as const)
+            : ("files" as const),
+        hasMore: page.hasMore,
+        nextOffset: page.nextOffset,
+        appliedLimit: page.appliedLimit,
+        appliedOffset: appliedOffset(input.offset),
+        searchIncomplete: rg.truncated,
+        contextMayBeIncomplete:
+          mode === "content" &&
+          rg.truncated &&
+          (requestedContext.before > 0 || requestedContext.after > 0),
+        truncated: rg.truncated || page.appliedLimit !== undefined || undefined,
+        error: partialWarning,
+      };
+      if (mode === "files_with_matches") {
+        return { ...base, ...pagination, filenames, numFiles: filenames.length };
+      }
+      if (mode === "count" || mode === "count-matches") {
+        const counts = page.items.flatMap((record) =>
+          record.kind === "count"
+            ? [{ filePath: record.filePath, count: record.count }]
+            : [],
         );
-        const filenames = entries.map((entry) => entry.filePath);
-
         return {
           ...base,
+          ...pagination,
           filenames,
           numFiles: filenames.length,
-          content: entries
-            .map((entry) => `${entry.filePath}:${entry.count}`)
+          counts,
+          content: counts
+            .map((entry) => `${formatGrepPath(entry.filePath)}:${entry.count}`)
             .join("\n"),
-          numMatches: entries.reduce((total, entry) => total + entry.count, 0),
-          appliedLimit: page.appliedLimit,
-          appliedOffset: appliedOffset(input.offset),
-          truncated: rg.truncated || page.appliedLimit !== undefined || undefined,
-          error: partialWarning,
+          numMatches: counts.reduce((sum, entry) => sum + entry.count, 0),
         };
       }
-
-      const page = applyHeadLimit(rg.lines, input.head_limit, input.offset ?? 0);
-      const contentLines = page.items.map((line) =>
-        relativizeContentLine(line, options.workspaceRoot),
-      );
-      const filenames = extractContentFilenames(contentLines);
-
+      const contentLines = page.items.flatMap((record) => {
+        if (record.kind !== "match" && record.kind !== "context") return [];
+        const separator = record.kind === "match" ? ":" : "-";
+        return record.lines.map((text, index) => {
+          const position =
+            input.lineNumbers === false
+              ? ""
+              : `${record.lineNumber + index}${separator}`;
+          return `${formatGrepPath(record.filePath)}${separator}${position}${text}`;
+        });
+      });
       return {
         ...base,
+        ...pagination,
         filenames,
         numFiles: filenames.length,
         content: contentLines.join("\n"),
         numLines: contentLines.length,
-        appliedLimit: page.appliedLimit,
-        appliedOffset: appliedOffset(input.offset),
-        truncated: rg.truncated || page.appliedLimit !== undefined || undefined,
-        error: partialWarning,
       };
     },
   });
@@ -270,36 +325,24 @@ export function createGrepToolExecutor(options: GrepToolOptions): ToolExecutor {
 export function buildRipgrepArgs(
   input: GrepArgs,
   mode: GrepOutputMode,
-  absoluteSearchPath: string,
+  searchPathArgument: string,
 ): string[] {
-  const args = ["--hidden", "--max-columns", "500"];
+  // Sorting in rg fixes cross-file order before any pagination, including partial output.
+  const args = ["--no-config", "--hidden", "--sort", "path", "--color", "never"];
 
-  for (const directory of ignoredDirectories) {
+  for (const directory of defaultExcludedDirectories) {
     args.push("--glob", `!${directory}`);
   }
 
   if (mode === "files_with_matches") {
-    args.push("-l");
-  } else if (mode === "count") {
-    args.push("-c", "--with-filename");
+    args.push("-l", "--null");
+  } else if (mode === "count" || mode === "count-matches") {
+    args.push(mode === "count" ? "-c" : "--count-matches", "--with-filename", "--null");
   } else {
-    args.push("--with-filename");
-    if (input.lineNumbers !== false) {
-      args.push("-n");
-    }
+    args.push("--json");
 
-    if (input.context !== undefined) {
-      args.push("-C", String(input.context));
-    } else if (input.contextAlias !== undefined) {
-      args.push("-C", String(input.contextAlias));
-    } else {
-      if (input.before !== undefined) {
-        args.push("-B", String(input.before));
-      }
-      if (input.after !== undefined) {
-        args.push("-A", String(input.after));
-      }
-    }
+    const context = resolveGrepContext(input);
+    args.push("-B", String(context.before), "-A", String(context.after));
   }
 
   if (input.multiline === true) {
@@ -318,26 +361,15 @@ export function buildRipgrepArgs(
     args.push("--glob", input.glob);
   }
 
-  args.push("-e", input.pattern, absoluteSearchPath);
+  args.push("-e", input.pattern, searchPathArgument);
   return args;
 }
 
-export function applyHeadLimit<T>(
-  items: T[],
-  limit: number | undefined,
-  offset = 0,
-): { items: T[]; appliedLimit?: number } {
-  if (limit === 0) {
-    return { items: items.slice(offset) };
-  }
-
-  const effectiveLimit = limit ?? defaultHeadLimit;
-  const itemsPage = items.slice(offset, offset + effectiveLimit);
-  const wasTruncated = items.length - offset > effectiveLimit;
-
+function resolveGrepContext(input: GrepArgs) {
+  const both = input.context ?? input.contextAlias;
   return {
-    items: itemsPage,
-    appliedLimit: wasTruncated ? effectiveLimit : undefined,
+    before: both ?? input.before ?? 0,
+    after: both ?? input.after ?? 0,
   };
 }
 
@@ -368,18 +400,29 @@ function parseGrepArgs(args: unknown): ParsedGrepArgs {
     args.output_mode !== undefined &&
     args.output_mode !== "content" &&
     args.output_mode !== "files_with_matches" &&
-    args.output_mode !== "count"
+    args.output_mode !== "count" &&
+    args.output_mode !== "count-matches"
   ) {
     return {
       ok: false,
       pattern,
       error:
-        'Grep.output_mode must be one of "content", "files_with_matches", or "count".',
+        'Grep.output_mode must be one of "content", "files_with_matches", "count", or "count-matches".',
     };
   }
 
   const mode = args.output_mode;
 
+  if (mode === "count" && args.multiline === true) {
+    return {
+      ok: false,
+      pattern,
+      mode,
+      error:
+        'Grep output_mode="count" counts matching lines and cannot be combined with multiline=true. Use output_mode="count-matches" to count multiline matches.',
+    };
+  }
+
   for (const name of ["-B", "-A", "-C", "context", "head_limit", "offset"]) {
     const value = args[name];
     if (value !== undefined && !isNonNegativeInteger(value)) {
@@ -449,74 +492,17 @@ function resolveSearchPath(options: GrepToolOptions, inputPath?: string): string
 
 async function ensureFileOrDirectory(
   targetPath: string,
-): Promise<{ ok: true } | { ok: false; error: string }> {
+): Promise<{ ok: true; isDirectory: boolean } | { ok: false; error: string }> {
   try {
     const info = await stat(targetPath);
     return info.isFile() || info.isDirectory()
-      ? { ok: true }
+      ? { ok: true, isDirectory: info.isDirectory() }
       : { ok: false, error: "Grep.path must be a file or directory." };
   } catch {
     return { ok: false, error: "Grep.path does not exist." };
   }
 }
 
-async function sortFilesForOutput(files: string[]): Promise<string[]> {
-  if (process.env.NODE_ENV === "test") {
-    return [...files].sort((left, right) => left.localeCompare(right));
-  }
-
-  const withMtime = await Promise.all(
-    files.map(async (file) => {
-      try {
-        const info = await stat(file);
-        return { file, mtimeMs: info.mtimeMs };
-      } catch {
-        return { file, mtimeMs: 0 };
-      }
-    }),
-  );
-
-  return withMtime
-    .sort((left, right) =>
-      right.mtimeMs !== left.mtimeMs
-        ? right.mtimeMs - left.mtimeMs
-        : left.file.localeCompare(right.file),
-    )
-    .map((entry) => entry.file);
-}
-
-function parseCountLine(
-  line: string,
-  workspaceRoot: string,
-): { filePath: string; count: number } {
-  const separator = line.lastIndexOf(":");
-  const absolutePath = separator === -1 ? line : line.slice(0, separator);
-  const count = separator === -1 ? 0 : Number(line.slice(separator + 1));
-
-  return {
-    filePath: toDisplayPath(workspaceRoot, absolutePath),
-    count: Number.isFinite(count) ? count : 0,
-  };
-}
-
-function relativizeContentLine(line: string, workspaceRoot: string): string {
-  const prefix = path.resolve(workspaceRoot) + path.sep;
-  return line.startsWith(prefix) ? line.slice(prefix.length) : line;
-}
-
-function extractContentFilenames(lines: string[]): string[] {
-  const filenames = new Set<string>();
-
-  for (const line of lines) {
-    const match = /^(.+?):\d+[:-]/.exec(line);
-    if (match?.[1] !== undefined) {
-      filenames.add(match[1]);
-    }
-  }
-
-  return [...filenames];
-}
-
 function appliedOffset(offset: number | undefined): number | undefined {
   return offset !== undefined && offset > 0 ? offset : undefined;
 }
@@ -537,7 +523,6 @@ function grepFailure(input: {
     mode: input.mode,
     filenames: [],
     numFiles: 0,
-    ignored: ignoredDirectories,
     truncated: input.truncated,
     error: input.error,
   };

package/src/tools/read.ts

@@ -121,15 +121,6 @@ export function createReadToolExecutor(options: ReadToolOptions): ToolExecutor {
         const offset = input.offset ?? 1;
 
         if (lines.length === 0) {
-          if (input.offset !== undefined || input.limit !== undefined) {
-            return {
-              ok: false,
-              filePath: input.file_path,
-              absolutePath,
-              error: "File is empty. Omit offset and limit to read the empty file.",
-            };
-          }
-
           options.snapshots.set(absolutePath, {
             sha256,
             mtimeMs: currentInfo.mtimeMs,

package/src/tools/ripgrep.ts

@@ -1,4 +1,5 @@
 import { execFile } from "node:child_process";
+import path from "node:path";
 import { cancellationError, throwIfTurnCancelled } from "../agent/turn-cancellation";
 import { DEFAULT_PUBLIC_TOOLING_CONFIG } from "../cli/public-config-contract";
 
@@ -7,7 +8,8 @@ export const RIPGREP_MISSING_ERROR =
 
 export type RipgrepResult = {
   ok: boolean;
-  lines: string[];
+  /** Raw stdout; the caller must decode its selected record protocol. */
+  stdout: string;
   exitCode?: number;
   truncated: boolean;
   error?: string;
@@ -16,6 +18,7 @@ export type RipgrepResult = {
 export type RipgrepOptions = {
   signal: AbortSignal;
   command?: string;
+  cwd?: string;
   timeoutMs?: number;
   maxBufferBytes?: number;
 };
@@ -32,7 +35,13 @@ export async function ripGrep(
   const timeoutMs = options.timeoutMs ?? DEFAULT_PUBLIC_TOOLING_CONFIG.grepTimeoutMs;
   const maxBufferBytes =
     options.maxBufferBytes ?? DEFAULT_PUBLIC_TOOLING_CONFIG.grepMaxBufferBytes;
-  const command = findRipgrepCommand(options.command);
+  const configuredCommand = findRipgrepCommand(options.command);
+  // Preserve relative executable paths when only the search subprocess changes cwd.
+  const command =
+    options.cwd !== undefined &&
+    (configuredCommand.includes("/") || configuredCommand.includes(path.sep))
+      ? path.resolve(configuredCommand)
+      : configuredCommand;
 
   const first = await runRipgrep(
     command,
@@ -40,6 +49,7 @@ export async function ripGrep(
     timeoutMs,
     maxBufferBytes,
     options.signal,
+    options.cwd,
   );
   if (first.retryWithSingleThread) {
     throwIfTurnCancelled(options.signal);
@@ -50,6 +60,7 @@ export async function ripGrep(
         timeoutMs,
         maxBufferBytes,
         options.signal,
+        options.cwd,
       ),
     );
   }
@@ -72,6 +83,7 @@ function runRipgrep(
   timeoutMs: number,
   maxBufferBytes: number,
   signal: AbortSignal,
+  cwd?: string,
 ): Promise<RipgrepAttempt> {
   return new Promise((resolve, reject) => {
     if (signal.aborted) {
@@ -82,7 +94,7 @@ function runRipgrep(
     execFile(
       command,
       args,
-      { timeout: timeoutMs, maxBuffer: maxBufferBytes, signal },
+      { timeout: timeoutMs, maxBuffer: maxBufferBytes, signal, cwd },
       (error, stdout, stderr) => {
         if (signal.aborted) {
           reject(cancellationError(signal, error));
@@ -158,21 +170,19 @@ function runRipgrep(
 }
 
 function finalizeResult(attempt: RipgrepAttempt): RipgrepResult {
-  const lines = splitCompleteLines(attempt.stdout, attempt.truncated);
-
   if (attempt.ok) {
     return {
       ok: true,
-      lines,
+      stdout: attempt.stdout,
       exitCode: attempt.exitCode,
       truncated: false,
     };
   }
 
-  if (attempt.truncated && lines.length > 0) {
+  if (attempt.truncated && attempt.stdout.length > 0) {
     return {
       ok: true,
-      lines,
+      stdout: attempt.stdout,
       exitCode: attempt.exitCode,
       truncated: true,
       error: attempt.error,
@@ -181,30 +191,13 @@ function finalizeResult(attempt: RipgrepAttempt): RipgrepResult {
 
   return {
     ok: false,
-    lines: [],
+    stdout: "",
     exitCode: attempt.exitCode,
     truncated: attempt.truncated,
     error: attempt.error ?? "ripgrep failed.",
   };
 }
 
-function splitCompleteLines(stdout: string, droppedPartialTail: boolean): string[] {
-  if (stdout === "") {
-    return [];
-  }
-
-  const lines = stdout.split("\n");
-  const last = lines.at(-1);
-
-  if (last === "") {
-    lines.pop();
-  } else if (droppedPartialTail) {
-    lines.pop();
-  }
-
-  return lines;
-}
-
 function isEagainError(
   error: Error & { code?: number | string },
   stderr: string,