tinker-agent 2.8.0 → 2.10.0

package/src/tools/grep-snippets.ts

@@ -0,0 +1,111 @@
+const maxLineCodePoints = 500;
+const matchContextCodePoints = 100;
+
+type MatchRange = { start: number; end: number };
+
+/** rg submatch offsets address the original bytes of the entire JSON event. */
+export function excerptGrepLines(bytes: Buffer, submatches: unknown): string[] {
+  const matches = parseSubmatches(submatches, bytes.length);
+  const lines: string[] = [];
+  let start = 0;
+  let matchIndex = 0;
+  while (start < bytes.length) {
+    const newline = bytes.indexOf(10, start);
+    const end = newline === -1 ? bytes.length : newline;
+    const textEnd = end > start && bytes[end - 1] === 13 ? end - 1 : end;
+    while (matchIndex < matches.length && matches[matchIndex].end < start) {
+      matchIndex++;
+    }
+    lines.push(excerptLine(bytes.subarray(start, textEnd), matches, matchIndex, start));
+    start = end + 1;
+  }
+  return lines;
+}
+
+function excerptLine(
+  bytes: Buffer,
+  matches: MatchRange[],
+  matchIndex: number,
+  lineStart: number,
+): string {
+  const points = [...bytes.toString("utf8")];
+  if (points.length <= maxLineCodePoints) return points.join("");
+
+  const windows: MatchRange[] = [];
+  const codePointOffset = createCodePointOffsetReader(bytes);
+  let remaining = maxLineCodePoints;
+  for (let index = matchIndex; index < matches.length && remaining > 0; index++) {
+    const match = matches[index];
+    if (match.start > lineStart + bytes.length) break;
+    if (match.end === lineStart && match.start < lineStart) continue;
+    const matchStart = codePointOffset(match.start - lineStart);
+    const matchEnd = codePointOffset(match.end - lineStart);
+    const previous = windows.at(-1);
+    // Reserve room for the match even when earlier windows used most of the budget.
+    const before = Math.min(matchContextCodePoints, Math.floor(remaining / 2));
+    const start = Math.max(0, matchStart - before);
+    const end = Math.min(points.length, matchEnd + matchContextCodePoints);
+    if (previous !== undefined && start <= previous.end) {
+      const extension = Math.min(remaining, Math.max(0, end - previous.end));
+      previous.end += extension;
+      remaining -= extension;
+    } else {
+      const keptEnd = Math.min(end, start + remaining);
+      windows.push({ start, end: keptEnd });
+      remaining -= keptEnd - start;
+    }
+  }
+  // Context events (and legacy fixtures without submatches) still retain useful text.
+  if (windows.length === 0) windows.push({ start: 0, end: maxLineCodePoints });
+
+  const parts: string[] = [];
+  let cursor = 0;
+  for (const window of windows) {
+    if (window.start > cursor) parts.push(omission(window.start - cursor));
+    parts.push(points.slice(window.start, window.end).join(""));
+    cursor = window.end;
+  }
+  if (cursor < points.length) parts.push(omission(points.length - cursor));
+  return parts.join("");
+}
+
+function createCodePointOffsetReader(bytes: Buffer): (offset: number) => number {
+  let byteCursor = 0;
+  let pointCursor = 0;
+  // Submatches are ordered and non-overlapping; scan each prefix only once.
+  return (offset) => {
+    const end = Math.max(0, Math.min(offset, bytes.length));
+    pointCursor += [...bytes.subarray(byteCursor, end).toString("utf8")].length;
+    byteCursor = end;
+    return pointCursor;
+  };
+}
+
+function omission(count: number): string {
+  return `[... ${count} code points omitted ...]`;
+}
+
+function parseSubmatches(value: unknown, byteLength: number): MatchRange[] {
+  if (value === undefined) return [];
+  if (!Array.isArray(value)) throw new Error("Invalid ripgrep submatches.");
+  let previousEnd = 0;
+  return value.map((item: unknown) => {
+    if (
+      typeof item !== "object" ||
+      item === null ||
+      !("start" in item) ||
+      !("end" in item) ||
+      typeof item.start !== "number" ||
+      typeof item.end !== "number" ||
+      !Number.isSafeInteger(item.start) ||
+      !Number.isSafeInteger(item.end) ||
+      item.start < previousEnd ||
+      item.end < item.start ||
+      item.end > byteLength
+    ) {
+      throw new Error("Invalid ripgrep submatch offsets.");
+    }
+    previousEnd = item.end;
+    return { start: item.start, end: item.end };
+  });
+}

package/src/tools/grep.ts

@@ -4,6 +4,9 @@ import { throwIfTurnCancelled } from "../agent/turn-cancellation";
 import { isWorkspaceLocalCwd, type CwdState } from "./cwd-state";
 import { resolveWorkspacePath, toDisplayPath } from "./path-safety";
 import { ripGrep } from "./ripgrep";
+import { parseGrepOutput } from "./grep-output";
+import { applyContentHeadLimit, applyHeadLimit } from "./grep-pagination";
+import { formatGrepPath } from "./grep-path";
 import { defineToolExecutor } from "./types";
 import type {
   GrepOutputMode,
@@ -39,7 +42,7 @@ export type GrepToolOptions = {
   };
 };
 
-const ignoredDirectories = [
+const defaultExcludedDirectories = [
   ".git",
   ".svn",
   ".hg",
@@ -50,35 +53,37 @@ const ignoredDirectories = [
   ".tinker",
 ];
 
-const defaultHeadLimit = 250;
-
 export function createGrepToolExecutor(options: GrepToolOptions): ToolExecutor {
   return defineToolExecutor("grep", {
     definition: {
       name: "Grep",
-      description: "Search file contents with ripgrep.",
+      description:
+        "Search file contents with ripgrep. Directory searches include hidden files, respect ignore rules (including .gitignore and .ignore), and exclude .git, .svn, .hg, .bzr, .jj, .sl, node_modules, and .tinker by default. Directory traversal does not follow symlinks; binary detection can stop searching a file. No matches means no matches within the effective search scope. Results use a consistent file-path order. Pagination is not a snapshot: file changes between calls may cause skipped or repeated results. Quoted paths use JSON escaping; pass the decoded path to file tools.",
       parameters: {
         type: "object",
         additionalProperties: false,
         properties: {
           pattern: {
             type: "string",
+            minLength: 1,
             description:
               "The regular expression pattern to search for in file contents.",
           },
           path: {
             type: "string",
             description:
-              "Optional workspace-relative or absolute file or directory to search in. Defaults to the current workspace-local cwd.",
+              "Optional workspace-relative or absolute file or directory to search in. Defaults to the current workspace-local cwd. For directories, ripgrep runs in that directory with . as its search path; explicitly selecting an excluded directory allows searching inside it. Files are passed as absolute paths: explicit files bypass ignore/glob/type filtering, and explicit symlink files are followed. Explicit binary files may yield matches but are not guaranteed to be searched completely.",
           },
           glob: {
             type: "string",
-            description: 'Glob pattern to filter files, such as "*.js" or "**/*.tsx".',
+            description:
+              "Passed unchanged to ripgrep's --glob option after default exclusions. A glob that matches files inside an excluded directory does not necessarily allow traversal into that directory: **/node_modules/** can return no matches because node_modules itself is still excluded. To search an excluded directory, set path to that directory (for example, node_modules/pkg), then use glob to filter files within it. Broad globs such as **/* can also match excluded directory entries and allow traversal. Does not enable symlink traversal or disable binary detection.",
           },
           output_mode: {
             type: "string",
-            enum: ["content", "files_with_matches", "count"],
-            description: 'Output mode. Defaults to "files_with_matches".',
+            enum: ["content", "files_with_matches", "count", "count-matches"],
+            description:
+              'Defaults to "files_with_matches" (matching file paths). "content" returns matching lines. Lines longer than 500 Unicode code points are excerpted: match windows include up to 100 code points on each side, with at most 500 source code points retained per line; later matches or long matches may be omitted. Long context lines retain their first 500 code points. Omission markers report skipped code points; use Read for full lines. "count" returns matching lines per file, counting a line once even with multiple matches; cannot be combined with multiline=true. "count-matches" returns non-overlapping matches per file, including multiple matches on one line. Count summaries cover only the displayed files when paginated.',
           },
           "-B": {
             type: "integer",
@@ -113,24 +118,24 @@ export function createGrepToolExecutor(options: GrepToolOptions): ToolExecutor {
           },
           type: {
             type: "string",
-            description: "File type to search, such as js, ts, py, rust, go, or java.",
+            description: "Passed unchanged to ripgrep's --type option.",
           },
           head_limit: {
             type: "integer",
             minimum: 0,
             description:
-              "Limit output to first N lines or entries. Defaults to 250. Pass 0 for unlimited.",
+              "Limit selected results. In content mode, counts matching lines (one per line even with multiple matches), or complete ripgrep match events with multiline=true. Each selected result includes its requested context when the search completes; context does not consume the limit. Nearby matches shown within that context are not consumed or expanded and may reappear on later pages. Overlapping context is deduplicated within a page. Other modes count file entries. Defaults to 250. Pass 0 for unlimited.",
           },
           offset: {
             type: "integer",
             minimum: 0,
             description:
-              "Skip first N lines or entries before applying head_limit. Defaults to 0.",
+              "Skip N results in the same units as head_limit, excluding context. Defaults to 0. To continue, pass nextOffset with the same search parameters. Pages may repeat context lines.",
           },
           multiline: {
             type: "boolean",
             description:
-              "Enable multiline mode where dot matches newlines. Defaults to false.",
+              'Enable multiline mode where dot matches newlines. Defaults to false. For counting, use output_mode="count-matches"; "count" rejects multiline=true.',
           },
         },
         required: ["pattern"],
@@ -178,10 +183,16 @@ export function createGrepToolExecutor(options: GrepToolOptions): ToolExecutor {
         });
       }
 
-      const rgArgs = buildRipgrepArgs(input, mode, absoluteSearchPath);
+      const searchCwd = pathCheck.isDirectory ? absoluteSearchPath : undefined;
+      const rgArgs = buildRipgrepArgs(
+        input,
+        mode,
+        pathCheck.isDirectory ? "." : absoluteSearchPath,
+      );
       const rg = await ripGrep(rgArgs, {
         signal: context.signal,
         ...options.ripgrep,
+        cwd: searchCwd,
       });
 
       if (!rg.ok) {
@@ -201,67 +212,111 @@ export function createGrepToolExecutor(options: GrepToolOptions): ToolExecutor {
         searchPath,
         absoluteSearchPath,
         mode,
-        ignored: ignoredDirectories,
       };
       const partialWarning = rg.truncated ? rg.error : undefined;
 
-      if (mode === "files_with_matches") {
-        const sorted = await sortFilesForOutput([...new Set(rg.lines)]);
-        const page = applyHeadLimit(sorted, input.head_limit, input.offset ?? 0);
-        const filenames = page.items.map((file) =>
-          toDisplayPath(options.workspaceRoot, file),
+      let records;
+      try {
+        records = parseGrepOutput(
+          rg.stdout,
+          mode,
+          options.workspaceRoot,
+          rg.truncated,
+          searchCwd,
         );
-
-        return {
-          ...base,
-          filenames,
-          numFiles: filenames.length,
-          appliedLimit: page.appliedLimit,
-          appliedOffset: appliedOffset(input.offset),
-          truncated: rg.truncated || page.appliedLimit !== undefined || undefined,
-          error: partialWarning,
-        };
+      } catch (error) {
+        return grepFailure({
+          pattern: input.pattern,
+          searchPath,
+          absoluteSearchPath,
+          mode,
+          error: `Invalid ripgrep output: ${errorMessage(error)}`,
+        });
+      }
+      if (rg.truncated && records.length === 0) {
+        return grepFailure({
+          pattern: input.pattern,
+          searchPath,
+          absoluteSearchPath,
+          mode,
+          truncated: true,
+          error:
+            partialWarning ??
+            "Incomplete ripgrep output contained no complete records.",
+        });
       }
 
-      if (mode === "count") {
-        const sorted = [...rg.lines].sort((left, right) => left.localeCompare(right));
-        const page = applyHeadLimit(sorted, input.head_limit, input.offset ?? 0);
-        const entries = page.items.map((line) =>
-          parseCountLine(line, options.workspaceRoot),
+      const requestedContext = resolveGrepContext(input);
+      const page =
+        mode === "content"
+          ? applyContentHeadLimit(
+              records,
+              input.head_limit,
+              input.offset ?? 0,
+              requestedContext,
+            )
+          : applyHeadLimit(records, input.head_limit, input.offset ?? 0);
+      const filenames = [...new Set(page.items.map((record) => record.filePath))];
+      const pagination = {
+        totalResults: page.totalResults,
+        returnedResults: page.returnedResults,
+        paginationUnit:
+          mode === "content"
+            ? input.multiline === true
+              ? ("match_events" as const)
+              : ("matching_lines" as const)
+            : ("files" as const),
+        hasMore: page.hasMore,
+        nextOffset: page.nextOffset,
+        appliedLimit: page.appliedLimit,
+        appliedOffset: appliedOffset(input.offset),
+        searchIncomplete: rg.truncated,
+        contextMayBeIncomplete:
+          mode === "content" &&
+          rg.truncated &&
+          (requestedContext.before > 0 || requestedContext.after > 0),
+        truncated: rg.truncated || page.appliedLimit !== undefined || undefined,
+        error: partialWarning,
+      };
+      if (mode === "files_with_matches") {
+        return { ...base, ...pagination, filenames, numFiles: filenames.length };
+      }
+      if (mode === "count" || mode === "count-matches") {
+        const counts = page.items.flatMap((record) =>
+          record.kind === "count"
+            ? [{ filePath: record.filePath, count: record.count }]
+            : [],
         );
-        const filenames = entries.map((entry) => entry.filePath);
-
         return {
           ...base,
+          ...pagination,
           filenames,
           numFiles: filenames.length,
-          content: entries
-            .map((entry) => `${entry.filePath}:${entry.count}`)
+          counts,
+          content: counts
+            .map((entry) => `${formatGrepPath(entry.filePath)}:${entry.count}`)
             .join("\n"),
-          numMatches: entries.reduce((total, entry) => total + entry.count, 0),
-          appliedLimit: page.appliedLimit,
-          appliedOffset: appliedOffset(input.offset),
-          truncated: rg.truncated || page.appliedLimit !== undefined || undefined,
-          error: partialWarning,
+          numMatches: counts.reduce((sum, entry) => sum + entry.count, 0),
         };
       }
-
-      const page = applyHeadLimit(rg.lines, input.head_limit, input.offset ?? 0);
-      const contentLines = page.items.map((line) =>
-        relativizeContentLine(line, options.workspaceRoot),
-      );
-      const filenames = extractContentFilenames(contentLines);
-
+      const contentLines = page.items.flatMap((record) => {
+        if (record.kind !== "match" && record.kind !== "context") return [];
+        const separator = record.kind === "match" ? ":" : "-";
+        return record.lines.map((text, index) => {
+          const position =
+            input.lineNumbers === false
+              ? ""
+              : `${record.lineNumber + index}${separator}`;
+          return `${formatGrepPath(record.filePath)}${separator}${position}${text}`;
+        });
+      });
       return {
         ...base,
+        ...pagination,
         filenames,
         numFiles: filenames.length,
         content: contentLines.join("\n"),
         numLines: contentLines.length,
-        appliedLimit: page.appliedLimit,
-        appliedOffset: appliedOffset(input.offset),
-        truncated: rg.truncated || page.appliedLimit !== undefined || undefined,
-        error: partialWarning,
       };
     },
   });
@@ -270,36 +325,24 @@ export function createGrepToolExecutor(options: GrepToolOptions): ToolExecutor {
 export function buildRipgrepArgs(
   input: GrepArgs,
   mode: GrepOutputMode,
-  absoluteSearchPath: string,
+  searchPathArgument: string,
 ): string[] {
-  const args = ["--hidden", "--max-columns", "500"];
+  // Sorting in rg fixes cross-file order before any pagination, including partial output.
+  const args = ["--no-config", "--hidden", "--sort", "path", "--color", "never"];
 
-  for (const directory of ignoredDirectories) {
+  for (const directory of defaultExcludedDirectories) {
     args.push("--glob", `!${directory}`);
   }
 
   if (mode === "files_with_matches") {
-    args.push("-l");
-  } else if (mode === "count") {
-    args.push("-c", "--with-filename");
+    args.push("-l", "--null");
+  } else if (mode === "count" || mode === "count-matches") {
+    args.push(mode === "count" ? "-c" : "--count-matches", "--with-filename", "--null");
   } else {
-    args.push("--with-filename");
-    if (input.lineNumbers !== false) {
-      args.push("-n");
-    }
+    args.push("--json");
 
-    if (input.context !== undefined) {
-      args.push("-C", String(input.context));
-    } else if (input.contextAlias !== undefined) {
-      args.push("-C", String(input.contextAlias));
-    } else {
-      if (input.before !== undefined) {
-        args.push("-B", String(input.before));
-      }
-      if (input.after !== undefined) {
-        args.push("-A", String(input.after));
-      }
-    }
+    const context = resolveGrepContext(input);
+    args.push("-B", String(context.before), "-A", String(context.after));
   }
 
   if (input.multiline === true) {
@@ -318,26 +361,15 @@ export function buildRipgrepArgs(
     args.push("--glob", input.glob);
   }
 
-  args.push("-e", input.pattern, absoluteSearchPath);
+  args.push("-e", input.pattern, searchPathArgument);
   return args;
 }
 
-export function applyHeadLimit<T>(
-  items: T[],
-  limit: number | undefined,
-  offset = 0,
-): { items: T[]; appliedLimit?: number } {
-  if (limit === 0) {
-    return { items: items.slice(offset) };
-  }
-
-  const effectiveLimit = limit ?? defaultHeadLimit;
-  const itemsPage = items.slice(offset, offset + effectiveLimit);
-  const wasTruncated = items.length - offset > effectiveLimit;
-
+function resolveGrepContext(input: GrepArgs) {
+  const both = input.context ?? input.contextAlias;
   return {
-    items: itemsPage,
-    appliedLimit: wasTruncated ? effectiveLimit : undefined,
+    before: both ?? input.before ?? 0,
+    after: both ?? input.after ?? 0,
   };
 }
 
@@ -368,18 +400,29 @@ function parseGrepArgs(args: unknown): ParsedGrepArgs {
     args.output_mode !== undefined &&
     args.output_mode !== "content" &&
     args.output_mode !== "files_with_matches" &&
-    args.output_mode !== "count"
+    args.output_mode !== "count" &&
+    args.output_mode !== "count-matches"
   ) {
     return {
       ok: false,
       pattern,
       error:
-        'Grep.output_mode must be one of "content", "files_with_matches", or "count".',
+        'Grep.output_mode must be one of "content", "files_with_matches", "count", or "count-matches".',
     };
   }
 
   const mode = args.output_mode;
 
+  if (mode === "count" && args.multiline === true) {
+    return {
+      ok: false,
+      pattern,
+      mode,
+      error:
+        'Grep output_mode="count" counts matching lines and cannot be combined with multiline=true. Use output_mode="count-matches" to count multiline matches.',
+    };
+  }
+
   for (const name of ["-B", "-A", "-C", "context", "head_limit", "offset"]) {
     const value = args[name];
     if (value !== undefined && !isNonNegativeInteger(value)) {
@@ -449,74 +492,17 @@ function resolveSearchPath(options: GrepToolOptions, inputPath?: string): string
 
 async function ensureFileOrDirectory(
   targetPath: string,
-): Promise<{ ok: true } | { ok: false; error: string }> {
+): Promise<{ ok: true; isDirectory: boolean } | { ok: false; error: string }> {
   try {
     const info = await stat(targetPath);
     return info.isFile() || info.isDirectory()
-      ? { ok: true }
+      ? { ok: true, isDirectory: info.isDirectory() }
       : { ok: false, error: "Grep.path must be a file or directory." };
   } catch {
     return { ok: false, error: "Grep.path does not exist." };
   }
 }
 
-async function sortFilesForOutput(files: string[]): Promise<string[]> {
-  if (process.env.NODE_ENV === "test") {
-    return [...files].sort((left, right) => left.localeCompare(right));
-  }
-
-  const withMtime = await Promise.all(
-    files.map(async (file) => {
-      try {
-        const info = await stat(file);
-        return { file, mtimeMs: info.mtimeMs };
-      } catch {
-        return { file, mtimeMs: 0 };
-      }
-    }),
-  );
-
-  return withMtime
-    .sort((left, right) =>
-      right.mtimeMs !== left.mtimeMs
-        ? right.mtimeMs - left.mtimeMs
-        : left.file.localeCompare(right.file),
-    )
-    .map((entry) => entry.file);
-}
-
-function parseCountLine(
-  line: string,
-  workspaceRoot: string,
-): { filePath: string; count: number } {
-  const separator = line.lastIndexOf(":");
-  const absolutePath = separator === -1 ? line : line.slice(0, separator);
-  const count = separator === -1 ? 0 : Number(line.slice(separator + 1));
-
-  return {
-    filePath: toDisplayPath(workspaceRoot, absolutePath),
-    count: Number.isFinite(count) ? count : 0,
-  };
-}
-
-function relativizeContentLine(line: string, workspaceRoot: string): string {
-  const prefix = path.resolve(workspaceRoot) + path.sep;
-  return line.startsWith(prefix) ? line.slice(prefix.length) : line;
-}
-
-function extractContentFilenames(lines: string[]): string[] {
-  const filenames = new Set<string>();
-
-  for (const line of lines) {
-    const match = /^(.+?):\d+[:-]/.exec(line);
-    if (match?.[1] !== undefined) {
-      filenames.add(match[1]);
-    }
-  }
-
-  return [...filenames];
-}
-
 function appliedOffset(offset: number | undefined): number | undefined {
   return offset !== undefined && offset > 0 ? offset : undefined;
 }
@@ -537,7 +523,6 @@ function grepFailure(input: {
     mode: input.mode,
     filenames: [],
     numFiles: 0,
-    ignored: ignoredDirectories,
     truncated: input.truncated,
     error: input.error,
   };

package/src/tools/read.ts

@@ -121,15 +121,6 @@ export function createReadToolExecutor(options: ReadToolOptions): ToolExecutor {
         const offset = input.offset ?? 1;
 
         if (lines.length === 0) {
-          if (input.offset !== undefined || input.limit !== undefined) {
-            return {
-              ok: false,
-              filePath: input.file_path,
-              absolutePath,
-              error: "File is empty. Omit offset and limit to read the empty file.",
-            };
-          }
-
           options.snapshots.set(absolutePath, {
             sha256,
             mtimeMs: currentInfo.mtimeMs,