tinker-agent 2.9.0 → 2.10.0

package/src/tools/grep-output.ts

@@ -0,0 +1,130 @@
+import path from "node:path";
+import { toDisplayPath } from "./path-safety";
+import type { GrepOutputMode } from "./types";
+import { excerptGrepLines } from "./grep-snippets";
+
+export type GrepContentRecord = {
+  kind: "match" | "context";
+  filePath: string;
+  lineNumber: number;
+  /** Keep each JSON event intact so pagination cannot split a multiline match. */
+  lines: string[];
+};
+
+export type GrepRecord =
+  | { kind: "file"; filePath: string }
+  | { kind: "count"; filePath: string; count: number }
+  | GrepContentRecord;
+
+/** Decode complete protocol records only. A truncated tail is never a record. */
+export function parseGrepOutput(
+  stdout: string,
+  mode: GrepOutputMode,
+  workspaceRoot: string,
+  truncated: boolean,
+  searchCwd: string = workspaceRoot,
+): GrepRecord[] {
+  if (mode === "content") {
+    return parseJsonOutput(stdout, workspaceRoot, truncated, searchCwd);
+  }
+  const records: GrepRecord[] = [];
+  let start = 0;
+  while (start < stdout.length) {
+    const nul = stdout.indexOf("\0", start);
+    if (nul === -1) {
+      if (truncated) break;
+      throw new Error("Missing NUL path delimiter.");
+    }
+    const reportedPath = stdout.slice(start, nul);
+    if (reportedPath === "") throw new Error("Empty path in ripgrep output.");
+    const filePath = toDisplayPath(
+      workspaceRoot,
+      path.resolve(searchCwd, reportedPath),
+    );
+    if (mode === "files_with_matches") {
+      records.push({ kind: "file", filePath });
+      start = nul + 1;
+      continue;
+    }
+    const end = stdout.indexOf("\n", nul + 1);
+    if (end === -1) {
+      if (truncated) break;
+      throw new Error("Unterminated count record.");
+    }
+    const value = stdout.slice(nul + 1, end);
+    if (!/^\d+$/.test(value) || !Number.isSafeInteger(Number(value))) {
+      throw new Error("Invalid count in ripgrep output.");
+    }
+    records.push({ kind: "count", filePath, count: Number(value) });
+    start = end + 1;
+  }
+  return records;
+}
+
+function parseJsonOutput(
+  stdout: string,
+  workspaceRoot: string,
+  truncated: boolean,
+  searchCwd: string,
+): GrepRecord[] {
+  const records: GrepRecord[] = [];
+  let start = 0;
+  while (start < stdout.length) {
+    const end = stdout.indexOf("\n", start);
+    if (end === -1) {
+      if (truncated) break;
+      throw new Error("Unterminated JSON event.");
+    }
+    const event: unknown = JSON.parse(stdout.slice(start, end));
+    start = end + 1;
+    if (!isRecord(event) || typeof event.type !== "string" || !isRecord(event.data)) {
+      throw new Error("Invalid ripgrep JSON event.");
+    }
+    if (["begin", "end", "summary"].includes(event.type)) continue;
+    if (event.type !== "match" && event.type !== "context") {
+      throw new Error("Unexpected ripgrep JSON event type.");
+    }
+    const data = event.data;
+    const reportedPath = decodeText(data.path, true);
+    if (reportedPath === "") throw new Error("Empty path in ripgrep JSON event.");
+    if (
+      typeof data.line_number !== "number" ||
+      !Number.isSafeInteger(data.line_number) ||
+      data.line_number < 1
+    ) {
+      throw new Error("Invalid line number in ripgrep JSON event.");
+    }
+    const filePath = toDisplayPath(
+      workspaceRoot,
+      path.resolve(searchCwd, reportedPath),
+    );
+    records.push({
+      kind: event.type,
+      filePath,
+      lineNumber: data.line_number,
+      lines: excerptGrepLines(decodeBytes(data.lines), data.submatches),
+    });
+  }
+  return records;
+}
+
+function decodeText(value: unknown, isPath: boolean): string {
+  if (isRecord(value) && typeof value.text === "string") return value.text;
+  return new TextDecoder("utf-8", { fatal: isPath }).decode(decodeBytes(value));
+}
+
+function decodeBytes(value: unknown): Buffer {
+  if (!isRecord(value)) throw new Error("Invalid text field in ripgrep JSON event.");
+  if (typeof value.text === "string") return Buffer.from(value.text, "utf8");
+  if (
+    typeof value.bytes === "string" &&
+    /^(?:[A-Za-z0-9+/]{4})*(?:[A-Za-z0-9+/]{2}==|[A-Za-z0-9+/]{3}=)?$/.test(value.bytes)
+  ) {
+    return Buffer.from(value.bytes, "base64");
+  }
+  throw new Error("Invalid text encoding in ripgrep JSON event.");
+}
+
+function isRecord(value: unknown): value is Record<string, unknown> {
+  return typeof value === "object" && value !== null && !Array.isArray(value);
+}

package/src/tools/grep-pagination.ts

@@ -0,0 +1,73 @@
+import type { GrepContentRecord, GrepRecord } from "./grep-output";
+
+const defaultHeadLimit = 250;
+
+export function applyHeadLimit<T>(items: T[], limit: number | undefined, offset = 0) {
+  const effectiveLimit = limit ?? defaultHeadLimit;
+  const selected =
+    effectiveLimit === 0
+      ? items.slice(offset)
+      : items.slice(offset, offset + effectiveLimit);
+  const hasMore = offset + selected.length < items.length;
+  return {
+    items: selected,
+    totalResults: items.length,
+    returnedResults: selected.length,
+    hasMore,
+    nextOffset: hasMore ? offset + selected.length : undefined,
+    appliedLimit: hasMore ? effectiveLimit : undefined,
+  };
+}
+
+/** Match events select windows; matches encountered inside a window never expand it. */
+export function applyContentHeadLimit(
+  records: GrepRecord[],
+  limit: number | undefined,
+  offset: number,
+  context: { before: number; after: number },
+) {
+  const matches = records.filter(
+    (record): record is GrepContentRecord => record.kind === "match",
+  );
+  const page = applyHeadLimit(matches, limit, offset);
+  const windows = new Map<string, { start: number; end: number }[]>();
+  for (const match of page.items) {
+    const ranges = windows.get(match.filePath) ?? [];
+    const start = Math.max(1, match.lineNumber - context.before);
+    const end = match.lineNumber + match.lines.length - 1 + context.after;
+    const last = ranges.at(-1);
+    if (last !== undefined && start <= last.end + 1) {
+      last.end = Math.max(last.end, end);
+    } else {
+      ranges.push({ start, end });
+    }
+    windows.set(match.filePath, ranges);
+  }
+
+  const items: GrepContentRecord[] = [];
+  // rg emits each physical line once, in file/line order. Walk merged windows
+  // linearly rather than rescanning all records for every selected match.
+  const cursors = new Map<string, number>();
+  for (const record of records) {
+    if (record.kind !== "match" && record.kind !== "context") continue;
+    const ranges = windows.get(record.filePath);
+    if (ranges === undefined) continue;
+    let cursor = cursors.get(record.filePath) ?? 0;
+    for (const [index, text] of record.lines.entries()) {
+      const lineNumber = record.lineNumber + index;
+      while (cursor < ranges.length && ranges[cursor].end < lineNumber) cursor++;
+      const range = ranges[cursor];
+      if (range === undefined) break;
+      if (lineNumber >= range.start) {
+        items.push({
+          kind: record.kind,
+          filePath: record.filePath,
+          lineNumber,
+          lines: [text],
+        });
+      }
+    }
+    cursors.set(record.filePath, cursor);
+  }
+  return { ...page, items };
+}

package/src/tools/grep-path.ts

@@ -0,0 +1,11 @@
+// Quote only paths that need escaping; ordinary paths remain easy to scan/copy.
+export function formatGrepPath(filePath: string): string {
+  return /[\p{Cc}\p{Cf}"\\\u2028\u2029]/u.test(filePath)
+    ? JSON.stringify(filePath).replace(/[\p{Cc}\p{Cf}\u2028\u2029]/gu, (character) =>
+        character
+          .split("")
+          .map((unit) => `\\u${unit.charCodeAt(0).toString(16).padStart(4, "0")}`)
+          .join(""),
+      )
+    : filePath;
+}

package/src/tools/grep-snippets.ts

@@ -0,0 +1,111 @@
+const maxLineCodePoints = 500;
+const matchContextCodePoints = 100;
+
+type MatchRange = { start: number; end: number };
+
+/** rg submatch offsets address the original bytes of the entire JSON event. */
+export function excerptGrepLines(bytes: Buffer, submatches: unknown): string[] {
+  const matches = parseSubmatches(submatches, bytes.length);
+  const lines: string[] = [];
+  let start = 0;
+  let matchIndex = 0;
+  while (start < bytes.length) {
+    const newline = bytes.indexOf(10, start);
+    const end = newline === -1 ? bytes.length : newline;
+    const textEnd = end > start && bytes[end - 1] === 13 ? end - 1 : end;
+    while (matchIndex < matches.length && matches[matchIndex].end < start) {
+      matchIndex++;
+    }
+    lines.push(excerptLine(bytes.subarray(start, textEnd), matches, matchIndex, start));
+    start = end + 1;
+  }
+  return lines;
+}
+
+function excerptLine(
+  bytes: Buffer,
+  matches: MatchRange[],
+  matchIndex: number,
+  lineStart: number,
+): string {
+  const points = [...bytes.toString("utf8")];
+  if (points.length <= maxLineCodePoints) return points.join("");
+
+  const windows: MatchRange[] = [];
+  const codePointOffset = createCodePointOffsetReader(bytes);
+  let remaining = maxLineCodePoints;
+  for (let index = matchIndex; index < matches.length && remaining > 0; index++) {
+    const match = matches[index];
+    if (match.start > lineStart + bytes.length) break;
+    if (match.end === lineStart && match.start < lineStart) continue;
+    const matchStart = codePointOffset(match.start - lineStart);
+    const matchEnd = codePointOffset(match.end - lineStart);
+    const previous = windows.at(-1);
+    // Reserve room for the match even when earlier windows used most of the budget.
+    const before = Math.min(matchContextCodePoints, Math.floor(remaining / 2));
+    const start = Math.max(0, matchStart - before);
+    const end = Math.min(points.length, matchEnd + matchContextCodePoints);
+    if (previous !== undefined && start <= previous.end) {
+      const extension = Math.min(remaining, Math.max(0, end - previous.end));
+      previous.end += extension;
+      remaining -= extension;
+    } else {
+      const keptEnd = Math.min(end, start + remaining);
+      windows.push({ start, end: keptEnd });
+      remaining -= keptEnd - start;
+    }
+  }
+  // Context events (and legacy fixtures without submatches) still retain useful text.
+  if (windows.length === 0) windows.push({ start: 0, end: maxLineCodePoints });
+
+  const parts: string[] = [];
+  let cursor = 0;
+  for (const window of windows) {
+    if (window.start > cursor) parts.push(omission(window.start - cursor));
+    parts.push(points.slice(window.start, window.end).join(""));
+    cursor = window.end;
+  }
+  if (cursor < points.length) parts.push(omission(points.length - cursor));
+  return parts.join("");
+}
+
+function createCodePointOffsetReader(bytes: Buffer): (offset: number) => number {
+  let byteCursor = 0;
+  let pointCursor = 0;
+  // Submatches are ordered and non-overlapping; scan each prefix only once.
+  return (offset) => {
+    const end = Math.max(0, Math.min(offset, bytes.length));
+    pointCursor += [...bytes.subarray(byteCursor, end).toString("utf8")].length;
+    byteCursor = end;
+    return pointCursor;
+  };
+}
+
+function omission(count: number): string {
+  return `[... ${count} code points omitted ...]`;
+}
+
+function parseSubmatches(value: unknown, byteLength: number): MatchRange[] {
+  if (value === undefined) return [];
+  if (!Array.isArray(value)) throw new Error("Invalid ripgrep submatches.");
+  let previousEnd = 0;
+  return value.map((item: unknown) => {
+    if (
+      typeof item !== "object" ||
+      item === null ||
+      !("start" in item) ||
+      !("end" in item) ||
+      typeof item.start !== "number" ||
+      typeof item.end !== "number" ||
+      !Number.isSafeInteger(item.start) ||
+      !Number.isSafeInteger(item.end) ||
+      item.start < previousEnd ||
+      item.end < item.start ||
+      item.end > byteLength
+    ) {
+      throw new Error("Invalid ripgrep submatch offsets.");
+    }
+    previousEnd = item.end;
+    return { start: item.start, end: item.end };
+  });
+}