pim-agent 0.0.1 → 0.10.0

package/packages/core/src/extensions/glob/schema.ts

@@ -0,0 +1,66 @@
+import { StringEnum } from "@earendil-works/pi-ai";
+import { type Static, Type } from "typebox";
+
+export const GLOB_HEAD_LIMIT_MAX = 1000;
+
+export const GLOB_PATH_FORMATS = ["relative", "absolute"] as const;
+
+export const globSchema = Type.Object({
+  pattern: Type.String({
+    description:
+      "Glob pattern relative to path (eg. **/*.ts). Brace expansion spans sibling dirs (eg. {src,docs}/**/*.ts). A bare directory (eg. src/tools) expands to everything under it.",
+  }),
+  path: Type.Optional(
+    Type.String({
+      description:
+        "Absolute or relative path to directory (resolved against cwd). Defaults to cwd.",
+    })
+  ),
+  exclude: Type.Optional(
+    Type.Array(Type.String(), {
+      description:
+        "Glob patterns to exclude, relative to path (eg. **/*.test.ts or dist/**).",
+    })
+  ),
+  includeDotfiles: Type.Optional(
+    Type.Boolean({
+      description:
+        "Include dot-prefixed files and directories such as .env or .github. Defaults to false.",
+    })
+  ),
+  includeIgnored: Type.Optional(
+    Type.Boolean({
+      description:
+        "Include gitignored and normally ignored paths such as node_modules. Defaults to false.",
+    })
+  ),
+  pathFormat: Type.Optional(
+    StringEnum(GLOB_PATH_FORMATS, {
+      description:
+        "`relative` (default): output paths relative to cwd when possible. `absolute`: output absolute paths.",
+    })
+  ),
+  headLimit: Type.Optional(
+    Type.Integer({
+      minimum: 1,
+      maximum: GLOB_HEAD_LIMIT_MAX,
+      description: `Maximum returned entries. Defaults to ${GLOB_HEAD_LIMIT_MAX}.`,
+    })
+  ),
+});
+
+export type GlobInput = Static<typeof globSchema>;
+export type GlobPathFormat = (typeof GLOB_PATH_FORMATS)[number];
+
+export type GlobDetails = {
+  readonly absolutePath: string;
+  readonly pattern: string;
+  readonly exclude: readonly string[] | undefined;
+  readonly includeDotfiles: boolean;
+  readonly includeIgnored: boolean;
+  readonly pathFormat: GlobPathFormat;
+  readonly fileCount: number;
+  readonly totalItems: number;
+  readonly visibleItems: number;
+  readonly truncated: boolean;
+};

package/packages/core/src/extensions/grep/grep.ts

@@ -0,0 +1,256 @@
+import { FileScanner, type FileScanOptions } from "../../shared/FileScanner";
+import { FsErrors } from "../../shared/FsErrors";
+import { Lines } from "../../shared/Lines";
+import { Paths } from "../../shared/Paths";
+import { Pool } from "../../shared/Pool";
+
+const MATCH_CONCURRENCY = 16;
+
+export type GrepLine = {
+  readonly lineNumber: number;
+  readonly text: string;
+};
+
+export type GrepLineRange = {
+  readonly startLineNumber: number;
+  readonly endLineNumber: number;
+};
+
+export type GrepMatch = {
+  readonly filePath: string;
+  readonly mtime: number;
+  readonly lines: readonly GrepLine[];
+  readonly ranges: readonly GrepLineRange[];
+  readonly fileLines: readonly string[];
+};
+
+export type GrepMatcher = {
+  readonly regex: RegExp;
+  readonly matchAcrossLines: boolean;
+  /** Raw-byte needle for the literal fast path; undefined leaves the regex path unchanged. */
+  readonly literal: Buffer | undefined;
+};
+
+export type GrepScanOptions = FileScanOptions & {
+  readonly retainFileLines?: boolean;
+};
+
+// ASCII-only: these stand for themselves in both a default-flag regex and raw UTF-8 bytes.
+const PURE_LITERAL = /^[A-Za-z0-9_ \-/]+$/;
+
+function literalNeedle(
+  pattern: string,
+  caseInsensitive: boolean,
+  matchAcrossLines: boolean
+): Buffer | undefined {
+  if (caseInsensitive || matchAcrossLines || !PURE_LITERAL.test(pattern)) {
+    return undefined;
+  }
+  return Buffer.from(pattern, "utf8");
+}
+
+export function buildMatcher(options: {
+  readonly pattern: string;
+  readonly caseInsensitive: boolean;
+  readonly matchAcrossLines: boolean;
+}): GrepMatcher {
+  const flags = `${options.matchAcrossLines ? "s" : ""}${
+    options.caseInsensitive ? "i" : ""
+  }`;
+
+  try {
+    return {
+      regex: new RegExp(options.pattern, flags),
+      matchAcrossLines: options.matchAcrossLines,
+      literal: literalNeedle(
+        options.pattern,
+        options.caseInsensitive,
+        options.matchAcrossLines
+      ),
+    };
+  } catch (error) {
+    const message = error instanceof Error ? error.message : String(error);
+    throw new Error(
+      `Invalid regular expression /${options.pattern}/${flags}: ${message}. Escape regex metacharacters for a literal search or simplify the pattern.`
+    );
+  }
+}
+
+export async function findMatches(
+  path: string,
+  glob: string | undefined,
+  matcher: GrepMatcher,
+  options: GrepScanOptions
+): Promise<readonly GrepMatch[]> {
+  const metadata = await FsErrors.statOrThrow(path);
+  const files = metadata.isFile()
+    ? [path]
+    : (await FileScanner.scan(path, glob ?? "**/*", options)).toSorted(
+        Paths.compare
+      );
+  const scanned = await Pool.mapPooled(files, MATCH_CONCURRENCY, (filePath) =>
+    matchFile(filePath, matcher, options.retainFileLines ?? true)
+  );
+
+  return scanned.filter((match) => match !== undefined);
+}
+
+// An entry may not be a readable regular file: an unreadable one is a non-match, never a failure.
+async function matchFile(
+  filePath: string,
+  matcher: GrepMatcher,
+  retainFileLines: boolean
+): Promise<GrepMatch | undefined> {
+  const file = Bun.file(filePath);
+
+  let text: string;
+  try {
+    if (await Lines.isBinary(file)) {
+      return undefined;
+    }
+
+    if (matcher.literal !== undefined) {
+      // Literal fast path: an ASCII needle's raw-byte hit/miss matches the decoded result.
+      const bytes = Buffer.from(await file.arrayBuffer());
+      if (bytes.indexOf(matcher.literal) < 0) {
+        return undefined;
+      }
+      text = bytes.toString("utf8");
+    } else {
+      text = await file.text();
+    }
+  } catch {
+    return undefined;
+  }
+
+  const content = Lines.normalize(text);
+  const fileLines = Lines.splitNormalized(content);
+  const ranges = matcher.matchAcrossLines
+    ? regexRanges(content, matcher.regex)
+    : matchLineByLine(fileLines, matcher.regex);
+
+  if (ranges.length === 0) {
+    return undefined;
+  }
+
+  return {
+    filePath,
+    mtime: file.lastModified,
+    lines: linesForRanges(fileLines, ranges),
+    ranges,
+    fileLines: retainFileLines ? fileLines : [],
+  };
+}
+
+function matchLineByLine(
+  lines: readonly string[],
+  regex: RegExp
+): readonly GrepLineRange[] {
+  const ranges: GrepLineRange[] = [];
+
+  for (const [index, line] of lines.entries()) {
+    if (regex.test(line)) {
+      const lineNumber = index + 1;
+      ranges.push({ startLineNumber: lineNumber, endLineNumber: lineNumber });
+    }
+  }
+
+  return ranges;
+}
+
+function regexRanges(content: string, regex: RegExp): readonly GrepLineRange[] {
+  const globalRegex = new RegExp(regex.source, addFlag(regex.flags, "g"));
+  const ranges: GrepLineRange[] = [];
+  const cursor: LineCursor = { offset: 0, line: 1 };
+
+  while (true) {
+    const match = globalRegex.exec(content);
+
+    if (match === null) {
+      break;
+    }
+
+    ranges.push(
+      lineRangeForOffsets(
+        content,
+        match.index,
+        match.index + match[0].length,
+        cursor
+      )
+    );
+
+    if (match[0].length === 0) {
+      globalRegex.lastIndex += 1;
+    }
+  }
+
+  return ranges;
+}
+
+function addFlag(flags: string, flag: string): string {
+  return flags.includes(flag) ? flags : `${flags}${flag}`;
+}
+
+type LineCursor = {
+  offset: number;
+  line: number;
+};
+
+function lineRangeForOffsets(
+  content: string,
+  startOffset: number,
+  endOffset: number,
+  cursor: LineCursor
+): GrepLineRange {
+  return {
+    startLineNumber: lineNumberForOffset(content, startOffset, cursor),
+    endLineNumber: lineNumberForOffset(
+      content,
+      Math.max(startOffset, endOffset - 1),
+      cursor
+    ),
+  };
+}
+
+function lineNumberForOffset(
+  content: string,
+  offset: number,
+  cursor: LineCursor
+): number {
+  const target = Math.min(offset, content.length);
+
+  for (let index = cursor.offset; index < target; index += 1) {
+    if (content[index] === "\n") {
+      cursor.line += 1;
+    }
+  }
+
+  cursor.offset = target;
+
+  return cursor.line;
+}
+
+function linesForRanges(
+  fileLines: readonly string[],
+  ranges: readonly GrepLineRange[]
+): readonly GrepLine[] {
+  const seen = new Set<number>();
+  const lines: GrepLine[] = [];
+
+  for (const range of ranges) {
+    for (
+      let lineNumber = range.startLineNumber;
+      lineNumber <= range.endLineNumber;
+      lineNumber += 1
+    ) {
+      if (seen.has(lineNumber)) {
+        continue;
+      }
+
+      seen.add(lineNumber);
+      lines.push({ lineNumber, text: fileLines[lineNumber - 1] ?? "" });
+    }
+  }
+
+  return lines;
+}

package/packages/core/src/extensions/grep/index.ts

@@ -0,0 +1,114 @@
+import type { ExtensionAPI } from "@earendil-works/pi-coding-agent";
+import { Paths } from "../../shared/Paths";
+import { Renderer } from "../../shared/Renderer";
+import { Tools } from "../../shared/Tools";
+import { buildMatcher, findMatches } from "./grep";
+import { buildView, type GrepViewDetails, renderMatches } from "./render";
+import {
+  GREP_HEAD_LIMIT_MAX,
+  type GrepInput,
+  type GrepOutputMode,
+  type GrepPathFormat,
+  grepSchema,
+} from "./schema";
+
+const DEFAULT_OUTPUT_MODE: GrepOutputMode = "files_with_matches";
+const DEFAULT_PATH_FORMAT: GrepPathFormat = "relative";
+
+export default function (pi: ExtensionAPI): void {
+  Tools.register(pi, {
+    name: "grep",
+    label: "grep",
+    description:
+      "Search UTF-8 text files with a JavaScript regex. " +
+      "Directory scans skip binary files, gitignored paths, and dotfiles unless requested; direct file paths are always searched. " +
+      "Use grep to search file contents instead of bash with grep, rg, ag, find -exec, or similar.",
+    parameters: grepSchema,
+    renderShell: "self",
+    effect: { kind: "readOnly" },
+    executionMode: "parallel",
+    async execute(_toolCallId, params, signal, _onUpdate, ctx) {
+      const {
+        pattern,
+        path,
+        glob,
+        exclude,
+        outputMode,
+        matchAcrossLines,
+        context,
+        includeDotfiles,
+        includeIgnored,
+        pathFormat,
+        caseInsensitive,
+        headLimit,
+      } = params as GrepInput;
+
+      if (signal?.aborted) {
+        throw new Error("Grep aborted before execution.");
+      }
+
+      const resolvedPathFormat = pathFormat ?? DEFAULT_PATH_FORMAT;
+      const resolvedContext = context ?? 0;
+      const resolvedOutputMode = outputMode ?? DEFAULT_OUTPUT_MODE;
+      const limit = Math.min(
+        headLimit ?? GREP_HEAD_LIMIT_MAX,
+        GREP_HEAD_LIMIT_MAX
+      );
+      const matcher = buildMatcher({
+        pattern,
+        caseInsensitive: caseInsensitive ?? false,
+        matchAcrossLines: matchAcrossLines ?? false,
+      });
+      const absolutePath = Paths.resolve(path ?? ".", ctx.cwd);
+      const matches = await findMatches(absolutePath, glob, matcher, {
+        exclude,
+        includeDotfiles: includeDotfiles ?? false,
+        includeIgnored: includeIgnored ?? false,
+        retainFileLines:
+          resolvedOutputMode === "content" && resolvedContext > 0,
+      });
+      const outcome = renderMatches(matches, resolvedOutputMode, limit, {
+        cwd: ctx.cwd,
+        pathFormat: resolvedPathFormat,
+        context: resolvedContext,
+      });
+      const content: Array<{ type: "text"; text: string }> = [
+        { type: "text", text: outcome.body },
+      ];
+
+      if (outcome.truncated) {
+        content.push({
+          type: "text",
+          text: `[grep tool: showing ${outcome.visibleItems} of ${outcome.totalItems} ${outcome.itemNoun}; narrow the pattern, scope to a specific path, or use a glob filter to reduce results.]`,
+        });
+      }
+
+      return {
+        content,
+        details: {
+          absolutePath,
+          outputMode: resolvedOutputMode,
+          exclude,
+          matchAcrossLines: matchAcrossLines ?? false,
+          context: resolvedContext,
+          includeDotfiles: includeDotfiles ?? false,
+          includeIgnored: includeIgnored ?? false,
+          pathFormat: resolvedPathFormat,
+          fileCount: outcome.fileCount,
+          totalMatches: outcome.totalMatches,
+          totalItems: outcome.totalItems,
+          visibleItems: outcome.visibleItems,
+          truncated: outcome.truncated,
+        },
+      };
+    },
+    toViewModel({ args, result, cwd }) {
+      return buildView({
+        args: (args ?? {}) as Partial<GrepInput>,
+        body: Renderer.firstText(result),
+        details: result?.details as GrepViewDetails | undefined,
+        cwd,
+      });
+    },
+  });
+}

package/packages/core/src/extensions/grep/render.ts

@@ -0,0 +1,271 @@
+import { Format } from "../../shared/Format";
+import { OutputBudget } from "../../shared/OutputBudget";
+import { Paths } from "../../shared/Paths";
+import {
+  NO_MATCHES,
+  type SearchList,
+  SearchRender,
+} from "../../shared/SearchRender";
+import type { ToolView, ViewBlock } from "../../view/ViewBlock";
+import type { GrepLineRange, GrepMatch } from "./grep";
+import type { GrepInput, GrepOutputMode, GrepPathFormat } from "./schema";
+
+export type RenderOutcome = SearchList & {
+  readonly fileCount: number;
+  readonly totalMatches: number;
+  readonly itemNoun: string;
+};
+
+export type RenderOptions = {
+  readonly cwd: string;
+  readonly pathFormat: GrepPathFormat;
+  readonly context: number;
+};
+
+const renderers: Record<
+  GrepOutputMode,
+  {
+    readonly itemNoun: string;
+    readonly toLines: (
+      matches: readonly GrepMatch[],
+      options: RenderOptions
+    ) => readonly string[];
+  }
+> = {
+  files_with_matches: { itemNoun: "files", toLines: renderFiles },
+  content: { itemNoun: "lines", toLines: renderContent },
+  count: { itemNoun: "files", toLines: renderCounts },
+};
+
+export function renderMatches(
+  matches: readonly GrepMatch[],
+  outputMode: GrepOutputMode,
+  headLimit: number,
+  options: RenderOptions
+): RenderOutcome {
+  const totalMatches = matches.reduce(
+    (sum, match) => sum + matchCount(match),
+    0
+  );
+  const { itemNoun, toLines } = renderers[outputMode];
+  const lines = toLines(matches, options);
+  const list = SearchRender.capList(lines, headLimit);
+  const empty = list.totalItems === 0;
+
+  return {
+    ...list,
+    fileCount: empty ? 0 : matches.length,
+    totalMatches: empty ? 0 : totalMatches,
+    itemNoun,
+  };
+}
+
+export type TitleOptions = {
+  readonly pattern: string | undefined;
+  readonly path: string | undefined;
+  readonly glob: string | undefined;
+  readonly cwd: string;
+};
+
+export function formatTitle(options: TitleOptions): string {
+  return SearchRender.subjectTitle({
+    subject: formatPattern(options.pattern),
+    path: options.path,
+    suffix: options.glob,
+    cwd: options.cwd,
+  });
+}
+
+export type GrepViewDetails = {
+  readonly outputMode?: GrepOutputMode;
+  readonly fileCount?: number;
+};
+
+export type GrepViewInput = {
+  /** Partially streamed while the call is in flight; every field is optional. */
+  readonly args: Partial<GrepInput>;
+  /** The result body the tool already rendered for the model; empty in flight. */
+  readonly body: string;
+  readonly details?: GrepViewDetails;
+  readonly cwd: string;
+};
+
+export function buildView({
+  args,
+  body,
+  details,
+  cwd,
+}: GrepViewInput): ToolView {
+  const fileCount = details?.fileCount;
+  return {
+    label: "Grep",
+    icon: "search",
+    title: [
+      {
+        kind: "text",
+        text: formatTitle({
+          pattern: args.pattern,
+          path: args.path,
+          glob: args.glob,
+          cwd,
+        }),
+      },
+      ...(fileCount === undefined
+        ? []
+        : ([
+            {
+              kind: "text",
+              tone: "muted",
+              text: Format.count(fileCount, "file"),
+            },
+          ] as const)),
+    ],
+    body: bodyBlocks(body, details?.outputMode),
+  };
+}
+
+function bodyBlocks(
+  body: string,
+  outputMode: GrepOutputMode | undefined
+): readonly ViewBlock[] {
+  if (
+    outputMode === "files_with_matches" &&
+    body !== "" &&
+    body !== NO_MATCHES
+  ) {
+    return body.split("\n").map((path) => ({ kind: "file", path }) as const);
+  }
+  return [{ kind: "text", text: body }];
+}
+
+function formatPattern(pattern: string | undefined): string {
+  return pattern === undefined ? "..." : `/${pattern}/`;
+}
+
+function renderFiles(
+  matches: readonly GrepMatch[],
+  options: RenderOptions
+): readonly string[] {
+  return byRecency(matches).map((match) =>
+    SearchRender.formatPath(match.filePath, options)
+  );
+}
+
+function renderContent(
+  matches: readonly GrepMatch[],
+  options: RenderOptions
+): readonly string[] {
+  if (options.context > 0) {
+    return byRecency(matches).flatMap((match) =>
+      renderContextContent(match, options)
+    );
+  }
+
+  return byRecency(matches).flatMap((match) =>
+    match.lines.map(
+      (line) =>
+        `${SearchRender.formatPath(match.filePath, options)}:${line.lineNumber}:${OutputBudget.truncateLine(line.text)}`
+    )
+  );
+}
+
+function renderContextContent(
+  match: GrepMatch,
+  options: RenderOptions
+): readonly string[] {
+  const path = SearchRender.formatPath(match.filePath, options);
+  const blocks = contextBlocks(
+    match.ranges,
+    match.fileLines.length,
+    options.context
+  );
+  const lines: string[] = [];
+
+  for (const [blockIndex, block] of blocks.entries()) {
+    if (blockIndex > 0) {
+      lines.push("--");
+    }
+
+    for (
+      let lineNumber = block.startLineNumber;
+      lineNumber <= block.endLineNumber;
+      lineNumber += 1
+    ) {
+      const marker = isMatchLine(match.ranges, lineNumber) ? ">" : " ";
+      const text = OutputBudget.truncateLine(
+        match.fileLines[lineNumber - 1] ?? ""
+      );
+      lines.push(`${marker} ${path}:${lineNumber}:${text}`);
+    }
+  }
+
+  return lines;
+}
+
+function renderCounts(
+  matches: readonly GrepMatch[],
+  options: RenderOptions
+): readonly string[] {
+  return [...matches]
+    .sort(
+      (left, right) =>
+        matchCount(right) - matchCount(left) ||
+        right.mtime - left.mtime ||
+        Paths.compare(left.filePath, right.filePath)
+    )
+    .map(
+      (match) =>
+        `${SearchRender.formatPath(match.filePath, options)}:${matchCount(match)}`
+    );
+}
+
+function contextBlocks(
+  ranges: readonly GrepLineRange[],
+  lineCount: number,
+  context: number
+): readonly GrepLineRange[] {
+  const blocks: GrepLineRange[] = [];
+
+  for (const range of ranges) {
+    const expanded = {
+      startLineNumber: Math.max(1, range.startLineNumber - context),
+      endLineNumber: Math.min(lineCount, range.endLineNumber + context),
+    };
+    const previous = blocks.at(-1);
+
+    if (
+      previous !== undefined &&
+      expanded.startLineNumber <= previous.endLineNumber + 1
+    ) {
+      blocks[blocks.length - 1] = {
+        startLineNumber: previous.startLineNumber,
+        endLineNumber: Math.max(previous.endLineNumber, expanded.endLineNumber),
+      };
+    } else {
+      blocks.push(expanded);
+    }
+  }
+
+  return blocks;
+}
+
+function matchCount(match: GrepMatch): number {
+  return match.ranges.length;
+}
+
+function isMatchLine(
+  ranges: readonly GrepLineRange[],
+  lineNumber: number
+): boolean {
+  return ranges.some(
+    (range) =>
+      lineNumber >= range.startLineNumber && lineNumber <= range.endLineNumber
+  );
+}
+
+function byRecency(matches: readonly GrepMatch[]): readonly GrepMatch[] {
+  return [...matches].sort(
+    (left, right) =>
+      right.mtime - left.mtime || Paths.compare(left.filePath, right.filePath)
+  );
+}