pi-ketch 0.1.0

package/src/ketch.ts

@@ -0,0 +1,291 @@
+import { spawn } from "node:child_process";
+
+export type KetchStatus =
+  | "ok"
+  | "not_found"
+  | "validation"
+  | "upstream"
+  | "precondition"
+  | "cancelled"
+  | "partial"
+  | "error";
+
+export interface RunKetchOptions {
+  cwd: string;
+  args: string[];
+  signal?: AbortSignal;
+  timeoutMs?: number;
+  env?: NodeJS.ProcessEnv;
+}
+
+export interface KetchProcessResult {
+  command: string;
+  bin: string;
+  args: string[];
+  cwd: string;
+  stdout: string;
+  stderr: string;
+  code: number;
+  status: KetchStatus;
+  durationMs: number;
+  termination?: "timeout" | "abort";
+}
+
+export interface CrawlRunOptions extends RunKetchOptions {
+  maxPages: number;
+  maxChars: number;
+}
+
+export interface CrawlPage {
+  url: string;
+  title?: string;
+  words?: number;
+  status?: string;
+  source?: string;
+  body: string;
+}
+
+export interface CrawlRunResult extends KetchProcessResult {
+  pages: CrawlPage[];
+  parseErrors: string[];
+  stopped?: "max_pages" | "timeout";
+}
+
+export class KetchError extends Error {
+  constructor(
+    message: string,
+    readonly result: KetchProcessResult,
+    readonly cause?: unknown,
+  ) {
+    super(message);
+    this.name = "KetchError";
+  }
+}
+
+export function resolveKetchBinary(env: NodeJS.ProcessEnv = process.env): string {
+  return env.KETCH_BIN?.trim() || "ketch";
+}
+
+export function buildKetchEnv(base: NodeJS.ProcessEnv = process.env): NodeJS.ProcessEnv {
+  return { ...base, NO_COLOR: "1", TERM: "dumb" };
+}
+
+function shellQuote(value: string): string {
+  return /^[A-Za-z0-9_./:@=,+-]+$/.test(value) ? value : JSON.stringify(value);
+}
+
+export function formatCommand(bin: string, args: string[]): string {
+  return [bin, ...args].map(shellQuote).join(" ");
+}
+
+export function statusForExitCode(code: number): KetchStatus {
+  switch (code) {
+    case 0:
+      return "ok";
+    case 2:
+      return "validation";
+    case 3:
+      return "not_found";
+    case 4:
+      return "upstream";
+    case 5:
+      return "precondition";
+    case 6:
+      return "cancelled";
+    default:
+      return "error";
+  }
+}
+
+function missingBinaryMessage(): string {
+  return [
+    "Ketch is unavailable because the `ketch` command was not found.",
+    "Install it with `brew install 1broseidon/tap/ketch`, put it on PATH, or set KETCH_BIN.",
+  ].join("\n");
+}
+
+function failureMessage(result: KetchProcessResult): string {
+  const hint = result.stderr.trim() || result.stdout.trim();
+  const lead: Record<KetchStatus, string> = {
+    ok: "Ketch completed",
+    not_found: "Ketch found no matching resource",
+    validation: "Ketch rejected the request",
+    upstream: "Ketch's upstream backend failed",
+    precondition: "Ketch requires operator setup",
+    cancelled: "Ketch was cancelled",
+    partial: "Ketch stopped with partial results",
+    error: "Ketch failed",
+  };
+  return `${lead[result.status]} (exit ${result.code}).${hint ? `\n${hint}` : ""}`;
+}
+
+export async function runKetch(options: RunKetchOptions): Promise<KetchProcessResult> {
+  const bin = resolveKetchBinary(options.env);
+  const result = await runChild({ ...options, bin });
+  if (result.code === 0 || result.code === 3) return result;
+  throw new KetchError(failureMessage(result), result);
+}
+
+interface ChildOptions extends RunKetchOptions {
+  bin: string;
+  onStdoutChunk?: (chunk: string, child: ReturnType<typeof spawn>) => void;
+  captureStdout?: boolean;
+}
+
+async function runChild(options: ChildOptions): Promise<KetchProcessResult> {
+  const command = formatCommand(options.bin, options.args);
+  const started = Date.now();
+  if (options.signal?.aborted) {
+    const result: KetchProcessResult = {
+      command,
+      bin: options.bin,
+      args: [...options.args],
+      cwd: options.cwd,
+      stdout: "",
+      stderr: "",
+      code: 6,
+      status: "cancelled",
+      durationMs: 0,
+      termination: "abort",
+    };
+    throw new KetchError("Ketch was cancelled before it started.", result);
+  }
+
+  return await new Promise<KetchProcessResult>((resolve, reject) => {
+    const child = spawn(options.bin, options.args, {
+      cwd: options.cwd,
+      env: buildKetchEnv(options.env),
+      stdio: ["ignore", "pipe", "pipe"],
+    });
+    let stdout = "";
+    let stderr = "";
+    let settled = false;
+    let timedOut = false;
+
+    const snapshot = (code: number): KetchProcessResult => ({
+      command,
+      bin: options.bin,
+      args: [...options.args],
+      cwd: options.cwd,
+      stdout,
+      stderr,
+      code,
+      status: statusForExitCode(code),
+      durationMs: Date.now() - started,
+    });
+
+    const cleanup = () => {
+      options.signal?.removeEventListener("abort", onAbort);
+      if (timer) clearTimeout(timer);
+    };
+    const finishReject = (error: unknown, code: number) => {
+      if (settled) return;
+      settled = true;
+      cleanup();
+      const cause = error as NodeJS.ErrnoException;
+      const result = snapshot(code);
+      const message = cause.code === "ENOENT" ? missingBinaryMessage() : cause.message || failureMessage(result);
+      reject(new KetchError(message, result, error));
+    };
+    const terminate = () => {
+      child.kill("SIGTERM");
+      const killTimer = setTimeout(() => child.kill("SIGKILL"), 500);
+      killTimer.unref();
+    };
+    const onAbort = () => {
+      if (settled) return;
+      terminate();
+    };
+    const timer = options.timeoutMs
+      ? setTimeout(() => {
+          if (settled) return;
+          timedOut = true;
+          terminate();
+        }, options.timeoutMs)
+      : undefined;
+
+    options.signal?.addEventListener("abort", onAbort, { once: true });
+    child.stdout.setEncoding("utf8");
+    child.stderr.setEncoding("utf8");
+    child.stdout.on("data", (chunk: string) => {
+      if (options.captureStdout !== false) stdout += chunk;
+      options.onStdoutChunk?.(chunk, child);
+    });
+    child.stderr.on("data", (chunk: string) => {
+      stderr += chunk;
+    });
+    child.on("error", (error: NodeJS.ErrnoException) => finishReject(error, error.code === "ENOENT" ? 127 : 1));
+    child.on("close", (code) => {
+      if (settled) return;
+      settled = true;
+      cleanup();
+      const aborted = options.signal?.aborted;
+      const exitCode = aborted ? 6 : timedOut ? 6 : (code ?? 1);
+      const result = snapshot(exitCode);
+      if (aborted) result.termination = "abort";
+      else if (timedOut) result.termination = "timeout";
+      if (aborted) {
+        reject(new KetchError("Ketch was cancelled.", result));
+      } else {
+        resolve(result);
+      }
+    });
+  });
+}
+
+function truncateCharacters(value: string, maxChars: number): string {
+  if (maxChars <= 0 || value.length <= maxChars) return value;
+  return `${value.slice(0, maxChars)}\n\n[truncated]`;
+}
+
+export async function runKetchCrawl(options: CrawlRunOptions): Promise<CrawlRunResult> {
+  const bin = resolveKetchBinary(options.env);
+  const pages: CrawlPage[] = [];
+  const parseErrors: string[] = [];
+  let buffered = "";
+  let stopped: CrawlRunResult["stopped"];
+
+  const parseLine = (line: string, child?: ReturnType<typeof spawn>) => {
+    const trimmed = line.trim();
+    if (!trimmed || pages.length >= options.maxPages) return;
+    try {
+      const parsed = JSON.parse(trimmed) as Partial<CrawlPage> & { body?: unknown; url?: unknown };
+      if (typeof parsed.url !== "string" || typeof parsed.body !== "string") {
+        parseErrors.push(`Invalid crawl record: ${trimmed.slice(0, 240)}`);
+        return;
+      }
+      pages.push({
+        url: parsed.url,
+        title: typeof parsed.title === "string" ? parsed.title : undefined,
+        words: typeof parsed.words === "number" ? parsed.words : undefined,
+        status: typeof parsed.status === "string" ? parsed.status : undefined,
+        source: typeof parsed.source === "string" ? parsed.source : undefined,
+        body: truncateCharacters(parsed.body, options.maxChars),
+      });
+      if (pages.length >= options.maxPages && child) {
+        stopped = "max_pages";
+        child.kill("SIGTERM");
+      }
+    } catch {
+      parseErrors.push(`Could not parse crawl JSON: ${trimmed.slice(0, 240)}`);
+    }
+  };
+
+  const result = await runChild({
+    ...options,
+    bin,
+    captureStdout: false,
+    onStdoutChunk(chunk, child) {
+      buffered += chunk;
+      const lines = buffered.split("\n");
+      buffered = lines.pop() ?? "";
+      for (const line of lines) parseLine(line, child);
+    },
+  });
+  if (buffered.trim()) parseLine(buffered);
+  if (!stopped && result.termination === "timeout") stopped = "timeout";
+  const status: KetchStatus = stopped ? "partial" : result.status;
+  const crawlResult: CrawlRunResult = { ...result, status, pages, parseErrors, stopped };
+  if (result.code !== 0 && !stopped) throw new KetchError(failureMessage(crawlResult), crawlResult);
+  return crawlResult;
+}

package/src/output.ts

@@ -0,0 +1,97 @@
+import { mkdtemp, writeFile } from "node:fs/promises";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import {
+  DEFAULT_MAX_BYTES,
+  DEFAULT_MAX_LINES,
+  formatSize,
+  truncateHead,
+  withFileMutationQueue,
+  type TruncationResult,
+} from "@earendil-works/pi-coding-agent";
+import type { KetchProcessResult, KetchStatus } from "./ketch.js";
+
+export interface KetchOutputDetails {
+  command: string;
+  args: string[];
+  cwd: string;
+  exitCode: number;
+  status: KetchStatus;
+  durationMs: number;
+  resultCount?: number;
+  errorCount?: number;
+  stopped?: string;
+  parsed?: unknown;
+  stderr?: string;
+  truncated: boolean;
+  truncation?: TruncationResult;
+  fullOutputPath?: string;
+}
+
+export interface KetchToolResult {
+  content: Array<{ type: "text"; text: string }>;
+  details: KetchOutputDetails;
+}
+
+export interface FormatKetchOutputOptions {
+  process: KetchProcessResult;
+  text: string;
+  parsed?: unknown;
+  fullText?: string;
+  resultCount?: number;
+  errorCount?: number;
+  stopped?: string;
+  maxBytes?: number;
+  maxLines?: number;
+}
+
+export async function formatKetchOutput(options: FormatKetchOutputOptions): Promise<KetchToolResult> {
+  const details: KetchOutputDetails = {
+    command: options.process.command,
+    args: [...options.process.args],
+    cwd: options.process.cwd,
+    exitCode: options.process.code,
+    status: options.process.status,
+    durationMs: options.process.durationMs,
+    resultCount: options.resultCount,
+    errorCount: options.errorCount,
+    stopped: options.stopped,
+    parsed: options.parsed,
+    stderr: options.process.stderr.trim() || undefined,
+    truncated: false,
+  };
+  const fullText = options.fullText ?? options.text;
+  const truncation = truncateHead(options.text, {
+    maxBytes: options.maxBytes ?? DEFAULT_MAX_BYTES,
+    maxLines: options.maxLines ?? DEFAULT_MAX_LINES,
+  });
+  let visible = truncation.content;
+
+  if (truncation.truncated) {
+    const dir = await mkdtemp(join(tmpdir(), "pi-ketch-"));
+    const fullOutputPath = join(dir, "output.txt");
+    await withFileMutationQueue(fullOutputPath, async () => writeFile(fullOutputPath, fullText, "utf8"));
+    details.truncated = true;
+    details.truncation = truncation;
+    details.fullOutputPath = fullOutputPath;
+    const omittedLines = truncation.totalLines - truncation.outputLines;
+    const omittedBytes = truncation.totalBytes - truncation.outputBytes;
+    visible += [
+      "",
+      `[Output truncated: showing ${truncation.outputLines} of ${truncation.totalLines} lines ` +
+        `(${formatSize(truncation.outputBytes)} of ${formatSize(truncation.totalBytes)}). ` +
+        `${omittedLines} lines (${formatSize(omittedBytes)}) omitted. Full output: ${fullOutputPath}]`,
+    ].join("\n");
+  }
+
+  if (!visible.trim()) visible = options.process.status === "not_found" ? "No results found." : "No output.";
+  return { content: [{ type: "text", text: visible }], details };
+}
+
+export function parseJson<T>(result: KetchProcessResult): T {
+  try {
+    return JSON.parse(result.stdout) as T;
+  } catch (error) {
+    throw new Error(`Ketch returned invalid JSON for ${result.command}: ${(error as Error).message}`);
+  }
+}

package/src/render.ts

@@ -0,0 +1,87 @@
+import { keyHint } from "@earendil-works/pi-coding-agent";
+import { Text } from "@earendil-works/pi-tui";
+import type { KetchToolResult } from "./output.js";
+
+interface Theme {
+  fg(slot: string, text: string): string;
+  bold(text: string): string;
+}
+interface Context {
+  lastComponent?: unknown;
+  expanded: boolean;
+  isError?: boolean;
+}
+interface ResultOptions {
+  expanded: boolean;
+  isPartial: boolean;
+}
+
+function component(previous: unknown): Text {
+  return previous instanceof Text ? previous : new Text("", 0, 0);
+}
+
+function compactValue(value: unknown): string | undefined {
+  if (value === undefined || value === null || value === false) return undefined;
+  const text = typeof value === "string" ? JSON.stringify(value) : JSON.stringify(value);
+  return text.length > 72 ? `${text.slice(0, 71)}…` : text;
+}
+
+function compactArgs(args: unknown): string {
+  if (!args || typeof args !== "object") return "";
+  return Object.entries(args)
+    .map(([key, value]) => {
+      const rendered = compactValue(value);
+      return rendered === undefined ? undefined : `${key}=${rendered}`;
+    })
+    .filter(Boolean)
+    .slice(0, 4)
+    .join(" ");
+}
+
+function hint(): string {
+  try {
+    return keyHint("app.tools.expand", "to expand");
+  } catch {
+    return "expand for details";
+  }
+}
+
+export function ketchRenderers(name: string) {
+  return {
+    renderCall(args: unknown, theme: Theme, context: Context): Text {
+      const text = component(context.lastComponent);
+      const title = theme.fg("toolTitle", theme.bold(name));
+      if (context.expanded) {
+        text.setText(`${title}\n${JSON.stringify(args, null, 2)}`);
+      } else {
+        const summary = compactArgs(args);
+        text.setText(summary ? `${title} ${theme.fg("muted", summary)}` : title);
+      }
+      return text;
+    },
+    renderResult(result: unknown, options: ResultOptions, theme: Theme, context: Context): Text {
+      const text = component(context.lastComponent);
+      if (options.isPartial) {
+        text.setText(theme.fg("muted", "Running Ketch…"));
+        return text;
+      }
+      const value = result as KetchToolResult;
+      const output = value.content.map((item) => item.text).join("\n");
+      if (options.expanded) {
+        text.setText(output || theme.fg("muted", "No output"));
+        return text;
+      }
+      const status = context.isError ? "error" : value.details.status;
+      const icon = status === "error" ? "✗" : status === "not_found" ? "!" : "✓";
+      const color = status === "error" ? "error" : status === "not_found" || status === "partial" ? "warning" : "success";
+      const count = value.details.resultCount;
+      const failures = value.details.errorCount;
+      const parts = [status.replace(/_/g, " ")];
+      if (count !== undefined) parts.push(`${count} result${count === 1 ? "" : "s"}`);
+      if (failures) parts.push(`${failures} failed`);
+      if (value.details.truncated) parts.push("output truncated");
+      text.setText(`${theme.fg(color, `${icon} ${parts.join(" · ")}`)} · ${hint()}`);
+      return text;
+    },
+  };
+}

package/src/schemas.ts

@@ -0,0 +1,64 @@
+import { StringEnum } from "@earendil-works/pi-ai";
+import { Type, type Static } from "typebox";
+
+const SearchBackend = StringEnum(["brave", "ddg", "searxng", "exa", "firecrawl", "keenable"] as const);
+const CodeBackend = StringEnum(["grepapp", "sourcegraph", "github"] as const);
+const DocsBackend = StringEnum(["context7"] as const);
+
+export const SearchParams = Type.Object({
+  query: Type.String({ description: "Web search query." }),
+  backend: Type.Optional(SearchBackend),
+  multi: Type.Optional(Type.Array(SearchBackend, { description: "Federated-search backends. Use all six entries to query every configured backend.", maxItems: 6 })),
+  limit: Type.Optional(Type.Integer({ description: "Maximum results. Defaults to Ketch config; capped at 20.", minimum: 1, maximum: 20 })),
+  scrape: Type.Optional(Type.Boolean({ description: "Fetch result pages and include extracted content." })),
+  trim: Type.Optional(Type.Boolean({ description: "Strip Markdown formatting from scraped content." })),
+  maxChars: Type.Optional(Type.Integer({ description: "Maximum scraped characters per result. Defaults to 6000 when scrape is enabled.", minimum: 1, maximum: 20000 })),
+  searxngUrl: Type.Optional(Type.String({ description: "Override the configured SearXNG endpoint." })),
+});
+export type SearchArgs = Static<typeof SearchParams>;
+
+export const CodeParams = Type.Object({
+  query: Type.String({ description: "Literal or regular-expression code query for public OSS repositories." }),
+  backend: Type.Optional(CodeBackend),
+  lang: Type.Optional(Type.String({ description: "Language filter, such as go, typescript, or python." })),
+  limit: Type.Optional(Type.Integer({ description: "Maximum results. Capped at 20.", minimum: 1, maximum: 20 })),
+  regex: Type.Optional(Type.Boolean({ description: "Interpret query as a regular expression. Supported by grepapp and sourcegraph, not GitHub." })),
+});
+export type CodeArgs = Static<typeof CodeParams>;
+
+export const DocsParams = Type.Object({
+  query: Type.String({ description: "Library name to resolve or documentation query." }),
+  backend: Type.Optional(DocsBackend),
+  library: Type.Optional(Type.String({ description: "Context7 library ID. Skips name resolution." })),
+  resolve: Type.Optional(Type.Boolean({ description: "Resolve a library name instead of searching documentation." })),
+  limit: Type.Optional(Type.Integer({ description: "Maximum matches for search or resolve. Capped at 20.", minimum: 1, maximum: 20 })),
+  tokens: Type.Optional(Type.Integer({ description: "Context7 token budget. Defaults to 4000; capped at 12000.", minimum: 100, maximum: 12000 })),
+});
+export type DocsArgs = Static<typeof DocsParams>;
+
+export const ScrapeParams = Type.Object({
+  url: Type.Optional(Type.String({ description: "One HTTP(S) URL to scrape. Provide exactly one of url or urls." })),
+  urls: Type.Optional(Type.Array(Type.String(), { description: "HTTP(S) URLs to scrape in one batch. Provide exactly one of url or urls.", minItems: 1, maxItems: 20 })),
+  selector: Type.Optional(Type.String({ description: "CSS selector to extract. Incompatible with raw." })),
+  trim: Type.Optional(Type.Boolean({ description: "Strip Markdown formatting. Incompatible with raw." })),
+  maxChars: Type.Optional(Type.Integer({ description: "Maximum characters per page. Defaults to 6000.", minimum: 1, maximum: 20000 })),
+  noCache: Type.Optional(Type.Boolean({ description: "Bypass Ketch's page cache." })),
+  raw: Type.Optional(Type.Boolean({ description: "Return raw HTML instead of extracted Markdown." })),
+  forceBrowser: Type.Optional(Type.Boolean({ description: "Force configured headless-browser rendering." })),
+  noLlmsTxt: Type.Optional(Type.Boolean({ description: "Disable automatic /llms.txt detection for bare domains." })),
+  concurrency: Type.Optional(Type.Integer({ description: "Batch concurrency. Defaults to 5; capped at 16.", minimum: 1, maximum: 16 })),
+});
+export type ScrapeArgs = Static<typeof ScrapeParams>;
+
+export const CrawlParams = Type.Object({
+  url: Type.String({ description: "HTTP(S) seed URL. Ketch crawls the same host breadth-first." }),
+  depth: Type.Optional(Type.Integer({ description: "Maximum crawl depth. Defaults to 3; capped at 5.", minimum: 1, maximum: 5 })),
+  sitemap: Type.Optional(Type.Boolean({ description: "Treat the seed as a sitemap." })),
+  maxPages: Type.Optional(Type.Integer({ description: "Stop after this many successful pages. Defaults to 30; capped at 100.", minimum: 1, maximum: 100 })),
+  allow: Type.Optional(Type.Array(Type.String(), { description: "Path substrings; a URL must match at least one." })),
+  deny: Type.Optional(Type.Array(Type.String(), { description: "Regular expressions for URLs to skip." })),
+  maxChars: Type.Optional(Type.Integer({ description: "Maximum Markdown characters per page. Defaults to 6000.", minimum: 1, maximum: 20000 })),
+  noCache: Type.Optional(Type.Boolean({ description: "Bypass Ketch's page cache." })),
+  concurrency: Type.Optional(Type.Integer({ description: "Worker count. Defaults to 8; capped at 16.", minimum: 1, maximum: 16 })),
+});
+export type CrawlArgs = Static<typeof CrawlParams>;

package/src/tools/code.ts

@@ -0,0 +1,46 @@
+import { defineTool, type ExtensionAPI } from "@earendil-works/pi-coding-agent";
+import { formatCodeResults, type CodeResult } from "../format.js";
+import { runKetch } from "../ketch.js";
+import { formatKetchOutput, parseJson } from "../output.js";
+import { ketchRenderers } from "../render.js";
+import { CodeParams, type CodeArgs } from "../schemas.js";
+import { addFlag } from "./common.js";
+
+export function buildCodeArgs(params: CodeArgs): string[] {
+  if (params.regex && params.backend === "github") throw new Error("ketch_code: regex is unsupported by the GitHub backend; use grepapp or sourcegraph.");
+  const args = ["code"];
+  addFlag(args, "--backend", params.backend);
+  addFlag(args, "--lang", params.lang);
+  addFlag(args, "--limit", params.limit);
+  addFlag(args, "--regex", params.regex);
+  args.push("--json", "--", params.query);
+  return args;
+}
+
+export function registerCodeTool(pi: ExtensionAPI): void {
+  pi.registerTool(
+    defineTool({
+      name: "ketch_code",
+      label: "Ketch Public Code Search",
+      description: "Search public open-source repositories through Ketch's grep.app, Sourcegraph, or GitHub backends. This is not local repository search.",
+      parameters: CodeParams,
+      promptSnippet: "ketch_code: Find real-world API usage in public OSS repositories; use local code tools for the current project.",
+      promptGuidelines: [
+        "Use ketch_code for public OSS examples and real-world API usage, not for searching the current local repository.",
+        "Cite the repository URLs returned by ketch_code when drawing conclusions from examples.",
+      ],
+      ...ketchRenderers("ketch_code"),
+      async execute(_id, params: CodeArgs, signal, _update, ctx) {
+        const process = await runKetch({ cwd: ctx.cwd, args: buildCodeArgs(params), signal, timeoutMs: 60_000 });
+        const results = process.stdout.trim() ? parseJson<CodeResult[]>(process) : [];
+        if (!results.length) process.status = "not_found";
+        return await formatKetchOutput({
+          process,
+          text: formatCodeResults(results),
+          parsed: results.map(({ repo, path, line, language, stars, url, source }) => ({ repo, path, line, language, stars, url, source })),
+          resultCount: results.length,
+        });
+      },
+    }),
+  );
+}

package/src/tools/common.ts

@@ -0,0 +1,26 @@
+export function addFlag(args: string[], flag: string, value: string | number | boolean | undefined): void {
+  if (value === undefined || value === false) return;
+  args.push(flag);
+  if (value !== true) args.push(String(value));
+}
+
+export function addListFlag(args: string[], flag: string, values: readonly string[] | undefined): void {
+  if (!values?.length) return;
+  args.push(flag, values.join(","));
+}
+
+export function validateHttpUrl(value: string, label = "url"): void {
+  let url: URL;
+  try {
+    url = new URL(value);
+  } catch {
+    throw new Error(`${label} must be a valid absolute URL.`);
+  }
+  if (url.protocol !== "http:" && url.protocol !== "https:") {
+    throw new Error(`${label} must use http or https.`);
+  }
+}
+
+export function warningCount(stderr: string): number {
+  return stderr.split(/\r?\n/).filter((line) => /^warn:/i.test(line.trim())).length;
+}

package/src/tools/crawl.ts

@@ -0,0 +1,65 @@
+import { defineTool, type ExtensionAPI } from "@earendil-works/pi-coding-agent";
+import { formatScrapePages } from "../format.js";
+import { runKetchCrawl } from "../ketch.js";
+import { formatKetchOutput } from "../output.js";
+import { ketchRenderers } from "../render.js";
+import { CrawlParams, type CrawlArgs } from "../schemas.js";
+import { addFlag, addListFlag, validateHttpUrl, warningCount } from "./common.js";
+
+export function buildCrawlArgs(params: CrawlArgs): string[] {
+  validateHttpUrl(params.url);
+  const args = ["crawl", params.url];
+  addFlag(args, "--depth", params.depth ?? 3);
+  addFlag(args, "--concurrency", params.concurrency ?? 8);
+  addListFlag(args, "--allow", params.allow);
+  addListFlag(args, "--deny", params.deny);
+  addFlag(args, "--sitemap", params.sitemap);
+  addFlag(args, "--no-cache", params.noCache);
+  args.push("--json");
+  return args;
+}
+
+export function registerCrawlTool(pi: ExtensionAPI): void {
+  pi.registerTool(
+    defineTool({
+      name: "ketch_crawl",
+      label: "Ketch Site Crawl",
+      description: "Breadth-first crawl one host through Ketch with Pi-enforced page, depth, per-page character, concurrency, and time bounds. Returns partial pages when a bound is reached.",
+      parameters: CrawlParams,
+      promptSnippet: "ketch_crawl: Read a bounded set of pages from one site; prefer scrape when one known page is enough.",
+      promptGuidelines: [
+        "Use ketch_crawl only when multiple pages from one host are required; prefer ketch_scrape for one known page.",
+        "Keep ketch_crawl bounded with maxPages and maxChars, and cite the page URLs used in conclusions.",
+      ],
+      ...ketchRenderers("ketch_crawl"),
+      async execute(_id, params: CrawlArgs, signal, _update, ctx) {
+        const maxPages = params.maxPages ?? 30;
+        const maxChars = params.maxChars ?? 6000;
+        const process = await runKetchCrawl({
+          cwd: ctx.cwd,
+          args: buildCrawlArgs(params),
+          signal,
+          timeoutMs: 180_000,
+          maxPages,
+          maxChars,
+        });
+        const errors = warningCount(process.stderr) + process.parseErrors.length;
+        let text = formatScrapePages(process.pages.map((page) => ({ url: page.url, title: page.title, markdown: page.body, source: page.source })));
+        const notices = [
+          process.stopped ? `Crawl stopped at the ${process.stopped.replace("_", " ")} bound; results are partial.` : undefined,
+          ...process.parseErrors,
+          process.stderr.trim() || undefined,
+        ].filter(Boolean);
+        if (notices.length) text += `\n\n## Crawl notices\n\n${notices.join("\n")}`;
+        return await formatKetchOutput({
+          process,
+          text,
+          parsed: process.pages.map(({ url, title, words, status, source }) => ({ url, title, words, status, source })),
+          resultCount: process.pages.length,
+          errorCount: errors,
+          stopped: process.stopped,
+        });
+      },
+    }),
+  );
+}