surgent 0.7.0-alpha.0

package/src/mcp-client/types.ts

@@ -0,0 +1,34 @@
+export type McpTransport = "stdio" | "http";
+
+interface McpServerBase {
+  transport: McpTransport;
+  enabled?: boolean;
+  description?: string;
+}
+
+export interface StdioMcpServer extends McpServerBase {
+  transport: "stdio";
+  command: string;
+  args?: string[];
+  cwd?: string;
+  env?: Record<string, string>;
+}
+
+export interface HttpMcpServer extends McpServerBase {
+  transport: "http";
+  url: string;
+  headers?: Record<string, string>;
+}
+
+export type McpServer = StdioMcpServer | HttpMcpServer;
+
+export type ResolvedMcpServer = McpServer & {
+  name: string;
+  scope: "project" | "global";
+};
+
+export interface McpToolCallDetails {
+  server: string;
+  transport: McpTransport;
+  remoteTool: string;
+}

package/src/mcp-client/validation.ts

@@ -0,0 +1,115 @@
+import { Type } from "typebox";
+import Value from "typebox/value";
+import type { ResolvedMcpServer } from "./types.js";
+
+const editableResolvedMcpServerSchema = Type.Union([
+  Type.Object(
+    {
+      name: Type.String({ minLength: 1 }),
+      scope: Type.Union([Type.Literal("project"), Type.Literal("global")]),
+      transport: Type.Literal("stdio"),
+      enabled: Type.Boolean(),
+      description: Type.Optional(Type.String()),
+      command: Type.String({ minLength: 1 }),
+      args: Type.Optional(Type.Array(Type.String())),
+      cwd: Type.Optional(Type.String({ minLength: 1 })),
+      env: Type.Optional(Type.Record(Type.String(), Type.String())),
+    },
+    { additionalProperties: false },
+  ),
+  Type.Object(
+    {
+      name: Type.String({ minLength: 1 }),
+      scope: Type.Union([Type.Literal("project"), Type.Literal("global")]),
+      transport: Type.Literal("http"),
+      enabled: Type.Boolean(),
+      description: Type.Optional(Type.String()),
+      url: Type.String({ minLength: 1 }),
+      headers: Type.Optional(Type.Record(Type.String(), Type.String())),
+    },
+    { additionalProperties: false },
+  ),
+]);
+
+export function parseEditConfigValues(values: Record<string, string>): ResolvedMcpServer {
+  const enabledText = (values.enabled ?? "").trim().toLowerCase();
+  if (enabledText !== "true" && enabledText !== "false") {
+    throw new Error('enabled must be "true" or "false".');
+  }
+  const enabled = enabledText === "true";
+
+  const name = (values.name ?? "").trim();
+  const scope = (values.scope ?? "").trim();
+  const transport = (values.transport ?? "").trim();
+  const description = (values.description ?? "").trim();
+
+  if (transport === "stdio") {
+    const args = parseOptionalJsonValue(values.args, "args");
+    const env = parseOptionalJsonValue(values.env, "env");
+    const stdioServer = {
+      name,
+      scope,
+      transport: "stdio",
+      enabled,
+      command: (values.command ?? "").trim(),
+      ...(description ? { description } : {}),
+      ...((values.cwd ?? "").trim() ? { cwd: (values.cwd ?? "").trim() } : {}),
+      ...(args !== undefined ? { args } : {}),
+      ...(env !== undefined ? { env } : {}),
+    };
+
+    validateWithTypebox(stdioServer);
+    return stdioServer as ResolvedMcpServer;
+  }
+
+  if (transport === "http") {
+    const headers = parseOptionalJsonValue(values.headers, "headers");
+    const httpServer = {
+      name,
+      scope,
+      transport: "http",
+      enabled,
+      url: (values.url ?? "").trim(),
+      ...(description ? { description } : {}),
+      ...(headers !== undefined ? { headers } : {}),
+    };
+
+    validateWithTypebox(httpServer);
+    return httpServer as ResolvedMcpServer;
+  }
+
+  throw new Error('transport must be "stdio" or "http".');
+}
+
+function parseOptionalJsonValue(rawValue: string | undefined, fieldName: string): unknown {
+  const trimmedValue = (rawValue ?? "").trim();
+  if (!trimmedValue) return;
+
+  try {
+    return JSON.parse(trimmedValue);
+  } catch (error) {
+    throw new Error(
+      `Invalid JSON in ${fieldName}: ${error instanceof Error ? error.message : String(error)}`,
+    );
+  }
+}
+
+function validateWithTypebox(server: unknown) {
+  if (Value.Check(editableResolvedMcpServerSchema, server)) {
+    return;
+  }
+
+  const validationErrors = Value.Errors(editableResolvedMcpServerSchema, server);
+  const firstError = validationErrors[0];
+  if (!firstError) {
+    throw new Error("Invalid MCP server config.");
+  }
+
+  const fieldPath = firstError.instancePath.replace(/^\//, "").replaceAll("/", ".").trim();
+
+  if (fieldPath.length > 0) {
+    throw new Error(`Invalid ${fieldPath}: ${firstError.message}`);
+  }
+
+  throw new Error(`Invalid MCP server config: ${firstError.message}`);
+}

package/src/optimizers/compactor/bash.ts

@@ -0,0 +1,159 @@
+import { stripVTControlCharacters } from "node:util";
+import { createLocalBashOperations, type BashOperations } from "@earendil-works/pi-coding-agent";
+
+const localBash = createLocalBashOperations();
+const MIN_SIMILAR_LENGTH = 10;
+const SIMILARITY_THRESHOLD = 0.8;
+
+function isSimilarLine(previousLine: string, nextLine: string) {
+  if (Math.min(previousLine.length, nextLine.length) < MIN_SIMILAR_LENGTH) return false;
+
+  const longestLength = Math.max(previousLine.length, nextLine.length);
+  if (1 - Math.abs(previousLine.length - nextLine.length) / longestLength < SIMILARITY_THRESHOLD) {
+    return false;
+  }
+
+  const distances = new Uint32Array(nextLine.length + 1);
+  for (let nextIndex = 0; nextIndex <= nextLine.length; nextIndex++) {
+    distances[nextIndex] = nextIndex;
+  }
+
+  for (let previousIndex = 1; previousIndex <= previousLine.length; previousIndex++) {
+    let diagonalDistance = distances[0]!;
+    distances[0] = previousIndex;
+
+    for (let nextIndex = 1; nextIndex <= nextLine.length; nextIndex++) {
+      const upperDistance = distances[nextIndex]!;
+      const editCost = previousLine[previousIndex - 1] === nextLine[nextIndex - 1] ? 0 : 1;
+      distances[nextIndex] = Math.min(
+        upperDistance + 1,
+        distances[nextIndex - 1]! + 1,
+        diagonalDistance + editCost,
+      );
+      diagonalDistance = upperDistance;
+    }
+  }
+
+  return 1 - distances[nextLine.length]! / longestLength >= SIMILARITY_THRESHOLD;
+}
+
+export class BashResultCompactor {
+  private readonly decoder = new TextDecoder();
+  private readonly onData: (data: Buffer) => void;
+  // Exact regex and carriage-return handling require holding one logical line.
+  private currentLine = "";
+  private carriageLine: string | undefined;
+  private hasPendingLine = false;
+  private pendingLine: string | undefined;
+  private pendingTerminated = false;
+  private omittedLines = 0;
+  private hasSimilarLines = false;
+
+  constructor(onData: (data: Buffer) => void) {
+    this.onData = onData;
+  }
+
+  append(data: Buffer) {
+    this.process(this.decoder.decode(data, { stream: true }));
+  }
+
+  finish() {
+    this.process(this.decoder.decode());
+    if (this.hasPendingLine) {
+      this.emit(this.getNormalizedLine(), false);
+      this.resetLine();
+    }
+    this.flushPending();
+  }
+
+  private process(text: string) {
+    for (const character of text) {
+      if (character === "\r") {
+        if (
+          this.carriageLine === undefined ||
+          stripVTControlCharacters(this.currentLine).length > 0
+        ) {
+          this.carriageLine = this.currentLine;
+        }
+        this.currentLine = "";
+        this.hasPendingLine = true;
+        continue;
+      }
+
+      if (character === "\n") {
+        this.emit(this.getNormalizedLine(), true);
+        this.resetLine();
+        continue;
+      }
+
+      this.currentLine += character;
+      this.hasPendingLine = true;
+    }
+  }
+
+  private getNormalizedLine() {
+    const normalizedLine = stripVTControlCharacters(this.currentLine);
+    if (this.carriageLine !== undefined && normalizedLine.length === 0) {
+      return stripVTControlCharacters(this.carriageLine);
+    }
+    return normalizedLine;
+  }
+
+  private emit(normalizedLine: string, terminated: boolean) {
+    if (this.pendingLine === undefined) {
+      this.pendingLine = normalizedLine;
+      this.pendingTerminated = terminated;
+      return;
+    }
+
+    if (normalizedLine === this.pendingLine) {
+      this.omittedLines++;
+      return;
+    }
+
+    if (isSimilarLine(this.pendingLine, normalizedLine)) {
+      this.pendingLine = normalizedLine;
+      this.pendingTerminated = terminated;
+      this.omittedLines++;
+      this.hasSimilarLines = true;
+      return;
+    }
+
+    this.flushPending();
+    this.pendingLine = normalizedLine;
+    this.pendingTerminated = terminated;
+  }
+
+  private flushPending() {
+    if (this.pendingLine === undefined) return;
+
+    const suffix = this.hasSimilarLines ? ` (similar line x${this.omittedLines})` : "";
+    this.onData(Buffer.from(`${this.pendingLine}${suffix}${this.pendingTerminated ? "\n" : ""}`));
+    this.pendingLine = undefined;
+    this.pendingTerminated = false;
+    this.omittedLines = 0;
+    this.hasSimilarLines = false;
+  }
+
+  private resetLine() {
+    this.currentLine = "";
+    this.carriageLine = undefined;
+    this.hasPendingLine = false;
+  }
+}
+
+export function createCompactingBashOperations(filter?: string): BashOperations {
+  return {
+    async exec(command, cwd, options) {
+      const compactor = new BashResultCompactor(options.onData);
+      try {
+        return await localBash.exec(command, cwd, {
+          ...options,
+          onData: (data) => compactor.append(data),
+        });
+      } finally {
+        compactor.finish();
+      }
+    },
+  };
+}

package/src/optimizers/compactor/grep.ts

@@ -0,0 +1,141 @@
+import {
+  appendFileSync,
+  existsSync,
+  readFileSync,
+  renameSync,
+  unlinkSync,
+  writeFileSync,
+} from "node:fs";
+
+export function extractGrepSummary(contentText: string): string | null {
+  if (contentText === "No matches found") return null;
+  const notice = contentText.match(/\n\n(\[[^\n]+\])$/)?.[1];
+
+  const fileLines = new Map<string, Set<number>>();
+  const contentLines = contentText.split("\n");
+  let filePath: string | undefined;
+  let matched = false;
+
+  for (let lineIndex = 0; lineIndex < contentLines.length; lineIndex++) {
+    const contentLine = contentLines[lineIndex] ?? "";
+    const lineMatch = contentLine.match(/^(\d+): /);
+    if (!lineMatch) {
+      const nextLine = contentLines[lineIndex + 1] ?? "";
+      if (contentLine && !/^\d+- /.test(contentLine) && /^\d+[:-] /.test(nextLine)) {
+        filePath = contentLine;
+      }
+      continue;
+    }
+
+    const lineNumStr = lineMatch[1];
+    if (!filePath || !lineNumStr) continue;
+
+    matched = true;
+    const lineNum = parseInt(lineNumStr, 10);
+    const existing = fileLines.get(filePath);
+    if (existing) {
+      existing.add(lineNum);
+    } else {
+      fileLines.set(filePath, new Set([lineNum]));
+    }
+  }
+
+  if (!matched) return null;
+
+  return (
+    Array.from(fileLines.entries())
+      .map(([filePath, lines]) => `${filePath}: lines_matched=[${Array.from(lines).join(", ")}]`)
+      .join("\n") + (notice ? `\n\n${notice}` : "")
+  );
+}
+
+export function rewriteTailWithSummaries(
+  sessionFile: string,
+  offset: number,
+  summaries: Map<string, string>,
+) {
+  if (summaries.size === 0) return;
+
+  const sessionBuffer = readFileSync(sessionFile);
+  if (offset > sessionBuffer.length) return;
+
+  const prefixBuffer = sessionBuffer.subarray(0, offset);
+  const tailText = sessionBuffer.subarray(offset).toString("utf-8");
+  if (tailText.length === 0) return;
+
+  let changed = false;
+  const rewrittenTail = tailText
+    .split("\n")
+    .filter((line) => line.trim().length > 0)
+    .map((line) => {
+      const entry = JSON.parse(line) as Record<string, unknown>;
+      const message = entry.message as Record<string, unknown> | undefined;
+      if (
+        entry.type === "message" &&
+        message?.role === "toolResult" &&
+        typeof message.toolCallId === "string" &&
+        summaries.has(message.toolCallId)
+      ) {
+        changed = true;
+        return JSON.stringify({
+          ...entry,
+          message: {
+            ...message,
+            content: [{ type: "text", text: summaries.get(message.toolCallId)! }],
+          },
+        });
+      }
+      return line;
+    })
+    .join("\n");
+
+  if (!changed) return;
+  const tempFile = `${sessionFile}.${process.pid}.${Date.now()}.tmp`;
+
+  try {
+    writeFileSync(tempFile, prefixBuffer);
+    appendFileSync(tempFile, `${rewrittenTail}\n`, "utf8");
+    renameSync(tempFile, sessionFile);
+  } finally {
+    if (existsSync(tempFile)) {
+      unlinkSync(tempFile);
+    }
+  }
+}
+
+export function formatGrepResult(content: string): string {
+  const formattedLines: string[] = [];
+  const formattedIndexes = new Map<string, number>();
+  let currentFilePath: string | undefined;
+  let changed = false;
+
+  for (const line of content.split("\n")) {
+    const matchLine = line.match(/^(.+?):(\d+): (.*)$/);
+    const contextLine = matchLine ? null : line.match(/^(.+?)-(\d+)- (.*)$/);
+    const filePath = matchLine?.[1] ?? contextLine?.[1];
+    const lineNumber = matchLine?.[2] ?? contextLine?.[2];
+    const lineText = matchLine?.[3] ?? contextLine?.[3];
+
+    if (!filePath || !lineNumber || lineText === undefined) {
+      formattedLines.push(line);
+      continue;
+    }
+
+    const lineKey = `${filePath}\0${lineNumber}`;
+    const formattedIndex = formattedIndexes.get(lineKey);
+    if (formattedIndex !== undefined) {
+      if (matchLine) formattedLines[formattedIndex] = `${lineNumber}: ${lineText}`;
+      continue;
+    }
+
+    if (filePath !== currentFilePath) {
+      formattedLines.push(filePath);
+      currentFilePath = filePath;
+    }
+    formattedIndexes.set(lineKey, formattedLines.length);
+    formattedLines.push(`${lineNumber}${matchLine ? ":" : "-"} ${lineText}`);
+    changed = true;
+  }
+
+  return changed ? formattedLines.join("\n") : content;
+}

package/src/optimizers/compactor/index.ts

@@ -0,0 +1,132 @@
+import { statSync } from "node:fs";
+import {
+  createBashToolDefinition,
+  createGrepToolDefinition,
+  isGrepToolResult,
+  type ExtensionAPI,
+} from "@earendil-works/pi-coding-agent";
+import { createCompactingBashOperations } from "./bash.js";
+import { rewriteTailWithSummaries, extractGrepSummary, formatGrepResult } from "./grep.js";
+import Type from "typebox";
+
+export default function (pi: ExtensionAPI) {
+  let writeStartOffset = 0;
+  const store = new Map<string, string>();
+  const grepTool = createGrepToolDefinition(process.cwd());
+  const bashTool = createBashToolDefinition(process.cwd(), {
+    operations: createCompactingBashOperations(),
+  });
+
+  pi.registerTool({
+    ...grepTool,
+    parameters: Type.Object({
+      ...grepTool.parameters.properties,
+      context: Type.Optional(
+        Type.Number({
+          description: "Number of lines to show before and after each match (default: 0)",
+          maximum: 3,
+        }),
+      ),
+    }),
+  });
+
+  pi.registerTool({
+    ...bashTool,
+    description:
+      "Execute a bash command in the current working directory. Strips ANSI escapes, collapses carriage-return updates, removes consecutive duplicate lines, then optionally filters lines with a JavaScript regex before truncating to the last 2000 lines or 50KB. Saved full output is compacted.",
+    parameters: Type.Object({
+      ...bashTool.parameters.properties,
+      purpose: Type.String({
+        description: "Briefly explain what this command will do and why before running it",
+        minLength: 1,
+        maxLength: 256,
+        pattern: "\\S",
+      }),
+    }),
+    prepareArguments: undefined,
+    execute(toolCallId, params, signal, onUpdate, ctx) {
+      return bashTool.execute(
+        toolCallId,
+        { command: params.command, timeout: params.timeout },
+        signal,
+        onUpdate,
+        ctx,
+      );
+    },
+  });
+
+  pi.on("agent_start", async (_event, ctx) => {
+    if (store.size > 0) return;
+
+    const sessionFile = ctx.sessionManager.getSessionFile();
+    if (!sessionFile) {
+      writeStartOffset = 0;
+      return;
+    }
+
+    try {
+      writeStartOffset = statSync(sessionFile).size;
+    } catch {
+      writeStartOffset = 0;
+    }
+  });
+
+  pi.on("agent_end", async (event) => {
+    for (const message of event.messages) {
+      if (message.role !== "toolResult" || message.toolName !== "grep") continue;
+      const text = message.content.find((content) => content.type === "text")?.text;
+      if (text === undefined) continue;
+
+      const summary = extractGrepSummary(text);
+      if (!summary) continue;
+      store.set(message.toolCallId, summary);
+    }
+  });
+
+  pi.on("session_shutdown", (event, ctx) => {
+    if (store.size === 0) return;
+
+    const sessionFile = ctx.sessionManager.getSessionFile();
+    if (sessionFile) {
+      rewriteTailWithSummaries(sessionFile, writeStartOffset, store);
+    }
+    if (event.targetSessionFile && event.targetSessionFile !== sessionFile) {
+      rewriteTailWithSummaries(event.targetSessionFile, writeStartOffset, store);
+    }
+  });
+
+  pi.on("context", async (event) => {
+    if (store.size === 0) return;
+
+    let changed = false;
+    for (const message of event.messages) {
+      if (message.role !== "toolResult" || message.toolName !== "grep") continue;
+
+      const summary = store.get(message.toolCallId);
+      if (summary === undefined) continue;
+
+      message.content = [{ type: "text", text: summary }];
+      changed = true;
+    }
+
+    if (!changed) return;
+    return { messages: event.messages };
+  });
+
+  pi.on("tool_result", async (event) => {
+    if (!isGrepToolResult(event) || event.isError) return;
+
+    let changed = false;
+    const content = event.content.map((item) => {
+      if (item.type !== "text") return item;
+
+      const text = formatGrepResult(item.text);
+      if (text === item.text) return item;
+
+      changed = true;
+      return { ...item, text };
+    });
+
+    if (changed) return { content };
+  });
+}

package/src/optimizers/deduplicator/helpers.ts

@@ -0,0 +1,75 @@
+import type { Range } from "../inspector/types.js";
+import type { ContextEvent } from "@earendil-works/pi-coding-agent";
+import type { DeduplicatorState } from "./types.js";
+
+export function filterDeduplicatedMessages(
+  messages: ContextEvent["messages"],
+  state: DeduplicatorState,
+): { changed: boolean; messages: ContextEvent["messages"] } {
+  const prunedIds = new Set<string>();
+  for (const [toolCallId, repeatIds] of state.replacements) {
+    if (repeatIds.every((repeatId) => state.resultEntryIds.has(repeatId))) {
+      prunedIds.add(toolCallId);
+    }
+  }
+  if (prunedIds.size === 0) {
+    return { changed: false, messages };
+  }
+
+  const retained: ContextEvent["messages"] = [];
+  let changed = false;
+  for (const message of messages) {
+    if (message.role === "toolResult" && prunedIds.has(message.toolCallId)) {
+      changed = true;
+      continue;
+    }
+    if (message.role !== "assistant") {
+      retained.push(message);
+      continue;
+    }
+
+    const retainedContent = message.content.filter(
+      (block) => block.type !== "toolCall" || !prunedIds.has(block.id),
+    );
+    if (retainedContent.length === message.content.length) {
+      retained.push(message);
+      continue;
+    }
+    changed = true;
+    if (retainedContent.some((block) => block.type !== "thinking")) {
+      retained.push({ ...message, content: retainedContent });
+    }
+  }
+  return { changed, messages: retained };
+}
+
+export function mergeRanges(ranges: Range[]) {
+  const sorted = ranges.toSorted(([firstStart], [secondStart]) => firstStart - secondStart);
+  const merged: Range[] = [];
+
+  for (const range of sorted) {
+    const previousRange = merged.at(-1);
+    if (!previousRange || previousRange[1] < range[0] - 1) {
+      merged.push([range[0], range[1]]);
+      continue;
+    }
+    previousRange[1] = Math.max(previousRange[1], range[1]);
+  }
+
+  return merged;
+}
+
+export function hasFullCoverage(range: Range, candidates: Range[]): boolean {
+  const intersections: Range[] = [];
+  for (const candidate of candidates) {
+    const start = Math.max(range[0], candidate[0]);
+    const end = Math.min(range[1], candidate[1]);
+    if (start <= end) intersections.push([start, end]);
+  }
+  if (intersections.length === 0) return false;
+
+  const merged = mergeRanges(intersections);
+  return (
+    merged.reduce((lines, [start, end]) => lines + end - start + 1, 0) === range[1] - range[0] + 1
+  );
+}

package/src/optimizers/deduplicator/index.ts

@@ -0,0 +1,23 @@
+import { type ExtensionAPI } from "@earendil-works/pi-coding-agent";
+import { readSessionEntries } from "../entries.js";
+import { filterDeduplicatedMessages } from "./helpers.js";
+import { buildDeduplicatorState } from "./state.js";
+
+export default function (pi: ExtensionAPI) {
+  let state = buildDeduplicatorState([], null, "");
+
+  pi.on("session_start", (_event, ctx) => {
+    const entries = readSessionEntries(ctx.sessionManager.getSessionFile()) ?? [];
+    state = buildDeduplicatorState(entries, ctx.sessionManager.getLeafId(), ctx.cwd);
+  });
+
+  pi.on("session_tree", (event, ctx) => {
+    const entries = readSessionEntries(ctx.sessionManager.getSessionFile()) ?? [];
+    state = buildDeduplicatorState(entries, event.newLeafId, ctx.cwd);
+  });
+
+  pi.on("context", (event) => {
+    const deduplicated = filterDeduplicatedMessages(event.messages, state);
+    if (deduplicated.changed) return { messages: deduplicated.messages };
+  });
+}