vellum 0.2.13 → 0.3.2

package/src/daemon/computer-use-session.ts

@@ -1,903 +0,0 @@
-/**
- * Computer-use session orchestrator.
- *
- * Manages the observation -> infer -> action loop for computer-use tasks,
- * bridging the macOS client (which captures screen state and executes actions)
- * with the AgentLoop (which runs inference via the Anthropic API with CU tools).
- */
-
-import { v4 as uuid } from 'uuid';
-import type { Provider, Message, ContentBlock, ToolDefinition } from '../providers/types.js';
-import { INTERACTIVE_SURFACE_TYPES } from './ipc-protocol.js';
-import type { ServerMessage, CuObservation, SurfaceType, SurfaceData, FileUploadSurfaceData, UiSurfaceShow } from './ipc-protocol.js';
-import type { Tool, ToolExecutionResult } from '../tools/types.js';
-import { AgentLoop } from '../agent/loop.js';
-import { ToolExecutor } from '../tools/executor.js';
-import { PermissionPrompter } from '../permissions/prompter.js';
-import { SecretPrompter } from '../permissions/secret-prompter.js';
-import type { UserDecision } from '../permissions/types.js';
-import { allUiSurfaceTools } from '../tools/ui-surface/definitions.js';
-import { allComputerUseTools } from '../tools/computer-use/definitions.js';
-import { registerSkillTools } from '../tools/registry.js';
-import { buildComputerUseSystemPrompt } from '../config/computer-use-prompt.js';
-import { getSandboxWorkingDir } from '../util/platform.js';
-import { getConfig } from '../config/loader.js';
-import { projectSkillTools, resetSkillToolProjection, type SkillProjectionCache } from './session-skill-tools.js';
-import { getLogger } from '../util/logger.js';
-
-const log = getLogger('computer-use-session');
-
-const MAX_STEPS = 50;
-const SESSION_TIMEOUT_MS = 30 * 60 * 1000; // 30 minutes
-const MAX_HISTORY_ENTRIES = 10;
-const LOOP_DETECTION_WINDOW = 3;
-const CONSECUTIVE_UNCHANGED_WARNING_THRESHOLD = 2;
-
-/** Number of most-recent AX tree snapshots to keep in conversation history. */
-const MAX_AX_TREES_IN_HISTORY = 2;
-
-/** Regex that matches the `<ax-tree>…</ax-tree>` markers injected by buildObservationResultContent. */
-const AX_TREE_PATTERN = /<ax-tree>[\s\S]*?<\/ax-tree>/g;
-const AX_TREE_PLACEHOLDER = '<ax_tree_omitted />';
-
-type SessionState = 'idle' | 'awaiting_observation' | 'inferring' | 'complete' | 'error';
-
-interface ActionRecord {
-  step: number;
-  toolName: string;
-  input: Record<string, unknown>;
-  reasoning?: string;
-  result?: string;
-}
-
-export class ComputerUseSession {
-  private readonly sessionId: string;
-  private readonly task: string;
-  private readonly screenWidth: number;
-  private readonly screenHeight: number;
-  private readonly provider: Provider;
-  private sendToClient: (msg: ServerMessage) => void;
-  private readonly interactionType: 'computer_use' | 'text_qa';
-  private readonly onTerminal?: (sessionId: string) => void;
-  private readonly preactivatedSkillIds: string[];
-  private readonly skillProjectionState = new Map<string, string>();
-  private readonly skillProjectionCache: SkillProjectionCache = {};
-
-  private state: SessionState = 'idle';
-  private stepCount = 0;
-  private actionHistory: ActionRecord[] = [];
-  private previousAXTree: string | undefined;
-  private consecutiveUnchangedSteps = 0;
-  private abortController: AbortController | null = null;
-  private sessionTimer: ReturnType<typeof setTimeout> | null = null;
-
-  private pendingObservation: {
-    resolve: (result: ToolExecutionResult) => void;
-  } | null = null;
-
-  private pendingSurfaceActions = new Map<string, {
-    resolve: (result: ToolExecutionResult) => void;
-  }>();
-  private surfaceState = new Map<string, { surfaceType: SurfaceType; data: SurfaceData }>();
-  private terminalNotified = false;
-  private prompter: PermissionPrompter | null = null;
-
-  // Tracks the agent loop promise so callers can await session completion
-  private loopPromise: Promise<void> | null = null;
-
-  constructor(
-    sessionId: string,
-    task: string,
-    screenWidth: number,
-    screenHeight: number,
-    provider: Provider,
-    sendToClient: (msg: ServerMessage) => void,
-    interactionType?: 'computer_use' | 'text_qa',
-    onTerminal?: (sessionId: string) => void,
-    preactivatedSkillIds?: string[],
-  ) {
-    this.sessionId = sessionId;
-    this.task = task;
-    this.screenWidth = screenWidth;
-    this.screenHeight = screenHeight;
-    this.provider = provider;
-    this.sendToClient = sendToClient;
-    this.interactionType = interactionType ?? 'computer_use';
-    this.onTerminal = onTerminal;
-    this.preactivatedSkillIds = preactivatedSkillIds ?? ['computer-use'];
-  }
-
-  // ---------------------------------------------------------------------------
-  // Public API
-  // ---------------------------------------------------------------------------
-
-  async handleObservation(obs: CuObservation): Promise<void> {
-    if (this.state === 'complete' || this.state === 'error') {
-      log.warn({ sessionId: this.sessionId, state: this.state }, 'Observation received after session ended');
-      return;
-    }
-
-    // Track consecutive unchanged steps
-    const hadPreviousAXTree = this.previousAXTree != null;
-    if (this.stepCount > 0) {
-      if (obs.axDiff == null && hadPreviousAXTree && obs.axTree != null) {
-        this.consecutiveUnchangedSteps++;
-      } else if (obs.axDiff != null) {
-        this.consecutiveUnchangedSteps = 0;
-      }
-    }
-
-    // Capture previous AX tree for next turn
-    if (obs.axTree != null) {
-      this.previousAXTree = obs.axTree;
-    }
-
-    if (this.state === 'awaiting_observation' && this.pendingObservation) {
-      // Resolve the pending proxy tool result with updated screen context
-      const content = this.buildObservationResultContent(obs, hadPreviousAXTree);
-      const result: ToolExecutionResult = obs.executionError
-        ? { content: `Action failed: ${obs.executionError}\n\n${content}`, isError: true }
-        : { content, isError: false };
-      this.state = 'inferring';
-      this.pendingObservation.resolve(result);
-      this.pendingObservation = null;
-      // The agent loop continues automatically after resolution
-      return;
-    }
-
-    // First observation — start the agent loop
-    this.state = 'inferring';
-    this.abortController = new AbortController();
-
-    // Safety net: abort the session if it runs longer than SESSION_TIMEOUT_MS
-    this.sessionTimer = setTimeout(() => {
-      log.warn({ sessionId: this.sessionId, timeoutMs: SESSION_TIMEOUT_MS }, 'Session timeout reached, aborting');
-      this.abort();
-    }, SESSION_TIMEOUT_MS);
-
-    const messages = this.buildMessages(obs, hadPreviousAXTree);
-    this.loopPromise = this.runAgentLoop(messages).catch((err) => {
-      // Catches errors from setup code (e.g. skill projection failures) that
-      // occur before runAgentLoop's internal try-catch takes over.
-      const message = err instanceof Error ? err.message : String(err);
-      log.error({ err, sessionId: this.sessionId }, 'Agent loop startup failed');
-      if (this.sessionTimer) {
-        clearTimeout(this.sessionTimer);
-        this.sessionTimer = null;
-      }
-      if (this.state !== 'complete' && this.state !== 'error') {
-        this.state = 'error';
-        this.sendToClient({
-          type: 'cu_error',
-          sessionId: this.sessionId,
-          message,
-        });
-        this.notifyTerminal();
-      }
-    });
-
-    await this.loopPromise;
-  }
-
-  abort(): void {
-    if (this.state === 'complete' || this.state === 'error') return;
-
-    log.info({ sessionId: this.sessionId }, 'Aborting computer-use session');
-    if (this.sessionTimer) {
-      clearTimeout(this.sessionTimer);
-      this.sessionTimer = null;
-    }
-    this.abortController?.abort();
-
-    // If waiting for an observation, resolve it as cancelled
-    if (this.pendingObservation) {
-      this.pendingObservation.resolve({ content: 'Session aborted', isError: true });
-      this.pendingObservation = null;
-    }
-
-    // Dispose prompter to clear pending permission timers and reject promises
-    this.prompter?.dispose();
-
-    // Resolve any pending surface actions
-    for (const [, pending] of this.pendingSurfaceActions) {
-      pending.resolve({ content: 'Session aborted', isError: true });
-    }
-    this.pendingSurfaceActions.clear();
-    this.surfaceState.clear();
-
-    this.state = 'error';
-    this.sendToClient({
-      type: 'cu_error',
-      sessionId: this.sessionId,
-      message: 'Session aborted by user',
-    });
-    this.notifyTerminal();
-  }
-
-  isComplete(): boolean {
-    return this.state === 'complete';
-  }
-
-  getState(): string {
-    return this.state;
-  }
-
-  /**
-   * Compute CU tool definitions from the bundled computer-use skill via
-   * skill projection. Returns null if projection fails so the caller can
-   * fall back to legacy hardcoded tool definitions.
-   */
-  private getProjectedCuToolDefinitions(): ToolDefinition[] | null {
-    if (this.preactivatedSkillIds.length === 0) {
-      log.warn('No preactivatedSkillIds configured, falling back to legacy CU tools');
-      return null;
-    }
-
-    try {
-      const projection = projectSkillTools([], {
-        preactivatedSkillIds: this.preactivatedSkillIds,
-        previouslyActiveSkillIds: this.skillProjectionState,
-        cache: this.skillProjectionCache,
-      });
-
-      if (projection.toolDefinitions.length === 0) {
-        log.warn(
-          { preactivatedSkillIds: this.preactivatedSkillIds },
-          'Skill projection produced no tool definitions, falling back to legacy CU tools',
-        );
-        return null;
-      }
-
-      return projection.toolDefinitions;
-    } catch (err) {
-      log.warn({ err }, 'Skill projection failed, falling back to legacy CU tools');
-      return null;
-    }
-  }
-
-  handleSurfaceAction(surfaceId: string, actionId: string, data?: Record<string, unknown>): void {
-    const pending = this.pendingSurfaceActions.get(surfaceId);
-    if (!pending) {
-      log.warn({ surfaceId, actionId }, 'No pending surface action found');
-      return;
-    }
-    // selection_changed is a non-terminal state update — don't consume the
-    // pending entry. The selection state will be in the action button payload.
-    if (actionId === 'selection_changed') {
-      return;
-    }
-    this.pendingSurfaceActions.delete(surfaceId);
-    pending.resolve({
-      content: JSON.stringify({ actionId, data: data ?? {} }),
-      isError: false,
-    });
-  }
-
-  // ---------------------------------------------------------------------------
-  // Agent loop execution
-  // ---------------------------------------------------------------------------
-
-  private async runAgentLoop(messages: Message[]): Promise<void> {
-    const systemPrompt = buildComputerUseSystemPrompt(this.screenWidth, this.screenHeight);
-
-    let cuToolDefs = this.getProjectedCuToolDefinitions();
-    if (!cuToolDefs) {
-      // Fallback: register the legacy CU tools as skill-origin tools so
-      // ToolExecutor can resolve them via getTool(), but using the same
-      // ownerSkillId as the bundled computer-use skill. This avoids
-      // core-vs-skill collisions that would permanently block skill
-      // projection recovery on subsequent sessions.
-      const fallbackSkillId = this.preactivatedSkillIds[0] ?? 'computer-use';
-      const fallbackTools: Tool[] = allComputerUseTools.map((t) => ({
-        ...t,
-        origin: 'skill' as const,
-        ownerSkillId: fallbackSkillId,
-        ownerSkillBundled: true,
-      }));
-      registerSkillTools(fallbackTools);
-      // Track in the session map so resetSkillToolProjection cleans up
-      this.skillProjectionState.set(fallbackSkillId, 'fallback');
-      cuToolDefs = allComputerUseTools.map((t) => t.getDefinition());
-    }
-
-    const toolDefs: ToolDefinition[] = [
-      ...cuToolDefs,
-      ...allUiSurfaceTools
-        .filter((t) => t.name !== 'request_file')
-        .map((t) => t.getDefinition()),
-    ];
-
-    this.prompter = new PermissionPrompter(this.sendToClient);
-    const prompter = this.prompter;
-    const secretPrompter = new SecretPrompter(this.sendToClient);
-    const executor = new ToolExecutor(prompter);
-
-    const proxyResolver = async (
-      toolName: string,
-      input: Record<string, unknown>,
-    ): Promise<ToolExecutionResult> => {
-      // ── Surface tool proxying ──────────────────────────────────────
-      if (toolName === 'ui_show') {
-        const surfaceId = uuid();
-        const surfaceType = input.surface_type as SurfaceType;
-        const title = typeof input.title === 'string' ? input.title : undefined;
-        const data = input.data as SurfaceData;
-        const actions = input.actions as Array<{ id: string; label: string; style?: string }> | undefined;
-        // Interactive surfaces default to awaiting user action.
-        // Tables and lists only block when explicit action buttons are provided;
-        // selectionMode alone should not gate blocking because selection_changed
-        // fires on every click and would immediately resolve multi-select surfaces.
-        const hasActions = Array.isArray(actions) && actions.length > 0;
-        const isInteractive = surfaceType === 'list'
-          ? hasActions
-          : surfaceType === 'table'
-            ? hasActions
-            : INTERACTIVE_SURFACE_TYPES.includes(surfaceType);
-        const awaitAction = (input.await_action as boolean) ?? isInteractive;
-
-        // Track surface state for ui_update merging
-        this.surfaceState.set(surfaceId, { surfaceType, data });
-
-        this.sendToClient({
-          type: 'ui_surface_show',
-          sessionId: this.sessionId,
-          surfaceId,
-          surfaceType,
-          title,
-          data,
-          actions: actions?.map(a => ({ id: a.id, label: a.label, style: (a.style ?? 'secondary') as 'primary' | 'secondary' | 'destructive' })),
-        } as unknown as UiSurfaceShow);
-
-        if (awaitAction) {
-          return new Promise<ToolExecutionResult>((resolve) => {
-            this.pendingSurfaceActions.set(surfaceId, { resolve });
-          });
-        }
-        return { content: JSON.stringify({ surfaceId }), isError: false };
-      }
-
-      if (toolName === 'ui_update') {
-        const surfaceId = input.surface_id as string;
-        const patch = input.data as Record<string, unknown>;
-
-        // Merge the partial patch into the stored full surface data
-        const stored = this.surfaceState.get(surfaceId);
-        let mergedData: SurfaceData;
-        if (stored) {
-          mergedData = { ...stored.data, ...patch } as SurfaceData;
-          stored.data = mergedData;
-        } else {
-          mergedData = patch as unknown as SurfaceData;
-        }
-
-        this.sendToClient({
-          type: 'ui_surface_update',
-          sessionId: this.sessionId,
-          surfaceId,
-          data: mergedData,
-        });
-        return { content: 'Surface updated', isError: false };
-      }
-
-      if (toolName === 'ui_dismiss') {
-        const surfaceId = input.surface_id as string;
-        this.sendToClient({
-          type: 'ui_surface_dismiss',
-          sessionId: this.sessionId,
-          surfaceId,
-        });
-        this.pendingSurfaceActions.delete(surfaceId);
-        this.surfaceState.delete(surfaceId);
-        return { content: 'Surface dismissed', isError: false };
-      }
-
-      // ── File request proxying ──────────────────────────────────────
-      if (toolName === 'request_file') {
-        const surfaceId = uuid();
-        const prompt = typeof input.prompt === 'string' ? input.prompt : 'Please share a file';
-        const acceptedTypes = Array.isArray(input.accepted_types) ? input.accepted_types as string[] : undefined;
-        const maxFiles = typeof input.max_files === 'number' ? input.max_files : 1;
-
-        const data: FileUploadSurfaceData = {
-          prompt,
-          acceptedTypes,
-          maxFiles,
-        };
-
-        this.surfaceState.set(surfaceId, { surfaceType: 'file_upload', data });
-
-        this.sendToClient({
-          type: 'ui_surface_show',
-          sessionId: this.sessionId,
-          surfaceId,
-          surfaceType: 'file_upload',
-          title: 'File Request',
-          data,
-        } as UiSurfaceShow);
-
-        // Always await — file upload is interactive
-        return new Promise<ToolExecutionResult>((resolve) => {
-          this.pendingSurfaceActions.set(surfaceId, { resolve });
-        });
-      }
-
-      // ── Computer-use tool proxying ─────────────────────────────────
-      const reasoning = typeof input.reasoning === 'string' ? input.reasoning : undefined;
-
-      // Record action in history
-      this.actionHistory.push({
-        step: this.stepCount + 1,
-        toolName,
-        input,
-        reasoning,
-      });
-
-      // Check for terminal tools
-      if (toolName === 'computer_use_done' || toolName === 'computer_use_respond') {
-        const summary =
-          toolName === 'computer_use_done'
-            ? (typeof input.summary === 'string' ? input.summary : 'Task completed')
-            : (typeof input.answer === 'string' ? input.answer : 'No answer provided');
-
-        this.sendToClient({
-          type: 'cu_complete',
-          sessionId: this.sessionId,
-          summary,
-          stepCount: this.stepCount,
-          isResponse: toolName === 'computer_use_respond' ? true : undefined,
-        });
-        this.state = 'complete';
-        // Stop AgentLoop immediately so terminal tools cannot trigger extra provider calls.
-        this.abortController?.abort();
-        this.notifyTerminal();
-        return { content: 'Session complete', isError: false };
-      }
-
-      this.stepCount++;
-
-      // Enforce step limit — abort the loop so toolChoice:'any' can't force another turn
-      if (this.stepCount > MAX_STEPS) {
-        this.state = 'error';
-        this.sendToClient({
-          type: 'cu_error',
-          sessionId: this.sessionId,
-          message: `Step limit (${MAX_STEPS}) exceeded`,
-        });
-        this.abortController?.abort();
-        this.notifyTerminal();
-        return { content: `Step limit (${MAX_STEPS}) exceeded`, isError: true };
-      }
-
-      // Send action to client for execution
-      this.sendToClient({
-        type: 'cu_action',
-        sessionId: this.sessionId,
-        toolName,
-        input,
-        reasoning,
-        stepNumber: this.stepCount,
-      });
-
-      // Wait for next observation from client
-      this.state = 'awaiting_observation';
-      return new Promise<ToolExecutionResult>((resolve) => {
-        this.pendingObservation = { resolve };
-      });
-    };
-
-    // Build a set of tool names the CU session is allowed to execute.
-    // This prevents tools registered globally (e.g. computer_use_request_control)
-    // but not advertised to the CU model from executing during CU sessions.
-    const allowedToolNames = new Set(toolDefs.map((td) => td.name));
-
-    const toolExecutor = async (
-      name: string,
-      input: Record<string, unknown>,
-    ): Promise<ToolExecutionResult> => {
-      return executor.execute(name, input, {
-        workingDir: getSandboxWorkingDir(),
-        sessionId: this.sessionId,
-        conversationId: this.sessionId,
-        proxyToolResolver: proxyResolver,
-        allowedToolNames,
-        requestSecret: async (params) => {
-          return secretPrompter.prompt(
-            params.service, params.field, params.label,
-            params.description, params.placeholder,
-            this.sessionId,
-            params.purpose, params.allowedTools, params.allowedDomains,
-          );
-        },
-      });
-    };
-
-    // Wrap the provider so that old AX tree snapshots are stripped from
-    // conversation history before each API call, keeping only the most recent
-    // MAX_AX_TREES_IN_HISTORY entries.  This prevents TTFT from growing
-    // linearly with step count.
-    const compactingProvider: Provider = {
-      name: this.provider.name,
-      sendMessage: (msgs, tools, sys, opts) => {
-        const compacted = ComputerUseSession.compactHistory(msgs);
-        return this.provider.sendMessage(compacted, tools, sys, opts);
-      },
-    };
-
-    const cuConfig = getConfig();
-    const agentLoop = new AgentLoop(
-      compactingProvider,
-      systemPrompt,
-      {
-        maxTokens: 4096,
-        maxInputTokens: cuConfig.contextWindow.maxInputTokens,
-        toolChoice: { type: 'any' },
-        // Allow MAX_STEPS non-terminal actions plus one terminal turn
-        // (computer_use_done/computer_use_respond), since AgentLoop caps tool turns globally.
-        maxToolUseTurns: MAX_STEPS + 1,
-      },
-      toolDefs,
-      toolExecutor,
-    );
-
-    try {
-      await agentLoop.run(
-        messages,
-        (event) => {
-          switch (event.type) {
-            case 'error':
-              log.error({ err: event.error, sessionId: this.sessionId }, 'Agent loop error');
-              if (this.state !== 'complete') {
-                this.state = 'error';
-                this.sendToClient({
-                  type: 'cu_error',
-                  sessionId: this.sessionId,
-                  message: event.error.message,
-                });
-                this.notifyTerminal();
-              }
-              break;
-            case 'usage':
-              log.info({
-                sessionId: this.sessionId,
-                inputTokens: event.inputTokens,
-                outputTokens: event.outputTokens,
-                model: event.model,
-              }, 'Usage');
-              break;
-            // Other events (text_delta, thinking_delta, etc.) are not surfaced to the CU client
-          }
-        },
-        this.abortController?.signal,
-      );
-
-      // If the loop exits without completing, treat as error
-      if (this.state !== 'complete' && this.state !== 'error') {
-        this.state = 'error';
-        this.sendToClient({
-          type: 'cu_error',
-          sessionId: this.sessionId,
-          message: 'Agent loop ended unexpectedly',
-        });
-        this.notifyTerminal();
-      }
-    } catch (err) {
-      if (this.abortController?.signal.aborted) {
-        log.info({ sessionId: this.sessionId }, 'Agent loop aborted');
-        return;
-      }
-      const message = err instanceof Error ? err.message : String(err);
-      log.error({ err, sessionId: this.sessionId }, 'Agent loop failed');
-      if (this.state !== 'complete') {
-        this.state = 'error';
-        this.sendToClient({
-          type: 'cu_error',
-          sessionId: this.sessionId,
-          message,
-        });
-        this.notifyTerminal();
-      }
-    } finally {
-      // Always clean up skill projection state and session timer
-      resetSkillToolProjection(this.skillProjectionState);
-      if (this.sessionTimer) {
-        clearTimeout(this.sessionTimer);
-        this.sessionTimer = null;
-      }
-    }
-  }
-
-  private notifyTerminal(): void {
-    if (this.terminalNotified) return;
-    this.terminalNotified = true;
-    resetSkillToolProjection(this.skillProjectionState);
-    this.onTerminal?.(this.sessionId);
-  }
-
-  // ---------------------------------------------------------------------------
-  // History compaction — strip old AX tree snapshots from tool results
-  // ---------------------------------------------------------------------------
-
-  /**
-   * Returns a shallow copy of `messages` where all but the most recent
-   * `MAX_AX_TREES_IN_HISTORY` `<ax-tree>` blocks have been replaced with a
-   * short placeholder.  This keeps the conversation context small so that
-   * TTFT does not grow linearly with step count.
-   */
-  static compactHistory(messages: Message[]): Message[] {
-    // Collect indices of user messages that contain an <ax-tree> block
-    const indicesWithAxTree: number[] = [];
-    for (let i = 0; i < messages.length; i++) {
-      const msg = messages[i];
-      if (msg.role !== 'user') continue;
-      for (const block of msg.content) {
-        if (
-          block.type === 'tool_result' &&
-          typeof block.content === 'string' &&
-          block.content.includes('<ax-tree>')
-        ) {
-          indicesWithAxTree.push(i);
-          break;
-        }
-      }
-    }
-
-    if (indicesWithAxTree.length <= MAX_AX_TREES_IN_HISTORY) {
-      return messages;
-    }
-
-    const toStrip = new Set(indicesWithAxTree.slice(0, -MAX_AX_TREES_IN_HISTORY));
-
-    return messages.map((msg, idx) => {
-      if (!toStrip.has(idx)) return msg;
-      return {
-        ...msg,
-        content: msg.content.map((block) => {
-          if (
-            block.type === 'tool_result' &&
-            typeof block.content === 'string' &&
-            block.content.includes('<ax-tree>')
-          ) {
-            return {
-              ...block,
-              content: block.content.replace(AX_TREE_PATTERN, AX_TREE_PLACEHOLDER),
-            };
-          }
-          return block;
-        }),
-      };
-    });
-  }
-
-  /**
-   * Escapes any literal `</ax-tree>` occurrences inside AX tree content so
-   * that the non-greedy compaction regex (`AX_TREE_PATTERN`) does not stop
-   * prematurely when the user happens to be viewing XML/HTML source that
-   * contains the closing tag.  The escaped content does not need to be
-   * unescaped because compaction replaces the entire block with a placeholder.
-   */
-  static escapeAxTreeContent(content: string): string {
-    return content.replace(/<\/ax-tree>/gi, '&lt;/ax-tree&gt;');
-  }
-
-  // ---------------------------------------------------------------------------
-  // Build rich tool-result content from an observation so the model sees
-  // updated screen state on each turn (not just "Action executed").
-  // ---------------------------------------------------------------------------
-
-  private buildObservationResultContent(obs: CuObservation, hadPreviousAXTree: boolean): string {
-    const parts: string[] = [];
-
-    if (obs.executionResult) {
-      parts.push(obs.executionResult);
-      parts.push('');
-    }
-
-    // AX tree diff
-    if (obs.axDiff) {
-      parts.push(obs.axDiff);
-      parts.push('');
-    } else if (hadPreviousAXTree && obs.axTree != null) {
-      const lastAction = this.actionHistory[this.actionHistory.length - 1];
-      const wasWait = lastAction?.toolName === 'computer_use_wait';
-      if (this.consecutiveUnchangedSteps >= CONSECUTIVE_UNCHANGED_WARNING_THRESHOLD) {
-        parts.push(
-          `WARNING: ${this.consecutiveUnchangedSteps} consecutive actions had NO VISIBLE EFFECT on the UI. You MUST try a completely different approach.`,
-        );
-      } else if (!wasWait) {
-        parts.push('Your last action had NO VISIBLE EFFECT on the UI. Try something different.');
-      }
-      parts.push('');
-    }
-
-    // Current screen state — wrapped in markers so compactHistory can strip old snapshots
-    if (obs.axTree) {
-      parts.push('<ax-tree>');
-      parts.push('CURRENT SCREEN STATE:');
-      parts.push(ComputerUseSession.escapeAxTreeContent(obs.axTree));
-      parts.push('</ax-tree>');
-    }
-
-    const screenshotMetadata = this.formatScreenshotMetadata(obs);
-    if (screenshotMetadata.length > 0) {
-      parts.push('');
-      parts.push(...screenshotMetadata);
-    }
-
-    return parts.join('\n').trim() || 'Action executed';
-  }
-
-  // ---------------------------------------------------------------------------
-  // Message building (replicates AnthropicProvider.buildMessages from Swift)
-  // ---------------------------------------------------------------------------
-
-  private buildMessages(obs: CuObservation, hadPreviousAXTree: boolean): Message[] {
-    const contentBlocks: ContentBlock[] = [];
-
-    // Screenshot image block
-    if (obs.screenshot) {
-      contentBlocks.push({
-        type: 'image',
-        source: {
-          type: 'base64',
-          media_type: 'image/jpeg',
-          data: obs.screenshot,
-        },
-      });
-    }
-
-    // Text block
-    const textParts: string[] = [];
-    const trimmedTask = this.task.trim();
-    if (trimmedTask) {
-      textParts.push(`TASK: ${trimmedTask}`);
-    } else {
-      textParts.push('TASK: No explicit task provided.');
-    }
-    textParts.push('');
-
-    // AX tree diff (compact summary of what changed)
-    if (obs.axDiff && this.actionHistory.length > 0) {
-      textParts.push(obs.axDiff);
-      textParts.push('');
-    } else if (hadPreviousAXTree && obs.axTree != null && this.actionHistory.length > 0) {
-      // AX tree unchanged — tell the model its action had no effect
-      const lastAction = this.actionHistory[this.actionHistory.length - 1];
-      const wasWait = lastAction?.toolName === 'computer_use_wait';
-      textParts.push('CHANGES SINCE LAST ACTION:');
-      if (this.consecutiveUnchangedSteps >= CONSECUTIVE_UNCHANGED_WARNING_THRESHOLD) {
-        textParts.push(
-          `WARNING: ${this.consecutiveUnchangedSteps} consecutive actions had NO VISIBLE EFFECT on the UI. You MUST try a completely different approach — do not repeat any of your recent actions.`,
-        );
-      } else if (!wasWait) {
-        const actionDesc = `${lastAction?.toolName ?? 'unknown'}`;
-        textParts.push(
-          `Your last action (${actionDesc}) had NO VISIBLE EFFECT on the UI. The screen is identical to the previous step. Do NOT repeat the same action — try something different.`,
-        );
-      } else {
-        textParts.push('No visible changes detected — the UI is identical to the previous step.');
-      }
-      textParts.push('');
-    }
-
-    // Current screen state
-    if (obs.axTree) {
-      textParts.push('CURRENT SCREEN STATE (accessibility tree of the focused window):');
-      textParts.push(obs.axTree);
-      textParts.push('');
-      textParts.push('Use element_id with the [ID] numbers shown above to target elements.');
-
-      // Secondary windows for cross-app awareness
-      if (obs.secondaryWindows) {
-        textParts.push('');
-        textParts.push(obs.secondaryWindows);
-        textParts.push('');
-        textParts.push(
-          "Note: The element [ID]s above are from other windows — you can reference them for context but can only interact with the focused window's elements.",
-        );
-      }
-
-      if (obs.screenshot) {
-        textParts.push('');
-        textParts.push(
-          'A screenshot of the FULL SCREEN is also attached above. Use it to see content outside the focused window (e.g., reference documents, PDFs, other apps visible behind the current window).',
-        );
-        const screenshotMetadata = this.formatScreenshotMetadata(obs);
-        if (screenshotMetadata.length > 0) {
-          textParts.push(...screenshotMetadata);
-        }
-      }
-    } else if (obs.screenshot) {
-      textParts.push('CURRENT SCREEN STATE:');
-      textParts.push('See the screenshot above. No accessibility tree available — estimate coordinates from the image.');
-      const screenshotMetadata = this.formatScreenshotMetadata(obs);
-      if (screenshotMetadata.length > 0) {
-        textParts.push(...screenshotMetadata);
-      }
-    } else {
-      textParts.push('CURRENT SCREEN STATE:');
-      textParts.push('No screen data available.');
-    }
-
-    // Action history
-    if (this.actionHistory.length > 0) {
-      textParts.push('');
-      textParts.push('ACTIONS TAKEN SO FAR:');
-      let windowedHistory: ActionRecord[];
-      if (this.actionHistory.length > MAX_HISTORY_ENTRIES) {
-        textParts.push(
-          `  [... ${this.actionHistory.length - MAX_HISTORY_ENTRIES} earlier actions omitted]`,
-        );
-        windowedHistory = this.actionHistory.slice(-MAX_HISTORY_ENTRIES);
-      } else {
-        windowedHistory = this.actionHistory;
-      }
-      for (const record of windowedHistory) {
-        const result = record.result ?? 'executed';
-        textParts.push(`  ${record.step}. ${record.toolName} → ${result}`);
-      }
-    }
-
-    // Loop detection warning
-    if (this.actionHistory.length >= LOOP_DETECTION_WINDOW) {
-      const recent = this.actionHistory.slice(-LOOP_DETECTION_WINDOW);
-      const allIdentical = recent.every(
-        (r) =>
-          r.toolName === recent[0].toolName &&
-          JSON.stringify(r.input) === JSON.stringify(recent[0].input),
-      );
-      if (allIdentical) {
-        textParts.push('');
-        textParts.push(
-          `WARNING: You have repeated the exact same action (${recent[0].toolName}) ${LOOP_DETECTION_WINDOW} times in a row. You MUST try a completely different approach or call computer_use_done with an explanation of why you are stuck.`,
-        );
-      }
-    }
-
-    // Prompt for next action
-    textParts.push('');
-    if (this.actionHistory.length === 0) {
-      textParts.push('This is the first action. Examine the screen state and decide what to do first.');
-    } else {
-      textParts.push('Decide the next action to take.');
-    }
-
-    contentBlocks.push({
-      type: 'text',
-      text: textParts.join('\n'),
-    });
-
-    return [{ role: 'user', content: contentBlocks }];
-  }
-
-  private formatScreenshotMetadata(obs: CuObservation): string[] {
-    if (!obs.screenshot) return [];
-
-    const lines: string[] = [];
-    if (obs.screenshotWidthPx != null && obs.screenshotHeightPx != null) {
-      lines.push(`Screenshot metadata: ${obs.screenshotWidthPx}x${obs.screenshotHeightPx} px`);
-    }
-    if (obs.screenWidthPt != null && obs.screenHeightPt != null) {
-      lines.push(`Screen metadata: ${obs.screenWidthPt}x${obs.screenHeightPt} pt`);
-    }
-    if (obs.coordinateOrigin) {
-      lines.push(`Coordinate origin: ${obs.coordinateOrigin}`);
-    }
-    if (obs.captureDisplayId != null) {
-      lines.push(`Capture display ID: ${obs.captureDisplayId}`);
-    }
-    return lines;
-  }
-
-  hasPendingConfirmation(requestId: string): boolean {
-    return this.prompter?.hasPendingRequest(requestId) ?? false;
-  }
-
-  handleConfirmationResponse(
-    requestId: string,
-    decision: UserDecision,
-    selectedPattern?: string,
-    selectedScope?: string,
-  ): void {
-    this.prompter?.resolveConfirmation(requestId, decision, selectedPattern, selectedScope);
-  }
-}