tinker-agent 1.11.0 → 2.1.0

package/src/model/model-client.ts

@@ -2,14 +2,13 @@ import type { AgentMessage, AssistantMessage, IterationIdentity } from "../agent
 import type { RuntimeSessionContext } from "../agent/runtime-session";
 import type { ToolDefinition } from "../tools/types";
 import type { ImageAssetStore } from "../image/image-asset-store";
-import type { CodePointRange, ImageAssetId, ImageMimeType } from "../image/image-types";
-import type { InputTokenEstimator } from "./input-token-estimator";
+import type { ImageAssetRef } from "../image/image-types";
 import type { ReasoningEffortController } from "./reasoning-effort";
 
 export interface ModelClient {
   readonly messageProtocol: ModelMessageProtocol;
-  readonly inputTokenEstimator?: InputTokenEstimator;
-  readonly inputModalities?: readonly ("text" | "image")[];
+  readonly inputModalities: readonly ModelInputModality[];
+  readonly toolResultModalities: readonly ToolResultModality[];
   readonly reasoningEffort?: ReasoningEffortController;
   prepare(input: ModelRequestInput): PreparedModelRequest;
   materialize?(
@@ -22,6 +21,59 @@ export interface ModelClient {
   ): Promise<ModelRequestOutput>;
 }
 
+export type ModelInputModality = "text" | "image";
+export type ToolResultModality = "text" | "image";
+
+export function validateModelModalities(input: {
+  readonly profileName?: string;
+  readonly adapter: ModelMessageProtocol["adapter"];
+  readonly inputModalities: readonly ModelInputModality[];
+  readonly toolResultModalities: readonly ToolResultModality[];
+  readonly adapterToolResultModalities: readonly ToolResultModality[];
+}): {
+  readonly inputModalities: readonly ModelInputModality[];
+  readonly toolResultModalities: readonly ToolResultModality[];
+} {
+  const inputModalities = normalizeModalities(input.inputModalities, "model input");
+  const toolResultModalities = normalizeModalities(
+    input.toolResultModalities,
+    "tool result",
+  );
+  if (toolResultModalities.includes("image") && !inputModalities.includes("image")) {
+    throw new Error(
+      'Image tool results require "image" in the model input modalities.',
+    );
+  }
+  const unsupported = toolResultModalities.find(
+    (modality) => !input.adapterToolResultModalities.includes(modality),
+  );
+  if (unsupported !== undefined) {
+    const subject =
+      input.profileName === undefined
+        ? "Model configuration"
+        : `Profile ${JSON.stringify(input.profileName)}`;
+    throw new Error(
+      `${subject} declares ${unsupported} tool results, but adapter ${JSON.stringify(input.adapter)} does not support them.`,
+    );
+  }
+  return Object.freeze({ inputModalities, toolResultModalities });
+}
+
+function normalizeModalities(
+  modalities: readonly (ModelInputModality | ToolResultModality)[],
+  label: string,
+): readonly ("text" | "image")[] {
+  if (
+    modalities.length === 0 ||
+    modalities.some((modality) => modality !== "text" && modality !== "image") ||
+    new Set(modalities).size !== modalities.length ||
+    !modalities.includes("text")
+  ) {
+    throw new Error(`${label} modalities must be unique and include "text".`);
+  }
+  return Object.freeze(modalities.includes("image") ? ["text", "image"] : ["text"]);
+}
+
 export class ModelRequestMediaAggregateError extends Error {
   readonly code = "MODEL_REQUEST_MEDIA_AGGREGATE_LIMIT";
 
@@ -67,15 +119,14 @@ export type PreparedPromptSegmentKind =
 export type PreparedPromptSegment = {
   kind: PreparedPromptSegmentKind;
   normalizedText: string;
-  media?: readonly PreparedMediaDescriptor[];
+  media?: readonly PreparedMediaOccurrence[];
 };
 
-export type PreparedMediaDescriptor = {
-  assetId: ImageAssetId;
-  label: string;
-  range: CodePointRange;
-  mimeType: ImageMimeType;
-  byteLength: number;
+export type PreparedMediaOccurrence = {
+  readonly asset: ImageAssetRef;
+  readonly source: "user_attachment" | "tool_result";
+  readonly messageOrdinal: number;
+  readonly blockPosition: number;
   width: number;
   height: number;
   planningTokens: number;

package/src/model/model-request-preflight.ts

@@ -3,7 +3,6 @@ import type { ModelContextBudget } from "./model-context-profile";
 
 export type ContextUsageSource =
   | "estimated_full"
-  | "provider_estimated"
   | "provider_measured"
   | "measured_plus_estimated_delta";
 

package/src/model/openai-chat-mapping.ts

@@ -21,6 +21,7 @@ import type {
 } from "openai/resources/chat/completions";
 import { validateUserMessage, type ImageAssetId } from "../image/image-types";
 import { imageAssetUrlMarker } from "./openai-image-mapping";
+import { toolResultText } from "../agent/tool-result-content";
 
 type DeepSeekAssistantMessageParam = ChatCompletionAssistantMessageParam & {
   reasoning_content?: string | null;
@@ -57,7 +58,7 @@ export function toOpenAIChatMessages(
       return {
         role: "tool",
         tool_call_id: message.providerToolCallId,
-        content: message.content,
+        content: toolResultText(message.content),
       };
     }
 

package/src/model/openai-chat-model-client.ts

@@ -9,8 +9,7 @@ import {
   IMAGE_INPUT_POLICY_VERSION,
 } from "../image/image-input-policy";
 import type { ModelContextBudget } from "./model-context-profile";
-import type { InputTokenEstimator } from "./input-token-estimator";
-import { ProviderResponseError } from "./model-client";
+import { ProviderResponseError, validateModelModalities } from "./model-client";
 import type {
   MaterializedModelRequest,
   ModelClient,
@@ -30,13 +29,13 @@ import {
 import { OpenAIChatCompletionStreamAccumulator } from "./openai-chat-stream";
 import {
   deepFreeze,
+  imageToolSegment,
   imageUserSegment,
   materializeOpenAIRequest,
   normalizedEndpointPolicy,
   sanitizedProviderError,
   segmentKind,
 } from "./openai-model-utils";
-import { MoonshotInputTokenEstimator } from "./moonshot-input-token-estimator";
 import type { ReasoningEffortController } from "./reasoning-effort";
 import { sha256, stableJsonStringify } from "./model-request-preflight";
 
@@ -48,7 +47,6 @@ export class OpenAIChatModelClient implements ModelClient {
     adapter: "openai-chat",
     serializationVersion: OPENAI_CHAT_SERIALIZATION_VERSION,
   });
-  readonly inputTokenEstimator?: InputTokenEstimator;
   readonly reasoningEffort?: ReasoningEffortController;
   private readonly client: OpenAI;
   private readonly preparedRequests = new WeakSet<object>();
@@ -56,6 +54,7 @@ export class OpenAIChatModelClient implements ModelClient {
   private readonly provider: string;
   private readonly stream: boolean;
   readonly inputModalities: readonly ("text" | "image")[];
+  readonly toolResultModalities: readonly ("text" | "image")[];
 
   constructor(
     private readonly options: {
@@ -64,14 +63,8 @@ export class OpenAIChatModelClient implements ModelClient {
       baseURL?: string;
       includeReasoningContent?: boolean;
       inputModalities?: readonly ("text" | "image")[];
-      tokenEstimator?: {
-        kind: "moonshot-estimate-token-count-v1";
-        model: string;
-        apiBase: string;
-        apiKey: string;
-        timeoutMs: number;
-        maxRetries: 0;
-      };
+      toolResultModalities?: readonly ("text" | "image")[];
+      profileName?: string;
       model: string;
       providerName?: string;
       reasoningEffort?: ReasoningEffortController;
@@ -83,14 +76,15 @@ export class OpenAIChatModelClient implements ModelClient {
     this.provider = options.providerName ?? "openai-compatible";
     this.stream = options.stream ?? true;
     this.reasoningEffort = options.reasoningEffort;
-    this.inputModalities = Object.freeze([...(options.inputModalities ?? ["text"])]);
-    const supportsImages = this.inputModalities.includes("image");
-    if (!this.inputModalities.includes("text")) {
-      throw new Error('OpenAI chat input modalities must include "text".');
-    }
-    if (supportsImages && options.tokenEstimator === undefined) {
-      throw new Error("Image-capable OpenAI chat requires a token estimator.");
-    }
+    const modalities = validateModelModalities({
+      profileName: options.profileName,
+      adapter: this.messageProtocol.adapter,
+      inputModalities: options.inputModalities ?? ["text"],
+      toolResultModalities: options.toolResultModalities ?? ["text"],
+      adapterToolResultModalities: ["text"],
+    });
+    this.inputModalities = modalities.inputModalities;
+    this.toolResultModalities = modalities.toolResultModalities;
     this.client = new OpenAI({
       apiKey: options.apiKey,
       baseURL: options.baseURL,
@@ -100,15 +94,6 @@ export class OpenAIChatModelClient implements ModelClient {
       maxRetries: 0,
       fetch: options.fetch,
     });
-    if (options.tokenEstimator !== undefined) {
-      this.inputTokenEstimator = new MoonshotInputTokenEstimator({
-        apiKey: options.tokenEstimator.apiKey,
-        baseURL: options.tokenEstimator.apiBase,
-        model: options.tokenEstimator.model,
-        timeoutMs: options.tokenEstimator.timeoutMs,
-        fetch: options.fetch,
-      });
-    }
   }
 
   prepare(input: ModelRequestInput): PreparedModelRequest {
@@ -139,11 +124,13 @@ export class OpenAIChatModelClient implements ModelClient {
     const messageSegments = input.messages.map(
       (message, index): PreparedPromptSegment =>
         message.role === "user" && message.attachments !== undefined
-          ? imageUserSegment(message)
-          : {
-              kind: segmentKind(message.role),
-              normalizedText: stableJsonStringify(messages[index]),
-            },
+          ? imageUserSegment(message, index + 1)
+          : message.role === "tool"
+            ? imageToolSegment(message, index + 1, stableJsonStringify(messages[index]))
+            : {
+                kind: segmentKind(message.role),
+                normalizedText: stableJsonStringify(messages[index]),
+              },
     );
     const mediaOccurrenceCount = messageSegments.reduce(
       (total, segment) => total + (segment.media?.length ?? 0),
@@ -159,14 +146,12 @@ export class OpenAIChatModelClient implements ModelClient {
         includeReasoningContent: this.options.includeReasoningContent === true,
         stream: this.stream,
         inputModalities: this.inputModalities,
+        toolResultModalities: this.toolResultModalities,
         requestPolicy: { toolChoice: "auto" },
         imagePolicy: {
           version: IMAGE_INPUT_POLICY_VERSION,
           ...IMAGE_INPUT_POLICY,
         },
-        ...(this.inputTokenEstimator === undefined
-          ? {}
-          : { tokenEstimator: this.inputTokenEstimator.compatibility }),
       }),
     );
     const prepared: PreparedModelRequest = {
@@ -299,4 +284,7 @@ export class OpenAIChatModelClient implements ModelClient {
   }
 }
 
-export { exactJsonBodyBytes } from "./openai-model-utils";
+export {
+  assertOpenAIRequestBodyLimit,
+  exactJsonBodyBytes,
+} from "./openai-model-utils";

package/src/model/openai-model-utils.ts

@@ -1,14 +1,19 @@
 import OpenAI from "openai";
-import type { UserMessage } from "../agent/types";
-import { IMAGE_INPUT_POLICY } from "../image/image-input-policy";
+import type { AgentMessage, UserMessage } from "../agent/types";
+import {
+  IMAGE_INPUT_POLICY,
+  imagePlanningTokens,
+  providerImageDimensions,
+} from "../image/image-input-policy";
 import type { ImageAssetId, ImageAssetRef } from "../image/image-types";
+import { materializeProviderImage, type ProviderImage } from "../image/provider-image";
 import {
   ModelRequestMediaAggregateError,
   ProviderResponseError,
   type MaterializedModelRequest,
   type ModelMaterializeOptions,
   type ModelRequestInput,
-  type PreparedMediaDescriptor,
+  type PreparedMediaOccurrence,
   type PreparedModelRequest,
   type PreparedPromptSegment,
   type ProviderResponseErrorCode,
@@ -28,26 +33,22 @@ export async function materializeOpenAIRequest(
   }
 
   const assets = distinctPreparedAssets(prepared.promptSegments);
-  const lowerLengths = new Map<ImageAssetId, number>();
-  for (const asset of assets.values()) {
-    lowerLengths.set(asset.assetId, dataUrlLength(asset));
-  }
   const markerCount = countImageMarkers(prepared.payload);
   if (markerCount !== prepared.mediaOccurrenceCount) {
     throw new Error("Prepared image marker count does not match media descriptors.");
   }
-  const lowerBodyBytes = exactJsonBodyBytes(prepared.payload, lowerLengths);
-  assertBodyLimit(lowerBodyBytes, prepared.mediaOccurrenceCount);
-
   const dataUrls = new Map<ImageAssetId, string>();
+  const providerImages = new Map<ImageAssetId, ProviderImage>();
   for (const asset of assets.values()) {
     options.signal.throwIfAborted();
     const bytes = await options.assetStore.readVerified(asset, {
       signal: options.signal,
     });
+    const image = await materializeProviderImage(bytes, asset.mimeType);
+    providerImages.set(asset.assetId, image);
     dataUrls.set(
       asset.assetId,
-      `data:${asset.mimeType};base64,${bytes.toString("base64")}`,
+      `data:${image.mimeType};base64,${image.bytes.toString("base64")}`,
     );
     await yieldToEventLoop();
   }
@@ -56,9 +57,14 @@ export async function materializeOpenAIRequest(
     [...dataUrls].map(([assetId, value]) => [assetId, value.length] as const),
   );
   const bodyBytes = exactJsonBodyBytes(prepared.payload, exactLengths);
-  assertBodyLimit(bodyBytes, prepared.mediaOccurrenceCount);
+  assertOpenAIRequestBodyLimit(bodyBytes, prepared.mediaOccurrenceCount);
   const payload = deepFreeze(materializePayload(prepared.payload, dataUrls));
-  return Object.freeze({ ...prepared, payload, bodyBytes });
+  return Object.freeze({
+    ...prepared,
+    payload,
+    promptSegments: materializedPromptSegments(prepared.promptSegments, providerImages),
+    bodyBytes,
+  });
 }
 
 export function exactJsonBodyBytes(
@@ -118,19 +124,29 @@ export function exactJsonBodyBytes(
   );
 }
 
-export function imageUserSegment(message: UserMessage): PreparedPromptSegment {
+export function imageUserSegment(
+  message: UserMessage,
+  messageOrdinal: number,
+): PreparedPromptSegment {
   const media = message.attachments!.map(
-    (attachment): PreparedMediaDescriptor =>
-      Object.freeze({
-        assetId: attachment.assetId,
-        label: attachment.label,
-        range: Object.freeze({ ...attachment.range }),
-        mimeType: attachment.mimeType,
-        byteLength: attachment.byteLength,
-        width: attachment.width,
-        height: attachment.height,
-        planningTokens: IMAGE_INPUT_POLICY.planningTokensPerImage,
-      }),
+    (attachment, blockPosition): PreparedMediaOccurrence => {
+      const dimensions = providerImageDimensions(attachment.width, attachment.height);
+      return Object.freeze({
+        asset: Object.freeze({
+          assetId: attachment.assetId,
+          mimeType: attachment.mimeType,
+          byteLength: attachment.byteLength,
+          width: attachment.width,
+          height: attachment.height,
+        }),
+        source: "user_attachment",
+        messageOrdinal,
+        blockPosition,
+        width: dimensions.width,
+        height: dimensions.height,
+        planningTokens: imagePlanningTokens(dimensions.width, dimensions.height),
+      });
+    },
   );
   return Object.freeze({
     kind: "user",
@@ -139,6 +155,35 @@ export function imageUserSegment(message: UserMessage): PreparedPromptSegment {
   });
 }
 
+export function imageToolSegment(
+  message: Extract<AgentMessage, { role: "tool" }>,
+  messageOrdinal: number,
+  normalizedText: string,
+): PreparedPromptSegment {
+  const media = message.content.flatMap((block, blockPosition) => {
+    if (block.type !== "image") {
+      return [];
+    }
+    const dimensions = providerImageDimensions(block.asset.width, block.asset.height);
+    return [
+      Object.freeze<PreparedMediaOccurrence>({
+        asset: Object.freeze({ ...block.asset }),
+        source: "tool_result",
+        messageOrdinal,
+        blockPosition,
+        width: dimensions.width,
+        height: dimensions.height,
+        planningTokens: imagePlanningTokens(dimensions.width, dimensions.height),
+      }),
+    ];
+  });
+  return Object.freeze({
+    kind: "tool",
+    normalizedText,
+    ...(media.length === 0 ? {} : { media: Object.freeze(media) }),
+  });
+}
+
 export function normalizedEndpointPolicy(baseURL: string | undefined): string {
   const url = new URL(baseURL ?? "https://api.openai.com/v1");
   url.username = "";
@@ -200,21 +245,15 @@ function distinctPreparedAssets(
   const assets = new Map<ImageAssetId, ImageAssetRef>();
   for (const segment of segments) {
     for (const media of segment.media ?? []) {
-      const asset = Object.freeze({
-        assetId: media.assetId,
-        mimeType: media.mimeType,
-        byteLength: media.byteLength,
-        width: media.width,
-        height: media.height,
-      });
-      const existing = assets.get(media.assetId);
+      const asset = media.asset;
+      const existing = assets.get(asset.assetId);
       if (
         existing !== undefined &&
         stableJsonStringify(existing) !== stableJsonStringify(asset)
       ) {
-        throw new Error(`Conflicting descriptors for image ${media.assetId}.`);
+        throw new Error(`Conflicting descriptors for image ${asset.assetId}.`);
       }
-      assets.set(media.assetId, asset);
+      assets.set(asset.assetId, asset);
     }
   }
   return assets;
@@ -265,11 +304,10 @@ function countImageMarkers(value: unknown): number {
   return 0;
 }
 
-function dataUrlLength(asset: ImageAssetRef): number {
-  return `data:${asset.mimeType};base64,`.length + 4 * Math.ceil(asset.byteLength / 3);
-}
-
-function assertBodyLimit(bodyBytes: number, imageCount: number): void {
+export function assertOpenAIRequestBodyLimit(
+  bodyBytes: number,
+  imageCount: number,
+): void {
   if (bodyBytes > IMAGE_INPUT_POLICY.maxRequestBodyBytes) {
     throw new ModelRequestMediaAggregateError(
       `Model request is ${bodyBytes} UTF-8 bytes with ${imageCount} images; maximum is ${IMAGE_INPUT_POLICY.maxRequestBodyBytes}.`,
@@ -277,6 +315,37 @@ function assertBodyLimit(bodyBytes: number, imageCount: number): void {
   }
 }
 
+function materializedPromptSegments(
+  segments: readonly PreparedPromptSegment[],
+  images: ReadonlyMap<ImageAssetId, ProviderImage>,
+): readonly PreparedPromptSegment[] {
+  return Object.freeze(
+    segments.map((segment) =>
+      segment.media === undefined
+        ? segment
+        : Object.freeze({
+            ...segment,
+            media: Object.freeze(
+              segment.media.map((media) => {
+                const image = images.get(media.asset.assetId);
+                if (image === undefined) {
+                  throw new Error(
+                    `Image ${media.asset.assetId.slice(0, 12)}… was not materialized.`,
+                  );
+                }
+                return Object.freeze({
+                  ...media,
+                  width: image.width,
+                  height: image.height,
+                  planningTokens: image.planningTokens,
+                });
+              }),
+            ),
+          }),
+    ),
+  );
+}
+
 function yieldToEventLoop(): Promise<void> {
   return new Promise((resolve) => setTimeout(resolve, 0));
 }

package/src/model/openai-responses-mapping.ts

@@ -10,6 +10,10 @@ import type {
   ToolCall,
   UserMessage,
 } from "../agent/types";
+import {
+  toolResultText,
+  validateToolResultContent,
+} from "../agent/tool-result-content";
 import type { RuntimeSessionContext } from "../agent/runtime-session";
 import {
   parseImageAssetId,
@@ -61,11 +65,31 @@ export function toOpenAIResponsesItems(
   }
 
   if (message.role === "tool") {
+    validateToolResultContent(message.content);
+    const hasImage = message.content.some((block) => block.type === "image");
     return [
       {
         type: "function_call_output",
         call_id: message.providerToolCallId,
-        output: message.content,
+        output: hasImage
+          ? message.content.map((block) =>
+              block.type === "text"
+                ? ({ type: "input_text", text: block.text } as const)
+                : ({
+                    type: "input_image",
+                    detail: "auto" as const,
+                    image_url:
+                      options.materializedImages === undefined
+                        ? (imageAssetUrlMarker(
+                            block.asset.assetId,
+                          ) as unknown as string)
+                        : requireMaterializedImage(
+                            options.materializedImages,
+                            block.asset.assetId,
+                          ),
+                  } as const),
+            )
+          : toolResultText(message.content),
       },
     ];
   }

package/src/model/openai-responses-model-client.ts

@@ -8,9 +8,8 @@ import {
   IMAGE_INPUT_POLICY,
   IMAGE_INPUT_POLICY_VERSION,
 } from "../image/image-input-policy";
-import type { InputTokenEstimator } from "./input-token-estimator";
 import type { ModelContextBudget } from "./model-context-profile";
-import { ProviderResponseError } from "./model-client";
+import { ProviderResponseError, validateModelModalities } from "./model-client";
 import type {
   MaterializedModelRequest,
   ModelClient,
@@ -22,9 +21,9 @@ import type {
   PreparedModelRequest,
   PreparedPromptSegment,
 } from "./model-client";
-import { MoonshotInputTokenEstimator } from "./moonshot-input-token-estimator";
 import {
   deepFreeze,
+  imageToolSegment,
   imageUserSegment,
   materializeOpenAIRequest,
   normalizedEndpointPolicy,
@@ -38,11 +37,10 @@ import {
   toOpenAIResponsesTools,
 } from "./openai-responses-mapping";
 import { OpenAIResponsesStreamAccumulator } from "./openai-responses-stream";
-import { responsesPayloadForChatTokenEstimator } from "./openai-responses-token-estimator";
 import type { ReasoningEffortController } from "./reasoning-effort";
 import { sha256, stableJsonStringify } from "./model-request-preflight";
 
-const OPENAI_RESPONSES_SERIALIZATION_VERSION = "openai-responses-v1";
+const OPENAI_RESPONSES_SERIALIZATION_VERSION = "openai-responses-v2";
 const OPENAI_RESPONSES_TIMEOUT_MS = 30 * 60 * 1_000;
 
 export class OpenAIResponsesModelClient implements ModelClient {
@@ -50,8 +48,8 @@ export class OpenAIResponsesModelClient implements ModelClient {
     adapter: "openai-responses",
     serializationVersion: OPENAI_RESPONSES_SERIALIZATION_VERSION,
   });
-  readonly inputTokenEstimator?: InputTokenEstimator;
   readonly inputModalities: readonly ("text" | "image")[];
+  readonly toolResultModalities: readonly ("text" | "image")[];
   readonly reasoningEffort?: ReasoningEffortController;
   private readonly client: OpenAI;
   private readonly preparedRequests = new WeakSet<object>();
@@ -65,14 +63,8 @@ export class OpenAIResponsesModelClient implements ModelClient {
       contextBudget: ModelContextBudget;
       baseURL?: string;
       inputModalities?: readonly ("text" | "image")[];
-      tokenEstimator?: {
-        kind: "moonshot-estimate-token-count-v1";
-        model: string;
-        apiBase: string;
-        apiKey: string;
-        timeoutMs: number;
-        maxRetries: 0;
-      };
+      toolResultModalities?: readonly ("text" | "image")[];
+      profileName?: string;
       model: string;
       providerName?: string;
       reasoningEffort?: ReasoningEffortController;
@@ -84,14 +76,15 @@ export class OpenAIResponsesModelClient implements ModelClient {
     this.provider = options.providerName ?? "responses-compatible";
     this.stream = options.stream ?? true;
     this.reasoningEffort = options.reasoningEffort;
-    this.inputModalities = Object.freeze([...(options.inputModalities ?? ["text"])]);
-    const supportsImages = this.inputModalities.includes("image");
-    if (!this.inputModalities.includes("text")) {
-      throw new Error('OpenAI Responses input modalities must include "text".');
-    }
-    if (supportsImages && options.tokenEstimator === undefined) {
-      throw new Error("Image-capable OpenAI Responses requires a token estimator.");
-    }
+    const modalities = validateModelModalities({
+      profileName: options.profileName,
+      adapter: this.messageProtocol.adapter,
+      inputModalities: options.inputModalities ?? ["text"],
+      toolResultModalities: options.toolResultModalities ?? ["text"],
+      adapterToolResultModalities: ["text", "image"],
+    });
+    this.inputModalities = modalities.inputModalities;
+    this.toolResultModalities = modalities.toolResultModalities;
     this.client = new OpenAI({
       apiKey: options.apiKey,
       baseURL: options.baseURL,
@@ -99,16 +92,6 @@ export class OpenAIResponsesModelClient implements ModelClient {
       maxRetries: 0,
       fetch: options.fetch,
     });
-    if (options.tokenEstimator !== undefined) {
-      this.inputTokenEstimator = new MoonshotInputTokenEstimator({
-        apiKey: options.tokenEstimator.apiKey,
-        baseURL: options.tokenEstimator.apiBase,
-        model: options.tokenEstimator.model,
-        timeoutMs: options.tokenEstimator.timeoutMs,
-        fetch: options.fetch,
-        payloadMapper: responsesPayloadForChatTokenEstimator,
-      });
-    }
   }
 
   prepare(input: ModelRequestInput): PreparedModelRequest {
@@ -139,11 +122,17 @@ export class OpenAIResponsesModelClient implements ModelClient {
     const messageSegments = input.messages.map(
       (message, index): PreparedPromptSegment =>
         message.role === "user" && message.attachments !== undefined
-          ? imageUserSegment(message)
-          : {
-              kind: segmentKind(message.role),
-              normalizedText: stableJsonStringify(itemsByMessage[index]),
-            },
+          ? imageUserSegment(message, index + 1)
+          : message.role === "tool"
+            ? imageToolSegment(
+                message,
+                index + 1,
+                stableJsonStringify(itemsByMessage[index]),
+              )
+            : {
+                kind: segmentKind(message.role),
+                normalizedText: stableJsonStringify(itemsByMessage[index]),
+              },
     );
     const mediaOccurrenceCount = messageSegments.reduce(
       (total, segment) => total + (segment.media?.length ?? 0),
@@ -158,14 +147,12 @@ export class OpenAIResponsesModelClient implements ModelClient {
         requestMaxOutputTokens: this.options.contextBudget.requestMaxOutputTokens,
         stream: this.stream,
         inputModalities: this.inputModalities,
+        toolResultModalities: this.toolResultModalities,
         requestPolicy: { store: false, toolChoice: "auto" },
         imagePolicy: {
           version: IMAGE_INPUT_POLICY_VERSION,
           ...IMAGE_INPUT_POLICY,
         },
-        ...(this.inputTokenEstimator === undefined
-          ? {}
-          : { tokenEstimator: this.inputTokenEstimator.compatibility }),
       }),
     );
     const prepared: PreparedModelRequest = {