tinker-agent 1.11.0 → 2.0.0

package/src/image/image-input-policy.ts

@@ -4,11 +4,62 @@ export const IMAGE_INPUT_POLICY = Object.freeze({
   maxBytesPerImage: 20 * 1024 * 1024,
   maxImagesPerMessage: 8,
   maxImagesPerRequest: 8,
+  maxProviderLongEdge: 2048,
   maxLongEdge: 4096,
   maxPixels: 8_847_360,
   maxRequestBodyBytes: 90_000_000,
-  planningTokensPerImage: 2048,
+  allowUpscale: false,
+  resizePolicy: "sharp-auto-orient-lanczos3-round-v1",
+  outputEncodingPolicy: "preserve-input-format-fixed-encoding-v1",
+  imageTokenBuckets: Object.freeze([
+    Object.freeze({ maxLongEdge: 512, planningTokens: 384 }),
+    Object.freeze({ maxLongEdge: 1024, planningTokens: 1408 }),
+    Object.freeze({ maxLongEdge: 1536, planningTokens: 3072 }),
+    Object.freeze({ maxLongEdge: 2048, planningTokens: 5504 }),
+  ] as const),
+  imageTokensUseTextCorrectionFactor: false,
   retryPolicy: "none",
 } as const);
 
-export const IMAGE_INPUT_POLICY_VERSION = "image-input-policy-v1" as const;
+export const IMAGE_INPUT_POLICY_VERSION = "image-input-policy-v2" as const;
+
+export function providerImageDimensions(
+  width: number,
+  height: number,
+): { readonly width: number; readonly height: number } {
+  requireDimension(width, "width");
+  requireDimension(height, "height");
+  const longEdge = Math.max(width, height);
+  if (longEdge <= IMAGE_INPUT_POLICY.maxProviderLongEdge) {
+    return Object.freeze({ width, height });
+  }
+  const scale = IMAGE_INPUT_POLICY.maxProviderLongEdge / longEdge;
+  let targetWidth = Math.max(1, Math.round(width * scale));
+  let targetHeight = Math.max(1, Math.round(height * scale));
+  if (targetWidth > IMAGE_INPUT_POLICY.maxProviderLongEdge) {
+    targetWidth = IMAGE_INPUT_POLICY.maxProviderLongEdge;
+  }
+  if (targetHeight > IMAGE_INPUT_POLICY.maxProviderLongEdge) {
+    targetHeight = IMAGE_INPUT_POLICY.maxProviderLongEdge;
+  }
+  return Object.freeze({ width: targetWidth, height: targetHeight });
+}
+
+export function imagePlanningTokens(width: number, height: number): number {
+  requireDimension(width, "width");
+  requireDimension(height, "height");
+  const longEdge = Math.max(width, height);
+  const bucket = IMAGE_INPUT_POLICY.imageTokenBuckets.find(
+    (candidate) => longEdge <= candidate.maxLongEdge,
+  );
+  if (bucket === undefined) {
+    throw new Error("Materialized image exceeds the provider image size policy.");
+  }
+  return bucket.planningTokens;
+}
+
+function requireDimension(value: number, name: string): void {
+  if (!Number.isSafeInteger(value) || value < 1) {
+    throw new Error(`Image ${name} must be a positive safe integer.`);
+  }
+}

package/src/image/image-probe.ts

@@ -1,5 +1,6 @@
 import sharp from "sharp";
 import { IMAGE_INPUT_POLICY } from "./image-input-policy";
+import { orientedDimensions } from "./provider-image";
 import {
   imageAssetIdForBytes,
   type ImageAssetRef,
@@ -59,8 +60,13 @@ export async function probeImageBytes(
       `Image container and decoder disagree on format (${container.mimeType} vs ${mimeType}).`,
     );
   }
-  const width = requireDimension(metadata.width, "width");
-  const height = requireDimension(metadata.height, "height");
+  const decodedWidth = requireDimension(metadata.width, "width");
+  const decodedHeight = requireDimension(metadata.height, "height");
+  const { width, height } = orientedDimensions(
+    decodedWidth,
+    decodedHeight,
+    metadata.orientation,
+  );
   const decoderAnimated =
     (metadata.pages ?? 1) > 1 || metadata.pageHeight !== undefined;
   if (container.animated || decoderAnimated) {

package/src/image/provider-image.ts

@@ -0,0 +1,99 @@
+import sharp from "sharp";
+import {
+  IMAGE_INPUT_POLICY,
+  imagePlanningTokens,
+  providerImageDimensions,
+} from "./image-input-policy";
+import type { ImageMimeType } from "./image-types";
+
+export type ProviderImage = {
+  readonly bytes: Buffer;
+  readonly mimeType: ImageMimeType;
+  readonly width: number;
+  readonly height: number;
+  readonly planningTokens: number;
+};
+
+export async function materializeProviderImage(
+  bytes: Buffer,
+  mimeType: ImageMimeType,
+): Promise<ProviderImage> {
+  const input = sharp(bytes, {
+    failOn: "warning",
+    limitInputPixels: IMAGE_INPUT_POLICY.maxPixels,
+    unlimited: false,
+    sequentialRead: true,
+  });
+  const metadata = await input.metadata();
+  const sourceWidth = requireDimension(metadata.width, "width");
+  const sourceHeight = requireDimension(metadata.height, "height");
+  const oriented = orientedDimensions(sourceWidth, sourceHeight, metadata.orientation);
+  const target = providerImageDimensions(oriented.width, oriented.height);
+  const requiresOrientation =
+    metadata.orientation !== undefined && metadata.orientation !== 1;
+  const requiresResize =
+    target.width !== oriented.width || target.height !== oriented.height;
+
+  let outputBytes = bytes;
+  if (requiresOrientation || requiresResize) {
+    let pipeline = sharp(bytes, {
+      failOn: "warning",
+      limitInputPixels: IMAGE_INPUT_POLICY.maxPixels,
+      unlimited: false,
+      sequentialRead: true,
+    }).rotate();
+    if (requiresResize) {
+      pipeline = pipeline.resize(target.width, target.height, {
+        fit: "fill",
+        kernel: sharp.kernel.lanczos3,
+        withoutEnlargement: true,
+      });
+    }
+    outputBytes = await encodeInOriginalFormat(pipeline, mimeType);
+  }
+
+  const outputMetadata = await sharp(outputBytes).metadata();
+  const width = requireDimension(outputMetadata.width, "width");
+  const height = requireDimension(outputMetadata.height, "height");
+  if (Math.max(width, height) > IMAGE_INPUT_POLICY.maxProviderLongEdge) {
+    throw new Error("Materialized image exceeds the provider image size policy.");
+  }
+  return Object.freeze({
+    bytes: outputBytes,
+    mimeType,
+    width,
+    height,
+    planningTokens: imagePlanningTokens(width, height),
+  });
+}
+
+export function orientedDimensions(
+  width: number,
+  height: number,
+  orientation: number | undefined,
+): { readonly width: number; readonly height: number } {
+  return orientation !== undefined && orientation >= 5 && orientation <= 8
+    ? Object.freeze({ width: height, height: width })
+    : Object.freeze({ width, height });
+}
+
+async function encodeInOriginalFormat(
+  pipeline: ReturnType<typeof sharp>,
+  mimeType: ImageMimeType,
+): Promise<Buffer> {
+  switch (mimeType) {
+    case "image/png":
+      return pipeline.png({ compressionLevel: 6, adaptiveFiltering: false }).toBuffer();
+    case "image/jpeg":
+      return pipeline.jpeg({ quality: 80, chromaSubsampling: "4:2:0" }).toBuffer();
+    case "image/webp":
+      return pipeline.webp({ quality: 80, effort: 4 }).toBuffer();
+  }
+}
+
+function requireDimension(value: number | undefined, name: string): number {
+  if (!Number.isSafeInteger(value) || value === undefined || value < 1) {
+    throw new Error(`Decoded image ${name} is invalid.`);
+  }
+  return value;
+}

package/src/model/fake-model-client.ts

@@ -2,9 +2,13 @@ import { createHash } from "node:crypto";
 import { appendFile } from "node:fs/promises";
 import type { AgentMessage, AssistantMessage } from "../agent/types";
 import { cancellationError } from "../agent/turn-cancellation";
-import { IMAGE_INPUT_POLICY } from "../image/image-input-policy";
+import {
+  IMAGE_INPUT_POLICY,
+  imagePlanningTokens,
+  providerImageDimensions,
+} from "../image/image-input-policy";
 import type { ImageAssetId, ImageAssetRef } from "../image/image-types";
-import type { InputTokenEstimator } from "./input-token-estimator";
+import { materializeProviderImage } from "../image/provider-image";
 import type { ModelContextBudget } from "./model-context-profile";
 import type { ReasoningEffortController } from "./reasoning-effort";
 import type {
@@ -24,7 +28,6 @@ import { estimatePromptSegments } from "./token-estimator";
 
 export class FakeModelClient implements ModelClient {
   readonly inputModalities: readonly ("text" | "image")[];
-  readonly inputTokenEstimator?: InputTokenEstimator;
   readonly reasoningEffort?: ReasoningEffortController;
   readonly messageProtocol: ModelMessageProtocol = Object.freeze({
     adapter: "fake",
@@ -42,13 +45,6 @@ export class FakeModelClient implements ModelClient {
       inputModalities?: readonly ("text" | "image")[];
       reasoningEffort?: ReasoningEffortController;
       requestLogPath?: string;
-      tokenEstimator?: {
-        kind: "moonshot-estimate-token-count-v1";
-        model: string;
-        apiBase: string;
-        timeoutMs: number;
-        maxRetries: 0;
-      };
     },
   ) {
     this.reasoningEffort = options.reasoningEffort;
@@ -58,38 +54,6 @@ export class FakeModelClient implements ModelClient {
     if (!this.inputModalities.includes("text")) {
       throw new Error('Fake model input modalities must include "text".');
     }
-    if (
-      this.inputModalities.includes("image") &&
-      options.tokenEstimator === undefined
-    ) {
-      throw new Error("Image-capable fake model requires a token estimator.");
-    }
-    if (options.tokenEstimator !== undefined) {
-      const estimator = options.tokenEstimator;
-      const endpoint = tokenEstimatorEndpoint(estimator.apiBase);
-      this.inputTokenEstimator = Object.freeze({
-        kind: estimator.kind,
-        compatibility: Object.freeze({
-          kind: estimator.kind,
-          coverageVersion: "full-request-v1",
-          model: estimator.model,
-          endpoint,
-          timeoutMs: estimator.timeoutMs,
-          maxRetries: estimator.maxRetries,
-        }),
-        async estimate(
-          request: MaterializedModelRequest,
-          estimateOptions: { signal: AbortSignal },
-        ) {
-          estimateOptions.signal.throwIfAborted();
-          return Object.freeze({
-            inputTokens: estimatePromptSegments(request.promptSegments).totalTokens,
-            source: "provider_estimated" as const,
-            coverage: "full_request" as const,
-          });
-        },
-      });
-    }
   }
 
   prepare(input: ModelRequestInput): PreparedModelRequest {
@@ -113,9 +77,6 @@ export class FakeModelClient implements ModelClient {
         model: this.options.model,
         requestMaxOutputTokens: this.options.contextBudget.requestMaxOutputTokens,
         inputModalities: this.inputModalities,
-        ...(this.inputTokenEstimator === undefined
-          ? {}
-          : { tokenEstimator: this.inputTokenEstimator.compatibility }),
       }),
     );
     const prepared: PreparedModelRequest = Object.freeze({
@@ -167,6 +128,9 @@ export class FakeModelClient implements ModelClient {
     const materializedAssets: Array<{
       readonly assetId: ImageAssetId;
       readonly byteLength: number;
+      readonly width: number;
+      readonly height: number;
+      readonly planningTokens: number;
       readonly bytesSha256: string;
     }> = [];
     for (const asset of assets.values()) {
@@ -174,11 +138,15 @@ export class FakeModelClient implements ModelClient {
       const bytes = await options.assetStore.readVerified(asset, {
         signal: options.signal,
       });
+      const image = await materializeProviderImage(bytes, asset.mimeType);
       materializedAssets.push(
         Object.freeze({
           assetId: asset.assetId,
-          byteLength: bytes.byteLength,
-          bytesSha256: createHash("sha256").update(bytes).digest("hex"),
+          byteLength: image.bytes.byteLength,
+          width: image.width,
+          height: image.height,
+          planningTokens: image.planningTokens,
+          bytesSha256: createHash("sha256").update(image.bytes).digest("hex"),
         }),
       );
     }
@@ -191,6 +159,10 @@ export class FakeModelClient implements ModelClient {
     const materialized = Object.freeze({
       ...prepared,
       payload,
+      promptSegments: materializedFakePromptSegments(
+        prepared.promptSegments,
+        materializedAssets,
+      ),
       bodyBytes: Buffer.byteLength(stableJsonStringify(payload), "utf8"),
     });
     this.preparedInputs.set(materialized, input);
@@ -1455,19 +1427,21 @@ function lastMessageIndex(
 
 function toPromptSegment(message: AgentMessage): PreparedPromptSegment {
   if (message.role === "user" && message.attachments !== undefined) {
-    const media = message.attachments.map(
-      (attachment): PreparedMediaDescriptor =>
-        Object.freeze({
-          assetId: attachment.assetId,
-          label: attachment.label,
-          range: Object.freeze({ ...attachment.range }),
-          mimeType: attachment.mimeType,
-          byteLength: attachment.byteLength,
-          width: attachment.width,
-          height: attachment.height,
-          planningTokens: IMAGE_INPUT_POLICY.planningTokensPerImage,
-        }),
-    );
+    const media = message.attachments.map((attachment): PreparedMediaDescriptor => {
+      const dimensions = providerImageDimensions(attachment.width, attachment.height);
+      return Object.freeze({
+        assetId: attachment.assetId,
+        label: attachment.label,
+        range: Object.freeze({ ...attachment.range }),
+        mimeType: attachment.mimeType,
+        byteLength: attachment.byteLength,
+        sourceWidth: attachment.width,
+        sourceHeight: attachment.height,
+        width: dimensions.width,
+        height: dimensions.height,
+        planningTokens: imagePlanningTokens(dimensions.width, dimensions.height),
+      });
+    });
     return Object.freeze({
       kind: "user",
       normalizedText: message.content,
@@ -1495,8 +1469,8 @@ function distinctPreparedAssets(
         assetId: media.assetId,
         mimeType: media.mimeType,
         byteLength: media.byteLength,
-        width: media.width,
-        height: media.height,
+        width: media.sourceWidth,
+        height: media.sourceHeight,
       });
       const existing = assets.get(media.assetId);
       if (
@@ -1511,13 +1485,39 @@ function distinctPreparedAssets(
   return assets;
 }
 
-function tokenEstimatorEndpoint(apiBase: string): string {
-  const base = new URL(apiBase.endsWith("/") ? apiBase : `${apiBase}/`);
-  base.username = "";
-  base.password = "";
-  base.search = "";
-  base.hash = "";
-  return new URL("tokenizers/estimate-token-count", base).toString();
+function materializedFakePromptSegments(
+  segments: readonly PreparedPromptSegment[],
+  images: readonly {
+    assetId: ImageAssetId;
+    width: number;
+    height: number;
+    planningTokens: number;
+  }[],
+): readonly PreparedPromptSegment[] {
+  const byId = new Map(images.map((image) => [image.assetId, image] as const));
+  return Object.freeze(
+    segments.map((segment) =>
+      segment.media === undefined
+        ? segment
+        : Object.freeze({
+            ...segment,
+            media: Object.freeze(
+              segment.media.map((media) => {
+                const image = byId.get(media.assetId);
+                if (image === undefined) {
+                  throw new Error(`Fake image ${media.assetId} was not materialized.`);
+                }
+                return Object.freeze({
+                  ...media,
+                  width: image.width,
+                  height: image.height,
+                  planningTokens: image.planningTokens,
+                });
+              }),
+            ),
+          }),
+    ),
+  );
 }
 
 function recallMarker(

package/src/model/model-client.ts

@@ -3,12 +3,10 @@ import type { RuntimeSessionContext } from "../agent/runtime-session";
 import type { ToolDefinition } from "../tools/types";
 import type { ImageAssetStore } from "../image/image-asset-store";
 import type { CodePointRange, ImageAssetId, ImageMimeType } from "../image/image-types";
-import type { InputTokenEstimator } from "./input-token-estimator";
 import type { ReasoningEffortController } from "./reasoning-effort";
 
 export interface ModelClient {
   readonly messageProtocol: ModelMessageProtocol;
-  readonly inputTokenEstimator?: InputTokenEstimator;
   readonly inputModalities?: readonly ("text" | "image")[];
   readonly reasoningEffort?: ReasoningEffortController;
   prepare(input: ModelRequestInput): PreparedModelRequest;
@@ -76,6 +74,8 @@ export type PreparedMediaDescriptor = {
   range: CodePointRange;
   mimeType: ImageMimeType;
   byteLength: number;
+  sourceWidth: number;
+  sourceHeight: number;
   width: number;
   height: number;
   planningTokens: number;

package/src/model/model-request-preflight.ts

@@ -3,7 +3,6 @@ import type { ModelContextBudget } from "./model-context-profile";
 
 export type ContextUsageSource =
   | "estimated_full"
-  | "provider_estimated"
   | "provider_measured"
   | "measured_plus_estimated_delta";
 

package/src/model/openai-chat-model-client.ts

@@ -9,7 +9,6 @@ import {
   IMAGE_INPUT_POLICY_VERSION,
 } from "../image/image-input-policy";
 import type { ModelContextBudget } from "./model-context-profile";
-import type { InputTokenEstimator } from "./input-token-estimator";
 import { ProviderResponseError } from "./model-client";
 import type {
   MaterializedModelRequest,
@@ -36,7 +35,6 @@ import {
   sanitizedProviderError,
   segmentKind,
 } from "./openai-model-utils";
-import { MoonshotInputTokenEstimator } from "./moonshot-input-token-estimator";
 import type { ReasoningEffortController } from "./reasoning-effort";
 import { sha256, stableJsonStringify } from "./model-request-preflight";
 
@@ -48,7 +46,6 @@ export class OpenAIChatModelClient implements ModelClient {
     adapter: "openai-chat",
     serializationVersion: OPENAI_CHAT_SERIALIZATION_VERSION,
   });
-  readonly inputTokenEstimator?: InputTokenEstimator;
   readonly reasoningEffort?: ReasoningEffortController;
   private readonly client: OpenAI;
   private readonly preparedRequests = new WeakSet<object>();
@@ -64,14 +61,6 @@ export class OpenAIChatModelClient implements ModelClient {
       baseURL?: string;
       includeReasoningContent?: boolean;
       inputModalities?: readonly ("text" | "image")[];
-      tokenEstimator?: {
-        kind: "moonshot-estimate-token-count-v1";
-        model: string;
-        apiBase: string;
-        apiKey: string;
-        timeoutMs: number;
-        maxRetries: 0;
-      };
       model: string;
       providerName?: string;
       reasoningEffort?: ReasoningEffortController;
@@ -84,13 +73,9 @@ export class OpenAIChatModelClient implements ModelClient {
     this.stream = options.stream ?? true;
     this.reasoningEffort = options.reasoningEffort;
     this.inputModalities = Object.freeze([...(options.inputModalities ?? ["text"])]);
-    const supportsImages = this.inputModalities.includes("image");
     if (!this.inputModalities.includes("text")) {
       throw new Error('OpenAI chat input modalities must include "text".');
     }
-    if (supportsImages && options.tokenEstimator === undefined) {
-      throw new Error("Image-capable OpenAI chat requires a token estimator.");
-    }
     this.client = new OpenAI({
       apiKey: options.apiKey,
       baseURL: options.baseURL,
@@ -100,15 +85,6 @@ export class OpenAIChatModelClient implements ModelClient {
       maxRetries: 0,
       fetch: options.fetch,
     });
-    if (options.tokenEstimator !== undefined) {
-      this.inputTokenEstimator = new MoonshotInputTokenEstimator({
-        apiKey: options.tokenEstimator.apiKey,
-        baseURL: options.tokenEstimator.apiBase,
-        model: options.tokenEstimator.model,
-        timeoutMs: options.tokenEstimator.timeoutMs,
-        fetch: options.fetch,
-      });
-    }
   }
 
   prepare(input: ModelRequestInput): PreparedModelRequest {
@@ -164,9 +140,6 @@ export class OpenAIChatModelClient implements ModelClient {
           version: IMAGE_INPUT_POLICY_VERSION,
           ...IMAGE_INPUT_POLICY,
         },
-        ...(this.inputTokenEstimator === undefined
-          ? {}
-          : { tokenEstimator: this.inputTokenEstimator.compatibility }),
       }),
     );
     const prepared: PreparedModelRequest = {
@@ -299,4 +272,7 @@ export class OpenAIChatModelClient implements ModelClient {
   }
 }
 
-export { exactJsonBodyBytes } from "./openai-model-utils";
+export {
+  assertOpenAIRequestBodyLimit,
+  exactJsonBodyBytes,
+} from "./openai-model-utils";

package/src/model/openai-model-utils.ts

@@ -1,7 +1,12 @@
 import OpenAI from "openai";
 import type { UserMessage } from "../agent/types";
-import { IMAGE_INPUT_POLICY } from "../image/image-input-policy";
+import {
+  IMAGE_INPUT_POLICY,
+  imagePlanningTokens,
+  providerImageDimensions,
+} from "../image/image-input-policy";
 import type { ImageAssetId, ImageAssetRef } from "../image/image-types";
+import { materializeProviderImage, type ProviderImage } from "../image/provider-image";
 import {
   ModelRequestMediaAggregateError,
   ProviderResponseError,
@@ -28,26 +33,22 @@ export async function materializeOpenAIRequest(
   }
 
   const assets = distinctPreparedAssets(prepared.promptSegments);
-  const lowerLengths = new Map<ImageAssetId, number>();
-  for (const asset of assets.values()) {
-    lowerLengths.set(asset.assetId, dataUrlLength(asset));
-  }
   const markerCount = countImageMarkers(prepared.payload);
   if (markerCount !== prepared.mediaOccurrenceCount) {
     throw new Error("Prepared image marker count does not match media descriptors.");
   }
-  const lowerBodyBytes = exactJsonBodyBytes(prepared.payload, lowerLengths);
-  assertBodyLimit(lowerBodyBytes, prepared.mediaOccurrenceCount);
-
   const dataUrls = new Map<ImageAssetId, string>();
+  const providerImages = new Map<ImageAssetId, ProviderImage>();
   for (const asset of assets.values()) {
     options.signal.throwIfAborted();
     const bytes = await options.assetStore.readVerified(asset, {
       signal: options.signal,
     });
+    const image = await materializeProviderImage(bytes, asset.mimeType);
+    providerImages.set(asset.assetId, image);
     dataUrls.set(
       asset.assetId,
-      `data:${asset.mimeType};base64,${bytes.toString("base64")}`,
+      `data:${image.mimeType};base64,${image.bytes.toString("base64")}`,
     );
     await yieldToEventLoop();
   }
@@ -56,9 +57,14 @@ export async function materializeOpenAIRequest(
     [...dataUrls].map(([assetId, value]) => [assetId, value.length] as const),
   );
   const bodyBytes = exactJsonBodyBytes(prepared.payload, exactLengths);
-  assertBodyLimit(bodyBytes, prepared.mediaOccurrenceCount);
+  assertOpenAIRequestBodyLimit(bodyBytes, prepared.mediaOccurrenceCount);
   const payload = deepFreeze(materializePayload(prepared.payload, dataUrls));
-  return Object.freeze({ ...prepared, payload, bodyBytes });
+  return Object.freeze({
+    ...prepared,
+    payload,
+    promptSegments: materializedPromptSegments(prepared.promptSegments, providerImages),
+    bodyBytes,
+  });
 }
 
 export function exactJsonBodyBytes(
@@ -119,19 +125,21 @@ export function exactJsonBodyBytes(
 }
 
 export function imageUserSegment(message: UserMessage): PreparedPromptSegment {
-  const media = message.attachments!.map(
-    (attachment): PreparedMediaDescriptor =>
-      Object.freeze({
-        assetId: attachment.assetId,
-        label: attachment.label,
-        range: Object.freeze({ ...attachment.range }),
-        mimeType: attachment.mimeType,
-        byteLength: attachment.byteLength,
-        width: attachment.width,
-        height: attachment.height,
-        planningTokens: IMAGE_INPUT_POLICY.planningTokensPerImage,
-      }),
-  );
+  const media = message.attachments!.map((attachment): PreparedMediaDescriptor => {
+    const dimensions = providerImageDimensions(attachment.width, attachment.height);
+    return Object.freeze({
+      assetId: attachment.assetId,
+      label: attachment.label,
+      range: Object.freeze({ ...attachment.range }),
+      mimeType: attachment.mimeType,
+      byteLength: attachment.byteLength,
+      sourceWidth: attachment.width,
+      sourceHeight: attachment.height,
+      width: dimensions.width,
+      height: dimensions.height,
+      planningTokens: imagePlanningTokens(dimensions.width, dimensions.height),
+    });
+  });
   return Object.freeze({
     kind: "user",
     normalizedText: message.content,
@@ -204,8 +212,8 @@ function distinctPreparedAssets(
         assetId: media.assetId,
         mimeType: media.mimeType,
         byteLength: media.byteLength,
-        width: media.width,
-        height: media.height,
+        width: media.sourceWidth,
+        height: media.sourceHeight,
       });
       const existing = assets.get(media.assetId);
       if (
@@ -265,11 +273,10 @@ function countImageMarkers(value: unknown): number {
   return 0;
 }
 
-function dataUrlLength(asset: ImageAssetRef): number {
-  return `data:${asset.mimeType};base64,`.length + 4 * Math.ceil(asset.byteLength / 3);
-}
-
-function assertBodyLimit(bodyBytes: number, imageCount: number): void {
+export function assertOpenAIRequestBodyLimit(
+  bodyBytes: number,
+  imageCount: number,
+): void {
   if (bodyBytes > IMAGE_INPUT_POLICY.maxRequestBodyBytes) {
     throw new ModelRequestMediaAggregateError(
       `Model request is ${bodyBytes} UTF-8 bytes with ${imageCount} images; maximum is ${IMAGE_INPUT_POLICY.maxRequestBodyBytes}.`,
@@ -277,6 +284,37 @@ function assertBodyLimit(bodyBytes: number, imageCount: number): void {
   }
 }
 
+function materializedPromptSegments(
+  segments: readonly PreparedPromptSegment[],
+  images: ReadonlyMap<ImageAssetId, ProviderImage>,
+): readonly PreparedPromptSegment[] {
+  return Object.freeze(
+    segments.map((segment) =>
+      segment.media === undefined
+        ? segment
+        : Object.freeze({
+            ...segment,
+            media: Object.freeze(
+              segment.media.map((media) => {
+                const image = images.get(media.assetId);
+                if (image === undefined) {
+                  throw new Error(
+                    `Image ${media.assetId.slice(0, 12)}… was not materialized.`,
+                  );
+                }
+                return Object.freeze({
+                  ...media,
+                  width: image.width,
+                  height: image.height,
+                  planningTokens: image.planningTokens,
+                });
+              }),
+            ),
+          }),
+    ),
+  );
+}
+
 function yieldToEventLoop(): Promise<void> {
   return new Promise((resolve) => setTimeout(resolve, 0));
 }