tinker-agent 1.0.65 → 1.2.0

package/src/model/moonshot-input-token-estimator.ts

@@ -0,0 +1,107 @@
+import type { InputTokenEstimate, InputTokenEstimator } from "./input-token-estimator";
+import type { MaterializedModelRequest } from "./model-client";
+import { IMAGE_INPUT_POLICY } from "../image/image-input-policy";
+
+export class MoonshotInputTokenEstimator implements InputTokenEstimator {
+  readonly kind = "moonshot-estimate-token-count-v1";
+  readonly compatibility: InputTokenEstimator["compatibility"];
+  private readonly endpoint: string;
+
+  constructor(
+    private readonly options: {
+      apiKey: string;
+      baseURL: string;
+      model: string;
+      timeoutMs: number;
+      fetch?: typeof fetch;
+    },
+  ) {
+    const base = new URL(
+      options.baseURL.endsWith("/") ? options.baseURL : `${options.baseURL}/`,
+    );
+    base.username = "";
+    base.password = "";
+    base.search = "";
+    base.hash = "";
+    const endpoint = new URL("tokenizers/estimate-token-count", base);
+    this.endpoint = endpoint.toString();
+    this.compatibility = Object.freeze({
+      kind: "moonshot-estimate-token-count-v1",
+      coverageVersion: "full-request-v1",
+      model: options.model,
+      endpoint: this.endpoint,
+      timeoutMs: options.timeoutMs,
+      maxRetries: 0,
+    });
+  }
+
+  async estimate(
+    request: MaterializedModelRequest,
+    options: { signal: AbortSignal },
+  ): Promise<InputTokenEstimate> {
+    const chatPayload = requireRecord(request.payload, "materialized chat payload");
+    if (!Array.isArray(chatPayload.messages)) {
+      throw new Error("Materialized request has no token estimator messages.");
+    }
+    const payload = {
+      model: this.options.model,
+      messages: chatPayload.messages,
+      ...(Array.isArray(chatPayload.tools) ? { tools: chatPayload.tools } : {}),
+    };
+    const body = JSON.stringify(payload);
+    const bodyBytes = Buffer.byteLength(body, "utf8");
+    if (bodyBytes > IMAGE_INPUT_POLICY.maxRequestBodyBytes) {
+      throw new Error(
+        `Token estimate request is ${bodyBytes} bytes; maximum is ${IMAGE_INPUT_POLICY.maxRequestBodyBytes}.`,
+      );
+    }
+
+    const controller = new AbortController();
+    const timeout = setTimeout(
+      () => controller.abort(new Error("Token estimate timed out.")),
+      this.options.timeoutMs,
+    );
+    const onAbort = () => controller.abort(options.signal.reason);
+    options.signal.addEventListener("abort", onAbort, { once: true });
+    try {
+      options.signal.throwIfAborted();
+      const response = await (this.options.fetch ?? fetch)(this.endpoint, {
+        method: "POST",
+        headers: {
+          authorization: `Bearer ${this.options.apiKey}`,
+          "content-type": "application/json",
+        },
+        body,
+        signal: controller.signal,
+      });
+      if (!response.ok) {
+        throw new Error(`Token estimate endpoint returned HTTP ${response.status}.`);
+      }
+      const decoded: unknown = await response.json();
+      const root = requireRecord(decoded, "token estimate response");
+      if (root.error !== undefined && root.error !== null) {
+        throw new Error("Token estimate endpoint returned an error response.");
+      }
+      const data = requireRecord(root.data, "token estimate response data");
+      const inputTokens = data.total_tokens;
+      if (!Number.isSafeInteger(inputTokens) || (inputTokens as number) < 0) {
+        throw new Error("Token estimate response total_tokens is invalid.");
+      }
+      return Object.freeze({
+        inputTokens: inputTokens as number,
+        source: "provider_estimated",
+        coverage: "full_request",
+      });
+    } finally {
+      clearTimeout(timeout);
+      options.signal.removeEventListener("abort", onAbort);
+    }
+  }
+}
+
+function requireRecord(value: unknown, name: string): Record<string, unknown> {
+  if (typeof value !== "object" || value === null || Array.isArray(value)) {
+    throw new Error(`${name} must be an object.`);
+  }
+  return value as Record<string, unknown>;
+}

package/src/model/openai-chat-mapping.ts

@@ -1,4 +1,9 @@
-import type { AgentMessage, IterationIdentity, ToolCall } from "../agent/types";
+import type {
+  AgentMessage,
+  IterationIdentity,
+  ToolCall,
+  UserMessage,
+} from "../agent/types";
 import type { RuntimeSessionContext } from "../agent/runtime-session";
 import type { ModelRequestOutput, ModelUsage } from "./model-client";
 import type { ToolDefinition } from "../tools/types";
@@ -6,8 +11,20 @@ import type {
   ChatCompletionAssistantMessageParam,
   ChatCompletionMessageFunctionToolCall,
   ChatCompletionMessageParam,
+  ChatCompletionContentPart,
   ChatCompletionTool,
 } from "openai/resources/chat/completions";
+import {
+  parseImageAssetId,
+  validateUserMessage,
+  type ImageAssetId,
+} from "../image/image-types";
+
+const IMAGE_ASSET_URL_MARKER = Symbol("tinker.image-asset-url-marker");
+
+export type ImageAssetUrlMarker = {
+  readonly [IMAGE_ASSET_URL_MARKER]: ImageAssetId;
+};
 
 type DeepSeekAssistantMessageParam = ChatCompletionAssistantMessageParam & {
   reasoning_content?: string | null;
@@ -15,6 +32,7 @@ type DeepSeekAssistantMessageParam = ChatCompletionAssistantMessageParam & {
 
 export type OpenAIChatMessageMappingOptions = {
   includeReasoningContent?: boolean;
+  materializedImages?: ReadonlyMap<ImageAssetId, string>;
 };
 
 export function toOpenAIChatMessages(
@@ -47,10 +65,68 @@ export function toOpenAIChatMessages(
       };
     }
 
+    if (message.role === "user") {
+      return {
+        role: "user",
+        content: toOpenAIUserContent(message, options.materializedImages),
+      };
+    }
+
     return message;
   });
 }
 
+export function toOpenAIUserContent(
+  message: UserMessage,
+  materializedImages?: ReadonlyMap<ImageAssetId, string>,
+): string | ChatCompletionContentPart[] {
+  validateUserMessage(message);
+  const attachments = message.attachments;
+  if (attachments === undefined) {
+    return message.content;
+  }
+  return [
+    ...attachments.flatMap((attachment): ChatCompletionContentPart[] => [
+      { type: "text", text: `<image name=${attachment.label}>` },
+      {
+        type: "image_url",
+        image_url: {
+          url:
+            materializedImages === undefined
+              ? (imageAssetUrlMarker(attachment.assetId) as unknown as string)
+              : requireMaterializedImage(materializedImages, attachment.assetId),
+        },
+      },
+      { type: "text", text: "</image>" },
+    ]),
+    { type: "text", text: message.content },
+  ];
+}
+
+export function imageAssetUrlMarker(assetId: ImageAssetId): ImageAssetUrlMarker {
+  parseImageAssetId(assetId);
+  return Object.freeze({ [IMAGE_ASSET_URL_MARKER]: assetId });
+}
+
+export function parseImageAssetUrlMarker(value: unknown): ImageAssetId | undefined {
+  if (typeof value !== "object" || value === null) {
+    return undefined;
+  }
+  const assetId = (value as Partial<ImageAssetUrlMarker>)[IMAGE_ASSET_URL_MARKER];
+  return typeof assetId === "string" ? parseImageAssetId(assetId) : undefined;
+}
+
+function requireMaterializedImage(
+  materializedImages: ReadonlyMap<ImageAssetId, string>,
+  assetId: ImageAssetId,
+): string {
+  const dataUrl = materializedImages.get(assetId);
+  if (dataUrl === undefined) {
+    throw new Error(`Image asset ${assetId.slice(0, 12)}… was not materialized.`);
+  }
+  return dataUrl;
+}
+
 export function toOpenAIChatTools(
   tools: readonly ToolDefinition[],
 ): ChatCompletionTool[] {