tinker-agent 2.0.0 → 2.2.0

package/src/model/openai-model-utils.ts

@@ -1,5 +1,5 @@
 import OpenAI from "openai";
-import type { UserMessage } from "../agent/types";
+import type { AgentMessage, UserMessage } from "../agent/types";
 import {
   IMAGE_INPUT_POLICY,
   imagePlanningTokens,
@@ -13,7 +13,7 @@ import {
   type MaterializedModelRequest,
   type ModelMaterializeOptions,
   type ModelRequestInput,
-  type PreparedMediaDescriptor,
+  type PreparedMediaOccurrence,
   type PreparedModelRequest,
   type PreparedPromptSegment,
   type ProviderResponseErrorCode,
@@ -124,22 +124,30 @@ export function exactJsonBodyBytes(
   );
 }
 
-export function imageUserSegment(message: UserMessage): PreparedPromptSegment {
-  const media = message.attachments!.map((attachment): PreparedMediaDescriptor => {
-    const dimensions = providerImageDimensions(attachment.width, attachment.height);
-    return Object.freeze({
-      assetId: attachment.assetId,
-      label: attachment.label,
-      range: Object.freeze({ ...attachment.range }),
-      mimeType: attachment.mimeType,
-      byteLength: attachment.byteLength,
-      sourceWidth: attachment.width,
-      sourceHeight: attachment.height,
-      width: dimensions.width,
-      height: dimensions.height,
-      planningTokens: imagePlanningTokens(dimensions.width, dimensions.height),
-    });
-  });
+export function imageUserSegment(
+  message: UserMessage,
+  messageOrdinal: number,
+): PreparedPromptSegment {
+  const media = message.attachments!.map(
+    (attachment, blockPosition): PreparedMediaOccurrence => {
+      const dimensions = providerImageDimensions(attachment.width, attachment.height);
+      return Object.freeze({
+        asset: Object.freeze({
+          assetId: attachment.assetId,
+          mimeType: attachment.mimeType,
+          byteLength: attachment.byteLength,
+          width: attachment.width,
+          height: attachment.height,
+        }),
+        source: "user_attachment",
+        messageOrdinal,
+        blockPosition,
+        width: dimensions.width,
+        height: dimensions.height,
+        planningTokens: imagePlanningTokens(dimensions.width, dimensions.height),
+      });
+    },
+  );
   return Object.freeze({
     kind: "user",
     normalizedText: message.content,
@@ -147,6 +155,35 @@ export function imageUserSegment(message: UserMessage): PreparedPromptSegment {
   });
 }
 
+export function imageToolSegment(
+  message: Extract<AgentMessage, { role: "tool" }>,
+  messageOrdinal: number,
+  normalizedText: string,
+): PreparedPromptSegment {
+  const media = message.content.flatMap((block, blockPosition) => {
+    if (block.type !== "image") {
+      return [];
+    }
+    const dimensions = providerImageDimensions(block.asset.width, block.asset.height);
+    return [
+      Object.freeze<PreparedMediaOccurrence>({
+        asset: Object.freeze({ ...block.asset }),
+        source: "tool_result",
+        messageOrdinal,
+        blockPosition,
+        width: dimensions.width,
+        height: dimensions.height,
+        planningTokens: imagePlanningTokens(dimensions.width, dimensions.height),
+      }),
+    ];
+  });
+  return Object.freeze({
+    kind: "tool",
+    normalizedText,
+    ...(media.length === 0 ? {} : { media: Object.freeze(media) }),
+  });
+}
+
 export function normalizedEndpointPolicy(baseURL: string | undefined): string {
   const url = new URL(baseURL ?? "https://api.openai.com/v1");
   url.username = "";
@@ -208,21 +245,15 @@ function distinctPreparedAssets(
   const assets = new Map<ImageAssetId, ImageAssetRef>();
   for (const segment of segments) {
     for (const media of segment.media ?? []) {
-      const asset = Object.freeze({
-        assetId: media.assetId,
-        mimeType: media.mimeType,
-        byteLength: media.byteLength,
-        width: media.sourceWidth,
-        height: media.sourceHeight,
-      });
-      const existing = assets.get(media.assetId);
+      const asset = media.asset;
+      const existing = assets.get(asset.assetId);
       if (
         existing !== undefined &&
         stableJsonStringify(existing) !== stableJsonStringify(asset)
       ) {
-        throw new Error(`Conflicting descriptors for image ${media.assetId}.`);
+        throw new Error(`Conflicting descriptors for image ${asset.assetId}.`);
       }
-      assets.set(media.assetId, asset);
+      assets.set(asset.assetId, asset);
     }
   }
   return assets;
@@ -296,10 +327,10 @@ function materializedPromptSegments(
             ...segment,
             media: Object.freeze(
               segment.media.map((media) => {
-                const image = images.get(media.assetId);
+                const image = images.get(media.asset.assetId);
                 if (image === undefined) {
                   throw new Error(
-                    `Image ${media.assetId.slice(0, 12)}… was not materialized.`,
+                    `Image ${media.asset.assetId.slice(0, 12)}… was not materialized.`,
                   );
                 }
                 return Object.freeze({

package/src/model/openai-responses-mapping.ts

@@ -10,6 +10,10 @@ import type {
   ToolCall,
   UserMessage,
 } from "../agent/types";
+import {
+  toolResultText,
+  validateToolResultContent,
+} from "../agent/tool-result-content";
 import type { RuntimeSessionContext } from "../agent/runtime-session";
 import {
   parseImageAssetId,
@@ -61,11 +65,31 @@ export function toOpenAIResponsesItems(
   }
 
   if (message.role === "tool") {
+    validateToolResultContent(message.content);
+    const hasImage = message.content.some((block) => block.type === "image");
     return [
       {
         type: "function_call_output",
         call_id: message.providerToolCallId,
-        output: message.content,
+        output: hasImage
+          ? message.content.map((block) =>
+              block.type === "text"
+                ? ({ type: "input_text", text: block.text } as const)
+                : ({
+                    type: "input_image",
+                    detail: "auto" as const,
+                    image_url:
+                      options.materializedImages === undefined
+                        ? (imageAssetUrlMarker(
+                            block.asset.assetId,
+                          ) as unknown as string)
+                        : requireMaterializedImage(
+                            options.materializedImages,
+                            block.asset.assetId,
+                          ),
+                  } as const),
+            )
+          : toolResultText(message.content),
       },
     ];
   }

package/src/model/openai-responses-model-client.ts

@@ -9,7 +9,7 @@ import {
   IMAGE_INPUT_POLICY_VERSION,
 } from "../image/image-input-policy";
 import type { ModelContextBudget } from "./model-context-profile";
-import { ProviderResponseError } from "./model-client";
+import { ProviderResponseError, validateModelModalities } from "./model-client";
 import type {
   MaterializedModelRequest,
   ModelClient,
@@ -23,6 +23,7 @@ import type {
 } from "./model-client";
 import {
   deepFreeze,
+  imageToolSegment,
   imageUserSegment,
   materializeOpenAIRequest,
   normalizedEndpointPolicy,
@@ -39,7 +40,7 @@ import { OpenAIResponsesStreamAccumulator } from "./openai-responses-stream";
 import type { ReasoningEffortController } from "./reasoning-effort";
 import { sha256, stableJsonStringify } from "./model-request-preflight";
 
-const OPENAI_RESPONSES_SERIALIZATION_VERSION = "openai-responses-v1";
+const OPENAI_RESPONSES_SERIALIZATION_VERSION = "openai-responses-v2";
 const OPENAI_RESPONSES_TIMEOUT_MS = 30 * 60 * 1_000;
 
 export class OpenAIResponsesModelClient implements ModelClient {
@@ -48,6 +49,7 @@ export class OpenAIResponsesModelClient implements ModelClient {
     serializationVersion: OPENAI_RESPONSES_SERIALIZATION_VERSION,
   });
   readonly inputModalities: readonly ("text" | "image")[];
+  readonly toolResultModalities: readonly ("text" | "image")[];
   readonly reasoningEffort?: ReasoningEffortController;
   private readonly client: OpenAI;
   private readonly preparedRequests = new WeakSet<object>();
@@ -61,6 +63,8 @@ export class OpenAIResponsesModelClient implements ModelClient {
       contextBudget: ModelContextBudget;
       baseURL?: string;
       inputModalities?: readonly ("text" | "image")[];
+      toolResultModalities?: readonly ("text" | "image")[];
+      profileName?: string;
       model: string;
       providerName?: string;
       reasoningEffort?: ReasoningEffortController;
@@ -72,10 +76,15 @@ export class OpenAIResponsesModelClient implements ModelClient {
     this.provider = options.providerName ?? "responses-compatible";
     this.stream = options.stream ?? true;
     this.reasoningEffort = options.reasoningEffort;
-    this.inputModalities = Object.freeze([...(options.inputModalities ?? ["text"])]);
-    if (!this.inputModalities.includes("text")) {
-      throw new Error('OpenAI Responses input modalities must include "text".');
-    }
+    const modalities = validateModelModalities({
+      profileName: options.profileName,
+      adapter: this.messageProtocol.adapter,
+      inputModalities: options.inputModalities ?? ["text"],
+      toolResultModalities: options.toolResultModalities ?? ["text"],
+      adapterToolResultModalities: ["text", "image"],
+    });
+    this.inputModalities = modalities.inputModalities;
+    this.toolResultModalities = modalities.toolResultModalities;
     this.client = new OpenAI({
       apiKey: options.apiKey,
       baseURL: options.baseURL,
@@ -113,11 +122,17 @@ export class OpenAIResponsesModelClient implements ModelClient {
     const messageSegments = input.messages.map(
       (message, index): PreparedPromptSegment =>
         message.role === "user" && message.attachments !== undefined
-          ? imageUserSegment(message)
-          : {
-              kind: segmentKind(message.role),
-              normalizedText: stableJsonStringify(itemsByMessage[index]),
-            },
+          ? imageUserSegment(message, index + 1)
+          : message.role === "tool"
+            ? imageToolSegment(
+                message,
+                index + 1,
+                stableJsonStringify(itemsByMessage[index]),
+              )
+            : {
+                kind: segmentKind(message.role),
+                normalizedText: stableJsonStringify(itemsByMessage[index]),
+              },
     );
     const mediaOccurrenceCount = messageSegments.reduce(
       (total, segment) => total + (segment.media?.length ?? 0),
@@ -132,6 +147,7 @@ export class OpenAIResponsesModelClient implements ModelClient {
         requestMaxOutputTokens: this.options.contextBudget.requestMaxOutputTokens,
         stream: this.stream,
         inputModalities: this.inputModalities,
+        toolResultModalities: this.toolResultModalities,
         requestPolicy: { store: false, toolChoice: "auto" },
         imagePolicy: {
           version: IMAGE_INPUT_POLICY_VERSION,

package/src/model/token-estimator.ts

@@ -85,6 +85,16 @@ export function estimatePromptSegments(
   };
 }
 
+export function guardedContextTokens(
+  breakdown: RawContextBreakdown,
+  correctionFactor: number,
+): number {
+  return (
+    Math.ceil(breakdown.textAndProtocolTokens * correctionFactor) +
+    breakdown.imageTokens
+  );
+}
+
 export class RollingTokenCalibration {
   private readonly samples: number[] = [];
 

package/src/observation/observation-builder.ts

@@ -1,4 +1,9 @@
 import type { ToolCall } from "../agent/types";
+import type { ToolResultContent } from "../agent/types";
+import {
+  textToolResultContent,
+  toolResultDisplayText,
+} from "../agent/tool-result-content";
 import type {
   BashRawResult,
   DeleteFileRawResult,
@@ -6,6 +11,7 @@ import type {
   GenericToolRawResult,
   GlobRawResult,
   GrepRawResult,
+  MemoryGetRawResult,
   MemorySearchRawResult,
   McpToolRawResult,
   ReadFileRawResult,
@@ -17,66 +23,100 @@ import type {
   TaskStopRawResult,
   ToolRawResult,
   UpdatePlanRawResult,
+  ViewImageRawResult,
+  WaitRawResult,
   WebFetchRawResult,
   WebSearchRawResult,
   WriteFileRawResult,
 } from "../tools/types";
 
 export type ToolObservation = {
-  content: string;
+  readonly content: readonly ToolResultContent[];
+  readonly displayText: string;
 };
 
 export class ObservationBuilder {
   build(input: { call: ToolCall; raw: ToolRawResult }): ToolObservation {
     switch (input.raw.kind) {
       case "glob":
-        return { content: renderGlobObservation(input.raw) };
+        return textObservation(renderGlobObservation(input.raw));
       case "grep":
-        return { content: renderGrepObservation(input.raw) };
+        return textObservation(renderGrepObservation(input.raw));
       case "read":
-        return { content: renderReadObservation(input.raw) };
+        return textObservation(renderReadObservation(input.raw));
+      case "view_image":
+        return renderViewImageObservation(input.raw);
       case "recall":
-        return { content: renderRecallObservation(input.raw) };
+        return textObservation(renderRecallObservation(input.raw));
       case "memory_search":
-        return { content: renderMemorySearchObservation(input.raw) };
+        return textObservation(renderMemorySearchObservation(input.raw));
+      case "memory_get":
+        return textObservation(renderMemoryGetObservation(input.raw));
       case "skill":
-        return { content: renderSkillObservation(input.raw) };
+        return textObservation(renderSkillObservation(input.raw));
       case "write":
-        return { content: renderWriteObservation(input.raw) };
+        return textObservation(renderWriteObservation(input.raw));
       case "edit":
-        return { content: renderEditObservation(input.raw) };
+        return textObservation(renderEditObservation(input.raw));
       case "delete":
-        return { content: renderDeleteObservation(input.raw) };
+        return textObservation(renderDeleteObservation(input.raw));
       case "bash":
-        return { content: renderBashObservation(input.raw) };
+        return textObservation(renderBashObservation(input.raw));
       case "update_plan":
-        return { content: renderUpdatePlanObservation(input.raw) };
+        return textObservation(renderUpdatePlanObservation(input.raw));
+      case "wait":
+        return textObservation(renderWaitObservation(input.raw));
       case "task_list":
-        return { content: renderTaskListObservation(input.raw) };
+        return textObservation(renderTaskListObservation(input.raw));
       case "task_output":
-        return { content: renderTaskOutputObservation(input.raw) };
+        return textObservation(renderTaskOutputObservation(input.raw));
       case "task_input":
-        return { content: renderTaskInputObservation(input.raw) };
+        return textObservation(renderTaskInputObservation(input.raw));
       case "task_stop":
-        return { content: renderTaskStopObservation(input.raw) };
+        return textObservation(renderTaskStopObservation(input.raw));
       case "web_search":
-        return { content: renderWebSearchObservation(input.raw) };
+        return textObservation(renderWebSearchObservation(input.raw));
       case "web_fetch":
-        return { content: renderWebFetchObservation(input.raw) };
+        return textObservation(renderWebFetchObservation(input.raw));
       case "mcp":
-        return { content: renderMcpObservation(input.raw) };
+        return textObservation(renderMcpObservation(input.raw));
       case "generic":
-        return { content: renderGenericObservation(input.raw) };
+        return textObservation(renderGenericObservation(input.raw));
       default:
         return assertNever(input.raw);
     }
   }
 }
 
+function textObservation(text: string): ToolObservation {
+  return Object.freeze({ content: textToolResultContent(text), displayText: text });
+}
+
+function renderViewImageObservation(raw: ViewImageRawResult): ToolObservation {
+  if (!raw.ok || raw.asset === undefined) {
+    return textObservation(
+      `ViewImage failed for ${raw.filePath || "(unknown path)"}: ${raw.error ?? "Unknown error."}`,
+    );
+  }
+  const text = `Viewed image ${raw.filePath} (${raw.asset.mimeType}, ${raw.asset.width}x${raw.asset.height}, ${raw.asset.byteLength} bytes, asset=${raw.asset.assetId.slice(0, 12)}…).`;
+  const content = Object.freeze([
+    Object.freeze({ type: "text" as const, text }),
+    Object.freeze({ type: "image" as const, asset: raw.asset }),
+  ]);
+  const displayText = toolResultDisplayText(content);
+  return Object.freeze({ content, displayText });
+}
+
 function renderUpdatePlanObservation(raw: UpdatePlanRawResult): string {
   return raw.ok ? "Plan updated." : `UpdatePlan failed: ${raw.error}`;
 }
 
+function renderWaitObservation(raw: WaitRawResult): string {
+  return raw.ok
+    ? `Waited ${raw.seconds} second${raw.seconds === 1 ? "" : "s"}.`
+    : `Wait failed: ${raw.error}`;
+}
+
 function assertNever(value: never): never {
   throw new Error(`Unhandled tool raw result: ${JSON.stringify(value)}`);
 }
@@ -247,16 +287,51 @@ function renderMemorySearchObservation(raw: MemorySearchRawResult): string {
   if (!raw.ok) {
     return `MemorySearch unavailable: ${raw.error}`;
   }
+  const degradedNote =
+    raw.degraded === "vector"
+      ? " vector search unavailable; keyword results only."
+      : raw.degraded === "fts"
+        ? " keyword search unavailable; vector results only."
+        : "";
   if (raw.matches.length === 0) {
-    return "MemorySearch found no stored memories.";
+    return `MemorySearch found no stored memories.${degradedNote}`;
   }
-  const header = `MemorySearch returned ${raw.matches.length} derived memories. They may be stale or wrong; verify current workspace facts.`;
+  const header = `MemorySearch returned ${raw.matches.length} derived historical memory records.${degradedNote} They describe past turns and may be stale or wrong; verify current workspace facts with current tools before relying on them.`;
+  const footer =
+    "Use MemoryGet on a result's memory id when its summary is truncated or you need the exact stored record; use RecallSearch on its source session for the full original context.";
   return [
     header,
-    ...raw.matches.map(
-      (match, index) =>
-        `${index + 1}. score=${match.score.toFixed(3)} created_at=${match.createdAt} workspace=${match.sourceWorkspace}\n   ${match.text}`,
+    ...raw.matches.map((match, index) =>
+      [
+        `${index + 1}. score=${match.score.toFixed(3)} via=${match.via.join(",")} created_at=${match.createdAt} workspace=${match.sourceWorkspace} session=${match.sourceSessionId} memory=${match.memoryId}`,
+        `   ${match.text}`,
+        ...(match.summary === "" ? [] : [`   summary: ${match.summary}`]),
+      ].join("\n"),
     ),
+    footer,
+  ].join("\n\n");
+}
+
+function renderMemoryGetObservation(raw: MemoryGetRawResult): string {
+  if (!raw.ok) {
+    return `MemoryGet unavailable: ${raw.error}`;
+  }
+  if (raw.memory === null) {
+    return "MemoryGet found no stored memory with that id.";
+  }
+  const memory = raw.memory;
+  const header =
+    "MemoryGet returned one derived historical memory record. It describes a past turn and may be stale or wrong; verify current workspace facts with current tools before relying on it.";
+  const footer =
+    "Use RecallSearch on the source session when you need the full original context.";
+  return [
+    header,
+    [
+      `memory=${memory.memoryId} created_at=${memory.createdAt} workspace=${memory.sourceWorkspace} session=${memory.sourceSessionId} turn=${memory.sourceTurnId}`,
+      `text: ${memory.text}`,
+      ...(memory.summary === "" ? [] : [`summary: ${memory.summary}`]),
+    ].join("\n"),
+    footer,
   ].join("\n\n");
 }