tinker-agent 1.11.0 → 2.1.0

package/CHANGELOG.md

@@ -5,6 +5,47 @@ All notable user-facing changes to Tinker are documented here. The project follo
 
 ## [Unreleased]
 
+## [2.1.0] - 2026-08-29
+
+### Added
+
+- Add cross-session derived memory. Completed turns are distilled into memory
+  records that the agent can recall in later sessions and workspaces through
+  `MemorySearch`, which fuses vector similarity with FTS5 keyword matching via
+  RRF hybrid ranking, and `MemoryGet`, which reads a full record by its id.
+- Add a `toolResultModalities` model-profile setting. Image-capable profiles can
+  now receive `ViewImage` results as real image content in the model request
+  instead of text-only descriptions.
+- Add a `Wait` tool that pauses the agent loop for a cancellable whole number of
+  seconds (1 to 3600), useful for spacing polling attempts.
+- Expand a leading `~` to the user's home directory in `TINKER_MODELS` and
+  `TINKER_WORKSPACE` paths.
+
+### Changed
+
+- Show `MemorySearch` keywords alongside the query in TUI tool summaries, so
+  hybrid recall behavior is visible while observing the agent.
+
+### Fixed
+
+- Accept `memory_get` raw results in stored tool history, so sessions that
+  contain MemoryGet calls remain resumable.
+
+## [2.0.0] - 2026-08-20
+
+### Changed
+
+- Estimate image input locally with deterministic size buckets instead of a
+  provider-side token-estimation request. Images are normalized for orientation
+  and provider limits before estimation and upload, keeping preflight accounting
+  aligned with the payload sent to the model.
+
+### Removed
+
+- Remove the `tokenEstimator` model-profile setting. Existing image profiles must
+  delete that field; sessions created under the previous image policy remain
+  inspectable but cannot be resumed for execution.
+
 ## [1.11.0] - 2026-08-15
 
 ### Added
@@ -208,7 +249,9 @@ All notable user-facing changes to Tinker are documented here. The project follo
 - First formal npm release under the `tinker-agent` package name with the `tinker`
   executable.
 
-[Unreleased]: https://github.com/ishowshao/tinker/compare/v1.11.0...HEAD
+[Unreleased]: https://github.com/ishowshao/tinker/compare/v2.1.0...HEAD
+[2.1.0]: https://github.com/ishowshao/tinker/releases/tag/v2.1.0
+[2.0.0]: https://github.com/ishowshao/tinker/releases/tag/v2.0.0
 [1.11.0]: https://github.com/ishowshao/tinker/releases/tag/v1.11.0
 [1.10.1]: https://github.com/ishowshao/tinker/releases/tag/v1.10.1
 [1.9.0]: https://github.com/ishowshao/tinker/releases/tag/v1.9.0

package/README.md

@@ -138,7 +138,7 @@ are required. Boolean environment values accept case-insensitive `true/false`,
 <!-- BEGIN GENERATED: PUBLIC ENVIRONMENT VARIABLES -->
 | Variable | Area | Applies | Required | Type | Default | Secret | Description |
 | --- | --- | --- | --- | --- | --- | --- | --- |
-| `TINKER_MODELS` | Model | All modes | No | Non-empty string | — | No | Optional model profiles JSON path. Relative paths resolve from the process cwd. |
+| `TINKER_MODELS` | Model | All modes | No | Non-empty string | — | No | Optional model profiles JSON path. A leading ~ expands to the home directory; other relative paths resolve from the process cwd. |
 | `TINKER_MODEL` | Model | Env mode | Env mode | Non-empty string | — | No | Model name used when model profiles are not configured. |
 | `TINKER_API` | Model | Env mode | No | Non-empty string | `"chat-completions"` | No | Model API adapter: "chat-completions" or "responses". |
 | `TINKER_BASE_URL` | Model | Env mode | Env mode | Non-empty string | — | No | OpenAI-compatible API root URL; do not append /chat/completions or /responses. |
@@ -148,7 +148,7 @@ are required. Boolean environment values accept case-insensitive `true/false`,
 | `TINKER_INCLUDE_REASONING_CONTENT` | Model | Env mode | No | Boolean | `false` | No | Replay provider reasoning_content in Chat Completions history; ignored by Responses. |
 | `TINKER_STREAM` | Model | Env mode | No | Boolean | `true` | No | Use streaming transport for the selected model API. |
 | `TINKER_WEBFETCH_REFINE_MODEL` | Model | Env mode | No | Non-empty string | — | No | Optional WebFetch refiner model; currently must match TINKER_MODEL. |
-| `TINKER_WORKSPACE` | Workspace | All modes | No | Non-empty string | Process cwd | No | Workspace path. Relative paths resolve from the process cwd. |
+| `TINKER_WORKSPACE` | Workspace | All modes | No | Non-empty string | Process cwd | No | Workspace path. A leading ~ expands to the home directory; other relative paths resolve from the process cwd. |
 | `TINKER_MAX_ITERATIONS` | Workspace | All modes | No | Positive integer | `512` | No | Maximum agent-loop iterations per turn. |
 | `EXA_API_KEY` | Tooling | All modes | No | Non-empty string | — | Yes | Enables WebSearch and the Exa WebFetch backend when set. |
 | `TINKER_MCP_TIMEOUT_MS` | Tooling | All modes | No | Positive integer | `60000` | No | MCP tool-call timeout in milliseconds. |
@@ -194,7 +194,7 @@ Profile fields:
 | `includeReasoningContent` | No | JSON boolean | `false` | No | Replay provider reasoning_content in Chat Completions history; ignored by Responses. |
 | `stream` | No | JSON boolean | `true` | No | Use streaming transport for the selected model API. |
 | `inputModalities` | No | Normalized modality array | `["text"]` | No | Accepted model input modalities; normalizes to ["text"] or ["text", "image"]. |
-| `tokenEstimator` | With image | Object | — | Yes | Independent token estimator required for image profiles. |
+| `toolResultModalities` | No | Normalized modality array | `["text"]` | No | Accepted tool-result modalities; normalizes to ["text"] or ["text", "image"]. |
 
 `reasoning` fields:
 
@@ -205,17 +205,6 @@ Profile fields:
 
 The optional `reasoning` object declares provider-specific effort values. Efforts must be unique non-whitespace strings, `reset` is reserved by the TUI command, and `defaultEffort` must appear in `supportedEfforts`. Omitting `reasoning` sends no effort parameter and disables `/reasoning` for that profile.
 
-`tokenEstimator` fields:
-
-| Field | Type / constraint | Secret | Description |
-| --- | --- | --- | --- |
-| `kind` | Literal `"moonshot-estimate-token-count-v1"` | No | Estimator protocol discriminator. |
-| `model` | Non-empty string | No | Estimator model name. |
-| `apiBase` | Non-empty string | No | Estimator API base URL. |
-| `apiKey` | Non-empty string | Yes | Estimator API credential. |
-| `timeoutMs` | Integer 1000–60000 | No | Estimator request timeout in milliseconds. |
-| `maxRetries` | Literal `0` | No | Estimator retry count; retries are disabled. |
-
 Text-only profile example:
 
 ```json
@@ -224,6 +213,7 @@ Text-only profile example:
   "profiles": {
     "text": {
       "model": "example-text-model",
+      "api": "chat-completions",
       "apiBase": "https://api.example.com/v1",
       "apiKey": "your-model-api-key",
       "contextWindowTokens": 128000,
@@ -240,6 +230,9 @@ Text-only profile example:
       "stream": true,
       "inputModalities": [
         "text"
+      ],
+      "toolResultModalities": [
+        "text"
       ]
     }
   }
@@ -254,6 +247,7 @@ Image-capable profile example:
   "profiles": {
     "image": {
       "model": "example-vision-model",
+      "api": "responses",
       "apiBase": "https://api.example.com/v1",
       "apiKey": "your-model-api-key",
       "contextWindowTokens": 128000,
@@ -272,14 +266,10 @@ Image-capable profile example:
         "text",
         "image"
       ],
-      "tokenEstimator": {
-        "kind": "moonshot-estimate-token-count-v1",
-        "model": "example-token-estimator",
-        "apiBase": "https://estimator.example.com/v1",
-        "apiKey": "your-estimator-api-key",
-        "timeoutMs": 30000,
-        "maxRetries": 0
-      }
+      "toolResultModalities": [
+        "text",
+        "image"
+      ]
     }
   }
 }
@@ -313,6 +303,7 @@ Atomic-memory profile example:
   "profiles": {
     "text": {
       "model": "example-text-model",
+      "api": "chat-completions",
       "apiBase": "https://api.example.com/v1",
       "apiKey": "your-model-api-key",
       "contextWindowTokens": 128000,
@@ -329,6 +320,9 @@ Atomic-memory profile example:
       "stream": true,
       "inputModalities": [
         "text"
+      ],
+      "toolResultModalities": [
+        "text"
       ]
     }
   },
@@ -370,7 +364,7 @@ configured profile.
 ### Image Input
 
 Image attachment is enabled only for a profile whose `inputModalities` explicitly
-includes `image` and which supplies a valid `tokenEstimator`. In the interactive
+includes `image`. In the interactive
 TUI, type `@` and select a file that is inside the workspace and visible to the
 workspace search rules. One-shot commands, clipboard image bytes, remote URLs, and
 files outside or ignored by the workspace search are not supported.
@@ -378,10 +372,25 @@ files outside or ignored by the workspace search are not supported.
 Tinker accepts PNG (not APNG), JPEG, and static WebP. It rejects GIF, animated
 WebP, and other formats. A message and provider request may contain at most eight
 images; each image may be at most 20 MiB, 4096 pixels on either edge, and 8,847,360
-pixels in total. See the
+pixels in total. Provider requests preserve smaller images and proportionally
+downscale larger images to a maximum 2048-pixel long edge. Context planning uses
+fixed local token buckets derived from the materialized dimensions and performs no
+independent token-estimator request. See the
+[`image token bucket design`](docs/image-token-bucket-estimation-design.md) and
 [`multimodal image input design`](docs/multimodal-image-input-design.md) for the
 complete fixed policy and persistence contract.
 
+`ViewImage(file_path)` is registered only when the selected profile declares both
+`inputModalities: ["text", "image"]` and
+`toolResultModalities: ["text", "image"]`. The first implementation supports
+image tool results through the Responses adapter; Chat Completions remains
+text-only for tool results. Relative paths stay inside the workspace, absolute
+paths may explicitly select an external local file, and symbolic links are
+rejected. Canonical history stores content-addressed image references rather than
+Base64, while stdout, TUI, Recall, and logs show deterministic text summaries.
+See the [`ViewImage tool design`](docs/view-image-tool-design.md) for the complete
+capability, persistence, provider, and compaction contract.
+
 ### Built-in Slash Commands
 
 <!-- BEGIN GENERATED: BUILT-IN SLASH COMMANDS -->

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "tinker-agent",
-  "version": "1.11.0",
+  "version": "2.1.0",
   "description": "A personal coding agent with an interactive TUI and one-shot CLI.",
   "license": "Apache-2.0",
   "type": "module",
@@ -50,6 +50,7 @@
     "bench:long-session": "bun scripts/bench-long-session-memory.ts",
     "bench:i3-provider-smoke": "bun scripts/bench-i3-provider-smoke.ts",
     "bench:k3-image-live": "bun scripts/k3-image-live-smoke.ts",
+    "bench:view-image-live": "bun scripts/view-image-live-smoke.ts",
     "bench:i4-active-recall": "bun scripts/bench-i4-active-recall.ts",
     "qualify:i4-active-recall": "bun scripts/qualify-i4-active-recall.ts",
     "release:verify": "bun scripts/verify-release-package.ts",

package/src/agent/context-meter.ts

@@ -2,13 +2,8 @@ import type { ModelContextBudget } from "../model/model-context-profile";
 import type {
   ModelRequestOutput,
   ModelUsage,
-  MaterializedModelRequest,
   PreparedModelRequest,
 } from "../model/model-client";
-import type {
-  InputTokenEstimate,
-  InputTokenEstimator,
-} from "../model/input-token-estimator";
 import {
   assertContextBudget,
   contextPressure,
@@ -18,6 +13,7 @@ import {
 import { lastPromptPrefixHash, promptPrefixHashes } from "../model/prompt-prefix-hash";
 import {
   estimatePromptSegments,
+  guardedContextTokens,
   RollingTokenCalibration,
   type RawContextBreakdown,
 } from "../model/token-estimator";
@@ -68,7 +64,6 @@ export class ContextMeter {
   private anchor?: MeasuredContextAnchor;
   private lastProviderUsage?: ModelUsage;
   private calibrationIdentity?: string;
-  private readonly providerEstimateCache = new Map<string, InputTokenEstimate>();
 
   constructor(
     private readonly budget: ModelContextBudget,
@@ -129,13 +124,14 @@ export class ContextMeter {
     let guardedDeltaTokens: number | undefined;
     if (anchor === undefined) {
       source = "estimated_full";
-      usedInputTokens = Math.ceil(rawFullEstimate.totalTokens * correctionFactor);
+      usedInputTokens = guardedEstimate(rawFullEstimate, correctionFactor);
     } else {
       source = "measured_plus_estimated_delta";
-      rawDeltaTokens = estimatePromptSegments(
+      const rawDelta = estimatePromptSegments(
         prepared.promptSegments.slice(anchor.segmentCount),
-      ).totalTokens;
-      guardedDeltaTokens = Math.ceil(rawDeltaTokens * correctionFactor);
+      );
+      rawDeltaTokens = rawDelta.totalTokens;
+      guardedDeltaTokens = guardedEstimate(rawDelta, correctionFactor);
       usedInputTokens = anchor.totalTokens + guardedDeltaTokens;
     }
 
@@ -217,62 +213,6 @@ export class ContextMeter {
     };
   }
 
-  applyProviderEstimate(
-    prepared: PreparedModelRequest,
-    estimate: {
-      inputTokens: number;
-      coverage: "messages" | "full_request";
-    },
-  ): ContextUsageSnapshot {
-    if (!Number.isSafeInteger(estimate.inputTokens) || estimate.inputTokens < 0) {
-      throw new Error("Provider input estimate must be a non-negative safe integer.");
-    }
-    const local = this.measure(prepared);
-    const measurement = this.measurements.get(prepared)!;
-    const guardedTools =
-      estimate.coverage === "messages"
-        ? Math.ceil(
-            measurement.rawFullEstimate.toolSchemaTokens * local.correctionFactor,
-          )
-        : 0;
-    const providerGuarded = estimate.inputTokens + guardedTools;
-    if (providerGuarded <= local.usedInputTokens) {
-      return local;
-    }
-    const snapshot: ContextUsageSnapshot = {
-      ...local,
-      usedInputTokens: providerGuarded,
-      source: "provider_estimated",
-      pressure: contextPressure(providerGuarded, this.budget),
-    };
-    this.measurements.set(prepared, {
-      rawFullEstimate: measurement.rawFullEstimate,
-      snapshot,
-    });
-    return snapshot;
-  }
-
-  async estimateProviderInput(
-    prepared: MaterializedModelRequest,
-    estimator: InputTokenEstimator,
-    options: { signal: AbortSignal },
-  ): Promise<InputTokenEstimate> {
-    options.signal.throwIfAborted();
-    const key = providerEstimateCacheKey(
-      prepared,
-      estimator.compatibility.coverageVersion,
-    );
-    const cached = this.providerEstimateCache.get(key);
-    if (cached !== undefined) {
-      return cached;
-    }
-    const estimate = await estimator.estimate(prepared, options);
-    options.signal.throwIfAborted();
-    const frozen = Object.freeze({ ...estimate });
-    this.providerEstimateCache.set(key, frozen);
-    return frozen;
-  }
-
   assertWithinBudget(snapshot: ContextUsageSnapshot): void {
     assertContextBudget({
       usedInputTokens: snapshot.usedInputTokens,
@@ -295,7 +235,6 @@ export class ContextMeter {
       this.lastProviderUsage = undefined;
       this.measurements = new WeakMap();
       this.calibration.clear();
-      this.providerEstimateCache.clear();
       this.calibrationIdentity = nextIdentity;
       throw new Error(
         "Context revision changed the request configuration or tool schema.",
@@ -304,7 +243,6 @@ export class ContextMeter {
     this.anchor = undefined;
     this.lastProviderUsage = undefined;
     this.measurements = new WeakMap();
-    this.providerEstimateCache.clear();
     this.calibrationIdentity = nextIdentity;
   }
 
@@ -315,7 +253,6 @@ export class ContextMeter {
     this.measurements = new WeakMap();
     this.calibration.clear();
     this.calibrationIdentity = undefined;
-    this.providerEstimateCache.clear();
   }
 
   private usableAnchor(
@@ -330,10 +267,7 @@ export class ContextMeter {
       anchor.requestConfigHash !== prepared.requestConfigHash ||
       anchor.toolSchemaHash !== prepared.toolSchemaHash ||
       anchor.segmentCount > prepared.promptSegments.length ||
-      prefixHashes[anchor.segmentCount] !== anchor.prefixHash ||
-      prepared.promptSegments
-        .slice(anchor.segmentCount)
-        .some((segment) => (segment.media?.length ?? 0) > 0)
+      prefixHashes[anchor.segmentCount] !== anchor.prefixHash
     ) {
       this.anchor = undefined;
       return undefined;
@@ -359,18 +293,11 @@ export class ContextMeter {
   }
 }
 
-function providerEstimateCacheKey(
-  prepared: PreparedModelRequest,
-  coverageVersion: string,
-): string {
-  return [
-    prepared.requestConfigHash,
-    prepared.toolSchemaHash,
-    lastPromptPrefixHash(
-      promptPrefixHashes(prepared.requestConfigHash, prepared.promptSegments),
-    ),
-    coverageVersion,
-  ].join(":");
+function guardedEstimate(
+  breakdown: RawContextBreakdown,
+  correctionFactor: number,
+): number {
+  return guardedContextTokens(breakdown, correctionFactor);
 }
 
 function assertMeasuredContextAnchor(anchor: MeasuredContextAnchor): void {

package/src/agent/loop.ts

@@ -175,20 +175,7 @@ export async function runAgent(input: RunAgentInput): Promise<RunAgentResult> {
             assetStore: input.assetStore,
             signal: input.signal,
           });
-          if (preflight.source === "measured_plus_estimated_delta") {
-            preflight = input.contextMeter.measure(request);
-          } else {
-            const estimator = input.model.inputTokenEstimator;
-            if (estimator === undefined) {
-              throw new Error("Image model request has no input token estimator.");
-            }
-            const estimate = await input.contextMeter.estimateProviderInput(
-              request as MaterializedModelRequest,
-              estimator,
-              { signal: input.signal },
-            );
-            preflight = input.contextMeter.applyProviderEstimate(request, estimate);
-          }
+          preflight = input.contextMeter.measure(request);
           input.contextMeter.assertWithinBudget(preflight);
           await input.runtimeSession.append({
             type: "context.usage.updated",

package/src/agent/runtime-session.ts

@@ -308,6 +308,7 @@ type CommonRuntimeSessionInput = {
   webFetchRefiner?: Refiner;
   toolingConfig?: PublicToolingConfig;
   memorySearch?: ToolExecutor;
+  memoryGet?: ToolExecutor;
   completedTurnHook?: CompletedTurnHook;
   enableTurnUndo?: boolean;
   bashGuard?: {
@@ -580,12 +581,8 @@ class DefaultRuntimeSession implements RuntimeSession {
         includeReasoningContent: input.includeReasoningContent,
         contextProfile: input.contextProfile,
         messageProtocol: input.modelClient.messageProtocol,
-        inputModalities: input.modelClient.inputModalities ?? ["text"],
-        ...(input.modelClient.inputTokenEstimator === undefined
-          ? {}
-          : {
-              tokenEstimator: input.modelClient.inputTokenEstimator.compatibility,
-            }),
+        inputModalities: input.modelClient.inputModalities,
+        toolResultModalities: input.modelClient.toolResultModalities,
       });
       if (input.selection.mode === "resume") {
         store.assertSessionCompatibility(compatibility);
@@ -659,6 +656,10 @@ class DefaultRuntimeSession implements RuntimeSession {
         workspaceRoot: input.workspaceRoot,
         runtimeSession: session.context,
         historyReader: store.historyReader(),
+        imageAssetStore: assetStore,
+        supportsViewImage:
+          input.modelClient.inputModalities.includes("image") &&
+          input.modelClient.toolResultModalities.includes("image"),
         ...(input.enableTurnUndo === true ? { enableTurnUndo: true } : {}),
         webFetchRefiner: input.webFetchRefiner,
         toolingConfig: input.toolingConfig,
@@ -670,6 +671,7 @@ class DefaultRuntimeSession implements RuntimeSession {
         ...(input.memorySearch === undefined
           ? {}
           : { memorySearch: input.memorySearch }),
+        ...(input.memoryGet === undefined ? {} : { memoryGet: input.memoryGet }),
         ...(session.skillCatalog.skills.size === 0
           ? {}
           : {
@@ -1402,7 +1404,7 @@ class DefaultRuntimeSession implements RuntimeSession {
             messageId: message.messageId,
             frameId: message.frameId,
             ordinal: message.ordinal,
-            content: message.content,
+            content: message.displayText,
             contentSha256: message.contentSha256,
           },
           name: activation.name,
@@ -1641,24 +1643,6 @@ class DefaultRuntimeSession implements RuntimeSession {
         { assetStore: this.assetStore, signal: controller.signal },
       );
       admissionSnapshot = this.contextMeter.measure(admissionPrepared);
-      if (
-        prepared.mediaOccurrenceCount > 0 &&
-        admissionSnapshot.source !== "measured_plus_estimated_delta"
-      ) {
-        const estimator = this.input.modelClient.inputTokenEstimator;
-        if (estimator === undefined) {
-          throw new Error("Image model request has no input token estimator.");
-        }
-        const estimate = await this.contextMeter.estimateProviderInput(
-          admissionPrepared,
-          estimator,
-          { signal: controller.signal },
-        );
-        admissionSnapshot = this.contextMeter.applyProviderEstimate(
-          admissionPrepared,
-          estimate,
-        );
-      }
       this.contextMeter.assertWithinBudget(admissionSnapshot);
       controller.signal.throwIfAborted();
       const turn = this.stageTurn(input.userMessage);

package/src/agent/session-ledger.ts

@@ -26,12 +26,15 @@ import {
 } from "../context/context-protocol-validator";
 import {
   CURRENT_TOOL_OBSERVATION_FORMAT,
+  canonicalToolResultContentHash,
   contentHash,
+  displayTextForCompletion,
   immutableCanonicalClone,
   immutableRecord,
   observationForCompletion,
   rawResultHash,
   userMessageHash,
+  validateReturnedToolObservation,
   type CanonicalMessageRecord,
   type ProtocolContextView,
   type ProtocolFrame,
@@ -583,6 +586,7 @@ export class InMemorySessionLedger implements SessionLedger {
       assertSameToolCall(expectedCall, completionInput.call);
       validateCompletionInput(completionInput);
       const content = observationForCompletion(completionInput);
+      const displayText = displayTextForCompletion(completionInput);
       const createdAt = this.clock();
       const messageId = this.input.idFactory.createMessageId();
       const message = immutableRecord<CanonicalMessageRecord>({
@@ -590,7 +594,7 @@ export class InMemorySessionLedger implements SessionLedger {
         sessionId: this.input.sessionId,
         frameId: frameBefore.frameId,
         ordinal: this.view.messages.length + messages.length + 1,
-        contentSha256: contentHash(content),
+        contentSha256: canonicalToolResultContentHash(content),
         createdAt,
         role: "tool",
         turnId: pending.turn.turnId,
@@ -599,6 +603,7 @@ export class InMemorySessionLedger implements SessionLedger {
         providerToolCallId: expectedCall.providerToolCallId,
         name: expectedCall.name,
         content,
+        displayText,
         origin: completionInput.kind === "returned" ? "tool" : "runtime",
       });
       const completion = canonicalCompletion(completionInput);
@@ -608,7 +613,7 @@ export class InMemorySessionLedger implements SessionLedger {
         toolCallId: expectedCall.toolCallId,
         toolMessageId: messageId,
         completion,
-        observationSha256: contentHash(content),
+        observationSha256: canonicalToolResultContentHash(content),
         createdAt,
       });
       messages.push(message);
@@ -1006,9 +1011,11 @@ function canonicalCompletion(input: ToolCompletionInput): ToolCompletion {
 
 function validateCompletionInput(input: ToolCompletionInput): void {
   if (input.kind === "returned") {
-    if (typeof input.observation !== "string") {
-      throw new Error("Returned tool completion observation must be a string.");
-    }
+    validateReturnedToolObservation({
+      toolName: input.call.name,
+      raw: input.raw,
+      content: input.observation,
+    });
     immutableCanonicalClone(input.raw);
     return;
   }

package/src/agent/tool-result-content.ts

@@ -0,0 +1,76 @@
+import { sha256, stableJsonStringify } from "../model/model-request-preflight";
+import { validateImageAssetRef } from "../image/image-types";
+import { IMAGE_INPUT_POLICY } from "../image/image-input-policy";
+import type { ToolResultContent } from "./types";
+
+export function canonicalToolResultContentHash(
+  content: readonly ToolResultContent[],
+): string {
+  validateToolResultContent(content);
+  return sha256(stableJsonStringify(content));
+}
+
+export function validateToolResultContent(content: readonly ToolResultContent[]): void {
+  if (content.length === 0) {
+    throw new Error("Tool result content must be a non-empty array.");
+  }
+  let previousWasText = false;
+  let imageCount = 0;
+  for (const block of content) {
+    switch (block.type) {
+      case "text":
+        if (block.text.trim() === "") {
+          throw new Error("Tool result text blocks must not be empty.");
+        }
+        if (previousWasText) {
+          throw new Error("Consecutive tool result text blocks must be merged.");
+        }
+        previousWasText = true;
+        break;
+      case "image":
+        validateImageAssetRef(block.asset);
+        imageCount += 1;
+        previousWasText = false;
+        break;
+    }
+  }
+  if (imageCount > IMAGE_INPUT_POLICY.maxImagesPerMessage) {
+    throw new Error("Tool result content exceeds the per-message image limit.");
+  }
+}
+
+export function textToolResultContent(text: string): readonly ToolResultContent[] {
+  const content = Object.freeze([Object.freeze({ type: "text" as const, text })]);
+  validateToolResultContent(content);
+  return content;
+}
+
+export function toolResultDisplayText(content: readonly ToolResultContent[]): string {
+  validateToolResultContent(content);
+  return content
+    .map((block) =>
+      block.type === "text"
+        ? block.text
+        : `[Image: ${block.asset.mimeType}, ${block.asset.width}x${block.asset.height}, ${block.asset.byteLength} bytes, asset=${shortAssetId(block.asset.assetId)}]`,
+    )
+    .join("\n");
+}
+
+export function toolResultText(content: readonly ToolResultContent[]): string {
+  validateToolResultContent(content);
+  if (content.some((block) => block.type === "image")) {
+    throw new Error("Text-only tool result mapping received an image block.");
+  }
+  return content
+    .map((block) => {
+      if (block.type !== "text") {
+        throw new Error("Text-only tool result mapping received an image block.");
+      }
+      return block.text;
+    })
+    .join("\n");
+}
+
+function shortAssetId(assetId: string): string {
+  return `${assetId.slice(0, 12)}…`;
+}

package/src/agent/types.ts

@@ -1,5 +1,5 @@
 import type { IterationId, SessionId, ToolCallId, TurnId } from "../ids/runtime-id";
-import type { UserMessage } from "../image/image-types";
+import type { ImageAssetRef, UserMessage } from "../image/image-types";
 
 export type {
   CodePointRange,
@@ -45,12 +45,24 @@ export type AssistantMessage = {
   toolCalls?: readonly ToolCall[];
 };
 
+export type ToolResultTextContent = {
+  readonly type: "text";
+  readonly text: string;
+};
+
+export type ToolResultImageContent = {
+  readonly type: "image";
+  readonly asset: ImageAssetRef;
+};
+
+export type ToolResultContent = ToolResultTextContent | ToolResultImageContent;
+
 export type ToolMessage = {
   role: "tool";
   toolCallId: ToolCallId;
   providerToolCallId: string;
   name: string;
-  content: string;
+  content: readonly ToolResultContent[];
 };
 
 export type AgentMessage =