tinker-agent 2.0.0 → 2.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (50) hide show
  1. package/CHANGELOG.md +28 -1
  2. package/README.md +27 -2
  3. package/package.json +2 -1
  4. package/src/agent/context-meter.ts +2 -4
  5. package/src/agent/runtime-session.ts +9 -2
  6. package/src/agent/session-ledger.ts +12 -5
  7. package/src/agent/tool-result-content.ts +76 -0
  8. package/src/agent/types.ts +14 -2
  9. package/src/cli/config.ts +4 -0
  10. package/src/cli/model-profiles.ts +41 -2
  11. package/src/cli/public-config-contract.ts +30 -7
  12. package/src/cli/runner-dependencies.ts +5 -0
  13. package/src/cli/tui-memory.ts +1 -0
  14. package/src/cli/tui-runner.tsx +4 -0
  15. package/src/context/compiled-context-hash.ts +2 -1
  16. package/src/context/compiled-context-validator.ts +13 -4
  17. package/src/context/context-protocol-validator.ts +33 -2
  18. package/src/context/context-revision-compiler.ts +2 -1
  19. package/src/context/context-revision.ts +8 -2
  20. package/src/context/context-swap-renderer.ts +46 -12
  21. package/src/context/prefix-retirement-planner.ts +13 -9
  22. package/src/context/protocol-frame.ts +74 -7
  23. package/src/context/swap-planner.ts +19 -14
  24. package/src/events/observation-text-log.ts +1 -1
  25. package/src/events/stdout-event-printer.ts +6 -0
  26. package/src/image/image-asset-store.ts +32 -3
  27. package/src/memory/contracts.ts +61 -3
  28. package/src/memory/memory-coordinator.ts +313 -49
  29. package/src/memory/memory-extractor.ts +48 -48
  30. package/src/memory/memory-get-tool.ts +86 -0
  31. package/src/memory/memory-search-tool.ts +122 -33
  32. package/src/memory/memory-store.ts +227 -20
  33. package/src/model/fake-model-client.ts +129 -76
  34. package/src/model/model-client.ts +62 -11
  35. package/src/model/openai-chat-mapping.ts +2 -1
  36. package/src/model/openai-chat-model-client.ts +22 -10
  37. package/src/model/openai-model-utils.ts +61 -30
  38. package/src/model/openai-responses-mapping.ts +25 -1
  39. package/src/model/openai-responses-model-client.ts +27 -11
  40. package/src/model/token-estimator.ts +10 -0
  41. package/src/observation/observation-builder.ts +100 -25
  42. package/src/session/session-history-reader.ts +128 -5
  43. package/src/session/session-schema.ts +59 -9
  44. package/src/session/session-store.ts +343 -196
  45. package/src/tools/registry.ts +18 -0
  46. package/src/tools/types.ts +46 -0
  47. package/src/tools/view-image.ts +89 -0
  48. package/src/tools/wait.ts +85 -0
  49. package/src/tui/components/memory-browser.tsx +3 -0
  50. package/src/tui/event-store.ts +61 -2
@@ -1,4 +1,5 @@
1
1
  import type { AgentMessage } from "../agent/types";
2
+ import { toolResultDisplayText, toolResultText } from "../agent/tool-result-content";
2
3
  import { stableJsonStringify } from "../model/model-request-preflight";
3
4
  import { formatMessageSource } from "./context-source";
4
5
  import type { CompiledRevisionContext, SwapOverride } from "./context-revision";
@@ -157,7 +158,7 @@ export class CompiledContextValidator {
157
158
  entry.message.toolCallId !== record.toolCallId ||
158
159
  entry.message.providerToolCallId !== record.providerToolCallId ||
159
160
  entry.message.name !== record.name ||
160
- entry.message.content !== override.renderedContent
161
+ toolResultText(entry.message.content) !== override.renderedContent
161
162
  ) {
162
163
  fail(
163
164
  `Swapped tool entry changed protocol identity at ordinal ${entry.ordinal}.`,
@@ -197,7 +198,11 @@ function validateOverrideIdentity(
197
198
  fail(`Swap override changed canonical identity at ordinal ${record.ordinal}.`);
198
199
  }
199
200
  const originalBytes = Buffer.byteLength(
200
- record.role === "assistant" ? (record.content ?? "") : record.content,
201
+ record.role === "assistant"
202
+ ? (record.content ?? "")
203
+ : record.role === "tool"
204
+ ? record.displayText
205
+ : record.content,
201
206
  "utf8",
202
207
  );
203
208
  const renderedBytes = Buffer.byteLength(override.renderedContent, "utf8");
@@ -208,8 +213,11 @@ function validateOverrideIdentity(
208
213
  override.originalBytes !== originalBytes ||
209
214
  override.renderedBytes !== renderedBytes ||
210
215
  override.byteSavings !== originalBytes - renderedBytes ||
211
- (rendererFormat === "swap-observation-v1" && override.byteSavings <= 0) ||
216
+ ((rendererFormat === "swap-observation-v1" ||
217
+ rendererFormat === "skill-activation-receipt-v1") &&
218
+ override.byteSavings <= 0) ||
212
219
  (rendererFormat !== "swap-observation-v1" &&
220
+ rendererFormat !== "swap-tool-image-v1" &&
213
221
  rendererFormat !== "skill-activation-receipt-v1")
214
222
  ) {
215
223
  fail(`Swap override metadata is invalid at ordinal ${record.ordinal}.`);
@@ -266,7 +274,8 @@ function assertSameMessage(actual: AgentMessage, record: CanonicalMessageRecord)
266
274
  actual.toolCallId !== record.toolCallId ||
267
275
  actual.providerToolCallId !== record.providerToolCallId ||
268
276
  actual.name !== record.name ||
269
- actual.content !== record.content
277
+ stableJsonStringify(actual.content) !== stableJsonStringify(record.content) ||
278
+ toolResultDisplayText(actual.content) !== record.displayText
270
279
  ) {
271
280
  fail(`Canonical entry changed tool data at ordinal ${record.ordinal}.`);
272
281
  }
@@ -1,8 +1,14 @@
1
1
  import type { MessageId, ProtocolFrameId, ToolCallId } from "../ids/runtime-id";
2
+ import {
3
+ canonicalToolResultContentHash,
4
+ toolResultDisplayText,
5
+ validateToolResultContent,
6
+ } from "../agent/tool-result-content";
2
7
  import {
3
8
  contentHash,
4
9
  userMessageHash,
5
10
  rawResultHash,
11
+ validateReturnedToolObservation,
6
12
  type CanonicalMessageRecord,
7
13
  type ProtocolContextView,
8
14
  type ProtocolFrame,
@@ -113,7 +119,9 @@ export class ContextProtocolValidator {
113
119
  ? {}
114
120
  : { attachments: message.attachments }),
115
121
  })
116
- : contentHash(message.content))
122
+ : message.role === "tool"
123
+ ? canonicalToolResultContentHash(message.content)
124
+ : contentHash(message.content))
117
125
  ) {
118
126
  fail(
119
127
  "content_hash_mismatch",
@@ -405,11 +413,19 @@ function validateToolExchange(
405
413
  );
406
414
  }
407
415
  const result = requireItem(results, 0, "tool result");
416
+ validateToolResultContent(message.content);
417
+ if (message.displayText !== toolResultDisplayText(message.content)) {
418
+ fail(
419
+ "content_hash_mismatch",
420
+ `Tool display projection does not match message ${message.messageId}.`,
421
+ identityForMessage(message),
422
+ );
423
+ }
408
424
  if (
409
425
  result.sessionId !== frame.sessionId ||
410
426
  result.frameId !== frame.frameId ||
411
427
  result.toolMessageId !== message.messageId ||
412
- result.observationSha256 !== contentHash(message.content)
428
+ result.observationSha256 !== canonicalToolResultContentHash(message.content)
413
429
  ) {
414
430
  fail(
415
431
  "tool_result_mismatch",
@@ -428,6 +444,21 @@ function validateToolExchange(
428
444
  { ...identityForMessage(message), toolCallId: call.toolCallId },
429
445
  );
430
446
  }
447
+ if (input.fullIntegrity && result.completion.kind === "returned") {
448
+ try {
449
+ validateReturnedToolObservation({
450
+ toolName: message.name,
451
+ raw: result.completion.raw,
452
+ content: message.content,
453
+ });
454
+ } catch (error) {
455
+ fail(
456
+ "tool_result_mismatch",
457
+ `Tool result for ${call.toolCallId} has invalid canonical content: ${error instanceof Error ? error.message : String(error)}`,
458
+ { ...identityForMessage(message), toolCallId: call.toolCallId },
459
+ );
460
+ }
461
+ }
431
462
  input.usedResultCallIds.add(call.toolCallId);
432
463
  }
433
464
  }
@@ -1,4 +1,5 @@
1
1
  import type { AgentMessage } from "../agent/types";
2
+ import { textToolResultContent } from "../agent/tool-result-content";
2
3
  import type { ContextRevisionId } from "../ids/runtime-id";
3
4
  import { CompiledContextValidator } from "./compiled-context-validator";
4
5
  import {
@@ -263,7 +264,7 @@ function swappedToolMessage(
263
264
  }
264
265
  return {
265
266
  ...message,
266
- content: override.renderedContent,
267
+ content: textToolResultContent(override.renderedContent),
267
268
  };
268
269
  }
269
270
 
@@ -133,12 +133,18 @@ export type SwapOverride = {
133
133
  readonly originalBytes: number;
134
134
  readonly renderedBytes: number;
135
135
  readonly byteSavings: number;
136
- readonly rendererFormat?: "swap-observation-v1" | "skill-activation-receipt-v1";
136
+ readonly rendererFormat?:
137
+ | "swap-observation-v1"
138
+ | "swap-tool-image-v1"
139
+ | "skill-activation-receipt-v1";
137
140
  };
138
141
 
139
142
  export type StoredContextOverrideV8 = SwapOverride & {
140
143
  readonly introducedRevisionId: ContextRevisionId;
141
- readonly rendererFormat: "swap-observation-v1" | "skill-activation-receipt-v1";
144
+ readonly rendererFormat:
145
+ | "swap-observation-v1"
146
+ | "swap-tool-image-v1"
147
+ | "skill-activation-receipt-v1";
142
148
  readonly createdAt: string;
143
149
  };
144
150
 
@@ -9,6 +9,7 @@ const MAX_METADATA_BYTES = 1_024;
9
9
  const MAX_SCALAR_BYTES = 256;
10
10
 
11
11
  export const SWAP_OBSERVATION_FORMAT = "swap-observation-v1" as const;
12
+ export const SWAP_TOOL_IMAGE_FORMAT = "swap-tool-image-v1" as const;
12
13
 
13
14
  export const SWAPPABLE_RAW_KINDS = Object.freeze([
14
15
  "read",
@@ -19,6 +20,7 @@ export const SWAPPABLE_RAW_KINDS = Object.freeze([
19
20
  "web_search",
20
21
  "web_fetch",
21
22
  "mcp",
23
+ "view_image",
22
24
  ] as const satisfies readonly ToolRawResultKind[]);
23
25
 
24
26
  export type SwappableRawKind = (typeof SWAPPABLE_RAW_KINDS)[number];
@@ -66,19 +68,21 @@ export class ContextSwapRenderer {
66
68
  assertHistoricalRawIsStable(raw);
67
69
 
68
70
  const source = formatMessageSource(message.messageId);
69
- const metadata = renderMetadata(raw);
70
- const renderedContent = [
71
- "[Tinker historical tool observation swapped]",
72
- `source=${source}`,
73
- `contentSha256=${message.contentSha256}`,
74
- `tool=${stableJsonStringify(compactExternalString(message.name))}`,
75
- `metadata=${metadata}`,
76
- "historical=Use RecallGet with source to recover the original observation.",
77
- `current=${currentGuidance(raw.kind)}`,
78
- ].join("\n");
79
- const originalBytes = utf8Bytes(message.content);
71
+ const renderedContent =
72
+ raw.kind === "view_image"
73
+ ? renderImagePlaceholder(raw)
74
+ : [
75
+ "[Tinker historical tool observation swapped]",
76
+ `source=${source}`,
77
+ `contentSha256=${message.contentSha256}`,
78
+ `tool=${stableJsonStringify(compactExternalString(message.name))}`,
79
+ `metadata=${renderMetadata(raw)}`,
80
+ "historical=Use RecallGet with source to recover the original observation.",
81
+ `current=${currentGuidance(raw.kind)}`,
82
+ ].join("\n");
83
+ const originalBytes = utf8Bytes(message.displayText);
80
84
  const renderedBytes = utf8Bytes(renderedContent);
81
- if (renderedBytes >= originalBytes) {
85
+ if (raw.kind !== "view_image" && renderedBytes >= originalBytes) {
82
86
  throw new SwapRenderUnsupportedError(
83
87
  "placeholder_not_smaller",
84
88
  "Rendered placeholder is not smaller than its canonical observation.",
@@ -95,6 +99,7 @@ export class ContextSwapRenderer {
95
99
  originalBytes,
96
100
  renderedBytes,
97
101
  byteSavings: originalBytes - renderedBytes,
102
+ ...(raw.kind === "view_image" ? { rendererFormat: SWAP_TOOL_IMAGE_FORMAT } : {}),
98
103
  });
99
104
  }
100
105
  }
@@ -123,6 +128,25 @@ function assertHistoricalRawIsStable(
123
128
  "Running task output cannot be rendered as a historical placeholder.",
124
129
  );
125
130
  }
131
+ if (raw.kind === "view_image" && (!raw.ok || raw.asset === undefined)) {
132
+ throw new SwapRenderUnsupportedError(
133
+ "unsuccessful_image",
134
+ "Only successful ViewImage results can use the image swap renderer.",
135
+ );
136
+ }
137
+ }
138
+
139
+ function renderImagePlaceholder(
140
+ raw: Extract<ToolRawResult, { kind: "view_image" }>,
141
+ ): string {
142
+ if (!raw.ok || raw.asset === undefined) {
143
+ throw new SwapRenderUnsupportedError(
144
+ "unsuccessful_image",
145
+ "Only successful ViewImage results can use the image swap renderer.",
146
+ );
147
+ }
148
+ const asset = raw.asset;
149
+ return `[Tool image omitted from compacted context: ViewImage ${raw.filePath}, ${asset.mimeType}, ${asset.width}x${asset.height}, asset=${asset.assetId.slice(0, 12)}…. Use ViewImage again if the current image is required.]`;
126
150
  }
127
151
 
128
152
  function renderMetadata(
@@ -213,6 +237,14 @@ function metadataEntries(
213
237
  ["isError", raw.isError],
214
238
  ["contentBlockCount", raw.contentBlockCount],
215
239
  ];
240
+ case "view_image":
241
+ return [
242
+ ["filePath", raw.filePath],
243
+ ["mimeType", raw.asset?.mimeType],
244
+ ["width", raw.asset?.width],
245
+ ["height", raw.asset?.height],
246
+ ["assetId", raw.asset?.assetId],
247
+ ];
216
248
  }
217
249
  }
218
250
 
@@ -234,6 +266,8 @@ function currentGuidance(kind: SwappableRawKind): string {
234
266
  return "Use WebFetch when the current page content is required.";
235
267
  case "mcp":
236
268
  return "Call the MCP tool again only when current external state is required.";
269
+ case "view_image":
270
+ return "Use ViewImage again if the current image is required.";
237
271
  }
238
272
  }
239
273
 
@@ -7,7 +7,11 @@ import {
7
7
  type PromptPrefixFingerprint,
8
8
  } from "../model/prompt-prefix-hash";
9
9
  import { sha256, stableJsonStringify } from "../model/model-request-preflight";
10
- import { estimatePromptSegments } from "../model/token-estimator";
10
+ import {
11
+ estimatePromptSegments,
12
+ guardedContextTokens,
13
+ type RawContextBreakdown,
14
+ } from "../model/token-estimator";
11
15
  import type { ToolDefinition } from "../tools/types";
12
16
  import {
13
17
  activeOverrideManifestHash,
@@ -150,11 +154,10 @@ export class PrefixRetirementPlanner {
150
154
  validatePlanningInput(input);
151
155
  const activeFingerprint = promptPrefixFingerprint(input.activePrepared);
152
156
  assertActiveFingerprint(input, activeFingerprint);
153
- const rawTokensBefore = estimatePromptSegments(
154
- input.activePrepared.promptSegments,
155
- ).totalTokens;
157
+ const activeBreakdown = estimatePromptSegments(input.activePrepared.promptSegments);
158
+ const rawTokensBefore = activeBreakdown.totalTokens;
156
159
  const guardedTokensBefore = guardTokens(
157
- rawTokensBefore,
160
+ activeBreakdown,
158
161
  input.activeUsage.correctionFactor,
159
162
  );
160
163
  const targetTokens = planningTarget(input);
@@ -223,7 +226,8 @@ export class PrefixRetirementPlanner {
223
226
  );
224
227
  }
225
228
  assertProspectiveConfiguration(input.activePrepared, prepared);
226
- const rawTokens = estimatePromptSegments(prepared.promptSegments).totalTokens;
229
+ const breakdown = estimatePromptSegments(prepared.promptSegments);
230
+ const rawTokens = breakdown.totalTokens;
227
231
  const projection = Object.freeze({
228
232
  candidateIndex,
229
233
  boundary,
@@ -231,7 +235,7 @@ export class PrefixRetirementPlanner {
231
235
  compiled,
232
236
  prepared,
233
237
  rawTokens,
234
- guardedTokens: guardTokens(rawTokens, input.activeUsage.correctionFactor),
238
+ guardedTokens: guardTokens(breakdown, input.activeUsage.correctionFactor),
235
239
  });
236
240
  projections.set(candidateIndex, projection);
237
241
  return projection;
@@ -603,8 +607,8 @@ function createPlan(input: {
603
607
  });
604
608
  }
605
609
 
606
- function guardTokens(rawTokens: number, correctionFactor: number): number {
607
- return Math.ceil(rawTokens * correctionFactor);
610
+ function guardTokens(breakdown: RawContextBreakdown, correctionFactor: number): number {
611
+ return guardedContextTokens(breakdown, correctionFactor);
608
612
  }
609
613
 
610
614
  function fail(code: string, message: string): never {
@@ -6,18 +6,26 @@ import type {
6
6
  AssistantMessage,
7
7
  IterationIdentity,
8
8
  ToolCall,
9
+ ToolResultContent,
9
10
  TurnIdentity,
10
11
  } from "../agent/types";
12
+ import {
13
+ canonicalToolResultContentHash,
14
+ textToolResultContent,
15
+ toolResultDisplayText,
16
+ validateToolResultContent,
17
+ } from "../agent/tool-result-content";
11
18
  import {
12
19
  canonicalUserMessageHash,
20
+ validateImageAssetRef,
21
+ validateOriginalImageName,
13
22
  validateUserMessage,
14
23
  type UserImageAttachment,
15
24
  type UserMessage,
16
25
  } from "../image/image-types";
17
26
 
18
- export const CURRENT_TOOL_OBSERVATION_FORMAT = "tool-observation-v3" as const;
27
+ export const CURRENT_TOOL_OBSERVATION_FORMAT = "tool-observation-v4" as const;
19
28
  export const SUPPORTED_TOOL_OBSERVATION_FORMATS = [
20
- "tool-observation-v2",
21
29
  CURRENT_TOOL_OBSERVATION_FORMAT,
22
30
  ] as const;
23
31
  export type SupportedToolObservationFormat =
@@ -63,7 +71,8 @@ export type CanonicalMessageRecord =
63
71
  readonly toolCallId: ToolCall["toolCallId"];
64
72
  readonly providerToolCallId: string;
65
73
  readonly name: string;
66
- readonly content: string;
74
+ readonly content: readonly ToolResultContent[];
75
+ readonly displayText: string;
67
76
  readonly origin: "tool" | "runtime";
68
77
  });
69
78
 
@@ -123,7 +132,7 @@ export type ReturnedToolCompletionInput = {
123
132
  readonly call: ToolCall;
124
133
  readonly kind: "returned";
125
134
  readonly raw: ToolRawResult;
126
- readonly observation: string;
135
+ readonly observation: readonly ToolResultContent[];
127
136
  };
128
137
 
129
138
  export type SyntheticToolCompletionInput = {
@@ -193,12 +202,13 @@ export function materializeAgentMessages(
193
202
  return message;
194
203
  }
195
204
  case "tool":
205
+ validateToolResultContent(record.content);
196
206
  return {
197
207
  role: "tool",
198
208
  toolCallId: record.toolCallId,
199
209
  providerToolCallId: record.providerToolCallId,
200
210
  name: record.name,
201
- content: record.content,
211
+ content: canonicalClone(record.content),
202
212
  };
203
213
  }
204
214
  });
@@ -243,12 +253,69 @@ export function interruptedCompletionInputs(
243
253
  }));
244
254
  }
245
255
 
246
- export function observationForCompletion(input: ToolCompletionInput): string {
256
+ export function observationForCompletion(
257
+ input: ToolCompletionInput,
258
+ ): readonly ToolResultContent[] {
247
259
  return input.kind === "returned"
248
260
  ? input.observation
249
- : renderSyntheticToolObservation(input.reason, input.detail);
261
+ : textToolResultContent(renderSyntheticToolObservation(input.reason, input.detail));
262
+ }
263
+
264
+ export function displayTextForCompletion(input: ToolCompletionInput): string {
265
+ return toolResultDisplayText(observationForCompletion(input));
266
+ }
267
+
268
+ export function validateReturnedToolObservation(input: {
269
+ readonly toolName: string;
270
+ readonly raw: ToolRawResult;
271
+ readonly content: readonly ToolResultContent[];
272
+ }): void {
273
+ validateToolResultContent(input.content);
274
+ if (input.raw.kind !== "view_image") {
275
+ return;
276
+ }
277
+ if (input.toolName !== "ViewImage") {
278
+ throw new Error("ViewImage raw result has an invalid tool name.");
279
+ }
280
+ const raw = input.raw;
281
+ if (!raw.ok) {
282
+ if (raw.asset !== undefined || raw.originalName !== undefined) {
283
+ throw new Error("Failed ViewImage result cannot contain image metadata.");
284
+ }
285
+ const expected = `ViewImage failed for ${raw.filePath || "(unknown path)"}: ${raw.error ?? "Unknown error."}`;
286
+ if (
287
+ input.content.length !== 1 ||
288
+ input.content[0]?.type !== "text" ||
289
+ input.content[0].text !== expected
290
+ ) {
291
+ throw new Error("Failed ViewImage observation is not canonical.");
292
+ }
293
+ return;
294
+ }
295
+ if (
296
+ raw.asset === undefined ||
297
+ raw.originalName === undefined ||
298
+ raw.filePath.trim() === "" ||
299
+ raw.error !== undefined
300
+ ) {
301
+ throw new Error("Successful ViewImage result is missing required metadata.");
302
+ }
303
+ validateImageAssetRef(raw.asset);
304
+ validateOriginalImageName(raw.originalName);
305
+ const expectedText = `Viewed image ${raw.filePath} (${raw.asset.mimeType}, ${raw.asset.width}x${raw.asset.height}, ${raw.asset.byteLength} bytes, asset=${raw.asset.assetId.slice(0, 12)}…).`;
306
+ if (
307
+ input.content.length !== 2 ||
308
+ input.content[0]?.type !== "text" ||
309
+ input.content[0].text !== expectedText ||
310
+ input.content[1]?.type !== "image" ||
311
+ stableJsonStringify(input.content[1].asset) !== stableJsonStringify(raw.asset)
312
+ ) {
313
+ throw new Error("Successful ViewImage observation is not canonical.");
314
+ }
250
315
  }
251
316
 
317
+ export { canonicalToolResultContentHash };
318
+
252
319
  function requireSyntheticDetail(detail: string | undefined): string {
253
320
  if (detail === undefined || detail.trim() === "") {
254
321
  throw new Error("failed_active synthetic completion requires error detail.");
@@ -7,7 +7,11 @@ import {
7
7
  type PromptPrefixFingerprint,
8
8
  } from "../model/prompt-prefix-hash";
9
9
  import { sha256, stableJsonStringify } from "../model/model-request-preflight";
10
- import { estimatePromptSegments } from "../model/token-estimator";
10
+ import {
11
+ estimatePromptSegments,
12
+ guardedContextTokens,
13
+ type RawContextBreakdown,
14
+ } from "../model/token-estimator";
11
15
  import type { ToolDefinition } from "../tools/types";
12
16
  import { activeOverrideManifestHash } from "./compiled-context-hash";
13
17
  import { CompiledContextError } from "./compiled-context-validator";
@@ -148,11 +152,10 @@ export class SwapPlanner {
148
152
  validatePlanningInput(input);
149
153
  const activeFingerprint = promptPrefixFingerprint(input.activePrepared);
150
154
  assertActiveFingerprint(input, activeFingerprint);
151
- const rawTokensBefore = estimatePromptSegments(
152
- input.activePrepared.promptSegments,
153
- ).totalTokens;
155
+ const activeBreakdown = estimatePromptSegments(input.activePrepared.promptSegments);
156
+ const rawTokensBefore = activeBreakdown.totalTokens;
154
157
  const guardedTokensBefore = guardTokens(
155
- rawTokensBefore,
158
+ activeBreakdown,
156
159
  input.activeUsage.correctionFactor,
157
160
  );
158
161
  const targetTokens = planningTarget(input);
@@ -238,12 +241,13 @@ export class SwapPlanner {
238
241
  );
239
242
  }
240
243
  assertProspectiveConfiguration(input.activePrepared, prepared);
241
- const rawTokens = estimatePromptSegments(prepared.promptSegments).totalTokens;
244
+ const breakdown = estimatePromptSegments(prepared.promptSegments);
245
+ const rawTokens = breakdown.totalTokens;
242
246
  const projection = Object.freeze({
243
247
  count,
244
248
  prepared,
245
249
  rawTokens,
246
- guardedTokens: guardTokens(rawTokens, input.activeUsage.correctionFactor),
250
+ guardedTokens: guardTokens(breakdown, input.activeUsage.correctionFactor),
247
251
  });
248
252
  projectionCache.set(count, projection);
249
253
  return projection;
@@ -615,15 +619,16 @@ function basicExclusionReason(input: {
615
619
  ) {
616
620
  return "active_turn_unconsumed";
617
621
  }
618
- if (
619
- Buffer.byteLength(input.message.content, "utf8") < input.minimumObservationBytes
620
- ) {
621
- return "observation_too_small";
622
- }
623
622
  const raw = input.result.completion.raw;
624
623
  if (!isSwappableRawResult(raw)) {
625
624
  return "raw_kind_not_allowlisted";
626
625
  }
626
+ if (
627
+ raw.kind !== "view_image" &&
628
+ Buffer.byteLength(input.message.displayText, "utf8") < input.minimumObservationBytes
629
+ ) {
630
+ return "observation_too_small";
631
+ }
627
632
  if (
628
633
  (raw.kind === "bash" && raw.status === "running") ||
629
634
  (raw.kind === "task_output" &&
@@ -742,8 +747,8 @@ function increment(counts: Map<string, number>, key: string): void {
742
747
  counts.set(key, (counts.get(key) ?? 0) + 1);
743
748
  }
744
749
 
745
- function guardTokens(rawTokens: number, correctionFactor: number): number {
746
- return Math.ceil(rawTokens * correctionFactor);
750
+ function guardTokens(breakdown: RawContextBreakdown, correctionFactor: number): number {
751
+ return guardedContextTokens(breakdown, correctionFactor);
747
752
  }
748
753
 
749
754
  function isCanonicalPlanningError(error: unknown): boolean {
@@ -209,7 +209,7 @@ function toolCallSummary(call: ToolCall): string {
209
209
  }
210
210
 
211
211
  function observationContent(observation: ToolObservation): string {
212
- return observation.content;
212
+ return observation.displayText;
213
213
  }
214
214
 
215
215
  function asRecord(value: unknown): Record<string, unknown> {
@@ -122,6 +122,9 @@ export class StdoutEventPrinter implements EventSink {
122
122
  `${formatToolLine("tool.finished", event.data.call).trimEnd()} ok=${event.data.ok}\n`,
123
123
  );
124
124
  break;
125
+ case "tool.observation":
126
+ this.stdout.write(`${event.data.observation.displayText}\n`);
127
+ break;
125
128
  case "tool.confirmation.requested":
126
129
  this.stdout.write(
127
130
  `tool.confirmation.requested toolCallId=${event.toolCallId} reason=${JSON.stringify(event.data.reason)} command=${JSON.stringify(event.data.command)}\n`,
@@ -217,6 +220,7 @@ function formatToolRawResult(call: ToolCall, raw: ToolRawResult): string[] {
217
220
  case "skill":
218
221
  return [formatSkillResult(raw)];
219
222
  case "read":
223
+ case "view_image":
220
224
  case "delete":
221
225
  case "glob":
222
226
  case "grep":
@@ -224,6 +228,8 @@ function formatToolRawResult(call: ToolCall, raw: ToolRawResult): string[] {
224
228
  case "web_fetch":
225
229
  case "recall":
226
230
  case "memory_search":
231
+ case "memory_get":
232
+ case "wait":
227
233
  case "mcp":
228
234
  case "generic":
229
235
  return [];
@@ -70,18 +70,45 @@ export class ImageAssetStore {
70
70
  signal?: AbortSignal;
71
71
  accept?: (asset: ImageAssetRef) => void;
72
72
  } = {},
73
+ ): Promise<ImportedImageAsset> {
74
+ return this.importFileInternal(sourcePath, true, options);
75
+ }
76
+
77
+ async importFile(
78
+ sourcePath: string,
79
+ options: {
80
+ signal?: AbortSignal;
81
+ } = {},
82
+ ): Promise<ImportedImageAsset> {
83
+ return this.importFileInternal(sourcePath, false, options);
84
+ }
85
+
86
+ private async importFileInternal(
87
+ sourcePath: string,
88
+ workspaceOnly: boolean,
89
+ options: {
90
+ signal?: AbortSignal;
91
+ accept?: (asset: ImageAssetRef) => void;
92
+ },
73
93
  ): Promise<ImportedImageAsset> {
74
94
  throwIfAborted(options.signal);
95
+ if (typeof sourcePath !== "string" || sourcePath.trim() === "") {
96
+ throw new Error("Image source path must be a non-empty string.");
97
+ }
75
98
  const candidate = path.isAbsolute(sourcePath)
76
99
  ? path.normalize(sourcePath)
77
100
  : path.resolve(this.workspaceRoot, sourcePath);
78
- assertContained(this.workspaceRoot, candidate, "Image source path");
101
+ if (workspaceOnly || !path.isAbsolute(sourcePath)) {
102
+ assertContained(this.workspaceRoot, candidate, "Image source path");
103
+ }
79
104
  const pathStat = await lstat(candidate);
80
105
  if (pathStat.isSymbolicLink() || !pathStat.isFile()) {
81
106
  throw new Error("Image source must be a regular non-symlink file.");
82
107
  }
83
108
  const canonicalSource = await realpath(candidate);
84
- assertContained(this.workspaceRoot, canonicalSource, "Image source realpath");
109
+ if (workspaceOnly || !path.isAbsolute(sourcePath)) {
110
+ assertContained(this.workspaceRoot, canonicalSource, "Image source realpath");
111
+ }
85
112
 
86
113
  const handle = await open(canonicalSource, constants.O_RDONLY | noFollowFlag());
87
114
  let bytes: Buffer;
@@ -102,7 +129,9 @@ export class ImageAssetStore {
102
129
  `Image is ${handleStat.size} bytes; maximum is ${IMAGE_INPUT_POLICY.maxBytesPerImage}.`,
103
130
  );
104
131
  }
105
- bytes = await handle.readFile();
132
+ bytes = await handle.readFile(
133
+ options.signal === undefined ? undefined : { signal: options.signal },
134
+ );
106
135
  } finally {
107
136
  await handle.close();
108
137
  }