tinker-agent 1.11.0 → 2.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (58) hide show
  1. package/CHANGELOG.md +44 -1
  2. package/README.md +33 -24
  3. package/package.json +2 -1
  4. package/src/agent/context-meter.ts +12 -85
  5. package/src/agent/loop.ts +1 -14
  6. package/src/agent/runtime-session.ts +9 -25
  7. package/src/agent/session-ledger.ts +12 -5
  8. package/src/agent/tool-result-content.ts +76 -0
  9. package/src/agent/types.ts +14 -2
  10. package/src/cli/config.ts +4 -5
  11. package/src/cli/model-profiles.ts +38 -97
  12. package/src/cli/public-config-contract.ts +24 -88
  13. package/src/cli/runner-dependencies.ts +5 -7
  14. package/src/cli/tui-memory.ts +1 -3
  15. package/src/cli/tui-runner.tsx +4 -0
  16. package/src/context/compiled-context-hash.ts +2 -1
  17. package/src/context/compiled-context-validator.ts +13 -4
  18. package/src/context/context-protocol-validator.ts +33 -2
  19. package/src/context/context-revision-compiler.ts +2 -1
  20. package/src/context/context-revision.ts +8 -2
  21. package/src/context/context-swap-renderer.ts +46 -12
  22. package/src/context/prefix-retirement-planner.ts +13 -9
  23. package/src/context/protocol-frame.ts +74 -7
  24. package/src/context/swap-planner.ts +19 -14
  25. package/src/events/observation-text-log.ts +1 -1
  26. package/src/events/stdout-event-printer.ts +6 -0
  27. package/src/image/image-asset-store.ts +32 -3
  28. package/src/image/image-input-policy.ts +53 -2
  29. package/src/image/image-probe.ts +8 -2
  30. package/src/image/provider-image.ts +99 -0
  31. package/src/memory/contracts.ts +61 -3
  32. package/src/memory/memory-coordinator.ts +313 -49
  33. package/src/memory/memory-extractor.ts +48 -48
  34. package/src/memory/memory-get-tool.ts +86 -0
  35. package/src/memory/memory-search-tool.ts +122 -33
  36. package/src/memory/memory-store.ts +227 -20
  37. package/src/model/fake-model-client.ts +177 -124
  38. package/src/model/model-client.ts +62 -11
  39. package/src/model/model-request-preflight.ts +0 -1
  40. package/src/model/openai-chat-mapping.ts +2 -1
  41. package/src/model/openai-chat-model-client.ts +26 -38
  42. package/src/model/openai-model-utils.ts +109 -40
  43. package/src/model/openai-responses-mapping.ts +25 -1
  44. package/src/model/openai-responses-model-client.ts +27 -40
  45. package/src/model/token-estimator.ts +26 -3
  46. package/src/observation/observation-builder.ts +100 -25
  47. package/src/session/session-history-reader.ts +128 -5
  48. package/src/session/session-schema.ts +59 -9
  49. package/src/session/session-store.ts +342 -205
  50. package/src/tools/registry.ts +18 -0
  51. package/src/tools/types.ts +46 -0
  52. package/src/tools/view-image.ts +89 -0
  53. package/src/tools/wait.ts +85 -0
  54. package/src/tui/components/memory-browser.tsx +3 -0
  55. package/src/tui/event-store.ts +61 -2
  56. package/src/model/input-token-estimator.ts +0 -25
  57. package/src/model/moonshot-input-token-estimator.ts +0 -111
  58. package/src/model/openai-responses-token-estimator.ts +0 -155
@@ -32,7 +32,6 @@ import {
32
32
  MODEL_MESSAGE_PROTOCOL_ADAPTERS,
33
33
  type ModelMessageProtocol,
34
34
  } from "../model/model-client";
35
- import type { InputTokenEstimatorCompatibility } from "../model/input-token-estimator";
36
35
  import type { ToolDefinition, ToolRawResult } from "../tools/types";
37
36
  import { sha256, stableJsonStringify } from "../model/model-request-preflight";
38
37
  import {
@@ -51,6 +50,7 @@ import {
51
50
  import {
52
51
  ContextSwapRenderer,
53
52
  SWAP_OBSERVATION_FORMAT,
53
+ SWAP_TOOL_IMAGE_FORMAT,
54
54
  } from "../context/context-swap-renderer";
55
55
  import {
56
56
  SUPPORTED_TOOL_OBSERVATION_FORMATS,
@@ -79,10 +79,16 @@ import type {
79
79
  StoredContextOverrideV8,
80
80
  SwapOverride,
81
81
  } from "../context/context-revision";
82
- import type { IterationIdentity, ToolCall } from "../agent/types";
82
+ import type { IterationIdentity, ToolCall, ToolResultContent } from "../agent/types";
83
+ import {
84
+ canonicalToolResultContentHash,
85
+ toolResultDisplayText,
86
+ validateToolResultContent,
87
+ } from "../agent/tool-result-content";
83
88
  import {
84
89
  parseImageAssetId,
85
90
  parseImageAttachmentId,
91
+ validateImageAssetRef,
86
92
  validateOriginalImageName,
87
93
  validateUserMessage,
88
94
  type ImageAssetRef,
@@ -109,7 +115,7 @@ import {
109
115
  } from "./session-history-reader";
110
116
  import { SessionLease } from "./session-lock";
111
117
  import {
112
- SESSION_SCHEMA_V9_FINGERPRINT,
118
+ SESSION_SCHEMA_V10_FINGERPRINT,
113
119
  SESSION_SCHEMA_VERSION,
114
120
  upgradeActiveTurnRetirementContract,
115
121
  upgradeRecallIndexContract,
@@ -145,11 +151,11 @@ import {
145
151
  IMAGE_INPUT_POLICY_VERSION,
146
152
  } from "../image/image-input-policy";
147
153
 
148
- export type SessionImageInputCompatibility = {
154
+ export type SessionMediaCompatibility = {
149
155
  readonly policyVersion: string;
150
156
  readonly policySha256: string;
151
157
  readonly inputModalities: readonly ("text" | "image")[];
152
- readonly tokenEstimator?: InputTokenEstimatorCompatibility;
158
+ readonly toolResultModalities: readonly ("text" | "image")[];
153
159
  };
154
160
 
155
161
  export type CompletedTurnMessageSnapshot =
@@ -181,11 +187,11 @@ export type SessionCompatibilityContract = {
181
187
  includeReasoningContent: boolean;
182
188
  contextProfile: ModelContextProfile;
183
189
  messageProtocol: ModelMessageProtocol;
184
- imageInput: SessionImageInputCompatibility;
190
+ media: SessionMediaCompatibility;
185
191
  };
186
192
 
187
- export type StoredSessionMetaV9 = {
188
- schemaVersion: 9;
193
+ export type StoredSessionMetaV10 = {
194
+ schemaVersion: 10;
189
195
  schemaFingerprint: string;
190
196
  initializationState: "creating" | "ready";
191
197
  sessionId: SessionId;
@@ -212,7 +218,7 @@ export type StoredSessionMetaV9 = {
212
218
  | null;
213
219
  };
214
220
 
215
- export type SessionCloseReason = NonNullable<StoredSessionMetaV9["lastCloseReason"]>;
221
+ export type SessionCloseReason = NonNullable<StoredSessionMetaV10["lastCloseReason"]>;
216
222
 
217
223
  export type SessionRecoveryResult = {
218
224
  recoveredTurnId?: TurnId;
@@ -509,7 +515,7 @@ export class SessionStore implements SessionLedgerCommitter {
509
515
  )
510
516
  .run(
511
517
  SESSION_SCHEMA_VERSION,
512
- SESSION_SCHEMA_V9_FINGERPRINT,
518
+ SESSION_SCHEMA_V10_FINGERPRINT,
513
519
  input.sessionId,
514
520
  workspaceRoot,
515
521
  input.modelName,
@@ -1692,7 +1698,7 @@ export class SessionStore implements SessionLedgerCommitter {
1692
1698
  messageId: message.messageId,
1693
1699
  frameId: message.frameId,
1694
1700
  ordinal: message.ordinal,
1695
- content: message.content,
1701
+ content: message.displayText,
1696
1702
  contentSha256: message.contentSha256,
1697
1703
  },
1698
1704
  name: activation.name,
@@ -2076,13 +2082,14 @@ export class SessionStore implements SessionLedgerCommitter {
2076
2082
  for (const input of completionInputs) {
2077
2083
  const createdAt = this.clock();
2078
2084
  const content = observationForCompletion(input);
2085
+ const displayText = toolResultDisplayText(content);
2079
2086
  const messageId = idFactory.createMessageId();
2080
2087
  const message = immutableRecord<CanonicalMessageRecord>({
2081
2088
  messageId,
2082
2089
  sessionId: this.sessionId,
2083
2090
  frameId: frame.frameId,
2084
2091
  ordinal: view.messages.length + messages.length + 1,
2085
- contentSha256: contentHash(content),
2092
+ contentSha256: canonicalToolResultContentHash(content),
2086
2093
  createdAt,
2087
2094
  role: "tool",
2088
2095
  turnId,
@@ -2091,6 +2098,7 @@ export class SessionStore implements SessionLedgerCommitter {
2091
2098
  providerToolCallId: input.call.providerToolCallId,
2092
2099
  name: input.call.name,
2093
2100
  content,
2101
+ displayText,
2094
2102
  origin: "runtime",
2095
2103
  });
2096
2104
  const completion: ToolCompletion = immutableRecord({
@@ -2103,7 +2111,7 @@ export class SessionStore implements SessionLedgerCommitter {
2103
2111
  toolCallId: input.call.toolCallId,
2104
2112
  toolMessageId: messageId,
2105
2113
  completion,
2106
- observationSha256: contentHash(content),
2114
+ observationSha256: canonicalToolResultContentHash(content),
2107
2115
  createdAt,
2108
2116
  });
2109
2117
  messages.push(message);
@@ -2289,6 +2297,7 @@ export class SessionStore implements SessionLedgerCommitter {
2289
2297
  loadProtocolView(): ProtocolContextView {
2290
2298
  this.requireOpen();
2291
2299
  const imageAttachments = loadMessageImageAttachments(this.database);
2300
+ const toolContentBlocks = loadToolMessageContentBlocks(this.database);
2292
2301
  const frames = this.database
2293
2302
  .query("SELECT * FROM protocol_frames ORDER BY first_ordinal")
2294
2303
  .all()
@@ -2299,13 +2308,21 @@ export class SessionStore implements SessionLedgerCommitter {
2299
2308
  .map((row) => {
2300
2309
  const record = recordFromSql(row, "message");
2301
2310
  const messageId = stringFromSql(record.message_id, "message_id");
2302
- const message = decodeMessage(row, imageAttachments.get(messageId));
2311
+ const message = decodeMessage(
2312
+ row,
2313
+ imageAttachments.get(messageId),
2314
+ toolContentBlocks.get(messageId),
2315
+ );
2303
2316
  imageAttachments.delete(messageId);
2317
+ toolContentBlocks.delete(messageId);
2304
2318
  return message;
2305
2319
  });
2306
2320
  if (imageAttachments.size > 0) {
2307
2321
  throw new Error("Image attachment rows reference unknown messages.");
2308
2322
  }
2323
+ if (toolContentBlocks.size > 0) {
2324
+ throw new Error("Tool content block rows reference unknown messages.");
2325
+ }
2309
2326
  const toolResults = this.database
2310
2327
  .query(
2311
2328
  `SELECT tr.* FROM tool_results tr
@@ -2327,21 +2344,23 @@ export class SessionStore implements SessionLedgerCommitter {
2327
2344
  this.requireOpen();
2328
2345
  const distinct = new Map<string, ImageAssetRef>();
2329
2346
  for (const message of this.loadProtocolView().messages) {
2330
- if (message.role !== "user") {
2331
- continue;
2332
- }
2333
- for (const attachment of message.attachments ?? []) {
2334
- const previous = distinct.get(attachment.assetId);
2335
- const asset = imageAssetRefFromAttachment(attachment);
2347
+ const assets =
2348
+ message.role === "user"
2349
+ ? (message.attachments ?? []).map(imageAssetRefFromAttachment)
2350
+ : message.role === "tool"
2351
+ ? message.content.flatMap((block) =>
2352
+ block.type === "image" ? [block.asset] : [],
2353
+ )
2354
+ : [];
2355
+ for (const asset of assets) {
2356
+ const previous = distinct.get(asset.assetId);
2336
2357
  if (
2337
2358
  previous !== undefined &&
2338
2359
  stableJsonStringify(previous) !== stableJsonStringify(asset)
2339
2360
  ) {
2340
- throw new Error(
2341
- `Conflicting metadata for image asset ${attachment.assetId}.`,
2342
- );
2361
+ throw new Error(`Conflicting metadata for image asset ${asset.assetId}.`);
2343
2362
  }
2344
- distinct.set(attachment.assetId, asset);
2363
+ distinct.set(asset.assetId, asset);
2345
2364
  }
2346
2365
  }
2347
2366
  if (distinct.size === 0) {
@@ -2359,7 +2378,7 @@ export class SessionStore implements SessionLedgerCommitter {
2359
2378
  try {
2360
2379
  if (
2361
2380
  meta.sessionId !== this.sessionId ||
2362
- meta.schemaFingerprint !== SESSION_SCHEMA_V9_FINGERPRINT
2381
+ meta.schemaFingerprint !== SESSION_SCHEMA_V10_FINGERPRINT
2363
2382
  ) {
2364
2383
  throw new Error("Session metadata identity or schema fingerprint changed.");
2365
2384
  }
@@ -2405,7 +2424,7 @@ export class SessionStore implements SessionLedgerCommitter {
2405
2424
  }
2406
2425
  }
2407
2426
 
2408
- readMeta(): StoredSessionMetaV9 {
2427
+ readMeta(): StoredSessionMetaV10 {
2409
2428
  this.requireOpen();
2410
2429
  const rows = this.database.query("SELECT * FROM session_meta").all();
2411
2430
  if (rows.length !== 1) {
@@ -2541,7 +2560,7 @@ export class SessionStore implements SessionLedgerCommitter {
2541
2560
  const meta = this.readMeta();
2542
2561
  if (
2543
2562
  meta.sessionId !== this.sessionId ||
2544
- meta.schemaFingerprint !== SESSION_SCHEMA_V9_FINGERPRINT ||
2563
+ meta.schemaFingerprint !== SESSION_SCHEMA_V10_FINGERPRINT ||
2545
2564
  meta.initializationState !== "ready" ||
2546
2565
  meta.activeRevisionId === null
2547
2566
  ) {
@@ -3039,7 +3058,7 @@ export class SessionStore implements SessionLedgerCommitter {
3039
3058
  }
3040
3059
 
3041
3060
  private loadValidatedContextSnapshot(
3042
- meta: StoredSessionMetaV9,
3061
+ meta: StoredSessionMetaV10,
3043
3062
  canonical: ProtocolContextView,
3044
3063
  ): StoredContextSnapshotV8 {
3045
3064
  const activeRevisionId = requireActiveRevisionId(meta);
@@ -3325,7 +3344,8 @@ export class SessionStore implements SessionLedgerCommitter {
3325
3344
  if (
3326
3345
  (revision?.kind !== "swap_only" && revision?.kind !== "skills_update") ||
3327
3346
  (revision.kind === "swap_only" &&
3328
- override.rendererFormat !== SWAP_OBSERVATION_FORMAT) ||
3347
+ override.rendererFormat !== SWAP_OBSERVATION_FORMAT &&
3348
+ override.rendererFormat !== SWAP_TOOL_IMAGE_FORMAT) ||
3329
3349
  (revision.kind === "skills_update" &&
3330
3350
  override.rendererFormat !== SKILL_ACTIVATION_RECEIPT_FORMAT) ||
3331
3351
  revisionNumberById.get(revision.revisionId) === undefined ||
@@ -3342,7 +3362,8 @@ export class SessionStore implements SessionLedgerCommitter {
3342
3362
  throw new Error("Stored context override canonical identity is invalid.");
3343
3363
  }
3344
3364
  const rendered =
3345
- override.rendererFormat === SWAP_OBSERVATION_FORMAT
3365
+ override.rendererFormat === SWAP_OBSERVATION_FORMAT ||
3366
+ override.rendererFormat === SWAP_TOOL_IMAGE_FORMAT
3346
3367
  ? this.swapRenderer.render({ message, result })
3347
3368
  : this.renderStoredSkillReceipt(message, result, revision.revisionId);
3348
3369
  if (
@@ -3461,7 +3482,7 @@ export class SessionStore implements SessionLedgerCommitter {
3461
3482
  messageId: message.messageId,
3462
3483
  frameId: message.frameId,
3463
3484
  ordinal: message.ordinal,
3464
- content: message.content,
3485
+ content: message.displayText,
3465
3486
  contentSha256: message.contentSha256,
3466
3487
  },
3467
3488
  name: activation.name,
@@ -3487,7 +3508,10 @@ export class SessionStore implements SessionLedgerCommitter {
3487
3508
  : decodeMeasuredContextState(row, this.sessionId);
3488
3509
  }
3489
3510
 
3490
- private validateCounters(meta: StoredSessionMetaV9, view: ProtocolContextView): void {
3511
+ private validateCounters(
3512
+ meta: StoredSessionMetaV10,
3513
+ view: ProtocolContextView,
3514
+ ): void {
3491
3515
  const turns = this.database
3492
3516
  .query("SELECT * FROM turns ORDER BY turn_number")
3493
3517
  .all() as Array<Record<string, unknown>>;
@@ -3703,8 +3727,8 @@ export function createSessionCompatibilityContract(input: {
3703
3727
  includeReasoningContent: boolean;
3704
3728
  contextProfile: ModelContextProfile;
3705
3729
  messageProtocol: ModelMessageProtocol;
3706
- inputModalities?: readonly ("text" | "image")[];
3707
- tokenEstimator?: InputTokenEstimatorCompatibility;
3730
+ inputModalities: readonly ("text" | "image")[];
3731
+ toolResultModalities: readonly ("text" | "image")[];
3708
3732
  }): SessionCompatibilityContract {
3709
3733
  if (input.modelName.trim() === "") {
3710
3734
  throw new Error("Session compatibility model name must not be empty.");
@@ -3721,14 +3745,10 @@ export function createSessionCompatibilityContract(input: {
3721
3745
  ) {
3722
3746
  throw new Error("Session compatibility message protocol is invalid.");
3723
3747
  }
3724
- const inputModalities = normalizeInputModalities(input.inputModalities ?? ["text"]);
3725
- if (inputModalities.includes("image") && input.tokenEstimator === undefined) {
3726
- throw new Error(
3727
- "Session compatibility image input requires a token estimator identity.",
3728
- );
3729
- }
3730
- if (input.tokenEstimator !== undefined) {
3731
- validateTokenEstimatorCompatibility(input.tokenEstimator);
3748
+ const inputModalities = normalizeInputModalities(input.inputModalities);
3749
+ const toolResultModalities = normalizeInputModalities(input.toolResultModalities);
3750
+ if (toolResultModalities.includes("image") && !inputModalities.includes("image")) {
3751
+ throw new Error("Session compatibility image tool results require image input.");
3732
3752
  }
3733
3753
  return Object.freeze({
3734
3754
  modelName: input.modelName,
@@ -3736,7 +3756,7 @@ export function createSessionCompatibilityContract(input: {
3736
3756
  includeReasoningContent: input.includeReasoningContent,
3737
3757
  contextProfile: Object.freeze(createModelContextProfile(input.contextProfile)),
3738
3758
  messageProtocol: immutableCanonicalClone(input.messageProtocol),
3739
- imageInput: Object.freeze({
3759
+ media: Object.freeze({
3740
3760
  policyVersion: IMAGE_INPUT_POLICY_VERSION,
3741
3761
  policySha256: sha256(
3742
3762
  stableJsonStringify({
@@ -3745,9 +3765,7 @@ export function createSessionCompatibilityContract(input: {
3745
3765
  }),
3746
3766
  ),
3747
3767
  inputModalities,
3748
- ...(input.tokenEstimator === undefined
3749
- ? {}
3750
- : { tokenEstimator: immutableCanonicalClone(input.tokenEstimator) }),
3768
+ toolResultModalities,
3751
3769
  }),
3752
3770
  });
3753
3771
  }
@@ -3763,10 +3781,8 @@ function normalizeSessionCompatibilityContract(
3763
3781
  includeReasoningContent: contract.includeReasoningContent,
3764
3782
  contextProfile: contract.contextProfile,
3765
3783
  messageProtocol: contract.messageProtocol,
3766
- inputModalities: contract.imageInput.inputModalities,
3767
- ...(contract.imageInput.tokenEstimator === undefined
3768
- ? {}
3769
- : { tokenEstimator: contract.imageInput.tokenEstimator }),
3784
+ inputModalities: contract.media.inputModalities,
3785
+ toolResultModalities: contract.media.toolResultModalities,
3770
3786
  });
3771
3787
  }
3772
3788
 
@@ -3786,40 +3802,6 @@ function normalizeInputModalities(
3786
3802
  );
3787
3803
  }
3788
3804
 
3789
- function validateTokenEstimatorCompatibility(
3790
- estimator: InputTokenEstimatorCompatibility,
3791
- ): void {
3792
- if (
3793
- estimator.kind !== "moonshot-estimate-token-count-v1" ||
3794
- estimator.coverageVersion !== "full-request-v1" ||
3795
- estimator.model.trim() === "" ||
3796
- !isNormalizedEstimatorEndpoint(estimator.endpoint) ||
3797
- !Number.isSafeInteger(estimator.timeoutMs) ||
3798
- estimator.timeoutMs < 1_000 ||
3799
- estimator.timeoutMs > 60_000 ||
3800
- estimator.maxRetries !== 0
3801
- ) {
3802
- throw new Error("Session compatibility token estimator identity is invalid.");
3803
- }
3804
- }
3805
-
3806
- function isNormalizedEstimatorEndpoint(value: string): boolean {
3807
- try {
3808
- const url = new URL(value);
3809
- return (
3810
- (url.protocol === "https:" || url.protocol === "http:") &&
3811
- url.username === "" &&
3812
- url.password === "" &&
3813
- url.search === "" &&
3814
- url.hash === "" &&
3815
- url.pathname.endsWith("/tokenizers/estimate-token-count") &&
3816
- url.toString() === value
3817
- );
3818
- } catch {
3819
- return false;
3820
- }
3821
- }
3822
-
3823
3805
  export function sessionDatabasePath(
3824
3806
  workspaceRoot: string,
3825
3807
  sessionId: SessionId,
@@ -3873,7 +3855,7 @@ function insertMessage(database: Database, message: CanonicalMessageRecord): voi
3873
3855
  message.role,
3874
3856
  turnId,
3875
3857
  iterationId,
3876
- message.content,
3858
+ message.role === "tool" ? message.displayText : message.content,
3877
3859
  message.contentSha256,
3878
3860
  assistant?.reasoningContent ?? null,
3879
3861
  reasoningPresent,
@@ -3891,6 +3873,9 @@ function insertMessage(database: Database, message: CanonicalMessageRecord): voi
3891
3873
  if (message.role === "user" && message.attachments !== undefined) {
3892
3874
  insertMessageImageAttachments(database, message);
3893
3875
  }
3876
+ if (message.role === "tool") {
3877
+ insertToolMessageContentBlocks(database, message);
3878
+ }
3894
3879
  }
3895
3880
 
3896
3881
  function insertMessageImageAttachments(
@@ -3908,45 +3893,11 @@ function insertMessageImageAttachments(
3908
3893
  }
3909
3894
  for (let position = 0; position < message.attachments!.length; position += 1) {
3910
3895
  const attachment = requireItem(message.attachments!, position, "image attachment");
3911
- const existing = database
3912
- .query(
3913
- `SELECT mime_type, byte_length, width, height, created_at
3914
- FROM image_assets WHERE asset_id = ?`,
3915
- )
3916
- .get(attachment.assetId) as {
3917
- mime_type: unknown;
3918
- byte_length: unknown;
3919
- width: unknown;
3920
- height: unknown;
3921
- created_at: unknown;
3922
- } | null;
3923
- if (existing === null) {
3924
- database
3925
- .query(
3926
- `INSERT INTO image_assets (
3927
- asset_id, mime_type, byte_length, width, height, created_at
3928
- ) VALUES (?, ?, ?, ?, ?, ?)`,
3929
- )
3930
- .run(
3931
- attachment.assetId,
3932
- attachment.mimeType,
3933
- attachment.byteLength,
3934
- attachment.width,
3935
- attachment.height,
3936
- message.createdAt,
3937
- );
3938
- } else {
3939
- timestampFromSql(existing.created_at, "image asset created_at");
3940
- if (
3941
- existing.mime_type !== attachment.mimeType ||
3942
- numberFromSql(existing.byte_length, "image asset byte_length") !==
3943
- attachment.byteLength ||
3944
- numberFromSql(existing.width, "image asset width") !== attachment.width ||
3945
- numberFromSql(existing.height, "image asset height") !== attachment.height
3946
- ) {
3947
- throw new Error(`Image asset metadata conflicts for ${attachment.assetId}.`);
3948
- }
3949
- }
3896
+ ensureImageAsset(
3897
+ database,
3898
+ imageAssetRefFromAttachment(attachment),
3899
+ message.createdAt,
3900
+ );
3950
3901
  database
3951
3902
  .query(
3952
3903
  `INSERT INTO message_image_attachments (
@@ -3967,6 +3918,84 @@ function insertMessageImageAttachments(
3967
3918
  }
3968
3919
  }
3969
3920
 
3921
+ function insertToolMessageContentBlocks(
3922
+ database: Database,
3923
+ message: Extract<CanonicalMessageRecord, { role: "tool" }>,
3924
+ ): void {
3925
+ validateToolResultContent(message.content);
3926
+ if (
3927
+ canonicalToolResultContentHash(message.content) !== message.contentSha256 ||
3928
+ toolResultDisplayText(message.content) !== message.displayText
3929
+ ) {
3930
+ throw new Error("Tool content blocks do not match canonical message metadata.");
3931
+ }
3932
+ for (let position = 0; position < message.content.length; position += 1) {
3933
+ const block = requireItem(message.content, position, "tool content block");
3934
+ if (block.type === "image") {
3935
+ ensureImageAsset(database, block.asset, message.createdAt);
3936
+ }
3937
+ database
3938
+ .query(
3939
+ `INSERT INTO tool_message_content_blocks (
3940
+ message_id, position, kind, text_content, asset_id
3941
+ ) VALUES (?, ?, ?, ?, ?)`,
3942
+ )
3943
+ .run(
3944
+ message.messageId,
3945
+ position,
3946
+ block.type,
3947
+ block.type === "text" ? block.text : null,
3948
+ block.type === "image" ? block.asset.assetId : null,
3949
+ );
3950
+ }
3951
+ }
3952
+
3953
+ function ensureImageAsset(
3954
+ database: Database,
3955
+ asset: ImageAssetRef,
3956
+ createdAt: string,
3957
+ ): void {
3958
+ const existing = database
3959
+ .query(
3960
+ `SELECT mime_type, byte_length, width, height, created_at
3961
+ FROM image_assets WHERE asset_id = ?`,
3962
+ )
3963
+ .get(asset.assetId) as {
3964
+ mime_type: unknown;
3965
+ byte_length: unknown;
3966
+ width: unknown;
3967
+ height: unknown;
3968
+ created_at: unknown;
3969
+ } | null;
3970
+ if (existing === null) {
3971
+ database
3972
+ .query(
3973
+ `INSERT INTO image_assets (
3974
+ asset_id, mime_type, byte_length, width, height, created_at
3975
+ ) VALUES (?, ?, ?, ?, ?, ?)`,
3976
+ )
3977
+ .run(
3978
+ asset.assetId,
3979
+ asset.mimeType,
3980
+ asset.byteLength,
3981
+ asset.width,
3982
+ asset.height,
3983
+ createdAt,
3984
+ );
3985
+ return;
3986
+ }
3987
+ timestampFromSql(existing.created_at, "image asset created_at");
3988
+ if (
3989
+ existing.mime_type !== asset.mimeType ||
3990
+ numberFromSql(existing.byte_length, "image asset byte_length") !==
3991
+ asset.byteLength ||
3992
+ numberFromSql(existing.width, "image asset width") !== asset.width ||
3993
+ numberFromSql(existing.height, "image asset height") !== asset.height
3994
+ ) {
3995
+ throw new Error(`Image asset metadata conflicts for ${asset.assetId}.`);
3996
+ }
3997
+ }
3998
+
3970
3999
  function insertToolResult(database: Database, result: ToolResultRecord): void {
3971
4000
  const returned = result.completion.kind === "returned" ? result.completion : null;
3972
4001
  const synthetic = result.completion.kind === "synthetic" ? result.completion : null;
@@ -4120,9 +4149,74 @@ function loadMessageImageAttachments(
4120
4149
  );
4121
4150
  }
4122
4151
 
4152
+ function loadToolMessageContentBlocks(
4153
+ database: Database,
4154
+ ): Map<string, readonly ToolResultContent[]> {
4155
+ const rows = database
4156
+ .query(
4157
+ `SELECT tcb.*, ia.mime_type, ia.byte_length, ia.width, ia.height,
4158
+ ia.created_at AS asset_created_at
4159
+ FROM tool_message_content_blocks tcb
4160
+ LEFT JOIN image_assets ia ON ia.asset_id = tcb.asset_id
4161
+ JOIN messages m ON m.message_id = tcb.message_id
4162
+ ORDER BY m.ordinal, tcb.position`,
4163
+ )
4164
+ .all();
4165
+ const mutable = new Map<string, ToolResultContent[]>();
4166
+ for (const rowValue of rows) {
4167
+ const row = recordFromSql(rowValue, "tool message content block");
4168
+ const messageId = stringFromSql(row.message_id, "tool block message_id");
4169
+ const blocks = mutable.get(messageId) ?? [];
4170
+ const position = numberFromSql(row.position, "tool block position");
4171
+ if (position !== blocks.length) {
4172
+ throw new Error(
4173
+ `Tool content block positions for message ${messageId} are not continuous.`,
4174
+ );
4175
+ }
4176
+ const kind = enumFromSql(row.kind, ["text", "image"] as const, "tool block kind");
4177
+ const block: ToolResultContent =
4178
+ kind === "text"
4179
+ ? immutableRecord({
4180
+ type: "text",
4181
+ text: stringFromSql(row.text_content, "tool block text_content"),
4182
+ })
4183
+ : immutableRecord({
4184
+ type: "image",
4185
+ asset: immutableRecord({
4186
+ assetId: parseImageAssetId(
4187
+ stringFromSql(row.asset_id, "tool block asset_id"),
4188
+ ),
4189
+ mimeType: enumFromSql(
4190
+ row.mime_type,
4191
+ ["image/png", "image/jpeg", "image/webp"] as const,
4192
+ "tool block image mime_type",
4193
+ ),
4194
+ byteLength: numberFromSql(
4195
+ row.byte_length,
4196
+ "tool block image byte_length",
4197
+ ),
4198
+ width: numberFromSql(row.width, "tool block image width"),
4199
+ height: numberFromSql(row.height, "tool block image height"),
4200
+ }),
4201
+ });
4202
+ if (kind === "image") {
4203
+ timestampFromSql(row.asset_created_at, "tool block image asset created_at");
4204
+ }
4205
+ blocks.push(block);
4206
+ mutable.set(messageId, blocks);
4207
+ }
4208
+ return new Map(
4209
+ [...mutable].map(([messageId, blocks]) => {
4210
+ validateToolResultContent(blocks);
4211
+ return [messageId, Object.freeze(blocks)] as const;
4212
+ }),
4213
+ );
4214
+ }
4215
+
4123
4216
  function decodeMessage(
4124
4217
  rowValue: unknown,
4125
4218
  attachments?: readonly UserImageAttachment[],
4219
+ toolContentBlocks?: readonly ToolResultContent[],
4126
4220
  ): CanonicalMessageRecord {
4127
4221
  const row = recordFromSql(rowValue, "message");
4128
4222
  const base = {
@@ -4140,8 +4234,8 @@ function decodeMessage(
4140
4234
  );
4141
4235
  switch (role) {
4142
4236
  case "system":
4143
- if (attachments !== undefined) {
4144
- throw new Error("System messages cannot have image attachments.");
4237
+ if (attachments !== undefined || toolContentBlocks !== undefined) {
4238
+ throw new Error("System messages cannot have media relation rows.");
4145
4239
  }
4146
4240
  return immutableRecord({
4147
4241
  ...base,
@@ -4150,6 +4244,9 @@ function decodeMessage(
4150
4244
  origin: "runtime",
4151
4245
  });
4152
4246
  case "user": {
4247
+ if (toolContentBlocks !== undefined) {
4248
+ throw new Error("User messages cannot have tool content blocks.");
4249
+ }
4153
4250
  const message = {
4154
4251
  ...base,
4155
4252
  role,
@@ -4168,8 +4265,8 @@ function decodeMessage(
4168
4265
  return immutableRecord(message);
4169
4266
  }
4170
4267
  case "assistant": {
4171
- if (attachments !== undefined) {
4172
- throw new Error("Assistant messages cannot have image attachments.");
4268
+ if (attachments !== undefined || toolContentBlocks !== undefined) {
4269
+ throw new Error("Assistant messages cannot have media relation rows.");
4173
4270
  }
4174
4271
  const content = nullableTextFromSql(row.content, "content");
4175
4272
  const reasoningPresent = numberFromSql(
@@ -4205,10 +4302,18 @@ function decodeMessage(
4205
4302
  origin: "model",
4206
4303
  });
4207
4304
  }
4208
- case "tool":
4305
+ case "tool": {
4209
4306
  if (attachments !== undefined) {
4210
4307
  throw new Error("Tool messages cannot have image attachments.");
4211
4308
  }
4309
+ if (toolContentBlocks === undefined) {
4310
+ throw new Error("Tool messages must have persisted content blocks.");
4311
+ }
4312
+ validateToolResultContent(toolContentBlocks);
4313
+ const displayText = stringFromSql(row.content, "content");
4314
+ if (toolResultDisplayText(toolContentBlocks) !== displayText) {
4315
+ throw new Error("Tool message display projection does not match its blocks.");
4316
+ }
4212
4317
  return immutableRecord({
4213
4318
  ...base,
4214
4319
  role,
@@ -4220,9 +4325,11 @@ function decodeMessage(
4220
4325
  "provider_tool_call_id",
4221
4326
  ),
4222
4327
  name: stringFromSql(row.name, "name"),
4223
- content: stringFromSql(row.content, "content"),
4328
+ content: toolContentBlocks,
4329
+ displayText,
4224
4330
  origin: enumFromSql(row.origin, ["tool", "runtime"] as const, "tool origin"),
4225
4331
  });
4332
+ }
4226
4333
  }
4227
4334
  }
4228
4335
 
@@ -4604,7 +4711,11 @@ function decodeStoredSwapOverride(rowValue: unknown): StoredContextOverrideV8 {
4604
4711
  ordinal: numberFromSql(row.ordinal, "ordinal"),
4605
4712
  rendererFormat: enumFromSql(
4606
4713
  row.renderer_format,
4607
- [SWAP_OBSERVATION_FORMAT, SKILL_ACTIVATION_RECEIPT_FORMAT] as const,
4714
+ [
4715
+ SWAP_OBSERVATION_FORMAT,
4716
+ SWAP_TOOL_IMAGE_FORMAT,
4717
+ SKILL_ACTIVATION_RECEIPT_FORMAT,
4718
+ ] as const,
4608
4719
  "context override renderer format",
4609
4720
  ),
4610
4721
  source: stringFromSql(row.source, "source") as StoredContextOverrideV8["source"],
@@ -4944,6 +5055,7 @@ export function decodeStoredToolRawResult(value: unknown): ToolRawResult {
4944
5055
  raw.kind,
4945
5056
  [
4946
5057
  "read",
5058
+ "view_image",
4947
5059
  "write",
4948
5060
  "edit",
4949
5061
  "delete",
@@ -4959,6 +5071,8 @@ export function decodeStoredToolRawResult(value: unknown): ToolRawResult {
4959
5071
  "web_fetch",
4960
5072
  "recall",
4961
5073
  "memory_search",
5074
+ "memory_get",
5075
+ "wait",
4962
5076
  "skill",
4963
5077
  "mcp",
4964
5078
  "generic",
@@ -4971,9 +5085,65 @@ export function decodeStoredToolRawResult(value: unknown): ToolRawResult {
4971
5085
  if (kind === "skill") {
4972
5086
  return decodeStoredSkillRawResult(raw);
4973
5087
  }
5088
+ if (kind === "view_image") {
5089
+ return decodeStoredViewImageRawResult(raw);
5090
+ }
4974
5091
  return immutableCanonicalClone(raw) as ToolRawResult;
4975
5092
  }
4976
5093
 
5094
+ function decodeStoredViewImageRawResult(
5095
+ raw: Record<string, unknown>,
5096
+ ): Extract<ToolRawResult, { kind: "view_image" }> {
5097
+ assertObjectKeys(
5098
+ raw,
5099
+ ["kind", "ok", "filePath", "originalName", "asset", "error"],
5100
+ ["kind", "ok", "filePath"],
5101
+ "ViewImage raw result",
5102
+ );
5103
+ const filePath = stringFromSql(raw.filePath, "ViewImage filePath");
5104
+ if (raw.ok === false) {
5105
+ if (raw.originalName !== undefined || raw.asset !== undefined) {
5106
+ throw new Error("Failed ViewImage raw result cannot contain image metadata.");
5107
+ }
5108
+ const error = stringFromSql(raw.error, "ViewImage error");
5109
+ if (error.trim() === "") {
5110
+ throw new Error("Failed ViewImage raw result error must not be empty.");
5111
+ }
5112
+ return immutableRecord({ kind: "view_image", ok: false, filePath, error });
5113
+ }
5114
+ if (raw.ok !== true || raw.error !== undefined || filePath.trim() === "") {
5115
+ throw new Error("Successful ViewImage raw result is invalid.");
5116
+ }
5117
+ const originalName = stringFromSql(raw.originalName, "ViewImage originalName");
5118
+ validateOriginalImageName(originalName);
5119
+ const storedAsset = recordFromSql(raw.asset, "ViewImage asset");
5120
+ assertObjectKeys(
5121
+ storedAsset,
5122
+ ["assetId", "mimeType", "byteLength", "width", "height"],
5123
+ ["assetId", "mimeType", "byteLength", "width", "height"],
5124
+ "ViewImage asset",
5125
+ );
5126
+ const asset: ImageAssetRef = immutableRecord({
5127
+ assetId: parseImageAssetId(stringFromSql(storedAsset.assetId, "assetId")),
5128
+ mimeType: enumFromSql(
5129
+ storedAsset.mimeType,
5130
+ ["image/png", "image/jpeg", "image/webp"] as const,
5131
+ "ViewImage asset mimeType",
5132
+ ),
5133
+ byteLength: numberFromJson(storedAsset.byteLength, "ViewImage asset byteLength"),
5134
+ width: numberFromJson(storedAsset.width, "ViewImage asset width"),
5135
+ height: numberFromJson(storedAsset.height, "ViewImage asset height"),
5136
+ });
5137
+ validateImageAssetRef(asset);
5138
+ return immutableRecord({
5139
+ kind: "view_image",
5140
+ ok: true,
5141
+ filePath,
5142
+ originalName,
5143
+ asset,
5144
+ });
5145
+ }
5146
+
4977
5147
  function decodeStoredSkillRawResult(
4978
5148
  raw: Record<string, unknown>,
4979
5149
  ): Extract<ToolRawResult, { kind: "skill" }> {
@@ -5332,7 +5502,7 @@ function assertCommitPrefixRetirementRevisionInput(
5332
5502
  }
5333
5503
  }
5334
5504
 
5335
- function requireActiveRevisionId(meta: StoredSessionMetaV9): ContextRevisionId {
5505
+ function requireActiveRevisionId(meta: StoredSessionMetaV10): ContextRevisionId {
5336
5506
  if (meta.initializationState !== "ready" || meta.activeRevisionId === null) {
5337
5507
  throw new Error("Session has no active context revision.");
5338
5508
  }
@@ -5346,16 +5516,19 @@ function previousRevision(
5346
5516
  return revisions[revision.revisionNumber - 2];
5347
5517
  }
5348
5518
 
5349
- function decodeMeta(value: unknown, expectedSessionId: SessionId): StoredSessionMetaV9 {
5519
+ function decodeMeta(
5520
+ value: unknown,
5521
+ expectedSessionId: SessionId,
5522
+ ): StoredSessionMetaV10 {
5350
5523
  const row = recordFromSql(value, "session metadata");
5351
5524
  const sessionId = stringFromSql(row.session_id, "session_id") as SessionId;
5352
5525
  if (sessionId !== expectedSessionId) {
5353
5526
  throw new Error(`Metadata session ID ${sessionId} does not match directory.`);
5354
5527
  }
5355
5528
  const schemaVersion = numberFromSql(row.schema_version, "schema_version");
5356
- if (schemaVersion !== 9) {
5529
+ if (schemaVersion !== 10) {
5357
5530
  throw new Error(
5358
- `Session metadata schema version must be 9; received ${schemaVersion}.`,
5531
+ `Session metadata schema version must be 10; received ${schemaVersion}.`,
5359
5532
  );
5360
5533
  }
5361
5534
  const projectInstructionFile = nullableStringFromSql(
@@ -5484,7 +5657,7 @@ function compatibilityContractDifferences(
5484
5657
  "includeReasoningContent",
5485
5658
  "contextProfile",
5486
5659
  "messageProtocol",
5487
- "imageInput",
5660
+ "media",
5488
5661
  ];
5489
5662
  return fields.filter((key) => {
5490
5663
  const storedValue = record[key];
@@ -5511,14 +5684,14 @@ function decodeSessionCompatibilityContract(
5511
5684
  "includeReasoningContent",
5512
5685
  "contextProfile",
5513
5686
  "messageProtocol",
5514
- "imageInput",
5687
+ "media",
5515
5688
  ],
5516
5689
  [
5517
5690
  "modelName",
5518
5691
  "includeReasoningContent",
5519
5692
  "contextProfile",
5520
5693
  "messageProtocol",
5521
- "imageInput",
5694
+ "media",
5522
5695
  ],
5523
5696
  "session compatibility contract",
5524
5697
  );
@@ -5542,24 +5715,21 @@ function decodeSessionCompatibilityContract(
5542
5715
  ["adapter", "serializationVersion"],
5543
5716
  "session compatibility message protocol",
5544
5717
  );
5545
- const imageInput = recordFromSql(
5546
- record.imageInput,
5547
- "session compatibility image input",
5548
- );
5718
+ const media = recordFromSql(record.media, "session compatibility media");
5549
5719
  assertObjectKeys(
5550
- imageInput,
5551
- ["policyVersion", "policySha256", "inputModalities", "tokenEstimator"],
5552
- ["policyVersion", "policySha256", "inputModalities"],
5553
- "session compatibility image input",
5720
+ media,
5721
+ ["policyVersion", "policySha256", "inputModalities", "toolResultModalities"],
5722
+ ["policyVersion", "policySha256", "inputModalities", "toolResultModalities"],
5723
+ "session compatibility media",
5554
5724
  );
5555
- const inputModalities = imageInput.inputModalities;
5556
- if (!Array.isArray(inputModalities)) {
5557
- throw new Error("Session compatibility input modalities must be an array.");
5725
+ const inputModalities = decodeCompatibilityModalities(media.inputModalities, "input");
5726
+ const toolResultModalities = decodeCompatibilityModalities(
5727
+ media.toolResultModalities,
5728
+ "tool result",
5729
+ );
5730
+ if (toolResultModalities.includes("image") && !inputModalities.includes("image")) {
5731
+ throw new Error("Session compatibility image tool results require image input.");
5558
5732
  }
5559
- const tokenEstimator =
5560
- imageInput.tokenEstimator === undefined
5561
- ? undefined
5562
- : decodeTokenEstimatorCompatibility(imageInput.tokenEstimator);
5563
5733
  if (typeof record.includeReasoningContent !== "boolean") {
5564
5734
  throw new Error("Session compatibility reasoning replay flag must be boolean.");
5565
5735
  }
@@ -5589,20 +5759,12 @@ function decodeSessionCompatibilityContract(
5589
5759
  "compatibility serializationVersion",
5590
5760
  ),
5591
5761
  });
5592
- const modalities = normalizeInputModalities(
5593
- inputModalities.map((value) =>
5594
- enumFromSql(value, ["text", "image"] as const, "compatibility input modality"),
5595
- ),
5596
- );
5597
- if (modalities.includes("image") && tokenEstimator === undefined) {
5598
- throw new Error("Stored image input compatibility has no token estimator.");
5599
- }
5600
5762
  const policyVersion = stringFromSql(
5601
- imageInput.policyVersion,
5763
+ media.policyVersion,
5602
5764
  "compatibility image policyVersion",
5603
5765
  );
5604
5766
  const policySha256 = stringFromSql(
5605
- imageInput.policySha256,
5767
+ media.policySha256,
5606
5768
  "compatibility image policySha256",
5607
5769
  );
5608
5770
  if (!/^[0-9a-f]{64}$/.test(policySha256)) {
@@ -5614,49 +5776,31 @@ function decodeSessionCompatibilityContract(
5614
5776
  includeReasoningContent: record.includeReasoningContent,
5615
5777
  contextProfile: Object.freeze(context),
5616
5778
  messageProtocol: protocol,
5617
- imageInput: Object.freeze({
5779
+ media: Object.freeze({
5618
5780
  policyVersion,
5619
5781
  policySha256,
5620
- inputModalities: modalities,
5621
- ...(tokenEstimator === undefined ? {} : { tokenEstimator }),
5782
+ inputModalities,
5783
+ toolResultModalities,
5622
5784
  }),
5623
5785
  });
5624
5786
  }
5625
5787
 
5626
- function decodeTokenEstimatorCompatibility(
5788
+ function decodeCompatibilityModalities(
5627
5789
  value: unknown,
5628
- ): InputTokenEstimatorCompatibility {
5629
- const record = recordFromSql(value, "session compatibility token estimator");
5630
- assertObjectKeys(
5631
- record,
5632
- ["kind", "coverageVersion", "model", "endpoint", "timeoutMs", "maxRetries"],
5633
- ["kind", "coverageVersion", "model", "endpoint", "timeoutMs", "maxRetries"],
5634
- "session compatibility token estimator",
5635
- );
5636
- const estimator: InputTokenEstimatorCompatibility = {
5637
- kind: enumFromSql(
5638
- record.kind,
5639
- ["moonshot-estimate-token-count-v1"] as const,
5640
- "compatibility token estimator kind",
5641
- ),
5642
- coverageVersion: enumFromSql(
5643
- record.coverageVersion,
5644
- ["full-request-v1"] as const,
5645
- "compatibility token estimator coverage",
5646
- ),
5647
- model: stringFromSql(record.model, "compatibility token estimator model"),
5648
- endpoint: stringFromSql(record.endpoint, "compatibility token estimator endpoint"),
5649
- timeoutMs: numberFromJson(
5650
- record.timeoutMs,
5651
- "compatibility token estimator timeoutMs",
5652
- ),
5653
- maxRetries: zeroFromJson(
5654
- record.maxRetries,
5655
- "compatibility token estimator maxRetries",
5790
+ label: string,
5791
+ ): readonly ("text" | "image")[] {
5792
+ if (!Array.isArray(value)) {
5793
+ throw new Error(`Session compatibility ${label} modalities must be an array.`);
5794
+ }
5795
+ return normalizeInputModalities(
5796
+ value.map((modality) =>
5797
+ enumFromSql(
5798
+ modality,
5799
+ ["text", "image"] as const,
5800
+ `compatibility ${label} modality`,
5801
+ ),
5656
5802
  ),
5657
- };
5658
- validateTokenEstimatorCompatibility(estimator);
5659
- return Object.freeze(estimator);
5803
+ );
5660
5804
  }
5661
5805
 
5662
5806
  function openWritableDatabase(databasePath: string): Database {
@@ -6254,13 +6398,6 @@ function numberFromJson(value: unknown, name: string): number {
6254
6398
  return value as number;
6255
6399
  }
6256
6400
 
6257
- function zeroFromJson(value: unknown, name: string): 0 {
6258
- if (value !== 0) {
6259
- throw new Error(`${name} must be 0.`);
6260
- }
6261
- return 0;
6262
- }
6263
-
6264
6401
  function enumFromSql<const T extends readonly string[]>(
6265
6402
  value: unknown,
6266
6403
  values: T,