tinker-agent 1.9.0 → 1.11.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (52) hide show
  1. package/CHANGELOG.md +36 -1
  2. package/README.md +64 -6
  3. package/package.json +1 -1
  4. package/src/agent/loop.ts +17 -0
  5. package/src/agent/runtime-session.ts +341 -1
  6. package/src/agent/session-ledger.ts +100 -3
  7. package/src/cli/config.ts +11 -2
  8. package/src/cli/model-profiles.ts +58 -0
  9. package/src/cli/public-config-contract.ts +73 -7
  10. package/src/cli/run-runner.ts +4 -1
  11. package/src/cli/runner-dependencies.ts +28 -4
  12. package/src/cli/tui-memory.ts +4 -0
  13. package/src/cli/tui-runner.tsx +8 -1
  14. package/src/context/context-automation-policy.ts +22 -21
  15. package/src/context/context-manager.ts +91 -15
  16. package/src/context/context-policy.ts +0 -2
  17. package/src/context/context-swap-renderer.ts +1 -1
  18. package/src/context/prefix-retirement-planner.ts +58 -8
  19. package/src/context/recall-retirement-contract.ts +5 -4
  20. package/src/context/swap-planner.ts +33 -27
  21. package/src/events/observation-text-log.ts +4 -0
  22. package/src/events/stdout-event-printer.ts +5 -0
  23. package/src/events/types.ts +5 -1
  24. package/src/model/fake-model-client.ts +55 -16
  25. package/src/model/model-api.ts +12 -0
  26. package/src/model/model-client.ts +9 -1
  27. package/src/model/moonshot-input-token-estimator.ts +5 -1
  28. package/src/model/openai-chat-mapping.ts +2 -24
  29. package/src/model/openai-chat-model-client.ts +18 -294
  30. package/src/model/openai-image-mapping.ts +20 -0
  31. package/src/model/openai-model-utils.ts +304 -0
  32. package/src/model/openai-responses-mapping.ts +532 -0
  33. package/src/model/openai-responses-model-client.ts +295 -0
  34. package/src/model/openai-responses-stream.ts +96 -0
  35. package/src/model/openai-responses-token-estimator.ts +155 -0
  36. package/src/model/reasoning-effort.ts +60 -0
  37. package/src/session/session-catalog.ts +2 -2
  38. package/src/session/session-history-reader.ts +6 -1
  39. package/src/session/session-schema.ts +268 -4
  40. package/src/session/session-store.ts +134 -26
  41. package/src/skills/skill-context.ts +2 -2
  42. package/src/tools/bounded-output-preview.ts +276 -0
  43. package/src/tools/recall.ts +67 -36
  44. package/src/tools/registry.ts +7 -2
  45. package/src/tools/task-output-snapshot.ts +6 -22
  46. package/src/tools/task-output.ts +23 -27
  47. package/src/tui/app.tsx +153 -11
  48. package/src/tui/components/footer.tsx +6 -1
  49. package/src/tui/components/prompt-input.tsx +9 -1
  50. package/src/tui/event-store.ts +15 -0
  51. package/src/tui/slash-commands.ts +20 -0
  52. package/src/tui/tui-session-controller.ts +14 -0
@@ -6,6 +6,7 @@ import { IMAGE_INPUT_POLICY } from "../image/image-input-policy";
6
6
  import type { ImageAssetId, ImageAssetRef } from "../image/image-types";
7
7
  import type { InputTokenEstimator } from "./input-token-estimator";
8
8
  import type { ModelContextBudget } from "./model-context-profile";
9
+ import type { ReasoningEffortController } from "./reasoning-effort";
9
10
  import type {
10
11
  MaterializedModelRequest,
11
12
  ModelClient,
@@ -24,6 +25,7 @@ import { estimatePromptSegments } from "./token-estimator";
24
25
  export class FakeModelClient implements ModelClient {
25
26
  readonly inputModalities: readonly ("text" | "image")[];
26
27
  readonly inputTokenEstimator?: InputTokenEstimator;
28
+ readonly reasoningEffort?: ReasoningEffortController;
27
29
  readonly messageProtocol: ModelMessageProtocol = Object.freeze({
28
30
  adapter: "fake",
29
31
  serializationVersion: "fake-v1",
@@ -38,6 +40,7 @@ export class FakeModelClient implements ModelClient {
38
40
  model: string;
39
41
  contextBudget: ModelContextBudget;
40
42
  inputModalities?: readonly ("text" | "image")[];
43
+ reasoningEffort?: ReasoningEffortController;
41
44
  requestLogPath?: string;
42
45
  tokenEstimator?: {
43
46
  kind: "moonshot-estimate-token-count-v1";
@@ -48,6 +51,7 @@ export class FakeModelClient implements ModelClient {
48
51
  };
49
52
  },
50
53
  ) {
54
+ this.reasoningEffort = options.reasoningEffort;
51
55
  this.inputModalities = Object.freeze([
52
56
  ...(options.inputModalities ?? (["text"] as const)),
53
57
  ]);
@@ -100,6 +104,7 @@ export class FakeModelClient implements ModelClient {
100
104
  (total, segment) => total + (segment.media?.length ?? 0),
101
105
  0,
102
106
  );
107
+ const reasoningEffort = this.reasoningEffort?.snapshot().effort;
103
108
  const requestConfigHash = sha256(
104
109
  stableJsonStringify({
105
110
  adapter: this.messageProtocol.adapter,
@@ -120,6 +125,7 @@ export class FakeModelClient implements ModelClient {
120
125
  messages: Object.freeze([...input.messages]),
121
126
  tools: Object.freeze([...input.tools]),
122
127
  maxTokens: this.options.contextBudget.requestMaxOutputTokens,
128
+ ...(reasoningEffort === undefined ? {} : { reasoningEffort }),
123
129
  }),
124
130
  promptSegments: Object.freeze([...toolSegments, ...messageSegments]),
125
131
  requestConfigHash,
@@ -208,6 +214,9 @@ export class FakeModelClient implements ModelClient {
208
214
  mode: this.mode,
209
215
  model: this.options.model,
210
216
  prompt: lastUserMessage(input.messages),
217
+ ...(this.reasoningEffort === undefined
218
+ ? {}
219
+ : { reasoningEffort: this.reasoningEffort.snapshot().effort }),
211
220
  requestNumber: this.steps,
212
221
  })}\n`,
213
222
  "utf8",
@@ -232,6 +241,9 @@ export class FakeModelClient implements ModelClient {
232
241
  if (this.mode === "pty-incremental-output") {
233
242
  return this.ptyIncrementalOutput(input, prepared, options);
234
243
  }
244
+ if (this.mode === "pty-steering-notice") {
245
+ return this.ptySteeringNotice(input, prepared, options);
246
+ }
235
247
  if (this.mode === "pty-resume-layout") {
236
248
  return this.ptyResumeLayout(input, prepared, options);
237
249
  }
@@ -403,6 +415,32 @@ export class FakeModelClient implements ModelClient {
403
415
  return textOutput(prepared, chunks.join(""));
404
416
  }
405
417
 
418
+ private async ptySteeringNotice(
419
+ input: ModelRequestInput,
420
+ prepared: PreparedModelRequest,
421
+ options: ModelRequestOptions,
422
+ ): Promise<ModelRequestOutput> {
423
+ requireTools(input, ["Bash"]);
424
+ const prompt = lastUserMessage(input.messages);
425
+ if (prompt === "PTY_STEERING_START") {
426
+ await Bun.sleep(600);
427
+ options.signal.throwIfAborted();
428
+ return toolCallOutput(prepared, options, "Bash", {
429
+ command: "printf 'PTY_STEERING_TOOL_DONE\\n'",
430
+ description: "Create steering boundary",
431
+ });
432
+ }
433
+ if (prompt === "PTY_STEERING_FOLLOWUP") {
434
+ requireToolMessage(input.messages, "Bash", "PTY_STEERING_TOOL_DONE");
435
+ await Bun.sleep(1_500);
436
+ options.signal.throwIfAborted();
437
+ return textOutput(prepared, "PTY_STEERING_FINAL");
438
+ }
439
+ throw new Error(
440
+ `Unexpected pty-steering-notice prompt: ${JSON.stringify(prompt)}.`,
441
+ );
442
+ }
443
+
406
444
  private ptyResumeLayout(
407
445
  input: ModelRequestInput,
408
446
  prepared: PreparedModelRequest,
@@ -956,7 +994,7 @@ export class FakeModelClient implements ModelClient {
956
994
  prepared: PreparedModelRequest,
957
995
  options: ModelRequestOptions,
958
996
  ): ModelRequestOutput {
959
- requireTools(input, ["Read", "Recall"]);
997
+ requireTools(input, ["Read", "RecallSearch", "RecallGet"]);
960
998
  const prompt = lastUserMessage(input.messages);
961
999
  if (prompt === "PTY_CONTEXT_HEAVY") {
962
1000
  const read = toolMessagesAfterLastUser(input.messages).find(
@@ -1156,7 +1194,8 @@ export class FakeModelClient implements ModelClient {
1156
1194
  .reverse()
1157
1195
  .find(
1158
1196
  (message): message is Extract<AgentMessage, { role: "tool" }> =>
1159
- message.role === "tool" && message.name === "Recall",
1197
+ message.role === "tool" &&
1198
+ (message.name === "RecallSearch" || message.name === "RecallGet"),
1160
1199
  );
1161
1200
  if (latestRecallResult === undefined) {
1162
1201
  return outputWithUsage(
@@ -1170,8 +1209,8 @@ export class FakeModelClient implements ModelClient {
1170
1209
  1,
1171
1210
  ),
1172
1211
  providerToolCallId: "fake-recall-search-1",
1173
- name: "Recall",
1174
- args: { mode: "search", query: "recall-smoke-marker" },
1212
+ name: "RecallSearch",
1213
+ args: { query: "recall-smoke-marker" },
1175
1214
  },
1176
1215
  ],
1177
1216
  },
@@ -1183,7 +1222,7 @@ export class FakeModelClient implements ModelClient {
1183
1222
  /^source=(ctx:\/\/message\/[0-9a-f-]+)$/m,
1184
1223
  )?.[1];
1185
1224
  if (source === undefined) {
1186
- throw new Error("Fake Recall search did not return a source.");
1225
+ throw new Error("Fake RecallSearch did not return a source.");
1187
1226
  }
1188
1227
  return outputWithUsage(
1189
1228
  prepared,
@@ -1196,8 +1235,8 @@ export class FakeModelClient implements ModelClient {
1196
1235
  1,
1197
1236
  ),
1198
1237
  providerToolCallId: "fake-recall-get-1",
1199
- name: "Recall",
1200
- args: { mode: "get", source },
1238
+ name: "RecallGet",
1239
+ args: { source },
1201
1240
  },
1202
1241
  ],
1203
1242
  },
@@ -1205,13 +1244,13 @@ export class FakeModelClient implements ModelClient {
1205
1244
  );
1206
1245
  }
1207
1246
  if (!latestRecallResult.content.includes("recall-smoke-marker")) {
1208
- throw new Error("Fake Recall get did not recover the expected marker.");
1247
+ throw new Error("Fake RecallGet did not recover the expected marker.");
1209
1248
  }
1210
1249
  return outputWithUsage(
1211
1250
  prepared,
1212
1251
  {
1213
1252
  role: "assistant",
1214
- content: "Recall search and get completed.",
1253
+ content: "RecallSearch and RecallGet completed.",
1215
1254
  },
1216
1255
  "stop",
1217
1256
  );
@@ -1489,11 +1528,12 @@ function recallMarker(
1489
1528
  finalText: string,
1490
1529
  ): ModelRequestOutput {
1491
1530
  const latestRecallResult = toolMessagesAfterLastUser(input.messages)
1492
- .filter((message) => message.name === "Recall")
1531
+ .filter(
1532
+ (message) => message.name === "RecallSearch" || message.name === "RecallGet",
1533
+ )
1493
1534
  .at(-1);
1494
1535
  if (latestRecallResult === undefined) {
1495
- return toolCallOutput(prepared, options, "Recall", {
1496
- mode: "search",
1536
+ return toolCallOutput(prepared, options, "RecallSearch", {
1497
1537
  query: marker,
1498
1538
  });
1499
1539
  }
@@ -1502,15 +1542,14 @@ function recallMarker(
1502
1542
  /^source=(ctx:\/\/message\/[0-9a-f-]+)$/m,
1503
1543
  )?.[1];
1504
1544
  if (source === undefined) {
1505
- throw new Error("Fake PTY Recall search did not return a source.");
1545
+ throw new Error("Fake PTY RecallSearch did not return a source.");
1506
1546
  }
1507
- return toolCallOutput(prepared, options, "Recall", {
1508
- mode: "get",
1547
+ return toolCallOutput(prepared, options, "RecallGet", {
1509
1548
  source,
1510
1549
  });
1511
1550
  }
1512
1551
  if (!latestRecallResult.content.includes(marker)) {
1513
- throw new Error(`Fake PTY Recall get did not recover ${marker}.`);
1552
+ throw new Error(`Fake PTY RecallGet did not recover ${marker}.`);
1514
1553
  }
1515
1554
  return textOutput(prepared, finalText);
1516
1555
  }
@@ -0,0 +1,12 @@
1
+ export const MODEL_APIS = ["chat-completions", "responses"] as const;
2
+
3
+ export type ModelApi = (typeof MODEL_APIS)[number];
4
+
5
+ export function parseModelApi(value: unknown, name: string): ModelApi {
6
+ if (value === "chat-completions" || value === "responses") {
7
+ return value;
8
+ }
9
+ throw new Error(
10
+ `${name} must be one of ${MODEL_APIS.map((api) => JSON.stringify(api)).join(", ")}.`,
11
+ );
12
+ }
@@ -4,11 +4,13 @@ import type { ToolDefinition } from "../tools/types";
4
4
  import type { ImageAssetStore } from "../image/image-asset-store";
5
5
  import type { CodePointRange, ImageAssetId, ImageMimeType } from "../image/image-types";
6
6
  import type { InputTokenEstimator } from "./input-token-estimator";
7
+ import type { ReasoningEffortController } from "./reasoning-effort";
7
8
 
8
9
  export interface ModelClient {
9
10
  readonly messageProtocol: ModelMessageProtocol;
10
11
  readonly inputTokenEstimator?: InputTokenEstimator;
11
12
  readonly inputModalities?: readonly ("text" | "image")[];
13
+ readonly reasoningEffort?: ReasoningEffortController;
12
14
  prepare(input: ModelRequestInput): PreparedModelRequest;
13
15
  materialize?(
14
16
  prepared: PreparedModelRequest,
@@ -29,8 +31,14 @@ export class ModelRequestMediaAggregateError extends Error {
29
31
  }
30
32
  }
31
33
 
34
+ export const MODEL_MESSAGE_PROTOCOL_ADAPTERS = [
35
+ "openai-chat",
36
+ "openai-responses",
37
+ "fake",
38
+ ] as const;
39
+
32
40
  export type ModelMessageProtocol = {
33
- adapter: "openai-chat" | "fake";
41
+ adapter: (typeof MODEL_MESSAGE_PROTOCOL_ADAPTERS)[number];
34
42
  serializationVersion: string;
35
43
  };
36
44
 
@@ -14,6 +14,7 @@ export class MoonshotInputTokenEstimator implements InputTokenEstimator {
14
14
  model: string;
15
15
  timeoutMs: number;
16
16
  fetch?: typeof fetch;
17
+ payloadMapper?: (payload: unknown) => unknown;
17
18
  },
18
19
  ) {
19
20
  const base = new URL(
@@ -39,7 +40,10 @@ export class MoonshotInputTokenEstimator implements InputTokenEstimator {
39
40
  request: MaterializedModelRequest,
40
41
  options: { signal: AbortSignal },
41
42
  ): Promise<InputTokenEstimate> {
42
- const chatPayload = requireRecord(request.payload, "materialized chat payload");
43
+ const chatPayload = requireRecord(
44
+ this.options.payloadMapper?.(request.payload) ?? request.payload,
45
+ "materialized chat payload",
46
+ );
43
47
  if (!Array.isArray(chatPayload.messages)) {
44
48
  throw new Error("Materialized request has no token estimator messages.");
45
49
  }
@@ -19,17 +19,8 @@ import type {
19
19
  ChatCompletionContentPart,
20
20
  ChatCompletionTool,
21
21
  } from "openai/resources/chat/completions";
22
- import {
23
- parseImageAssetId,
24
- validateUserMessage,
25
- type ImageAssetId,
26
- } from "../image/image-types";
27
-
28
- const IMAGE_ASSET_URL_MARKER = Symbol("tinker.image-asset-url-marker");
29
-
30
- export type ImageAssetUrlMarker = {
31
- readonly [IMAGE_ASSET_URL_MARKER]: ImageAssetId;
32
- };
22
+ import { validateUserMessage, type ImageAssetId } from "../image/image-types";
23
+ import { imageAssetUrlMarker } from "./openai-image-mapping";
33
24
 
34
25
  type DeepSeekAssistantMessageParam = ChatCompletionAssistantMessageParam & {
35
26
  reasoning_content?: string | null;
@@ -108,19 +99,6 @@ export function toOpenAIUserContent(
108
99
  ];
109
100
  }
110
101
 
111
- export function imageAssetUrlMarker(assetId: ImageAssetId): ImageAssetUrlMarker {
112
- parseImageAssetId(assetId);
113
- return Object.freeze({ [IMAGE_ASSET_URL_MARKER]: assetId });
114
- }
115
-
116
- export function parseImageAssetUrlMarker(value: unknown): ImageAssetId | undefined {
117
- if (typeof value !== "object" || value === null) {
118
- return undefined;
119
- }
120
- const assetId = (value as Partial<ImageAssetUrlMarker>)[IMAGE_ASSET_URL_MARKER];
121
- return typeof assetId === "string" ? parseImageAssetId(assetId) : undefined;
122
- }
123
-
124
102
  function requireMaterializedImage(
125
103
  materializedImages: ReadonlyMap<ImageAssetId, string>,
126
104
  assetId: ImageAssetId,
@@ -3,19 +3,14 @@ import type {
3
3
  ChatCompletionCreateParamsNonStreaming,
4
4
  ChatCompletionCreateParamsStreaming,
5
5
  } from "openai/resources/chat/completions";
6
- import type { AssistantMessage, UserMessage } from "../agent/types";
6
+ import type { AssistantMessage } from "../agent/types";
7
7
  import {
8
8
  IMAGE_INPUT_POLICY,
9
9
  IMAGE_INPUT_POLICY_VERSION,
10
10
  } from "../image/image-input-policy";
11
- import type { ImageAssetId, ImageAssetRef } from "../image/image-types";
12
11
  import type { ModelContextBudget } from "./model-context-profile";
13
12
  import type { InputTokenEstimator } from "./input-token-estimator";
14
- import {
15
- ModelRequestMediaAggregateError,
16
- ProviderResponseError,
17
- type ProviderResponseErrorCode,
18
- } from "./model-client";
13
+ import { ProviderResponseError } from "./model-client";
19
14
  import type {
20
15
  MaterializedModelRequest,
21
16
  ModelClient,
@@ -24,18 +19,25 @@ import type {
24
19
  ModelRequestInput,
25
20
  ModelRequestOptions,
26
21
  ModelRequestOutput,
27
- PreparedMediaDescriptor,
28
22
  PreparedModelRequest,
29
23
  PreparedPromptSegment,
30
24
  } from "./model-client";
31
25
  import {
32
26
  fromOpenAIChatCompletion,
33
- parseImageAssetUrlMarker,
34
27
  toOpenAIChatMessages,
35
28
  toOpenAIChatTools,
36
29
  } from "./openai-chat-mapping";
37
30
  import { OpenAIChatCompletionStreamAccumulator } from "./openai-chat-stream";
31
+ import {
32
+ deepFreeze,
33
+ imageUserSegment,
34
+ materializeOpenAIRequest,
35
+ normalizedEndpointPolicy,
36
+ sanitizedProviderError,
37
+ segmentKind,
38
+ } from "./openai-model-utils";
38
39
  import { MoonshotInputTokenEstimator } from "./moonshot-input-token-estimator";
40
+ import type { ReasoningEffortController } from "./reasoning-effort";
39
41
  import { sha256, stableJsonStringify } from "./model-request-preflight";
40
42
 
41
43
  const OPENAI_CHAT_SERIALIZATION_VERSION = "openai-chat-v2";
@@ -47,6 +49,7 @@ export class OpenAIChatModelClient implements ModelClient {
47
49
  serializationVersion: OPENAI_CHAT_SERIALIZATION_VERSION,
48
50
  });
49
51
  readonly inputTokenEstimator?: InputTokenEstimator;
52
+ readonly reasoningEffort?: ReasoningEffortController;
50
53
  private readonly client: OpenAI;
51
54
  private readonly preparedRequests = new WeakSet<object>();
52
55
  private readonly materializedRequests = new WeakSet<object>();
@@ -71,6 +74,7 @@ export class OpenAIChatModelClient implements ModelClient {
71
74
  };
72
75
  model: string;
73
76
  providerName?: string;
77
+ reasoningEffort?: ReasoningEffortController;
74
78
  stream?: boolean;
75
79
  timeoutMs?: number;
76
80
  fetch?: typeof fetch;
@@ -78,6 +82,7 @@ export class OpenAIChatModelClient implements ModelClient {
78
82
  ) {
79
83
  this.provider = options.providerName ?? "openai-compatible";
80
84
  this.stream = options.stream ?? true;
85
+ this.reasoningEffort = options.reasoningEffort;
81
86
  this.inputModalities = Object.freeze([...(options.inputModalities ?? ["text"])]);
82
87
  const supportsImages = this.inputModalities.includes("image");
83
88
  if (!this.inputModalities.includes("text")) {
@@ -111,10 +116,12 @@ export class OpenAIChatModelClient implements ModelClient {
111
116
  includeReasoningContent: this.options.includeReasoningContent,
112
117
  });
113
118
  const tools = input.tools.length > 0 ? toOpenAIChatTools(input.tools) : undefined;
119
+ const reasoningEffort = this.reasoningEffort?.snapshot().effort;
114
120
  const payload = deepFreeze({
115
121
  model: this.options.model,
116
122
  messages,
117
123
  ...(tools === undefined ? {} : { tools, tool_choice: "auto" as const }),
124
+ ...(reasoningEffort === undefined ? {} : { reasoning_effort: reasoningEffort }),
118
125
  max_completion_tokens: this.options.contextBudget.requestMaxOutputTokens,
119
126
  ...(this.stream
120
127
  ? {
@@ -185,48 +192,10 @@ export class OpenAIChatModelClient implements ModelClient {
185
192
  options: ModelMaterializeOptions,
186
193
  ): Promise<MaterializedModelRequest> {
187
194
  this.assertPrepared(prepared);
188
- options.signal.throwIfAborted();
189
- if (prepared.mediaOccurrenceCount > IMAGE_INPUT_POLICY.maxImagesPerRequest) {
190
- throw new ModelRequestMediaAggregateError(
191
- `Model request has ${prepared.mediaOccurrenceCount} images; maximum is ${IMAGE_INPUT_POLICY.maxImagesPerRequest}.`,
192
- );
193
- }
194
195
  if (prepared.mediaOccurrenceCount > 0 && !this.inputModalities.includes("image")) {
195
196
  throw new Error("Current model profile does not support image input.");
196
197
  }
197
-
198
- const assets = distinctPreparedAssets(prepared.promptSegments);
199
- const lowerLengths = new Map<ImageAssetId, number>();
200
- for (const asset of assets.values()) {
201
- lowerLengths.set(asset.assetId, dataUrlLength(asset));
202
- }
203
- const markerCount = countImageMarkers(prepared.payload);
204
- if (markerCount !== prepared.mediaOccurrenceCount) {
205
- throw new Error("Prepared image marker count does not match media descriptors.");
206
- }
207
- const lowerBodyBytes = exactJsonBodyBytes(prepared.payload, lowerLengths);
208
- assertBodyLimit(lowerBodyBytes, prepared.mediaOccurrenceCount);
209
-
210
- const dataUrls = new Map<ImageAssetId, string>();
211
- for (const asset of assets.values()) {
212
- options.signal.throwIfAborted();
213
- const bytes = await options.assetStore.readVerified(asset, {
214
- signal: options.signal,
215
- });
216
- dataUrls.set(
217
- asset.assetId,
218
- `data:${asset.mimeType};base64,${bytes.toString("base64")}`,
219
- );
220
- await yieldToEventLoop();
221
- }
222
- options.signal.throwIfAborted();
223
- const exactLengths = new Map(
224
- [...dataUrls].map(([assetId, value]) => [assetId, value.length] as const),
225
- );
226
- const bodyBytes = exactJsonBodyBytes(prepared.payload, exactLengths);
227
- assertBodyLimit(bodyBytes, prepared.mediaOccurrenceCount);
228
- const payload = deepFreeze(materializePayload(prepared.payload, dataUrls));
229
- const materialized = Object.freeze({ ...prepared, payload, bodyBytes });
198
+ const materialized = await materializeOpenAIRequest(prepared, options);
230
199
  this.materializedRequests.add(materialized);
231
200
  return materialized;
232
201
  }
@@ -330,249 +299,4 @@ export class OpenAIChatModelClient implements ModelClient {
330
299
  }
331
300
  }
332
301
 
333
- export function exactJsonBodyBytes(
334
- value: unknown,
335
- imageDataUrlLengths: ReadonlyMap<ImageAssetId, number>,
336
- ): number {
337
- const assetId = parseImageAssetUrlMarker(value);
338
- if (assetId !== undefined) {
339
- const length = imageDataUrlLengths.get(assetId);
340
- if (!Number.isSafeInteger(length) || length === undefined || length < 1) {
341
- throw new Error(
342
- `Missing materialized length for image ${assetId.slice(0, 12)}….`,
343
- );
344
- }
345
- return length + 2;
346
- }
347
- if (
348
- value === null ||
349
- typeof value === "string" ||
350
- typeof value === "number" ||
351
- typeof value === "boolean"
352
- ) {
353
- const serialized: unknown = JSON.stringify(value);
354
- if (typeof serialized !== "string") {
355
- throw new Error("JSON primitive did not serialize to a string.");
356
- }
357
- return Buffer.byteLength(serialized, "utf8");
358
- }
359
- if (Array.isArray(value)) {
360
- const entries: readonly unknown[] = value;
361
- return (
362
- 2 +
363
- Math.max(0, entries.length - 1) +
364
- entries.reduce<number>(
365
- (total, entry) => total + exactJsonBodyBytes(entry, imageDataUrlLengths),
366
- 0,
367
- )
368
- );
369
- }
370
- if (typeof value !== "object" || value === undefined) {
371
- throw new Error(`Cannot size non-JSON value of type ${typeof value}.`);
372
- }
373
- const entries = Object.entries(value as Record<string, unknown>).filter(
374
- ([, entry]) => entry !== undefined,
375
- );
376
- return (
377
- 2 +
378
- Math.max(0, entries.length - 1) +
379
- entries.reduce(
380
- (total, [key, entry]) =>
381
- total +
382
- Buffer.byteLength(JSON.stringify(key), "utf8") +
383
- 1 +
384
- exactJsonBodyBytes(entry, imageDataUrlLengths),
385
- 0,
386
- )
387
- );
388
- }
389
-
390
- function imageUserSegment(message: UserMessage): PreparedPromptSegment {
391
- const media = message.attachments!.map(
392
- (attachment): PreparedMediaDescriptor =>
393
- Object.freeze({
394
- assetId: attachment.assetId,
395
- label: attachment.label,
396
- range: Object.freeze({ ...attachment.range }),
397
- mimeType: attachment.mimeType,
398
- byteLength: attachment.byteLength,
399
- width: attachment.width,
400
- height: attachment.height,
401
- planningTokens: IMAGE_INPUT_POLICY.planningTokensPerImage,
402
- }),
403
- );
404
- return Object.freeze({
405
- kind: "user",
406
- normalizedText: message.content,
407
- media: Object.freeze(media),
408
- });
409
- }
410
-
411
- function distinctPreparedAssets(
412
- segments: readonly PreparedPromptSegment[],
413
- ): Map<ImageAssetId, ImageAssetRef> {
414
- const assets = new Map<ImageAssetId, ImageAssetRef>();
415
- for (const segment of segments) {
416
- for (const media of segment.media ?? []) {
417
- const asset = Object.freeze({
418
- assetId: media.assetId,
419
- mimeType: media.mimeType,
420
- byteLength: media.byteLength,
421
- width: media.width,
422
- height: media.height,
423
- });
424
- const existing = assets.get(media.assetId);
425
- if (
426
- existing !== undefined &&
427
- stableJsonStringify(existing) !== stableJsonStringify(asset)
428
- ) {
429
- throw new Error(`Conflicting descriptors for image ${media.assetId}.`);
430
- }
431
- assets.set(media.assetId, asset);
432
- }
433
- }
434
- return assets;
435
- }
436
-
437
- function materializePayload(
438
- value: unknown,
439
- dataUrls: ReadonlyMap<ImageAssetId, string>,
440
- ): unknown {
441
- const assetId = parseImageAssetUrlMarker(value);
442
- if (assetId !== undefined) {
443
- const dataUrl = dataUrls.get(assetId);
444
- if (dataUrl === undefined) {
445
- throw new Error(`Image ${assetId.slice(0, 12)}… was not materialized.`);
446
- }
447
- return dataUrl;
448
- }
449
- if (Array.isArray(value)) {
450
- return value.map((entry) => materializePayload(entry, dataUrls));
451
- }
452
- if (typeof value === "object" && value !== null) {
453
- return Object.fromEntries(
454
- Object.entries(value)
455
- .filter(([, entry]) => entry !== undefined)
456
- .map(([key, entry]) => [key, materializePayload(entry, dataUrls)]),
457
- );
458
- }
459
- return value;
460
- }
461
-
462
- function countImageMarkers(value: unknown): number {
463
- if (parseImageAssetUrlMarker(value) !== undefined) {
464
- return 1;
465
- }
466
- if (Array.isArray(value)) {
467
- const entries: readonly unknown[] = value;
468
- return entries.reduce<number>(
469
- (total, entry) => total + countImageMarkers(entry),
470
- 0,
471
- );
472
- }
473
- if (typeof value === "object" && value !== null) {
474
- return Object.values(value as Record<string, unknown>).reduce<number>(
475
- (total, entry) => total + countImageMarkers(entry),
476
- 0,
477
- );
478
- }
479
- return 0;
480
- }
481
-
482
- function dataUrlLength(asset: ImageAssetRef): number {
483
- return `data:${asset.mimeType};base64,`.length + 4 * Math.ceil(asset.byteLength / 3);
484
- }
485
-
486
- function assertBodyLimit(bodyBytes: number, imageCount: number): void {
487
- if (bodyBytes > IMAGE_INPUT_POLICY.maxRequestBodyBytes) {
488
- throw new ModelRequestMediaAggregateError(
489
- `Model request is ${bodyBytes} UTF-8 bytes with ${imageCount} images; maximum is ${IMAGE_INPUT_POLICY.maxRequestBodyBytes}.`,
490
- );
491
- }
492
- }
493
-
494
- function normalizedEndpointPolicy(baseURL: string | undefined): string {
495
- const url = new URL(baseURL ?? "https://api.openai.com/v1");
496
- url.username = "";
497
- url.password = "";
498
- url.search = "";
499
- url.hash = "";
500
- const pathname = url.pathname.replace(/\/+$/, "") || "/";
501
- return `${url.protocol}//${url.host}${pathname}`;
502
- }
503
-
504
- function segmentKind(
505
- role: ModelRequestInput["messages"][number]["role"],
506
- ): PreparedPromptSegment["kind"] {
507
- switch (role) {
508
- case "system":
509
- return "kernel";
510
- case "user":
511
- return "user";
512
- case "assistant":
513
- return "assistant";
514
- case "tool":
515
- return "tool";
516
- }
517
- }
518
-
519
- function yieldToEventLoop(): Promise<void> {
520
- return new Promise((resolve) => setTimeout(resolve, 0));
521
- }
522
-
523
- function deepFreeze<T>(value: T): T {
524
- if (typeof value !== "object" || value === null || Object.isFrozen(value)) {
525
- return value;
526
- }
527
- for (const child of Object.values(value)) {
528
- deepFreeze(child);
529
- }
530
- return Object.freeze(value);
531
- }
532
-
533
- function sanitizedProviderError(
534
- error: unknown,
535
- provider: string,
536
- model: string,
537
- ): ProviderResponseError {
538
- const message = error instanceof Error ? error.message : String(error);
539
- const sanitized = message
540
- .replace(
541
- /data:image\/(?:png|jpeg|webp);base64,[A-Za-z0-9+/=]+/gu,
542
- "[redacted image data]",
543
- )
544
- .replace(/Bearer\s+[A-Za-z0-9._~+/-]+/giu, "Bearer [redacted]");
545
- return new ProviderResponseError(
546
- providerErrorCode(error),
547
- sanitized,
548
- { provider, model },
549
- { cause: error },
550
- );
551
- }
552
-
553
- function providerErrorCode(error: unknown): ProviderResponseErrorCode {
554
- const status = providerErrorStatus(error);
555
- if (status === 429) {
556
- return "provider_rate_limited";
557
- }
558
- if (status === 500 || status === 502 || status === 503 || status === 504) {
559
- return "provider_unavailable";
560
- }
561
- if (status === undefined && isProviderConnectionError(error)) {
562
- return "provider_unavailable";
563
- }
564
- return "provider_request_error";
565
- }
566
-
567
- function providerErrorStatus(error: unknown): number | undefined {
568
- if (typeof error !== "object" || error === null || !("status" in error)) {
569
- return undefined;
570
- }
571
- const status = (error as { status?: unknown }).status;
572
- return typeof status === "number" ? status : undefined;
573
- }
574
-
575
- function isProviderConnectionError(error: unknown): boolean {
576
- // APIConnectionTimeoutError extends APIConnectionError in the SDK.
577
- return error instanceof OpenAI.APIConnectionError;
578
- }
302
+ export { exactJsonBodyBytes } from "./openai-model-utils";