tinker-agent 1.8.0 → 1.10.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (54) hide show
  1. package/CHANGELOG.md +41 -1
  2. package/README.md +65 -6
  3. package/package.json +1 -1
  4. package/src/agent/loop.ts +13 -0
  5. package/src/agent/runtime-session.ts +165 -0
  6. package/src/agent/session-ledger.ts +20 -3
  7. package/src/cli/config.ts +11 -2
  8. package/src/cli/model-profiles.ts +58 -0
  9. package/src/cli/public-config-contract.ts +73 -7
  10. package/src/cli/run-runner.ts +4 -1
  11. package/src/cli/runner-dependencies.ts +35 -10
  12. package/src/cli/tui-memory.ts +4 -0
  13. package/src/cli/tui-runner.tsx +8 -1
  14. package/src/context/context-automation-policy.ts +22 -21
  15. package/src/context/context-manager.ts +91 -15
  16. package/src/context/context-policy.ts +0 -2
  17. package/src/context/context-swap-renderer.ts +1 -1
  18. package/src/context/prefix-retirement-planner.ts +58 -8
  19. package/src/context/recall-retirement-contract.ts +5 -4
  20. package/src/context/swap-planner.ts +33 -27
  21. package/src/events/stdout-event-printer.ts +17 -0
  22. package/src/model/fake-model-client.ts +26 -16
  23. package/src/model/model-api.ts +12 -0
  24. package/src/model/model-client.ts +9 -1
  25. package/src/model/moonshot-input-token-estimator.ts +5 -1
  26. package/src/model/openai-chat-mapping.ts +2 -24
  27. package/src/model/openai-chat-model-client.ts +18 -294
  28. package/src/model/openai-image-mapping.ts +20 -0
  29. package/src/model/openai-model-utils.ts +304 -0
  30. package/src/model/openai-responses-mapping.ts +532 -0
  31. package/src/model/openai-responses-model-client.ts +295 -0
  32. package/src/model/openai-responses-stream.ts +96 -0
  33. package/src/model/openai-responses-token-estimator.ts +155 -0
  34. package/src/model/reasoning-effort.ts +60 -0
  35. package/src/observation/observation-builder.ts +7 -0
  36. package/src/session/session-catalog.ts +2 -2
  37. package/src/session/session-history-reader.ts +6 -1
  38. package/src/session/session-schema.ts +268 -4
  39. package/src/session/session-store.ts +106 -26
  40. package/src/skills/skill-context.ts +2 -2
  41. package/src/tools/bounded-output-preview.ts +276 -0
  42. package/src/tools/recall.ts +67 -36
  43. package/src/tools/registry.ts +9 -2
  44. package/src/tools/task-output-snapshot.ts +6 -22
  45. package/src/tools/task-output.ts +23 -27
  46. package/src/tools/types.ts +19 -0
  47. package/src/tools/update-plan.ts +166 -0
  48. package/src/tui/app.tsx +82 -5
  49. package/src/tui/components/plan-view.tsx +43 -0
  50. package/src/tui/components/prompt-input.tsx +9 -1
  51. package/src/tui/components/timeline.tsx +7 -0
  52. package/src/tui/event-store.ts +28 -2
  53. package/src/tui/slash-commands.ts +20 -0
  54. package/src/tui/tui-session-controller.ts +7 -0
@@ -6,6 +6,7 @@ import { IMAGE_INPUT_POLICY } from "../image/image-input-policy";
6
6
  import type { ImageAssetId, ImageAssetRef } from "../image/image-types";
7
7
  import type { InputTokenEstimator } from "./input-token-estimator";
8
8
  import type { ModelContextBudget } from "./model-context-profile";
9
+ import type { ReasoningEffortController } from "./reasoning-effort";
9
10
  import type {
10
11
  MaterializedModelRequest,
11
12
  ModelClient,
@@ -24,6 +25,7 @@ import { estimatePromptSegments } from "./token-estimator";
24
25
  export class FakeModelClient implements ModelClient {
25
26
  readonly inputModalities: readonly ("text" | "image")[];
26
27
  readonly inputTokenEstimator?: InputTokenEstimator;
28
+ readonly reasoningEffort?: ReasoningEffortController;
27
29
  readonly messageProtocol: ModelMessageProtocol = Object.freeze({
28
30
  adapter: "fake",
29
31
  serializationVersion: "fake-v1",
@@ -38,6 +40,7 @@ export class FakeModelClient implements ModelClient {
38
40
  model: string;
39
41
  contextBudget: ModelContextBudget;
40
42
  inputModalities?: readonly ("text" | "image")[];
43
+ reasoningEffort?: ReasoningEffortController;
41
44
  requestLogPath?: string;
42
45
  tokenEstimator?: {
43
46
  kind: "moonshot-estimate-token-count-v1";
@@ -48,6 +51,7 @@ export class FakeModelClient implements ModelClient {
48
51
  };
49
52
  },
50
53
  ) {
54
+ this.reasoningEffort = options.reasoningEffort;
51
55
  this.inputModalities = Object.freeze([
52
56
  ...(options.inputModalities ?? (["text"] as const)),
53
57
  ]);
@@ -100,6 +104,7 @@ export class FakeModelClient implements ModelClient {
100
104
  (total, segment) => total + (segment.media?.length ?? 0),
101
105
  0,
102
106
  );
107
+ const reasoningEffort = this.reasoningEffort?.snapshot().effort;
103
108
  const requestConfigHash = sha256(
104
109
  stableJsonStringify({
105
110
  adapter: this.messageProtocol.adapter,
@@ -120,6 +125,7 @@ export class FakeModelClient implements ModelClient {
120
125
  messages: Object.freeze([...input.messages]),
121
126
  tools: Object.freeze([...input.tools]),
122
127
  maxTokens: this.options.contextBudget.requestMaxOutputTokens,
128
+ ...(reasoningEffort === undefined ? {} : { reasoningEffort }),
123
129
  }),
124
130
  promptSegments: Object.freeze([...toolSegments, ...messageSegments]),
125
131
  requestConfigHash,
@@ -208,6 +214,9 @@ export class FakeModelClient implements ModelClient {
208
214
  mode: this.mode,
209
215
  model: this.options.model,
210
216
  prompt: lastUserMessage(input.messages),
217
+ ...(this.reasoningEffort === undefined
218
+ ? {}
219
+ : { reasoningEffort: this.reasoningEffort.snapshot().effort }),
211
220
  requestNumber: this.steps,
212
221
  })}\n`,
213
222
  "utf8",
@@ -956,7 +965,7 @@ export class FakeModelClient implements ModelClient {
956
965
  prepared: PreparedModelRequest,
957
966
  options: ModelRequestOptions,
958
967
  ): ModelRequestOutput {
959
- requireTools(input, ["Read", "Recall"]);
968
+ requireTools(input, ["Read", "RecallSearch", "RecallGet"]);
960
969
  const prompt = lastUserMessage(input.messages);
961
970
  if (prompt === "PTY_CONTEXT_HEAVY") {
962
971
  const read = toolMessagesAfterLastUser(input.messages).find(
@@ -1156,7 +1165,8 @@ export class FakeModelClient implements ModelClient {
1156
1165
  .reverse()
1157
1166
  .find(
1158
1167
  (message): message is Extract<AgentMessage, { role: "tool" }> =>
1159
- message.role === "tool" && message.name === "Recall",
1168
+ message.role === "tool" &&
1169
+ (message.name === "RecallSearch" || message.name === "RecallGet"),
1160
1170
  );
1161
1171
  if (latestRecallResult === undefined) {
1162
1172
  return outputWithUsage(
@@ -1170,8 +1180,8 @@ export class FakeModelClient implements ModelClient {
1170
1180
  1,
1171
1181
  ),
1172
1182
  providerToolCallId: "fake-recall-search-1",
1173
- name: "Recall",
1174
- args: { mode: "search", query: "recall-smoke-marker" },
1183
+ name: "RecallSearch",
1184
+ args: { query: "recall-smoke-marker" },
1175
1185
  },
1176
1186
  ],
1177
1187
  },
@@ -1183,7 +1193,7 @@ export class FakeModelClient implements ModelClient {
1183
1193
  /^source=(ctx:\/\/message\/[0-9a-f-]+)$/m,
1184
1194
  )?.[1];
1185
1195
  if (source === undefined) {
1186
- throw new Error("Fake Recall search did not return a source.");
1196
+ throw new Error("Fake RecallSearch did not return a source.");
1187
1197
  }
1188
1198
  return outputWithUsage(
1189
1199
  prepared,
@@ -1196,8 +1206,8 @@ export class FakeModelClient implements ModelClient {
1196
1206
  1,
1197
1207
  ),
1198
1208
  providerToolCallId: "fake-recall-get-1",
1199
- name: "Recall",
1200
- args: { mode: "get", source },
1209
+ name: "RecallGet",
1210
+ args: { source },
1201
1211
  },
1202
1212
  ],
1203
1213
  },
@@ -1205,13 +1215,13 @@ export class FakeModelClient implements ModelClient {
1205
1215
  );
1206
1216
  }
1207
1217
  if (!latestRecallResult.content.includes("recall-smoke-marker")) {
1208
- throw new Error("Fake Recall get did not recover the expected marker.");
1218
+ throw new Error("Fake RecallGet did not recover the expected marker.");
1209
1219
  }
1210
1220
  return outputWithUsage(
1211
1221
  prepared,
1212
1222
  {
1213
1223
  role: "assistant",
1214
- content: "Recall search and get completed.",
1224
+ content: "RecallSearch and RecallGet completed.",
1215
1225
  },
1216
1226
  "stop",
1217
1227
  );
@@ -1489,11 +1499,12 @@ function recallMarker(
1489
1499
  finalText: string,
1490
1500
  ): ModelRequestOutput {
1491
1501
  const latestRecallResult = toolMessagesAfterLastUser(input.messages)
1492
- .filter((message) => message.name === "Recall")
1502
+ .filter(
1503
+ (message) => message.name === "RecallSearch" || message.name === "RecallGet",
1504
+ )
1493
1505
  .at(-1);
1494
1506
  if (latestRecallResult === undefined) {
1495
- return toolCallOutput(prepared, options, "Recall", {
1496
- mode: "search",
1507
+ return toolCallOutput(prepared, options, "RecallSearch", {
1497
1508
  query: marker,
1498
1509
  });
1499
1510
  }
@@ -1502,15 +1513,14 @@ function recallMarker(
1502
1513
  /^source=(ctx:\/\/message\/[0-9a-f-]+)$/m,
1503
1514
  )?.[1];
1504
1515
  if (source === undefined) {
1505
- throw new Error("Fake PTY Recall search did not return a source.");
1516
+ throw new Error("Fake PTY RecallSearch did not return a source.");
1506
1517
  }
1507
- return toolCallOutput(prepared, options, "Recall", {
1508
- mode: "get",
1518
+ return toolCallOutput(prepared, options, "RecallGet", {
1509
1519
  source,
1510
1520
  });
1511
1521
  }
1512
1522
  if (!latestRecallResult.content.includes(marker)) {
1513
- throw new Error(`Fake PTY Recall get did not recover ${marker}.`);
1523
+ throw new Error(`Fake PTY RecallGet did not recover ${marker}.`);
1514
1524
  }
1515
1525
  return textOutput(prepared, finalText);
1516
1526
  }
@@ -0,0 +1,12 @@
1
+ export const MODEL_APIS = ["chat-completions", "responses"] as const;
2
+
3
+ export type ModelApi = (typeof MODEL_APIS)[number];
4
+
5
+ export function parseModelApi(value: unknown, name: string): ModelApi {
6
+ if (value === "chat-completions" || value === "responses") {
7
+ return value;
8
+ }
9
+ throw new Error(
10
+ `${name} must be one of ${MODEL_APIS.map((api) => JSON.stringify(api)).join(", ")}.`,
11
+ );
12
+ }
@@ -4,11 +4,13 @@ import type { ToolDefinition } from "../tools/types";
4
4
  import type { ImageAssetStore } from "../image/image-asset-store";
5
5
  import type { CodePointRange, ImageAssetId, ImageMimeType } from "../image/image-types";
6
6
  import type { InputTokenEstimator } from "./input-token-estimator";
7
+ import type { ReasoningEffortController } from "./reasoning-effort";
7
8
 
8
9
  export interface ModelClient {
9
10
  readonly messageProtocol: ModelMessageProtocol;
10
11
  readonly inputTokenEstimator?: InputTokenEstimator;
11
12
  readonly inputModalities?: readonly ("text" | "image")[];
13
+ readonly reasoningEffort?: ReasoningEffortController;
12
14
  prepare(input: ModelRequestInput): PreparedModelRequest;
13
15
  materialize?(
14
16
  prepared: PreparedModelRequest,
@@ -29,8 +31,14 @@ export class ModelRequestMediaAggregateError extends Error {
29
31
  }
30
32
  }
31
33
 
34
+ export const MODEL_MESSAGE_PROTOCOL_ADAPTERS = [
35
+ "openai-chat",
36
+ "openai-responses",
37
+ "fake",
38
+ ] as const;
39
+
32
40
  export type ModelMessageProtocol = {
33
- adapter: "openai-chat" | "fake";
41
+ adapter: (typeof MODEL_MESSAGE_PROTOCOL_ADAPTERS)[number];
34
42
  serializationVersion: string;
35
43
  };
36
44
 
@@ -14,6 +14,7 @@ export class MoonshotInputTokenEstimator implements InputTokenEstimator {
14
14
  model: string;
15
15
  timeoutMs: number;
16
16
  fetch?: typeof fetch;
17
+ payloadMapper?: (payload: unknown) => unknown;
17
18
  },
18
19
  ) {
19
20
  const base = new URL(
@@ -39,7 +40,10 @@ export class MoonshotInputTokenEstimator implements InputTokenEstimator {
39
40
  request: MaterializedModelRequest,
40
41
  options: { signal: AbortSignal },
41
42
  ): Promise<InputTokenEstimate> {
42
- const chatPayload = requireRecord(request.payload, "materialized chat payload");
43
+ const chatPayload = requireRecord(
44
+ this.options.payloadMapper?.(request.payload) ?? request.payload,
45
+ "materialized chat payload",
46
+ );
43
47
  if (!Array.isArray(chatPayload.messages)) {
44
48
  throw new Error("Materialized request has no token estimator messages.");
45
49
  }
@@ -19,17 +19,8 @@ import type {
19
19
  ChatCompletionContentPart,
20
20
  ChatCompletionTool,
21
21
  } from "openai/resources/chat/completions";
22
- import {
23
- parseImageAssetId,
24
- validateUserMessage,
25
- type ImageAssetId,
26
- } from "../image/image-types";
27
-
28
- const IMAGE_ASSET_URL_MARKER = Symbol("tinker.image-asset-url-marker");
29
-
30
- export type ImageAssetUrlMarker = {
31
- readonly [IMAGE_ASSET_URL_MARKER]: ImageAssetId;
32
- };
22
+ import { validateUserMessage, type ImageAssetId } from "../image/image-types";
23
+ import { imageAssetUrlMarker } from "./openai-image-mapping";
33
24
 
34
25
  type DeepSeekAssistantMessageParam = ChatCompletionAssistantMessageParam & {
35
26
  reasoning_content?: string | null;
@@ -108,19 +99,6 @@ export function toOpenAIUserContent(
108
99
  ];
109
100
  }
110
101
 
111
- export function imageAssetUrlMarker(assetId: ImageAssetId): ImageAssetUrlMarker {
112
- parseImageAssetId(assetId);
113
- return Object.freeze({ [IMAGE_ASSET_URL_MARKER]: assetId });
114
- }
115
-
116
- export function parseImageAssetUrlMarker(value: unknown): ImageAssetId | undefined {
117
- if (typeof value !== "object" || value === null) {
118
- return undefined;
119
- }
120
- const assetId = (value as Partial<ImageAssetUrlMarker>)[IMAGE_ASSET_URL_MARKER];
121
- return typeof assetId === "string" ? parseImageAssetId(assetId) : undefined;
122
- }
123
-
124
102
  function requireMaterializedImage(
125
103
  materializedImages: ReadonlyMap<ImageAssetId, string>,
126
104
  assetId: ImageAssetId,
@@ -3,19 +3,14 @@ import type {
3
3
  ChatCompletionCreateParamsNonStreaming,
4
4
  ChatCompletionCreateParamsStreaming,
5
5
  } from "openai/resources/chat/completions";
6
- import type { AssistantMessage, UserMessage } from "../agent/types";
6
+ import type { AssistantMessage } from "../agent/types";
7
7
  import {
8
8
  IMAGE_INPUT_POLICY,
9
9
  IMAGE_INPUT_POLICY_VERSION,
10
10
  } from "../image/image-input-policy";
11
- import type { ImageAssetId, ImageAssetRef } from "../image/image-types";
12
11
  import type { ModelContextBudget } from "./model-context-profile";
13
12
  import type { InputTokenEstimator } from "./input-token-estimator";
14
- import {
15
- ModelRequestMediaAggregateError,
16
- ProviderResponseError,
17
- type ProviderResponseErrorCode,
18
- } from "./model-client";
13
+ import { ProviderResponseError } from "./model-client";
19
14
  import type {
20
15
  MaterializedModelRequest,
21
16
  ModelClient,
@@ -24,18 +19,25 @@ import type {
24
19
  ModelRequestInput,
25
20
  ModelRequestOptions,
26
21
  ModelRequestOutput,
27
- PreparedMediaDescriptor,
28
22
  PreparedModelRequest,
29
23
  PreparedPromptSegment,
30
24
  } from "./model-client";
31
25
  import {
32
26
  fromOpenAIChatCompletion,
33
- parseImageAssetUrlMarker,
34
27
  toOpenAIChatMessages,
35
28
  toOpenAIChatTools,
36
29
  } from "./openai-chat-mapping";
37
30
  import { OpenAIChatCompletionStreamAccumulator } from "./openai-chat-stream";
31
+ import {
32
+ deepFreeze,
33
+ imageUserSegment,
34
+ materializeOpenAIRequest,
35
+ normalizedEndpointPolicy,
36
+ sanitizedProviderError,
37
+ segmentKind,
38
+ } from "./openai-model-utils";
38
39
  import { MoonshotInputTokenEstimator } from "./moonshot-input-token-estimator";
40
+ import type { ReasoningEffortController } from "./reasoning-effort";
39
41
  import { sha256, stableJsonStringify } from "./model-request-preflight";
40
42
 
41
43
  const OPENAI_CHAT_SERIALIZATION_VERSION = "openai-chat-v2";
@@ -47,6 +49,7 @@ export class OpenAIChatModelClient implements ModelClient {
47
49
  serializationVersion: OPENAI_CHAT_SERIALIZATION_VERSION,
48
50
  });
49
51
  readonly inputTokenEstimator?: InputTokenEstimator;
52
+ readonly reasoningEffort?: ReasoningEffortController;
50
53
  private readonly client: OpenAI;
51
54
  private readonly preparedRequests = new WeakSet<object>();
52
55
  private readonly materializedRequests = new WeakSet<object>();
@@ -71,6 +74,7 @@ export class OpenAIChatModelClient implements ModelClient {
71
74
  };
72
75
  model: string;
73
76
  providerName?: string;
77
+ reasoningEffort?: ReasoningEffortController;
74
78
  stream?: boolean;
75
79
  timeoutMs?: number;
76
80
  fetch?: typeof fetch;
@@ -78,6 +82,7 @@ export class OpenAIChatModelClient implements ModelClient {
78
82
  ) {
79
83
  this.provider = options.providerName ?? "openai-compatible";
80
84
  this.stream = options.stream ?? true;
85
+ this.reasoningEffort = options.reasoningEffort;
81
86
  this.inputModalities = Object.freeze([...(options.inputModalities ?? ["text"])]);
82
87
  const supportsImages = this.inputModalities.includes("image");
83
88
  if (!this.inputModalities.includes("text")) {
@@ -111,10 +116,12 @@ export class OpenAIChatModelClient implements ModelClient {
111
116
  includeReasoningContent: this.options.includeReasoningContent,
112
117
  });
113
118
  const tools = input.tools.length > 0 ? toOpenAIChatTools(input.tools) : undefined;
119
+ const reasoningEffort = this.reasoningEffort?.snapshot().effort;
114
120
  const payload = deepFreeze({
115
121
  model: this.options.model,
116
122
  messages,
117
123
  ...(tools === undefined ? {} : { tools, tool_choice: "auto" as const }),
124
+ ...(reasoningEffort === undefined ? {} : { reasoning_effort: reasoningEffort }),
118
125
  max_completion_tokens: this.options.contextBudget.requestMaxOutputTokens,
119
126
  ...(this.stream
120
127
  ? {
@@ -185,48 +192,10 @@ export class OpenAIChatModelClient implements ModelClient {
185
192
  options: ModelMaterializeOptions,
186
193
  ): Promise<MaterializedModelRequest> {
187
194
  this.assertPrepared(prepared);
188
- options.signal.throwIfAborted();
189
- if (prepared.mediaOccurrenceCount > IMAGE_INPUT_POLICY.maxImagesPerRequest) {
190
- throw new ModelRequestMediaAggregateError(
191
- `Model request has ${prepared.mediaOccurrenceCount} images; maximum is ${IMAGE_INPUT_POLICY.maxImagesPerRequest}.`,
192
- );
193
- }
194
195
  if (prepared.mediaOccurrenceCount > 0 && !this.inputModalities.includes("image")) {
195
196
  throw new Error("Current model profile does not support image input.");
196
197
  }
197
-
198
- const assets = distinctPreparedAssets(prepared.promptSegments);
199
- const lowerLengths = new Map<ImageAssetId, number>();
200
- for (const asset of assets.values()) {
201
- lowerLengths.set(asset.assetId, dataUrlLength(asset));
202
- }
203
- const markerCount = countImageMarkers(prepared.payload);
204
- if (markerCount !== prepared.mediaOccurrenceCount) {
205
- throw new Error("Prepared image marker count does not match media descriptors.");
206
- }
207
- const lowerBodyBytes = exactJsonBodyBytes(prepared.payload, lowerLengths);
208
- assertBodyLimit(lowerBodyBytes, prepared.mediaOccurrenceCount);
209
-
210
- const dataUrls = new Map<ImageAssetId, string>();
211
- for (const asset of assets.values()) {
212
- options.signal.throwIfAborted();
213
- const bytes = await options.assetStore.readVerified(asset, {
214
- signal: options.signal,
215
- });
216
- dataUrls.set(
217
- asset.assetId,
218
- `data:${asset.mimeType};base64,${bytes.toString("base64")}`,
219
- );
220
- await yieldToEventLoop();
221
- }
222
- options.signal.throwIfAborted();
223
- const exactLengths = new Map(
224
- [...dataUrls].map(([assetId, value]) => [assetId, value.length] as const),
225
- );
226
- const bodyBytes = exactJsonBodyBytes(prepared.payload, exactLengths);
227
- assertBodyLimit(bodyBytes, prepared.mediaOccurrenceCount);
228
- const payload = deepFreeze(materializePayload(prepared.payload, dataUrls));
229
- const materialized = Object.freeze({ ...prepared, payload, bodyBytes });
198
+ const materialized = await materializeOpenAIRequest(prepared, options);
230
199
  this.materializedRequests.add(materialized);
231
200
  return materialized;
232
201
  }
@@ -330,249 +299,4 @@ export class OpenAIChatModelClient implements ModelClient {
330
299
  }
331
300
  }
332
301
 
333
- export function exactJsonBodyBytes(
334
- value: unknown,
335
- imageDataUrlLengths: ReadonlyMap<ImageAssetId, number>,
336
- ): number {
337
- const assetId = parseImageAssetUrlMarker(value);
338
- if (assetId !== undefined) {
339
- const length = imageDataUrlLengths.get(assetId);
340
- if (!Number.isSafeInteger(length) || length === undefined || length < 1) {
341
- throw new Error(
342
- `Missing materialized length for image ${assetId.slice(0, 12)}….`,
343
- );
344
- }
345
- return length + 2;
346
- }
347
- if (
348
- value === null ||
349
- typeof value === "string" ||
350
- typeof value === "number" ||
351
- typeof value === "boolean"
352
- ) {
353
- const serialized: unknown = JSON.stringify(value);
354
- if (typeof serialized !== "string") {
355
- throw new Error("JSON primitive did not serialize to a string.");
356
- }
357
- return Buffer.byteLength(serialized, "utf8");
358
- }
359
- if (Array.isArray(value)) {
360
- const entries: readonly unknown[] = value;
361
- return (
362
- 2 +
363
- Math.max(0, entries.length - 1) +
364
- entries.reduce<number>(
365
- (total, entry) => total + exactJsonBodyBytes(entry, imageDataUrlLengths),
366
- 0,
367
- )
368
- );
369
- }
370
- if (typeof value !== "object" || value === undefined) {
371
- throw new Error(`Cannot size non-JSON value of type ${typeof value}.`);
372
- }
373
- const entries = Object.entries(value as Record<string, unknown>).filter(
374
- ([, entry]) => entry !== undefined,
375
- );
376
- return (
377
- 2 +
378
- Math.max(0, entries.length - 1) +
379
- entries.reduce(
380
- (total, [key, entry]) =>
381
- total +
382
- Buffer.byteLength(JSON.stringify(key), "utf8") +
383
- 1 +
384
- exactJsonBodyBytes(entry, imageDataUrlLengths),
385
- 0,
386
- )
387
- );
388
- }
389
-
390
- function imageUserSegment(message: UserMessage): PreparedPromptSegment {
391
- const media = message.attachments!.map(
392
- (attachment): PreparedMediaDescriptor =>
393
- Object.freeze({
394
- assetId: attachment.assetId,
395
- label: attachment.label,
396
- range: Object.freeze({ ...attachment.range }),
397
- mimeType: attachment.mimeType,
398
- byteLength: attachment.byteLength,
399
- width: attachment.width,
400
- height: attachment.height,
401
- planningTokens: IMAGE_INPUT_POLICY.planningTokensPerImage,
402
- }),
403
- );
404
- return Object.freeze({
405
- kind: "user",
406
- normalizedText: message.content,
407
- media: Object.freeze(media),
408
- });
409
- }
410
-
411
- function distinctPreparedAssets(
412
- segments: readonly PreparedPromptSegment[],
413
- ): Map<ImageAssetId, ImageAssetRef> {
414
- const assets = new Map<ImageAssetId, ImageAssetRef>();
415
- for (const segment of segments) {
416
- for (const media of segment.media ?? []) {
417
- const asset = Object.freeze({
418
- assetId: media.assetId,
419
- mimeType: media.mimeType,
420
- byteLength: media.byteLength,
421
- width: media.width,
422
- height: media.height,
423
- });
424
- const existing = assets.get(media.assetId);
425
- if (
426
- existing !== undefined &&
427
- stableJsonStringify(existing) !== stableJsonStringify(asset)
428
- ) {
429
- throw new Error(`Conflicting descriptors for image ${media.assetId}.`);
430
- }
431
- assets.set(media.assetId, asset);
432
- }
433
- }
434
- return assets;
435
- }
436
-
437
- function materializePayload(
438
- value: unknown,
439
- dataUrls: ReadonlyMap<ImageAssetId, string>,
440
- ): unknown {
441
- const assetId = parseImageAssetUrlMarker(value);
442
- if (assetId !== undefined) {
443
- const dataUrl = dataUrls.get(assetId);
444
- if (dataUrl === undefined) {
445
- throw new Error(`Image ${assetId.slice(0, 12)}… was not materialized.`);
446
- }
447
- return dataUrl;
448
- }
449
- if (Array.isArray(value)) {
450
- return value.map((entry) => materializePayload(entry, dataUrls));
451
- }
452
- if (typeof value === "object" && value !== null) {
453
- return Object.fromEntries(
454
- Object.entries(value)
455
- .filter(([, entry]) => entry !== undefined)
456
- .map(([key, entry]) => [key, materializePayload(entry, dataUrls)]),
457
- );
458
- }
459
- return value;
460
- }
461
-
462
- function countImageMarkers(value: unknown): number {
463
- if (parseImageAssetUrlMarker(value) !== undefined) {
464
- return 1;
465
- }
466
- if (Array.isArray(value)) {
467
- const entries: readonly unknown[] = value;
468
- return entries.reduce<number>(
469
- (total, entry) => total + countImageMarkers(entry),
470
- 0,
471
- );
472
- }
473
- if (typeof value === "object" && value !== null) {
474
- return Object.values(value as Record<string, unknown>).reduce<number>(
475
- (total, entry) => total + countImageMarkers(entry),
476
- 0,
477
- );
478
- }
479
- return 0;
480
- }
481
-
482
- function dataUrlLength(asset: ImageAssetRef): number {
483
- return `data:${asset.mimeType};base64,`.length + 4 * Math.ceil(asset.byteLength / 3);
484
- }
485
-
486
- function assertBodyLimit(bodyBytes: number, imageCount: number): void {
487
- if (bodyBytes > IMAGE_INPUT_POLICY.maxRequestBodyBytes) {
488
- throw new ModelRequestMediaAggregateError(
489
- `Model request is ${bodyBytes} UTF-8 bytes with ${imageCount} images; maximum is ${IMAGE_INPUT_POLICY.maxRequestBodyBytes}.`,
490
- );
491
- }
492
- }
493
-
494
- function normalizedEndpointPolicy(baseURL: string | undefined): string {
495
- const url = new URL(baseURL ?? "https://api.openai.com/v1");
496
- url.username = "";
497
- url.password = "";
498
- url.search = "";
499
- url.hash = "";
500
- const pathname = url.pathname.replace(/\/+$/, "") || "/";
501
- return `${url.protocol}//${url.host}${pathname}`;
502
- }
503
-
504
- function segmentKind(
505
- role: ModelRequestInput["messages"][number]["role"],
506
- ): PreparedPromptSegment["kind"] {
507
- switch (role) {
508
- case "system":
509
- return "kernel";
510
- case "user":
511
- return "user";
512
- case "assistant":
513
- return "assistant";
514
- case "tool":
515
- return "tool";
516
- }
517
- }
518
-
519
- function yieldToEventLoop(): Promise<void> {
520
- return new Promise((resolve) => setTimeout(resolve, 0));
521
- }
522
-
523
- function deepFreeze<T>(value: T): T {
524
- if (typeof value !== "object" || value === null || Object.isFrozen(value)) {
525
- return value;
526
- }
527
- for (const child of Object.values(value)) {
528
- deepFreeze(child);
529
- }
530
- return Object.freeze(value);
531
- }
532
-
533
- function sanitizedProviderError(
534
- error: unknown,
535
- provider: string,
536
- model: string,
537
- ): ProviderResponseError {
538
- const message = error instanceof Error ? error.message : String(error);
539
- const sanitized = message
540
- .replace(
541
- /data:image\/(?:png|jpeg|webp);base64,[A-Za-z0-9+/=]+/gu,
542
- "[redacted image data]",
543
- )
544
- .replace(/Bearer\s+[A-Za-z0-9._~+/-]+/giu, "Bearer [redacted]");
545
- return new ProviderResponseError(
546
- providerErrorCode(error),
547
- sanitized,
548
- { provider, model },
549
- { cause: error },
550
- );
551
- }
552
-
553
- function providerErrorCode(error: unknown): ProviderResponseErrorCode {
554
- const status = providerErrorStatus(error);
555
- if (status === 429) {
556
- return "provider_rate_limited";
557
- }
558
- if (status === 500 || status === 502 || status === 503 || status === 504) {
559
- return "provider_unavailable";
560
- }
561
- if (status === undefined && isProviderConnectionError(error)) {
562
- return "provider_unavailable";
563
- }
564
- return "provider_request_error";
565
- }
566
-
567
- function providerErrorStatus(error: unknown): number | undefined {
568
- if (typeof error !== "object" || error === null || !("status" in error)) {
569
- return undefined;
570
- }
571
- const status = (error as { status?: unknown }).status;
572
- return typeof status === "number" ? status : undefined;
573
- }
574
-
575
- function isProviderConnectionError(error: unknown): boolean {
576
- // APIConnectionTimeoutError extends APIConnectionError in the SDK.
577
- return error instanceof OpenAI.APIConnectionError;
578
- }
302
+ export { exactJsonBodyBytes } from "./openai-model-utils";
@@ -0,0 +1,20 @@
1
+ import { parseImageAssetId, type ImageAssetId } from "../image/image-types";
2
+
3
+ const IMAGE_ASSET_URL_MARKER = Symbol("tinker.image-asset-url-marker");
4
+
5
+ export type ImageAssetUrlMarker = {
6
+ readonly [IMAGE_ASSET_URL_MARKER]: ImageAssetId;
7
+ };
8
+
9
+ export function imageAssetUrlMarker(assetId: ImageAssetId): ImageAssetUrlMarker {
10
+ parseImageAssetId(assetId);
11
+ return Object.freeze({ [IMAGE_ASSET_URL_MARKER]: assetId });
12
+ }
13
+
14
+ export function parseImageAssetUrlMarker(value: unknown): ImageAssetId | undefined {
15
+ if (typeof value !== "object" || value === null) {
16
+ return undefined;
17
+ }
18
+ const assetId = (value as Partial<ImageAssetUrlMarker>)[IMAGE_ASSET_URL_MARKER];
19
+ return typeof assetId === "string" ? parseImageAssetId(assetId) : undefined;
20
+ }