tinker-agent 1.8.0 → 1.10.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +41 -1
- package/README.md +65 -6
- package/package.json +1 -1
- package/src/agent/loop.ts +13 -0
- package/src/agent/runtime-session.ts +165 -0
- package/src/agent/session-ledger.ts +20 -3
- package/src/cli/config.ts +11 -2
- package/src/cli/model-profiles.ts +58 -0
- package/src/cli/public-config-contract.ts +73 -7
- package/src/cli/run-runner.ts +4 -1
- package/src/cli/runner-dependencies.ts +35 -10
- package/src/cli/tui-memory.ts +4 -0
- package/src/cli/tui-runner.tsx +8 -1
- package/src/context/context-automation-policy.ts +22 -21
- package/src/context/context-manager.ts +91 -15
- package/src/context/context-policy.ts +0 -2
- package/src/context/context-swap-renderer.ts +1 -1
- package/src/context/prefix-retirement-planner.ts +58 -8
- package/src/context/recall-retirement-contract.ts +5 -4
- package/src/context/swap-planner.ts +33 -27
- package/src/events/stdout-event-printer.ts +17 -0
- package/src/model/fake-model-client.ts +26 -16
- package/src/model/model-api.ts +12 -0
- package/src/model/model-client.ts +9 -1
- package/src/model/moonshot-input-token-estimator.ts +5 -1
- package/src/model/openai-chat-mapping.ts +2 -24
- package/src/model/openai-chat-model-client.ts +18 -294
- package/src/model/openai-image-mapping.ts +20 -0
- package/src/model/openai-model-utils.ts +304 -0
- package/src/model/openai-responses-mapping.ts +532 -0
- package/src/model/openai-responses-model-client.ts +295 -0
- package/src/model/openai-responses-stream.ts +96 -0
- package/src/model/openai-responses-token-estimator.ts +155 -0
- package/src/model/reasoning-effort.ts +60 -0
- package/src/observation/observation-builder.ts +7 -0
- package/src/session/session-catalog.ts +2 -2
- package/src/session/session-history-reader.ts +6 -1
- package/src/session/session-schema.ts +268 -4
- package/src/session/session-store.ts +106 -26
- package/src/skills/skill-context.ts +2 -2
- package/src/tools/bounded-output-preview.ts +276 -0
- package/src/tools/recall.ts +67 -36
- package/src/tools/registry.ts +9 -2
- package/src/tools/task-output-snapshot.ts +6 -22
- package/src/tools/task-output.ts +23 -27
- package/src/tools/types.ts +19 -0
- package/src/tools/update-plan.ts +166 -0
- package/src/tui/app.tsx +82 -5
- package/src/tui/components/plan-view.tsx +43 -0
- package/src/tui/components/prompt-input.tsx +9 -1
- package/src/tui/components/timeline.tsx +7 -0
- package/src/tui/event-store.ts +28 -2
- package/src/tui/slash-commands.ts +20 -0
- package/src/tui/tui-session-controller.ts +7 -0
|
@@ -6,6 +6,7 @@ import { IMAGE_INPUT_POLICY } from "../image/image-input-policy";
|
|
|
6
6
|
import type { ImageAssetId, ImageAssetRef } from "../image/image-types";
|
|
7
7
|
import type { InputTokenEstimator } from "./input-token-estimator";
|
|
8
8
|
import type { ModelContextBudget } from "./model-context-profile";
|
|
9
|
+
import type { ReasoningEffortController } from "./reasoning-effort";
|
|
9
10
|
import type {
|
|
10
11
|
MaterializedModelRequest,
|
|
11
12
|
ModelClient,
|
|
@@ -24,6 +25,7 @@ import { estimatePromptSegments } from "./token-estimator";
|
|
|
24
25
|
export class FakeModelClient implements ModelClient {
|
|
25
26
|
readonly inputModalities: readonly ("text" | "image")[];
|
|
26
27
|
readonly inputTokenEstimator?: InputTokenEstimator;
|
|
28
|
+
readonly reasoningEffort?: ReasoningEffortController;
|
|
27
29
|
readonly messageProtocol: ModelMessageProtocol = Object.freeze({
|
|
28
30
|
adapter: "fake",
|
|
29
31
|
serializationVersion: "fake-v1",
|
|
@@ -38,6 +40,7 @@ export class FakeModelClient implements ModelClient {
|
|
|
38
40
|
model: string;
|
|
39
41
|
contextBudget: ModelContextBudget;
|
|
40
42
|
inputModalities?: readonly ("text" | "image")[];
|
|
43
|
+
reasoningEffort?: ReasoningEffortController;
|
|
41
44
|
requestLogPath?: string;
|
|
42
45
|
tokenEstimator?: {
|
|
43
46
|
kind: "moonshot-estimate-token-count-v1";
|
|
@@ -48,6 +51,7 @@ export class FakeModelClient implements ModelClient {
|
|
|
48
51
|
};
|
|
49
52
|
},
|
|
50
53
|
) {
|
|
54
|
+
this.reasoningEffort = options.reasoningEffort;
|
|
51
55
|
this.inputModalities = Object.freeze([
|
|
52
56
|
...(options.inputModalities ?? (["text"] as const)),
|
|
53
57
|
]);
|
|
@@ -100,6 +104,7 @@ export class FakeModelClient implements ModelClient {
|
|
|
100
104
|
(total, segment) => total + (segment.media?.length ?? 0),
|
|
101
105
|
0,
|
|
102
106
|
);
|
|
107
|
+
const reasoningEffort = this.reasoningEffort?.snapshot().effort;
|
|
103
108
|
const requestConfigHash = sha256(
|
|
104
109
|
stableJsonStringify({
|
|
105
110
|
adapter: this.messageProtocol.adapter,
|
|
@@ -120,6 +125,7 @@ export class FakeModelClient implements ModelClient {
|
|
|
120
125
|
messages: Object.freeze([...input.messages]),
|
|
121
126
|
tools: Object.freeze([...input.tools]),
|
|
122
127
|
maxTokens: this.options.contextBudget.requestMaxOutputTokens,
|
|
128
|
+
...(reasoningEffort === undefined ? {} : { reasoningEffort }),
|
|
123
129
|
}),
|
|
124
130
|
promptSegments: Object.freeze([...toolSegments, ...messageSegments]),
|
|
125
131
|
requestConfigHash,
|
|
@@ -208,6 +214,9 @@ export class FakeModelClient implements ModelClient {
|
|
|
208
214
|
mode: this.mode,
|
|
209
215
|
model: this.options.model,
|
|
210
216
|
prompt: lastUserMessage(input.messages),
|
|
217
|
+
...(this.reasoningEffort === undefined
|
|
218
|
+
? {}
|
|
219
|
+
: { reasoningEffort: this.reasoningEffort.snapshot().effort }),
|
|
211
220
|
requestNumber: this.steps,
|
|
212
221
|
})}\n`,
|
|
213
222
|
"utf8",
|
|
@@ -956,7 +965,7 @@ export class FakeModelClient implements ModelClient {
|
|
|
956
965
|
prepared: PreparedModelRequest,
|
|
957
966
|
options: ModelRequestOptions,
|
|
958
967
|
): ModelRequestOutput {
|
|
959
|
-
requireTools(input, ["Read", "
|
|
968
|
+
requireTools(input, ["Read", "RecallSearch", "RecallGet"]);
|
|
960
969
|
const prompt = lastUserMessage(input.messages);
|
|
961
970
|
if (prompt === "PTY_CONTEXT_HEAVY") {
|
|
962
971
|
const read = toolMessagesAfterLastUser(input.messages).find(
|
|
@@ -1156,7 +1165,8 @@ export class FakeModelClient implements ModelClient {
|
|
|
1156
1165
|
.reverse()
|
|
1157
1166
|
.find(
|
|
1158
1167
|
(message): message is Extract<AgentMessage, { role: "tool" }> =>
|
|
1159
|
-
message.role === "tool" &&
|
|
1168
|
+
message.role === "tool" &&
|
|
1169
|
+
(message.name === "RecallSearch" || message.name === "RecallGet"),
|
|
1160
1170
|
);
|
|
1161
1171
|
if (latestRecallResult === undefined) {
|
|
1162
1172
|
return outputWithUsage(
|
|
@@ -1170,8 +1180,8 @@ export class FakeModelClient implements ModelClient {
|
|
|
1170
1180
|
1,
|
|
1171
1181
|
),
|
|
1172
1182
|
providerToolCallId: "fake-recall-search-1",
|
|
1173
|
-
name: "
|
|
1174
|
-
args: {
|
|
1183
|
+
name: "RecallSearch",
|
|
1184
|
+
args: { query: "recall-smoke-marker" },
|
|
1175
1185
|
},
|
|
1176
1186
|
],
|
|
1177
1187
|
},
|
|
@@ -1183,7 +1193,7 @@ export class FakeModelClient implements ModelClient {
|
|
|
1183
1193
|
/^source=(ctx:\/\/message\/[0-9a-f-]+)$/m,
|
|
1184
1194
|
)?.[1];
|
|
1185
1195
|
if (source === undefined) {
|
|
1186
|
-
throw new Error("Fake
|
|
1196
|
+
throw new Error("Fake RecallSearch did not return a source.");
|
|
1187
1197
|
}
|
|
1188
1198
|
return outputWithUsage(
|
|
1189
1199
|
prepared,
|
|
@@ -1196,8 +1206,8 @@ export class FakeModelClient implements ModelClient {
|
|
|
1196
1206
|
1,
|
|
1197
1207
|
),
|
|
1198
1208
|
providerToolCallId: "fake-recall-get-1",
|
|
1199
|
-
name: "
|
|
1200
|
-
args: {
|
|
1209
|
+
name: "RecallGet",
|
|
1210
|
+
args: { source },
|
|
1201
1211
|
},
|
|
1202
1212
|
],
|
|
1203
1213
|
},
|
|
@@ -1205,13 +1215,13 @@ export class FakeModelClient implements ModelClient {
|
|
|
1205
1215
|
);
|
|
1206
1216
|
}
|
|
1207
1217
|
if (!latestRecallResult.content.includes("recall-smoke-marker")) {
|
|
1208
|
-
throw new Error("Fake
|
|
1218
|
+
throw new Error("Fake RecallGet did not recover the expected marker.");
|
|
1209
1219
|
}
|
|
1210
1220
|
return outputWithUsage(
|
|
1211
1221
|
prepared,
|
|
1212
1222
|
{
|
|
1213
1223
|
role: "assistant",
|
|
1214
|
-
content: "
|
|
1224
|
+
content: "RecallSearch and RecallGet completed.",
|
|
1215
1225
|
},
|
|
1216
1226
|
"stop",
|
|
1217
1227
|
);
|
|
@@ -1489,11 +1499,12 @@ function recallMarker(
|
|
|
1489
1499
|
finalText: string,
|
|
1490
1500
|
): ModelRequestOutput {
|
|
1491
1501
|
const latestRecallResult = toolMessagesAfterLastUser(input.messages)
|
|
1492
|
-
.filter(
|
|
1502
|
+
.filter(
|
|
1503
|
+
(message) => message.name === "RecallSearch" || message.name === "RecallGet",
|
|
1504
|
+
)
|
|
1493
1505
|
.at(-1);
|
|
1494
1506
|
if (latestRecallResult === undefined) {
|
|
1495
|
-
return toolCallOutput(prepared, options, "
|
|
1496
|
-
mode: "search",
|
|
1507
|
+
return toolCallOutput(prepared, options, "RecallSearch", {
|
|
1497
1508
|
query: marker,
|
|
1498
1509
|
});
|
|
1499
1510
|
}
|
|
@@ -1502,15 +1513,14 @@ function recallMarker(
|
|
|
1502
1513
|
/^source=(ctx:\/\/message\/[0-9a-f-]+)$/m,
|
|
1503
1514
|
)?.[1];
|
|
1504
1515
|
if (source === undefined) {
|
|
1505
|
-
throw new Error("Fake PTY
|
|
1516
|
+
throw new Error("Fake PTY RecallSearch did not return a source.");
|
|
1506
1517
|
}
|
|
1507
|
-
return toolCallOutput(prepared, options, "
|
|
1508
|
-
mode: "get",
|
|
1518
|
+
return toolCallOutput(prepared, options, "RecallGet", {
|
|
1509
1519
|
source,
|
|
1510
1520
|
});
|
|
1511
1521
|
}
|
|
1512
1522
|
if (!latestRecallResult.content.includes(marker)) {
|
|
1513
|
-
throw new Error(`Fake PTY
|
|
1523
|
+
throw new Error(`Fake PTY RecallGet did not recover ${marker}.`);
|
|
1514
1524
|
}
|
|
1515
1525
|
return textOutput(prepared, finalText);
|
|
1516
1526
|
}
|
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
export const MODEL_APIS = ["chat-completions", "responses"] as const;
|
|
2
|
+
|
|
3
|
+
export type ModelApi = (typeof MODEL_APIS)[number];
|
|
4
|
+
|
|
5
|
+
export function parseModelApi(value: unknown, name: string): ModelApi {
|
|
6
|
+
if (value === "chat-completions" || value === "responses") {
|
|
7
|
+
return value;
|
|
8
|
+
}
|
|
9
|
+
throw new Error(
|
|
10
|
+
`${name} must be one of ${MODEL_APIS.map((api) => JSON.stringify(api)).join(", ")}.`,
|
|
11
|
+
);
|
|
12
|
+
}
|
|
@@ -4,11 +4,13 @@ import type { ToolDefinition } from "../tools/types";
|
|
|
4
4
|
import type { ImageAssetStore } from "../image/image-asset-store";
|
|
5
5
|
import type { CodePointRange, ImageAssetId, ImageMimeType } from "../image/image-types";
|
|
6
6
|
import type { InputTokenEstimator } from "./input-token-estimator";
|
|
7
|
+
import type { ReasoningEffortController } from "./reasoning-effort";
|
|
7
8
|
|
|
8
9
|
export interface ModelClient {
|
|
9
10
|
readonly messageProtocol: ModelMessageProtocol;
|
|
10
11
|
readonly inputTokenEstimator?: InputTokenEstimator;
|
|
11
12
|
readonly inputModalities?: readonly ("text" | "image")[];
|
|
13
|
+
readonly reasoningEffort?: ReasoningEffortController;
|
|
12
14
|
prepare(input: ModelRequestInput): PreparedModelRequest;
|
|
13
15
|
materialize?(
|
|
14
16
|
prepared: PreparedModelRequest,
|
|
@@ -29,8 +31,14 @@ export class ModelRequestMediaAggregateError extends Error {
|
|
|
29
31
|
}
|
|
30
32
|
}
|
|
31
33
|
|
|
34
|
+
export const MODEL_MESSAGE_PROTOCOL_ADAPTERS = [
|
|
35
|
+
"openai-chat",
|
|
36
|
+
"openai-responses",
|
|
37
|
+
"fake",
|
|
38
|
+
] as const;
|
|
39
|
+
|
|
32
40
|
export type ModelMessageProtocol = {
|
|
33
|
-
adapter:
|
|
41
|
+
adapter: (typeof MODEL_MESSAGE_PROTOCOL_ADAPTERS)[number];
|
|
34
42
|
serializationVersion: string;
|
|
35
43
|
};
|
|
36
44
|
|
|
@@ -14,6 +14,7 @@ export class MoonshotInputTokenEstimator implements InputTokenEstimator {
|
|
|
14
14
|
model: string;
|
|
15
15
|
timeoutMs: number;
|
|
16
16
|
fetch?: typeof fetch;
|
|
17
|
+
payloadMapper?: (payload: unknown) => unknown;
|
|
17
18
|
},
|
|
18
19
|
) {
|
|
19
20
|
const base = new URL(
|
|
@@ -39,7 +40,10 @@ export class MoonshotInputTokenEstimator implements InputTokenEstimator {
|
|
|
39
40
|
request: MaterializedModelRequest,
|
|
40
41
|
options: { signal: AbortSignal },
|
|
41
42
|
): Promise<InputTokenEstimate> {
|
|
42
|
-
const chatPayload = requireRecord(
|
|
43
|
+
const chatPayload = requireRecord(
|
|
44
|
+
this.options.payloadMapper?.(request.payload) ?? request.payload,
|
|
45
|
+
"materialized chat payload",
|
|
46
|
+
);
|
|
43
47
|
if (!Array.isArray(chatPayload.messages)) {
|
|
44
48
|
throw new Error("Materialized request has no token estimator messages.");
|
|
45
49
|
}
|
|
@@ -19,17 +19,8 @@ import type {
|
|
|
19
19
|
ChatCompletionContentPart,
|
|
20
20
|
ChatCompletionTool,
|
|
21
21
|
} from "openai/resources/chat/completions";
|
|
22
|
-
import {
|
|
23
|
-
|
|
24
|
-
validateUserMessage,
|
|
25
|
-
type ImageAssetId,
|
|
26
|
-
} from "../image/image-types";
|
|
27
|
-
|
|
28
|
-
const IMAGE_ASSET_URL_MARKER = Symbol("tinker.image-asset-url-marker");
|
|
29
|
-
|
|
30
|
-
export type ImageAssetUrlMarker = {
|
|
31
|
-
readonly [IMAGE_ASSET_URL_MARKER]: ImageAssetId;
|
|
32
|
-
};
|
|
22
|
+
import { validateUserMessage, type ImageAssetId } from "../image/image-types";
|
|
23
|
+
import { imageAssetUrlMarker } from "./openai-image-mapping";
|
|
33
24
|
|
|
34
25
|
type DeepSeekAssistantMessageParam = ChatCompletionAssistantMessageParam & {
|
|
35
26
|
reasoning_content?: string | null;
|
|
@@ -108,19 +99,6 @@ export function toOpenAIUserContent(
|
|
|
108
99
|
];
|
|
109
100
|
}
|
|
110
101
|
|
|
111
|
-
export function imageAssetUrlMarker(assetId: ImageAssetId): ImageAssetUrlMarker {
|
|
112
|
-
parseImageAssetId(assetId);
|
|
113
|
-
return Object.freeze({ [IMAGE_ASSET_URL_MARKER]: assetId });
|
|
114
|
-
}
|
|
115
|
-
|
|
116
|
-
export function parseImageAssetUrlMarker(value: unknown): ImageAssetId | undefined {
|
|
117
|
-
if (typeof value !== "object" || value === null) {
|
|
118
|
-
return undefined;
|
|
119
|
-
}
|
|
120
|
-
const assetId = (value as Partial<ImageAssetUrlMarker>)[IMAGE_ASSET_URL_MARKER];
|
|
121
|
-
return typeof assetId === "string" ? parseImageAssetId(assetId) : undefined;
|
|
122
|
-
}
|
|
123
|
-
|
|
124
102
|
function requireMaterializedImage(
|
|
125
103
|
materializedImages: ReadonlyMap<ImageAssetId, string>,
|
|
126
104
|
assetId: ImageAssetId,
|
|
@@ -3,19 +3,14 @@ import type {
|
|
|
3
3
|
ChatCompletionCreateParamsNonStreaming,
|
|
4
4
|
ChatCompletionCreateParamsStreaming,
|
|
5
5
|
} from "openai/resources/chat/completions";
|
|
6
|
-
import type { AssistantMessage
|
|
6
|
+
import type { AssistantMessage } from "../agent/types";
|
|
7
7
|
import {
|
|
8
8
|
IMAGE_INPUT_POLICY,
|
|
9
9
|
IMAGE_INPUT_POLICY_VERSION,
|
|
10
10
|
} from "../image/image-input-policy";
|
|
11
|
-
import type { ImageAssetId, ImageAssetRef } from "../image/image-types";
|
|
12
11
|
import type { ModelContextBudget } from "./model-context-profile";
|
|
13
12
|
import type { InputTokenEstimator } from "./input-token-estimator";
|
|
14
|
-
import {
|
|
15
|
-
ModelRequestMediaAggregateError,
|
|
16
|
-
ProviderResponseError,
|
|
17
|
-
type ProviderResponseErrorCode,
|
|
18
|
-
} from "./model-client";
|
|
13
|
+
import { ProviderResponseError } from "./model-client";
|
|
19
14
|
import type {
|
|
20
15
|
MaterializedModelRequest,
|
|
21
16
|
ModelClient,
|
|
@@ -24,18 +19,25 @@ import type {
|
|
|
24
19
|
ModelRequestInput,
|
|
25
20
|
ModelRequestOptions,
|
|
26
21
|
ModelRequestOutput,
|
|
27
|
-
PreparedMediaDescriptor,
|
|
28
22
|
PreparedModelRequest,
|
|
29
23
|
PreparedPromptSegment,
|
|
30
24
|
} from "./model-client";
|
|
31
25
|
import {
|
|
32
26
|
fromOpenAIChatCompletion,
|
|
33
|
-
parseImageAssetUrlMarker,
|
|
34
27
|
toOpenAIChatMessages,
|
|
35
28
|
toOpenAIChatTools,
|
|
36
29
|
} from "./openai-chat-mapping";
|
|
37
30
|
import { OpenAIChatCompletionStreamAccumulator } from "./openai-chat-stream";
|
|
31
|
+
import {
|
|
32
|
+
deepFreeze,
|
|
33
|
+
imageUserSegment,
|
|
34
|
+
materializeOpenAIRequest,
|
|
35
|
+
normalizedEndpointPolicy,
|
|
36
|
+
sanitizedProviderError,
|
|
37
|
+
segmentKind,
|
|
38
|
+
} from "./openai-model-utils";
|
|
38
39
|
import { MoonshotInputTokenEstimator } from "./moonshot-input-token-estimator";
|
|
40
|
+
import type { ReasoningEffortController } from "./reasoning-effort";
|
|
39
41
|
import { sha256, stableJsonStringify } from "./model-request-preflight";
|
|
40
42
|
|
|
41
43
|
const OPENAI_CHAT_SERIALIZATION_VERSION = "openai-chat-v2";
|
|
@@ -47,6 +49,7 @@ export class OpenAIChatModelClient implements ModelClient {
|
|
|
47
49
|
serializationVersion: OPENAI_CHAT_SERIALIZATION_VERSION,
|
|
48
50
|
});
|
|
49
51
|
readonly inputTokenEstimator?: InputTokenEstimator;
|
|
52
|
+
readonly reasoningEffort?: ReasoningEffortController;
|
|
50
53
|
private readonly client: OpenAI;
|
|
51
54
|
private readonly preparedRequests = new WeakSet<object>();
|
|
52
55
|
private readonly materializedRequests = new WeakSet<object>();
|
|
@@ -71,6 +74,7 @@ export class OpenAIChatModelClient implements ModelClient {
|
|
|
71
74
|
};
|
|
72
75
|
model: string;
|
|
73
76
|
providerName?: string;
|
|
77
|
+
reasoningEffort?: ReasoningEffortController;
|
|
74
78
|
stream?: boolean;
|
|
75
79
|
timeoutMs?: number;
|
|
76
80
|
fetch?: typeof fetch;
|
|
@@ -78,6 +82,7 @@ export class OpenAIChatModelClient implements ModelClient {
|
|
|
78
82
|
) {
|
|
79
83
|
this.provider = options.providerName ?? "openai-compatible";
|
|
80
84
|
this.stream = options.stream ?? true;
|
|
85
|
+
this.reasoningEffort = options.reasoningEffort;
|
|
81
86
|
this.inputModalities = Object.freeze([...(options.inputModalities ?? ["text"])]);
|
|
82
87
|
const supportsImages = this.inputModalities.includes("image");
|
|
83
88
|
if (!this.inputModalities.includes("text")) {
|
|
@@ -111,10 +116,12 @@ export class OpenAIChatModelClient implements ModelClient {
|
|
|
111
116
|
includeReasoningContent: this.options.includeReasoningContent,
|
|
112
117
|
});
|
|
113
118
|
const tools = input.tools.length > 0 ? toOpenAIChatTools(input.tools) : undefined;
|
|
119
|
+
const reasoningEffort = this.reasoningEffort?.snapshot().effort;
|
|
114
120
|
const payload = deepFreeze({
|
|
115
121
|
model: this.options.model,
|
|
116
122
|
messages,
|
|
117
123
|
...(tools === undefined ? {} : { tools, tool_choice: "auto" as const }),
|
|
124
|
+
...(reasoningEffort === undefined ? {} : { reasoning_effort: reasoningEffort }),
|
|
118
125
|
max_completion_tokens: this.options.contextBudget.requestMaxOutputTokens,
|
|
119
126
|
...(this.stream
|
|
120
127
|
? {
|
|
@@ -185,48 +192,10 @@ export class OpenAIChatModelClient implements ModelClient {
|
|
|
185
192
|
options: ModelMaterializeOptions,
|
|
186
193
|
): Promise<MaterializedModelRequest> {
|
|
187
194
|
this.assertPrepared(prepared);
|
|
188
|
-
options.signal.throwIfAborted();
|
|
189
|
-
if (prepared.mediaOccurrenceCount > IMAGE_INPUT_POLICY.maxImagesPerRequest) {
|
|
190
|
-
throw new ModelRequestMediaAggregateError(
|
|
191
|
-
`Model request has ${prepared.mediaOccurrenceCount} images; maximum is ${IMAGE_INPUT_POLICY.maxImagesPerRequest}.`,
|
|
192
|
-
);
|
|
193
|
-
}
|
|
194
195
|
if (prepared.mediaOccurrenceCount > 0 && !this.inputModalities.includes("image")) {
|
|
195
196
|
throw new Error("Current model profile does not support image input.");
|
|
196
197
|
}
|
|
197
|
-
|
|
198
|
-
const assets = distinctPreparedAssets(prepared.promptSegments);
|
|
199
|
-
const lowerLengths = new Map<ImageAssetId, number>();
|
|
200
|
-
for (const asset of assets.values()) {
|
|
201
|
-
lowerLengths.set(asset.assetId, dataUrlLength(asset));
|
|
202
|
-
}
|
|
203
|
-
const markerCount = countImageMarkers(prepared.payload);
|
|
204
|
-
if (markerCount !== prepared.mediaOccurrenceCount) {
|
|
205
|
-
throw new Error("Prepared image marker count does not match media descriptors.");
|
|
206
|
-
}
|
|
207
|
-
const lowerBodyBytes = exactJsonBodyBytes(prepared.payload, lowerLengths);
|
|
208
|
-
assertBodyLimit(lowerBodyBytes, prepared.mediaOccurrenceCount);
|
|
209
|
-
|
|
210
|
-
const dataUrls = new Map<ImageAssetId, string>();
|
|
211
|
-
for (const asset of assets.values()) {
|
|
212
|
-
options.signal.throwIfAborted();
|
|
213
|
-
const bytes = await options.assetStore.readVerified(asset, {
|
|
214
|
-
signal: options.signal,
|
|
215
|
-
});
|
|
216
|
-
dataUrls.set(
|
|
217
|
-
asset.assetId,
|
|
218
|
-
`data:${asset.mimeType};base64,${bytes.toString("base64")}`,
|
|
219
|
-
);
|
|
220
|
-
await yieldToEventLoop();
|
|
221
|
-
}
|
|
222
|
-
options.signal.throwIfAborted();
|
|
223
|
-
const exactLengths = new Map(
|
|
224
|
-
[...dataUrls].map(([assetId, value]) => [assetId, value.length] as const),
|
|
225
|
-
);
|
|
226
|
-
const bodyBytes = exactJsonBodyBytes(prepared.payload, exactLengths);
|
|
227
|
-
assertBodyLimit(bodyBytes, prepared.mediaOccurrenceCount);
|
|
228
|
-
const payload = deepFreeze(materializePayload(prepared.payload, dataUrls));
|
|
229
|
-
const materialized = Object.freeze({ ...prepared, payload, bodyBytes });
|
|
198
|
+
const materialized = await materializeOpenAIRequest(prepared, options);
|
|
230
199
|
this.materializedRequests.add(materialized);
|
|
231
200
|
return materialized;
|
|
232
201
|
}
|
|
@@ -330,249 +299,4 @@ export class OpenAIChatModelClient implements ModelClient {
|
|
|
330
299
|
}
|
|
331
300
|
}
|
|
332
301
|
|
|
333
|
-
export
|
|
334
|
-
value: unknown,
|
|
335
|
-
imageDataUrlLengths: ReadonlyMap<ImageAssetId, number>,
|
|
336
|
-
): number {
|
|
337
|
-
const assetId = parseImageAssetUrlMarker(value);
|
|
338
|
-
if (assetId !== undefined) {
|
|
339
|
-
const length = imageDataUrlLengths.get(assetId);
|
|
340
|
-
if (!Number.isSafeInteger(length) || length === undefined || length < 1) {
|
|
341
|
-
throw new Error(
|
|
342
|
-
`Missing materialized length for image ${assetId.slice(0, 12)}….`,
|
|
343
|
-
);
|
|
344
|
-
}
|
|
345
|
-
return length + 2;
|
|
346
|
-
}
|
|
347
|
-
if (
|
|
348
|
-
value === null ||
|
|
349
|
-
typeof value === "string" ||
|
|
350
|
-
typeof value === "number" ||
|
|
351
|
-
typeof value === "boolean"
|
|
352
|
-
) {
|
|
353
|
-
const serialized: unknown = JSON.stringify(value);
|
|
354
|
-
if (typeof serialized !== "string") {
|
|
355
|
-
throw new Error("JSON primitive did not serialize to a string.");
|
|
356
|
-
}
|
|
357
|
-
return Buffer.byteLength(serialized, "utf8");
|
|
358
|
-
}
|
|
359
|
-
if (Array.isArray(value)) {
|
|
360
|
-
const entries: readonly unknown[] = value;
|
|
361
|
-
return (
|
|
362
|
-
2 +
|
|
363
|
-
Math.max(0, entries.length - 1) +
|
|
364
|
-
entries.reduce<number>(
|
|
365
|
-
(total, entry) => total + exactJsonBodyBytes(entry, imageDataUrlLengths),
|
|
366
|
-
0,
|
|
367
|
-
)
|
|
368
|
-
);
|
|
369
|
-
}
|
|
370
|
-
if (typeof value !== "object" || value === undefined) {
|
|
371
|
-
throw new Error(`Cannot size non-JSON value of type ${typeof value}.`);
|
|
372
|
-
}
|
|
373
|
-
const entries = Object.entries(value as Record<string, unknown>).filter(
|
|
374
|
-
([, entry]) => entry !== undefined,
|
|
375
|
-
);
|
|
376
|
-
return (
|
|
377
|
-
2 +
|
|
378
|
-
Math.max(0, entries.length - 1) +
|
|
379
|
-
entries.reduce(
|
|
380
|
-
(total, [key, entry]) =>
|
|
381
|
-
total +
|
|
382
|
-
Buffer.byteLength(JSON.stringify(key), "utf8") +
|
|
383
|
-
1 +
|
|
384
|
-
exactJsonBodyBytes(entry, imageDataUrlLengths),
|
|
385
|
-
0,
|
|
386
|
-
)
|
|
387
|
-
);
|
|
388
|
-
}
|
|
389
|
-
|
|
390
|
-
function imageUserSegment(message: UserMessage): PreparedPromptSegment {
|
|
391
|
-
const media = message.attachments!.map(
|
|
392
|
-
(attachment): PreparedMediaDescriptor =>
|
|
393
|
-
Object.freeze({
|
|
394
|
-
assetId: attachment.assetId,
|
|
395
|
-
label: attachment.label,
|
|
396
|
-
range: Object.freeze({ ...attachment.range }),
|
|
397
|
-
mimeType: attachment.mimeType,
|
|
398
|
-
byteLength: attachment.byteLength,
|
|
399
|
-
width: attachment.width,
|
|
400
|
-
height: attachment.height,
|
|
401
|
-
planningTokens: IMAGE_INPUT_POLICY.planningTokensPerImage,
|
|
402
|
-
}),
|
|
403
|
-
);
|
|
404
|
-
return Object.freeze({
|
|
405
|
-
kind: "user",
|
|
406
|
-
normalizedText: message.content,
|
|
407
|
-
media: Object.freeze(media),
|
|
408
|
-
});
|
|
409
|
-
}
|
|
410
|
-
|
|
411
|
-
function distinctPreparedAssets(
|
|
412
|
-
segments: readonly PreparedPromptSegment[],
|
|
413
|
-
): Map<ImageAssetId, ImageAssetRef> {
|
|
414
|
-
const assets = new Map<ImageAssetId, ImageAssetRef>();
|
|
415
|
-
for (const segment of segments) {
|
|
416
|
-
for (const media of segment.media ?? []) {
|
|
417
|
-
const asset = Object.freeze({
|
|
418
|
-
assetId: media.assetId,
|
|
419
|
-
mimeType: media.mimeType,
|
|
420
|
-
byteLength: media.byteLength,
|
|
421
|
-
width: media.width,
|
|
422
|
-
height: media.height,
|
|
423
|
-
});
|
|
424
|
-
const existing = assets.get(media.assetId);
|
|
425
|
-
if (
|
|
426
|
-
existing !== undefined &&
|
|
427
|
-
stableJsonStringify(existing) !== stableJsonStringify(asset)
|
|
428
|
-
) {
|
|
429
|
-
throw new Error(`Conflicting descriptors for image ${media.assetId}.`);
|
|
430
|
-
}
|
|
431
|
-
assets.set(media.assetId, asset);
|
|
432
|
-
}
|
|
433
|
-
}
|
|
434
|
-
return assets;
|
|
435
|
-
}
|
|
436
|
-
|
|
437
|
-
function materializePayload(
|
|
438
|
-
value: unknown,
|
|
439
|
-
dataUrls: ReadonlyMap<ImageAssetId, string>,
|
|
440
|
-
): unknown {
|
|
441
|
-
const assetId = parseImageAssetUrlMarker(value);
|
|
442
|
-
if (assetId !== undefined) {
|
|
443
|
-
const dataUrl = dataUrls.get(assetId);
|
|
444
|
-
if (dataUrl === undefined) {
|
|
445
|
-
throw new Error(`Image ${assetId.slice(0, 12)}… was not materialized.`);
|
|
446
|
-
}
|
|
447
|
-
return dataUrl;
|
|
448
|
-
}
|
|
449
|
-
if (Array.isArray(value)) {
|
|
450
|
-
return value.map((entry) => materializePayload(entry, dataUrls));
|
|
451
|
-
}
|
|
452
|
-
if (typeof value === "object" && value !== null) {
|
|
453
|
-
return Object.fromEntries(
|
|
454
|
-
Object.entries(value)
|
|
455
|
-
.filter(([, entry]) => entry !== undefined)
|
|
456
|
-
.map(([key, entry]) => [key, materializePayload(entry, dataUrls)]),
|
|
457
|
-
);
|
|
458
|
-
}
|
|
459
|
-
return value;
|
|
460
|
-
}
|
|
461
|
-
|
|
462
|
-
function countImageMarkers(value: unknown): number {
|
|
463
|
-
if (parseImageAssetUrlMarker(value) !== undefined) {
|
|
464
|
-
return 1;
|
|
465
|
-
}
|
|
466
|
-
if (Array.isArray(value)) {
|
|
467
|
-
const entries: readonly unknown[] = value;
|
|
468
|
-
return entries.reduce<number>(
|
|
469
|
-
(total, entry) => total + countImageMarkers(entry),
|
|
470
|
-
0,
|
|
471
|
-
);
|
|
472
|
-
}
|
|
473
|
-
if (typeof value === "object" && value !== null) {
|
|
474
|
-
return Object.values(value as Record<string, unknown>).reduce<number>(
|
|
475
|
-
(total, entry) => total + countImageMarkers(entry),
|
|
476
|
-
0,
|
|
477
|
-
);
|
|
478
|
-
}
|
|
479
|
-
return 0;
|
|
480
|
-
}
|
|
481
|
-
|
|
482
|
-
function dataUrlLength(asset: ImageAssetRef): number {
|
|
483
|
-
return `data:${asset.mimeType};base64,`.length + 4 * Math.ceil(asset.byteLength / 3);
|
|
484
|
-
}
|
|
485
|
-
|
|
486
|
-
function assertBodyLimit(bodyBytes: number, imageCount: number): void {
|
|
487
|
-
if (bodyBytes > IMAGE_INPUT_POLICY.maxRequestBodyBytes) {
|
|
488
|
-
throw new ModelRequestMediaAggregateError(
|
|
489
|
-
`Model request is ${bodyBytes} UTF-8 bytes with ${imageCount} images; maximum is ${IMAGE_INPUT_POLICY.maxRequestBodyBytes}.`,
|
|
490
|
-
);
|
|
491
|
-
}
|
|
492
|
-
}
|
|
493
|
-
|
|
494
|
-
function normalizedEndpointPolicy(baseURL: string | undefined): string {
|
|
495
|
-
const url = new URL(baseURL ?? "https://api.openai.com/v1");
|
|
496
|
-
url.username = "";
|
|
497
|
-
url.password = "";
|
|
498
|
-
url.search = "";
|
|
499
|
-
url.hash = "";
|
|
500
|
-
const pathname = url.pathname.replace(/\/+$/, "") || "/";
|
|
501
|
-
return `${url.protocol}//${url.host}${pathname}`;
|
|
502
|
-
}
|
|
503
|
-
|
|
504
|
-
function segmentKind(
|
|
505
|
-
role: ModelRequestInput["messages"][number]["role"],
|
|
506
|
-
): PreparedPromptSegment["kind"] {
|
|
507
|
-
switch (role) {
|
|
508
|
-
case "system":
|
|
509
|
-
return "kernel";
|
|
510
|
-
case "user":
|
|
511
|
-
return "user";
|
|
512
|
-
case "assistant":
|
|
513
|
-
return "assistant";
|
|
514
|
-
case "tool":
|
|
515
|
-
return "tool";
|
|
516
|
-
}
|
|
517
|
-
}
|
|
518
|
-
|
|
519
|
-
function yieldToEventLoop(): Promise<void> {
|
|
520
|
-
return new Promise((resolve) => setTimeout(resolve, 0));
|
|
521
|
-
}
|
|
522
|
-
|
|
523
|
-
function deepFreeze<T>(value: T): T {
|
|
524
|
-
if (typeof value !== "object" || value === null || Object.isFrozen(value)) {
|
|
525
|
-
return value;
|
|
526
|
-
}
|
|
527
|
-
for (const child of Object.values(value)) {
|
|
528
|
-
deepFreeze(child);
|
|
529
|
-
}
|
|
530
|
-
return Object.freeze(value);
|
|
531
|
-
}
|
|
532
|
-
|
|
533
|
-
function sanitizedProviderError(
|
|
534
|
-
error: unknown,
|
|
535
|
-
provider: string,
|
|
536
|
-
model: string,
|
|
537
|
-
): ProviderResponseError {
|
|
538
|
-
const message = error instanceof Error ? error.message : String(error);
|
|
539
|
-
const sanitized = message
|
|
540
|
-
.replace(
|
|
541
|
-
/data:image\/(?:png|jpeg|webp);base64,[A-Za-z0-9+/=]+/gu,
|
|
542
|
-
"[redacted image data]",
|
|
543
|
-
)
|
|
544
|
-
.replace(/Bearer\s+[A-Za-z0-9._~+/-]+/giu, "Bearer [redacted]");
|
|
545
|
-
return new ProviderResponseError(
|
|
546
|
-
providerErrorCode(error),
|
|
547
|
-
sanitized,
|
|
548
|
-
{ provider, model },
|
|
549
|
-
{ cause: error },
|
|
550
|
-
);
|
|
551
|
-
}
|
|
552
|
-
|
|
553
|
-
function providerErrorCode(error: unknown): ProviderResponseErrorCode {
|
|
554
|
-
const status = providerErrorStatus(error);
|
|
555
|
-
if (status === 429) {
|
|
556
|
-
return "provider_rate_limited";
|
|
557
|
-
}
|
|
558
|
-
if (status === 500 || status === 502 || status === 503 || status === 504) {
|
|
559
|
-
return "provider_unavailable";
|
|
560
|
-
}
|
|
561
|
-
if (status === undefined && isProviderConnectionError(error)) {
|
|
562
|
-
return "provider_unavailable";
|
|
563
|
-
}
|
|
564
|
-
return "provider_request_error";
|
|
565
|
-
}
|
|
566
|
-
|
|
567
|
-
function providerErrorStatus(error: unknown): number | undefined {
|
|
568
|
-
if (typeof error !== "object" || error === null || !("status" in error)) {
|
|
569
|
-
return undefined;
|
|
570
|
-
}
|
|
571
|
-
const status = (error as { status?: unknown }).status;
|
|
572
|
-
return typeof status === "number" ? status : undefined;
|
|
573
|
-
}
|
|
574
|
-
|
|
575
|
-
function isProviderConnectionError(error: unknown): boolean {
|
|
576
|
-
// APIConnectionTimeoutError extends APIConnectionError in the SDK.
|
|
577
|
-
return error instanceof OpenAI.APIConnectionError;
|
|
578
|
-
}
|
|
302
|
+
export { exactJsonBodyBytes } from "./openai-model-utils";
|
|
@@ -0,0 +1,20 @@
|
|
|
1
|
+
import { parseImageAssetId, type ImageAssetId } from "../image/image-types";
|
|
2
|
+
|
|
3
|
+
const IMAGE_ASSET_URL_MARKER = Symbol("tinker.image-asset-url-marker");
|
|
4
|
+
|
|
5
|
+
export type ImageAssetUrlMarker = {
|
|
6
|
+
readonly [IMAGE_ASSET_URL_MARKER]: ImageAssetId;
|
|
7
|
+
};
|
|
8
|
+
|
|
9
|
+
export function imageAssetUrlMarker(assetId: ImageAssetId): ImageAssetUrlMarker {
|
|
10
|
+
parseImageAssetId(assetId);
|
|
11
|
+
return Object.freeze({ [IMAGE_ASSET_URL_MARKER]: assetId });
|
|
12
|
+
}
|
|
13
|
+
|
|
14
|
+
export function parseImageAssetUrlMarker(value: unknown): ImageAssetId | undefined {
|
|
15
|
+
if (typeof value !== "object" || value === null) {
|
|
16
|
+
return undefined;
|
|
17
|
+
}
|
|
18
|
+
const assetId = (value as Partial<ImageAssetUrlMarker>)[IMAGE_ASSET_URL_MARKER];
|
|
19
|
+
return typeof assetId === "string" ? parseImageAssetId(assetId) : undefined;
|
|
20
|
+
}
|