tinker-agent 1.10.1 → 2.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +27 -1
- package/README.md +7 -23
- package/package.json +1 -1
- package/src/agent/context-meter.ts +14 -85
- package/src/agent/loop.ts +5 -14
- package/src/agent/runtime-session.ts +176 -24
- package/src/agent/session-ledger.ts +80 -0
- package/src/cli/config.ts +0 -5
- package/src/cli/model-profiles.ts +0 -98
- package/src/cli/public-config-contract.ts +1 -88
- package/src/cli/runner-dependencies.ts +0 -7
- package/src/cli/tui-memory.ts +0 -3
- package/src/events/observation-text-log.ts +4 -0
- package/src/events/stdout-event-printer.ts +5 -0
- package/src/events/types.ts +5 -1
- package/src/image/image-input-policy.ts +53 -2
- package/src/image/image-probe.ts +8 -2
- package/src/image/provider-image.ts +99 -0
- package/src/model/fake-model-client.ts +98 -69
- package/src/model/model-client.ts +2 -2
- package/src/model/model-request-preflight.ts +0 -1
- package/src/model/openai-chat-model-client.ts +4 -28
- package/src/model/openai-model-utils.ts +69 -31
- package/src/model/openai-responses-model-client.ts +0 -29
- package/src/model/token-estimator.ts +16 -3
- package/src/session/session-store.ts +42 -23
- package/src/tui/app.tsx +72 -7
- package/src/tui/components/footer.tsx +6 -1
- package/src/tui/event-store.ts +15 -0
- package/src/tui/tui-session-controller.ts +7 -0
- package/src/model/input-token-estimator.ts +0 -25
- package/src/model/moonshot-input-token-estimator.ts +0 -111
- package/src/model/openai-responses-token-estimator.ts +0 -155
|
@@ -100,6 +100,9 @@ export type PendingLedgerTurn = {
|
|
|
100
100
|
};
|
|
101
101
|
|
|
102
102
|
export type AgentTurnLedger = {
|
|
103
|
+
appendSteeringUserMessages(
|
|
104
|
+
messages: readonly UserMessage[],
|
|
105
|
+
): readonly CanonicalMessageRecord[];
|
|
103
106
|
appendAssistant(input: {
|
|
104
107
|
iteration: IterationIdentity;
|
|
105
108
|
message: AssistantMessage;
|
|
@@ -129,6 +132,13 @@ export type LedgerMutation =
|
|
|
129
132
|
admissionBase?: AdmissionBaseToken;
|
|
130
133
|
next: ProtocolContextView;
|
|
131
134
|
}
|
|
135
|
+
| {
|
|
136
|
+
kind: "append_steering_users";
|
|
137
|
+
turn: TurnIdentity;
|
|
138
|
+
frames: readonly ProtocolFrame[];
|
|
139
|
+
messages: readonly CanonicalMessageRecord[];
|
|
140
|
+
next: ProtocolContextView;
|
|
141
|
+
}
|
|
132
142
|
| {
|
|
133
143
|
kind: "append_assistant";
|
|
134
144
|
iteration: IterationIdentity;
|
|
@@ -396,6 +406,74 @@ export class InMemorySessionLedger implements SessionLedger {
|
|
|
396
406
|
this.pending = undefined;
|
|
397
407
|
}
|
|
398
408
|
|
|
409
|
+
appendSteeringUserMessages(
|
|
410
|
+
pending: InMemoryPendingLedgerTurn,
|
|
411
|
+
userMessages: readonly UserMessage[],
|
|
412
|
+
): readonly CanonicalMessageRecord[] {
|
|
413
|
+
this.requirePending(pending, "append steering user messages");
|
|
414
|
+
this.assertNoOpenFrame();
|
|
415
|
+
if (userMessages.length === 0) {
|
|
416
|
+
return Object.freeze([]);
|
|
417
|
+
}
|
|
418
|
+
for (const userMessage of userMessages) {
|
|
419
|
+
validateUserMessage(userMessage);
|
|
420
|
+
}
|
|
421
|
+
|
|
422
|
+
const frames: ProtocolFrame[] = [];
|
|
423
|
+
const messages: CanonicalMessageRecord[] = [];
|
|
424
|
+
for (const userMessage of userMessages) {
|
|
425
|
+
const createdAt = this.clock();
|
|
426
|
+
const ordinal = this.view.messages.length + messages.length + 1;
|
|
427
|
+
const frameId = this.input.idFactory.createProtocolFrameId();
|
|
428
|
+
messages.push(
|
|
429
|
+
immutableRecord<CanonicalMessageRecord>({
|
|
430
|
+
messageId: this.input.idFactory.createMessageId(),
|
|
431
|
+
sessionId: this.input.sessionId,
|
|
432
|
+
frameId,
|
|
433
|
+
ordinal,
|
|
434
|
+
contentSha256: userMessageHash(userMessage),
|
|
435
|
+
createdAt,
|
|
436
|
+
role: "user",
|
|
437
|
+
turnId: pending.turn.turnId,
|
|
438
|
+
content: userMessage.content,
|
|
439
|
+
...(userMessage.attachments === undefined
|
|
440
|
+
? {}
|
|
441
|
+
: {
|
|
442
|
+
attachments: Object.freeze(
|
|
443
|
+
immutableCanonicalClone(userMessage.attachments),
|
|
444
|
+
),
|
|
445
|
+
}),
|
|
446
|
+
origin: "user",
|
|
447
|
+
}),
|
|
448
|
+
);
|
|
449
|
+
frames.push(
|
|
450
|
+
immutableRecord<ProtocolFrame>({
|
|
451
|
+
frameId,
|
|
452
|
+
sessionId: this.input.sessionId,
|
|
453
|
+
turnId: pending.turn.turnId,
|
|
454
|
+
kind: "user",
|
|
455
|
+
state: "closed",
|
|
456
|
+
firstOrdinal: ordinal,
|
|
457
|
+
lastOrdinal: ordinal,
|
|
458
|
+
createdAt,
|
|
459
|
+
closedAt: createdAt,
|
|
460
|
+
}),
|
|
461
|
+
);
|
|
462
|
+
}
|
|
463
|
+
const immutableFrames = Object.freeze(frames);
|
|
464
|
+
const immutableMessages = Object.freeze(messages);
|
|
465
|
+
const next = appendView(this.view, immutableFrames, immutableMessages, []);
|
|
466
|
+
this.validator.validate(next, { fullIntegrity: true });
|
|
467
|
+
this.commit({
|
|
468
|
+
kind: "append_steering_users",
|
|
469
|
+
turn: pending.turn,
|
|
470
|
+
frames: immutableFrames,
|
|
471
|
+
messages: immutableMessages,
|
|
472
|
+
next,
|
|
473
|
+
});
|
|
474
|
+
return immutableMessages;
|
|
475
|
+
}
|
|
476
|
+
|
|
399
477
|
appendAssistant(
|
|
400
478
|
pending: InMemoryPendingLedgerTurn,
|
|
401
479
|
input: {
|
|
@@ -776,6 +854,8 @@ class InMemoryPendingLedgerTurn implements PendingLedgerTurn {
|
|
|
776
854
|
readonly turn: TurnIdentity,
|
|
777
855
|
) {
|
|
778
856
|
this.agent = {
|
|
857
|
+
appendSteeringUserMessages: (messages) =>
|
|
858
|
+
this.ledger.appendSteeringUserMessages(this, messages),
|
|
779
859
|
appendAssistant: (input) => this.ledger.appendAssistant(this, input),
|
|
780
860
|
assertCanExecuteTool: (call) => this.ledger.assertCanExecuteTool(this, call),
|
|
781
861
|
commitToolCompletions: (completions) =>
|
package/src/cli/config.ts
CHANGED
|
@@ -14,7 +14,6 @@ import {
|
|
|
14
14
|
type ModelInputModality,
|
|
15
15
|
type ModelProfile,
|
|
16
16
|
type ModelProfiles,
|
|
17
|
-
type ModelTokenEstimatorProfile,
|
|
18
17
|
unknownProfileError,
|
|
19
18
|
} from "./model-profiles";
|
|
20
19
|
import type { MemoryEmbeddingConfig } from "../memory/contracts";
|
|
@@ -40,7 +39,6 @@ export type RunnerConfig = {
|
|
|
40
39
|
readonly contextBudget: ModelContextBudget;
|
|
41
40
|
readonly profileName?: string;
|
|
42
41
|
readonly inputModalities: readonly ModelInputModality[];
|
|
43
|
-
readonly tokenEstimator?: ModelTokenEstimatorProfile;
|
|
44
42
|
readonly bashGuardMode: "guard" | "yolo";
|
|
45
43
|
readonly bashGuardSource: "default" | "environment" | "cli";
|
|
46
44
|
};
|
|
@@ -184,9 +182,6 @@ function runnerConfigTemplateFromProfile(
|
|
|
184
182
|
inputModalities: profile.inputModalities,
|
|
185
183
|
bashGuardMode: environment.bashGuardMode,
|
|
186
184
|
bashGuardSource: environment.bashGuardSource,
|
|
187
|
-
...(profile.tokenEstimator === undefined
|
|
188
|
-
? {}
|
|
189
|
-
: { tokenEstimator: profile.tokenEstimator }),
|
|
190
185
|
});
|
|
191
186
|
}
|
|
192
187
|
|
|
@@ -10,9 +10,6 @@ import {
|
|
|
10
10
|
MODEL_PROFILE_FIELDS,
|
|
11
11
|
MODEL_PROFILES_DOCUMENT_FIELDS,
|
|
12
12
|
MODEL_REASONING_FIELDS,
|
|
13
|
-
MODEL_TOKEN_ESTIMATOR_FIELDS,
|
|
14
|
-
type ModelTokenEstimatorKind,
|
|
15
|
-
type ModelTokenEstimatorMaxRetries,
|
|
16
13
|
} from "./public-config-contract";
|
|
17
14
|
import type { MemoryEmbeddingConfig } from "../memory/contracts";
|
|
18
15
|
import type { ReasoningEffortConfig } from "../model/reasoning-effort";
|
|
@@ -29,20 +26,10 @@ export type ModelProfile = {
|
|
|
29
26
|
readonly includeReasoningContent: boolean;
|
|
30
27
|
readonly stream: boolean;
|
|
31
28
|
readonly inputModalities: readonly ModelInputModality[];
|
|
32
|
-
readonly tokenEstimator?: ModelTokenEstimatorProfile;
|
|
33
29
|
};
|
|
34
30
|
|
|
35
31
|
export type ModelInputModality = "text" | "image";
|
|
36
32
|
|
|
37
|
-
export type ModelTokenEstimatorProfile = {
|
|
38
|
-
readonly kind: ModelTokenEstimatorKind;
|
|
39
|
-
readonly model: string;
|
|
40
|
-
readonly apiBase: string;
|
|
41
|
-
readonly apiKey: string;
|
|
42
|
-
readonly timeoutMs: number;
|
|
43
|
-
readonly maxRetries: ModelTokenEstimatorMaxRetries;
|
|
44
|
-
};
|
|
45
|
-
|
|
46
33
|
export type ModelProfiles = {
|
|
47
34
|
readonly defaultProfile: string;
|
|
48
35
|
readonly profiles: ReadonlyMap<string, ModelProfile>;
|
|
@@ -285,16 +272,6 @@ function parseProfile(
|
|
|
285
272
|
value.inputModalities,
|
|
286
273
|
`${where}: "inputModalities"`,
|
|
287
274
|
);
|
|
288
|
-
const tokenEstimator =
|
|
289
|
-
value.tokenEstimator === undefined
|
|
290
|
-
? undefined
|
|
291
|
-
: parseTokenEstimator(value.tokenEstimator, `${where}: "tokenEstimator"`);
|
|
292
|
-
if (inputModalities.includes("image") && tokenEstimator === undefined) {
|
|
293
|
-
throw new Error(
|
|
294
|
-
`${where}: image input requires a complete "tokenEstimator" configuration.`,
|
|
295
|
-
);
|
|
296
|
-
}
|
|
297
|
-
|
|
298
275
|
createModelContextProfile({
|
|
299
276
|
contextWindowTokens,
|
|
300
277
|
maxSupportedOutputTokens,
|
|
@@ -312,7 +289,6 @@ function parseProfile(
|
|
|
312
289
|
includeReasoningContent,
|
|
313
290
|
stream,
|
|
314
291
|
inputModalities,
|
|
315
|
-
...(tokenEstimator === undefined ? {} : { tokenEstimator }),
|
|
316
292
|
});
|
|
317
293
|
}
|
|
318
294
|
|
|
@@ -434,54 +410,7 @@ function parseInputModalities(
|
|
|
434
410
|
);
|
|
435
411
|
}
|
|
436
412
|
|
|
437
|
-
function parseTokenEstimator(value: unknown, name: string): ModelTokenEstimatorProfile {
|
|
438
|
-
if (!isRecord(value)) {
|
|
439
|
-
throw new Error(`${name} must be an object.`);
|
|
440
|
-
}
|
|
441
|
-
assertKnownKeys(
|
|
442
|
-
value,
|
|
443
|
-
MODEL_TOKEN_ESTIMATOR_FIELDS.map((field) => field.name),
|
|
444
|
-
name,
|
|
445
|
-
);
|
|
446
|
-
const kindField = tokenEstimatorField("kind");
|
|
447
|
-
if (value.kind !== kindField.literalValue) {
|
|
448
|
-
throw new Error(`${name}.kind must be ${JSON.stringify(kindField.literalValue)}.`);
|
|
449
|
-
}
|
|
450
|
-
const model = parseTokenEstimatorString(value, "model", name);
|
|
451
|
-
const apiBase = parseTokenEstimatorString(value, "apiBase", name);
|
|
452
|
-
const apiKey = parseTokenEstimatorString(value, "apiKey", name);
|
|
453
|
-
const timeoutField = tokenEstimatorField("timeoutMs");
|
|
454
|
-
if (timeoutField.valueKind !== "positive-integer") {
|
|
455
|
-
throw new Error("Token estimator timeoutMs contract kind is invalid.");
|
|
456
|
-
}
|
|
457
|
-
const timeoutMs = requirePositiveInteger(value.timeoutMs, `${name}.timeoutMs`);
|
|
458
|
-
if (
|
|
459
|
-
timeoutField.minimum === undefined ||
|
|
460
|
-
timeoutField.maximum === undefined ||
|
|
461
|
-
timeoutMs < timeoutField.minimum ||
|
|
462
|
-
timeoutMs > timeoutField.maximum
|
|
463
|
-
) {
|
|
464
|
-
throw new Error(
|
|
465
|
-
`${name}.timeoutMs must be between ${timeoutField.minimum} and ${timeoutField.maximum}.`,
|
|
466
|
-
);
|
|
467
|
-
}
|
|
468
|
-
const maxRetriesField = tokenEstimatorField("maxRetries");
|
|
469
|
-
if (value.maxRetries !== maxRetriesField.literalValue) {
|
|
470
|
-
throw new Error(`${name}.maxRetries must be ${maxRetriesField.literalValue}.`);
|
|
471
|
-
}
|
|
472
|
-
return Object.freeze({
|
|
473
|
-
kind: kindField.literalValue,
|
|
474
|
-
model,
|
|
475
|
-
apiBase,
|
|
476
|
-
apiKey,
|
|
477
|
-
timeoutMs,
|
|
478
|
-
maxRetries: maxRetriesField.literalValue,
|
|
479
|
-
});
|
|
480
|
-
}
|
|
481
|
-
|
|
482
413
|
type ModelProfileFieldName = (typeof MODEL_PROFILE_FIELDS)[number]["name"];
|
|
483
|
-
type ModelTokenEstimatorFieldName =
|
|
484
|
-
(typeof MODEL_TOKEN_ESTIMATOR_FIELDS)[number]["name"];
|
|
485
414
|
|
|
486
415
|
function modelProfileField<Name extends ModelProfileFieldName>(
|
|
487
416
|
name: Name,
|
|
@@ -496,21 +425,6 @@ function modelProfileField<Name extends ModelProfileFieldName>(
|
|
|
496
425
|
>;
|
|
497
426
|
}
|
|
498
427
|
|
|
499
|
-
function tokenEstimatorField<Name extends ModelTokenEstimatorFieldName>(
|
|
500
|
-
name: Name,
|
|
501
|
-
): Extract<(typeof MODEL_TOKEN_ESTIMATOR_FIELDS)[number], { readonly name: Name }> {
|
|
502
|
-
const field = MODEL_TOKEN_ESTIMATOR_FIELDS.find(
|
|
503
|
-
(candidate) => candidate.name === name,
|
|
504
|
-
);
|
|
505
|
-
if (field === undefined) {
|
|
506
|
-
throw new Error(`Missing token estimator field contract for ${name}.`);
|
|
507
|
-
}
|
|
508
|
-
return field as Extract<
|
|
509
|
-
(typeof MODEL_TOKEN_ESTIMATOR_FIELDS)[number],
|
|
510
|
-
{ readonly name: Name }
|
|
511
|
-
>;
|
|
512
|
-
}
|
|
513
|
-
|
|
514
428
|
function parseProfileString(
|
|
515
429
|
value: Record<string, unknown>,
|
|
516
430
|
name: "model" | "apiBase" | "apiKey",
|
|
@@ -555,18 +469,6 @@ function parseProfileBoolean(
|
|
|
555
469
|
: parseBoolean(value[name], `${where}: ${JSON.stringify(name)}`);
|
|
556
470
|
}
|
|
557
471
|
|
|
558
|
-
function parseTokenEstimatorString(
|
|
559
|
-
value: Record<string, unknown>,
|
|
560
|
-
name: "model" | "apiBase" | "apiKey",
|
|
561
|
-
where: string,
|
|
562
|
-
): string {
|
|
563
|
-
const field = tokenEstimatorField(name);
|
|
564
|
-
if (field.valueKind !== "non-empty-string") {
|
|
565
|
-
throw new Error(`Token estimator field ${name} has an invalid contract kind.`);
|
|
566
|
-
}
|
|
567
|
-
return requireString(value[name], `${where}.${name}`);
|
|
568
|
-
}
|
|
569
|
-
|
|
570
472
|
function assertKnownKeys(
|
|
571
473
|
value: Record<string, unknown>,
|
|
572
474
|
allowed: readonly string[],
|
|
@@ -242,11 +242,7 @@ export const PUBLIC_CONFIG_FIELDS = Object.freeze([
|
|
|
242
242
|
|
|
243
243
|
export type ModelProfileField = {
|
|
244
244
|
readonly name: string;
|
|
245
|
-
readonly valueKind:
|
|
246
|
-
| PublicConfigValueKind
|
|
247
|
-
| "input-modalities"
|
|
248
|
-
| "reasoning"
|
|
249
|
-
| "token-estimator";
|
|
245
|
+
readonly valueKind: PublicConfigValueKind | "input-modalities" | "reasoning";
|
|
250
246
|
readonly required: boolean;
|
|
251
247
|
readonly defaultValue?: string | number | boolean | readonly string[];
|
|
252
248
|
readonly secret: boolean;
|
|
@@ -337,13 +333,6 @@ export const MODEL_PROFILE_FIELDS = Object.freeze([
|
|
|
337
333
|
description:
|
|
338
334
|
'Accepted model input modalities; normalizes to ["text"] or ["text", "image"].',
|
|
339
335
|
}),
|
|
340
|
-
profileField({
|
|
341
|
-
name: "tokenEstimator",
|
|
342
|
-
valueKind: "token-estimator",
|
|
343
|
-
required: false,
|
|
344
|
-
secret: true,
|
|
345
|
-
description: "Independent token estimator required for image profiles.",
|
|
346
|
-
}),
|
|
347
336
|
]);
|
|
348
337
|
|
|
349
338
|
export type ModelReasoningField = {
|
|
@@ -375,82 +364,6 @@ export const MODEL_REASONING_FIELDS = Object.freeze([
|
|
|
375
364
|
}),
|
|
376
365
|
]);
|
|
377
366
|
|
|
378
|
-
export type ModelTokenEstimatorField = {
|
|
379
|
-
readonly name: string;
|
|
380
|
-
readonly valueKind: PublicConfigValueKind | "literal-string" | "literal-number";
|
|
381
|
-
readonly required: true;
|
|
382
|
-
readonly secret: boolean;
|
|
383
|
-
readonly literalValue?: string | number;
|
|
384
|
-
readonly minimum?: number;
|
|
385
|
-
readonly maximum?: number;
|
|
386
|
-
readonly description: string;
|
|
387
|
-
};
|
|
388
|
-
|
|
389
|
-
function estimatorField<const T extends ModelTokenEstimatorField>(
|
|
390
|
-
field: T,
|
|
391
|
-
): Readonly<T> {
|
|
392
|
-
return Object.freeze(field);
|
|
393
|
-
}
|
|
394
|
-
|
|
395
|
-
export const MODEL_TOKEN_ESTIMATOR_FIELDS = Object.freeze([
|
|
396
|
-
estimatorField({
|
|
397
|
-
name: "kind",
|
|
398
|
-
valueKind: "literal-string",
|
|
399
|
-
required: true,
|
|
400
|
-
secret: false,
|
|
401
|
-
literalValue: "moonshot-estimate-token-count-v1",
|
|
402
|
-
description: "Estimator protocol discriminator.",
|
|
403
|
-
}),
|
|
404
|
-
estimatorField({
|
|
405
|
-
name: "model",
|
|
406
|
-
valueKind: "non-empty-string",
|
|
407
|
-
required: true,
|
|
408
|
-
secret: false,
|
|
409
|
-
description: "Estimator model name.",
|
|
410
|
-
}),
|
|
411
|
-
estimatorField({
|
|
412
|
-
name: "apiBase",
|
|
413
|
-
valueKind: "non-empty-string",
|
|
414
|
-
required: true,
|
|
415
|
-
secret: false,
|
|
416
|
-
description: "Estimator API base URL.",
|
|
417
|
-
}),
|
|
418
|
-
estimatorField({
|
|
419
|
-
name: "apiKey",
|
|
420
|
-
valueKind: "non-empty-string",
|
|
421
|
-
required: true,
|
|
422
|
-
secret: true,
|
|
423
|
-
description: "Estimator API credential.",
|
|
424
|
-
}),
|
|
425
|
-
estimatorField({
|
|
426
|
-
name: "timeoutMs",
|
|
427
|
-
valueKind: "positive-integer",
|
|
428
|
-
required: true,
|
|
429
|
-
secret: false,
|
|
430
|
-
minimum: 1_000,
|
|
431
|
-
maximum: 60_000,
|
|
432
|
-
description: "Estimator request timeout in milliseconds.",
|
|
433
|
-
}),
|
|
434
|
-
estimatorField({
|
|
435
|
-
name: "maxRetries",
|
|
436
|
-
valueKind: "literal-number",
|
|
437
|
-
required: true,
|
|
438
|
-
secret: false,
|
|
439
|
-
literalValue: 0,
|
|
440
|
-
description: "Estimator retry count; retries are disabled.",
|
|
441
|
-
}),
|
|
442
|
-
]);
|
|
443
|
-
|
|
444
|
-
export type ModelTokenEstimatorKind = Extract<
|
|
445
|
-
(typeof MODEL_TOKEN_ESTIMATOR_FIELDS)[number],
|
|
446
|
-
{ readonly name: "kind" }
|
|
447
|
-
>["literalValue"];
|
|
448
|
-
|
|
449
|
-
export type ModelTokenEstimatorMaxRetries = Extract<
|
|
450
|
-
(typeof MODEL_TOKEN_ESTIMATOR_FIELDS)[number],
|
|
451
|
-
{ readonly name: "maxRetries" }
|
|
452
|
-
>["literalValue"];
|
|
453
|
-
|
|
454
367
|
export type MemoryConfigField = {
|
|
455
368
|
readonly name: "profile" | "embedding";
|
|
456
369
|
readonly valueKind: "non-empty-string" | "embedding-profile";
|
|
@@ -66,7 +66,6 @@ export function createModelClient(
|
|
|
66
66
|
| "apiKey"
|
|
67
67
|
| "apiBase"
|
|
68
68
|
| "inputModalities"
|
|
69
|
-
| "tokenEstimator"
|
|
70
69
|
>,
|
|
71
70
|
env: NodeJS.ProcessEnv = process.env,
|
|
72
71
|
reasoningEffort?: ReasoningEffortController,
|
|
@@ -82,9 +81,6 @@ export function createModelClient(
|
|
|
82
81
|
...(activeReasoningEffort === undefined
|
|
83
82
|
? {}
|
|
84
83
|
: { reasoningEffort: activeReasoningEffort }),
|
|
85
|
-
...(config.tokenEstimator === undefined
|
|
86
|
-
? {}
|
|
87
|
-
: { tokenEstimator: config.tokenEstimator }),
|
|
88
84
|
...(env.TINKER_TEST_FAKE_MODEL_REQUEST_LOG === undefined ||
|
|
89
85
|
env.TINKER_TEST_FAKE_MODEL_REQUEST_LOG === ""
|
|
90
86
|
? {}
|
|
@@ -102,9 +98,6 @@ export function createModelClient(
|
|
|
102
98
|
...(activeReasoningEffort === undefined
|
|
103
99
|
? {}
|
|
104
100
|
: { reasoningEffort: activeReasoningEffort }),
|
|
105
|
-
...(config.tokenEstimator === undefined
|
|
106
|
-
? {}
|
|
107
|
-
: { tokenEstimator: config.tokenEstimator }),
|
|
108
101
|
};
|
|
109
102
|
if (config.api === "responses") {
|
|
110
103
|
return new OpenAIResponsesModelClient(common);
|
package/src/cli/tui-memory.ts
CHANGED
|
@@ -47,9 +47,6 @@ export async function initializeTuiMemory(input: {
|
|
|
47
47
|
stream: profile.stream,
|
|
48
48
|
contextBudget: memoryConfig.contextBudget,
|
|
49
49
|
inputModalities: profile.inputModalities,
|
|
50
|
-
...(profile.tokenEstimator === undefined
|
|
51
|
-
? {}
|
|
52
|
-
: { tokenEstimator: profile.tokenEstimator }),
|
|
53
50
|
},
|
|
54
51
|
input.env,
|
|
55
52
|
);
|
|
@@ -35,6 +35,10 @@ export function renderObservationLogEvent(event: AgentEvent): string | undefined
|
|
|
35
35
|
].join("\n");
|
|
36
36
|
case "turn.started":
|
|
37
37
|
return renderTurnStarted(event);
|
|
38
|
+
case "turn.steering.applied":
|
|
39
|
+
return ["## User follow-up", "", event.data.userPrompt.text, "", "---", ""].join(
|
|
40
|
+
"\n",
|
|
41
|
+
);
|
|
38
42
|
case "assistant.progress":
|
|
39
43
|
return renderAssistantProgress(event);
|
|
40
44
|
case "tool.observation":
|
|
@@ -37,6 +37,11 @@ export class StdoutEventPrinter implements EventSink {
|
|
|
37
37
|
`turn.started turn=${event.turnNumber} turnId=${event.turnId}\n`,
|
|
38
38
|
);
|
|
39
39
|
break;
|
|
40
|
+
case "turn.steering.applied":
|
|
41
|
+
this.stdout.write(
|
|
42
|
+
`turn.steering.applied turn=${event.turnNumber} ordinal=${event.data.ordinal}\n`,
|
|
43
|
+
);
|
|
44
|
+
break;
|
|
40
45
|
case "agent.iteration.started":
|
|
41
46
|
this.stdout.write(
|
|
42
47
|
`agent.iteration.started iteration=${event.iterationNumber} iterationId=${event.iterationId}\n`,
|
package/src/events/types.ts
CHANGED
|
@@ -307,6 +307,10 @@ export type AgentEventDataMap = {
|
|
|
307
307
|
"session.interrupted_frame_recovered": InterruptedFrameRecoveredData;
|
|
308
308
|
"session.finished": SessionFinishedData;
|
|
309
309
|
"turn.started": { userPrompt: UserPromptProjection };
|
|
310
|
+
"turn.steering.applied": {
|
|
311
|
+
userPrompt: UserPromptProjection;
|
|
312
|
+
ordinal: number;
|
|
313
|
+
};
|
|
310
314
|
"turn.finished": TurnFinishedData;
|
|
311
315
|
"turn.failed": { error: string };
|
|
312
316
|
"turn.cancelled": { cancellation: TurnCancellation };
|
|
@@ -411,7 +415,7 @@ export type AgentEventInput =
|
|
|
411
415
|
>
|
|
412
416
|
| SessionEventInput<"mcp.server.connected" | "mcp.server.failed">
|
|
413
417
|
| SessionEventInput<"diagnostic.sink_failed">
|
|
414
|
-
| TurnEventInput<"turn.started" | "turn.finished">
|
|
418
|
+
| TurnEventInput<"turn.started" | "turn.steering.applied" | "turn.finished">
|
|
415
419
|
| (
|
|
416
420
|
| TurnEventInput<"turn.failed" | "turn.cancelled">
|
|
417
421
|
| IterationEventInput<"turn.failed" | "turn.cancelled">
|
|
@@ -4,11 +4,62 @@ export const IMAGE_INPUT_POLICY = Object.freeze({
|
|
|
4
4
|
maxBytesPerImage: 20 * 1024 * 1024,
|
|
5
5
|
maxImagesPerMessage: 8,
|
|
6
6
|
maxImagesPerRequest: 8,
|
|
7
|
+
maxProviderLongEdge: 2048,
|
|
7
8
|
maxLongEdge: 4096,
|
|
8
9
|
maxPixels: 8_847_360,
|
|
9
10
|
maxRequestBodyBytes: 90_000_000,
|
|
10
|
-
|
|
11
|
+
allowUpscale: false,
|
|
12
|
+
resizePolicy: "sharp-auto-orient-lanczos3-round-v1",
|
|
13
|
+
outputEncodingPolicy: "preserve-input-format-fixed-encoding-v1",
|
|
14
|
+
imageTokenBuckets: Object.freeze([
|
|
15
|
+
Object.freeze({ maxLongEdge: 512, planningTokens: 384 }),
|
|
16
|
+
Object.freeze({ maxLongEdge: 1024, planningTokens: 1408 }),
|
|
17
|
+
Object.freeze({ maxLongEdge: 1536, planningTokens: 3072 }),
|
|
18
|
+
Object.freeze({ maxLongEdge: 2048, planningTokens: 5504 }),
|
|
19
|
+
] as const),
|
|
20
|
+
imageTokensUseTextCorrectionFactor: false,
|
|
11
21
|
retryPolicy: "none",
|
|
12
22
|
} as const);
|
|
13
23
|
|
|
14
|
-
export const IMAGE_INPUT_POLICY_VERSION = "image-input-policy-
|
|
24
|
+
export const IMAGE_INPUT_POLICY_VERSION = "image-input-policy-v2" as const;
|
|
25
|
+
|
|
26
|
+
export function providerImageDimensions(
|
|
27
|
+
width: number,
|
|
28
|
+
height: number,
|
|
29
|
+
): { readonly width: number; readonly height: number } {
|
|
30
|
+
requireDimension(width, "width");
|
|
31
|
+
requireDimension(height, "height");
|
|
32
|
+
const longEdge = Math.max(width, height);
|
|
33
|
+
if (longEdge <= IMAGE_INPUT_POLICY.maxProviderLongEdge) {
|
|
34
|
+
return Object.freeze({ width, height });
|
|
35
|
+
}
|
|
36
|
+
const scale = IMAGE_INPUT_POLICY.maxProviderLongEdge / longEdge;
|
|
37
|
+
let targetWidth = Math.max(1, Math.round(width * scale));
|
|
38
|
+
let targetHeight = Math.max(1, Math.round(height * scale));
|
|
39
|
+
if (targetWidth > IMAGE_INPUT_POLICY.maxProviderLongEdge) {
|
|
40
|
+
targetWidth = IMAGE_INPUT_POLICY.maxProviderLongEdge;
|
|
41
|
+
}
|
|
42
|
+
if (targetHeight > IMAGE_INPUT_POLICY.maxProviderLongEdge) {
|
|
43
|
+
targetHeight = IMAGE_INPUT_POLICY.maxProviderLongEdge;
|
|
44
|
+
}
|
|
45
|
+
return Object.freeze({ width: targetWidth, height: targetHeight });
|
|
46
|
+
}
|
|
47
|
+
|
|
48
|
+
export function imagePlanningTokens(width: number, height: number): number {
|
|
49
|
+
requireDimension(width, "width");
|
|
50
|
+
requireDimension(height, "height");
|
|
51
|
+
const longEdge = Math.max(width, height);
|
|
52
|
+
const bucket = IMAGE_INPUT_POLICY.imageTokenBuckets.find(
|
|
53
|
+
(candidate) => longEdge <= candidate.maxLongEdge,
|
|
54
|
+
);
|
|
55
|
+
if (bucket === undefined) {
|
|
56
|
+
throw new Error("Materialized image exceeds the provider image size policy.");
|
|
57
|
+
}
|
|
58
|
+
return bucket.planningTokens;
|
|
59
|
+
}
|
|
60
|
+
|
|
61
|
+
function requireDimension(value: number, name: string): void {
|
|
62
|
+
if (!Number.isSafeInteger(value) || value < 1) {
|
|
63
|
+
throw new Error(`Image ${name} must be a positive safe integer.`);
|
|
64
|
+
}
|
|
65
|
+
}
|
package/src/image/image-probe.ts
CHANGED
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import sharp from "sharp";
|
|
2
2
|
import { IMAGE_INPUT_POLICY } from "./image-input-policy";
|
|
3
|
+
import { orientedDimensions } from "./provider-image";
|
|
3
4
|
import {
|
|
4
5
|
imageAssetIdForBytes,
|
|
5
6
|
type ImageAssetRef,
|
|
@@ -59,8 +60,13 @@ export async function probeImageBytes(
|
|
|
59
60
|
`Image container and decoder disagree on format (${container.mimeType} vs ${mimeType}).`,
|
|
60
61
|
);
|
|
61
62
|
}
|
|
62
|
-
const
|
|
63
|
-
const
|
|
63
|
+
const decodedWidth = requireDimension(metadata.width, "width");
|
|
64
|
+
const decodedHeight = requireDimension(metadata.height, "height");
|
|
65
|
+
const { width, height } = orientedDimensions(
|
|
66
|
+
decodedWidth,
|
|
67
|
+
decodedHeight,
|
|
68
|
+
metadata.orientation,
|
|
69
|
+
);
|
|
64
70
|
const decoderAnimated =
|
|
65
71
|
(metadata.pages ?? 1) > 1 || metadata.pageHeight !== undefined;
|
|
66
72
|
if (container.animated || decoderAnimated) {
|
|
@@ -0,0 +1,99 @@
|
|
|
1
|
+
import sharp from "sharp";
|
|
2
|
+
import {
|
|
3
|
+
IMAGE_INPUT_POLICY,
|
|
4
|
+
imagePlanningTokens,
|
|
5
|
+
providerImageDimensions,
|
|
6
|
+
} from "./image-input-policy";
|
|
7
|
+
import type { ImageMimeType } from "./image-types";
|
|
8
|
+
|
|
9
|
+
export type ProviderImage = {
|
|
10
|
+
readonly bytes: Buffer;
|
|
11
|
+
readonly mimeType: ImageMimeType;
|
|
12
|
+
readonly width: number;
|
|
13
|
+
readonly height: number;
|
|
14
|
+
readonly planningTokens: number;
|
|
15
|
+
};
|
|
16
|
+
|
|
17
|
+
export async function materializeProviderImage(
|
|
18
|
+
bytes: Buffer,
|
|
19
|
+
mimeType: ImageMimeType,
|
|
20
|
+
): Promise<ProviderImage> {
|
|
21
|
+
const input = sharp(bytes, {
|
|
22
|
+
failOn: "warning",
|
|
23
|
+
limitInputPixels: IMAGE_INPUT_POLICY.maxPixels,
|
|
24
|
+
unlimited: false,
|
|
25
|
+
sequentialRead: true,
|
|
26
|
+
});
|
|
27
|
+
const metadata = await input.metadata();
|
|
28
|
+
const sourceWidth = requireDimension(metadata.width, "width");
|
|
29
|
+
const sourceHeight = requireDimension(metadata.height, "height");
|
|
30
|
+
const oriented = orientedDimensions(sourceWidth, sourceHeight, metadata.orientation);
|
|
31
|
+
const target = providerImageDimensions(oriented.width, oriented.height);
|
|
32
|
+
const requiresOrientation =
|
|
33
|
+
metadata.orientation !== undefined && metadata.orientation !== 1;
|
|
34
|
+
const requiresResize =
|
|
35
|
+
target.width !== oriented.width || target.height !== oriented.height;
|
|
36
|
+
|
|
37
|
+
let outputBytes = bytes;
|
|
38
|
+
if (requiresOrientation || requiresResize) {
|
|
39
|
+
let pipeline = sharp(bytes, {
|
|
40
|
+
failOn: "warning",
|
|
41
|
+
limitInputPixels: IMAGE_INPUT_POLICY.maxPixels,
|
|
42
|
+
unlimited: false,
|
|
43
|
+
sequentialRead: true,
|
|
44
|
+
}).rotate();
|
|
45
|
+
if (requiresResize) {
|
|
46
|
+
pipeline = pipeline.resize(target.width, target.height, {
|
|
47
|
+
fit: "fill",
|
|
48
|
+
kernel: sharp.kernel.lanczos3,
|
|
49
|
+
withoutEnlargement: true,
|
|
50
|
+
});
|
|
51
|
+
}
|
|
52
|
+
outputBytes = await encodeInOriginalFormat(pipeline, mimeType);
|
|
53
|
+
}
|
|
54
|
+
|
|
55
|
+
const outputMetadata = await sharp(outputBytes).metadata();
|
|
56
|
+
const width = requireDimension(outputMetadata.width, "width");
|
|
57
|
+
const height = requireDimension(outputMetadata.height, "height");
|
|
58
|
+
if (Math.max(width, height) > IMAGE_INPUT_POLICY.maxProviderLongEdge) {
|
|
59
|
+
throw new Error("Materialized image exceeds the provider image size policy.");
|
|
60
|
+
}
|
|
61
|
+
return Object.freeze({
|
|
62
|
+
bytes: outputBytes,
|
|
63
|
+
mimeType,
|
|
64
|
+
width,
|
|
65
|
+
height,
|
|
66
|
+
planningTokens: imagePlanningTokens(width, height),
|
|
67
|
+
});
|
|
68
|
+
}
|
|
69
|
+
|
|
70
|
+
export function orientedDimensions(
|
|
71
|
+
width: number,
|
|
72
|
+
height: number,
|
|
73
|
+
orientation: number | undefined,
|
|
74
|
+
): { readonly width: number; readonly height: number } {
|
|
75
|
+
return orientation !== undefined && orientation >= 5 && orientation <= 8
|
|
76
|
+
? Object.freeze({ width: height, height: width })
|
|
77
|
+
: Object.freeze({ width, height });
|
|
78
|
+
}
|
|
79
|
+
|
|
80
|
+
async function encodeInOriginalFormat(
|
|
81
|
+
pipeline: ReturnType<typeof sharp>,
|
|
82
|
+
mimeType: ImageMimeType,
|
|
83
|
+
): Promise<Buffer> {
|
|
84
|
+
switch (mimeType) {
|
|
85
|
+
case "image/png":
|
|
86
|
+
return pipeline.png({ compressionLevel: 6, adaptiveFiltering: false }).toBuffer();
|
|
87
|
+
case "image/jpeg":
|
|
88
|
+
return pipeline.jpeg({ quality: 80, chromaSubsampling: "4:2:0" }).toBuffer();
|
|
89
|
+
case "image/webp":
|
|
90
|
+
return pipeline.webp({ quality: 80, effort: 4 }).toBuffer();
|
|
91
|
+
}
|
|
92
|
+
}
|
|
93
|
+
|
|
94
|
+
function requireDimension(value: number | undefined, name: string): number {
|
|
95
|
+
if (!Number.isSafeInteger(value) || value === undefined || value < 1) {
|
|
96
|
+
throw new Error(`Decoded image ${name} is invalid.`);
|
|
97
|
+
}
|
|
98
|
+
return value;
|
|
99
|
+
}
|