tinker-agent 1.10.1 → 2.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -100,6 +100,9 @@ export type PendingLedgerTurn = {
100
100
  };
101
101
 
102
102
  export type AgentTurnLedger = {
103
+ appendSteeringUserMessages(
104
+ messages: readonly UserMessage[],
105
+ ): readonly CanonicalMessageRecord[];
103
106
  appendAssistant(input: {
104
107
  iteration: IterationIdentity;
105
108
  message: AssistantMessage;
@@ -129,6 +132,13 @@ export type LedgerMutation =
129
132
  admissionBase?: AdmissionBaseToken;
130
133
  next: ProtocolContextView;
131
134
  }
135
+ | {
136
+ kind: "append_steering_users";
137
+ turn: TurnIdentity;
138
+ frames: readonly ProtocolFrame[];
139
+ messages: readonly CanonicalMessageRecord[];
140
+ next: ProtocolContextView;
141
+ }
132
142
  | {
133
143
  kind: "append_assistant";
134
144
  iteration: IterationIdentity;
@@ -396,6 +406,74 @@ export class InMemorySessionLedger implements SessionLedger {
396
406
  this.pending = undefined;
397
407
  }
398
408
 
409
+ appendSteeringUserMessages(
410
+ pending: InMemoryPendingLedgerTurn,
411
+ userMessages: readonly UserMessage[],
412
+ ): readonly CanonicalMessageRecord[] {
413
+ this.requirePending(pending, "append steering user messages");
414
+ this.assertNoOpenFrame();
415
+ if (userMessages.length === 0) {
416
+ return Object.freeze([]);
417
+ }
418
+ for (const userMessage of userMessages) {
419
+ validateUserMessage(userMessage);
420
+ }
421
+
422
+ const frames: ProtocolFrame[] = [];
423
+ const messages: CanonicalMessageRecord[] = [];
424
+ for (const userMessage of userMessages) {
425
+ const createdAt = this.clock();
426
+ const ordinal = this.view.messages.length + messages.length + 1;
427
+ const frameId = this.input.idFactory.createProtocolFrameId();
428
+ messages.push(
429
+ immutableRecord<CanonicalMessageRecord>({
430
+ messageId: this.input.idFactory.createMessageId(),
431
+ sessionId: this.input.sessionId,
432
+ frameId,
433
+ ordinal,
434
+ contentSha256: userMessageHash(userMessage),
435
+ createdAt,
436
+ role: "user",
437
+ turnId: pending.turn.turnId,
438
+ content: userMessage.content,
439
+ ...(userMessage.attachments === undefined
440
+ ? {}
441
+ : {
442
+ attachments: Object.freeze(
443
+ immutableCanonicalClone(userMessage.attachments),
444
+ ),
445
+ }),
446
+ origin: "user",
447
+ }),
448
+ );
449
+ frames.push(
450
+ immutableRecord<ProtocolFrame>({
451
+ frameId,
452
+ sessionId: this.input.sessionId,
453
+ turnId: pending.turn.turnId,
454
+ kind: "user",
455
+ state: "closed",
456
+ firstOrdinal: ordinal,
457
+ lastOrdinal: ordinal,
458
+ createdAt,
459
+ closedAt: createdAt,
460
+ }),
461
+ );
462
+ }
463
+ const immutableFrames = Object.freeze(frames);
464
+ const immutableMessages = Object.freeze(messages);
465
+ const next = appendView(this.view, immutableFrames, immutableMessages, []);
466
+ this.validator.validate(next, { fullIntegrity: true });
467
+ this.commit({
468
+ kind: "append_steering_users",
469
+ turn: pending.turn,
470
+ frames: immutableFrames,
471
+ messages: immutableMessages,
472
+ next,
473
+ });
474
+ return immutableMessages;
475
+ }
476
+
399
477
  appendAssistant(
400
478
  pending: InMemoryPendingLedgerTurn,
401
479
  input: {
@@ -776,6 +854,8 @@ class InMemoryPendingLedgerTurn implements PendingLedgerTurn {
776
854
  readonly turn: TurnIdentity,
777
855
  ) {
778
856
  this.agent = {
857
+ appendSteeringUserMessages: (messages) =>
858
+ this.ledger.appendSteeringUserMessages(this, messages),
779
859
  appendAssistant: (input) => this.ledger.appendAssistant(this, input),
780
860
  assertCanExecuteTool: (call) => this.ledger.assertCanExecuteTool(this, call),
781
861
  commitToolCompletions: (completions) =>
package/src/cli/config.ts CHANGED
@@ -14,7 +14,6 @@ import {
14
14
  type ModelInputModality,
15
15
  type ModelProfile,
16
16
  type ModelProfiles,
17
- type ModelTokenEstimatorProfile,
18
17
  unknownProfileError,
19
18
  } from "./model-profiles";
20
19
  import type { MemoryEmbeddingConfig } from "../memory/contracts";
@@ -40,7 +39,6 @@ export type RunnerConfig = {
40
39
  readonly contextBudget: ModelContextBudget;
41
40
  readonly profileName?: string;
42
41
  readonly inputModalities: readonly ModelInputModality[];
43
- readonly tokenEstimator?: ModelTokenEstimatorProfile;
44
42
  readonly bashGuardMode: "guard" | "yolo";
45
43
  readonly bashGuardSource: "default" | "environment" | "cli";
46
44
  };
@@ -184,9 +182,6 @@ function runnerConfigTemplateFromProfile(
184
182
  inputModalities: profile.inputModalities,
185
183
  bashGuardMode: environment.bashGuardMode,
186
184
  bashGuardSource: environment.bashGuardSource,
187
- ...(profile.tokenEstimator === undefined
188
- ? {}
189
- : { tokenEstimator: profile.tokenEstimator }),
190
185
  });
191
186
  }
192
187
 
@@ -10,9 +10,6 @@ import {
10
10
  MODEL_PROFILE_FIELDS,
11
11
  MODEL_PROFILES_DOCUMENT_FIELDS,
12
12
  MODEL_REASONING_FIELDS,
13
- MODEL_TOKEN_ESTIMATOR_FIELDS,
14
- type ModelTokenEstimatorKind,
15
- type ModelTokenEstimatorMaxRetries,
16
13
  } from "./public-config-contract";
17
14
  import type { MemoryEmbeddingConfig } from "../memory/contracts";
18
15
  import type { ReasoningEffortConfig } from "../model/reasoning-effort";
@@ -29,20 +26,10 @@ export type ModelProfile = {
29
26
  readonly includeReasoningContent: boolean;
30
27
  readonly stream: boolean;
31
28
  readonly inputModalities: readonly ModelInputModality[];
32
- readonly tokenEstimator?: ModelTokenEstimatorProfile;
33
29
  };
34
30
 
35
31
  export type ModelInputModality = "text" | "image";
36
32
 
37
- export type ModelTokenEstimatorProfile = {
38
- readonly kind: ModelTokenEstimatorKind;
39
- readonly model: string;
40
- readonly apiBase: string;
41
- readonly apiKey: string;
42
- readonly timeoutMs: number;
43
- readonly maxRetries: ModelTokenEstimatorMaxRetries;
44
- };
45
-
46
33
  export type ModelProfiles = {
47
34
  readonly defaultProfile: string;
48
35
  readonly profiles: ReadonlyMap<string, ModelProfile>;
@@ -285,16 +272,6 @@ function parseProfile(
285
272
  value.inputModalities,
286
273
  `${where}: "inputModalities"`,
287
274
  );
288
- const tokenEstimator =
289
- value.tokenEstimator === undefined
290
- ? undefined
291
- : parseTokenEstimator(value.tokenEstimator, `${where}: "tokenEstimator"`);
292
- if (inputModalities.includes("image") && tokenEstimator === undefined) {
293
- throw new Error(
294
- `${where}: image input requires a complete "tokenEstimator" configuration.`,
295
- );
296
- }
297
-
298
275
  createModelContextProfile({
299
276
  contextWindowTokens,
300
277
  maxSupportedOutputTokens,
@@ -312,7 +289,6 @@ function parseProfile(
312
289
  includeReasoningContent,
313
290
  stream,
314
291
  inputModalities,
315
- ...(tokenEstimator === undefined ? {} : { tokenEstimator }),
316
292
  });
317
293
  }
318
294
 
@@ -434,54 +410,7 @@ function parseInputModalities(
434
410
  );
435
411
  }
436
412
 
437
- function parseTokenEstimator(value: unknown, name: string): ModelTokenEstimatorProfile {
438
- if (!isRecord(value)) {
439
- throw new Error(`${name} must be an object.`);
440
- }
441
- assertKnownKeys(
442
- value,
443
- MODEL_TOKEN_ESTIMATOR_FIELDS.map((field) => field.name),
444
- name,
445
- );
446
- const kindField = tokenEstimatorField("kind");
447
- if (value.kind !== kindField.literalValue) {
448
- throw new Error(`${name}.kind must be ${JSON.stringify(kindField.literalValue)}.`);
449
- }
450
- const model = parseTokenEstimatorString(value, "model", name);
451
- const apiBase = parseTokenEstimatorString(value, "apiBase", name);
452
- const apiKey = parseTokenEstimatorString(value, "apiKey", name);
453
- const timeoutField = tokenEstimatorField("timeoutMs");
454
- if (timeoutField.valueKind !== "positive-integer") {
455
- throw new Error("Token estimator timeoutMs contract kind is invalid.");
456
- }
457
- const timeoutMs = requirePositiveInteger(value.timeoutMs, `${name}.timeoutMs`);
458
- if (
459
- timeoutField.minimum === undefined ||
460
- timeoutField.maximum === undefined ||
461
- timeoutMs < timeoutField.minimum ||
462
- timeoutMs > timeoutField.maximum
463
- ) {
464
- throw new Error(
465
- `${name}.timeoutMs must be between ${timeoutField.minimum} and ${timeoutField.maximum}.`,
466
- );
467
- }
468
- const maxRetriesField = tokenEstimatorField("maxRetries");
469
- if (value.maxRetries !== maxRetriesField.literalValue) {
470
- throw new Error(`${name}.maxRetries must be ${maxRetriesField.literalValue}.`);
471
- }
472
- return Object.freeze({
473
- kind: kindField.literalValue,
474
- model,
475
- apiBase,
476
- apiKey,
477
- timeoutMs,
478
- maxRetries: maxRetriesField.literalValue,
479
- });
480
- }
481
-
482
413
  type ModelProfileFieldName = (typeof MODEL_PROFILE_FIELDS)[number]["name"];
483
- type ModelTokenEstimatorFieldName =
484
- (typeof MODEL_TOKEN_ESTIMATOR_FIELDS)[number]["name"];
485
414
 
486
415
  function modelProfileField<Name extends ModelProfileFieldName>(
487
416
  name: Name,
@@ -496,21 +425,6 @@ function modelProfileField<Name extends ModelProfileFieldName>(
496
425
  >;
497
426
  }
498
427
 
499
- function tokenEstimatorField<Name extends ModelTokenEstimatorFieldName>(
500
- name: Name,
501
- ): Extract<(typeof MODEL_TOKEN_ESTIMATOR_FIELDS)[number], { readonly name: Name }> {
502
- const field = MODEL_TOKEN_ESTIMATOR_FIELDS.find(
503
- (candidate) => candidate.name === name,
504
- );
505
- if (field === undefined) {
506
- throw new Error(`Missing token estimator field contract for ${name}.`);
507
- }
508
- return field as Extract<
509
- (typeof MODEL_TOKEN_ESTIMATOR_FIELDS)[number],
510
- { readonly name: Name }
511
- >;
512
- }
513
-
514
428
  function parseProfileString(
515
429
  value: Record<string, unknown>,
516
430
  name: "model" | "apiBase" | "apiKey",
@@ -555,18 +469,6 @@ function parseProfileBoolean(
555
469
  : parseBoolean(value[name], `${where}: ${JSON.stringify(name)}`);
556
470
  }
557
471
 
558
- function parseTokenEstimatorString(
559
- value: Record<string, unknown>,
560
- name: "model" | "apiBase" | "apiKey",
561
- where: string,
562
- ): string {
563
- const field = tokenEstimatorField(name);
564
- if (field.valueKind !== "non-empty-string") {
565
- throw new Error(`Token estimator field ${name} has an invalid contract kind.`);
566
- }
567
- return requireString(value[name], `${where}.${name}`);
568
- }
569
-
570
472
  function assertKnownKeys(
571
473
  value: Record<string, unknown>,
572
474
  allowed: readonly string[],
@@ -242,11 +242,7 @@ export const PUBLIC_CONFIG_FIELDS = Object.freeze([
242
242
 
243
243
  export type ModelProfileField = {
244
244
  readonly name: string;
245
- readonly valueKind:
246
- | PublicConfigValueKind
247
- | "input-modalities"
248
- | "reasoning"
249
- | "token-estimator";
245
+ readonly valueKind: PublicConfigValueKind | "input-modalities" | "reasoning";
250
246
  readonly required: boolean;
251
247
  readonly defaultValue?: string | number | boolean | readonly string[];
252
248
  readonly secret: boolean;
@@ -337,13 +333,6 @@ export const MODEL_PROFILE_FIELDS = Object.freeze([
337
333
  description:
338
334
  'Accepted model input modalities; normalizes to ["text"] or ["text", "image"].',
339
335
  }),
340
- profileField({
341
- name: "tokenEstimator",
342
- valueKind: "token-estimator",
343
- required: false,
344
- secret: true,
345
- description: "Independent token estimator required for image profiles.",
346
- }),
347
336
  ]);
348
337
 
349
338
  export type ModelReasoningField = {
@@ -375,82 +364,6 @@ export const MODEL_REASONING_FIELDS = Object.freeze([
375
364
  }),
376
365
  ]);
377
366
 
378
- export type ModelTokenEstimatorField = {
379
- readonly name: string;
380
- readonly valueKind: PublicConfigValueKind | "literal-string" | "literal-number";
381
- readonly required: true;
382
- readonly secret: boolean;
383
- readonly literalValue?: string | number;
384
- readonly minimum?: number;
385
- readonly maximum?: number;
386
- readonly description: string;
387
- };
388
-
389
- function estimatorField<const T extends ModelTokenEstimatorField>(
390
- field: T,
391
- ): Readonly<T> {
392
- return Object.freeze(field);
393
- }
394
-
395
- export const MODEL_TOKEN_ESTIMATOR_FIELDS = Object.freeze([
396
- estimatorField({
397
- name: "kind",
398
- valueKind: "literal-string",
399
- required: true,
400
- secret: false,
401
- literalValue: "moonshot-estimate-token-count-v1",
402
- description: "Estimator protocol discriminator.",
403
- }),
404
- estimatorField({
405
- name: "model",
406
- valueKind: "non-empty-string",
407
- required: true,
408
- secret: false,
409
- description: "Estimator model name.",
410
- }),
411
- estimatorField({
412
- name: "apiBase",
413
- valueKind: "non-empty-string",
414
- required: true,
415
- secret: false,
416
- description: "Estimator API base URL.",
417
- }),
418
- estimatorField({
419
- name: "apiKey",
420
- valueKind: "non-empty-string",
421
- required: true,
422
- secret: true,
423
- description: "Estimator API credential.",
424
- }),
425
- estimatorField({
426
- name: "timeoutMs",
427
- valueKind: "positive-integer",
428
- required: true,
429
- secret: false,
430
- minimum: 1_000,
431
- maximum: 60_000,
432
- description: "Estimator request timeout in milliseconds.",
433
- }),
434
- estimatorField({
435
- name: "maxRetries",
436
- valueKind: "literal-number",
437
- required: true,
438
- secret: false,
439
- literalValue: 0,
440
- description: "Estimator retry count; retries are disabled.",
441
- }),
442
- ]);
443
-
444
- export type ModelTokenEstimatorKind = Extract<
445
- (typeof MODEL_TOKEN_ESTIMATOR_FIELDS)[number],
446
- { readonly name: "kind" }
447
- >["literalValue"];
448
-
449
- export type ModelTokenEstimatorMaxRetries = Extract<
450
- (typeof MODEL_TOKEN_ESTIMATOR_FIELDS)[number],
451
- { readonly name: "maxRetries" }
452
- >["literalValue"];
453
-
454
367
  export type MemoryConfigField = {
455
368
  readonly name: "profile" | "embedding";
456
369
  readonly valueKind: "non-empty-string" | "embedding-profile";
@@ -66,7 +66,6 @@ export function createModelClient(
66
66
  | "apiKey"
67
67
  | "apiBase"
68
68
  | "inputModalities"
69
- | "tokenEstimator"
70
69
  >,
71
70
  env: NodeJS.ProcessEnv = process.env,
72
71
  reasoningEffort?: ReasoningEffortController,
@@ -82,9 +81,6 @@ export function createModelClient(
82
81
  ...(activeReasoningEffort === undefined
83
82
  ? {}
84
83
  : { reasoningEffort: activeReasoningEffort }),
85
- ...(config.tokenEstimator === undefined
86
- ? {}
87
- : { tokenEstimator: config.tokenEstimator }),
88
84
  ...(env.TINKER_TEST_FAKE_MODEL_REQUEST_LOG === undefined ||
89
85
  env.TINKER_TEST_FAKE_MODEL_REQUEST_LOG === ""
90
86
  ? {}
@@ -102,9 +98,6 @@ export function createModelClient(
102
98
  ...(activeReasoningEffort === undefined
103
99
  ? {}
104
100
  : { reasoningEffort: activeReasoningEffort }),
105
- ...(config.tokenEstimator === undefined
106
- ? {}
107
- : { tokenEstimator: config.tokenEstimator }),
108
101
  };
109
102
  if (config.api === "responses") {
110
103
  return new OpenAIResponsesModelClient(common);
@@ -47,9 +47,6 @@ export async function initializeTuiMemory(input: {
47
47
  stream: profile.stream,
48
48
  contextBudget: memoryConfig.contextBudget,
49
49
  inputModalities: profile.inputModalities,
50
- ...(profile.tokenEstimator === undefined
51
- ? {}
52
- : { tokenEstimator: profile.tokenEstimator }),
53
50
  },
54
51
  input.env,
55
52
  );
@@ -35,6 +35,10 @@ export function renderObservationLogEvent(event: AgentEvent): string | undefined
35
35
  ].join("\n");
36
36
  case "turn.started":
37
37
  return renderTurnStarted(event);
38
+ case "turn.steering.applied":
39
+ return ["## User follow-up", "", event.data.userPrompt.text, "", "---", ""].join(
40
+ "\n",
41
+ );
38
42
  case "assistant.progress":
39
43
  return renderAssistantProgress(event);
40
44
  case "tool.observation":
@@ -37,6 +37,11 @@ export class StdoutEventPrinter implements EventSink {
37
37
  `turn.started turn=${event.turnNumber} turnId=${event.turnId}\n`,
38
38
  );
39
39
  break;
40
+ case "turn.steering.applied":
41
+ this.stdout.write(
42
+ `turn.steering.applied turn=${event.turnNumber} ordinal=${event.data.ordinal}\n`,
43
+ );
44
+ break;
40
45
  case "agent.iteration.started":
41
46
  this.stdout.write(
42
47
  `agent.iteration.started iteration=${event.iterationNumber} iterationId=${event.iterationId}\n`,
@@ -307,6 +307,10 @@ export type AgentEventDataMap = {
307
307
  "session.interrupted_frame_recovered": InterruptedFrameRecoveredData;
308
308
  "session.finished": SessionFinishedData;
309
309
  "turn.started": { userPrompt: UserPromptProjection };
310
+ "turn.steering.applied": {
311
+ userPrompt: UserPromptProjection;
312
+ ordinal: number;
313
+ };
310
314
  "turn.finished": TurnFinishedData;
311
315
  "turn.failed": { error: string };
312
316
  "turn.cancelled": { cancellation: TurnCancellation };
@@ -411,7 +415,7 @@ export type AgentEventInput =
411
415
  >
412
416
  | SessionEventInput<"mcp.server.connected" | "mcp.server.failed">
413
417
  | SessionEventInput<"diagnostic.sink_failed">
414
- | TurnEventInput<"turn.started" | "turn.finished">
418
+ | TurnEventInput<"turn.started" | "turn.steering.applied" | "turn.finished">
415
419
  | (
416
420
  | TurnEventInput<"turn.failed" | "turn.cancelled">
417
421
  | IterationEventInput<"turn.failed" | "turn.cancelled">
@@ -4,11 +4,62 @@ export const IMAGE_INPUT_POLICY = Object.freeze({
4
4
  maxBytesPerImage: 20 * 1024 * 1024,
5
5
  maxImagesPerMessage: 8,
6
6
  maxImagesPerRequest: 8,
7
+ maxProviderLongEdge: 2048,
7
8
  maxLongEdge: 4096,
8
9
  maxPixels: 8_847_360,
9
10
  maxRequestBodyBytes: 90_000_000,
10
- planningTokensPerImage: 2048,
11
+ allowUpscale: false,
12
+ resizePolicy: "sharp-auto-orient-lanczos3-round-v1",
13
+ outputEncodingPolicy: "preserve-input-format-fixed-encoding-v1",
14
+ imageTokenBuckets: Object.freeze([
15
+ Object.freeze({ maxLongEdge: 512, planningTokens: 384 }),
16
+ Object.freeze({ maxLongEdge: 1024, planningTokens: 1408 }),
17
+ Object.freeze({ maxLongEdge: 1536, planningTokens: 3072 }),
18
+ Object.freeze({ maxLongEdge: 2048, planningTokens: 5504 }),
19
+ ] as const),
20
+ imageTokensUseTextCorrectionFactor: false,
11
21
  retryPolicy: "none",
12
22
  } as const);
13
23
 
14
- export const IMAGE_INPUT_POLICY_VERSION = "image-input-policy-v1" as const;
24
+ export const IMAGE_INPUT_POLICY_VERSION = "image-input-policy-v2" as const;
25
+
26
+ export function providerImageDimensions(
27
+ width: number,
28
+ height: number,
29
+ ): { readonly width: number; readonly height: number } {
30
+ requireDimension(width, "width");
31
+ requireDimension(height, "height");
32
+ const longEdge = Math.max(width, height);
33
+ if (longEdge <= IMAGE_INPUT_POLICY.maxProviderLongEdge) {
34
+ return Object.freeze({ width, height });
35
+ }
36
+ const scale = IMAGE_INPUT_POLICY.maxProviderLongEdge / longEdge;
37
+ let targetWidth = Math.max(1, Math.round(width * scale));
38
+ let targetHeight = Math.max(1, Math.round(height * scale));
39
+ if (targetWidth > IMAGE_INPUT_POLICY.maxProviderLongEdge) {
40
+ targetWidth = IMAGE_INPUT_POLICY.maxProviderLongEdge;
41
+ }
42
+ if (targetHeight > IMAGE_INPUT_POLICY.maxProviderLongEdge) {
43
+ targetHeight = IMAGE_INPUT_POLICY.maxProviderLongEdge;
44
+ }
45
+ return Object.freeze({ width: targetWidth, height: targetHeight });
46
+ }
47
+
48
+ export function imagePlanningTokens(width: number, height: number): number {
49
+ requireDimension(width, "width");
50
+ requireDimension(height, "height");
51
+ const longEdge = Math.max(width, height);
52
+ const bucket = IMAGE_INPUT_POLICY.imageTokenBuckets.find(
53
+ (candidate) => longEdge <= candidate.maxLongEdge,
54
+ );
55
+ if (bucket === undefined) {
56
+ throw new Error("Materialized image exceeds the provider image size policy.");
57
+ }
58
+ return bucket.planningTokens;
59
+ }
60
+
61
+ function requireDimension(value: number, name: string): void {
62
+ if (!Number.isSafeInteger(value) || value < 1) {
63
+ throw new Error(`Image ${name} must be a positive safe integer.`);
64
+ }
65
+ }
@@ -1,5 +1,6 @@
1
1
  import sharp from "sharp";
2
2
  import { IMAGE_INPUT_POLICY } from "./image-input-policy";
3
+ import { orientedDimensions } from "./provider-image";
3
4
  import {
4
5
  imageAssetIdForBytes,
5
6
  type ImageAssetRef,
@@ -59,8 +60,13 @@ export async function probeImageBytes(
59
60
  `Image container and decoder disagree on format (${container.mimeType} vs ${mimeType}).`,
60
61
  );
61
62
  }
62
- const width = requireDimension(metadata.width, "width");
63
- const height = requireDimension(metadata.height, "height");
63
+ const decodedWidth = requireDimension(metadata.width, "width");
64
+ const decodedHeight = requireDimension(metadata.height, "height");
65
+ const { width, height } = orientedDimensions(
66
+ decodedWidth,
67
+ decodedHeight,
68
+ metadata.orientation,
69
+ );
64
70
  const decoderAnimated =
65
71
  (metadata.pages ?? 1) > 1 || metadata.pageHeight !== undefined;
66
72
  if (container.animated || decoderAnimated) {
@@ -0,0 +1,99 @@
1
+ import sharp from "sharp";
2
+ import {
3
+ IMAGE_INPUT_POLICY,
4
+ imagePlanningTokens,
5
+ providerImageDimensions,
6
+ } from "./image-input-policy";
7
+ import type { ImageMimeType } from "./image-types";
8
+
9
+ export type ProviderImage = {
10
+ readonly bytes: Buffer;
11
+ readonly mimeType: ImageMimeType;
12
+ readonly width: number;
13
+ readonly height: number;
14
+ readonly planningTokens: number;
15
+ };
16
+
17
+ export async function materializeProviderImage(
18
+ bytes: Buffer,
19
+ mimeType: ImageMimeType,
20
+ ): Promise<ProviderImage> {
21
+ const input = sharp(bytes, {
22
+ failOn: "warning",
23
+ limitInputPixels: IMAGE_INPUT_POLICY.maxPixels,
24
+ unlimited: false,
25
+ sequentialRead: true,
26
+ });
27
+ const metadata = await input.metadata();
28
+ const sourceWidth = requireDimension(metadata.width, "width");
29
+ const sourceHeight = requireDimension(metadata.height, "height");
30
+ const oriented = orientedDimensions(sourceWidth, sourceHeight, metadata.orientation);
31
+ const target = providerImageDimensions(oriented.width, oriented.height);
32
+ const requiresOrientation =
33
+ metadata.orientation !== undefined && metadata.orientation !== 1;
34
+ const requiresResize =
35
+ target.width !== oriented.width || target.height !== oriented.height;
36
+
37
+ let outputBytes = bytes;
38
+ if (requiresOrientation || requiresResize) {
39
+ let pipeline = sharp(bytes, {
40
+ failOn: "warning",
41
+ limitInputPixels: IMAGE_INPUT_POLICY.maxPixels,
42
+ unlimited: false,
43
+ sequentialRead: true,
44
+ }).rotate();
45
+ if (requiresResize) {
46
+ pipeline = pipeline.resize(target.width, target.height, {
47
+ fit: "fill",
48
+ kernel: sharp.kernel.lanczos3,
49
+ withoutEnlargement: true,
50
+ });
51
+ }
52
+ outputBytes = await encodeInOriginalFormat(pipeline, mimeType);
53
+ }
54
+
55
+ const outputMetadata = await sharp(outputBytes).metadata();
56
+ const width = requireDimension(outputMetadata.width, "width");
57
+ const height = requireDimension(outputMetadata.height, "height");
58
+ if (Math.max(width, height) > IMAGE_INPUT_POLICY.maxProviderLongEdge) {
59
+ throw new Error("Materialized image exceeds the provider image size policy.");
60
+ }
61
+ return Object.freeze({
62
+ bytes: outputBytes,
63
+ mimeType,
64
+ width,
65
+ height,
66
+ planningTokens: imagePlanningTokens(width, height),
67
+ });
68
+ }
69
+
70
+ export function orientedDimensions(
71
+ width: number,
72
+ height: number,
73
+ orientation: number | undefined,
74
+ ): { readonly width: number; readonly height: number } {
75
+ return orientation !== undefined && orientation >= 5 && orientation <= 8
76
+ ? Object.freeze({ width: height, height: width })
77
+ : Object.freeze({ width, height });
78
+ }
79
+
80
+ async function encodeInOriginalFormat(
81
+ pipeline: ReturnType<typeof sharp>,
82
+ mimeType: ImageMimeType,
83
+ ): Promise<Buffer> {
84
+ switch (mimeType) {
85
+ case "image/png":
86
+ return pipeline.png({ compressionLevel: 6, adaptiveFiltering: false }).toBuffer();
87
+ case "image/jpeg":
88
+ return pipeline.jpeg({ quality: 80, chromaSubsampling: "4:2:0" }).toBuffer();
89
+ case "image/webp":
90
+ return pipeline.webp({ quality: 80, effort: 4 }).toBuffer();
91
+ }
92
+ }
93
+
94
+ function requireDimension(value: number | undefined, name: string): number {
95
+ if (!Number.isSafeInteger(value) || value === undefined || value < 1) {
96
+ throw new Error(`Decoded image ${name} is invalid.`);
97
+ }
98
+ return value;
99
+ }