pi-openai-toolkit 0.17.3 → 0.19.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +5 -5
- package/README.zh.md +5 -5
- package/config.schema.json +2 -1
- package/docs/configuration.md +7 -7
- package/package.json +5 -5
- package/src/context-management/codex-provider.ts +18 -8
- package/src/extension-runtime.ts +47 -26
- package/src/image-generation/artifacts.ts +28 -7
- package/src/image-generation/client.ts +72 -8
- package/src/image-generation/eligibility.ts +13 -8
- package/src/image-generation/extension.ts +22 -6
- package/src/image-generation/protocol.ts +228 -128
- package/src/image-generation/render.ts +16 -8
- package/src/image-generation/service.ts +384 -255
- package/src/image-generation/types.ts +117 -6
- package/src/payload-rewrite.ts +16 -1
- package/src/responses-headers.ts +13 -3
- package/src/runtime.ts +295 -28
- package/src/serializer.ts +214 -0
- package/src/supported-environment.ts +5 -0
- package/src/types.ts +9 -5
- package/src/web-search/extension.ts +4 -3
- package/src/web-search/types.ts +25 -3
|
@@ -1,18 +1,23 @@
|
|
|
1
|
+
import { VIRTUAL_MODEL_API } from "../runtime";
|
|
1
2
|
import type { ImageGenerationConfig } from "../types";
|
|
2
3
|
import { IMAGE_GENERATION_CAPABLE_APIS, type ImageGenerationModel } from "./types";
|
|
3
4
|
|
|
4
5
|
/**
|
|
5
|
-
* The
|
|
6
|
-
* specific model
|
|
7
|
-
*
|
|
6
|
+
* The independent Images API is exposed only for Responses-capable runtimes rather than for a
|
|
7
|
+
* specific conversation model. `enabled` is the single opt-in switch and defaults to off because
|
|
8
|
+
* every successful provider request may incur a charge.
|
|
8
9
|
*/
|
|
9
10
|
export function isImageGenerationEnabledForModel(
|
|
10
11
|
model: ImageGenerationModel | undefined,
|
|
11
12
|
config: ImageGenerationConfig,
|
|
13
|
+
effectiveModel: ImageGenerationModel | undefined = model,
|
|
12
14
|
): boolean {
|
|
13
|
-
return
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
15
|
+
if (!config.enabled) return false;
|
|
16
|
+
// A virtual model is only a candidate until execution resolves its physical
|
|
17
|
+
// model. The service performs that resolution and fails closed if it is not
|
|
18
|
+
// available; this keeps the tool discoverable instead of silently hiding a
|
|
19
|
+
// valid virtual route at startup.
|
|
20
|
+
if (model?.api === VIRTUAL_MODEL_API && effectiveModel === model) return true;
|
|
21
|
+
return typeof effectiveModel?.api === "string" &&
|
|
22
|
+
(IMAGE_GENERATION_CAPABLE_APIS as readonly string[]).includes(effectiveModel.api);
|
|
18
23
|
}
|
|
@@ -2,6 +2,7 @@ import { StringEnum, Type, type Static } from "@earendil-works/pi-ai";
|
|
|
2
2
|
import type { ExtensionAPI } from "@earendil-works/pi-coding-agent";
|
|
3
3
|
import { loadToolkitConfig, resolveToolkitConfig } from "../config";
|
|
4
4
|
import { notifyConfigIssues } from "../config/notifications";
|
|
5
|
+
import { registerPhysicalModelObserver } from "../runtime";
|
|
5
6
|
import { executeImageGeneration } from "./service";
|
|
6
7
|
import { isImageGenerationEnabledForModel } from "./eligibility";
|
|
7
8
|
import { renderImageGenerationResult } from "./render";
|
|
@@ -9,13 +10,14 @@ import {
|
|
|
9
10
|
IMAGE_GENERATION_QUALITIES,
|
|
10
11
|
IMAGE_GENERATION_SIZES,
|
|
11
12
|
IMAGE_GENERATION_TOOL_NAME,
|
|
13
|
+
MAX_IMAGE_BATCH_SIZE,
|
|
12
14
|
MAX_IMAGE_MODEL_ID_CHARS,
|
|
13
15
|
MAX_IMAGE_PATH_CHARS,
|
|
14
16
|
MAX_IMAGE_PROMPT_CHARS,
|
|
15
17
|
MAX_REFERENCE_IMAGE_COUNT,
|
|
16
18
|
ImageGenerationError,
|
|
17
19
|
sanitizeImageDiagnostic,
|
|
18
|
-
type
|
|
20
|
+
type ImageGenerationToolDetails,
|
|
19
21
|
} from "./types";
|
|
20
22
|
|
|
21
23
|
const registeredApis = new WeakSet<object>();
|
|
@@ -53,13 +55,13 @@ const GenerateImageParameters = Type.Object(
|
|
|
53
55
|
],
|
|
54
56
|
{
|
|
55
57
|
description:
|
|
56
|
-
"Use null when the user did not explicitly request a destination; otherwise provide
|
|
58
|
+
"Use null when the user did not explicitly request a destination; otherwise provide an explicit .png, .jpg, .jpeg, or .webp file path matching the provider output format.",
|
|
57
59
|
},
|
|
58
60
|
),
|
|
59
61
|
),
|
|
60
62
|
size: Type.Optional(
|
|
61
63
|
StringEnum(IMAGE_GENERATION_SIZES, {
|
|
62
|
-
description: "Requested
|
|
64
|
+
description: "Requested raster-image dimensions, or auto.",
|
|
63
65
|
}),
|
|
64
66
|
),
|
|
65
67
|
quality: Type.Optional(
|
|
@@ -82,6 +84,18 @@ const GenerateImageParameters = Type.Object(
|
|
|
82
84
|
},
|
|
83
85
|
),
|
|
84
86
|
),
|
|
87
|
+
batchSize: Type.Optional(
|
|
88
|
+
Type.Union(
|
|
89
|
+
[
|
|
90
|
+
Type.Null(),
|
|
91
|
+
Type.Integer({ minimum: 1, maximum: MAX_IMAGE_BATCH_SIZE }),
|
|
92
|
+
],
|
|
93
|
+
{
|
|
94
|
+
description:
|
|
95
|
+
"Run this many independent image requests concurrently (1-10). Batch requests use generated artifacts and cannot set outputPath.",
|
|
96
|
+
},
|
|
97
|
+
),
|
|
98
|
+
),
|
|
85
99
|
},
|
|
86
100
|
{ additionalProperties: false },
|
|
87
101
|
);
|
|
@@ -110,13 +124,14 @@ export function registerImageGenerationExtension(
|
|
|
110
124
|
): void {
|
|
111
125
|
if (registeredApis.has(pi)) return;
|
|
112
126
|
registeredApis.add(pi);
|
|
127
|
+
registerPhysicalModelObserver(pi);
|
|
113
128
|
|
|
114
|
-
pi.registerTool<typeof GenerateImageParameters,
|
|
129
|
+
pi.registerTool<typeof GenerateImageParameters, ImageGenerationToolDetails, Record<string, never>>({
|
|
115
130
|
name: IMAGE_GENERATION_TOOL_NAME,
|
|
116
131
|
label: "OpenAI Generate Image",
|
|
117
132
|
description:
|
|
118
|
-
"Generate
|
|
119
|
-
promptSnippet: "Generate or edit
|
|
133
|
+
"Generate raster images, or edit from one to five user-approved local reference images, through an independent Images API request using a configured image model. PNG, JPEG, and WebP responses are validated and saved in their actual format. This is a paid provider operation; batchSize runs independent requests concurrently.",
|
|
134
|
+
promptSnippet: "Generate or edit raster images through an independent Images API request and a configured imageGeneration.allowedModels entry.",
|
|
120
135
|
promptGuidelines: [
|
|
121
136
|
"Use openai_generate_image when the user explicitly asks to create, draw, render, or edit a raster image and the active model speaks a Responses API.",
|
|
122
137
|
"Do not call openai_generate_image speculatively: it consumes the user's provider or gateway image quota.",
|
|
@@ -124,6 +139,7 @@ export function registerImageGenerationExtension(
|
|
|
124
139
|
"Set referenceImagePaths to null unless the user explicitly identified local files; never invent paths or placeholder strings, and remember upload requires user approval.",
|
|
125
140
|
"Set outputPath to null unless the user explicitly asks for a destination; never invent a destination or placeholder string, and otherwise use the default Pi agent artifact.",
|
|
126
141
|
"Set model to null unless the user named an image model configured in imageGeneration.allowedModels; never invent or guess a model id, and remember an unlisted model fails before the paid request.",
|
|
142
|
+
"Use batchSize only when the user asks for multiple variants; each batch item is an independent paid request, and batchSize cannot be combined with outputPath.",
|
|
127
143
|
"Do not substitute Python, browser automation, shell scripts, or unrelated image tools for an eligible image request.",
|
|
128
144
|
],
|
|
129
145
|
parameters: GenerateImageParameters,
|
|
@@ -1,9 +1,10 @@
|
|
|
1
1
|
import { DEFAULT_IMAGE_GENERATION_MODELS } from "../types";
|
|
2
2
|
import {
|
|
3
|
-
|
|
3
|
+
IMAGE_GENERATION_MIME_TYPES,
|
|
4
4
|
IMAGE_GENERATION_QUALITIES,
|
|
5
5
|
IMAGE_GENERATION_SIZES,
|
|
6
6
|
MAX_GENERATED_IMAGE_BYTES,
|
|
7
|
+
MAX_IMAGE_BATCH_SIZE,
|
|
7
8
|
MAX_IMAGE_DIAGNOSTIC_CHARS,
|
|
8
9
|
MAX_IMAGE_DIMENSION,
|
|
9
10
|
MAX_IMAGE_IDENTIFIER_CHARS,
|
|
@@ -14,6 +15,7 @@ import {
|
|
|
14
15
|
ImageGenerationError,
|
|
15
16
|
sanitizeImageDiagnostic,
|
|
16
17
|
type GenerateImageParams,
|
|
18
|
+
type ImageGenerationMimeType,
|
|
17
19
|
type NormalizedGenerateImageParams,
|
|
18
20
|
type ParsedGeneratedImage,
|
|
19
21
|
type PreparedReferenceImage,
|
|
@@ -26,26 +28,13 @@ const BASE64_PATTERN = /^(?:[A-Za-z0-9+/]{4})*(?:[A-Za-z0-9+/]{2}==|[A-Za-z0-9+/
|
|
|
26
28
|
const DATA_URL_PATTERN = /^data:image\/[a-z0-9.+-]+;base64,(.*)$/is;
|
|
27
29
|
|
|
28
30
|
export type ImageGenerationRequestBody = {
|
|
31
|
+
action: NormalizedGenerateImageParams["action"];
|
|
29
32
|
model: string;
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
content: Array<
|
|
36
|
-
| { type: "input_text"; text: string }
|
|
37
|
-
| { type: "input_image"; image_url: string; detail: "auto" }
|
|
38
|
-
>;
|
|
39
|
-
}>;
|
|
40
|
-
tools: Array<{
|
|
41
|
-
type: "image_generation";
|
|
42
|
-
model: string;
|
|
43
|
-
action: "generate" | "edit";
|
|
44
|
-
size: NormalizedGenerateImageParams["size"];
|
|
45
|
-
quality: NormalizedGenerateImageParams["quality"];
|
|
46
|
-
output_format: "png";
|
|
47
|
-
}>;
|
|
48
|
-
tool_choice: { type: "image_generation" };
|
|
33
|
+
prompt: string;
|
|
34
|
+
n: 1;
|
|
35
|
+
size: NormalizedGenerateImageParams["size"];
|
|
36
|
+
quality: NormalizedGenerateImageParams["quality"];
|
|
37
|
+
references: readonly PreparedReferenceImage[];
|
|
49
38
|
};
|
|
50
39
|
|
|
51
40
|
export type ParsedImageResponseResult =
|
|
@@ -137,6 +126,22 @@ function normalizeRequestedModel(value: unknown): string | undefined {
|
|
|
137
126
|
return trimmed;
|
|
138
127
|
}
|
|
139
128
|
|
|
129
|
+
function normalizeBatchSize(value: unknown): number {
|
|
130
|
+
if (value === null || value === undefined) return 1;
|
|
131
|
+
if (
|
|
132
|
+
typeof value !== "number" ||
|
|
133
|
+
!Number.isInteger(value) ||
|
|
134
|
+
value < 1 ||
|
|
135
|
+
value > MAX_IMAGE_BATCH_SIZE
|
|
136
|
+
) {
|
|
137
|
+
throw new ImageGenerationError(
|
|
138
|
+
"invalid-parameters",
|
|
139
|
+
`batchSize must be an integer from 1 to ${MAX_IMAGE_BATCH_SIZE}.`,
|
|
140
|
+
);
|
|
141
|
+
}
|
|
142
|
+
return value;
|
|
143
|
+
}
|
|
144
|
+
|
|
140
145
|
/**
|
|
141
146
|
* Configured model ids are only trusted as far as the shared bounds: unusable entries are
|
|
142
147
|
* dropped, and an empty or malformed list falls back to the shipped default model.
|
|
@@ -153,7 +158,7 @@ function usableImageModels(models: readonly string[] | undefined): string[] {
|
|
|
153
158
|
}
|
|
154
159
|
|
|
155
160
|
/**
|
|
156
|
-
* Resolve the
|
|
161
|
+
* Resolve the independent Images API model for one call. An omitted request selects the
|
|
157
162
|
* explicit default when supplied (v2), or the first configured model for legacy callers.
|
|
158
163
|
* An explicit request must match a configured id exactly before any paid dispatch.
|
|
159
164
|
*/
|
|
@@ -213,6 +218,13 @@ export function normalizeGenerateImageParams(
|
|
|
213
218
|
|
|
214
219
|
const referenceImagePaths = normalizePathList(params.referenceImagePaths);
|
|
215
220
|
const outputPath = normalizeOutputPath(params.outputPath);
|
|
221
|
+
const batchSize = normalizeBatchSize(params.batchSize);
|
|
222
|
+
if (batchSize > 1 && outputPath !== undefined) {
|
|
223
|
+
throw new ImageGenerationError(
|
|
224
|
+
"invalid-parameters",
|
|
225
|
+
"outputPath cannot be used when batchSize is greater than 1; use the generated artifacts instead.",
|
|
226
|
+
);
|
|
227
|
+
}
|
|
216
228
|
|
|
217
229
|
return {
|
|
218
230
|
prompt,
|
|
@@ -221,12 +233,12 @@ export function normalizeGenerateImageParams(
|
|
|
221
233
|
size,
|
|
222
234
|
quality,
|
|
223
235
|
model: normalizeRequestedModel(params.model),
|
|
236
|
+
batchSize,
|
|
224
237
|
action: referenceImagePaths.length > 0 ? "edit" : "generate",
|
|
225
238
|
};
|
|
226
239
|
}
|
|
227
240
|
|
|
228
241
|
export function buildImageGenerationRequest(args: {
|
|
229
|
-
routingModel: string;
|
|
230
242
|
imageModel: string;
|
|
231
243
|
params: NormalizedGenerateImageParams;
|
|
232
244
|
references: readonly PreparedReferenceImage[];
|
|
@@ -238,34 +250,14 @@ export function buildImageGenerationRequest(args: {
|
|
|
238
250
|
"A configured image generation model is required.",
|
|
239
251
|
);
|
|
240
252
|
}
|
|
241
|
-
const content: ImageGenerationRequestBody["input"][number]["content"] = [
|
|
242
|
-
{ type: "input_text", text: args.params.prompt },
|
|
243
|
-
];
|
|
244
|
-
for (const reference of args.references) {
|
|
245
|
-
content.push({
|
|
246
|
-
type: "input_image",
|
|
247
|
-
image_url: `data:${reference.mimeType};base64,${reference.bytes.toString("base64")}`,
|
|
248
|
-
detail: "auto",
|
|
249
|
-
});
|
|
250
|
-
}
|
|
251
|
-
|
|
252
253
|
return {
|
|
253
|
-
|
|
254
|
-
|
|
255
|
-
|
|
256
|
-
|
|
257
|
-
|
|
258
|
-
|
|
259
|
-
|
|
260
|
-
type: "image_generation",
|
|
261
|
-
model: imageModel,
|
|
262
|
-
action: args.params.action,
|
|
263
|
-
size: args.params.size,
|
|
264
|
-
quality: args.params.quality,
|
|
265
|
-
output_format: "png",
|
|
266
|
-
},
|
|
267
|
-
],
|
|
268
|
-
tool_choice: { type: "image_generation" },
|
|
254
|
+
action: args.params.action,
|
|
255
|
+
model: imageModel,
|
|
256
|
+
prompt: args.params.prompt,
|
|
257
|
+
n: 1,
|
|
258
|
+
size: args.params.size,
|
|
259
|
+
quality: args.params.quality,
|
|
260
|
+
references: args.references,
|
|
269
261
|
};
|
|
270
262
|
}
|
|
271
263
|
|
|
@@ -278,35 +270,7 @@ function hasPngEndChunk(buffer: Buffer): boolean {
|
|
|
278
270
|
}
|
|
279
271
|
|
|
280
272
|
export function detectReferenceImageMimeType(bytes: Uint8Array): ReferenceImageMimeType | undefined {
|
|
281
|
-
|
|
282
|
-
if (readPngDimensions(buffer) && hasPngEndChunk(buffer)) return "image/png";
|
|
283
|
-
if (
|
|
284
|
-
buffer.length >= 4 &&
|
|
285
|
-
buffer[0] === 0xff &&
|
|
286
|
-
buffer[1] === 0xd8 &&
|
|
287
|
-
buffer[2] === 0xff &&
|
|
288
|
-
buffer.lastIndexOf(Buffer.from([0xff, 0xd9])) >= 3
|
|
289
|
-
) {
|
|
290
|
-
return "image/jpeg";
|
|
291
|
-
}
|
|
292
|
-
if (
|
|
293
|
-
buffer.length >= 20 &&
|
|
294
|
-
buffer.toString("ascii", 0, 4) === "RIFF" &&
|
|
295
|
-
buffer.toString("ascii", 8, 12) === "WEBP"
|
|
296
|
-
) {
|
|
297
|
-
const declaredSize = buffer.readUInt32LE(4) + 8;
|
|
298
|
-
const chunkType = buffer.toString("ascii", 12, 16);
|
|
299
|
-
const chunkSize = buffer.readUInt32LE(16);
|
|
300
|
-
const paddedChunkSize = chunkSize + (chunkSize % 2);
|
|
301
|
-
if (
|
|
302
|
-
declaredSize === buffer.length &&
|
|
303
|
-
(chunkType === "VP8 " || chunkType === "VP8L" || chunkType === "VP8X") &&
|
|
304
|
-
20 + paddedChunkSize <= buffer.length
|
|
305
|
-
) {
|
|
306
|
-
return "image/webp";
|
|
307
|
-
}
|
|
308
|
-
}
|
|
309
|
-
return undefined;
|
|
273
|
+
return readImageMetadata(bytes)?.mimeType;
|
|
310
274
|
}
|
|
311
275
|
|
|
312
276
|
export function isValidPng(bytes: Uint8Array): boolean {
|
|
@@ -328,14 +292,145 @@ export function readPngDimensions(bytes: Uint8Array): { width: number; height: n
|
|
|
328
292
|
return { width, height };
|
|
329
293
|
}
|
|
330
294
|
|
|
295
|
+
function boundedDimensions(width: number, height: number): { width: number; height: number } | undefined {
|
|
296
|
+
if (
|
|
297
|
+
!Number.isInteger(width) ||
|
|
298
|
+
!Number.isInteger(height) ||
|
|
299
|
+
width <= 0 ||
|
|
300
|
+
height <= 0 ||
|
|
301
|
+
width > MAX_IMAGE_DIMENSION ||
|
|
302
|
+
height > MAX_IMAGE_DIMENSION
|
|
303
|
+
) {
|
|
304
|
+
return undefined;
|
|
305
|
+
}
|
|
306
|
+
return { width, height };
|
|
307
|
+
}
|
|
308
|
+
|
|
309
|
+
const JPEG_DIMENSION_MARKERS = new Set([
|
|
310
|
+
0xc0,
|
|
311
|
+
0xc1,
|
|
312
|
+
0xc2,
|
|
313
|
+
0xc3,
|
|
314
|
+
0xc5,
|
|
315
|
+
0xc6,
|
|
316
|
+
0xc7,
|
|
317
|
+
0xc9,
|
|
318
|
+
0xca,
|
|
319
|
+
0xcb,
|
|
320
|
+
0xcd,
|
|
321
|
+
0xce,
|
|
322
|
+
0xcf,
|
|
323
|
+
]);
|
|
324
|
+
|
|
325
|
+
function readJpegDimensions(bytes: Uint8Array): { width: number; height: number } | undefined {
|
|
326
|
+
const buffer = Buffer.from(bytes.buffer, bytes.byteOffset, bytes.byteLength);
|
|
327
|
+
if (buffer.length < 4 || buffer[0] !== 0xff || buffer[1] !== 0xd8) return undefined;
|
|
328
|
+
let offset = 2;
|
|
329
|
+
while (offset < buffer.length) {
|
|
330
|
+
if (buffer[offset] !== 0xff) {
|
|
331
|
+
offset += 1;
|
|
332
|
+
continue;
|
|
333
|
+
}
|
|
334
|
+
while (offset < buffer.length && buffer[offset] === 0xff) offset += 1;
|
|
335
|
+
if (offset >= buffer.length) return undefined;
|
|
336
|
+
const marker = buffer[offset++];
|
|
337
|
+
if (marker === 0xd9 || marker === 0xda) return undefined;
|
|
338
|
+
if (marker === 0x01 || (marker >= 0xd0 && marker <= 0xd7)) continue;
|
|
339
|
+
if (offset + 2 > buffer.length) return undefined;
|
|
340
|
+
const segmentLength = buffer.readUInt16BE(offset);
|
|
341
|
+
if (segmentLength < 2 || offset + segmentLength > buffer.length) return undefined;
|
|
342
|
+
if (JPEG_DIMENSION_MARKERS.has(marker) && segmentLength >= 7) {
|
|
343
|
+
const height = buffer.readUInt16BE(offset + 3);
|
|
344
|
+
const width = buffer.readUInt16BE(offset + 5);
|
|
345
|
+
return boundedDimensions(width, height);
|
|
346
|
+
}
|
|
347
|
+
offset += segmentLength;
|
|
348
|
+
}
|
|
349
|
+
return undefined;
|
|
350
|
+
}
|
|
351
|
+
|
|
352
|
+
function hasJpegEndMarker(bytes: Uint8Array): boolean {
|
|
353
|
+
const buffer = Buffer.from(bytes.buffer, bytes.byteOffset, bytes.byteLength);
|
|
354
|
+
return buffer.length >= 2 && buffer[buffer.length - 2] === 0xff && buffer[buffer.length - 1] === 0xd9;
|
|
355
|
+
}
|
|
356
|
+
|
|
357
|
+
function readWebpDimensions(bytes: Uint8Array): { width: number; height: number } | undefined {
|
|
358
|
+
const buffer = Buffer.from(bytes.buffer, bytes.byteOffset, bytes.byteLength);
|
|
359
|
+
if (
|
|
360
|
+
buffer.length < 20 ||
|
|
361
|
+
buffer.toString("ascii", 0, 4) !== "RIFF" ||
|
|
362
|
+
buffer.toString("ascii", 8, 12) !== "WEBP"
|
|
363
|
+
) {
|
|
364
|
+
return undefined;
|
|
365
|
+
}
|
|
366
|
+
const declaredSize = buffer.readUInt32LE(4) + 8;
|
|
367
|
+
if (declaredSize !== buffer.length) return undefined;
|
|
368
|
+
let offset = 12;
|
|
369
|
+
while (offset + 8 <= buffer.length) {
|
|
370
|
+
const chunkType = buffer.toString("ascii", offset, offset + 4);
|
|
371
|
+
const chunkSize = buffer.readUInt32LE(offset + 4);
|
|
372
|
+
const dataStart = offset + 8;
|
|
373
|
+
const dataEnd = dataStart + chunkSize;
|
|
374
|
+
if (dataEnd > buffer.length) return undefined;
|
|
375
|
+
if (chunkType === "VP8X" && chunkSize >= 10) {
|
|
376
|
+
const width = 1 + buffer.readUIntLE(dataStart + 4, 3);
|
|
377
|
+
const height = 1 + buffer.readUIntLE(dataStart + 7, 3);
|
|
378
|
+
return boundedDimensions(width, height);
|
|
379
|
+
}
|
|
380
|
+
if (chunkType === "VP8 " && chunkSize >= 10) {
|
|
381
|
+
if (
|
|
382
|
+
buffer[dataStart + 3] === 0x9d &&
|
|
383
|
+
buffer[dataStart + 4] === 0x01 &&
|
|
384
|
+
buffer[dataStart + 5] === 0x2a
|
|
385
|
+
) {
|
|
386
|
+
const width = buffer.readUInt16LE(dataStart + 6) & 0x3fff;
|
|
387
|
+
const height = buffer.readUInt16LE(dataStart + 8) & 0x3fff;
|
|
388
|
+
return boundedDimensions(width, height);
|
|
389
|
+
}
|
|
390
|
+
}
|
|
391
|
+
if (chunkType === "VP8L" && chunkSize >= 5 && buffer[dataStart] === 0x2f) {
|
|
392
|
+
const width = 1 + ((buffer[dataStart + 1] | (buffer[dataStart + 2] << 8)) & 0x3fff);
|
|
393
|
+
const height = 1 + ((buffer[dataStart + 2] >> 6) | (buffer[dataStart + 3] << 2) | (buffer[dataStart + 4] << 10));
|
|
394
|
+
return boundedDimensions(width, height);
|
|
395
|
+
}
|
|
396
|
+
offset = dataEnd + (chunkSize % 2);
|
|
397
|
+
}
|
|
398
|
+
return undefined;
|
|
399
|
+
}
|
|
400
|
+
|
|
331
401
|
function extractBase64(value: string): string {
|
|
332
402
|
const trimmed = value.trim();
|
|
333
403
|
const dataUrl = DATA_URL_PATTERN.exec(trimmed);
|
|
334
404
|
return (dataUrl?.[1] ?? trimmed).trim();
|
|
335
405
|
}
|
|
336
406
|
|
|
337
|
-
|
|
338
|
-
|
|
407
|
+
function normalizeImageMimeType(value: unknown): ImageGenerationMimeType | undefined {
|
|
408
|
+
if (typeof value !== "string") return undefined;
|
|
409
|
+
const normalized = value.trim().toLowerCase();
|
|
410
|
+
if (normalized === "image/jpg") return "image/jpeg";
|
|
411
|
+
return (IMAGE_GENERATION_MIME_TYPES as readonly string[]).includes(normalized)
|
|
412
|
+
? normalized as ImageGenerationMimeType
|
|
413
|
+
: undefined;
|
|
414
|
+
}
|
|
415
|
+
|
|
416
|
+
export function readImageMetadata(bytes: Uint8Array):
|
|
417
|
+
| { mimeType: ImageGenerationMimeType; width: number; height: number }
|
|
418
|
+
| undefined {
|
|
419
|
+
if (readPngDimensions(bytes) && hasPngEndChunk(Buffer.from(bytes))) {
|
|
420
|
+
const dimensions = readPngDimensions(bytes)!;
|
|
421
|
+
return { mimeType: "image/png", ...dimensions };
|
|
422
|
+
}
|
|
423
|
+
if (hasJpegEndMarker(bytes)) {
|
|
424
|
+
const dimensions = readJpegDimensions(bytes);
|
|
425
|
+
if (dimensions) return { mimeType: "image/jpeg", ...dimensions };
|
|
426
|
+
}
|
|
427
|
+
const dimensions = readWebpDimensions(bytes);
|
|
428
|
+
if (dimensions) return { mimeType: "image/webp", ...dimensions };
|
|
429
|
+
return undefined;
|
|
430
|
+
}
|
|
431
|
+
|
|
432
|
+
export function decodeGeneratedImage(value: string, hintedMimeType?: unknown):
|
|
433
|
+
| { ok: true; bytes: Buffer; mimeType: ImageGenerationMimeType; width: number; height: number }
|
|
339
434
|
| { ok: false; reason: "malformed-response" | "oversized-response"; errorMessage: string } {
|
|
340
435
|
const base64 = extractBase64(value);
|
|
341
436
|
const maxBase64Chars = Math.ceil(MAX_GENERATED_IMAGE_BYTES / 3) * 4;
|
|
@@ -367,16 +462,40 @@ export function decodeGeneratedPng(value: string):
|
|
|
367
462
|
errorMessage: "Image generation returned invalid or oversized image data.",
|
|
368
463
|
};
|
|
369
464
|
}
|
|
370
|
-
const
|
|
371
|
-
if (!
|
|
465
|
+
const metadata = readImageMetadata(bytes);
|
|
466
|
+
if (!metadata) {
|
|
372
467
|
bytes.fill(0);
|
|
373
468
|
return {
|
|
374
469
|
ok: false,
|
|
375
470
|
reason: "malformed-response",
|
|
376
|
-
errorMessage: "Image generation returned data that is not a
|
|
471
|
+
errorMessage: "Image generation returned data that is not a supported PNG, JPEG, or WebP image.",
|
|
377
472
|
};
|
|
378
473
|
}
|
|
379
|
-
|
|
474
|
+
if (typeof hintedMimeType === "string" && hintedMimeType.trim()) {
|
|
475
|
+
const hinted = normalizeImageMimeType(hintedMimeType);
|
|
476
|
+
if (!hinted || hinted !== metadata.mimeType) {
|
|
477
|
+
bytes.fill(0);
|
|
478
|
+
return {
|
|
479
|
+
ok: false,
|
|
480
|
+
reason: "malformed-response",
|
|
481
|
+
errorMessage: "Image generation returned an image with inconsistent MIME metadata.",
|
|
482
|
+
};
|
|
483
|
+
}
|
|
484
|
+
}
|
|
485
|
+
return { ok: true, bytes, ...metadata };
|
|
486
|
+
}
|
|
487
|
+
|
|
488
|
+
export function decodeGeneratedPng(value: string):
|
|
489
|
+
| { ok: true; bytes: Buffer; width: number; height: number }
|
|
490
|
+
| { ok: false; reason: "malformed-response" | "oversized-response"; errorMessage: string } {
|
|
491
|
+
const decoded = decodeGeneratedImage(value, "image/png");
|
|
492
|
+
if (!decoded.ok) return decoded;
|
|
493
|
+
return {
|
|
494
|
+
ok: true,
|
|
495
|
+
bytes: decoded.bytes,
|
|
496
|
+
width: decoded.width,
|
|
497
|
+
height: decoded.height,
|
|
498
|
+
};
|
|
380
499
|
}
|
|
381
500
|
|
|
382
501
|
export function extractProviderErrorMessage(value: unknown, fallback: string): string {
|
|
@@ -390,7 +509,7 @@ export function extractProviderErrorMessage(value: unknown, fallback: string): s
|
|
|
390
509
|
return fallback;
|
|
391
510
|
}
|
|
392
511
|
|
|
393
|
-
export function
|
|
512
|
+
export function parseImagesApiResponse(value: unknown): ParsedImageResponseResult {
|
|
394
513
|
if (!isRecord(value)) {
|
|
395
514
|
return {
|
|
396
515
|
ok: false,
|
|
@@ -398,75 +517,56 @@ export function parseImageGenerationResponse(value: unknown): ParsedImageRespons
|
|
|
398
517
|
errorMessage: "Image generation returned a non-object response.",
|
|
399
518
|
};
|
|
400
519
|
}
|
|
401
|
-
|
|
402
|
-
if (value.status !== "completed") {
|
|
520
|
+
if (!Array.isArray(value.data)) {
|
|
403
521
|
return {
|
|
404
522
|
ok: false,
|
|
405
|
-
reason: "
|
|
406
|
-
errorMessage:
|
|
407
|
-
value,
|
|
408
|
-
`Image generation did not complete (status: ${String(value.status ?? "unknown")}).`,
|
|
409
|
-
),
|
|
523
|
+
reason: "malformed-response",
|
|
524
|
+
errorMessage: "Image generation response did not contain a data array.",
|
|
410
525
|
};
|
|
411
526
|
}
|
|
412
|
-
if (
|
|
527
|
+
if (value.data.length === 0) {
|
|
413
528
|
return {
|
|
414
529
|
ok: false,
|
|
415
|
-
reason: "
|
|
416
|
-
errorMessage: "Image generation
|
|
530
|
+
reason: "no-image",
|
|
531
|
+
errorMessage: "Image generation completed without a usable image result.",
|
|
417
532
|
};
|
|
418
533
|
}
|
|
419
|
-
|
|
420
|
-
|
|
421
|
-
|
|
422
|
-
|
|
423
|
-
|
|
424
|
-
|
|
425
|
-
const result = typeof item.result === "string"
|
|
426
|
-
? item.result
|
|
427
|
-
: typeof item.b64_json === "string"
|
|
428
|
-
? item.b64_json
|
|
429
|
-
: undefined;
|
|
430
|
-
if (!result?.trim()) continue;
|
|
431
|
-
const id = normalizedOptionalString(item.id, MAX_IMAGE_IDENTIFIER_CHARS) ?? `image_generation_${index}`;
|
|
432
|
-
const revisedPrompt = normalizedOptionalString(item.revised_prompt, MAX_IMAGE_DIAGNOSTIC_CHARS);
|
|
433
|
-
const existing = calls.get(id);
|
|
434
|
-
if (existing && existing.result !== result) {
|
|
435
|
-
return {
|
|
436
|
-
ok: false,
|
|
437
|
-
reason: "malformed-response",
|
|
438
|
-
errorMessage: "Image generation returned conflicting results for one image call.",
|
|
439
|
-
};
|
|
440
|
-
}
|
|
441
|
-
calls.set(id, { result, revisedPrompt: revisedPrompt ?? existing?.revisedPrompt });
|
|
534
|
+
if (value.data.length !== 1) {
|
|
535
|
+
return {
|
|
536
|
+
ok: false,
|
|
537
|
+
reason: "malformed-response",
|
|
538
|
+
errorMessage: "Image generation returned more than one image for a single request.",
|
|
539
|
+
};
|
|
442
540
|
}
|
|
443
541
|
|
|
444
|
-
|
|
542
|
+
const item = value.data[0];
|
|
543
|
+
if (!isRecord(item) || typeof item.b64_json !== "string" || !item.b64_json.trim()) {
|
|
445
544
|
return {
|
|
446
545
|
ok: false,
|
|
447
546
|
reason: "no-image",
|
|
448
|
-
errorMessage: "Image generation
|
|
547
|
+
errorMessage: "Image generation response did not contain base64 image data.",
|
|
449
548
|
};
|
|
450
549
|
}
|
|
451
|
-
|
|
550
|
+
const mimeHint = item.mime_type ?? item.mimeType;
|
|
551
|
+
if (mimeHint !== undefined && typeof mimeHint !== "string") {
|
|
452
552
|
return {
|
|
453
553
|
ok: false,
|
|
454
554
|
reason: "malformed-response",
|
|
455
|
-
errorMessage: "Image generation returned
|
|
555
|
+
errorMessage: "Image generation returned invalid MIME metadata.",
|
|
456
556
|
};
|
|
457
557
|
}
|
|
458
|
-
|
|
459
|
-
const [imageCallId, call] = [...calls.entries()][0]!;
|
|
460
|
-
const decoded = decodeGeneratedPng(call.result);
|
|
558
|
+
const decoded = decodeGeneratedImage(item.b64_json, mimeHint);
|
|
461
559
|
if (!decoded.ok) return decoded;
|
|
462
560
|
|
|
463
561
|
return {
|
|
464
562
|
ok: true,
|
|
465
563
|
image: {
|
|
466
564
|
bytes: decoded.bytes,
|
|
467
|
-
|
|
565
|
+
mimeType: decoded.mimeType,
|
|
566
|
+
imageCallId:
|
|
567
|
+
normalizedOptionalString(item.id, MAX_IMAGE_IDENTIFIER_CHARS) ?? "image_generation_0",
|
|
468
568
|
responseId: normalizedOptionalString(value.id, MAX_IMAGE_IDENTIFIER_CHARS),
|
|
469
|
-
revisedPrompt:
|
|
569
|
+
revisedPrompt: normalizedOptionalString(item.revised_prompt, MAX_IMAGE_DIAGNOSTIC_CHARS),
|
|
470
570
|
width: decoded.width,
|
|
471
571
|
height: decoded.height,
|
|
472
572
|
},
|
|
@@ -475,5 +575,5 @@ export function parseImageGenerationResponse(value: unknown): ParsedImageRespons
|
|
|
475
575
|
|
|
476
576
|
export const _protocolTest = {
|
|
477
577
|
PNG_SIGNATURE,
|
|
478
|
-
|
|
578
|
+
IMAGE_GENERATION_MIME_TYPES,
|
|
479
579
|
};
|