pi-openai-toolkit 0.17.3 → 0.18.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +2 -2
- package/README.zh.md +2 -2
- package/docs/configuration.md +1 -1
- package/package.json +1 -1
- package/src/image-generation/client.ts +70 -8
- package/src/image-generation/eligibility.ts +3 -3
- package/src/image-generation/extension.ts +18 -4
- package/src/image-generation/protocol.ts +57 -94
- package/src/image-generation/render.ts +3 -3
- package/src/image-generation/service.ts +356 -255
- package/src/image-generation/types.ts +94 -3
- package/src/responses-headers.ts +7 -2
- package/src/types.ts +3 -3
package/README.md
CHANGED
|
@@ -168,7 +168,7 @@ Toolkit does not fall back between routes. See [search configuration](docs/confi
|
|
|
168
168
|
|
|
169
169
|
### Generate an image
|
|
170
170
|
|
|
171
|
-
Requires
|
|
171
|
+
Requires an eligible Responses-capable session and may incur provider charges. The local Pi tool sends a separate Images API request to the current provider (`/images/generations` or `/images/edits`; Codex uses `/codex/images/...`) rather than adding an image tool to the conversation request. Enable it with an image model your provider supports:
|
|
172
172
|
|
|
173
173
|
```json
|
|
174
174
|
{
|
|
@@ -183,7 +183,7 @@ Requires a Responses session and may incur provider charges. Enable it with an i
|
|
|
183
183
|
}
|
|
184
184
|
```
|
|
185
185
|
|
|
186
|
-
Ask Pi to generate an image or edit explicitly supplied local references using `openai_generate_image`. The
|
|
186
|
+
Ask Pi to generate an image or edit explicitly supplied local references using `openai_generate_image`. The selected image model is sent in the independent request; the active conversation model remains the routing/auth model. Set the tool's optional `model` to another exact `allowedModels` entry, or use `batchSize` (1-10) for concurrent independent variants. Batch calls save canonical artifacts and cannot use a single-file `outputPath`; image settings are global. See [image configuration](docs/configuration.md#images).
|
|
187
187
|
|
|
188
188
|
### Review tool calls automatically
|
|
189
189
|
|
package/README.zh.md
CHANGED
|
@@ -168,7 +168,7 @@ Toolkit 不会在路由之间自动回退。后端要求见[搜索配置](docs/c
|
|
|
168
168
|
|
|
169
169
|
### 生成图片
|
|
170
170
|
|
|
171
|
-
|
|
171
|
+
需要符合条件的 Responses-capable 会话,并可能产生服务商费用。Pi 工具会向当前服务商发送独立的 Images API 请求(`/images/generations` 或 `/images/edits`;Codex 使用 `/codex/images/...`),不会把生图 tool 加入普通对话请求。选择服务商支持的生图模型后启用:
|
|
172
172
|
|
|
173
173
|
```json
|
|
174
174
|
{
|
|
@@ -183,7 +183,7 @@ Toolkit 不会在路由之间自动回退。后端要求见[搜索配置](docs/c
|
|
|
183
183
|
}
|
|
184
184
|
```
|
|
185
185
|
|
|
186
|
-
让 Pi 通过 `openai_generate_image`
|
|
186
|
+
让 Pi 通过 `openai_generate_image` 生成图片,或编辑明确提供的本地参考图。选中的生图模型会写入独立请求;当前对话模型仍只负责路由和认证。工具的可选 `model` 必须精确匹配 `allowedModels`,也可以使用 `batchSize`(1-10)并发生成多个独立变体。批量调用会保存 canonical artifacts,不能同时指定单文件 `outputPath`;生图设置全局生效。详见[生图配置](docs/configuration.md#images)。
|
|
187
187
|
|
|
188
188
|
### 启用工具调用自动审查
|
|
189
189
|
|
package/docs/configuration.md
CHANGED
|
@@ -104,7 +104,7 @@ There is no v2 search master switch. `local` and `unmanaged` differ in payload c
|
|
|
104
104
|
|
|
105
105
|
Image settings are global-only under `defaults.imageGeneration`. `enabled` defaults to `false`. `defaultModel` defaults to `"gpt-image-2.5"`; `allowedModels` defaults to `["gpt-image-2.5"]`.
|
|
106
106
|
|
|
107
|
-
These are bare output-model IDs, not active-session model keys. Each normalized ID must contain 1-256 characters. The allowed list must remain nonempty after normalization and contain the explicit default.
|
|
107
|
+
These are bare output-model IDs, not active-session model keys. Each normalized ID must contain 1-256 characters. The allowed list must remain nonempty after normalization and contain the explicit default. A caller's one-call `model` must be a member; policy and membership are checked before auth, local-reference preparation/upload, and paid dispatch. The active session remains the routing/auth model, while the selected image model is sent in a separate Images API request. Generation uses `/images/generations`; edits use `/images/edits` (Codex uses the corresponding `/codex/images/...` paths). The wrapper never injects a Responses `image_generation` tool. `batchSize` is a tool argument from 1-10; each item is an independent `n: 1` request, partial success preserves saved artifacts, and batch calls cannot use a single-file `outputPath`. Config does not grant permission to upload arbitrary local files or imply that a provider supports a particular image model.
|
|
108
108
|
|
|
109
109
|
---
|
|
110
110
|
|
package/package.json
CHANGED
|
@@ -1,10 +1,11 @@
|
|
|
1
1
|
import { buildResponsesRequestHeaders } from "../responses-headers";
|
|
2
|
-
import type
|
|
2
|
+
import { normalizeBaseUrl, type ResponsesRuntime } from "../runtime";
|
|
3
3
|
import {
|
|
4
4
|
extractProviderErrorMessage,
|
|
5
|
-
|
|
5
|
+
parseImagesApiResponse,
|
|
6
6
|
type ImageGenerationRequestBody,
|
|
7
7
|
} from "./protocol";
|
|
8
|
+
import type { PreparedReferenceImage } from "./types";
|
|
8
9
|
import {
|
|
9
10
|
IMAGE_REQUEST_TIMEOUT_MS,
|
|
10
11
|
MAX_IMAGE_ERROR_BYTES,
|
|
@@ -39,8 +40,62 @@ export type ImageGenerationFetch = (
|
|
|
39
40
|
init?: RequestInit,
|
|
40
41
|
) => Promise<Response>;
|
|
41
42
|
|
|
42
|
-
function buildRequestHeaders(runtime: ResponsesRuntime): Headers {
|
|
43
|
-
return buildResponsesRequestHeaders(runtime, { accept: "application/json" });
|
|
43
|
+
function buildRequestHeaders(runtime: ResponsesRuntime, contentType: string | null = "application/json"): Headers {
|
|
44
|
+
return buildResponsesRequestHeaders(runtime, { accept: "application/json", contentType });
|
|
45
|
+
}
|
|
46
|
+
|
|
47
|
+
function buildImagesEndpoint(runtime: ResponsesRuntime, action: ImageGenerationRequestBody["action"]): string {
|
|
48
|
+
const baseUrl = normalizeBaseUrl(runtime.baseUrl);
|
|
49
|
+
if (!baseUrl) throw new Error("Image generation requires a provider base URL.");
|
|
50
|
+
if (baseUrl.endsWith("/responses") || baseUrl.endsWith("/codex/responses")) {
|
|
51
|
+
throw new Error("Image generation requires the provider base URL, not a resolved Responses URL.");
|
|
52
|
+
}
|
|
53
|
+
const suffix = runtime.api === "openai-codex-responses" ? "codex/images" : "images";
|
|
54
|
+
const baseAlreadyIncludesCodex = runtime.api === "openai-codex-responses" && baseUrl.endsWith("/codex");
|
|
55
|
+
return `${baseUrl}/${baseAlreadyIncludesCodex ? "images" : suffix}/${action === "edit" ? "edits" : "generations"}`;
|
|
56
|
+
}
|
|
57
|
+
|
|
58
|
+
function dataUrl(reference: PreparedReferenceImage): string {
|
|
59
|
+
return `data:${reference.mimeType};base64,${reference.bytes.toString("base64")}`;
|
|
60
|
+
}
|
|
61
|
+
|
|
62
|
+
function jsonRequestBody(runtime: ResponsesRuntime, body: ImageGenerationRequestBody): Record<string, unknown> {
|
|
63
|
+
const common = {
|
|
64
|
+
model: body.model,
|
|
65
|
+
prompt: body.prompt,
|
|
66
|
+
n: body.n,
|
|
67
|
+
size: body.size,
|
|
68
|
+
quality: body.quality,
|
|
69
|
+
};
|
|
70
|
+
if (body.action === "edit") {
|
|
71
|
+
return {
|
|
72
|
+
...common,
|
|
73
|
+
images: body.references.map((reference) => ({ image_url: dataUrl(reference) })),
|
|
74
|
+
};
|
|
75
|
+
}
|
|
76
|
+
return runtime.api === "openai-codex-responses"
|
|
77
|
+
? common
|
|
78
|
+
: { ...common, output_format: "png", response_format: "b64_json" };
|
|
79
|
+
}
|
|
80
|
+
|
|
81
|
+
function multipartRequestBody(body: ImageGenerationRequestBody): FormData {
|
|
82
|
+
const form = new FormData();
|
|
83
|
+
form.append("model", body.model);
|
|
84
|
+
form.append("prompt", body.prompt);
|
|
85
|
+
form.append("n", String(body.n));
|
|
86
|
+
form.append("size", body.size);
|
|
87
|
+
form.append("quality", body.quality);
|
|
88
|
+
form.append("output_format", "png");
|
|
89
|
+
form.append("response_format", "b64_json");
|
|
90
|
+
for (const [index, reference] of body.references.entries()) {
|
|
91
|
+
const extension = reference.mimeType === "image/jpeg" ? "jpg" : reference.mimeType.slice("image/".length);
|
|
92
|
+
form.append(
|
|
93
|
+
"image",
|
|
94
|
+
new Blob([new Uint8Array(reference.bytes)], { type: reference.mimeType }),
|
|
95
|
+
`reference-${index + 1}.${extension}`,
|
|
96
|
+
);
|
|
97
|
+
}
|
|
98
|
+
return form;
|
|
44
99
|
}
|
|
45
100
|
|
|
46
101
|
async function readBoundedBody(
|
|
@@ -141,12 +196,16 @@ export async function requestGeneratedImage(args: {
|
|
|
141
196
|
? AbortSignal.any([args.signal, timeoutSignal])
|
|
142
197
|
: timeoutSignal;
|
|
143
198
|
|
|
199
|
+
const multipart = args.body.action === "edit" && args.runtime.api === "openai-responses";
|
|
200
|
+
const transportBody: BodyInit = multipart
|
|
201
|
+
? multipartRequestBody(args.body)
|
|
202
|
+
: JSON.stringify(jsonRequestBody(args.runtime, args.body));
|
|
144
203
|
let response: Response;
|
|
145
204
|
try {
|
|
146
|
-
response = await fetchFn(args.runtime.
|
|
205
|
+
response = await fetchFn(buildImagesEndpoint(args.runtime, args.body.action), {
|
|
147
206
|
method: "POST",
|
|
148
|
-
headers: buildRequestHeaders(args.runtime),
|
|
149
|
-
body:
|
|
207
|
+
headers: buildRequestHeaders(args.runtime, multipart ? null : "application/json"),
|
|
208
|
+
body: transportBody,
|
|
150
209
|
signal,
|
|
151
210
|
});
|
|
152
211
|
} catch (error) {
|
|
@@ -230,7 +289,7 @@ export async function requestGeneratedImage(args: {
|
|
|
230
289
|
};
|
|
231
290
|
}
|
|
232
291
|
|
|
233
|
-
const parsed =
|
|
292
|
+
const parsed = parseImagesApiResponse(payload);
|
|
234
293
|
if (!parsed.ok) {
|
|
235
294
|
return {
|
|
236
295
|
ok: false,
|
|
@@ -244,5 +303,8 @@ export async function requestGeneratedImage(args: {
|
|
|
244
303
|
|
|
245
304
|
export const _clientTest = {
|
|
246
305
|
buildRequestHeaders,
|
|
306
|
+
buildImagesEndpoint,
|
|
307
|
+
jsonRequestBody,
|
|
308
|
+
multipartRequestBody,
|
|
247
309
|
readBoundedBody,
|
|
248
310
|
};
|
|
@@ -2,9 +2,9 @@ import type { ImageGenerationConfig } from "../types";
|
|
|
2
2
|
import { IMAGE_GENERATION_CAPABLE_APIS, type ImageGenerationModel } from "./types";
|
|
3
3
|
|
|
4
4
|
/**
|
|
5
|
-
* The
|
|
6
|
-
* specific model
|
|
7
|
-
*
|
|
5
|
+
* The independent Images API is exposed only for Responses-capable runtimes rather than for a
|
|
6
|
+
* specific conversation model. `enabled` is the single opt-in switch and defaults to off because
|
|
7
|
+
* every successful provider request may incur a charge.
|
|
8
8
|
*/
|
|
9
9
|
export function isImageGenerationEnabledForModel(
|
|
10
10
|
model: ImageGenerationModel | undefined,
|
|
@@ -9,13 +9,14 @@ import {
|
|
|
9
9
|
IMAGE_GENERATION_QUALITIES,
|
|
10
10
|
IMAGE_GENERATION_SIZES,
|
|
11
11
|
IMAGE_GENERATION_TOOL_NAME,
|
|
12
|
+
MAX_IMAGE_BATCH_SIZE,
|
|
12
13
|
MAX_IMAGE_MODEL_ID_CHARS,
|
|
13
14
|
MAX_IMAGE_PATH_CHARS,
|
|
14
15
|
MAX_IMAGE_PROMPT_CHARS,
|
|
15
16
|
MAX_REFERENCE_IMAGE_COUNT,
|
|
16
17
|
ImageGenerationError,
|
|
17
18
|
sanitizeImageDiagnostic,
|
|
18
|
-
type
|
|
19
|
+
type ImageGenerationToolDetails,
|
|
19
20
|
} from "./types";
|
|
20
21
|
|
|
21
22
|
const registeredApis = new WeakSet<object>();
|
|
@@ -82,6 +83,18 @@ const GenerateImageParameters = Type.Object(
|
|
|
82
83
|
},
|
|
83
84
|
),
|
|
84
85
|
),
|
|
86
|
+
batchSize: Type.Optional(
|
|
87
|
+
Type.Union(
|
|
88
|
+
[
|
|
89
|
+
Type.Null(),
|
|
90
|
+
Type.Integer({ minimum: 1, maximum: MAX_IMAGE_BATCH_SIZE }),
|
|
91
|
+
],
|
|
92
|
+
{
|
|
93
|
+
description:
|
|
94
|
+
"Run this many independent image requests concurrently (1-10). Batch requests use generated artifacts and cannot set outputPath.",
|
|
95
|
+
},
|
|
96
|
+
),
|
|
97
|
+
),
|
|
85
98
|
},
|
|
86
99
|
{ additionalProperties: false },
|
|
87
100
|
);
|
|
@@ -111,12 +124,12 @@ export function registerImageGenerationExtension(
|
|
|
111
124
|
if (registeredApis.has(pi)) return;
|
|
112
125
|
registeredApis.add(pi);
|
|
113
126
|
|
|
114
|
-
pi.registerTool<typeof GenerateImageParameters,
|
|
127
|
+
pi.registerTool<typeof GenerateImageParameters, ImageGenerationToolDetails, Record<string, never>>({
|
|
115
128
|
name: IMAGE_GENERATION_TOOL_NAME,
|
|
116
129
|
label: "OpenAI Generate Image",
|
|
117
130
|
description:
|
|
118
|
-
"Generate
|
|
119
|
-
promptSnippet: "Generate or edit PNG images through
|
|
131
|
+
"Generate PNG images, or edit from one to five user-approved local reference images, through an independent Images API request using a configured image model. This is a paid provider operation; batchSize runs independent requests concurrently.",
|
|
132
|
+
promptSnippet: "Generate or edit PNG images through an independent Images API request and a configured imageGeneration.allowedModels entry.",
|
|
120
133
|
promptGuidelines: [
|
|
121
134
|
"Use openai_generate_image when the user explicitly asks to create, draw, render, or edit a raster image and the active model speaks a Responses API.",
|
|
122
135
|
"Do not call openai_generate_image speculatively: it consumes the user's provider or gateway image quota.",
|
|
@@ -124,6 +137,7 @@ export function registerImageGenerationExtension(
|
|
|
124
137
|
"Set referenceImagePaths to null unless the user explicitly identified local files; never invent paths or placeholder strings, and remember upload requires user approval.",
|
|
125
138
|
"Set outputPath to null unless the user explicitly asks for a destination; never invent a destination or placeholder string, and otherwise use the default Pi agent artifact.",
|
|
126
139
|
"Set model to null unless the user named an image model configured in imageGeneration.allowedModels; never invent or guess a model id, and remember an unlisted model fails before the paid request.",
|
|
140
|
+
"Use batchSize only when the user asks for multiple variants; each batch item is an independent paid request, and batchSize cannot be combined with outputPath.",
|
|
127
141
|
"Do not substitute Python, browser automation, shell scripts, or unrelated image tools for an eligible image request.",
|
|
128
142
|
],
|
|
129
143
|
parameters: GenerateImageParameters,
|
|
@@ -4,6 +4,7 @@ import {
|
|
|
4
4
|
IMAGE_GENERATION_QUALITIES,
|
|
5
5
|
IMAGE_GENERATION_SIZES,
|
|
6
6
|
MAX_GENERATED_IMAGE_BYTES,
|
|
7
|
+
MAX_IMAGE_BATCH_SIZE,
|
|
7
8
|
MAX_IMAGE_DIAGNOSTIC_CHARS,
|
|
8
9
|
MAX_IMAGE_DIMENSION,
|
|
9
10
|
MAX_IMAGE_IDENTIFIER_CHARS,
|
|
@@ -26,26 +27,13 @@ const BASE64_PATTERN = /^(?:[A-Za-z0-9+/]{4})*(?:[A-Za-z0-9+/]{2}==|[A-Za-z0-9+/
|
|
|
26
27
|
const DATA_URL_PATTERN = /^data:image\/[a-z0-9.+-]+;base64,(.*)$/is;
|
|
27
28
|
|
|
28
29
|
export type ImageGenerationRequestBody = {
|
|
30
|
+
action: NormalizedGenerateImageParams["action"];
|
|
29
31
|
model: string;
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
content: Array<
|
|
36
|
-
| { type: "input_text"; text: string }
|
|
37
|
-
| { type: "input_image"; image_url: string; detail: "auto" }
|
|
38
|
-
>;
|
|
39
|
-
}>;
|
|
40
|
-
tools: Array<{
|
|
41
|
-
type: "image_generation";
|
|
42
|
-
model: string;
|
|
43
|
-
action: "generate" | "edit";
|
|
44
|
-
size: NormalizedGenerateImageParams["size"];
|
|
45
|
-
quality: NormalizedGenerateImageParams["quality"];
|
|
46
|
-
output_format: "png";
|
|
47
|
-
}>;
|
|
48
|
-
tool_choice: { type: "image_generation" };
|
|
32
|
+
prompt: string;
|
|
33
|
+
n: 1;
|
|
34
|
+
size: NormalizedGenerateImageParams["size"];
|
|
35
|
+
quality: NormalizedGenerateImageParams["quality"];
|
|
36
|
+
references: readonly PreparedReferenceImage[];
|
|
49
37
|
};
|
|
50
38
|
|
|
51
39
|
export type ParsedImageResponseResult =
|
|
@@ -137,6 +125,22 @@ function normalizeRequestedModel(value: unknown): string | undefined {
|
|
|
137
125
|
return trimmed;
|
|
138
126
|
}
|
|
139
127
|
|
|
128
|
+
function normalizeBatchSize(value: unknown): number {
|
|
129
|
+
if (value === null || value === undefined) return 1;
|
|
130
|
+
if (
|
|
131
|
+
typeof value !== "number" ||
|
|
132
|
+
!Number.isInteger(value) ||
|
|
133
|
+
value < 1 ||
|
|
134
|
+
value > MAX_IMAGE_BATCH_SIZE
|
|
135
|
+
) {
|
|
136
|
+
throw new ImageGenerationError(
|
|
137
|
+
"invalid-parameters",
|
|
138
|
+
`batchSize must be an integer from 1 to ${MAX_IMAGE_BATCH_SIZE}.`,
|
|
139
|
+
);
|
|
140
|
+
}
|
|
141
|
+
return value;
|
|
142
|
+
}
|
|
143
|
+
|
|
140
144
|
/**
|
|
141
145
|
* Configured model ids are only trusted as far as the shared bounds: unusable entries are
|
|
142
146
|
* dropped, and an empty or malformed list falls back to the shipped default model.
|
|
@@ -153,7 +157,7 @@ function usableImageModels(models: readonly string[] | undefined): string[] {
|
|
|
153
157
|
}
|
|
154
158
|
|
|
155
159
|
/**
|
|
156
|
-
* Resolve the
|
|
160
|
+
* Resolve the independent Images API model for one call. An omitted request selects the
|
|
157
161
|
* explicit default when supplied (v2), or the first configured model for legacy callers.
|
|
158
162
|
* An explicit request must match a configured id exactly before any paid dispatch.
|
|
159
163
|
*/
|
|
@@ -213,6 +217,13 @@ export function normalizeGenerateImageParams(
|
|
|
213
217
|
|
|
214
218
|
const referenceImagePaths = normalizePathList(params.referenceImagePaths);
|
|
215
219
|
const outputPath = normalizeOutputPath(params.outputPath);
|
|
220
|
+
const batchSize = normalizeBatchSize(params.batchSize);
|
|
221
|
+
if (batchSize > 1 && outputPath !== undefined) {
|
|
222
|
+
throw new ImageGenerationError(
|
|
223
|
+
"invalid-parameters",
|
|
224
|
+
"outputPath cannot be used when batchSize is greater than 1; use the generated artifacts instead.",
|
|
225
|
+
);
|
|
226
|
+
}
|
|
216
227
|
|
|
217
228
|
return {
|
|
218
229
|
prompt,
|
|
@@ -221,12 +232,12 @@ export function normalizeGenerateImageParams(
|
|
|
221
232
|
size,
|
|
222
233
|
quality,
|
|
223
234
|
model: normalizeRequestedModel(params.model),
|
|
235
|
+
batchSize,
|
|
224
236
|
action: referenceImagePaths.length > 0 ? "edit" : "generate",
|
|
225
237
|
};
|
|
226
238
|
}
|
|
227
239
|
|
|
228
240
|
export function buildImageGenerationRequest(args: {
|
|
229
|
-
routingModel: string;
|
|
230
241
|
imageModel: string;
|
|
231
242
|
params: NormalizedGenerateImageParams;
|
|
232
243
|
references: readonly PreparedReferenceImage[];
|
|
@@ -238,34 +249,14 @@ export function buildImageGenerationRequest(args: {
|
|
|
238
249
|
"A configured image generation model is required.",
|
|
239
250
|
);
|
|
240
251
|
}
|
|
241
|
-
const content: ImageGenerationRequestBody["input"][number]["content"] = [
|
|
242
|
-
{ type: "input_text", text: args.params.prompt },
|
|
243
|
-
];
|
|
244
|
-
for (const reference of args.references) {
|
|
245
|
-
content.push({
|
|
246
|
-
type: "input_image",
|
|
247
|
-
image_url: `data:${reference.mimeType};base64,${reference.bytes.toString("base64")}`,
|
|
248
|
-
detail: "auto",
|
|
249
|
-
});
|
|
250
|
-
}
|
|
251
|
-
|
|
252
252
|
return {
|
|
253
|
-
|
|
254
|
-
|
|
255
|
-
|
|
256
|
-
|
|
257
|
-
|
|
258
|
-
|
|
259
|
-
|
|
260
|
-
type: "image_generation",
|
|
261
|
-
model: imageModel,
|
|
262
|
-
action: args.params.action,
|
|
263
|
-
size: args.params.size,
|
|
264
|
-
quality: args.params.quality,
|
|
265
|
-
output_format: "png",
|
|
266
|
-
},
|
|
267
|
-
],
|
|
268
|
-
tool_choice: { type: "image_generation" },
|
|
253
|
+
action: args.params.action,
|
|
254
|
+
model: imageModel,
|
|
255
|
+
prompt: args.params.prompt,
|
|
256
|
+
n: 1,
|
|
257
|
+
size: args.params.size,
|
|
258
|
+
quality: args.params.quality,
|
|
259
|
+
references: args.references,
|
|
269
260
|
};
|
|
270
261
|
}
|
|
271
262
|
|
|
@@ -390,7 +381,7 @@ export function extractProviderErrorMessage(value: unknown, fallback: string): s
|
|
|
390
381
|
return fallback;
|
|
391
382
|
}
|
|
392
383
|
|
|
393
|
-
export function
|
|
384
|
+
export function parseImagesApiResponse(value: unknown): ParsedImageResponseResult {
|
|
394
385
|
if (!isRecord(value)) {
|
|
395
386
|
return {
|
|
396
387
|
ok: false,
|
|
@@ -398,75 +389,47 @@ export function parseImageGenerationResponse(value: unknown): ParsedImageRespons
|
|
|
398
389
|
errorMessage: "Image generation returned a non-object response.",
|
|
399
390
|
};
|
|
400
391
|
}
|
|
401
|
-
|
|
402
|
-
if (value.status !== "completed") {
|
|
403
|
-
return {
|
|
404
|
-
ok: false,
|
|
405
|
-
reason: "request-rejected",
|
|
406
|
-
errorMessage: extractProviderErrorMessage(
|
|
407
|
-
value,
|
|
408
|
-
`Image generation did not complete (status: ${String(value.status ?? "unknown")}).`,
|
|
409
|
-
),
|
|
410
|
-
};
|
|
411
|
-
}
|
|
412
|
-
if (!Array.isArray(value.output)) {
|
|
392
|
+
if (!Array.isArray(value.data)) {
|
|
413
393
|
return {
|
|
414
394
|
ok: false,
|
|
415
395
|
reason: "malformed-response",
|
|
416
|
-
errorMessage: "Image generation response did not contain
|
|
396
|
+
errorMessage: "Image generation response did not contain a data array.",
|
|
417
397
|
};
|
|
418
398
|
}
|
|
419
|
-
|
|
420
|
-
const calls = new Map<string, { result: string; revisedPrompt?: string }>();
|
|
421
|
-
for (let index = 0; index < value.output.length; index += 1) {
|
|
422
|
-
const item = value.output[index];
|
|
423
|
-
if (!isRecord(item) || item.type !== "image_generation_call") continue;
|
|
424
|
-
if (item.status !== "completed") continue;
|
|
425
|
-
const result = typeof item.result === "string"
|
|
426
|
-
? item.result
|
|
427
|
-
: typeof item.b64_json === "string"
|
|
428
|
-
? item.b64_json
|
|
429
|
-
: undefined;
|
|
430
|
-
if (!result?.trim()) continue;
|
|
431
|
-
const id = normalizedOptionalString(item.id, MAX_IMAGE_IDENTIFIER_CHARS) ?? `image_generation_${index}`;
|
|
432
|
-
const revisedPrompt = normalizedOptionalString(item.revised_prompt, MAX_IMAGE_DIAGNOSTIC_CHARS);
|
|
433
|
-
const existing = calls.get(id);
|
|
434
|
-
if (existing && existing.result !== result) {
|
|
435
|
-
return {
|
|
436
|
-
ok: false,
|
|
437
|
-
reason: "malformed-response",
|
|
438
|
-
errorMessage: "Image generation returned conflicting results for one image call.",
|
|
439
|
-
};
|
|
440
|
-
}
|
|
441
|
-
calls.set(id, { result, revisedPrompt: revisedPrompt ?? existing?.revisedPrompt });
|
|
442
|
-
}
|
|
443
|
-
|
|
444
|
-
if (calls.size === 0) {
|
|
399
|
+
if (value.data.length === 0) {
|
|
445
400
|
return {
|
|
446
401
|
ok: false,
|
|
447
402
|
reason: "no-image",
|
|
448
403
|
errorMessage: "Image generation completed without a usable image result.",
|
|
449
404
|
};
|
|
450
405
|
}
|
|
451
|
-
if (
|
|
406
|
+
if (value.data.length !== 1) {
|
|
452
407
|
return {
|
|
453
408
|
ok: false,
|
|
454
409
|
reason: "malformed-response",
|
|
455
|
-
errorMessage: "Image generation returned more than one image
|
|
410
|
+
errorMessage: "Image generation returned more than one image for a single request.",
|
|
456
411
|
};
|
|
457
412
|
}
|
|
458
413
|
|
|
459
|
-
const
|
|
460
|
-
|
|
414
|
+
const item = value.data[0];
|
|
415
|
+
if (!isRecord(item) || typeof item.b64_json !== "string" || !item.b64_json.trim()) {
|
|
416
|
+
return {
|
|
417
|
+
ok: false,
|
|
418
|
+
reason: "no-image",
|
|
419
|
+
errorMessage: "Image generation response did not contain base64 image data.",
|
|
420
|
+
};
|
|
421
|
+
}
|
|
422
|
+
const decoded = decodeGeneratedPng(item.b64_json);
|
|
461
423
|
if (!decoded.ok) return decoded;
|
|
462
424
|
|
|
463
425
|
return {
|
|
464
426
|
ok: true,
|
|
465
427
|
image: {
|
|
466
428
|
bytes: decoded.bytes,
|
|
467
|
-
imageCallId
|
|
429
|
+
imageCallId:
|
|
430
|
+
normalizedOptionalString(item.id, MAX_IMAGE_IDENTIFIER_CHARS) ?? "image_generation_0",
|
|
468
431
|
responseId: normalizedOptionalString(value.id, MAX_IMAGE_IDENTIFIER_CHARS),
|
|
469
|
-
revisedPrompt:
|
|
432
|
+
revisedPrompt: normalizedOptionalString(item.revised_prompt, MAX_IMAGE_DIAGNOSTIC_CHARS),
|
|
470
433
|
width: decoded.width,
|
|
471
434
|
height: decoded.height,
|
|
472
435
|
},
|
|
@@ -21,7 +21,7 @@ import {
|
|
|
21
21
|
MAX_GENERATED_IMAGE_BYTES,
|
|
22
22
|
isImageGenerationDetails,
|
|
23
23
|
type GenerateImageParams,
|
|
24
|
-
type
|
|
24
|
+
type ImageGenerationToolDetails,
|
|
25
25
|
} from "./types";
|
|
26
26
|
|
|
27
27
|
export type ImageGenerationRenderState = Record<string, never>;
|
|
@@ -54,7 +54,7 @@ function isPathInsideOrEqual(root: string, candidate: string): boolean {
|
|
|
54
54
|
return rel === "" || (!rel.startsWith(`..${sep}`) && rel !== ".." && !isAbsolute(rel));
|
|
55
55
|
}
|
|
56
56
|
|
|
57
|
-
function resultText(result: AgentToolResult<
|
|
57
|
+
function resultText(result: AgentToolResult<ImageGenerationToolDetails>): string {
|
|
58
58
|
return result.content
|
|
59
59
|
.filter((block): block is { type: "text"; text: string } => block.type === "text")
|
|
60
60
|
.map((block) => block.text)
|
|
@@ -148,7 +148,7 @@ function textOnly(args: {
|
|
|
148
148
|
export function createImageGenerationResultRenderer(
|
|
149
149
|
deps: ImageRendererDependencies = DEFAULT_RENDER_DEPS,
|
|
150
150
|
): (
|
|
151
|
-
result: AgentToolResult<
|
|
151
|
+
result: AgentToolResult<ImageGenerationToolDetails>,
|
|
152
152
|
options: ToolRenderResultOptions,
|
|
153
153
|
theme: Theme,
|
|
154
154
|
context: ImageGenerationRenderContext,
|
|
@@ -1,255 +1,356 @@
|
|
|
1
|
-
import { getAgentDir, type ExtensionContext } from "@earendil-works/pi-coding-agent";
|
|
2
|
-
import { assertConfigValid, loadToolkitConfig, resolveToolkitConfig } from "../config";
|
|
3
|
-
import { notifyConfigIssues } from "../config/notifications";
|
|
4
|
-
import { resolveResponsesEnvironment } from "../runtime";
|
|
5
|
-
import {
|
|
6
|
-
copyImageToExplicitPath,
|
|
7
|
-
prepareExplicitOutputPath,
|
|
8
|
-
saveCanonicalImage,
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
import {
|
|
12
|
-
import {
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
type
|
|
29
|
-
type
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
case "
|
|
99
|
-
case "
|
|
100
|
-
case "
|
|
101
|
-
case "
|
|
102
|
-
case "
|
|
103
|
-
case "
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
}
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
})
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
|
|
155
|
-
|
|
156
|
-
|
|
157
|
-
|
|
158
|
-
|
|
159
|
-
|
|
160
|
-
|
|
161
|
-
|
|
162
|
-
|
|
163
|
-
|
|
164
|
-
|
|
165
|
-
|
|
166
|
-
|
|
167
|
-
|
|
168
|
-
|
|
169
|
-
|
|
170
|
-
|
|
171
|
-
|
|
172
|
-
|
|
173
|
-
|
|
174
|
-
|
|
175
|
-
|
|
176
|
-
|
|
177
|
-
|
|
178
|
-
|
|
179
|
-
|
|
180
|
-
|
|
181
|
-
|
|
182
|
-
|
|
183
|
-
|
|
184
|
-
|
|
185
|
-
|
|
186
|
-
|
|
187
|
-
|
|
188
|
-
|
|
189
|
-
|
|
190
|
-
|
|
191
|
-
|
|
192
|
-
|
|
193
|
-
|
|
194
|
-
|
|
195
|
-
|
|
196
|
-
|
|
197
|
-
|
|
198
|
-
|
|
199
|
-
|
|
200
|
-
|
|
201
|
-
|
|
202
|
-
|
|
203
|
-
|
|
204
|
-
})
|
|
205
|
-
|
|
206
|
-
|
|
207
|
-
|
|
208
|
-
|
|
209
|
-
|
|
210
|
-
|
|
211
|
-
|
|
212
|
-
|
|
213
|
-
|
|
214
|
-
|
|
215
|
-
|
|
216
|
-
|
|
217
|
-
|
|
218
|
-
|
|
219
|
-
|
|
220
|
-
|
|
221
|
-
|
|
222
|
-
|
|
223
|
-
|
|
224
|
-
|
|
225
|
-
|
|
226
|
-
|
|
227
|
-
|
|
228
|
-
|
|
229
|
-
|
|
230
|
-
|
|
231
|
-
|
|
232
|
-
|
|
233
|
-
|
|
234
|
-
|
|
235
|
-
|
|
236
|
-
|
|
237
|
-
|
|
238
|
-
|
|
239
|
-
|
|
240
|
-
|
|
241
|
-
|
|
242
|
-
|
|
243
|
-
|
|
244
|
-
|
|
245
|
-
|
|
246
|
-
|
|
247
|
-
|
|
248
|
-
|
|
249
|
-
|
|
250
|
-
|
|
251
|
-
|
|
252
|
-
|
|
253
|
-
|
|
254
|
-
|
|
255
|
-
|
|
1
|
+
import { getAgentDir, type ExtensionContext } from "@earendil-works/pi-coding-agent";
|
|
2
|
+
import { assertConfigValid, loadToolkitConfig, resolveToolkitConfig } from "../config";
|
|
3
|
+
import { notifyConfigIssues } from "../config/notifications";
|
|
4
|
+
import { resolveResponsesEnvironment } from "../runtime";
|
|
5
|
+
import {
|
|
6
|
+
copyImageToExplicitPath,
|
|
7
|
+
prepareExplicitOutputPath,
|
|
8
|
+
saveCanonicalImage,
|
|
9
|
+
type ExplicitOutputPlan,
|
|
10
|
+
} from "./artifacts";
|
|
11
|
+
import { requestGeneratedImage } from "./client";
|
|
12
|
+
import { isImageGenerationEnabledForModel } from "./eligibility";
|
|
13
|
+
import {
|
|
14
|
+
buildImageGenerationRequest,
|
|
15
|
+
normalizeGenerateImageParams,
|
|
16
|
+
selectImageGenerationModel,
|
|
17
|
+
} from "./protocol";
|
|
18
|
+
import {
|
|
19
|
+
clearPreparedReferences,
|
|
20
|
+
prepareReferenceImages,
|
|
21
|
+
} from "./references";
|
|
22
|
+
import {
|
|
23
|
+
IMAGE_GENERATION_CAPABLE_APIS,
|
|
24
|
+
IMAGE_GENERATION_MIME_TYPE,
|
|
25
|
+
MAX_IMAGE_IDENTIFIER_CHARS,
|
|
26
|
+
ImageGenerationError,
|
|
27
|
+
sanitizeImageDiagnostic,
|
|
28
|
+
type GenerateImageParams,
|
|
29
|
+
type ImageGenerationBatchDetails,
|
|
30
|
+
type ImageGenerationExecutionResult,
|
|
31
|
+
type ImageGenerationFailureCode,
|
|
32
|
+
type ImageGenerationDetails,
|
|
33
|
+
type ParsedGeneratedImage,
|
|
34
|
+
} from "./types";
|
|
35
|
+
|
|
36
|
+
export type ImageGenerationServiceDependencies = {
|
|
37
|
+
loadConfig: typeof loadToolkitConfig;
|
|
38
|
+
resolveRuntime: typeof resolveResponsesEnvironment;
|
|
39
|
+
getAgentDir: typeof getAgentDir;
|
|
40
|
+
prepareOutput: typeof prepareExplicitOutputPath;
|
|
41
|
+
prepareReferences: typeof prepareReferenceImages;
|
|
42
|
+
requestImage: typeof requestGeneratedImage;
|
|
43
|
+
saveCanonical: typeof saveCanonicalImage;
|
|
44
|
+
copyExplicit: typeof copyImageToExplicitPath;
|
|
45
|
+
clearReferences: typeof clearPreparedReferences;
|
|
46
|
+
};
|
|
47
|
+
|
|
48
|
+
const DEFAULT_SERVICE_DEPS: ImageGenerationServiceDependencies = {
|
|
49
|
+
loadConfig: loadToolkitConfig,
|
|
50
|
+
resolveRuntime: resolveResponsesEnvironment,
|
|
51
|
+
getAgentDir,
|
|
52
|
+
prepareOutput: prepareExplicitOutputPath,
|
|
53
|
+
prepareReferences: prepareReferenceImages,
|
|
54
|
+
requestImage: requestGeneratedImage,
|
|
55
|
+
saveCanonical: saveCanonicalImage,
|
|
56
|
+
copyExplicit: copyImageToExplicitPath,
|
|
57
|
+
clearReferences: clearPreparedReferences,
|
|
58
|
+
};
|
|
59
|
+
|
|
60
|
+
function runtimeFailure(
|
|
61
|
+
reason: string,
|
|
62
|
+
errorMessage?: string,
|
|
63
|
+
): ImageGenerationError {
|
|
64
|
+
switch (reason) {
|
|
65
|
+
case "disabled":
|
|
66
|
+
return new ImageGenerationError("disabled", "Image generation is disabled.");
|
|
67
|
+
case "unsupported-api":
|
|
68
|
+
case "missing-model":
|
|
69
|
+
return new ImageGenerationError(
|
|
70
|
+
"unsupported-model",
|
|
71
|
+
"Image generation is unavailable for the current model.",
|
|
72
|
+
);
|
|
73
|
+
case "missing-api-key":
|
|
74
|
+
case "auth-resolution-failed":
|
|
75
|
+
return new ImageGenerationError(
|
|
76
|
+
"authentication",
|
|
77
|
+
sanitizeImageDiagnostic(
|
|
78
|
+
errorMessage,
|
|
79
|
+
"Unable to resolve authentication for image generation with the current model.",
|
|
80
|
+
),
|
|
81
|
+
);
|
|
82
|
+
default:
|
|
83
|
+
return new ImageGenerationError(
|
|
84
|
+
"missing-runtime",
|
|
85
|
+
"Unable to resolve the current model's Images endpoint for image generation.",
|
|
86
|
+
);
|
|
87
|
+
}
|
|
88
|
+
}
|
|
89
|
+
|
|
90
|
+
function throwIfAborted(signal: AbortSignal | undefined): void {
|
|
91
|
+
if (signal?.aborted) {
|
|
92
|
+
throw new ImageGenerationError("aborted", "Image generation was cancelled.");
|
|
93
|
+
}
|
|
94
|
+
}
|
|
95
|
+
|
|
96
|
+
function clientFailureCode(reason: string): ImageGenerationFailureCode {
|
|
97
|
+
switch (reason) {
|
|
98
|
+
case "aborted":
|
|
99
|
+
case "timeout":
|
|
100
|
+
case "authentication":
|
|
101
|
+
case "rate-limit":
|
|
102
|
+
case "request-rejected":
|
|
103
|
+
case "backend-unavailable":
|
|
104
|
+
case "network":
|
|
105
|
+
case "oversized-response":
|
|
106
|
+
case "malformed-response":
|
|
107
|
+
case "no-image":
|
|
108
|
+
return reason;
|
|
109
|
+
default:
|
|
110
|
+
return "network";
|
|
111
|
+
}
|
|
112
|
+
}
|
|
113
|
+
|
|
114
|
+
function asImageGenerationError(value: unknown): ImageGenerationError {
|
|
115
|
+
if (value instanceof ImageGenerationError) {
|
|
116
|
+
return new ImageGenerationError(
|
|
117
|
+
value.code,
|
|
118
|
+
sanitizeImageDiagnostic(value.message, "Image generation failed."),
|
|
119
|
+
);
|
|
120
|
+
}
|
|
121
|
+
return new ImageGenerationError(
|
|
122
|
+
"network",
|
|
123
|
+
sanitizeImageDiagnostic(value instanceof Error ? value.message : value, "Image generation failed."),
|
|
124
|
+
);
|
|
125
|
+
}
|
|
126
|
+
|
|
127
|
+
function formatResultText(args: {
|
|
128
|
+
artifactPath: string;
|
|
129
|
+
outputPath?: string;
|
|
130
|
+
warning?: string;
|
|
131
|
+
edited: boolean;
|
|
132
|
+
width: number;
|
|
133
|
+
height: number;
|
|
134
|
+
}): string {
|
|
135
|
+
const lines = [
|
|
136
|
+
`${args.edited ? "Edited" : "Generated"} PNG image (${args.width}x${args.height}).`,
|
|
137
|
+
`Artifact: ${args.artifactPath}`,
|
|
138
|
+
];
|
|
139
|
+
if (args.outputPath) lines.push(`Copied to: ${args.outputPath}`);
|
|
140
|
+
if (args.warning) lines.push(`Warning: ${args.warning}`);
|
|
141
|
+
return lines.join("\n");
|
|
142
|
+
}
|
|
143
|
+
|
|
144
|
+
function formatBatchResultText(details: ImageGenerationBatchDetails): string {
|
|
145
|
+
const lines = [`Generated ${details.succeededCount}/${details.requestedCount} PNG images.`];
|
|
146
|
+
const failedIndexes = new Set((details.failures ?? []).map((failure) => failure.index));
|
|
147
|
+
let imageIndex = 0;
|
|
148
|
+
for (let batchIndex = 0; batchIndex < details.requestedCount; batchIndex += 1) {
|
|
149
|
+
if (failedIndexes.has(batchIndex)) continue;
|
|
150
|
+
const image = details.images[imageIndex++];
|
|
151
|
+
if (image) lines.push(`Image ${batchIndex + 1}: ${image.artifactPath}`);
|
|
152
|
+
}
|
|
153
|
+
for (const failure of details.failures ?? []) {
|
|
154
|
+
lines.push(`Image ${failure.index + 1} failed: ${failure.message}`);
|
|
155
|
+
}
|
|
156
|
+
return lines.join("\n");
|
|
157
|
+
}
|
|
158
|
+
|
|
159
|
+
function batchImageCallId(imageCallId: string, batchIndex: number, batchSize: number): string {
|
|
160
|
+
if (batchSize === 1) return imageCallId;
|
|
161
|
+
const suffix = `-batch-${batchIndex + 1}`;
|
|
162
|
+
return `${imageCallId.slice(0, Math.max(1, MAX_IMAGE_IDENTIFIER_CHARS - suffix.length))}${suffix}`;
|
|
163
|
+
}
|
|
164
|
+
|
|
165
|
+
async function persistGeneratedImage(args: {
|
|
166
|
+
generated: ParsedGeneratedImage;
|
|
167
|
+
params: ReturnType<typeof normalizeGenerateImageParams>;
|
|
168
|
+
batchIndex: number;
|
|
169
|
+
toolCallId: string;
|
|
170
|
+
batchSize: number;
|
|
171
|
+
imageModel: string;
|
|
172
|
+
routingModel: string;
|
|
173
|
+
agentDir: string;
|
|
174
|
+
sessionId: string;
|
|
175
|
+
referencesCount: number;
|
|
176
|
+
explicitOutput?: ExplicitOutputPlan;
|
|
177
|
+
deps: ImageGenerationServiceDependencies;
|
|
178
|
+
}): Promise<ImageGenerationDetails> {
|
|
179
|
+
const imageCallId = batchImageCallId(args.generated.imageCallId || args.toolCallId, args.batchIndex, args.batchSize);
|
|
180
|
+
try {
|
|
181
|
+
const artifactPath = await args.deps.saveCanonical({
|
|
182
|
+
bytes: args.generated.bytes,
|
|
183
|
+
agentDir: args.agentDir,
|
|
184
|
+
sessionId: args.sessionId,
|
|
185
|
+
imageCallId,
|
|
186
|
+
});
|
|
187
|
+
|
|
188
|
+
let outputPath: string | undefined;
|
|
189
|
+
let warning: string | undefined;
|
|
190
|
+
if (args.explicitOutput) {
|
|
191
|
+
try {
|
|
192
|
+
await args.deps.copyExplicit({ bytes: args.generated.bytes, plan: args.explicitOutput });
|
|
193
|
+
outputPath = args.explicitOutput.path;
|
|
194
|
+
} catch (error) {
|
|
195
|
+
warning = sanitizeImageDiagnostic(
|
|
196
|
+
error instanceof Error ? error.message : error,
|
|
197
|
+
`The generated image was saved to the canonical artifact, but copying to ${args.explicitOutput.path} failed.`,
|
|
198
|
+
);
|
|
199
|
+
}
|
|
200
|
+
}
|
|
201
|
+
|
|
202
|
+
return {
|
|
203
|
+
artifactPath,
|
|
204
|
+
...(outputPath ? { outputPath } : {}),
|
|
205
|
+
routingModel: args.routingModel,
|
|
206
|
+
imageModel: args.imageModel,
|
|
207
|
+
imageCallId,
|
|
208
|
+
...(args.generated.responseId ? { responseId: args.generated.responseId } : {}),
|
|
209
|
+
mimeType: IMAGE_GENERATION_MIME_TYPE,
|
|
210
|
+
byteCount: args.generated.bytes.length,
|
|
211
|
+
width: args.generated.width,
|
|
212
|
+
height: args.generated.height,
|
|
213
|
+
edited: args.params.action === "edit",
|
|
214
|
+
referenceCount: args.referencesCount,
|
|
215
|
+
...(args.generated.revisedPrompt ? { revisedPrompt: args.generated.revisedPrompt } : {}),
|
|
216
|
+
...(warning ? { warning } : {}),
|
|
217
|
+
};
|
|
218
|
+
} finally {
|
|
219
|
+
args.generated.bytes.fill(0);
|
|
220
|
+
}
|
|
221
|
+
}
|
|
222
|
+
|
|
223
|
+
export function createImageGenerationExecutor(
|
|
224
|
+
deps: ImageGenerationServiceDependencies = DEFAULT_SERVICE_DEPS,
|
|
225
|
+
): (args: {
|
|
226
|
+
params: GenerateImageParams;
|
|
227
|
+
toolCallId: string;
|
|
228
|
+
signal?: AbortSignal;
|
|
229
|
+
ctx: ExtensionContext;
|
|
230
|
+
}) => Promise<ImageGenerationExecutionResult> {
|
|
231
|
+
return async (args) => {
|
|
232
|
+
const params = normalizeGenerateImageParams(args.params);
|
|
233
|
+
throwIfAborted(args.signal);
|
|
234
|
+
const resolved = resolveToolkitConfig(deps.loadConfig(), args.ctx.model);
|
|
235
|
+
notifyConfigIssues(args.ctx, resolved);
|
|
236
|
+
assertConfigValid(resolved, "imageGeneration", "compatibility");
|
|
237
|
+
const { config } = resolved;
|
|
238
|
+
if (!isImageGenerationEnabledForModel(args.ctx.model, config.imageGeneration)) {
|
|
239
|
+
throw new ImageGenerationError(
|
|
240
|
+
"unsupported-model",
|
|
241
|
+
"Image generation is not enabled for the current provider/model-id.",
|
|
242
|
+
);
|
|
243
|
+
}
|
|
244
|
+
|
|
245
|
+
const imageModel = selectImageGenerationModel({
|
|
246
|
+
requestedModel: params.model,
|
|
247
|
+
configuredModels: resolved.policy.imageGeneration.allowedModels,
|
|
248
|
+
defaultModel: resolved.policy.imageGeneration.defaultModel,
|
|
249
|
+
});
|
|
250
|
+
|
|
251
|
+
const runtimeResolution = await deps.resolveRuntime(args.ctx, {
|
|
252
|
+
enabled: config.imageGeneration.enabled,
|
|
253
|
+
responsesApis: IMAGE_GENERATION_CAPABLE_APIS,
|
|
254
|
+
...(resolved.format === "v2" ? { codexGatewayModels: resolved.gatewayModelKeys } : {}),
|
|
255
|
+
});
|
|
256
|
+
if (!runtimeResolution.ok) {
|
|
257
|
+
throw runtimeFailure(runtimeResolution.reason, runtimeResolution.errorMessage);
|
|
258
|
+
}
|
|
259
|
+
|
|
260
|
+
const agentDir = deps.getAgentDir();
|
|
261
|
+
const explicitOutput = await deps.prepareOutput({
|
|
262
|
+
rawPath: params.outputPath,
|
|
263
|
+
agentDir,
|
|
264
|
+
ctx: args.ctx,
|
|
265
|
+
signal: args.signal,
|
|
266
|
+
});
|
|
267
|
+
|
|
268
|
+
const references = await deps.prepareReferences({
|
|
269
|
+
paths: params.referenceImagePaths,
|
|
270
|
+
ctx: args.ctx,
|
|
271
|
+
signal: args.signal,
|
|
272
|
+
});
|
|
273
|
+
try {
|
|
274
|
+
const jobs = Array.from({ length: params.batchSize }, (_, batchIndex) =>
|
|
275
|
+
(async () => {
|
|
276
|
+
throwIfAborted(args.signal);
|
|
277
|
+
const body = buildImageGenerationRequest({
|
|
278
|
+
imageModel,
|
|
279
|
+
params,
|
|
280
|
+
references,
|
|
281
|
+
});
|
|
282
|
+
const response = await deps.requestImage({
|
|
283
|
+
runtime: runtimeResolution.runtime,
|
|
284
|
+
body,
|
|
285
|
+
signal: args.signal,
|
|
286
|
+
});
|
|
287
|
+
if (!response.ok) {
|
|
288
|
+
throw new ImageGenerationError(
|
|
289
|
+
clientFailureCode(response.reason),
|
|
290
|
+
sanitizeImageDiagnostic(response.errorMessage, "Image generation request failed."),
|
|
291
|
+
);
|
|
292
|
+
}
|
|
293
|
+
return persistGeneratedImage({
|
|
294
|
+
generated: response.image,
|
|
295
|
+
params,
|
|
296
|
+
batchIndex,
|
|
297
|
+
toolCallId: args.toolCallId,
|
|
298
|
+
batchSize: params.batchSize,
|
|
299
|
+
imageModel,
|
|
300
|
+
routingModel: `${runtimeResolution.runtime.provider}/${runtimeResolution.runtime.model}`,
|
|
301
|
+
agentDir,
|
|
302
|
+
sessionId: args.ctx.sessionManager.getSessionId(),
|
|
303
|
+
referencesCount: references.length,
|
|
304
|
+
explicitOutput,
|
|
305
|
+
deps,
|
|
306
|
+
});
|
|
307
|
+
})(),
|
|
308
|
+
);
|
|
309
|
+
const settled = await Promise.allSettled(jobs);
|
|
310
|
+
const images: ImageGenerationDetails[] = [];
|
|
311
|
+
const failures: ImageGenerationBatchDetails["failures"] = [];
|
|
312
|
+
for (const [index, result] of settled.entries()) {
|
|
313
|
+
if (result.status === "fulfilled") {
|
|
314
|
+
images.push(result.value);
|
|
315
|
+
} else {
|
|
316
|
+
const error = asImageGenerationError(result.reason);
|
|
317
|
+
failures.push({ index, code: error.code, message: error.message });
|
|
318
|
+
}
|
|
319
|
+
}
|
|
320
|
+
|
|
321
|
+
if (images.length === 0) {
|
|
322
|
+
const firstFailure = settled.find((result) => result.status === "rejected");
|
|
323
|
+
throw asImageGenerationError(firstFailure && firstFailure.status === "rejected" ? firstFailure.reason : "Image generation failed.");
|
|
324
|
+
}
|
|
325
|
+
|
|
326
|
+
if (params.batchSize === 1) {
|
|
327
|
+
const details = images[0]!;
|
|
328
|
+
return {
|
|
329
|
+
details,
|
|
330
|
+
text: formatResultText({
|
|
331
|
+
artifactPath: details.artifactPath,
|
|
332
|
+
outputPath: details.outputPath,
|
|
333
|
+
warning: details.warning,
|
|
334
|
+
edited: details.edited,
|
|
335
|
+
width: details.width,
|
|
336
|
+
height: details.height,
|
|
337
|
+
}),
|
|
338
|
+
};
|
|
339
|
+
}
|
|
340
|
+
|
|
341
|
+
const batchDetails: ImageGenerationBatchDetails = {
|
|
342
|
+
batch: true,
|
|
343
|
+
requestedCount: params.batchSize,
|
|
344
|
+
succeededCount: images.length,
|
|
345
|
+
failedCount: failures.length,
|
|
346
|
+
images,
|
|
347
|
+
...(failures.length > 0 ? { failures } : {}),
|
|
348
|
+
};
|
|
349
|
+
return { details: batchDetails, text: formatBatchResultText(batchDetails) };
|
|
350
|
+
} finally {
|
|
351
|
+
deps.clearReferences(references);
|
|
352
|
+
}
|
|
353
|
+
};
|
|
354
|
+
}
|
|
355
|
+
|
|
356
|
+
export const executeImageGeneration = createImageGenerationExecutor();
|
|
@@ -23,8 +23,9 @@ export const IMAGE_REQUEST_TIMEOUT_MS = 5 * 60 * 1000;
|
|
|
23
23
|
export const MAX_IMAGE_PATH_CHARS = 4096;
|
|
24
24
|
export const MAX_IMAGE_DIAGNOSTIC_CHARS = 4096;
|
|
25
25
|
export const MAX_IMAGE_IDENTIFIER_CHARS = 256;
|
|
26
|
-
/** Upper bound for a configured or requested
|
|
26
|
+
/** Upper bound for a configured or requested Images API model id. */
|
|
27
27
|
export const MAX_IMAGE_MODEL_ID_CHARS = 256;
|
|
28
|
+
export const MAX_IMAGE_BATCH_SIZE = 10;
|
|
28
29
|
export const MAX_IMAGE_DIMENSION = 100_000;
|
|
29
30
|
|
|
30
31
|
export type ImageGenerationCapableApi = (typeof IMAGE_GENERATION_CAPABLE_APIS)[number];
|
|
@@ -45,8 +46,10 @@ export type GenerateImageParams = {
|
|
|
45
46
|
outputPath?: string | null;
|
|
46
47
|
size?: ImageGenerationSize;
|
|
47
48
|
quality?: ImageGenerationQuality;
|
|
48
|
-
/** Optional bare image model id that must exist in
|
|
49
|
+
/** Optional bare image model id that must exist in the configured image model allowlist. */
|
|
49
50
|
model?: string | null;
|
|
51
|
+
/** Number of independent image requests to run concurrently. */
|
|
52
|
+
batchSize?: number | null;
|
|
50
53
|
};
|
|
51
54
|
|
|
52
55
|
export type NormalizedGenerateImageParams = {
|
|
@@ -56,6 +59,7 @@ export type NormalizedGenerateImageParams = {
|
|
|
56
59
|
size: ImageGenerationSize;
|
|
57
60
|
quality: ImageGenerationQuality;
|
|
58
61
|
action: ImageGenerationAction;
|
|
62
|
+
batchSize: number;
|
|
59
63
|
/** Trimmed requested model id, or undefined when the caller omitted it. */
|
|
60
64
|
model?: string;
|
|
61
65
|
};
|
|
@@ -92,8 +96,25 @@ export type ImageGenerationDetails = {
|
|
|
92
96
|
warning?: string;
|
|
93
97
|
};
|
|
94
98
|
|
|
99
|
+
export type ImageGenerationBatchFailure = {
|
|
100
|
+
index: number;
|
|
101
|
+
code: ImageGenerationFailureCode;
|
|
102
|
+
message: string;
|
|
103
|
+
};
|
|
104
|
+
|
|
105
|
+
export type ImageGenerationBatchDetails = {
|
|
106
|
+
batch: true;
|
|
107
|
+
requestedCount: number;
|
|
108
|
+
succeededCount: number;
|
|
109
|
+
failedCount: number;
|
|
110
|
+
images: ImageGenerationDetails[];
|
|
111
|
+
failures?: ImageGenerationBatchFailure[];
|
|
112
|
+
};
|
|
113
|
+
|
|
114
|
+
export type ImageGenerationToolDetails = ImageGenerationDetails | ImageGenerationBatchDetails;
|
|
115
|
+
|
|
95
116
|
export type ImageGenerationExecutionResult = {
|
|
96
|
-
details:
|
|
117
|
+
details: ImageGenerationToolDetails;
|
|
97
118
|
text: string;
|
|
98
119
|
};
|
|
99
120
|
|
|
@@ -120,6 +141,35 @@ export type ImageGenerationFailureCode =
|
|
|
120
141
|
| "no-image"
|
|
121
142
|
| "artifact-write-failed";
|
|
122
143
|
|
|
144
|
+
function isImageGenerationFailureCode(value: unknown): value is ImageGenerationFailureCode {
|
|
145
|
+
switch (value) {
|
|
146
|
+
case "aborted":
|
|
147
|
+
case "timeout":
|
|
148
|
+
case "disabled":
|
|
149
|
+
case "unsupported-model":
|
|
150
|
+
case "missing-runtime":
|
|
151
|
+
case "invalid-parameters":
|
|
152
|
+
case "reference-input-invalid":
|
|
153
|
+
case "reference-upload-approval-required":
|
|
154
|
+
case "reference-upload-declined":
|
|
155
|
+
case "output-path-invalid":
|
|
156
|
+
case "output-path-approval-required":
|
|
157
|
+
case "output-path-declined":
|
|
158
|
+
case "authentication":
|
|
159
|
+
case "rate-limit":
|
|
160
|
+
case "request-rejected":
|
|
161
|
+
case "backend-unavailable":
|
|
162
|
+
case "network":
|
|
163
|
+
case "oversized-response":
|
|
164
|
+
case "malformed-response":
|
|
165
|
+
case "no-image":
|
|
166
|
+
case "artifact-write-failed":
|
|
167
|
+
return true;
|
|
168
|
+
default:
|
|
169
|
+
return false;
|
|
170
|
+
}
|
|
171
|
+
}
|
|
172
|
+
|
|
123
173
|
export class ImageGenerationError extends Error {
|
|
124
174
|
readonly code: ImageGenerationFailureCode;
|
|
125
175
|
|
|
@@ -170,6 +220,47 @@ export function isImageGenerationDetails(value: unknown): value is ImageGenerati
|
|
|
170
220
|
);
|
|
171
221
|
}
|
|
172
222
|
|
|
223
|
+
function isBoundedIndex(value: unknown, maximum: number): value is number {
|
|
224
|
+
return typeof value === "number" && Number.isInteger(value) && value >= 0 && value < maximum;
|
|
225
|
+
}
|
|
226
|
+
|
|
227
|
+
export function isImageGenerationBatchDetails(value: unknown): value is ImageGenerationBatchDetails {
|
|
228
|
+
if (!isRecord(value) || value.batch !== true) return false;
|
|
229
|
+
if (
|
|
230
|
+
typeof value.requestedCount !== "number" ||
|
|
231
|
+
!Number.isInteger(value.requestedCount) ||
|
|
232
|
+
value.requestedCount < 2 ||
|
|
233
|
+
value.requestedCount > MAX_IMAGE_BATCH_SIZE ||
|
|
234
|
+
typeof value.succeededCount !== "number" ||
|
|
235
|
+
!Number.isInteger(value.succeededCount) ||
|
|
236
|
+
value.succeededCount < 1 ||
|
|
237
|
+
value.succeededCount > value.requestedCount ||
|
|
238
|
+
typeof value.failedCount !== "number" ||
|
|
239
|
+
!Number.isInteger(value.failedCount) ||
|
|
240
|
+
value.failedCount !== value.requestedCount - value.succeededCount ||
|
|
241
|
+
!Array.isArray(value.images) ||
|
|
242
|
+
value.images.length !== value.succeededCount ||
|
|
243
|
+
!value.images.every(isImageGenerationDetails)
|
|
244
|
+
) {
|
|
245
|
+
return false;
|
|
246
|
+
}
|
|
247
|
+
if (value.failedCount === 0) return value.failures === undefined;
|
|
248
|
+
if (!Array.isArray(value.failures) || value.failures.length !== value.failedCount) return false;
|
|
249
|
+
const requestedCount = value.requestedCount;
|
|
250
|
+
const indexes = new Set<number>();
|
|
251
|
+
return value.failures.every((failure) => {
|
|
252
|
+
if (!isRecord(failure) || !isBoundedIndex(failure.index, requestedCount)) return false;
|
|
253
|
+
if (!isImageGenerationFailureCode(failure.code) || !isBoundedString(failure.message, MAX_IMAGE_DIAGNOSTIC_CHARS)) return false;
|
|
254
|
+
if (indexes.has(failure.index)) return false;
|
|
255
|
+
indexes.add(failure.index);
|
|
256
|
+
return true;
|
|
257
|
+
});
|
|
258
|
+
}
|
|
259
|
+
|
|
260
|
+
export function isImageGenerationToolDetails(value: unknown): value is ImageGenerationToolDetails {
|
|
261
|
+
return isImageGenerationDetails(value) || isImageGenerationBatchDetails(value);
|
|
262
|
+
}
|
|
263
|
+
|
|
173
264
|
export function sanitizeImageDiagnostic(value: unknown, fallback: string): string {
|
|
174
265
|
if (typeof value !== "string") return fallback;
|
|
175
266
|
const normalized = value
|
package/src/responses-headers.ts
CHANGED
|
@@ -107,7 +107,8 @@ export function clampCodexSessionId(sessionId: string | undefined): string | und
|
|
|
107
107
|
|
|
108
108
|
export type ResponsesHeaderOptions = {
|
|
109
109
|
accept: string;
|
|
110
|
-
|
|
110
|
+
/** `null` omits content-type so a multipart FormData body can set its boundary. */
|
|
111
|
+
contentType?: string | null;
|
|
111
112
|
/** Conversation identity for Codex cache-affinity headers; clamped to 64 chars here. */
|
|
112
113
|
sessionId?: string;
|
|
113
114
|
};
|
|
@@ -121,7 +122,11 @@ export function buildResponsesRequestHeaders(
|
|
|
121
122
|
runtime.codexAffinity ? filterCodexGatewayHeaders(mergedHeaders) : mergedHeaders,
|
|
122
123
|
);
|
|
123
124
|
headers.set("accept", options.accept);
|
|
124
|
-
|
|
125
|
+
if (options.contentType === null) {
|
|
126
|
+
headers.delete("content-type");
|
|
127
|
+
} else {
|
|
128
|
+
headers.set("content-type", options.contentType ?? "application/json");
|
|
129
|
+
}
|
|
125
130
|
if (!headers.has("authorization")) {
|
|
126
131
|
headers.set("authorization", `Bearer ${runtime.apiKey}`);
|
|
127
132
|
}
|
package/src/types.ts
CHANGED
|
@@ -151,9 +151,9 @@ export type WebSearchConfig = {
|
|
|
151
151
|
export type ImageGenerationConfig = {
|
|
152
152
|
enabled: boolean;
|
|
153
153
|
/**
|
|
154
|
-
* Bare
|
|
155
|
-
* The first entry is the default; `openai_generate_image`'s optional `model` argument
|
|
156
|
-
* select any other entry in this order-preserving list.
|
|
154
|
+
* Bare Images API model ids this toolkit may send, without a `provider/` prefix.
|
|
155
|
+
* The first legacy entry is the default; `openai_generate_image`'s optional `model` argument
|
|
156
|
+
* may select any other entry in this order-preserving list.
|
|
157
157
|
*/
|
|
158
158
|
models: string[];
|
|
159
159
|
};
|