pi-openai-toolkit 0.17.2 → 0.18.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +2 -2
- package/README.zh.md +2 -2
- package/docs/configuration.md +1 -1
- package/package.json +1 -1
- package/src/codex-astra/extension.ts +0 -15
- package/src/context-management/codex-provider.ts +0 -1
- package/src/image-generation/client.ts +70 -8
- package/src/image-generation/eligibility.ts +3 -3
- package/src/image-generation/extension.ts +18 -4
- package/src/image-generation/protocol.ts +57 -94
- package/src/image-generation/render.ts +3 -3
- package/src/image-generation/service.ts +356 -255
- package/src/image-generation/types.ts +94 -3
- package/src/responses-headers.ts +10 -9
- package/src/types.ts +3 -3
package/README.md
CHANGED
|
@@ -168,7 +168,7 @@ Toolkit does not fall back between routes. See [search configuration](docs/confi
|
|
|
168
168
|
|
|
169
169
|
### Generate an image
|
|
170
170
|
|
|
171
|
-
Requires
|
|
171
|
+
Requires an eligible Responses-capable session and may incur provider charges. The local Pi tool sends a separate Images API request to the current provider (`/images/generations` or `/images/edits`; Codex uses `/codex/images/...`) rather than adding an image tool to the conversation request. Enable it with an image model your provider supports:
|
|
172
172
|
|
|
173
173
|
```json
|
|
174
174
|
{
|
|
@@ -183,7 +183,7 @@ Requires a Responses session and may incur provider charges. Enable it with an i
|
|
|
183
183
|
}
|
|
184
184
|
```
|
|
185
185
|
|
|
186
|
-
Ask Pi to generate an image or edit explicitly supplied local references using `openai_generate_image`. The
|
|
186
|
+
Ask Pi to generate an image or edit explicitly supplied local references using `openai_generate_image`. The selected image model is sent in the independent request; the active conversation model remains the routing/auth model. Set the tool's optional `model` to another exact `allowedModels` entry, or use `batchSize` (1-10) for concurrent independent variants. Batch calls save canonical artifacts and cannot use a single-file `outputPath`; image settings are global. See [image configuration](docs/configuration.md#images).
|
|
187
187
|
|
|
188
188
|
### Review tool calls automatically
|
|
189
189
|
|
package/README.zh.md
CHANGED
|
@@ -168,7 +168,7 @@ Toolkit 不会在路由之间自动回退。后端要求见[搜索配置](docs/c
|
|
|
168
168
|
|
|
169
169
|
### 生成图片
|
|
170
170
|
|
|
171
|
-
|
|
171
|
+
需要符合条件的 Responses-capable 会话,并可能产生服务商费用。Pi 工具会向当前服务商发送独立的 Images API 请求(`/images/generations` 或 `/images/edits`;Codex 使用 `/codex/images/...`),不会把生图 tool 加入普通对话请求。选择服务商支持的生图模型后启用:
|
|
172
172
|
|
|
173
173
|
```json
|
|
174
174
|
{
|
|
@@ -183,7 +183,7 @@ Toolkit 不会在路由之间自动回退。后端要求见[搜索配置](docs/c
|
|
|
183
183
|
}
|
|
184
184
|
```
|
|
185
185
|
|
|
186
|
-
让 Pi 通过 `openai_generate_image`
|
|
186
|
+
让 Pi 通过 `openai_generate_image` 生成图片,或编辑明确提供的本地参考图。选中的生图模型会写入独立请求;当前对话模型仍只负责路由和认证。工具的可选 `model` 必须精确匹配 `allowedModels`,也可以使用 `batchSize`(1-10)并发生成多个独立变体。批量调用会保存 canonical artifacts,不能同时指定单文件 `outputPath`;生图设置全局生效。详见[生图配置](docs/configuration.md#images)。
|
|
187
187
|
|
|
188
188
|
### 启用工具调用自动审查
|
|
189
189
|
|
package/docs/configuration.md
CHANGED
|
@@ -104,7 +104,7 @@ There is no v2 search master switch. `local` and `unmanaged` differ in payload c
|
|
|
104
104
|
|
|
105
105
|
Image settings are global-only under `defaults.imageGeneration`. `enabled` defaults to `false`. `defaultModel` defaults to `"gpt-image-2.5"`; `allowedModels` defaults to `["gpt-image-2.5"]`.
|
|
106
106
|
|
|
107
|
-
These are bare output-model IDs, not active-session model keys. Each normalized ID must contain 1-256 characters. The allowed list must remain nonempty after normalization and contain the explicit default.
|
|
107
|
+
These are bare output-model IDs, not active-session model keys. Each normalized ID must contain 1-256 characters. The allowed list must remain nonempty after normalization and contain the explicit default. A caller's one-call `model` must be a member; policy and membership are checked before auth, local-reference preparation/upload, and paid dispatch. The active session remains the routing/auth model, while the selected image model is sent in a separate Images API request. Generation uses `/images/generations`; edits use `/images/edits` (Codex uses the corresponding `/codex/images/...` paths). The wrapper never injects a Responses `image_generation` tool. `batchSize` is a tool argument from 1-10; each item is an independent `n: 1` request, partial success preserves saved artifacts, and batch calls cannot use a single-file `outputPath`. Config does not grant permission to upload arbitrary local files or imply that a provider supports a particular image model.
|
|
108
108
|
|
|
109
109
|
---
|
|
110
110
|
|
package/package.json
CHANGED
|
@@ -1,6 +1,5 @@
|
|
|
1
1
|
import type { ExtensionAPI } from "@earendil-works/pi-coding-agent";
|
|
2
2
|
import { ASTRA_MODEL_ID } from "../types";
|
|
3
|
-
import { CODEX_CLIENT_VERSION } from "../responses-headers";
|
|
4
3
|
import {
|
|
5
4
|
getEffortControlState,
|
|
6
5
|
planStableEffort,
|
|
@@ -101,20 +100,6 @@ export function registerCodexAstraExtension(pi: ExtensionAPI): void {
|
|
|
101
100
|
return undefined;
|
|
102
101
|
}
|
|
103
102
|
});
|
|
104
|
-
|
|
105
|
-
pi.on("before_provider_headers", (event, ctx) => {
|
|
106
|
-
try {
|
|
107
|
-
const model = ctx.model;
|
|
108
|
-
if (!model || model.api !== "openai-codex-responses") return;
|
|
109
|
-
// Live requests need the same backend version gate as our synthetic
|
|
110
|
-
// ones: an old or missing `version` never reaches gated SKUs.
|
|
111
|
-
if (!event.headers.version) {
|
|
112
|
-
event.headers.version = CODEX_CLIENT_VERSION;
|
|
113
|
-
}
|
|
114
|
-
} catch {
|
|
115
|
-
// Header injection is best-effort.
|
|
116
|
-
}
|
|
117
|
-
});
|
|
118
103
|
}
|
|
119
104
|
|
|
120
105
|
export default function codexAstraExtension(pi: ExtensionAPI): void {
|
|
@@ -146,7 +146,6 @@ export function codexContextProviderHeaders(
|
|
|
146
146
|
headers.set("ChatGPT-Account-ID", provider.accountId);
|
|
147
147
|
headers.set("originator", CODEX_TOOL_ORIGINATOR);
|
|
148
148
|
headers.set("User-Agent", userAgent(CODEX_TOOL_VERSION));
|
|
149
|
-
headers.set("version", CODEX_TOOL_VERSION);
|
|
150
149
|
} else {
|
|
151
150
|
headers.set("Authorization", `Bearer ${provider.apiKey}`);
|
|
152
151
|
headers.delete("ChatGPT-Account-ID");
|
|
@@ -1,10 +1,11 @@
|
|
|
1
1
|
import { buildResponsesRequestHeaders } from "../responses-headers";
|
|
2
|
-
import type
|
|
2
|
+
import { normalizeBaseUrl, type ResponsesRuntime } from "../runtime";
|
|
3
3
|
import {
|
|
4
4
|
extractProviderErrorMessage,
|
|
5
|
-
|
|
5
|
+
parseImagesApiResponse,
|
|
6
6
|
type ImageGenerationRequestBody,
|
|
7
7
|
} from "./protocol";
|
|
8
|
+
import type { PreparedReferenceImage } from "./types";
|
|
8
9
|
import {
|
|
9
10
|
IMAGE_REQUEST_TIMEOUT_MS,
|
|
10
11
|
MAX_IMAGE_ERROR_BYTES,
|
|
@@ -39,8 +40,62 @@ export type ImageGenerationFetch = (
|
|
|
39
40
|
init?: RequestInit,
|
|
40
41
|
) => Promise<Response>;
|
|
41
42
|
|
|
42
|
-
function buildRequestHeaders(runtime: ResponsesRuntime): Headers {
|
|
43
|
-
return buildResponsesRequestHeaders(runtime, { accept: "application/json" });
|
|
43
|
+
function buildRequestHeaders(runtime: ResponsesRuntime, contentType: string | null = "application/json"): Headers {
|
|
44
|
+
return buildResponsesRequestHeaders(runtime, { accept: "application/json", contentType });
|
|
45
|
+
}
|
|
46
|
+
|
|
47
|
+
function buildImagesEndpoint(runtime: ResponsesRuntime, action: ImageGenerationRequestBody["action"]): string {
|
|
48
|
+
const baseUrl = normalizeBaseUrl(runtime.baseUrl);
|
|
49
|
+
if (!baseUrl) throw new Error("Image generation requires a provider base URL.");
|
|
50
|
+
if (baseUrl.endsWith("/responses") || baseUrl.endsWith("/codex/responses")) {
|
|
51
|
+
throw new Error("Image generation requires the provider base URL, not a resolved Responses URL.");
|
|
52
|
+
}
|
|
53
|
+
const suffix = runtime.api === "openai-codex-responses" ? "codex/images" : "images";
|
|
54
|
+
const baseAlreadyIncludesCodex = runtime.api === "openai-codex-responses" && baseUrl.endsWith("/codex");
|
|
55
|
+
return `${baseUrl}/${baseAlreadyIncludesCodex ? "images" : suffix}/${action === "edit" ? "edits" : "generations"}`;
|
|
56
|
+
}
|
|
57
|
+
|
|
58
|
+
function dataUrl(reference: PreparedReferenceImage): string {
|
|
59
|
+
return `data:${reference.mimeType};base64,${reference.bytes.toString("base64")}`;
|
|
60
|
+
}
|
|
61
|
+
|
|
62
|
+
function jsonRequestBody(runtime: ResponsesRuntime, body: ImageGenerationRequestBody): Record<string, unknown> {
|
|
63
|
+
const common = {
|
|
64
|
+
model: body.model,
|
|
65
|
+
prompt: body.prompt,
|
|
66
|
+
n: body.n,
|
|
67
|
+
size: body.size,
|
|
68
|
+
quality: body.quality,
|
|
69
|
+
};
|
|
70
|
+
if (body.action === "edit") {
|
|
71
|
+
return {
|
|
72
|
+
...common,
|
|
73
|
+
images: body.references.map((reference) => ({ image_url: dataUrl(reference) })),
|
|
74
|
+
};
|
|
75
|
+
}
|
|
76
|
+
return runtime.api === "openai-codex-responses"
|
|
77
|
+
? common
|
|
78
|
+
: { ...common, output_format: "png", response_format: "b64_json" };
|
|
79
|
+
}
|
|
80
|
+
|
|
81
|
+
function multipartRequestBody(body: ImageGenerationRequestBody): FormData {
|
|
82
|
+
const form = new FormData();
|
|
83
|
+
form.append("model", body.model);
|
|
84
|
+
form.append("prompt", body.prompt);
|
|
85
|
+
form.append("n", String(body.n));
|
|
86
|
+
form.append("size", body.size);
|
|
87
|
+
form.append("quality", body.quality);
|
|
88
|
+
form.append("output_format", "png");
|
|
89
|
+
form.append("response_format", "b64_json");
|
|
90
|
+
for (const [index, reference] of body.references.entries()) {
|
|
91
|
+
const extension = reference.mimeType === "image/jpeg" ? "jpg" : reference.mimeType.slice("image/".length);
|
|
92
|
+
form.append(
|
|
93
|
+
"image",
|
|
94
|
+
new Blob([new Uint8Array(reference.bytes)], { type: reference.mimeType }),
|
|
95
|
+
`reference-${index + 1}.${extension}`,
|
|
96
|
+
);
|
|
97
|
+
}
|
|
98
|
+
return form;
|
|
44
99
|
}
|
|
45
100
|
|
|
46
101
|
async function readBoundedBody(
|
|
@@ -141,12 +196,16 @@ export async function requestGeneratedImage(args: {
|
|
|
141
196
|
? AbortSignal.any([args.signal, timeoutSignal])
|
|
142
197
|
: timeoutSignal;
|
|
143
198
|
|
|
199
|
+
const multipart = args.body.action === "edit" && args.runtime.api === "openai-responses";
|
|
200
|
+
const transportBody: BodyInit = multipart
|
|
201
|
+
? multipartRequestBody(args.body)
|
|
202
|
+
: JSON.stringify(jsonRequestBody(args.runtime, args.body));
|
|
144
203
|
let response: Response;
|
|
145
204
|
try {
|
|
146
|
-
response = await fetchFn(args.runtime.
|
|
205
|
+
response = await fetchFn(buildImagesEndpoint(args.runtime, args.body.action), {
|
|
147
206
|
method: "POST",
|
|
148
|
-
headers: buildRequestHeaders(args.runtime),
|
|
149
|
-
body:
|
|
207
|
+
headers: buildRequestHeaders(args.runtime, multipart ? null : "application/json"),
|
|
208
|
+
body: transportBody,
|
|
150
209
|
signal,
|
|
151
210
|
});
|
|
152
211
|
} catch (error) {
|
|
@@ -230,7 +289,7 @@ export async function requestGeneratedImage(args: {
|
|
|
230
289
|
};
|
|
231
290
|
}
|
|
232
291
|
|
|
233
|
-
const parsed =
|
|
292
|
+
const parsed = parseImagesApiResponse(payload);
|
|
234
293
|
if (!parsed.ok) {
|
|
235
294
|
return {
|
|
236
295
|
ok: false,
|
|
@@ -244,5 +303,8 @@ export async function requestGeneratedImage(args: {
|
|
|
244
303
|
|
|
245
304
|
export const _clientTest = {
|
|
246
305
|
buildRequestHeaders,
|
|
306
|
+
buildImagesEndpoint,
|
|
307
|
+
jsonRequestBody,
|
|
308
|
+
multipartRequestBody,
|
|
247
309
|
readBoundedBody,
|
|
248
310
|
};
|
|
@@ -2,9 +2,9 @@ import type { ImageGenerationConfig } from "../types";
|
|
|
2
2
|
import { IMAGE_GENERATION_CAPABLE_APIS, type ImageGenerationModel } from "./types";
|
|
3
3
|
|
|
4
4
|
/**
|
|
5
|
-
* The
|
|
6
|
-
* specific model
|
|
7
|
-
*
|
|
5
|
+
* The independent Images API is exposed only for Responses-capable runtimes rather than for a
|
|
6
|
+
* specific conversation model. `enabled` is the single opt-in switch and defaults to off because
|
|
7
|
+
* every successful provider request may incur a charge.
|
|
8
8
|
*/
|
|
9
9
|
export function isImageGenerationEnabledForModel(
|
|
10
10
|
model: ImageGenerationModel | undefined,
|
|
@@ -9,13 +9,14 @@ import {
|
|
|
9
9
|
IMAGE_GENERATION_QUALITIES,
|
|
10
10
|
IMAGE_GENERATION_SIZES,
|
|
11
11
|
IMAGE_GENERATION_TOOL_NAME,
|
|
12
|
+
MAX_IMAGE_BATCH_SIZE,
|
|
12
13
|
MAX_IMAGE_MODEL_ID_CHARS,
|
|
13
14
|
MAX_IMAGE_PATH_CHARS,
|
|
14
15
|
MAX_IMAGE_PROMPT_CHARS,
|
|
15
16
|
MAX_REFERENCE_IMAGE_COUNT,
|
|
16
17
|
ImageGenerationError,
|
|
17
18
|
sanitizeImageDiagnostic,
|
|
18
|
-
type
|
|
19
|
+
type ImageGenerationToolDetails,
|
|
19
20
|
} from "./types";
|
|
20
21
|
|
|
21
22
|
const registeredApis = new WeakSet<object>();
|
|
@@ -82,6 +83,18 @@ const GenerateImageParameters = Type.Object(
|
|
|
82
83
|
},
|
|
83
84
|
),
|
|
84
85
|
),
|
|
86
|
+
batchSize: Type.Optional(
|
|
87
|
+
Type.Union(
|
|
88
|
+
[
|
|
89
|
+
Type.Null(),
|
|
90
|
+
Type.Integer({ minimum: 1, maximum: MAX_IMAGE_BATCH_SIZE }),
|
|
91
|
+
],
|
|
92
|
+
{
|
|
93
|
+
description:
|
|
94
|
+
"Run this many independent image requests concurrently (1-10). Batch requests use generated artifacts and cannot set outputPath.",
|
|
95
|
+
},
|
|
96
|
+
),
|
|
97
|
+
),
|
|
85
98
|
},
|
|
86
99
|
{ additionalProperties: false },
|
|
87
100
|
);
|
|
@@ -111,12 +124,12 @@ export function registerImageGenerationExtension(
|
|
|
111
124
|
if (registeredApis.has(pi)) return;
|
|
112
125
|
registeredApis.add(pi);
|
|
113
126
|
|
|
114
|
-
pi.registerTool<typeof GenerateImageParameters,
|
|
127
|
+
pi.registerTool<typeof GenerateImageParameters, ImageGenerationToolDetails, Record<string, never>>({
|
|
115
128
|
name: IMAGE_GENERATION_TOOL_NAME,
|
|
116
129
|
label: "OpenAI Generate Image",
|
|
117
130
|
description:
|
|
118
|
-
"Generate
|
|
119
|
-
promptSnippet: "Generate or edit PNG images through
|
|
131
|
+
"Generate PNG images, or edit from one to five user-approved local reference images, through an independent Images API request using a configured image model. This is a paid provider operation; batchSize runs independent requests concurrently.",
|
|
132
|
+
promptSnippet: "Generate or edit PNG images through an independent Images API request and a configured imageGeneration.allowedModels entry.",
|
|
120
133
|
promptGuidelines: [
|
|
121
134
|
"Use openai_generate_image when the user explicitly asks to create, draw, render, or edit a raster image and the active model speaks a Responses API.",
|
|
122
135
|
"Do not call openai_generate_image speculatively: it consumes the user's provider or gateway image quota.",
|
|
@@ -124,6 +137,7 @@ export function registerImageGenerationExtension(
|
|
|
124
137
|
"Set referenceImagePaths to null unless the user explicitly identified local files; never invent paths or placeholder strings, and remember upload requires user approval.",
|
|
125
138
|
"Set outputPath to null unless the user explicitly asks for a destination; never invent a destination or placeholder string, and otherwise use the default Pi agent artifact.",
|
|
126
139
|
"Set model to null unless the user named an image model configured in imageGeneration.allowedModels; never invent or guess a model id, and remember an unlisted model fails before the paid request.",
|
|
140
|
+
"Use batchSize only when the user asks for multiple variants; each batch item is an independent paid request, and batchSize cannot be combined with outputPath.",
|
|
127
141
|
"Do not substitute Python, browser automation, shell scripts, or unrelated image tools for an eligible image request.",
|
|
128
142
|
],
|
|
129
143
|
parameters: GenerateImageParameters,
|
|
@@ -4,6 +4,7 @@ import {
|
|
|
4
4
|
IMAGE_GENERATION_QUALITIES,
|
|
5
5
|
IMAGE_GENERATION_SIZES,
|
|
6
6
|
MAX_GENERATED_IMAGE_BYTES,
|
|
7
|
+
MAX_IMAGE_BATCH_SIZE,
|
|
7
8
|
MAX_IMAGE_DIAGNOSTIC_CHARS,
|
|
8
9
|
MAX_IMAGE_DIMENSION,
|
|
9
10
|
MAX_IMAGE_IDENTIFIER_CHARS,
|
|
@@ -26,26 +27,13 @@ const BASE64_PATTERN = /^(?:[A-Za-z0-9+/]{4})*(?:[A-Za-z0-9+/]{2}==|[A-Za-z0-9+/
|
|
|
26
27
|
const DATA_URL_PATTERN = /^data:image\/[a-z0-9.+-]+;base64,(.*)$/is;
|
|
27
28
|
|
|
28
29
|
export type ImageGenerationRequestBody = {
|
|
30
|
+
action: NormalizedGenerateImageParams["action"];
|
|
29
31
|
model: string;
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
content: Array<
|
|
36
|
-
| { type: "input_text"; text: string }
|
|
37
|
-
| { type: "input_image"; image_url: string; detail: "auto" }
|
|
38
|
-
>;
|
|
39
|
-
}>;
|
|
40
|
-
tools: Array<{
|
|
41
|
-
type: "image_generation";
|
|
42
|
-
model: string;
|
|
43
|
-
action: "generate" | "edit";
|
|
44
|
-
size: NormalizedGenerateImageParams["size"];
|
|
45
|
-
quality: NormalizedGenerateImageParams["quality"];
|
|
46
|
-
output_format: "png";
|
|
47
|
-
}>;
|
|
48
|
-
tool_choice: { type: "image_generation" };
|
|
32
|
+
prompt: string;
|
|
33
|
+
n: 1;
|
|
34
|
+
size: NormalizedGenerateImageParams["size"];
|
|
35
|
+
quality: NormalizedGenerateImageParams["quality"];
|
|
36
|
+
references: readonly PreparedReferenceImage[];
|
|
49
37
|
};
|
|
50
38
|
|
|
51
39
|
export type ParsedImageResponseResult =
|
|
@@ -137,6 +125,22 @@ function normalizeRequestedModel(value: unknown): string | undefined {
|
|
|
137
125
|
return trimmed;
|
|
138
126
|
}
|
|
139
127
|
|
|
128
|
+
function normalizeBatchSize(value: unknown): number {
|
|
129
|
+
if (value === null || value === undefined) return 1;
|
|
130
|
+
if (
|
|
131
|
+
typeof value !== "number" ||
|
|
132
|
+
!Number.isInteger(value) ||
|
|
133
|
+
value < 1 ||
|
|
134
|
+
value > MAX_IMAGE_BATCH_SIZE
|
|
135
|
+
) {
|
|
136
|
+
throw new ImageGenerationError(
|
|
137
|
+
"invalid-parameters",
|
|
138
|
+
`batchSize must be an integer from 1 to ${MAX_IMAGE_BATCH_SIZE}.`,
|
|
139
|
+
);
|
|
140
|
+
}
|
|
141
|
+
return value;
|
|
142
|
+
}
|
|
143
|
+
|
|
140
144
|
/**
|
|
141
145
|
* Configured model ids are only trusted as far as the shared bounds: unusable entries are
|
|
142
146
|
* dropped, and an empty or malformed list falls back to the shipped default model.
|
|
@@ -153,7 +157,7 @@ function usableImageModels(models: readonly string[] | undefined): string[] {
|
|
|
153
157
|
}
|
|
154
158
|
|
|
155
159
|
/**
|
|
156
|
-
* Resolve the
|
|
160
|
+
* Resolve the independent Images API model for one call. An omitted request selects the
|
|
157
161
|
* explicit default when supplied (v2), or the first configured model for legacy callers.
|
|
158
162
|
* An explicit request must match a configured id exactly before any paid dispatch.
|
|
159
163
|
*/
|
|
@@ -213,6 +217,13 @@ export function normalizeGenerateImageParams(
|
|
|
213
217
|
|
|
214
218
|
const referenceImagePaths = normalizePathList(params.referenceImagePaths);
|
|
215
219
|
const outputPath = normalizeOutputPath(params.outputPath);
|
|
220
|
+
const batchSize = normalizeBatchSize(params.batchSize);
|
|
221
|
+
if (batchSize > 1 && outputPath !== undefined) {
|
|
222
|
+
throw new ImageGenerationError(
|
|
223
|
+
"invalid-parameters",
|
|
224
|
+
"outputPath cannot be used when batchSize is greater than 1; use the generated artifacts instead.",
|
|
225
|
+
);
|
|
226
|
+
}
|
|
216
227
|
|
|
217
228
|
return {
|
|
218
229
|
prompt,
|
|
@@ -221,12 +232,12 @@ export function normalizeGenerateImageParams(
|
|
|
221
232
|
size,
|
|
222
233
|
quality,
|
|
223
234
|
model: normalizeRequestedModel(params.model),
|
|
235
|
+
batchSize,
|
|
224
236
|
action: referenceImagePaths.length > 0 ? "edit" : "generate",
|
|
225
237
|
};
|
|
226
238
|
}
|
|
227
239
|
|
|
228
240
|
export function buildImageGenerationRequest(args: {
|
|
229
|
-
routingModel: string;
|
|
230
241
|
imageModel: string;
|
|
231
242
|
params: NormalizedGenerateImageParams;
|
|
232
243
|
references: readonly PreparedReferenceImage[];
|
|
@@ -238,34 +249,14 @@ export function buildImageGenerationRequest(args: {
|
|
|
238
249
|
"A configured image generation model is required.",
|
|
239
250
|
);
|
|
240
251
|
}
|
|
241
|
-
const content: ImageGenerationRequestBody["input"][number]["content"] = [
|
|
242
|
-
{ type: "input_text", text: args.params.prompt },
|
|
243
|
-
];
|
|
244
|
-
for (const reference of args.references) {
|
|
245
|
-
content.push({
|
|
246
|
-
type: "input_image",
|
|
247
|
-
image_url: `data:${reference.mimeType};base64,${reference.bytes.toString("base64")}`,
|
|
248
|
-
detail: "auto",
|
|
249
|
-
});
|
|
250
|
-
}
|
|
251
|
-
|
|
252
252
|
return {
|
|
253
|
-
|
|
254
|
-
|
|
255
|
-
|
|
256
|
-
|
|
257
|
-
|
|
258
|
-
|
|
259
|
-
|
|
260
|
-
type: "image_generation",
|
|
261
|
-
model: imageModel,
|
|
262
|
-
action: args.params.action,
|
|
263
|
-
size: args.params.size,
|
|
264
|
-
quality: args.params.quality,
|
|
265
|
-
output_format: "png",
|
|
266
|
-
},
|
|
267
|
-
],
|
|
268
|
-
tool_choice: { type: "image_generation" },
|
|
253
|
+
action: args.params.action,
|
|
254
|
+
model: imageModel,
|
|
255
|
+
prompt: args.params.prompt,
|
|
256
|
+
n: 1,
|
|
257
|
+
size: args.params.size,
|
|
258
|
+
quality: args.params.quality,
|
|
259
|
+
references: args.references,
|
|
269
260
|
};
|
|
270
261
|
}
|
|
271
262
|
|
|
@@ -390,7 +381,7 @@ export function extractProviderErrorMessage(value: unknown, fallback: string): s
|
|
|
390
381
|
return fallback;
|
|
391
382
|
}
|
|
392
383
|
|
|
393
|
-
export function
|
|
384
|
+
export function parseImagesApiResponse(value: unknown): ParsedImageResponseResult {
|
|
394
385
|
if (!isRecord(value)) {
|
|
395
386
|
return {
|
|
396
387
|
ok: false,
|
|
@@ -398,75 +389,47 @@ export function parseImageGenerationResponse(value: unknown): ParsedImageRespons
|
|
|
398
389
|
errorMessage: "Image generation returned a non-object response.",
|
|
399
390
|
};
|
|
400
391
|
}
|
|
401
|
-
|
|
402
|
-
if (value.status !== "completed") {
|
|
403
|
-
return {
|
|
404
|
-
ok: false,
|
|
405
|
-
reason: "request-rejected",
|
|
406
|
-
errorMessage: extractProviderErrorMessage(
|
|
407
|
-
value,
|
|
408
|
-
`Image generation did not complete (status: ${String(value.status ?? "unknown")}).`,
|
|
409
|
-
),
|
|
410
|
-
};
|
|
411
|
-
}
|
|
412
|
-
if (!Array.isArray(value.output)) {
|
|
392
|
+
if (!Array.isArray(value.data)) {
|
|
413
393
|
return {
|
|
414
394
|
ok: false,
|
|
415
395
|
reason: "malformed-response",
|
|
416
|
-
errorMessage: "Image generation response did not contain
|
|
396
|
+
errorMessage: "Image generation response did not contain a data array.",
|
|
417
397
|
};
|
|
418
398
|
}
|
|
419
|
-
|
|
420
|
-
const calls = new Map<string, { result: string; revisedPrompt?: string }>();
|
|
421
|
-
for (let index = 0; index < value.output.length; index += 1) {
|
|
422
|
-
const item = value.output[index];
|
|
423
|
-
if (!isRecord(item) || item.type !== "image_generation_call") continue;
|
|
424
|
-
if (item.status !== "completed") continue;
|
|
425
|
-
const result = typeof item.result === "string"
|
|
426
|
-
? item.result
|
|
427
|
-
: typeof item.b64_json === "string"
|
|
428
|
-
? item.b64_json
|
|
429
|
-
: undefined;
|
|
430
|
-
if (!result?.trim()) continue;
|
|
431
|
-
const id = normalizedOptionalString(item.id, MAX_IMAGE_IDENTIFIER_CHARS) ?? `image_generation_${index}`;
|
|
432
|
-
const revisedPrompt = normalizedOptionalString(item.revised_prompt, MAX_IMAGE_DIAGNOSTIC_CHARS);
|
|
433
|
-
const existing = calls.get(id);
|
|
434
|
-
if (existing && existing.result !== result) {
|
|
435
|
-
return {
|
|
436
|
-
ok: false,
|
|
437
|
-
reason: "malformed-response",
|
|
438
|
-
errorMessage: "Image generation returned conflicting results for one image call.",
|
|
439
|
-
};
|
|
440
|
-
}
|
|
441
|
-
calls.set(id, { result, revisedPrompt: revisedPrompt ?? existing?.revisedPrompt });
|
|
442
|
-
}
|
|
443
|
-
|
|
444
|
-
if (calls.size === 0) {
|
|
399
|
+
if (value.data.length === 0) {
|
|
445
400
|
return {
|
|
446
401
|
ok: false,
|
|
447
402
|
reason: "no-image",
|
|
448
403
|
errorMessage: "Image generation completed without a usable image result.",
|
|
449
404
|
};
|
|
450
405
|
}
|
|
451
|
-
if (
|
|
406
|
+
if (value.data.length !== 1) {
|
|
452
407
|
return {
|
|
453
408
|
ok: false,
|
|
454
409
|
reason: "malformed-response",
|
|
455
|
-
errorMessage: "Image generation returned more than one image
|
|
410
|
+
errorMessage: "Image generation returned more than one image for a single request.",
|
|
456
411
|
};
|
|
457
412
|
}
|
|
458
413
|
|
|
459
|
-
const
|
|
460
|
-
|
|
414
|
+
const item = value.data[0];
|
|
415
|
+
if (!isRecord(item) || typeof item.b64_json !== "string" || !item.b64_json.trim()) {
|
|
416
|
+
return {
|
|
417
|
+
ok: false,
|
|
418
|
+
reason: "no-image",
|
|
419
|
+
errorMessage: "Image generation response did not contain base64 image data.",
|
|
420
|
+
};
|
|
421
|
+
}
|
|
422
|
+
const decoded = decodeGeneratedPng(item.b64_json);
|
|
461
423
|
if (!decoded.ok) return decoded;
|
|
462
424
|
|
|
463
425
|
return {
|
|
464
426
|
ok: true,
|
|
465
427
|
image: {
|
|
466
428
|
bytes: decoded.bytes,
|
|
467
|
-
imageCallId
|
|
429
|
+
imageCallId:
|
|
430
|
+
normalizedOptionalString(item.id, MAX_IMAGE_IDENTIFIER_CHARS) ?? "image_generation_0",
|
|
468
431
|
responseId: normalizedOptionalString(value.id, MAX_IMAGE_IDENTIFIER_CHARS),
|
|
469
|
-
revisedPrompt:
|
|
432
|
+
revisedPrompt: normalizedOptionalString(item.revised_prompt, MAX_IMAGE_DIAGNOSTIC_CHARS),
|
|
470
433
|
width: decoded.width,
|
|
471
434
|
height: decoded.height,
|
|
472
435
|
},
|
|
@@ -21,7 +21,7 @@ import {
|
|
|
21
21
|
MAX_GENERATED_IMAGE_BYTES,
|
|
22
22
|
isImageGenerationDetails,
|
|
23
23
|
type GenerateImageParams,
|
|
24
|
-
type
|
|
24
|
+
type ImageGenerationToolDetails,
|
|
25
25
|
} from "./types";
|
|
26
26
|
|
|
27
27
|
export type ImageGenerationRenderState = Record<string, never>;
|
|
@@ -54,7 +54,7 @@ function isPathInsideOrEqual(root: string, candidate: string): boolean {
|
|
|
54
54
|
return rel === "" || (!rel.startsWith(`..${sep}`) && rel !== ".." && !isAbsolute(rel));
|
|
55
55
|
}
|
|
56
56
|
|
|
57
|
-
function resultText(result: AgentToolResult<
|
|
57
|
+
function resultText(result: AgentToolResult<ImageGenerationToolDetails>): string {
|
|
58
58
|
return result.content
|
|
59
59
|
.filter((block): block is { type: "text"; text: string } => block.type === "text")
|
|
60
60
|
.map((block) => block.text)
|
|
@@ -148,7 +148,7 @@ function textOnly(args: {
|
|
|
148
148
|
export function createImageGenerationResultRenderer(
|
|
149
149
|
deps: ImageRendererDependencies = DEFAULT_RENDER_DEPS,
|
|
150
150
|
): (
|
|
151
|
-
result: AgentToolResult<
|
|
151
|
+
result: AgentToolResult<ImageGenerationToolDetails>,
|
|
152
152
|
options: ToolRenderResultOptions,
|
|
153
153
|
theme: Theme,
|
|
154
154
|
context: ImageGenerationRenderContext,
|