pi-codex-image-gen 0.1.11 → 0.1.13
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +33 -0
- package/CONTRIBUTING.md +32 -3
- package/README.md +43 -9
- package/SECURITY.md +8 -0
- package/extensions/index.ts +314 -202
- package/package.json +11 -7
- package/skills/imagegen/SKILL.md +29 -24
- package/skills/imagegen/references/cli.md +7 -5
- package/skills/imagegen/references/image-api.md +18 -8
- package/skills/imagegen/references/prompting.md +2 -2
- package/skills/imagegen/references/sample-prompts.md +2 -2
- package/skills/imagegen/scripts/image_gen.py +29 -20
- package/src/codex-response.ts +286 -0
- package/src/install-telemetry.ts +19 -46
package/extensions/index.ts
CHANGED
|
@@ -3,16 +3,19 @@
|
|
|
3
3
|
*
|
|
4
4
|
* Registers `codex_generate_image`, a tool that uses Pi's existing
|
|
5
5
|
* openai-codex ChatGPT/Codex auth to call the Codex Responses backend with the
|
|
6
|
-
* native `image_generation` tool. The backend
|
|
6
|
+
* native `image_generation` tool. The backend selects the image model.
|
|
7
7
|
*/
|
|
8
8
|
|
|
9
9
|
import { readFileSync } from "node:fs";
|
|
10
|
-
import {
|
|
10
|
+
import { constants } from "node:fs";
|
|
11
|
+
import { mkdir, open, writeFile } from "node:fs/promises";
|
|
12
|
+
import { homedir } from "node:os";
|
|
11
13
|
import { isAbsolute, join, resolve } from "node:path";
|
|
12
14
|
import { StringEnum } from "@earendil-works/pi-ai";
|
|
13
|
-
import { type ExtensionAPI, getAgentDir, withFileMutationQueue } from "@earendil-works/pi-coding-agent";
|
|
15
|
+
import { CONFIG_DIR_NAME, type ExtensionAPI, getAgentDir, withFileMutationQueue } from "@earendil-works/pi-coding-agent";
|
|
14
16
|
import { type Static, Type } from "typebox";
|
|
15
17
|
import { reportInstallTelemetry } from "../src/install-telemetry.js";
|
|
18
|
+
import { abortable, httpFailure, MAX_IMAGE_BYTES, parseCodexSse, withRequestDeadline, type ParsedCodexResponse } from "../src/codex-response.js";
|
|
16
19
|
|
|
17
20
|
const PACKAGE_NAME = "pi-codex-image-gen";
|
|
18
21
|
const PROVIDER = "openai-codex";
|
|
@@ -23,6 +26,11 @@ const DEFAULT_SAVE_MODE = "global";
|
|
|
23
26
|
const OPENAI_BETA_HEADER = "responses=experimental";
|
|
24
27
|
const MAX_RETRIES = 3;
|
|
25
28
|
const BASE_DELAY_MS = 1000;
|
|
29
|
+
const MAX_RETRY_DELAY_MS = 30_000;
|
|
30
|
+
const MAX_EDIT_IMAGES = 5;
|
|
31
|
+
const MAX_INPUT_IMAGE_BYTES = 20 * 1024 * 1024;
|
|
32
|
+
const MAX_TOTAL_INPUT_BYTES = 50 * 1024 * 1024;
|
|
33
|
+
const MAX_PROMPT_CHARS = 32_000;
|
|
26
34
|
|
|
27
35
|
const SAVE_MODES = ["none", "project", "global", "custom"] as const;
|
|
28
36
|
type SaveMode = (typeof SAVE_MODES)[number];
|
|
@@ -32,22 +40,58 @@ type OutputFormat = (typeof OUTPUT_FORMATS)[number];
|
|
|
32
40
|
|
|
33
41
|
// --- #1: Retry helpers with exponential backoff + jitter ---
|
|
34
42
|
|
|
35
|
-
function
|
|
36
|
-
if (
|
|
37
|
-
|
|
43
|
+
export function parseRetryAfter(value: string | null, nowMs = Date.now()): number | undefined {
|
|
44
|
+
if (!value) return undefined;
|
|
45
|
+
const trimmed = value.trim();
|
|
46
|
+
if (/^\d+(?:\.\d+)?$/.test(trimmed)) {
|
|
47
|
+
const milliseconds = Number(trimmed) * 1000;
|
|
48
|
+
return Number.isFinite(milliseconds) ? Math.min(milliseconds, MAX_RETRY_DELAY_MS) : undefined;
|
|
49
|
+
}
|
|
50
|
+
const dateMs = Date.parse(trimmed);
|
|
51
|
+
if (!Number.isFinite(dateMs) || dateMs <= nowMs) return undefined;
|
|
52
|
+
return Math.min(dateMs - nowMs, MAX_RETRY_DELAY_MS);
|
|
38
53
|
}
|
|
39
54
|
|
|
40
|
-
function
|
|
41
|
-
|
|
42
|
-
|
|
55
|
+
export function retryDelayMs(
|
|
56
|
+
attempt: number,
|
|
57
|
+
retryAfter: string | null,
|
|
58
|
+
random = Math.random,
|
|
59
|
+
nowMs = Date.now(),
|
|
60
|
+
): number {
|
|
61
|
+
const serverDelay = parseRetryAfter(retryAfter, nowMs);
|
|
62
|
+
if (serverDelay !== undefined) {
|
|
63
|
+
return Math.floor(Math.min(serverDelay * (1 + random() * 0.1), MAX_RETRY_DELAY_MS));
|
|
64
|
+
}
|
|
65
|
+
const exponential = Math.min(BASE_DELAY_MS * 2 ** (attempt - 1), MAX_RETRY_DELAY_MS);
|
|
66
|
+
return Math.floor(exponential * (0.9 + random() * 0.2));
|
|
67
|
+
}
|
|
68
|
+
|
|
69
|
+
export function abortableDelay(milliseconds: number, signal?: AbortSignal): Promise<void> {
|
|
70
|
+
if (signal?.aborted) return Promise.reject(new Error("Image generation was aborted."));
|
|
71
|
+
return new Promise<void>((resolve, reject) => {
|
|
72
|
+
const timer = setTimeout(finish, milliseconds);
|
|
73
|
+
function cleanup() {
|
|
74
|
+
clearTimeout(timer);
|
|
75
|
+
signal?.removeEventListener("abort", abort);
|
|
76
|
+
}
|
|
77
|
+
function finish() {
|
|
78
|
+
cleanup();
|
|
79
|
+
resolve();
|
|
80
|
+
}
|
|
81
|
+
function abort() {
|
|
82
|
+
cleanup();
|
|
83
|
+
reject(new Error("Image generation was aborted."));
|
|
84
|
+
}
|
|
85
|
+
signal?.addEventListener("abort", abort, { once: true });
|
|
86
|
+
});
|
|
43
87
|
}
|
|
44
88
|
|
|
45
89
|
// --- Tool parameter schema ---
|
|
46
90
|
|
|
47
91
|
const TOOL_PARAMS = Type.Object({
|
|
48
|
-
prompt: Type.String({ description: "The image prompt. Be specific about subject, composition, style, text, and constraints." }),
|
|
92
|
+
prompt: Type.String({ minLength: 1, maxLength: MAX_PROMPT_CHARS, description: "The image prompt. Be specific about subject, composition, style, text, and constraints." }),
|
|
49
93
|
model: Type.Optional(
|
|
50
|
-
Type.String({ description: `Codex model
|
|
94
|
+
Type.String({ minLength: 1, maxLength: 200, description: `Codex routing model, not an image model selector. Defaults to ${DEFAULT_MODEL}.` }),
|
|
51
95
|
),
|
|
52
96
|
outputFormat: Type.Optional(StringEnum(OUTPUT_FORMATS)),
|
|
53
97
|
save: Type.Optional(StringEnum(SAVE_MODES)),
|
|
@@ -56,6 +100,19 @@ const TOOL_PARAMS = Type.Object({
|
|
|
56
100
|
description: "Directory to save the image when save=custom. Relative paths resolve under the current workspace.",
|
|
57
101
|
}),
|
|
58
102
|
),
|
|
103
|
+
referencedImagePaths: Type.Optional(
|
|
104
|
+
Type.Array(Type.String(), {
|
|
105
|
+
maxItems: MAX_EDIT_IMAGES,
|
|
106
|
+
description: "Up to five local image paths to edit. Relative paths resolve under the current workspace.",
|
|
107
|
+
}),
|
|
108
|
+
),
|
|
109
|
+
numLastImagesToInclude: Type.Optional(
|
|
110
|
+
Type.Integer({
|
|
111
|
+
minimum: 1,
|
|
112
|
+
maximum: MAX_EDIT_IMAGES,
|
|
113
|
+
description: "Use the most recent one to five images from the current conversation as edit inputs.",
|
|
114
|
+
}),
|
|
115
|
+
),
|
|
59
116
|
});
|
|
60
117
|
|
|
61
118
|
type ToolParams = Static<typeof TOOL_PARAMS>;
|
|
@@ -73,39 +130,11 @@ interface SaveConfig {
|
|
|
73
130
|
outputDir?: string;
|
|
74
131
|
}
|
|
75
132
|
|
|
76
|
-
interface
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
result: string;
|
|
80
|
-
revisedPrompt?: string;
|
|
81
|
-
}
|
|
82
|
-
|
|
83
|
-
interface ParsedCodexResponse {
|
|
84
|
-
image?: GeneratedImage;
|
|
85
|
-
text: string[];
|
|
86
|
-
responseId?: string;
|
|
87
|
-
usage?: unknown;
|
|
133
|
+
interface InputImage {
|
|
134
|
+
data: string;
|
|
135
|
+
mimeType: string;
|
|
88
136
|
}
|
|
89
137
|
|
|
90
|
-
// --- #11: Typed SSE event discriminated union ---
|
|
91
|
-
|
|
92
|
-
type CodexSseEvent =
|
|
93
|
-
| { type: "error"; message?: string; code?: string }
|
|
94
|
-
| { type: "response.failed"; response?: { error?: { message?: string } } }
|
|
95
|
-
| { type: "response.created"; response?: { id?: string } }
|
|
96
|
-
| { type: "response.output_text.delta"; delta?: string }
|
|
97
|
-
| {
|
|
98
|
-
type: "response.output_item.done";
|
|
99
|
-
item?: {
|
|
100
|
-
type?: string;
|
|
101
|
-
id?: string | number;
|
|
102
|
-
status?: string;
|
|
103
|
-
result?: string;
|
|
104
|
-
revised_prompt?: string;
|
|
105
|
-
};
|
|
106
|
-
}
|
|
107
|
-
| { type: "response.completed"; response?: { id?: string; usage?: unknown } };
|
|
108
|
-
|
|
109
138
|
// --- JWT helpers ---
|
|
110
139
|
|
|
111
140
|
function decodeJwtPayload(token: string): Record<string, unknown> {
|
|
@@ -115,8 +144,8 @@ function decodeJwtPayload(token: string): Record<string, unknown> {
|
|
|
115
144
|
}
|
|
116
145
|
try {
|
|
117
146
|
return JSON.parse(Buffer.from(parts[1], "base64url").toString("utf8")) as Record<string, unknown>;
|
|
118
|
-
} catch
|
|
119
|
-
throw new Error(
|
|
147
|
+
} catch {
|
|
148
|
+
throw new Error("Failed to decode OpenAI Codex auth token. Run /login for openai-codex again.");
|
|
120
149
|
}
|
|
121
150
|
}
|
|
122
151
|
|
|
@@ -143,20 +172,23 @@ function readConfigFile(path: string): ExtensionConfig {
|
|
|
143
172
|
}
|
|
144
173
|
}
|
|
145
174
|
|
|
146
|
-
function loadConfig(cwd: string): ExtensionConfig {
|
|
147
|
-
const globalConfig = readConfigFile(join(
|
|
148
|
-
|
|
175
|
+
export function loadConfig(cwd: string, projectTrusted: boolean, agentDir = getAgentDir()): ExtensionConfig {
|
|
176
|
+
const globalConfig = readConfigFile(join(agentDir, "extensions", "codex-image-gen.json"));
|
|
177
|
+
if (!projectTrusted) return globalConfig;
|
|
178
|
+
const projectConfig = readConfigFile(join(cwd, CONFIG_DIR_NAME, "extensions", "codex-image-gen.json"));
|
|
149
179
|
return { ...globalConfig, ...projectConfig };
|
|
150
180
|
}
|
|
151
181
|
|
|
152
182
|
// --- Path helpers ---
|
|
153
183
|
|
|
154
|
-
function resolveUnderCwd(cwd: string, path: string): string {
|
|
184
|
+
export function resolveUnderCwd(cwd: string, path: string, homeDir = homedir()): string {
|
|
185
|
+
if (path === "~") return homeDir;
|
|
186
|
+
if (path.startsWith("~/")) return resolve(homeDir, path.slice(2));
|
|
155
187
|
return isAbsolute(path) ? path : resolve(cwd, path);
|
|
156
188
|
}
|
|
157
189
|
|
|
158
190
|
function sanitizePathPart(value: string, fallback: string): string {
|
|
159
|
-
const sanitized = value
|
|
191
|
+
const sanitized = value.slice(0, 128)
|
|
160
192
|
.split("")
|
|
161
193
|
.map((ch) => (/[a-zA-Z0-9_-]/.test(ch) ? ch : "_"))
|
|
162
194
|
.join("")
|
|
@@ -174,7 +206,7 @@ function resolveSaveConfig(params: ToolParams, cwd: string, sessionId: string, c
|
|
|
174
206
|
throw new Error(`Invalid save mode: ${mode}. Expected one of ${SAVE_MODES.join(", ")}.`);
|
|
175
207
|
}
|
|
176
208
|
if (mode === "project") {
|
|
177
|
-
return { mode, outputDir: join(cwd,
|
|
209
|
+
return { mode, outputDir: join(cwd, CONFIG_DIR_NAME, "generated-images", safeSessionId) };
|
|
178
210
|
}
|
|
179
211
|
if (mode === "global") {
|
|
180
212
|
return { mode, outputDir: join(getAgentDir(), "generated-images", safeSessionId) };
|
|
@@ -199,22 +231,152 @@ function mimeForFormat(outputFormat: OutputFormat): string {
|
|
|
199
231
|
return outputFormat === "jpeg" ? "image/jpeg" : `image/${outputFormat}`;
|
|
200
232
|
}
|
|
201
233
|
|
|
202
|
-
|
|
234
|
+
function imagePath(outputFormat: OutputFormat, outputDir: string, imageCallId: string): string {
|
|
203
235
|
const filename = `${sanitizePathPart(imageCallId, "image_generation")}.${extensionForFormat(outputFormat)}`;
|
|
204
|
-
|
|
236
|
+
return join(outputDir, filename);
|
|
237
|
+
}
|
|
238
|
+
|
|
239
|
+
export function decodeImageData(base64Data: string, outputFormat: OutputFormat): Buffer {
|
|
240
|
+
if (base64Data.length > Math.ceil(MAX_IMAGE_BYTES / 3) * 4) throw new Error("Codex image exceeded the 32 MiB size limit.");
|
|
241
|
+
const value = base64Data.trim();
|
|
242
|
+
if (!value || value.length % 4 !== 0 || /[^A-Za-z0-9+/=]/.test(value)) {
|
|
243
|
+
throw new Error("Codex returned invalid base64 image data.");
|
|
244
|
+
}
|
|
245
|
+
const bytes = Buffer.from(value, "base64");
|
|
246
|
+
if (bytes.length === 0 || bytes.length > MAX_IMAGE_BYTES || bytes.toString("base64") !== value) {
|
|
247
|
+
throw new Error("Codex returned invalid base64 image data.");
|
|
248
|
+
}
|
|
249
|
+
const validSignature =
|
|
250
|
+
(outputFormat === "png" && bytes.length >= 8 && bytes.subarray(0, 8).equals(Buffer.from([0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a]))) ||
|
|
251
|
+
(outputFormat === "jpeg" && bytes.length >= 3 && bytes[0] === 0xff && bytes[1] === 0xd8 && bytes[2] === 0xff) ||
|
|
252
|
+
(outputFormat === "webp" && bytes.length >= 12 && bytes.toString("ascii", 0, 4) === "RIFF" && bytes.toString("ascii", 8, 12) === "WEBP");
|
|
253
|
+
if (!validSignature) throw new Error(`Codex returned image data that does not match ${outputFormat}.`);
|
|
254
|
+
return bytes;
|
|
255
|
+
}
|
|
256
|
+
|
|
257
|
+
async function saveImage(
|
|
258
|
+
bytes: Buffer,
|
|
259
|
+
outputFormat: OutputFormat,
|
|
260
|
+
outputDir: string,
|
|
261
|
+
imageCallId: string,
|
|
262
|
+
): Promise<string> {
|
|
263
|
+
const filePath = imagePath(outputFormat, outputDir, imageCallId);
|
|
205
264
|
await withFileMutationQueue(filePath, async () => {
|
|
206
|
-
await mkdir(outputDir, { recursive: true });
|
|
207
|
-
await writeFile(filePath,
|
|
265
|
+
await mkdir(outputDir, { recursive: true, mode: 0o700 });
|
|
266
|
+
await writeFile(filePath, bytes, { flag: "wx", mode: 0o600 });
|
|
208
267
|
});
|
|
209
268
|
return filePath;
|
|
210
269
|
}
|
|
211
270
|
|
|
271
|
+
export function selectRecentImages(messages: unknown[], count: number): InputImage[] {
|
|
272
|
+
const images: InputImage[] = [];
|
|
273
|
+
for (let index = messages.length - 1; index >= 0 && images.length < count; index--) {
|
|
274
|
+
const message = messages[index] as { content?: unknown };
|
|
275
|
+
if (!Array.isArray(message?.content)) continue;
|
|
276
|
+
for (let contentIndex = message.content.length - 1; contentIndex >= 0 && images.length < count; contentIndex--) {
|
|
277
|
+
const block = message.content[contentIndex] as { type?: unknown; data?: unknown; mimeType?: unknown };
|
|
278
|
+
if (block?.type === "image" && typeof block.data === "string" && typeof block.mimeType === "string") {
|
|
279
|
+
images.push({ data: block.data, mimeType: block.mimeType });
|
|
280
|
+
}
|
|
281
|
+
}
|
|
282
|
+
}
|
|
283
|
+
return images.reverse();
|
|
284
|
+
}
|
|
285
|
+
|
|
286
|
+
function mimeFromBytes(bytes: Buffer, path: string): string {
|
|
287
|
+
if (bytes.length >= 8 && bytes.subarray(0, 8).equals(Buffer.from([0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a]))) return "image/png";
|
|
288
|
+
if (bytes.length >= 3 && bytes[0] === 0xff && bytes[1] === 0xd8 && bytes[2] === 0xff) return "image/jpeg";
|
|
289
|
+
if (bytes.length >= 12 && bytes.toString("ascii", 0, 4) === "RIFF" && bytes.toString("ascii", 8, 12) === "WEBP") return "image/webp";
|
|
290
|
+
throw new Error(`Referenced image is unavailable or unsupported: ${path}`);
|
|
291
|
+
}
|
|
292
|
+
|
|
293
|
+
async function readInputImage(path: string): Promise<Buffer> {
|
|
294
|
+
// O_NONBLOCK prevents named pipes from blocking before the regular-file check.
|
|
295
|
+
const file = await open(path, constants.O_RDONLY | (constants.O_NONBLOCK ?? 0));
|
|
296
|
+
try {
|
|
297
|
+
const info = await file.stat();
|
|
298
|
+
if (!info.isFile()) throw new Error("Referenced images must be regular files.");
|
|
299
|
+
if (info.size > MAX_INPUT_IMAGE_BYTES) throw new Error("Referenced image exceeds 20 MiB.");
|
|
300
|
+
const blocks: Buffer[] = [];
|
|
301
|
+
let total = 0;
|
|
302
|
+
while (true) {
|
|
303
|
+
const block = Buffer.alloc(Math.min(64 * 1024, MAX_INPUT_IMAGE_BYTES + 1 - total));
|
|
304
|
+
const { bytesRead } = await file.read(block, 0, block.length, null);
|
|
305
|
+
if (!bytesRead) break;
|
|
306
|
+
total += bytesRead;
|
|
307
|
+
if (total > MAX_INPUT_IMAGE_BYTES) throw new Error("Referenced image exceeds 20 MiB.");
|
|
308
|
+
blocks.push(block.subarray(0, bytesRead));
|
|
309
|
+
}
|
|
310
|
+
return Buffer.concat(blocks, total);
|
|
311
|
+
} finally {
|
|
312
|
+
await file.close();
|
|
313
|
+
}
|
|
314
|
+
}
|
|
315
|
+
|
|
316
|
+
export async function resolveInputImages(
|
|
317
|
+
params: ToolParams,
|
|
318
|
+
cwd: string,
|
|
319
|
+
messages: unknown[],
|
|
320
|
+
): Promise<InputImage[]> {
|
|
321
|
+
const paths = params.referencedImagePaths ?? [];
|
|
322
|
+
if (paths.length > 0 && params.numLastImagesToInclude !== undefined) {
|
|
323
|
+
throw new Error("Provide only one of referencedImagePaths or numLastImagesToInclude.");
|
|
324
|
+
}
|
|
325
|
+
if (paths.length > MAX_EDIT_IMAGES) throw new Error(`referencedImagePaths accepts at most ${MAX_EDIT_IMAGES} paths.`);
|
|
326
|
+
if (paths.length > 0) {
|
|
327
|
+
const images: InputImage[] = [];
|
|
328
|
+
let total = 0;
|
|
329
|
+
for (const path of paths) {
|
|
330
|
+
const normalized = path.startsWith("@") ? path.slice(1) : path;
|
|
331
|
+
const absolutePath = resolveUnderCwd(cwd, normalized);
|
|
332
|
+
let bytes: Buffer;
|
|
333
|
+
try {
|
|
334
|
+
bytes = await readInputImage(absolutePath);
|
|
335
|
+
} catch (error) {
|
|
336
|
+
throw new Error(`Unable to read referenced image at ${absolutePath}: ${error instanceof Error ? error.message : String(error)}`);
|
|
337
|
+
}
|
|
338
|
+
total += bytes.length;
|
|
339
|
+
if (total > MAX_TOTAL_INPUT_BYTES) throw new Error("Referenced images exceed 50 MiB in total.");
|
|
340
|
+
images.push({ data: bytes.toString("base64"), mimeType: mimeFromBytes(bytes, absolutePath) });
|
|
341
|
+
}
|
|
342
|
+
return images;
|
|
343
|
+
}
|
|
344
|
+
if (params.numLastImagesToInclude !== undefined) {
|
|
345
|
+
const count = params.numLastImagesToInclude;
|
|
346
|
+
if (!Number.isInteger(count) || count < 1 || count > MAX_EDIT_IMAGES) {
|
|
347
|
+
throw new Error(`numLastImagesToInclude must be between 1 and ${MAX_EDIT_IMAGES}.`);
|
|
348
|
+
}
|
|
349
|
+
const images = selectRecentImages(messages, count);
|
|
350
|
+
if (images.length !== count) {
|
|
351
|
+
throw new Error(`Requested the last ${count} conversation images, but only ${images.length} were available.`);
|
|
352
|
+
}
|
|
353
|
+
let total = 0;
|
|
354
|
+
for (const image of images) {
|
|
355
|
+
if (image.data.length > Math.ceil(MAX_INPUT_IMAGE_BYTES / 3) * 4) throw new Error("Conversation image exceeds 20 MiB.");
|
|
356
|
+
const format = OUTPUT_FORMATS.find(format => mimeForFormat(format) === image.mimeType);
|
|
357
|
+
if (!format) throw new Error("Conversation image has an unsupported format.");
|
|
358
|
+
const bytes = decodeImageData(image.data, format);
|
|
359
|
+
if (bytes.length > MAX_INPUT_IMAGE_BYTES) throw new Error("Conversation image exceeds 20 MiB.");
|
|
360
|
+
total += bytes.length;
|
|
361
|
+
if (total > MAX_TOTAL_INPUT_BYTES) throw new Error("Conversation images exceed 50 MiB in total.");
|
|
362
|
+
}
|
|
363
|
+
return images;
|
|
364
|
+
}
|
|
365
|
+
return [];
|
|
366
|
+
}
|
|
367
|
+
|
|
212
368
|
// --- Request building ---
|
|
213
369
|
// #2: prompt_cache_key set to sessionId
|
|
214
370
|
// #7: parallel_tool_calls: false
|
|
215
371
|
// #14: include removed (not needed without reasoning)
|
|
216
372
|
|
|
217
|
-
function buildRequestBody(
|
|
373
|
+
export function buildRequestBody(
|
|
374
|
+
params: ToolParams,
|
|
375
|
+
model: string,
|
|
376
|
+
outputFormat: OutputFormat,
|
|
377
|
+
sessionId: string,
|
|
378
|
+
inputImages: InputImage[] = [],
|
|
379
|
+
) {
|
|
218
380
|
return {
|
|
219
381
|
model,
|
|
220
382
|
store: false,
|
|
@@ -225,7 +387,13 @@ function buildRequestBody(params: ToolParams, model: string, outputFormat: Outpu
|
|
|
225
387
|
input: [
|
|
226
388
|
{
|
|
227
389
|
role: "user",
|
|
228
|
-
content: [
|
|
390
|
+
content: [
|
|
391
|
+
{ type: "input_text", text: params.prompt },
|
|
392
|
+
...inputImages.map((image) => ({
|
|
393
|
+
type: "input_image",
|
|
394
|
+
image_url: `data:${image.mimeType};base64,${image.data}`,
|
|
395
|
+
})),
|
|
396
|
+
],
|
|
229
397
|
},
|
|
230
398
|
],
|
|
231
399
|
tools: [{ type: "image_generation", output_format: outputFormat }],
|
|
@@ -235,108 +403,6 @@ function buildRequestBody(params: ToolParams, model: string, outputFormat: Outpu
|
|
|
235
403
|
};
|
|
236
404
|
}
|
|
237
405
|
|
|
238
|
-
// --- SSE parsing ---
|
|
239
|
-
|
|
240
|
-
function parseSseDataLines(chunk: string): string | undefined {
|
|
241
|
-
const data = chunk
|
|
242
|
-
.split("\n")
|
|
243
|
-
.filter((line) => line.startsWith("data:"))
|
|
244
|
-
.map((line) => line.slice(5).trim())
|
|
245
|
-
.join("\n")
|
|
246
|
-
.trim();
|
|
247
|
-
return data && data !== "[DONE]" ? data : undefined;
|
|
248
|
-
}
|
|
249
|
-
|
|
250
|
-
async function parseCodexSse(response: Response, signal?: AbortSignal): Promise<ParsedCodexResponse> {
|
|
251
|
-
if (!response.body) throw new Error("Codex response did not include a stream body.");
|
|
252
|
-
const reader = response.body.getReader();
|
|
253
|
-
const decoder = new TextDecoder();
|
|
254
|
-
let buffer = "";
|
|
255
|
-
const parsed: ParsedCodexResponse = { text: [] };
|
|
256
|
-
|
|
257
|
-
try {
|
|
258
|
-
while (true) {
|
|
259
|
-
if (signal?.aborted) throw new Error("Image generation was aborted.");
|
|
260
|
-
const { done, value } = await reader.read();
|
|
261
|
-
if (done) break;
|
|
262
|
-
buffer += decoder.decode(value, { stream: true });
|
|
263
|
-
|
|
264
|
-
let separator = buffer.indexOf("\n\n");
|
|
265
|
-
while (separator !== -1) {
|
|
266
|
-
const chunk = buffer.slice(0, separator);
|
|
267
|
-
buffer = buffer.slice(separator + 2);
|
|
268
|
-
const data = parseSseDataLines(chunk);
|
|
269
|
-
if (data) handleCodexEvent(JSON.parse(data) as CodexSseEvent, parsed);
|
|
270
|
-
separator = buffer.indexOf("\n\n");
|
|
271
|
-
}
|
|
272
|
-
}
|
|
273
|
-
const remaining = parseSseDataLines(buffer);
|
|
274
|
-
if (remaining) handleCodexEvent(JSON.parse(remaining) as CodexSseEvent, parsed);
|
|
275
|
-
} finally {
|
|
276
|
-
try {
|
|
277
|
-
await reader.cancel();
|
|
278
|
-
} catch {
|
|
279
|
-
// ignored: stream may already be closed
|
|
280
|
-
}
|
|
281
|
-
reader.releaseLock();
|
|
282
|
-
}
|
|
283
|
-
|
|
284
|
-
return parsed;
|
|
285
|
-
}
|
|
286
|
-
|
|
287
|
-
// --- #11: Typed event handler via discriminated union ---
|
|
288
|
-
|
|
289
|
-
function handleCodexEvent(event: CodexSseEvent, parsed: ParsedCodexResponse): void {
|
|
290
|
-
if (!event || typeof event !== "object") return;
|
|
291
|
-
|
|
292
|
-
switch (event.type) {
|
|
293
|
-
case "error": {
|
|
294
|
-
const e = event as Extract<CodexSseEvent, { type: "error" }>;
|
|
295
|
-
throw new Error(`Codex error: ${e.message || e.code || JSON.stringify(event)}`);
|
|
296
|
-
}
|
|
297
|
-
case "response.failed": {
|
|
298
|
-
const e = event as Extract<CodexSseEvent, { type: "response.failed" }>;
|
|
299
|
-
throw new Error(e.response?.error?.message || "Codex response failed.");
|
|
300
|
-
}
|
|
301
|
-
case "response.created": {
|
|
302
|
-
const e = event as Extract<CodexSseEvent, { type: "response.created" }>;
|
|
303
|
-
if (typeof e.response?.id === "string") {
|
|
304
|
-
parsed.responseId = e.response.id;
|
|
305
|
-
}
|
|
306
|
-
break;
|
|
307
|
-
}
|
|
308
|
-
case "response.output_text.delta": {
|
|
309
|
-
const e = event as Extract<CodexSseEvent, { type: "response.output_text.delta" }>;
|
|
310
|
-
if (typeof e.delta === "string") {
|
|
311
|
-
parsed.text.push(e.delta);
|
|
312
|
-
}
|
|
313
|
-
break;
|
|
314
|
-
}
|
|
315
|
-
case "response.output_item.done": {
|
|
316
|
-
const e = event as Extract<CodexSseEvent, { type: "response.output_item.done" }>;
|
|
317
|
-
const item = e.item;
|
|
318
|
-
if (item?.type === "image_generation_call") {
|
|
319
|
-
if (typeof item.result !== "string" || item.result.length === 0) {
|
|
320
|
-
throw new Error("Codex image_generation_call did not contain image data.");
|
|
321
|
-
}
|
|
322
|
-
parsed.image = {
|
|
323
|
-
id: String(item.id || "image_generation"),
|
|
324
|
-
status: String(item.status || "completed"),
|
|
325
|
-
result: item.result,
|
|
326
|
-
revisedPrompt: typeof item.revised_prompt === "string" ? item.revised_prompt : undefined,
|
|
327
|
-
};
|
|
328
|
-
}
|
|
329
|
-
break;
|
|
330
|
-
}
|
|
331
|
-
case "response.completed": {
|
|
332
|
-
const e = event as Extract<CodexSseEvent, { type: "response.completed" }>;
|
|
333
|
-
if (typeof e.response?.id === "string") parsed.responseId = e.response.id;
|
|
334
|
-
if (e.response?.usage) parsed.usage = e.response.usage;
|
|
335
|
-
break;
|
|
336
|
-
}
|
|
337
|
-
}
|
|
338
|
-
}
|
|
339
|
-
|
|
340
406
|
// --- #1: requestImage with retry + backoff + jitter ---
|
|
341
407
|
|
|
342
408
|
async function requestImage(
|
|
@@ -346,42 +412,48 @@ async function requestImage(
|
|
|
346
412
|
model: string,
|
|
347
413
|
outputFormat: OutputFormat,
|
|
348
414
|
sessionId: string,
|
|
415
|
+
inputImages: InputImage[],
|
|
349
416
|
signal?: AbortSignal,
|
|
417
|
+
onProgress?: (stage: string) => void,
|
|
350
418
|
): Promise<ParsedCodexResponse> {
|
|
351
|
-
const body = JSON.stringify(buildRequestBody(params, model, outputFormat, sessionId));
|
|
419
|
+
const body = JSON.stringify(buildRequestBody(params, model, outputFormat, sessionId, inputImages));
|
|
352
420
|
const headers: Record<string, string> = {
|
|
353
421
|
Authorization: `Bearer ${token}`,
|
|
354
422
|
"chatgpt-account-id": accountId,
|
|
355
423
|
originator: "pi",
|
|
424
|
+
"User-Agent": PACKAGE_NAME,
|
|
356
425
|
"OpenAI-Beta": OPENAI_BETA_HEADER,
|
|
357
426
|
accept: "text/event-stream",
|
|
358
427
|
"content-type": "application/json",
|
|
359
428
|
};
|
|
360
429
|
|
|
361
|
-
|
|
362
|
-
|
|
363
|
-
|
|
364
|
-
|
|
365
|
-
|
|
366
|
-
|
|
367
|
-
|
|
368
|
-
|
|
369
|
-
|
|
370
|
-
|
|
371
|
-
|
|
372
|
-
const errorText = await response.text();
|
|
373
|
-
if (attempt <= MAX_RETRIES && isRetryableStatus(response.status, errorText)) {
|
|
374
|
-
const delay = backoffMs(attempt);
|
|
375
|
-
await new Promise<void>((resolve) => setTimeout(resolve, delay));
|
|
376
|
-
continue;
|
|
430
|
+
return withRequestDeadline(signal, async (signal) => {
|
|
431
|
+
for (let attempt = 1; attempt <= MAX_RETRIES + 1; attempt++) {
|
|
432
|
+
signal.throwIfAborted();
|
|
433
|
+
let response: Response;
|
|
434
|
+
try {
|
|
435
|
+
response = await abortable(fetch(CODEX_RESPONSES_URL, {
|
|
436
|
+
method: "POST", headers, body, signal, redirect: "error",
|
|
437
|
+
}), signal);
|
|
438
|
+
} catch {
|
|
439
|
+
signal.throwIfAborted();
|
|
440
|
+
throw new Error("Codex connection failed. No automatic retry was made; check connectivity before trying again.");
|
|
377
441
|
}
|
|
378
|
-
throw new Error(`Codex image generation request failed (${response.status}): ${errorText}`);
|
|
379
|
-
}
|
|
380
442
|
|
|
381
|
-
|
|
382
|
-
|
|
443
|
+
if (!response.ok) {
|
|
444
|
+
const failure = await httpFailure(response, signal);
|
|
445
|
+
if (attempt <= MAX_RETRIES && failure.retry) {
|
|
446
|
+
const delay = retryDelayMs(attempt, response.headers.get("retry-after"));
|
|
447
|
+
await abortableDelay(delay, signal);
|
|
448
|
+
continue;
|
|
449
|
+
}
|
|
450
|
+
throw new Error(failure.message);
|
|
451
|
+
}
|
|
383
452
|
|
|
384
|
-
|
|
453
|
+
return parseCodexSse(response, signal, [token, accountId], onProgress);
|
|
454
|
+
}
|
|
455
|
+
throw new Error("Codex image generation request failed after all retries.");
|
|
456
|
+
});
|
|
385
457
|
}
|
|
386
458
|
|
|
387
459
|
// --- Extension entry point ---
|
|
@@ -393,58 +465,91 @@ export default function codexImageGen(pi: ExtensionAPI) {
|
|
|
393
465
|
name: "codex_generate_image",
|
|
394
466
|
label: "Codex Image",
|
|
395
467
|
description:
|
|
396
|
-
"Generate an image with the OpenAI Codex ChatGPT backend built-in image_generation tool (
|
|
397
|
-
promptSnippet: "Generate bitmap images via the OpenAI Codex ChatGPT backend
|
|
468
|
+
"Generate or edit an image with the OpenAI Codex ChatGPT backend built-in image_generation tool. The backend selects the image model. Accepts up to five local or recent conversation images (20 MiB each, 50 MiB total). Uses the existing openai-codex login; does not require OPENAI_API_KEY. Network deadline: 5 minutes; output image limit: 32 MiB; backend text is limited to 4,000 characters.",
|
|
469
|
+
promptSnippet: "Generate or edit bitmap images via the OpenAI Codex ChatGPT backend image_generation tool.",
|
|
398
470
|
promptGuidelines: [
|
|
399
|
-
"Use codex_generate_image when the user asks to generate a raster image
|
|
471
|
+
"Use codex_generate_image when the user asks to generate or edit a raster image with OpenAI/Codex image generation.",
|
|
400
472
|
"Do not use codex_generate_image without a clear image-generation request, because it consumes the user's Codex image quota.",
|
|
473
|
+
"The model parameter selects a Codex routing model, not an image model. Do not pass gpt-image-* IDs.",
|
|
474
|
+
"Output metadata is backend-reported, not independently verified. Check pixels for dimensions and transparency; do not infer a served model from appearance or a successful request.",
|
|
475
|
+
"Do not automatically repeat quota, connection, deadline, or incomplete-stream failures. The backend may already have consumed image quota.",
|
|
401
476
|
],
|
|
402
477
|
parameters: TOOL_PARAMS,
|
|
403
478
|
executionMode: "parallel", // #4: safe to run concurrently — no shared state, saves serialized per-path
|
|
404
479
|
async execute(toolCallId, params: ToolParams, signal, onUpdate, ctx) {
|
|
480
|
+
if (typeof params.prompt !== "string" || !params.prompt.trim() || params.prompt.length > MAX_PROMPT_CHARS) {
|
|
481
|
+
throw new Error("Image prompt must contain 1 to 32,000 characters.");
|
|
482
|
+
}
|
|
405
483
|
const outputFormat = params.outputFormat || "png";
|
|
406
|
-
|
|
484
|
+
if (!OUTPUT_FORMATS.includes(outputFormat)) throw new Error("Unsupported image output format.");
|
|
485
|
+
const projectTrusted = typeof ctx.isProjectTrusted === "function" && ctx.isProjectTrusted();
|
|
486
|
+
const config = loadConfig(ctx.cwd, projectTrusted); // #5: load once, pass to resolveSaveConfig
|
|
407
487
|
const requestedModel = params.model || config.model || DEFAULT_MODEL;
|
|
488
|
+
if (typeof requestedModel !== "string" || !requestedModel.trim() || requestedModel.length > 200) {
|
|
489
|
+
throw new Error("Codex routing model must contain 1 to 200 characters.");
|
|
490
|
+
}
|
|
491
|
+
if (requestedModel.startsWith("gpt-image-")) {
|
|
492
|
+
throw new Error("The model parameter selects a Codex routing model, not an image model. Subscription image-model selection is not verified.");
|
|
493
|
+
}
|
|
408
494
|
const model = ctx.modelRegistry.find(PROVIDER, requestedModel)?.id || requestedModel; // #6: removed dead FALLBACK_MODEL
|
|
495
|
+
const sessionId = ctx.sessionManager.getSessionId();
|
|
496
|
+
const saveConfig = resolveSaveConfig(params, ctx.cwd, sessionId, config);
|
|
409
497
|
const token = await ctx.modelRegistry.getApiKeyForProvider(PROVIDER);
|
|
410
498
|
if (!token) {
|
|
411
499
|
throw new Error(`Missing ${PROVIDER} credentials. Run /login and select ChatGPT Plus/Pro (Codex).`);
|
|
412
500
|
}
|
|
413
501
|
const accountId = extractChatGptAccountId(token);
|
|
414
|
-
const
|
|
502
|
+
const messages: unknown[] = [];
|
|
503
|
+
for (const entry of ctx.sessionManager.getBranch()) {
|
|
504
|
+
if (entry.type === "message") messages.push(entry.message);
|
|
505
|
+
if (entry.type === "custom_message") messages.push(entry);
|
|
506
|
+
}
|
|
507
|
+
const inputImages = await resolveInputImages(params, ctx.cwd, messages);
|
|
415
508
|
|
|
416
509
|
onUpdate?.({
|
|
417
|
-
content: [{ type: "text", text: `Requesting
|
|
418
|
-
details: { provider: PROVIDER, model, outputFormat },
|
|
510
|
+
content: [{ type: "text", text: `Requesting image ${inputImages.length > 0 ? "edit" : "generation"} through ${PROVIDER}/${model}...` }],
|
|
511
|
+
details: { provider: PROVIDER, model, outputFormat, inputImageCount: inputImages.length },
|
|
419
512
|
});
|
|
420
513
|
|
|
421
|
-
const
|
|
514
|
+
const started = Date.now();
|
|
515
|
+
const parsed = await requestImage(params, token, accountId, model, outputFormat, sessionId, inputImages, signal, (stage) => {
|
|
516
|
+
onUpdate?.({
|
|
517
|
+
content: [{ type: "text", text: `Codex image stage: ${stage}.` }],
|
|
518
|
+
details: { provider: PROVIDER, model, stage },
|
|
519
|
+
});
|
|
520
|
+
});
|
|
422
521
|
if (!parsed.image) {
|
|
423
522
|
const text = parsed.text.join("").trim();
|
|
424
523
|
throw new Error(text ? `Codex did not return an image. Response text: ${text}` : "Codex did not return an image.");
|
|
425
524
|
}
|
|
426
525
|
|
|
427
|
-
const
|
|
526
|
+
const imageBytes = decodeImageData(parsed.image.result, outputFormat);
|
|
527
|
+
const reportedImage = parsed.image.reported;
|
|
428
528
|
let savedPath: string | undefined;
|
|
529
|
+
let attemptedPath: string | undefined;
|
|
530
|
+
let saveWarning: string | undefined;
|
|
429
531
|
if (saveConfig.mode !== "none" && saveConfig.outputDir) {
|
|
430
|
-
|
|
431
|
-
|
|
432
|
-
|
|
433
|
-
|
|
434
|
-
|
|
435
|
-
provider: PROVIDER,
|
|
436
|
-
|
|
437
|
-
|
|
438
|
-
|
|
439
|
-
|
|
440
|
-
});
|
|
532
|
+
attemptedPath = imagePath(outputFormat, saveConfig.outputDir, parsed.image.id || toolCallId);
|
|
533
|
+
try {
|
|
534
|
+
savedPath = await saveImage(imageBytes, outputFormat, saveConfig.outputDir, parsed.image.id || toolCallId);
|
|
535
|
+
onUpdate?.({
|
|
536
|
+
content: [{ type: "text", text: `Image saved to ${savedPath}.` }],
|
|
537
|
+
details: { provider: PROVIDER, model, savedPath, byteCount: imageBytes.length },
|
|
538
|
+
});
|
|
539
|
+
} catch (error) {
|
|
540
|
+
saveWarning = `Image generation succeeded, but the image could not be saved to disk: ${error instanceof Error ? error.message : String(error)}`;
|
|
541
|
+
}
|
|
441
542
|
}
|
|
442
543
|
|
|
443
544
|
const summary = [
|
|
444
|
-
`Generated image via ${PROVIDER}/${model} using backend
|
|
545
|
+
`Generated image via ${PROVIDER}/${model} using the backend-selected image model.`,
|
|
445
546
|
`Status: ${parsed.image.status}.`,
|
|
547
|
+
reportedImage.size ? `Backend-reported size: ${reportedImage.size}.` : undefined,
|
|
548
|
+
reportedImage.quality ? `Backend-reported quality: ${reportedImage.quality}.` : undefined,
|
|
549
|
+
reportedImage.background ? `Backend-reported background: ${reportedImage.background}.` : undefined,
|
|
446
550
|
parsed.image.revisedPrompt ? `Revised prompt: ${parsed.image.revisedPrompt}` : undefined,
|
|
447
551
|
savedPath ? `Saved image to: ${savedPath}` : "Image was not saved to disk.",
|
|
552
|
+
saveWarning ? `Warning: ${saveWarning}` : undefined,
|
|
448
553
|
]
|
|
449
554
|
.filter(Boolean)
|
|
450
555
|
.join(" ");
|
|
@@ -457,10 +562,17 @@ export default function codexImageGen(pi: ExtensionAPI) {
|
|
|
457
562
|
details: {
|
|
458
563
|
provider: PROVIDER,
|
|
459
564
|
model,
|
|
460
|
-
backendImageModel: "
|
|
565
|
+
backendImageModel: reportedImage.model ?? "unknown",
|
|
566
|
+
reportedImage,
|
|
567
|
+
transport: "codex-responses",
|
|
568
|
+
generationDurationMs: Date.now() - started,
|
|
569
|
+
byteCount: imageBytes.length,
|
|
461
570
|
outputFormat,
|
|
462
571
|
saveMode: saveConfig.mode,
|
|
463
572
|
savedPath,
|
|
573
|
+
attemptedPath,
|
|
574
|
+
saveWarning,
|
|
575
|
+
inputImageCount: inputImages.length,
|
|
464
576
|
responseId: parsed.responseId,
|
|
465
577
|
imageGenerationId: parsed.image.id,
|
|
466
578
|
revisedPrompt: parsed.image.revisedPrompt,
|