pi-cliproxyapi-native 0.3.1 → 0.3.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/README.md CHANGED
@@ -13,6 +13,8 @@ Chat uses Pi's native API adapters and model metadata, with availability from yo
13
13
  - [Generate media](#generate-media)
14
14
  - [Resolution and aspect ratio](#resolution-and-aspect-ratio)
15
15
  - [OpenAI image controls and editing](#openai-image-controls-and-editing)
16
+ - [Image usage and cost](#image-usage-and-cost)
17
+ - [Pi image models](#pi-image-models)
16
18
  - [Supported media models](#supported-media-models)
17
19
  - [Files, status, and limits](#files-status-and-limits)
18
20
  - [Chat models](#chat-models)
@@ -34,10 +36,10 @@ The extension targets Pi 1.4.0 and CLIProxyAPI 8.0.48, with Node.js 22.19 or new
34
36
  If you installed `pi-cliproxy-native`, remove it before installing the renamed package. Both register the same `cliproxyapi` provider.
35
37
 
36
38
  ```sh
37
- pi install npm:pi-cliproxyapi-native@0.3.1
39
+ pi install npm:pi-cliproxyapi-native@0.3.2
38
40
  ```
39
41
 
40
- For Git installation, use `pi install git:github.com/hawkff/pi-cliproxyapi-native@v0.3.1`. For a local checkout, use `pi install /path/to/pi-cliproxyapi-native`.
42
+ For Git installation, use `pi install git:github.com/hawkff/pi-cliproxyapi-native@v0.3.2`. For a local checkout, use `pi install /path/to/pi-cliproxyapi-native`.
41
43
 
42
44
  The default proxy address is `http://localhost:8317`. For another address, set the [connection](#connection) before logging in. Restart Pi, then run:
43
45
 
@@ -154,17 +156,18 @@ Control values follow the [OpenAI image guide](https://developers.openai.com/api
154
156
 
155
157
  Use `cliproxyapi_media_models` to inspect controls for the exact advertised model ID. Omitted options keep the backend defaults. The extension rejects unsupported values before contacting the proxy.
156
158
 
157
- | Option | Supported values |
158
- | -------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
159
- | `quality` | `auto`, `low`, `medium`, `high`. Sunburst and Flare also accept `xhigh` and `max`. |
160
- | `background` | `auto`, `opaque`, `transparent`. Transparent output requires PNG or WebP; GPT Image 2 transparency is in preview upstream. |
161
- | `output_format` | `png`, `jpeg`, `webp`. JPEG and WebP can reduce file size and help keep results within the response limit. |
162
- | `output_compression` | Integer from 0 to 100, with an explicit `jpeg` or `webp` format. Zero is a valid value. |
163
- | `moderation` | `auto` or `low`. OpenAI still applies its content policies. |
164
- | `n` | Integer from 1 to 10 on an explicit `api/` image route. Other routes allow only 1 because subscription translation can drop the count. |
165
- | `images` | One to 16 local PNG, JPEG, or WebP paths. Supplying these selects `/v1/images/edits` instead of generation. |
166
- | `mask` | Optional local PNG mask. The first image must also be PNG with matching dimensions. The mask needs a grayscale-alpha or RGBA alpha channel; transparent regions mark edits. |
167
- | `input_fidelity` | `low` or `high`, for GPT Image 1.5 editing only. Omit it for other models. |
159
+ | Option | Supported values |
160
+ | -------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
161
+ | `quality` | `auto`, `low`, `medium`, `high`. Sunburst and Flare also accept `xhigh` and `max`. |
162
+ | `background` | `auto`, `opaque`, `transparent`. Transparent output requires PNG or WebP; GPT Image 2 transparency is in preview upstream. |
163
+ | `output_format` | `png`, `jpeg`, `webp`. JPEG and WebP can reduce file size and help keep results within the response limit. |
164
+ | `output_compression` | Integer from 0 to 100, with an explicit `jpeg` or `webp` format. Zero is a valid value. |
165
+ | `moderation` | `auto` or `low`. OpenAI still applies its content policies. |
166
+ | `n` | Integer from 1 to 10 on an explicit `api/` image route. Other routes allow only 1 because subscription translation can drop the count. |
167
+ | `partial_images` | Integer from 0 to 3. The proxy streams up to that many draft frames while the image renders, and Pi shows them as tool progress. Requires `n` of 1. Each draft adds about 100 image output tokens. |
168
+ | `images` | One to 16 local PNG, JPEG, or WebP paths. Supplying these selects `/v1/images/edits` instead of generation. |
169
+ | `mask` | Optional local PNG mask. The first image must also be PNG with matching dimensions. The mask needs a grayscale-alpha or RGBA alpha channel; transparent regions mark edits. |
170
+ | `input_fidelity` | `low` or `high`, for GPT Image 1.5 editing only. Omit it for other models. |
168
171
 
169
172
  Editing uploads the files you name to the configured proxy. Paths can be absolute or relative to the working directory. The extension accepts regular files, checks file signatures, and limits uploads to 14 MiB per file and 20 MiB total including the mask. It does not fetch input URLs, resize inputs, or modify source files. OpenAI performs full image decoding and validates the edit. These local limits keep base64 uploads bounded and may be lower than upstream limits.
170
173
 
@@ -173,9 +176,21 @@ Generate two images with api/gpt-image-2.5-sunburst at 1536x864, quality max, ou
173
176
  Edit ./photo.png with api/gpt-image-2.5-flare: replace the background with a garden, using ./mask.png as the mask.
174
177
  ```
175
178
 
176
- All returned images share the existing 32 MiB response limit and 4 MiB preview budget. Larger sizes, higher quality, and multiple images can increase cost and latency. Use fewer images or JPEG/WebP if the response exceeds the limit. Generation and editing do not retry after a failure.
179
+ All returned images share the existing 32 MiB response limit and 4 MiB preview budget. Streamed requests allow another 32 MiB for each draft frame. The extension saves only the final image and drops drafts it cannot decode. Larger sizes, higher quality, and multiple images can increase cost and latency. Use fewer images or JPEG/WebP if the response exceeds the limit. Generation and editing do not retry after a failure.
177
180
 
178
- The extension supports the verified options above, not a general OpenAI API passthrough. Streaming image previews, remote file IDs, annotation markup, audio, batches, and realtime APIs are not exposed. Adding an upstream API key does not enable endpoints that the proxy or extension cannot handle.
181
+ The extension supports the verified options above, not a general OpenAI API passthrough. Remote file IDs, annotation markup, audio, batches, and realtime APIs are not exposed. Adding an upstream API key does not enable endpoints that the proxy or extension cannot handle.
182
+
183
+ ### Image usage and cost
184
+
185
+ OpenAI image responses report token usage. The image tool returns it to Pi, so image calls count toward the session totals in the footer and `/session`.
186
+
187
+ Costs are estimates at OpenAI list prices per million tokens: $5 for text input, $8 for image input, and $30 for image output on GPT Image 2 and 2.5 Sunburst and Flare. GPT Image 1.5 output costs $32. The plain `gpt-image-2.5` subscription model has no published price, so it reports tokens at zero cost. Subscription routes show the same estimates when the proxy reports usage, even though the plan covers them. Google and xAI image results report no usage.
188
+
189
+ ### Pi image models
190
+
191
+ When Pi refreshes the catalog, at interactive startup or through `/cliproxyapi-refresh`, the advertised image models also register as Pi image models. Codemode scripts list them with `models.getAvailableOfType("image", "cliproxyapi")` and call `models.generateImages()`, and other extensions can use `ctx.modelRegistry.generateImages()`.
192
+
193
+ These calls send the prompt with backend defaults. Pi's image interface has no size, quality, count, mask, or preview options. Image blocks in the input select OpenAI editing, under the same 16-image, 14 MiB per image, and 20 MiB total limits. Google and xAI models accept text only. Results arrive as image blocks with usage and are not saved to disk. Use `cliproxyapi_generate_image` when you need controls, masks, or saved files.
179
194
 
180
195
  ### Supported media models
181
196
 
@@ -187,7 +202,7 @@ Your proxy must advertise the exact ID for a model to be available.
187
202
  | Google models | `gemini-2.5-flash-image`, `gemini-3.1-flash-image`, `gemini-3-pro-image`, `gemini-3.1-flash-lite-image` |
188
203
  | xAI models | `grok-imagine-image`, `grok-imagine-image-quality`, `grok-imagine-image-2.0`, `grok-imagine-video`, `grok-imagine-video-1.5`, `grok-imagine-video-1.5-preview` |
189
204
 
190
- Supported media IDs can also use advertised `api/` and `oauth/` credential routes. Gemini image models also support `vertex/` and `antigravity/` backend routes. Those backend prefixes do not enable OpenAI or xAI media execution; the picker shows those entries as disabled. Other prefixes, nested prefixes, media aliases, unknown media IDs, Google Veo, and non-OpenAI editing are unsupported.
205
+ Supported media IDs can also use advertised `api/` and `oauth/` credential routes. OpenAI's API has no plain `gpt-image-2.5`, so the picker disables `api/gpt-image-2.5` and points to Sunburst or Flare; subscription routes keep it. Gemini image models also support `vertex/` and `antigravity/` backend routes. Those backend prefixes do not enable OpenAI or xAI media execution; the picker shows those entries as disabled. Other prefixes, nested prefixes, media aliases, unknown media IDs, Google Veo, and non-OpenAI editing are unsupported.
191
206
 
192
207
  The extension disables these retired Imagen IDs, including their supported prefixed forms:
193
208
 
@@ -213,21 +228,23 @@ Video submission returns a request ID without waiting for generation. Call `clip
213
228
  | ---------------------------------- | ---------- | ---------------------------------------------- |
214
229
  | Catalog fetch | 10 seconds | 4 MiB |
215
230
  | Media listing or video status | 15 seconds | 4 MiB for the catalog; 64 KiB for video status |
216
- | OpenAI image generation or editing | 10 minutes | 32 MiB |
231
+ | OpenAI image generation or editing | 10 minutes | 32 MiB, plus 32 MiB per streamed draft |
217
232
  | Other image generation | 3 minutes | 32 MiB |
218
233
  | Video submission | 60 seconds | 64 KiB |
219
234
 
220
- Media deadlines include input-file reads, authentication, and network waits. Discovery and media requests reject redirects and report HTTP errors without upstream response bodies.
235
+ Media deadlines include input-file reads, authentication, and network waits. Discovery and media requests reject redirects. Media errors quote the `message` from an upstream JSON error, cut to 500 characters, with credentials and control characters removed. Other error bodies and discovery errors report only the HTTP status.
221
236
 
222
237
  The media tools do not retry generation. Cancellation or timeout can leave work running upstream. Check a known video request ID before submitting another generation.
223
238
 
224
239
  ## Chat models
225
240
 
226
- The extension discovers available IDs through `/v1/models` and matches them against Pi's built-in metadata for limits, prices, inputs, and thinking capabilities.
241
+ The extension discovers available IDs through `/v1/models` and matches them against Pi's Anthropic, OpenAI, OpenAI Codex, Google, and xAI metadata for limits, prices, inputs, and thinking capabilities. xAI Grok models use Pi's native Responses adapter through the proxy.
242
+
243
+ During a session, the extension also reads the provider catalogs that Pi refreshes from its catalog service. Models newer than Pi's bundled data can then register; for models Pi already bundles, the bundled metadata wins.
227
244
 
228
245
  ### Refresh
229
246
 
230
- Run `/cliproxyapi-refresh` to update the chat catalog.
247
+ Run `/cliproxyapi-refresh` to update the chat and image catalogs. It reports both counts and lists up to 20 advertised IDs that it could not register because Pi has no metadata for them or the match is ambiguous.
231
248
 
232
249
  ### Wait for quota reset
233
250
 
@@ -361,9 +378,9 @@ Run `/reload` after changing settings and `/cliproxyapi-refresh` before starting
361
378
 
362
379
  Claude uses the legacy fine-grained tool-streaming header in place of eager tool fields. The extension disables unverified deferred-tool and strict-tool capabilities. Responses function tools use `strict: null` when Pi omits strictness, so optional arguments remain optional. Explicit payload hooks retain final control.
363
380
 
364
- OpenAI images POST `model`, `prompt`, `n`, and supplied controls to `/v1/images/generations`, without `response_format`. Editing uses `/v1/images/edits` with JSON `images` and optional `mask` data URLs. xAI uses the same route with `response_format: "b64_json"`. Gemini uses `/v1beta/models/{id}:generateContent`. Media generation sends the prompt and generation options without chat history, system instructions, or tools.
381
+ OpenAI images POST `model`, `prompt`, `n`, and supplied controls to `/v1/images/generations`, without `response_format`. Editing uses `/v1/images/edits` with JSON `images` and optional `mask` data URLs. A nonzero `partial_images` adds `stream: true`, and the extension reads server-sent events. Each `*.partial_image` frame becomes a progress preview, and the `*.completed` frame carries the final image and usage. xAI uses the same route with `response_format: "b64_json"`. Gemini uses `/v1beta/models/{id}:generateContent`. Media generation sends the prompt and generation options without chat history, system instructions, or tools.
365
382
 
366
- OpenAI image POSTs use a dedicated HTTP connection and preserve Node's configured proxy routing. They disable socket timeouts so the ten-minute deadline controls the request, request identity encoding, and reject compressed responses.
383
+ OpenAI image POSTs use a dedicated HTTP connection. It follows `HTTP_PROXY`, `HTTPS_PROXY`, and `NO_PROXY` from the environment, where Pi also places its `httpProxy` setting, on Node 22.21+ and 24.5+. Older Node versions connect directly. The connection disables socket timeouts so the ten-minute deadline controls the request, requests identity encoding, and rejects compressed responses.
367
384
 
368
385
  The OpenAI image implementation follows CLIProxyAPI v8.0.48's [image handler](https://github.com/router-for-me/CLIProxyAPI/blob/v8.0.48/sdk/api/handlers/openai/openai_images_handlers.go) and [Codex image executor](https://github.com/router-for-me/CLIProxyAPI/blob/v8.0.48/internal/runtime/executor/codex_openai_images.go), plus the OpenAI [generation](https://developers.openai.com/api/reference/resources/images/methods/generate) and [editing](https://developers.openai.com/api/reference/resources/images/methods/edit) references. The xAI and Google routes follow v7.2.158's [video handler](https://github.com/router-for-me/CLIProxyAPI/blob/v7.2.158/sdk/api/handlers/openai/openai_videos_handlers.go), [Gemini handler](https://github.com/router-for-me/CLIProxyAPI/blob/v7.2.158/sdk/api/handlers/gemini/gemini_handlers.go), and [Vertex executor](https://github.com/router-for-me/CLIProxyAPI/blob/v7.2.158/internal/runtime/executor/gemini_vertex_executor.go). See [xAI's video documentation](https://docs.x.ai/developers/model-capabilities/video/generation) for duration and status semantics.
369
386
 
@@ -4,7 +4,7 @@ import { type ExtensionAPI, type ExtensionContext, getAgentDir } from "@earendil
4
4
  import { PROVIDER_ID, parseConfig } from "../src/config.ts";
5
5
  import { registerMediaTools } from "../src/media.ts";
6
6
  import { registerModelPicker } from "../src/picker.ts";
7
- import { createCliproxyProvider } from "../src/provider.ts";
7
+ import { createCliproxyProvider, metadataCatalog } from "../src/provider.ts";
8
8
  import { registerAutoRetry } from "../src/retry.ts";
9
9
 
10
10
  export default async function (pi: ExtensionAPI) {
@@ -37,32 +37,32 @@ export default async function (pi: ExtensionAPI) {
37
37
  context = undefined;
38
38
  waits.clear();
39
39
  });
40
- pi.registerProvider(
41
- createCliproxyProvider(
42
- config,
43
- undefined,
44
- (id, seconds) => {
45
- if (!context) return;
46
- if (seconds === undefined) waits.delete(id);
47
- else {
48
- if (!waits.has(id)) {
49
- const message = `CLIProxyAPI quota limit. Retrying at ${new Date(Date.now() + seconds * 1000).toLocaleString()}. Cancel the current request to stop waiting.`;
50
- if (context.mode === "print" || context.mode === "json") process.stderr.write(`${message}\n`);
51
- else context.ui.notify(message, "warning");
52
- }
53
- waits.set(id, seconds);
40
+ const provider = createCliproxyProvider(
41
+ config,
42
+ // Session context exposes Pi's refreshed catalogs, including models newer than its bundled data.
43
+ () => metadataCatalog(context && ((id) => context?.modelRegistry.getProvider(id)?.getModels())),
44
+ (id, seconds) => {
45
+ if (!context) return;
46
+ if (seconds === undefined) waits.delete(id);
47
+ else {
48
+ if (!waits.has(id)) {
49
+ const message = `CLIProxyAPI quota limit. Retrying at ${new Date(Date.now() + seconds * 1000).toLocaleString()}. Cancel the current request to stop waiting.`;
50
+ if (context.mode === "print" || context.mode === "json") process.stderr.write(`${message}\n`);
51
+ else context.ui.notify(message, "warning");
54
52
  }
55
- const remaining = Math.min(...waits.values());
56
- context.ui.setStatus(
57
- "cliproxyapi-quota",
58
- waits.size
59
- ? `CLIProxyAPI quota: retry in ${Math.floor(remaining / 3600)}h ${Math.floor(remaining / 60) % 60}m ${remaining % 60}s`
60
- : undefined,
61
- );
62
- },
63
- compactionSignals,
64
- ),
53
+ waits.set(id, seconds);
54
+ }
55
+ const remaining = Math.min(...waits.values());
56
+ context.ui.setStatus(
57
+ "cliproxyapi-quota",
58
+ waits.size
59
+ ? `CLIProxyAPI quota: retry in ${Math.floor(remaining / 3600)}h ${Math.floor(remaining / 60) % 60}m ${remaining % 60}s`
60
+ : undefined,
61
+ );
62
+ },
63
+ compactionSignals,
65
64
  );
65
+ pi.registerProvider(provider);
66
66
  registerAutoRetry(pi);
67
67
  registerMediaTools(pi, config);
68
68
  registerModelPicker(pi, config);
@@ -88,7 +88,16 @@ export default async function (pi: ExtensionAPI) {
88
88
  return;
89
89
  }
90
90
  const count = ctx.modelRegistry.getAll().filter((model) => model.provider === PROVIDER_ID).length;
91
- ctx.ui.notify(`CLIProxyAPI: ${count} models.`, "info");
91
+ const images = ctx.modelRegistry.getModelsOfType("image", PROVIDER_ID).length;
92
+ const skipped = provider.skippedModels();
93
+ const listed = [
94
+ ...skipped.slice(0, 20),
95
+ ...(skipped.length > 20 ? [`${skipped.length - 20} more`] : []),
96
+ ];
97
+ ctx.ui.notify(
98
+ `CLIProxyAPI: ${count} chat models, ${images} image models.${listed.length ? ` Not registered (missing or ambiguous Pi metadata): ${listed.join(", ")}.` : ""}`,
99
+ "info",
100
+ );
92
101
  },
93
102
  });
94
103
  }
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "pi-cliproxyapi-native",
3
- "version": "0.3.1",
3
+ "version": "0.3.2",
4
4
  "description": "CLIProxyAPI chat discovery, image generation and editing, and video generation with Pi's native adapters",
5
5
  "publishConfig": {
6
6
  "access": "public"
package/src/catalog.ts CHANGED
@@ -1,5 +1,5 @@
1
1
  import type { Api, Model } from "@earendil-works/pi-ai";
2
- import { type Config, isModelId, isRecord, PROVIDER_ID } from "./config.ts";
2
+ import { type Config, deadline, isModelId, isRecord, PROVIDER_ID, validateKey } from "./config.ts";
3
3
 
4
4
  export type CpaApi =
5
5
  | "anthropic-messages"
@@ -34,6 +34,51 @@ export function parseCatalog(value: unknown) {
34
34
  });
35
35
  }
36
36
 
37
+ export async function fetchCatalog(config: Config, key: string, signal: AbortSignal) {
38
+ validateKey(key);
39
+ const boundedSignal = deadline(signal, 10000);
40
+ let response: Response;
41
+ try {
42
+ response = await fetch(`${config.baseUrl}/v1/models`, {
43
+ headers: { Authorization: `Bearer ${key}` },
44
+ redirect: "error",
45
+ signal: boundedSignal,
46
+ });
47
+ } catch {
48
+ signal.throwIfAborted();
49
+ throw new Error("CLIProxyAPI discovery failed: connection, timeout, or redirect error.");
50
+ }
51
+ if (!response.ok) {
52
+ await response.body?.cancel();
53
+ throw new Error(`CLIProxyAPI discovery failed (HTTP ${response.status}).`);
54
+ }
55
+ let payload: unknown;
56
+ const reader = response.body?.getReader();
57
+ if (!reader) throw new Error("CLIProxyAPI discovery returned an empty response.");
58
+ try {
59
+ const decoder = new TextDecoder();
60
+ let text = "";
61
+ let size = 0;
62
+ while (true) {
63
+ const chunk = await reader.read();
64
+ if (chunk.done) break;
65
+ size += chunk.value.byteLength;
66
+ if (size > 4 * 1024 * 1024) {
67
+ await reader.cancel();
68
+ throw new Error("Catalog too large.");
69
+ }
70
+ text += decoder.decode(chunk.value, { stream: true });
71
+ }
72
+ payload = JSON.parse(text + decoder.decode());
73
+ } catch {
74
+ signal.throwIfAborted();
75
+ throw new Error("CLIProxyAPI catalog is invalid, incomplete, or larger than 4 MiB.");
76
+ } finally {
77
+ reader.releaseLock();
78
+ }
79
+ return payload;
80
+ }
81
+
37
82
  // Resolve metadata and capabilities without changing registry or request IDs.
38
83
  export function modelRoute(id: string) {
39
84
  const match = /^(vertex|antigravity|oauth|api)\/(.+)$/.exec(id);
@@ -59,17 +104,31 @@ export function routedName(id: string, name = id) {
59
104
  export type MediaPurpose = "image" | "video";
60
105
 
61
106
  const imagenRetirement = "Retired on Vertex (June 2026); disabled. Select a Nano Banana image model instead.";
107
+ // OpenAI list prices in USD per million tokens: text input, image input, image output.
108
+ const gptImage2Price = [5, 8, 30] as const;
62
109
 
63
110
  // Exact media IDs, including retired entries; vision input alone does not imply image output.
64
111
  const mediaModels = new Map<
65
112
  string,
66
- { name: string; purpose: MediaPurpose; route?: "xai" | "gemini" | "openai"; disabledReason?: string }
113
+ {
114
+ name: string;
115
+ purpose: MediaPurpose;
116
+ route?: "xai" | "gemini" | "openai";
117
+ disabledReason?: string;
118
+ price?: readonly [number, number, number];
119
+ }
67
120
  >([
68
- ["gpt-image-2.5-flare", { name: "GPT Image 2.5 Flare", purpose: "image", route: "openai" }],
69
- ["gpt-image-2.5-sunburst", { name: "GPT Image 2.5 Sunburst", purpose: "image", route: "openai" }],
121
+ [
122
+ "gpt-image-2.5-flare",
123
+ { name: "GPT Image 2.5 Flare", purpose: "image", route: "openai", price: gptImage2Price },
124
+ ],
125
+ [
126
+ "gpt-image-2.5-sunburst",
127
+ { name: "GPT Image 2.5 Sunburst", purpose: "image", route: "openai", price: gptImage2Price },
128
+ ],
70
129
  ["gpt-image-2.5", { name: "GPT Image 2.5", purpose: "image", route: "openai" }],
71
- ["gpt-image-2", { name: "GPT Image 2", purpose: "image", route: "openai" }],
72
- ["gpt-image-1.5", { name: "GPT Image 1.5", purpose: "image", route: "openai" }],
130
+ ["gpt-image-2", { name: "GPT Image 2", purpose: "image", route: "openai", price: gptImage2Price }],
131
+ ["gpt-image-1.5", { name: "GPT Image 1.5", purpose: "image", route: "openai", price: [5, 8, 32] }],
73
132
  ["grok-imagine-image", { name: "Grok Imagine Image", purpose: "image", route: "xai" }],
74
133
  ["grok-imagine-image-quality", { name: "Grok Imagine Image Quality", purpose: "image", route: "xai" }],
75
134
  ["grok-imagine-image-2.0", { name: "Grok Imagine Image 2.0", purpose: "image", route: "xai" }],
@@ -100,11 +159,20 @@ const mediaModels = new Map<
100
159
  ]);
101
160
 
102
161
  export function mediaCapability(id: string) {
103
- const { metadataId, backend } = modelRoute(id);
162
+ const { metadataId, backend, authType } = modelRoute(id);
104
163
  const capability = mediaModels.get(metadataId);
105
164
  if (backend && (capability?.route === "xai" || capability?.route === "openai")) {
106
165
  return { ...capability, route: undefined, disabledReason: `Unsupported media execution on ${backend}.` };
107
166
  }
167
+ // CLIProxyAPI 8.0.48 advertises it for API keys, but OpenAI's Images API rejects the bare 2.5 name.
168
+ if (capability && authType === "api-key" && metadataId === "gpt-image-2.5") {
169
+ return {
170
+ ...capability,
171
+ route: undefined,
172
+ disabledReason:
173
+ "OpenAI's API has no gpt-image-2.5 model. Use api/gpt-image-2.5-sunburst or api/gpt-image-2.5-flare.",
174
+ };
175
+ }
108
176
  return capability;
109
177
  }
110
178
 
@@ -160,6 +228,7 @@ export function mediaControls(id: string) {
160
228
  moderation: openai ? ["auto", "low"] : [],
161
229
  // Subscription image-tool translation does not forward n.
162
230
  n: openai ? { min: 1, max: authType === "api-key" ? 10 : 1 } : undefined,
231
+ partial_images: openai ? { min: 0, max: 3 } : undefined,
163
232
  editing: openai
164
233
  ? { max_images: 16, max_file_bytes: 14 * 1024 * 1024, max_total_bytes: 20 * 1024 * 1024, mask: "png" }
165
234
  : undefined,
@@ -167,6 +236,29 @@ export function mediaControls(id: string) {
167
236
  };
168
237
  }
169
238
 
239
+ export const IMAGE_API = "cliproxyapi-images";
240
+
241
+ // Pi image models for codemode and extensions; video stays tool-only because Pi has no video interface.
242
+ export function mapImageModels(value: unknown, config: Config) {
243
+ return mapMediaCatalog(value).flatMap(({ id, purpose, name, controls }) => {
244
+ if (purpose !== "image") return [];
245
+ const [input = 0, , output = 0] = mediaCapability(id)?.price ?? [];
246
+ return [
247
+ {
248
+ type: "image" as const,
249
+ id,
250
+ name,
251
+ api: IMAGE_API,
252
+ provider: PROVIDER_ID,
253
+ baseUrl: config.baseUrl,
254
+ input: controls.editing ? ["text" as const, "image" as const] : ["text" as const],
255
+ output: ["image" as const],
256
+ cost: { input, output, cacheRead: 0, cacheWrite: 0 },
257
+ },
258
+ ];
259
+ });
260
+ }
261
+
170
262
  export function mapMediaCatalog(value: unknown) {
171
263
  const models = new Map<
172
264
  string,
@@ -194,6 +286,7 @@ const owners: Readonly<Record<string, string>> = {
194
286
  google: "google",
195
287
  gemini: "google",
196
288
  "gemini-cli": "google",
289
+ xai: "xai",
197
290
  };
198
291
 
199
292
  // Built-in aliases for advertised IDs without native metadata; exact metadata still wins.
@@ -209,11 +302,7 @@ export function mapCatalog(value: unknown, config: Config, known: readonly Model
209
302
  const byId = Map.groupBy(known, (model) => model.id);
210
303
  const byReference = Map.groupBy(known, (model) => `${model.provider}/${model.id}`);
211
304
  for (const entry of parseCatalog(value)) {
212
- if (entry.hidden || models.has(entry.id)) continue;
213
- if (mediaPurpose(entry.id)) {
214
- skipped.add(entry.id);
215
- continue;
216
- }
305
+ if (entry.hidden || models.has(entry.id) || mediaPurpose(entry.id)) continue;
217
306
  const { metadataId, prefix } = modelRoute(entry.id);
218
307
  const routed = !entry.id.includes("/") || prefix;
219
308
  const alias = Object.hasOwn(config.aliases, entry.id)
package/src/config.ts CHANGED
@@ -9,6 +9,17 @@ export function isModelId(value: unknown): value is string {
9
9
  return typeof value === "string" && /^[!-~]{1,256}$/.test(value);
10
10
  }
11
11
 
12
+ export function validateKey(key: string) {
13
+ if (!/^[!-~]{1,4096}$/.test(key)) {
14
+ throw new Error("CLIProxyAPI requires a non-empty API key without whitespace or control characters.");
15
+ }
16
+ return key;
17
+ }
18
+
19
+ export function deadline(signal: AbortSignal | undefined, milliseconds: number) {
20
+ return AbortSignal.any([...(signal ? [signal] : []), AbortSignal.timeout(milliseconds)]);
21
+ }
22
+
12
23
  export function normalizeBaseUrl(value: string) {
13
24
  let url: URL;
14
25
  try {
package/src/media.ts CHANGED
@@ -4,12 +4,22 @@ import * as http from "node:http";
4
4
  import * as https from "node:https";
5
5
  import { join, resolve } from "node:path";
6
6
  import { addAbortSignal, Readable } from "node:stream";
7
- import { type ImageContent, type Static, type TextContent, Type } from "@earendil-works/pi-ai";
7
+ import {
8
+ type AssistantImages,
9
+ type ImageApi,
10
+ type ImageContent,
11
+ type ImageModel,
12
+ type ImagesContext,
13
+ type ImagesOptions,
14
+ type Static,
15
+ type TextContent,
16
+ Type,
17
+ type Usage,
18
+ } from "@earendil-works/pi-ai";
8
19
  import type { ExtensionAPI, ExtensionContext } from "@earendil-works/pi-coding-agent";
9
- import { mapMediaCatalog, mediaCapability, mediaControls } from "./catalog.ts";
10
- import { type Config, isRecord, PROVIDER_ID } from "./config.ts";
20
+ import { fetchCatalog, mapMediaCatalog, mediaCapability, mediaControls } from "./catalog.ts";
21
+ import { type Config, deadline, isRecord, PROVIDER_ID, validateKey } from "./config.ts";
11
22
  import { type DefaultsContext, effectiveMediaDefaults, resolveMediaModel } from "./media-defaults.ts";
12
- import { deadline, fetchCatalog, validateKey } from "./provider.ts";
13
23
 
14
24
  type MediaContext = Pick<ExtensionContext, "cwd" | "model"> &
15
25
  DefaultsContext & {
@@ -17,6 +27,7 @@ type MediaContext = Pick<ExtensionContext, "cwd" | "model"> &
17
27
  };
18
28
 
19
29
  const MAX_INLINE_IMAGE_BASE64_BYTES = 4 * 1024 * 1024;
30
+ const MAX_IMAGE_RESPONSE_BYTES = 32 * 1024 * 1024;
20
31
 
21
32
  const generationParameters = Type.Object({
22
33
  model: Type.Optional(
@@ -79,6 +90,14 @@ const imageParameters = Type.Object({
79
90
  "OpenAI image count. Values above 1 require an explicit api/ route; see controls.n. Default 1.",
80
91
  }),
81
92
  ),
93
+ partial_images: Type.Optional(
94
+ Type.Integer({
95
+ minimum: 0,
96
+ maximum: 3,
97
+ description:
98
+ "OpenAI only: stream up to 3 partial previews while generating. Requires n of 1; each preview adds about 100 output tokens. Omit or 0 for none.",
99
+ }),
100
+ ),
82
101
  images: Type.Optional(
83
102
  Type.Array(Type.String({ minLength: 1 }), {
84
103
  minItems: 1,
@@ -129,6 +148,7 @@ function mediaOutputOptions(model: string, params: Partial<Static<typeof imagePa
129
148
  moderation: params.moderation,
130
149
  n: params.n,
131
150
  input_fidelity: params.input_fidelity,
151
+ partial_images: params.partial_images,
132
152
  };
133
153
  const controls = mediaControls(model);
134
154
  for (const field of [
@@ -171,6 +191,17 @@ function mediaOutputOptions(model: string, params: Partial<Static<typeof imagePa
171
191
  (!controls.n || !Number.isInteger(options.n) || options.n < controls.n.min || options.n > controls.n.max)
172
192
  )
173
193
  throw new Error("CLIProxyAPI n is unsupported for this route. Use an api/ image model for 2–10 images.");
194
+ if (
195
+ options.partial_images !== undefined &&
196
+ (!controls.partial_images ||
197
+ !Number.isInteger(options.partial_images) ||
198
+ options.partial_images < controls.partial_images.min ||
199
+ options.partial_images > controls.partial_images.max ||
200
+ (options.partial_images > 0 && (options.n ?? 1) !== 1))
201
+ )
202
+ throw new Error(
203
+ "CLIProxyAPI partial_images requires an OpenAI image model, n of 1, and an integer from 0 to 3.",
204
+ );
174
205
  if (
175
206
  options.output_compression !== undefined &&
176
207
  (!controls.output_compression ||
@@ -296,64 +327,180 @@ export async function mediaKey(ctx: Pick<MediaContext, "modelRegistry">, signal:
296
327
  }
297
328
  }
298
329
 
299
- function openaiImageResponse(config: Config, key: string, path: string, body: object, signal: AbortSignal) {
300
- return new Promise<{ ok: boolean; status: number; body: ReturnType<typeof Readable.toWeb> | null }>(
301
- (resolve, reject) => {
302
- const url = new URL(`${config.baseUrl}/${path}`);
303
- const transport = url.protocol === "https:" ? https : http;
304
- const options = "options" in transport.globalAgent ? transport.globalAgent.options : undefined;
305
- // Preserve configured proxies without inheriting the global agent's socket timeout.
306
- const agent = new transport.Agent({
307
- ...(isRecord(options) ? options : {}),
308
- keepAlive: false,
330
+ // Pi's native image interface can supply transport options; the image tool uses none.
331
+ type TransportOptions = Pick<ImagesOptions, "fetch" | "headers" | "env"> & {
332
+ onResponse?: (response: { status: number; headers: Record<string, string> }) => unknown;
333
+ };
334
+
335
+ function requestHeaders(key: string, extra: TransportOptions["headers"] = {}) {
336
+ const headers: Record<string, string> = {};
337
+ for (const [name, value] of Object.entries(extra))
338
+ if (value !== null && !["authorization", "content-type", "accept-encoding"].includes(name.toLowerCase()))
339
+ headers[name] = value;
340
+ return { ...headers, Authorization: `Bearer ${key}`, "Content-Type": "application/json" };
341
+ }
342
+
343
+ function openaiImageResponse(
344
+ config: Config,
345
+ key: string,
346
+ path: string,
347
+ body: unknown,
348
+ signal: AbortSignal,
349
+ { headers, env }: TransportOptions,
350
+ ) {
351
+ return new Promise<{
352
+ ok: boolean;
353
+ status: number;
354
+ headers: Record<string, string>;
355
+ body: ResponseBody | null;
356
+ }>((resolve, reject) => {
357
+ const url = new URL(`${config.baseUrl}/${path}`);
358
+ const transport = url.protocol === "https:" ? https : http;
359
+ const options = "options" in transport.globalAgent ? transport.globalAgent.options : undefined;
360
+ // Pi exports its proxy setting to the environment. Node 22.21+ and 24.5+ apply proxyEnv,
361
+ // including NO_PROXY; older Node versions connect directly. The global socket timeout is dropped.
362
+ const agent = new transport.Agent({
363
+ ...(isRecord(options) ? options : {}),
364
+ proxyEnv: { ...process.env, ...env },
365
+ keepAlive: false,
366
+ timeout: 0,
367
+ });
368
+ const connect = agent.createConnection.bind(agent);
369
+ agent.createConnection = (connectionOptions, callback) => {
370
+ // Node assigns HTTPS proxy sockets to the request only after CONNECT completes.
371
+ const socket = connect(connectionOptions, callback);
372
+ return socket ? addAbortSignal(signal, socket) : socket;
373
+ };
374
+ const request = transport.request(
375
+ url,
376
+ {
377
+ method: "POST",
378
+ agent,
309
379
  timeout: 0,
310
- });
311
- const connect = agent.createConnection.bind(agent);
312
- agent.createConnection = (connectionOptions, callback) => {
313
- // Node assigns HTTPS proxy sockets to the request only after CONNECT completes.
314
- const socket = connect(connectionOptions, callback);
315
- return socket ? addAbortSignal(signal, socket) : socket;
316
- };
317
- const request = transport.request(
318
- url,
319
- {
320
- method: "POST",
321
- agent,
322
- timeout: 0,
323
- signal,
324
- headers: {
325
- Authorization: `Bearer ${key}`,
326
- "Content-Type": "application/json",
327
- "Accept-Encoding": "identity",
328
- },
329
- },
330
- (incoming) => {
331
- const status = incoming.statusCode ?? 500;
332
- if (status < 200 || status >= 300) {
333
- incoming.destroy();
334
- resolve({ ok: false, status, body: null });
335
- return;
336
- }
337
- if (incoming.headers["content-encoding"] && incoming.headers["content-encoding"] !== "identity") {
338
- incoming.destroy();
339
- reject(new Error("Unsupported image response encoding."));
340
- return;
341
- }
342
- resolve({
343
- ok: true,
344
- status,
345
- body: Readable.toWeb(addAbortSignal(signal, incoming), { strategy: { highWaterMark: 1 } }),
346
- });
347
- },
348
- );
349
- request.on("error", reject);
350
- request.on("upgrade", (incoming, socket) => {
351
- socket.destroy();
352
- resolve({ ok: false, status: incoming.statusCode ?? 101, body: null });
353
- });
354
- request.end(JSON.stringify(body));
355
- },
356
- );
380
+ signal,
381
+ headers: { ...requestHeaders(key, headers), "Accept-Encoding": "identity" },
382
+ },
383
+ (incoming) => {
384
+ const status = incoming.statusCode ?? 500;
385
+ const ok = status >= 200 && status < 300;
386
+ const received = Object.fromEntries(
387
+ Object.entries(incoming.headers).flatMap(([name, value]) =>
388
+ value === undefined ? [] : [[name, Array.isArray(value) ? value.join(", ") : value]],
389
+ ),
390
+ );
391
+ if (incoming.headers["content-encoding"] && incoming.headers["content-encoding"] !== "identity") {
392
+ incoming.destroy();
393
+ if (ok) reject(new Error("Unsupported image response encoding."));
394
+ else resolve({ ok, status, headers: received, body: null });
395
+ return;
396
+ }
397
+ resolve({
398
+ ok,
399
+ status,
400
+ headers: received,
401
+ body: Readable.toWeb(addAbortSignal(signal, incoming), { strategy: { highWaterMark: 1 } }),
402
+ });
403
+ },
404
+ );
405
+ request.on("error", reject);
406
+ request.on("upgrade", (incoming, socket) => {
407
+ socket.destroy();
408
+ resolve({ ok: false, status: incoming.statusCode ?? 101, headers: {}, body: null });
409
+ });
410
+ request.end(JSON.stringify(body));
411
+ });
412
+ }
413
+
414
+ type StreamReader = {
415
+ read(): Promise<{ done: boolean; value?: Uint8Array }>;
416
+ cancel(): Promise<void>;
417
+ };
418
+ // Fetch and Node web streams have incompatible DOM typings; both satisfy this shape.
419
+ type ResponseBody = { getReader(): StreamReader & { releaseLock(): void }; cancel(): Promise<void> };
420
+
421
+ // Upstream errors name the rejected option or model. Show a short JSON message without credentials.
422
+ async function errorDetail(reader: StreamReader, key: string) {
423
+ const chunks: Uint8Array[] = [];
424
+ let size = 0;
425
+ // A stalled error body must not hold the failure until the request deadline; cancel ends pending reads.
426
+ const stall = setTimeout(() => void reader.cancel().catch(() => {}), 5000);
427
+ try {
428
+ while (size <= 64 * 1024) {
429
+ const chunk = await reader.read();
430
+ if (chunk.done || !chunk.value) break;
431
+ chunks.push(chunk.value);
432
+ size += chunk.value.byteLength;
433
+ }
434
+ return errorText(JSON.parse(Buffer.concat(chunks).toString("utf8")), key);
435
+ } catch {
436
+ return "";
437
+ } finally {
438
+ clearTimeout(stall);
439
+ void reader.cancel().catch(() => {});
440
+ }
441
+ }
442
+
443
+ function errorText(value: unknown, key: string) {
444
+ const error = isRecord(value) && isRecord(value.error) ? value.error : value;
445
+ // OpenAI-style bodies nest { message }; xAI-style bodies use a string error.
446
+ const message = !isRecord(error)
447
+ ? undefined
448
+ : typeof error.message === "string"
449
+ ? error.message
450
+ : typeof error.error === "string"
451
+ ? error.error
452
+ : undefined;
453
+ if (message === undefined) return "";
454
+ // Covers the proxy key, bearer tokens, and OpenAI, Anthropic, xAI, and Google key formats.
455
+ return message
456
+ .split(key)
457
+ .join("[redacted]")
458
+ .replace(/\b(?:sk-|xai-|AIza|ya29\.|Bearer\s+)[\w.~+/=-]+/gi, "[redacted]")
459
+ .replace(/[\p{C}\s]+/gu, " ")
460
+ .trim()
461
+ .slice(0, 500);
462
+ }
463
+
464
+ // Server-sent events: partial previews go to the callback, other events are returned in order.
465
+ async function readEvents(
466
+ reader: StreamReader,
467
+ maxBytes: number,
468
+ onPartial: (event: Record<string, unknown>) => void,
469
+ ) {
470
+ const decoder = new TextDecoder();
471
+ const events: Record<string, unknown>[] = [];
472
+ let buffer = "";
473
+ // A CRLF pair can straddle two chunks; hold a trailing CR until the next chunk arrives.
474
+ let carry = "";
475
+ let size = 0;
476
+ const frame = (text: string) => {
477
+ const data = text
478
+ .split("\n")
479
+ .filter((line) => line.startsWith("data:"))
480
+ .map((line) => line.slice(5).replace(/^ /, "").replace(/\r$/, ""))
481
+ .join("\n");
482
+ if (!data || data === "[DONE]") return;
483
+ const event: unknown = JSON.parse(data);
484
+ if (!isRecord(event)) throw new Error("Invalid image event.");
485
+ if (typeof event.type === "string" && event.type.endsWith(".partial_image")) onPartial(event);
486
+ else events.push(event);
487
+ };
488
+ for (let searched = 0; ; ) {
489
+ const chunk = await reader.read();
490
+ if (chunk.done || !chunk.value) break;
491
+ size += chunk.value.byteLength;
492
+ if (size > maxBytes) throw new Error("Response too large.");
493
+ const text = carry + decoder.decode(chunk.value, { stream: true });
494
+ carry = text.endsWith("\r") ? "\r" : "";
495
+ buffer += (carry ? text.slice(0, -1) : text).replace(/\r\n/g, "\n");
496
+ for (let end = buffer.indexOf("\n\n", searched); end >= 0; end = buffer.indexOf("\n\n")) {
497
+ frame(buffer.slice(0, end));
498
+ buffer = buffer.slice(end + 2);
499
+ }
500
+ searched = Math.max(0, buffer.length - 1);
501
+ }
502
+ frame(buffer + (carry + decoder.decode()).replace(/\r\n/g, "\n"));
503
+ return events;
357
504
  }
358
505
 
359
506
  async function mediaRequest(
@@ -361,39 +508,65 @@ async function mediaRequest(
361
508
  key: string,
362
509
  path: string,
363
510
  signal: AbortSignal,
364
- maxBytes: number,
365
- body?: object,
366
- openaiImage = false,
511
+ {
512
+ maxBytes,
513
+ body,
514
+ openaiImage = false,
515
+ onPartial,
516
+ ...transport
517
+ }: TransportOptions & {
518
+ maxBytes: number;
519
+ body?: unknown;
520
+ openaiImage?: boolean;
521
+ onPartial?: (event: Record<string, unknown>) => void;
522
+ },
367
523
  ) {
368
- let response: Response | Awaited<ReturnType<typeof openaiImageResponse>>;
524
+ let response: Awaited<ReturnType<typeof openaiImageResponse>>;
369
525
  try {
370
- response =
371
- openaiImage && body
372
- ? await openaiImageResponse(config, key, path, body, signal)
373
- : await fetch(`${config.baseUrl}/${path}`, {
374
- method: body ? "POST" : "GET",
375
- headers: { Authorization: `Bearer ${key}`, "Content-Type": "application/json" },
376
- body: body ? JSON.stringify(body) : undefined,
377
- redirect: "error",
378
- signal,
379
- });
526
+ // A caller-supplied fetch replaces the dedicated image connection and owns its timeouts.
527
+ if (openaiImage && body && !transport.fetch)
528
+ response = await openaiImageResponse(config, key, path, body, signal, transport);
529
+ else {
530
+ const result = await (transport.fetch ?? fetch)(`${config.baseUrl}/${path}`, {
531
+ method: body ? "POST" : "GET",
532
+ headers: requestHeaders(key, transport.headers),
533
+ body: body ? JSON.stringify(body) : undefined,
534
+ redirect: "error",
535
+ signal,
536
+ });
537
+ response = {
538
+ ok: result.ok,
539
+ status: result.status,
540
+ headers: Object.fromEntries(result.headers),
541
+ body: result.body,
542
+ };
543
+ }
380
544
  } catch {
381
545
  throw new Error(
382
546
  "CLIProxyAPI media request failed: connection, cancellation, timeout, or redirect error. Generation was not retried.",
383
547
  );
384
548
  }
385
- if (!response.ok) {
386
- await response.body?.cancel();
387
- throw new Error(`CLIProxyAPI media request failed (HTTP ${response.status}).`);
549
+ try {
550
+ await transport.onResponse?.({ status: response.status, headers: response.headers });
551
+ } catch (error) {
552
+ await response.body?.cancel().catch(() => {});
553
+ throw error;
388
554
  }
389
555
  const reader = response.body?.getReader();
556
+ if (!response.ok) {
557
+ const detail = reader ? await errorDetail(reader, key) : "";
558
+ throw new Error(
559
+ `CLIProxyAPI media request failed (HTTP ${response.status})${detail ? `: ${detail}` : "."}`,
560
+ );
561
+ }
390
562
  if (!reader) throw new Error("CLIProxyAPI media returned an empty response.");
391
563
  try {
564
+ if (onPartial) return await readEvents(reader, maxBytes, onPartial);
392
565
  const chunks: Uint8Array[] = [];
393
566
  let size = 0;
394
567
  while (true) {
395
568
  const chunk = await reader.read();
396
- if (chunk.done) break;
569
+ if (chunk.done || !chunk.value) break;
397
570
  size += chunk.value.byteLength;
398
571
  if (size > maxBytes) {
399
572
  await reader.cancel();
@@ -404,6 +577,7 @@ async function mediaRequest(
404
577
  const result: unknown = JSON.parse(Buffer.concat(chunks).toString("utf8"));
405
578
  return result;
406
579
  } catch {
580
+ void reader.cancel().catch(() => {});
407
581
  throw new Error(
408
582
  "CLIProxyAPI media response is invalid, incomplete, cancelled, or exceeds the response size limit.",
409
583
  );
@@ -553,12 +727,199 @@ function parseGoogleImages(value: unknown) {
553
727
  return images;
554
728
  }
555
729
 
730
+ // OpenAI reports image token usage. Costs use list prices and stay zero for unpriced models.
731
+ function imageUsage(model: string, value: unknown) {
732
+ const tokens = (field: unknown) =>
733
+ typeof field === "number" && Number.isSafeInteger(field) && field >= 0 ? field : undefined;
734
+ if (!isRecord(value)) return undefined;
735
+ const input = tokens(value.input_tokens);
736
+ const output = tokens(value.output_tokens);
737
+ if (input === undefined || output === undefined) return undefined;
738
+ const imageInput = Math.min(
739
+ input,
740
+ tokens(isRecord(value.input_tokens_details) ? value.input_tokens_details.image_tokens : 0) ?? 0,
741
+ );
742
+ const [textPrice, imagePrice, outputPrice] = mediaCapability(model)?.price ?? [0, 0, 0];
743
+ const cost = {
744
+ input: ((input - imageInput) * textPrice + imageInput * imagePrice) / 1e6,
745
+ output: (output * outputPrice) / 1e6,
746
+ cacheRead: 0,
747
+ cacheWrite: 0,
748
+ total: 0,
749
+ };
750
+ cost.total = cost.input + cost.output;
751
+ const usage: Usage = { input, output, cacheRead: 0, cacheWrite: 0, totalTokens: input + output, cost };
752
+ return usage;
753
+ }
754
+
755
+ // Shared by the image tool and Pi's native image interface.
756
+ async function requestImages(
757
+ config: Config,
758
+ key: string,
759
+ model: string,
760
+ request: {
761
+ prompt: string;
762
+ output: Partial<ReturnType<typeof mediaOutputOptions>>;
763
+ images?: { image_url: string }[];
764
+ mask?: { image_url: string };
765
+ },
766
+ signal: AbortSignal,
767
+ onPartial?: (image: ImageContent, index: number) => void,
768
+ { onPayload, ...transport }: TransportOptions & { onPayload?: (payload: unknown) => unknown } = {},
769
+ ) {
770
+ const route = mediaCapability(model)?.route;
771
+ const { prompt, images, mask } = request;
772
+ const { partial_images = 0, ...output } = request.output;
773
+ const n = output.n ?? 1;
774
+ const stream = route === "openai" && partial_images > 0;
775
+ const body =
776
+ route === "xai"
777
+ ? {
778
+ model,
779
+ prompt,
780
+ n: 1,
781
+ response_format: "b64_json",
782
+ resolution: output.resolution,
783
+ aspect_ratio: output.aspect_ratio,
784
+ }
785
+ : route === "openai"
786
+ ? { model, prompt, ...output, n, images, mask, ...(stream ? { stream, partial_images } : {}) }
787
+ : {
788
+ contents: [{ role: "user", parts: [{ text: prompt }] }],
789
+ generationConfig: {
790
+ responseModalities: ["TEXT", "IMAGE"],
791
+ candidateCount: 1,
792
+ ...(output.resolution !== undefined || output.aspect_ratio !== undefined
793
+ ? { imageConfig: { imageSize: output.resolution, aspectRatio: output.aspect_ratio } }
794
+ : {}),
795
+ },
796
+ };
797
+ const payload = (await onPayload?.(body)) ?? body;
798
+ const path =
799
+ route === "gemini"
800
+ ? `v1beta/models/${model}:generateContent`
801
+ : images
802
+ ? "v1/images/edits"
803
+ : "v1/images/generations";
804
+ if (!stream) {
805
+ const result = await mediaRequest(config, key, path, signal, {
806
+ ...transport,
807
+ maxBytes: MAX_IMAGE_RESPONSE_BYTES,
808
+ body: payload,
809
+ openaiImage: route === "openai",
810
+ });
811
+ return {
812
+ images: route === "gemini" ? parseGoogleImages(result) : parseImages(result, n),
813
+ usage: route === "openai" && isRecord(result) ? imageUsage(model, result.usage) : undefined,
814
+ };
815
+ }
816
+ // Each frame carries one full image, so the stream may hold the final image plus every preview.
817
+ const events = await mediaRequest(config, key, path, signal, {
818
+ ...transport,
819
+ maxBytes: MAX_IMAGE_RESPONSE_BYTES * (1 + partial_images),
820
+ body: payload,
821
+ openaiImage: true,
822
+ onPartial: (event) => {
823
+ try {
824
+ const index = typeof event.partial_image_index === "number" ? event.partial_image_index : 0;
825
+ onPartial?.(parseImage(event).image, index);
826
+ } catch {
827
+ // A malformed preview does not affect the final image.
828
+ }
829
+ },
830
+ });
831
+ if (!Array.isArray(events)) throw new Error("CLIProxyAPI returned an invalid image stream.");
832
+ const failed = events.find(
833
+ (event) => isRecord(event) && (event.type === "error" || event.error !== undefined),
834
+ );
835
+ if (failed) {
836
+ const detail = errorText(failed, key);
837
+ throw new Error(`CLIProxyAPI image stream failed${detail ? `: ${detail}` : "."}`);
838
+ }
839
+ const completed = events.filter(
840
+ (event) => isRecord(event) && typeof event.type === "string" && event.type.endsWith(".completed"),
841
+ );
842
+ if (completed.length !== n) throw new Error(`CLIProxyAPI image stream must complete ${n} image(s).`);
843
+ return { images: completed.map(parseImage), usage: imageUsage(model, completed[0].usage) };
844
+ }
845
+
846
+ export async function generateNativeImages(
847
+ config: Config,
848
+ model: ImageModel<ImageApi>,
849
+ context: ImagesContext,
850
+ options?: ImagesOptions,
851
+ ) {
852
+ const result: AssistantImages = {
853
+ api: model.api,
854
+ provider: model.provider,
855
+ model: model.id,
856
+ output: [],
857
+ stopReason: "stop",
858
+ timestamp: Date.now(),
859
+ };
860
+ try {
861
+ const capability = mediaCapability(model.id);
862
+ if (capability?.purpose !== "image" || capability.disabledReason)
863
+ throw new Error(capability?.disabledReason ?? "Choose a supported CLIProxyAPI image model.");
864
+ const prompt = context.input.flatMap((block) => (block.type === "text" ? [block.text] : [])).join("\n\n");
865
+ if (!prompt.trim() || (capability.route === "openai" && prompt.length > 32000))
866
+ throw new Error(
867
+ "CLIProxyAPI requires a non-empty prompt; OpenAI prompts are limited to 32000 characters.",
868
+ );
869
+ const limits = mediaControls(model.id).editing;
870
+ const blocks = context.input.flatMap((block) => (block.type === "image" ? [block] : []));
871
+ let total = 0;
872
+ const images = blocks.map(({ data }) => {
873
+ const bytes = Buffer.from(data, "base64");
874
+ total += bytes.length;
875
+ if (
876
+ !limits ||
877
+ blocks.length > limits.max_images ||
878
+ bytes.length > limits.max_file_bytes ||
879
+ total > limits.max_total_bytes
880
+ )
881
+ throw new Error(
882
+ "CLIProxyAPI image inputs need an OpenAI image model: at most 16 images, 14 MiB each, 20 MiB total.",
883
+ );
884
+ let mimeType: string;
885
+ try {
886
+ ({ mimeType } = imageType(bytes));
887
+ } catch {
888
+ throw new Error("CLIProxyAPI image inputs must be PNG, JPEG, or WebP.");
889
+ }
890
+ return { image_url: `data:${mimeType};base64,${bytes.toString("base64")}` };
891
+ });
892
+ const generated = await requestImages(
893
+ config,
894
+ validateKey(options?.apiKey ?? ""),
895
+ model.id,
896
+ { prompt, output: {}, ...(images.length ? { images } : {}) },
897
+ deadline(options?.signal, capability.route === "openai" ? 600000 : 180000),
898
+ undefined,
899
+ {
900
+ fetch: options?.fetch,
901
+ headers: options?.headers,
902
+ env: options?.env,
903
+ onPayload: options?.onPayload && ((payload) => options.onPayload?.(payload, model)),
904
+ onResponse: options?.onResponse && ((response) => options.onResponse?.(response, model)),
905
+ },
906
+ );
907
+ result.output = generated.images.map(({ image }) => image);
908
+ result.usage = generated.usage;
909
+ } catch (error) {
910
+ result.stopReason = options?.signal?.aborted ? "aborted" : "error";
911
+ result.errorMessage = error instanceof Error ? error.message : String(error);
912
+ }
913
+ return result;
914
+ }
915
+
556
916
  export async function generateImage(
557
917
  config: Config,
558
918
  params: Static<typeof imageParameters>,
559
919
  ctx: MediaContext,
560
920
  signal?: AbortSignal,
561
921
  onProgress?: (text: string) => void,
922
+ onPartial?: (image: ImageContent, index: number) => void,
562
923
  ) {
563
924
  const model = resolveMediaModel(config, ctx, "image", params.model);
564
925
  const route = mediaCapability(model)?.route;
@@ -577,40 +938,14 @@ export async function generateImage(
577
938
  bounded.throwIfAborted();
578
939
  onProgress?.(inputs.images ? "Editing image..." : "Generating image...");
579
940
  bounded.throwIfAborted();
580
- const result = await mediaRequest(
941
+ const { images, usage } = await requestImages(
581
942
  config,
582
943
  key,
583
- route === "gemini"
584
- ? `v1beta/models/${model}:generateContent`
585
- : inputs.images
586
- ? "v1/images/edits"
587
- : "v1/images/generations",
944
+ model,
945
+ { prompt: params.prompt, output, ...inputs },
588
946
  bounded,
589
- 32 * 1024 * 1024,
590
- route === "xai"
591
- ? {
592
- model,
593
- prompt: params.prompt,
594
- n: 1,
595
- response_format: "b64_json",
596
- resolution: output.resolution,
597
- aspect_ratio: output.aspect_ratio,
598
- }
599
- : route === "openai"
600
- ? { model, prompt: params.prompt, ...output, n: output.n ?? 1, ...inputs }
601
- : {
602
- contents: [{ role: "user", parts: [{ text: params.prompt }] }],
603
- generationConfig: {
604
- responseModalities: ["TEXT", "IMAGE"],
605
- candidateCount: 1,
606
- ...(output.resolution !== undefined || output.aspect_ratio !== undefined
607
- ? { imageConfig: { imageSize: output.resolution, aspectRatio: output.aspect_ratio } }
608
- : {}),
609
- },
610
- },
611
- route === "openai",
947
+ onPartial,
612
948
  );
613
- const images = route === "gemini" ? parseGoogleImages(result) : parseImages(result, output.n ?? 1);
614
949
  bounded.throwIfAborted();
615
950
  onProgress?.("Saving image...");
616
951
  bounded.throwIfAborted();
@@ -647,7 +982,7 @@ export async function generateImage(
647
982
  previewBudget -= image.data.length;
648
983
  }
649
984
  }
650
- return { content, details: { model, files } };
985
+ return { content, details: { model, files }, usage };
651
986
  }
652
987
 
653
988
  function requestId(value: unknown) {
@@ -675,12 +1010,15 @@ export async function generateVideo(
675
1010
  const output = mediaOutputOptions(model, params);
676
1011
  const bounded = deadline(signal, 60000);
677
1012
  const key = await generationKey(config, { ...params, model }, ctx, bounded);
678
- const result = await mediaRequest(config, key, "v1/videos", bounded, 64 * 1024, {
679
- model,
680
- prompt: params.prompt,
681
- duration: params.duration,
682
- resolution: output.resolution,
683
- aspect_ratio: output.aspect_ratio,
1013
+ const result = await mediaRequest(config, key, "v1/videos", bounded, {
1014
+ maxBytes: 64 * 1024,
1015
+ body: {
1016
+ model,
1017
+ prompt: params.prompt,
1018
+ duration: params.duration,
1019
+ resolution: output.resolution,
1020
+ aspect_ratio: output.aspect_ratio,
1021
+ },
684
1022
  });
685
1023
  if (!isRecord(result)) throw new Error("CLIProxyAPI returned an invalid video submission.");
686
1024
  return { model, request_id: requestId(result.request_id), status: "pending" };
@@ -690,7 +1028,7 @@ export async function videoStatus(config: Config, id: string, ctx: MediaContext,
690
1028
  const request_id = requestId(id);
691
1029
  const bounded = deadline(signal, 15000);
692
1030
  const key = await mediaKey(ctx, bounded);
693
- const result = await mediaRequest(config, key, `v1/videos/${request_id}`, bounded, 64 * 1024);
1031
+ const result = await mediaRequest(config, key, `v1/videos/${request_id}`, bounded, { maxBytes: 64 * 1024 });
694
1032
  if (!isRecord(result)) throw new Error("CLIProxyAPI returned an invalid video status.");
695
1033
  if (result.status === "pending") return { request_id, status: "pending" };
696
1034
  if (result.status === "failed" || result.status === "expired") {
@@ -739,15 +1077,32 @@ export function registerMediaTools(pi: ExtensionAPI, config: Config) {
739
1077
  name: "cliproxyapi_generate_image",
740
1078
  label: "CLIProxyAPI image",
741
1079
  description:
742
- "Generate images, or edit local images with OpenAI models by supplying images and an optional mask. Supports OpenAI size, quality, background, format/compression and image count from cliproxyapi_media_models. Uses the selected model or automatic OpenAI/GPT default without changing chat. Saves final images under a new .pi/ directory with optional previews. Maximum response: 32 MiB; inline base64: 4 MiB. No automatic retries.",
1080
+ "Generate images, or edit local images with OpenAI models by supplying images and an optional mask. Supports OpenAI size, quality, background, format/compression, image count and streamed partial previews from cliproxyapi_media_models. Uses the selected model or automatic OpenAI/GPT default without changing chat. Saves final images under a new .pi/ directory with optional previews. Maximum response: 32 MiB in total, plus 32 MiB per streamed preview; inline base64: 4 MiB. No automatic retries.",
743
1081
  promptSnippet: "Generate requested raster images through CLIProxyAPI using the effective image default.",
744
1082
  promptGuidelines: [
745
1083
  "Use cliproxyapi_generate_image for requested raster images. Omit model to use the effective default unless a specific image model is requested; keep the current chat model.",
746
1084
  ],
747
1085
  parameters: imageParameters,
748
1086
  async execute(_id, params, signal, onUpdate, ctx) {
749
- return generateImage(config, params, ctx, signal, (text) =>
750
- onUpdate?.({ content: [{ type: "text", text }], details: {} }),
1087
+ return generateImage(
1088
+ config,
1089
+ params,
1090
+ ctx,
1091
+ signal,
1092
+ (text) => onUpdate?.({ content: [{ type: "text", text }], details: {} }),
1093
+ (image, index) => {
1094
+ const large = image.data.length > MAX_INLINE_IMAGE_BASE64_BYTES;
1095
+ onUpdate?.({
1096
+ content: [
1097
+ {
1098
+ type: "text",
1099
+ text: `Partial image ${index + 1}${large ? " (preview over 4 MiB omitted)" : ""}`,
1100
+ },
1101
+ ...(large ? [] : [image]),
1102
+ ],
1103
+ details: {},
1104
+ });
1105
+ },
751
1106
  );
752
1107
  },
753
1108
  });
package/src/picker.ts CHANGED
@@ -7,8 +7,8 @@ import {
7
7
  type TuiMouseEvent,
8
8
  truncateToWidth,
9
9
  } from "@earendil-works/pi-tui";
10
- import { backendLabel, mediaCapability, parseCatalog, routedName } from "./catalog.ts";
11
- import type { Config } from "./config.ts";
10
+ import { backendLabel, fetchCatalog, mediaCapability, parseCatalog, routedName } from "./catalog.ts";
11
+ import { type Config, deadline } from "./config.ts";
12
12
  import { mediaKey } from "./media.ts";
13
13
  import {
14
14
  effectiveMediaDefaults,
@@ -16,7 +16,6 @@ import {
16
16
  type MediaDefaults,
17
17
  readMediaDefaults,
18
18
  } from "./media-defaults.ts";
19
- import { deadline, fetchCatalog } from "./provider.ts";
20
19
 
21
20
  export function pickerCatalog(value: unknown) {
22
21
  const rows = new Map<
package/src/provider.ts CHANGED
@@ -18,10 +18,19 @@ import {
18
18
  openAIResponsesApi,
19
19
  } from "@earendil-works/pi-ai/compat";
20
20
  import { getBuiltinModels } from "@earendil-works/pi-ai/providers/all";
21
- import { type CpaApi, endpoint, mapCatalog, modelRoute } from "./catalog.ts";
21
+ import {
22
+ type CpaApi,
23
+ endpoint,
24
+ fetchCatalog,
25
+ IMAGE_API,
26
+ mapCatalog,
27
+ mapImageModels,
28
+ modelRoute,
29
+ } from "./catalog.ts";
22
30
  import { type QuotaWaitHandler, waitForClaudeQuota } from "./claude-quota.ts";
23
31
  import { withCompactionRecovery } from "./compaction.ts";
24
- import { type Config, isRecord, PROVIDER_ID } from "./config.ts";
32
+ import { type Config, isRecord, PROVIDER_ID, validateKey } from "./config.ts";
33
+ import { generateNativeImages } from "./media.ts";
25
34
 
26
35
  export type { QuotaWaitHandler } from "./claude-quota.ts";
27
36
 
@@ -31,63 +40,21 @@ export function builtinCatalog() {
31
40
  ...getBuiltinModels("openai"),
32
41
  ...getBuiltinModels("openai-codex"),
33
42
  ...getBuiltinModels("google"),
43
+ ...getBuiltinModels("xai"),
34
44
  ];
35
45
  }
36
46
 
37
- export function validateKey(key: string) {
38
- if (!/^[!-~]{1,4096}$/.test(key)) {
39
- throw new Error("CLIProxyAPI requires a non-empty API key without whitespace or control characters.");
40
- }
41
- return key;
42
- }
43
-
44
- export function deadline(signal: AbortSignal | undefined, milliseconds: number) {
45
- return AbortSignal.any([...(signal ? [signal] : []), AbortSignal.timeout(milliseconds)]);
46
- }
47
-
48
- export async function fetchCatalog(config: Config, key: string, signal: AbortSignal) {
49
- validateKey(key);
50
- const boundedSignal = deadline(signal, 10000);
51
- let response: Response;
52
- try {
53
- response = await fetch(`${config.baseUrl}/v1/models`, {
54
- headers: { Authorization: `Bearer ${key}` },
55
- redirect: "error",
56
- signal: boundedSignal,
57
- });
58
- } catch {
59
- signal.throwIfAborted();
60
- throw new Error("CLIProxyAPI discovery failed: connection, timeout, or redirect error.");
61
- }
62
- if (!response.ok) {
63
- await response.body?.cancel();
64
- throw new Error(`CLIProxyAPI discovery failed (HTTP ${response.status}).`);
65
- }
66
- let payload: unknown;
67
- const reader = response.body?.getReader();
68
- if (!reader) throw new Error("CLIProxyAPI discovery returned an empty response.");
69
- try {
70
- const decoder = new TextDecoder();
71
- let text = "";
72
- let size = 0;
73
- while (true) {
74
- const chunk = await reader.read();
75
- if (chunk.done) break;
76
- size += chunk.value.byteLength;
77
- if (size > 4 * 1024 * 1024) {
78
- await reader.cancel();
79
- throw new Error("Catalog too large.");
80
- }
81
- text += decoder.decode(chunk.value, { stream: true });
82
- }
83
- payload = JSON.parse(text + decoder.decode());
84
- } catch {
85
- signal.throwIfAborted();
86
- throw new Error("CLIProxyAPI catalog is invalid, incomplete, or larger than 4 MiB.");
87
- } finally {
88
- reader.releaseLock();
89
- }
90
- return payload;
47
+ // Pi's live catalogs add models released after this Pi build; bundled metadata wins for known IDs.
48
+ export function metadataCatalog(live?: (provider: string) => readonly Model<Api>[] | undefined) {
49
+ const bundled = builtinCatalog();
50
+ if (!live) return bundled;
51
+ const ids = new Set(bundled.map(({ provider, id }) => `${provider}/${id}`));
52
+ return [
53
+ ...bundled,
54
+ ...["anthropic", "openai", "openai-codex", "google", "xai"]
55
+ .flatMap((provider) => live(provider) ?? [])
56
+ .filter(({ provider, id }) => !ids.has(`${provider}/${id}`)),
57
+ ];
91
58
  }
92
59
 
93
60
  export async function discover(
@@ -345,10 +312,12 @@ function route(
345
312
 
346
313
  export function createCliproxyProvider(
347
314
  config: Config,
348
- known: readonly Model<Api>[] = builtinCatalog(),
315
+ known: readonly Model<Api>[] | (() => readonly Model<Api>[]) = builtinCatalog,
349
316
  onQuotaWait?: QuotaWaitHandler,
350
317
  compactionSignals = new WeakSet<AbortSignal>(),
351
318
  ) {
319
+ const metadata = typeof known === "function" ? known : () => known;
320
+ let skipped: string[] = [];
352
321
  const standardAuth = envApiKeyAuth("CLIProxyAPI API key", ["CLIPROXYAPI_API_KEY"]);
353
322
  const scope = createHash("sha256")
354
323
  .update(JSON.stringify([5, { baseUrl: config.baseUrl, aliases: config.aliases }]))
@@ -364,7 +333,7 @@ export function createCliproxyProvider(
364
333
  const credential = await standardAuth.login?.(interaction);
365
334
  if (!credential?.key) throw new Error("CLIProxyAPI requires an API key.");
366
335
  validateKey(credential.key);
367
- await discover(config, credential.key, interaction.signal, known);
336
+ await discover(config, credential.key, interaction.signal, metadata());
368
337
  return credential;
369
338
  },
370
339
  async resolve(input) {
@@ -380,7 +349,15 @@ export function createCliproxyProvider(
380
349
  if (context.credential?.type !== "api_key" || !context.credential.key) {
381
350
  throw new Error("Use /login cliproxyapi or set CLIPROXYAPI_API_KEY before refreshing.");
382
351
  }
383
- return (await discover(config, context.credential.key, context.signal, known)).models;
352
+ const payload = await fetchCatalog(config, context.credential.key, context.signal);
353
+ const chat = mapCatalog(payload, config, metadata());
354
+ skipped = chat.skipped;
355
+ return [...chat.models, ...mapImageModels(payload, config)];
356
+ },
357
+ images: {
358
+ [IMAGE_API]: {
359
+ generateImages: (model, context, options) => generateNativeImages(config, model, context, options),
360
+ },
384
361
  },
385
362
  api: {
386
363
  "anthropic-messages": route(anthropicMessagesApi(), config, compactionSignals, onQuotaWait),
@@ -392,6 +369,8 @@ export function createCliproxyProvider(
392
369
  const refresh = provider.refreshModels;
393
370
  return {
394
371
  ...provider,
372
+ // Chat IDs without Pi metadata from the latest network refresh, for diagnostics.
373
+ skippedModels: () => skipped,
395
374
  async refreshModels(context: RefreshModelsContext) {
396
375
  await refresh?.({
397
376
  ...context,