pi-meta-oauth 0.4.3 → 0.4.4

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/README.md CHANGED
@@ -61,10 +61,10 @@ Audio is captured only between the two Alt+V presses and streamed as 16 kHz mono
61
61
 
62
62
  **Windows:** uses `winmm.dll` `waveIn*` via a C# helper. On first run pi tries to compile `extensions/voice/windows-audio.cs` with `csc.exe` (in-box .NET Framework) to `~/.pi/agent/bin/pi-meta-oauth-voice-v1.exe` for best performance. If no compiler is found it falls back to `extensions/voice/windows-audio.ps1` executed with `powershell.exe -ExecutionPolicy Bypass` (PowerShell 5.1 inbox, or `pwsh` 7 if present) which compiles the same capture code at runtime via `Add-Type`. No extra install is required. If recording fails, check **Settings → Privacy & security → Microphone** that access is allowed for desktop apps.
63
63
 
64
- Optional overrides:
64
+ Optional overrides (the `PI_META_*` name takes precedence):
65
65
 
66
- - `PI_META_VOICE_ASR_ENDPOINT`
67
- - `PI_META_VOICE_ASR_MODEL`
66
+ - `PI_META_VOICE_ASR_ENDPOINT` (legacy alias: `MUSE_VOICE_ASR_ENDPOINT`)
67
+ - `PI_META_VOICE_ASR_MODEL` (legacy alias: `MUSE_VOICE_ASR_MODEL`)
68
68
 
69
69
  ## Media tools
70
70
 
@@ -91,6 +91,8 @@ Fallback models use a 1,048,576-token context window, up to 256K output tokens,
91
91
  | `muse-spark-1.2-contributor` | 0.10 / 0.20 / 0.002 |
92
92
  | `muse-spark-1.1` | 1.25 / 4.25 / 0.15 |
93
93
 
94
+ > **Contributor-model privacy:** the discounted contributor model allows Meta to use prompts and completions for product improvement, including training future Meta models. Use the standard `muse-spark-1.2` model if you do not want the contributor terms. See [Meta's model documentation](https://dev.meta.ai/docs/models).
95
+
94
96
  To scope Pi's model picker to Meta models:
95
97
 
96
98
  ```json
@@ -99,39 +101,22 @@ To scope Pi's model picker to Meta models:
99
101
 
100
102
  ### Making context windows visible to external tools
101
103
 
102
- Pi knows the Meta context window at runtime, but it does **not** persist
103
- catalogs for extension-registered providers to `~/.pi/agent/models-store.json`.
104
- Anything reading Pi's session data from outside the process therefore sees no
105
- window for `meta` models and can only show an absolute token count.
106
-
107
- This affects usage tooling such as
108
- [herdr-agent-usage](https://github.com/senna-lang/herdr-agent-usage), whose
109
- sidebar shows `⛁ 24k` instead of `⛁ 2% (24k)` for Muse panes.
110
-
111
- To publish the window, add an override to `~/.pi/agent/models.json`:
112
-
113
- ```json
114
- {
115
- "providers": {
116
- "meta": {
117
- "modelOverrides": {
118
- "muse-spark-1.2": { "contextWindow": 1007997 },
119
- "muse-spark-1.2-contributor": { "contextWindow": 1007997 }
120
- }
121
- }
122
- }
123
- }
124
- ```
125
-
126
- `1,007,997` is the effective limit reported for those two models by a cached
127
- Muse Code 0.1.0/R708.1 catalog snapshot observed on 2026-08-06. That snapshot
128
- did not include `muse-spark-1.1`, so no effective-limit override is claimed for
129
- 1.1 here. The bundled fallback in `extensions/meta.ts` uses the nominal
130
- `1048576`; use the lower value only when you want percentages to match that
131
- specific Muse snapshot.
132
-
133
- This only adds metadata — the provider itself is still registered by the
134
- extension, and `pi --list-models meta` is unchanged.
104
+ After a successful network model refresh, the extension persists the Meta
105
+ catalog to `~/.pi/agent/models-store.json`. External usage tools such as
106
+ [herdr-agent-usage](https://github.com/senna-lang/herdr-agent-usage) can then
107
+ show a percentage (for example, `⛁ 2% (24k)`) instead of only an absolute token
108
+ count. Pi writes the cache during interactive or RPC startup, and again after
109
+ `/login meta`. `pi --list-models meta` lists currently available models but does
110
+ not itself trigger a network catalog refresh. The cached catalog is also used
111
+ when Pi starts without network access.
112
+
113
+ The bundled fallback uses Meta's nominal `1,048,576`-token context window. A
114
+ cached Muse Code 0.1.0/R708.1 catalog observed on 2026-08-06 reported a lower
115
+ effective limit of `1,007,997` for `muse-spark-1.2` and
116
+ `muse-spark-1.2-contributor`. If you need percentages to match that specific
117
+ Muse snapshot, you can still set `contextWindow: 1007997` for those models in
118
+ `~/.pi/agent/models.json`; model overrides take precedence over the persisted
119
+ catalog.
135
120
 
136
121
  ## Verify
137
122
 
@@ -0,0 +1,131 @@
1
+ import { basename } from "node:path";
2
+ import { readFile } from "node:fs/promises";
3
+ import { statSync } from "node:fs";
4
+ import { META_FILES_URL } from "../meta.ts";
5
+ import {
6
+ AUTOMATIC_UPLOAD_EXPIRY_SECONDS,
7
+ FILES_API_LIMIT_BYTES,
8
+ } from "./limits.ts";
9
+ import { filenameForMime, mimeForPath } from "./mime.ts";
10
+
11
+ export interface UploadResult {
12
+ id: string;
13
+ filename: string;
14
+ bytes: number;
15
+ purpose: string;
16
+ status: string;
17
+ expires_at?: number;
18
+ }
19
+
20
+ function uploadErrorDetail(body: Record<string, unknown>, bodyText: string): string {
21
+ if (typeof body.error === "string") return body.error;
22
+ if (typeof body.message === "string") return body.message;
23
+ if (typeof body.error_description === "string") return body.error_description;
24
+ return bodyText.slice(0, 500);
25
+ }
26
+
27
+ export async function uploadMetaFile(
28
+ apiKey: string,
29
+ filePath: string,
30
+ expiresAfter?: { anchor: "created_at"; seconds: number },
31
+ signal?: AbortSignal,
32
+ ): Promise<UploadResult> {
33
+ const size = statSync(filePath).size;
34
+ if (size > FILES_API_LIMIT_BYTES) {
35
+ throw new Error(
36
+ `File too large for Files API (${size} bytes > 1 GiB)`,
37
+ );
38
+ }
39
+ const data = await readFile(filePath, { signal });
40
+ const mime = mimeForPath(filePath) ?? "application/octet-stream";
41
+ const blob = new Blob([data], { type: mime });
42
+ const form = new FormData();
43
+ form.append("file", blob, basename(filePath));
44
+ form.append("purpose", "user_data");
45
+ if (expiresAfter) {
46
+ form.append("expires_after[anchor]", expiresAfter.anchor);
47
+ form.append("expires_after[seconds]", String(expiresAfter.seconds));
48
+ }
49
+ const res = await fetch(META_FILES_URL, {
50
+ method: "POST",
51
+ headers: { Authorization: `Bearer ${apiKey}` },
52
+ body: form,
53
+ signal,
54
+ });
55
+ const bodyText = await res.text();
56
+ let body: Record<string, unknown> = {};
57
+ try {
58
+ body = bodyText ? (JSON.parse(bodyText) as Record<string, unknown>) : {};
59
+ } catch {
60
+ // Non-JSON body — keep empty and use raw text for error detail
61
+ }
62
+ if (!res.ok) {
63
+ const detail = uploadErrorDetail(body, bodyText);
64
+ if (res.status === 400 && detail.includes("storage")) {
65
+ throw new Error(
66
+ `Meta Files storage limit reached (100 GiB/team). Delete old files via DELETE /v1/files/{id}. Detail: ${detail}`,
67
+ );
68
+ }
69
+ throw new Error(
70
+ `Meta Files upload failed (HTTP ${res.status}): ${detail || res.statusText}`,
71
+ );
72
+ }
73
+ const id =
74
+ typeof body.id === "string"
75
+ ? body.id
76
+ : typeof body.file_id === "string"
77
+ ? body.file_id
78
+ : undefined;
79
+ if (!id) {
80
+ throw new Error(
81
+ `Meta Files upload returned no id: ${bodyText.slice(0, 500)}`,
82
+ );
83
+ }
84
+ return {
85
+ id,
86
+ filename:
87
+ typeof body.filename === "string" ? body.filename : basename(filePath),
88
+ bytes: typeof body.bytes === "number" ? body.bytes : data.byteLength,
89
+ purpose: typeof body.purpose === "string" ? body.purpose : "user_data",
90
+ status: typeof body.status === "string" ? body.status : "uploaded",
91
+ expires_at:
92
+ typeof body.expires_at === "number" ? body.expires_at : undefined,
93
+ };
94
+ }
95
+
96
+ export async function uploadInlineMedia(
97
+ apiKey: string,
98
+ base64: string,
99
+ mime: string,
100
+ filename: string,
101
+ signal?: AbortSignal,
102
+ ): Promise<string> {
103
+ const blob = new Blob([Buffer.from(base64, "base64")], { type: mime });
104
+ const form = new FormData();
105
+ form.append("file", blob, filename || filenameForMime(mime));
106
+ form.append("purpose", "user_data");
107
+ form.append("expires_after[anchor]", "created_at");
108
+ form.append("expires_after[seconds]", String(AUTOMATIC_UPLOAD_EXPIRY_SECONDS));
109
+ const res = await fetch(META_FILES_URL, {
110
+ method: "POST",
111
+ headers: { Authorization: `Bearer ${apiKey}` },
112
+ body: form,
113
+ signal,
114
+ });
115
+ const text = await res.text();
116
+ if (!res.ok) {
117
+ throw new Error(
118
+ `Files upload for large inline media failed: ${text.slice(0, 500)}`,
119
+ );
120
+ }
121
+ let json: Record<string, unknown>;
122
+ try {
123
+ json = JSON.parse(text) as Record<string, unknown>;
124
+ } catch (error) {
125
+ throw new Error(`Files upload returned invalid JSON: ${String(error)}`);
126
+ }
127
+ if (typeof json.id !== "string" || !json.id) {
128
+ throw new Error(`Files upload returned no id: ${text.slice(0, 500)}`);
129
+ }
130
+ return json.id;
131
+ }
@@ -0,0 +1,12 @@
1
+ export const INLINE_LIMIT_BYTES = 50_000_000;
2
+ export const FILES_API_LIMIT_BYTES = 1_073_741_824;
3
+ // Meta Responses `store=true` (default) has ~20 MB persistence limit even when
4
+ // inline limit is 50 MB — a 24 MB base64 payload triggers HTTP 413
5
+ // "payload_too_large … with `store=true`". Keep inline only for small files
6
+ // and force Files API or `store:false` above this threshold.
7
+ export const STORE_SAFE_INLINE_BYTES = 15_000_000;
8
+ export const AUTOMATIC_UPLOAD_EXPIRY_SECONDS = 24 * 60 * 60;
9
+ export const EXPLICIT_UPLOAD_EXPIRY_SECONDS = 7 * 24 * 60 * 60;
10
+ export const MIN_UPLOAD_EXPIRY_SECONDS = 3_600;
11
+ export const MAX_UPLOAD_EXPIRY_SECONDS = 2_592_000;
12
+ export const MAX_ANALYSIS_SOURCES = 50;
@@ -0,0 +1,178 @@
1
+ import { extname } from "node:path";
2
+
3
+ export interface ResponsesContentBlock {
4
+ type: string;
5
+ [key: string]: unknown;
6
+ }
7
+
8
+ type SupportedMime =
9
+ | "video/mp4"
10
+ | "audio/mpeg"
11
+ | "audio/wav"
12
+ | "application/pdf"
13
+ | "image/png"
14
+ | "image/jpeg"
15
+ | "image/gif"
16
+ | "image/webp"
17
+ | "image/x-icon"
18
+ | "text/plain"
19
+ | "application/json"
20
+ | "application/jsonl";
21
+
22
+ const EXT_TO_MIME: Record<string, SupportedMime> = {
23
+ ".mp4": "video/mp4",
24
+ ".m4v": "video/mp4",
25
+ ".mp3": "audio/mpeg",
26
+ ".wav": "audio/wav",
27
+ ".pdf": "application/pdf",
28
+ ".png": "image/png",
29
+ ".jpg": "image/jpeg",
30
+ ".jpeg": "image/jpeg",
31
+ ".gif": "image/gif",
32
+ ".webp": "image/webp",
33
+ ".ico": "image/x-icon",
34
+ ".txt": "text/plain",
35
+ ".json": "application/json",
36
+ ".jsonl": "application/jsonl",
37
+ };
38
+
39
+ export function mimeForPath(path: string): SupportedMime | undefined {
40
+ const ext = extname(path).toLowerCase();
41
+ return EXT_TO_MIME[ext];
42
+ }
43
+
44
+ export function mimeForDataUrl(dataUrl: string): string | undefined {
45
+ const m = dataUrl.match(/^data:([^;,\s]+)[;,]/);
46
+ return m?.[1]?.toLowerCase();
47
+ }
48
+
49
+ export function isVideoMime(m: string | undefined): boolean {
50
+ return m === "video/mp4";
51
+ }
52
+
53
+ export function isAudioMime(m: string | undefined): boolean {
54
+ return (
55
+ m === "audio/mpeg" ||
56
+ m === "audio/mp3" ||
57
+ m === "audio/wav" ||
58
+ m === "audio/x-wav"
59
+ );
60
+ }
61
+
62
+ export function isPdfMime(m: string | undefined): boolean {
63
+ return m === "application/pdf";
64
+ }
65
+
66
+ export function isImageMime(m: string | undefined): boolean {
67
+ return typeof m === "string" && m.startsWith("image/");
68
+ }
69
+
70
+ export function filenameForMime(
71
+ mime: string | undefined,
72
+ fallback = "file.bin",
73
+ ): string {
74
+ switch (mime) {
75
+ case "video/mp4":
76
+ return "video.mp4";
77
+ case "audio/mpeg":
78
+ case "audio/mp3":
79
+ return "audio.mp3";
80
+ case "audio/wav":
81
+ case "audio/x-wav":
82
+ return "audio.wav";
83
+ case "application/pdf":
84
+ return "document.pdf";
85
+ default:
86
+ return fallback;
87
+ }
88
+ }
89
+
90
+ export function audioFormatForMime(mime: string | undefined): "wav" | "mp3" {
91
+ return mime === "audio/wav" || mime === "audio/x-wav" ? "wav" : "mp3";
92
+ }
93
+
94
+ export function base64FromDataUrl(dataUrl: string): string {
95
+ const comma = dataUrl.indexOf(",");
96
+ return comma === -1 ? "" : dataUrl.slice(comma + 1);
97
+ }
98
+
99
+ export function base64ByteLength(base64: string): number {
100
+ const normalized = base64.replace(/\s/g, "");
101
+ if (!normalized) return 0;
102
+ const padding = normalized.endsWith("==")
103
+ ? 2
104
+ : normalized.endsWith("=")
105
+ ? 1
106
+ : 0;
107
+ return Math.max(0, Math.floor((normalized.length * 3) / 4) - padding);
108
+ }
109
+
110
+ export function isRemoteMediaSource(source: string): boolean {
111
+ return source.startsWith("https://") || source.startsWith("data:");
112
+ }
113
+
114
+ export function mediaPathFromUrl(url: string): string {
115
+ if (url.startsWith("data:")) return url;
116
+ try {
117
+ return new URL(url).pathname;
118
+ } catch {
119
+ const withoutHash = url.split("#")[0] ?? url;
120
+ return withoutHash.split("?")[0] ?? withoutHash;
121
+ }
122
+ }
123
+
124
+ export function mimeForSource(source: string): string | undefined {
125
+ if (source.startsWith("data:")) return mimeForDataUrl(source);
126
+ if (source.startsWith("https://")) {
127
+ try {
128
+ return mimeForPath(new URL(source).pathname);
129
+ } catch {
130
+ return undefined;
131
+ }
132
+ }
133
+ return mimeForPath(source);
134
+ }
135
+
136
+ /** Build a generic Meta Responses input_file block. */
137
+ export function inputFileFromSource(
138
+ source: string,
139
+ filename?: string,
140
+ ): ResponsesContentBlock {
141
+ if (source.startsWith("https://")) {
142
+ return { type: "input_file", file_url: source };
143
+ }
144
+ if (source.startsWith("data:")) {
145
+ const mime = mimeForDataUrl(source);
146
+ return {
147
+ type: "input_file",
148
+ file_data: source,
149
+ filename: filename ?? filenameForMime(mime),
150
+ };
151
+ }
152
+ return { type: "input_file", file_id: source };
153
+ }
154
+
155
+ /** Build the typed content block required by Meta for each media kind. */
156
+ export function mediaInputFromSource(
157
+ source: string,
158
+ filename?: string,
159
+ expectedMime?: string,
160
+ ): ResponsesContentBlock {
161
+ if (!source.startsWith("data:") && !source.startsWith("https://")) {
162
+ return { type: "input_file", file_id: source };
163
+ }
164
+ const mime = expectedMime ?? mimeForSource(source);
165
+ if (isVideoMime(mime)) {
166
+ return { type: "input_video", video_url: source };
167
+ }
168
+ if (isAudioMime(mime) && source.startsWith("data:")) {
169
+ return {
170
+ type: "input_audio",
171
+ input_audio: {
172
+ data: base64FromDataUrl(source),
173
+ format: audioFormatForMime(mime),
174
+ },
175
+ };
176
+ }
177
+ return inputFileFromSource(source, filename);
178
+ }
@@ -0,0 +1,303 @@
1
+ import { STORE_SAFE_INLINE_BYTES } from "./limits.ts";
2
+ import { uploadInlineMedia } from "./files.ts";
3
+ import {
4
+ base64ByteLength,
5
+ base64FromDataUrl,
6
+ filenameForMime,
7
+ isAudioMime,
8
+ isImageMime,
9
+ isPdfMime,
10
+ isVideoMime,
11
+ mediaInputFromSource,
12
+ mediaPathFromUrl,
13
+ mimeForDataUrl,
14
+ } from "./mime.ts";
15
+
16
+ function record(value: unknown): Record<string, unknown> | undefined {
17
+ return value !== null && typeof value === "object"
18
+ ? (value as Record<string, unknown>)
19
+ : undefined;
20
+ }
21
+
22
+ function isInputImageBlock(b: Record<string, unknown>): boolean {
23
+ return b.type === "input_image" && typeof b.image_url === "string";
24
+ }
25
+
26
+ function rewriteBlock(
27
+ block: Record<string, unknown>,
28
+ ): Record<string, unknown> | null {
29
+ if (!isInputImageBlock(block)) return null;
30
+ const url = block.image_url as string;
31
+ const mime = mimeForDataUrl(url);
32
+ const lowerPath = mediaPathFromUrl(url).toLowerCase();
33
+ const isHttps = url.startsWith("https://");
34
+ const isHttpsVideo =
35
+ isHttps && (lowerPath.endsWith(".mp4") || lowerPath.endsWith(".m4v"));
36
+ const isHttpsAudio =
37
+ isHttps && (lowerPath.endsWith(".mp3") || lowerPath.endsWith(".wav"));
38
+ const isHttpsPdf = isHttps && lowerPath.endsWith(".pdf");
39
+ const isHttpsImage =
40
+ isHttps &&
41
+ (lowerPath.endsWith(".png") ||
42
+ lowerPath.endsWith(".jpg") ||
43
+ lowerPath.endsWith(".jpeg") ||
44
+ lowerPath.endsWith(".gif") ||
45
+ lowerPath.endsWith(".webp"));
46
+
47
+ if (isHttpsImage || isImageMime(mime)) return null;
48
+
49
+ if (isVideoMime(mime) || isHttpsVideo) {
50
+ return mediaInputFromSource(url, "video.mp4", "video/mp4") as Record<
51
+ string,
52
+ unknown
53
+ >;
54
+ }
55
+ if (isAudioMime(mime) || isHttpsAudio) {
56
+ const effectiveMime =
57
+ typeof mime === "string"
58
+ ? mime
59
+ : lowerPath.endsWith(".wav")
60
+ ? "audio/wav"
61
+ : "audio/mpeg";
62
+ return mediaInputFromSource(
63
+ url,
64
+ filenameForMime(effectiveMime, "audio.bin"),
65
+ effectiveMime,
66
+ ) as Record<string, unknown>;
67
+ }
68
+ if (isPdfMime(mime) || isHttpsPdf) {
69
+ return mediaInputFromSource(
70
+ url,
71
+ "document.pdf",
72
+ "application/pdf",
73
+ ) as Record<string, unknown>;
74
+ }
75
+ return null;
76
+ }
77
+
78
+ function rewriteBlocks(
79
+ blocks: Record<string, unknown>[],
80
+ onlyImageUrls = false,
81
+ ): { blocks: Record<string, unknown>[]; rewritten: boolean } {
82
+ let rewritten = false;
83
+ const next = blocks.map((block) => {
84
+ if (
85
+ onlyImageUrls &&
86
+ !(block.type === "input_image" && typeof block.image_url === "string")
87
+ ) {
88
+ return block;
89
+ }
90
+ const replacement = rewriteBlock(block);
91
+ if (replacement) {
92
+ rewritten = true;
93
+ return replacement;
94
+ }
95
+ return block;
96
+ });
97
+ return { blocks: next, rewritten };
98
+ }
99
+
100
+ /** Rewrite smuggled media image_url blocks into Meta-supported typed blocks. */
101
+ export function rewriteResponsesPayload(payload: unknown): {
102
+ rewritten: boolean;
103
+ payload: unknown;
104
+ } {
105
+ if (!payload || typeof payload !== "object")
106
+ return { rewritten: false, payload };
107
+ const p = payload as Record<string, unknown>;
108
+ const input = p.input;
109
+ if (!Array.isArray(input)) return { rewritten: false, payload };
110
+ let rewritten = false;
111
+ const newInput = input.map((entry) => {
112
+ if (!entry || typeof entry !== "object") return entry;
113
+ const e = { ...(entry as Record<string, unknown>) };
114
+ if (Array.isArray(e.content)) {
115
+ const result = rewriteBlocks(e.content as Record<string, unknown>[]);
116
+ if (result.rewritten) {
117
+ rewritten = true;
118
+ e.content = result.blocks;
119
+ }
120
+ }
121
+ if (
122
+ (e.type === "function_call_output" ||
123
+ e.type === "custom_tool_call_output") &&
124
+ Array.isArray(e.output)
125
+ ) {
126
+ const result = rewriteBlocks(
127
+ e.output as Record<string, unknown>[],
128
+ true,
129
+ );
130
+ if (result.rewritten) {
131
+ rewritten = true;
132
+ e.output = result.blocks;
133
+ }
134
+ }
135
+ return e;
136
+ });
137
+ if (!rewritten) return { rewritten: false, payload };
138
+ return { rewritten: true, payload: { ...p, input: newInput } };
139
+ }
140
+
141
+ function inlineFromBlock(
142
+ block: Record<string, unknown>,
143
+ ): { base64: string; mime: string; filename: string } | undefined {
144
+ if (
145
+ block.type === "input_video" &&
146
+ typeof block.video_url === "string" &&
147
+ block.video_url.startsWith("data:")
148
+ ) {
149
+ return {
150
+ base64: base64FromDataUrl(block.video_url),
151
+ mime: mimeForDataUrl(block.video_url) ?? "video/mp4",
152
+ filename: "video.mp4",
153
+ };
154
+ }
155
+ if (block.type === "input_audio") {
156
+ const audio = record(block.input_audio);
157
+ if (typeof audio?.data === "string") {
158
+ const format = audio.format === "wav" ? "wav" : "mp3";
159
+ return {
160
+ base64: audio.data.startsWith("data:")
161
+ ? base64FromDataUrl(audio.data)
162
+ : audio.data,
163
+ mime: format === "wav" ? "audio/wav" : "audio/mpeg",
164
+ filename: `audio.${format}`,
165
+ };
166
+ }
167
+ }
168
+ if (
169
+ block.type === "input_file" &&
170
+ typeof block.file_data === "string" &&
171
+ block.file_data.startsWith("data:")
172
+ ) {
173
+ const mime = mimeForDataUrl(block.file_data) ?? "application/octet-stream";
174
+ return {
175
+ base64: base64FromDataUrl(block.file_data),
176
+ mime,
177
+ filename:
178
+ typeof block.filename === "string" && block.filename
179
+ ? block.filename
180
+ : filenameForMime(mime),
181
+ };
182
+ }
183
+ if (
184
+ block.type === "input_image" &&
185
+ typeof block.image_url === "string" &&
186
+ block.image_url.startsWith("data:")
187
+ ) {
188
+ const mime = mimeForDataUrl(block.image_url) ?? "image/png";
189
+ return {
190
+ base64: base64FromDataUrl(block.image_url),
191
+ mime,
192
+ filename: filenameForMime(mime, "image.bin"),
193
+ };
194
+ }
195
+ return undefined;
196
+ }
197
+
198
+ function isMediaBlock(block: Record<string, unknown>): boolean {
199
+ return (
200
+ block.type === "input_file" ||
201
+ block.type === "input_image" ||
202
+ block.type === "input_video" ||
203
+ block.type === "input_audio"
204
+ );
205
+ }
206
+
207
+ export function payloadContainsMedia(payload: unknown): boolean {
208
+ if (!payload || typeof payload !== "object") return false;
209
+ const input = (payload as Record<string, unknown>).input;
210
+ if (!Array.isArray(input)) return false;
211
+ for (const entry of input as Record<string, unknown>[]) {
212
+ if (!entry || typeof entry !== "object") continue;
213
+ const groups = [entry.content, entry.output];
214
+ for (const group of groups) {
215
+ if (!Array.isArray(group)) continue;
216
+ for (const block of group as Record<string, unknown>[]) {
217
+ if (block && typeof block === "object" && isMediaBlock(block))
218
+ return true;
219
+ }
220
+ }
221
+ }
222
+ return false;
223
+ }
224
+
225
+ async function promoteBlocks(
226
+ blocks: Record<string, unknown>[],
227
+ apiKey: string,
228
+ uploadThresholdBytes: number,
229
+ signal?: AbortSignal,
230
+ ): Promise<{
231
+ blocks: Record<string, unknown>[];
232
+ uploaded: number;
233
+ needsStoreFalse: boolean;
234
+ }> {
235
+ let uploaded = 0;
236
+ let needsStoreFalse = false;
237
+ const next: Record<string, unknown>[] = [];
238
+ for (const block of blocks) {
239
+ const inline = inlineFromBlock(block);
240
+ if (inline) {
241
+ needsStoreFalse = true;
242
+ if (base64ByteLength(inline.base64) > uploadThresholdBytes) {
243
+ const id = await uploadInlineMedia(
244
+ apiKey,
245
+ inline.base64,
246
+ inline.mime,
247
+ inline.filename,
248
+ signal,
249
+ );
250
+ uploaded += 1;
251
+ next.push({ type: "input_file", file_id: id });
252
+ continue;
253
+ }
254
+ } else if (isMediaBlock(block)) {
255
+ needsStoreFalse = true;
256
+ }
257
+ next.push(block);
258
+ }
259
+ return { blocks: next, uploaded, needsStoreFalse };
260
+ }
261
+
262
+ export async function maybeUploadLargeInlineBlocks(
263
+ payload: unknown,
264
+ apiKey: string | undefined,
265
+ uploadThresholdBytes = STORE_SAFE_INLINE_BYTES,
266
+ signal?: AbortSignal,
267
+ ): Promise<{ payload: unknown; uploaded: number }> {
268
+ if (!apiKey) return { payload, uploaded: 0 };
269
+ if (!payload || typeof payload !== "object") return { payload, uploaded: 0 };
270
+ const p = payload as Record<string, unknown>;
271
+ const input = p.input;
272
+ if (!Array.isArray(input)) return { payload, uploaded: 0 };
273
+ let uploaded = 0;
274
+ let needsStoreFalse = false;
275
+ const newInput: unknown[] = [];
276
+
277
+ for (const entry of input as Record<string, unknown>[]) {
278
+ if (!entry || typeof entry !== "object") {
279
+ newInput.push(entry);
280
+ continue;
281
+ }
282
+ const nextEntry = { ...entry };
283
+ for (const key of ["content", "output"] as const) {
284
+ if (!Array.isArray(nextEntry[key])) continue;
285
+ const promoted = await promoteBlocks(
286
+ nextEntry[key] as Record<string, unknown>[],
287
+ apiKey,
288
+ uploadThresholdBytes,
289
+ signal,
290
+ );
291
+ nextEntry[key] = promoted.blocks;
292
+ uploaded += promoted.uploaded;
293
+ needsStoreFalse ||= promoted.needsStoreFalse;
294
+ }
295
+ newInput.push(nextEntry);
296
+ }
297
+
298
+ if (!needsStoreFalse && uploaded === 0) return { payload, uploaded: 0 };
299
+ return {
300
+ payload: { ...p, input: newInput, store: false },
301
+ uploaded,
302
+ };
303
+ }