pi-meta-oauth 0.4.3 → 0.4.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +21 -36
- package/extensions/media/files.ts +131 -0
- package/extensions/media/limits.ts +12 -0
- package/extensions/media/mime.ts +178 -0
- package/extensions/media/payload.ts +303 -0
- package/extensions/media/responses.ts +186 -0
- package/extensions/media-output.ts +5 -1
- package/extensions/media.ts +108 -947
- package/extensions/meta.ts +165 -41
- package/extensions/voice/asr.ts +107 -0
- package/extensions/voice/helpers.ts +243 -0
- package/extensions/voice.ts +27 -333
- package/package.json +1 -1
package/README.md
CHANGED
|
@@ -61,10 +61,10 @@ Audio is captured only between the two Alt+V presses and streamed as 16 kHz mono
|
|
|
61
61
|
|
|
62
62
|
**Windows:** uses `winmm.dll` `waveIn*` via a C# helper. On first run pi tries to compile `extensions/voice/windows-audio.cs` with `csc.exe` (in-box .NET Framework) to `~/.pi/agent/bin/pi-meta-oauth-voice-v1.exe` for best performance. If no compiler is found it falls back to `extensions/voice/windows-audio.ps1` executed with `powershell.exe -ExecutionPolicy Bypass` (PowerShell 5.1 inbox, or `pwsh` 7 if present) which compiles the same capture code at runtime via `Add-Type`. No extra install is required. If recording fails, check **Settings → Privacy & security → Microphone** that access is allowed for desktop apps.
|
|
63
63
|
|
|
64
|
-
Optional overrides:
|
|
64
|
+
Optional overrides (the `PI_META_*` name takes precedence):
|
|
65
65
|
|
|
66
|
-
- `PI_META_VOICE_ASR_ENDPOINT`
|
|
67
|
-
- `PI_META_VOICE_ASR_MODEL`
|
|
66
|
+
- `PI_META_VOICE_ASR_ENDPOINT` (legacy alias: `MUSE_VOICE_ASR_ENDPOINT`)
|
|
67
|
+
- `PI_META_VOICE_ASR_MODEL` (legacy alias: `MUSE_VOICE_ASR_MODEL`)
|
|
68
68
|
|
|
69
69
|
## Media tools
|
|
70
70
|
|
|
@@ -91,6 +91,8 @@ Fallback models use a 1,048,576-token context window, up to 256K output tokens,
|
|
|
91
91
|
| `muse-spark-1.2-contributor` | 0.10 / 0.20 / 0.002 |
|
|
92
92
|
| `muse-spark-1.1` | 1.25 / 4.25 / 0.15 |
|
|
93
93
|
|
|
94
|
+
> **Contributor-model privacy:** the discounted contributor model allows Meta to use prompts and completions for product improvement, including training future Meta models. Use the standard `muse-spark-1.2` model if you do not want the contributor terms. See [Meta's model documentation](https://dev.meta.ai/docs/models).
|
|
95
|
+
|
|
94
96
|
To scope Pi's model picker to Meta models:
|
|
95
97
|
|
|
96
98
|
```json
|
|
@@ -99,39 +101,22 @@ To scope Pi's model picker to Meta models:
|
|
|
99
101
|
|
|
100
102
|
### Making context windows visible to external tools
|
|
101
103
|
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
"muse-spark-1.2": { "contextWindow": 1007997 },
|
|
119
|
-
"muse-spark-1.2-contributor": { "contextWindow": 1007997 }
|
|
120
|
-
}
|
|
121
|
-
}
|
|
122
|
-
}
|
|
123
|
-
}
|
|
124
|
-
```
|
|
125
|
-
|
|
126
|
-
`1,007,997` is the effective limit reported for those two models by a cached
|
|
127
|
-
Muse Code 0.1.0/R708.1 catalog snapshot observed on 2026-08-06. That snapshot
|
|
128
|
-
did not include `muse-spark-1.1`, so no effective-limit override is claimed for
|
|
129
|
-
1.1 here. The bundled fallback in `extensions/meta.ts` uses the nominal
|
|
130
|
-
`1048576`; use the lower value only when you want percentages to match that
|
|
131
|
-
specific Muse snapshot.
|
|
132
|
-
|
|
133
|
-
This only adds metadata — the provider itself is still registered by the
|
|
134
|
-
extension, and `pi --list-models meta` is unchanged.
|
|
104
|
+
After a successful network model refresh, the extension persists the Meta
|
|
105
|
+
catalog to `~/.pi/agent/models-store.json`. External usage tools such as
|
|
106
|
+
[herdr-agent-usage](https://github.com/senna-lang/herdr-agent-usage) can then
|
|
107
|
+
show a percentage (for example, `⛁ 2% (24k)`) instead of only an absolute token
|
|
108
|
+
count. Pi writes the cache during interactive or RPC startup, and again after
|
|
109
|
+
`/login meta`. `pi --list-models meta` lists currently available models but does
|
|
110
|
+
not itself trigger a network catalog refresh. The cached catalog is also used
|
|
111
|
+
when Pi starts without network access.
|
|
112
|
+
|
|
113
|
+
The bundled fallback uses Meta's nominal `1,048,576`-token context window. A
|
|
114
|
+
cached Muse Code 0.1.0/R708.1 catalog observed on 2026-08-06 reported a lower
|
|
115
|
+
effective limit of `1,007,997` for `muse-spark-1.2` and
|
|
116
|
+
`muse-spark-1.2-contributor`. If you need percentages to match that specific
|
|
117
|
+
Muse snapshot, you can still set `contextWindow: 1007997` for those models in
|
|
118
|
+
`~/.pi/agent/models.json`; model overrides take precedence over the persisted
|
|
119
|
+
catalog.
|
|
135
120
|
|
|
136
121
|
## Verify
|
|
137
122
|
|
|
@@ -0,0 +1,131 @@
|
|
|
1
|
+
import { basename } from "node:path";
|
|
2
|
+
import { readFile } from "node:fs/promises";
|
|
3
|
+
import { statSync } from "node:fs";
|
|
4
|
+
import { META_FILES_URL } from "../meta.ts";
|
|
5
|
+
import {
|
|
6
|
+
AUTOMATIC_UPLOAD_EXPIRY_SECONDS,
|
|
7
|
+
FILES_API_LIMIT_BYTES,
|
|
8
|
+
} from "./limits.ts";
|
|
9
|
+
import { filenameForMime, mimeForPath } from "./mime.ts";
|
|
10
|
+
|
|
11
|
+
export interface UploadResult {
|
|
12
|
+
id: string;
|
|
13
|
+
filename: string;
|
|
14
|
+
bytes: number;
|
|
15
|
+
purpose: string;
|
|
16
|
+
status: string;
|
|
17
|
+
expires_at?: number;
|
|
18
|
+
}
|
|
19
|
+
|
|
20
|
+
function uploadErrorDetail(body: Record<string, unknown>, bodyText: string): string {
|
|
21
|
+
if (typeof body.error === "string") return body.error;
|
|
22
|
+
if (typeof body.message === "string") return body.message;
|
|
23
|
+
if (typeof body.error_description === "string") return body.error_description;
|
|
24
|
+
return bodyText.slice(0, 500);
|
|
25
|
+
}
|
|
26
|
+
|
|
27
|
+
export async function uploadMetaFile(
|
|
28
|
+
apiKey: string,
|
|
29
|
+
filePath: string,
|
|
30
|
+
expiresAfter?: { anchor: "created_at"; seconds: number },
|
|
31
|
+
signal?: AbortSignal,
|
|
32
|
+
): Promise<UploadResult> {
|
|
33
|
+
const size = statSync(filePath).size;
|
|
34
|
+
if (size > FILES_API_LIMIT_BYTES) {
|
|
35
|
+
throw new Error(
|
|
36
|
+
`File too large for Files API (${size} bytes > 1 GiB)`,
|
|
37
|
+
);
|
|
38
|
+
}
|
|
39
|
+
const data = await readFile(filePath, { signal });
|
|
40
|
+
const mime = mimeForPath(filePath) ?? "application/octet-stream";
|
|
41
|
+
const blob = new Blob([data], { type: mime });
|
|
42
|
+
const form = new FormData();
|
|
43
|
+
form.append("file", blob, basename(filePath));
|
|
44
|
+
form.append("purpose", "user_data");
|
|
45
|
+
if (expiresAfter) {
|
|
46
|
+
form.append("expires_after[anchor]", expiresAfter.anchor);
|
|
47
|
+
form.append("expires_after[seconds]", String(expiresAfter.seconds));
|
|
48
|
+
}
|
|
49
|
+
const res = await fetch(META_FILES_URL, {
|
|
50
|
+
method: "POST",
|
|
51
|
+
headers: { Authorization: `Bearer ${apiKey}` },
|
|
52
|
+
body: form,
|
|
53
|
+
signal,
|
|
54
|
+
});
|
|
55
|
+
const bodyText = await res.text();
|
|
56
|
+
let body: Record<string, unknown> = {};
|
|
57
|
+
try {
|
|
58
|
+
body = bodyText ? (JSON.parse(bodyText) as Record<string, unknown>) : {};
|
|
59
|
+
} catch {
|
|
60
|
+
// Non-JSON body — keep empty and use raw text for error detail
|
|
61
|
+
}
|
|
62
|
+
if (!res.ok) {
|
|
63
|
+
const detail = uploadErrorDetail(body, bodyText);
|
|
64
|
+
if (res.status === 400 && detail.includes("storage")) {
|
|
65
|
+
throw new Error(
|
|
66
|
+
`Meta Files storage limit reached (100 GiB/team). Delete old files via DELETE /v1/files/{id}. Detail: ${detail}`,
|
|
67
|
+
);
|
|
68
|
+
}
|
|
69
|
+
throw new Error(
|
|
70
|
+
`Meta Files upload failed (HTTP ${res.status}): ${detail || res.statusText}`,
|
|
71
|
+
);
|
|
72
|
+
}
|
|
73
|
+
const id =
|
|
74
|
+
typeof body.id === "string"
|
|
75
|
+
? body.id
|
|
76
|
+
: typeof body.file_id === "string"
|
|
77
|
+
? body.file_id
|
|
78
|
+
: undefined;
|
|
79
|
+
if (!id) {
|
|
80
|
+
throw new Error(
|
|
81
|
+
`Meta Files upload returned no id: ${bodyText.slice(0, 500)}`,
|
|
82
|
+
);
|
|
83
|
+
}
|
|
84
|
+
return {
|
|
85
|
+
id,
|
|
86
|
+
filename:
|
|
87
|
+
typeof body.filename === "string" ? body.filename : basename(filePath),
|
|
88
|
+
bytes: typeof body.bytes === "number" ? body.bytes : data.byteLength,
|
|
89
|
+
purpose: typeof body.purpose === "string" ? body.purpose : "user_data",
|
|
90
|
+
status: typeof body.status === "string" ? body.status : "uploaded",
|
|
91
|
+
expires_at:
|
|
92
|
+
typeof body.expires_at === "number" ? body.expires_at : undefined,
|
|
93
|
+
};
|
|
94
|
+
}
|
|
95
|
+
|
|
96
|
+
export async function uploadInlineMedia(
|
|
97
|
+
apiKey: string,
|
|
98
|
+
base64: string,
|
|
99
|
+
mime: string,
|
|
100
|
+
filename: string,
|
|
101
|
+
signal?: AbortSignal,
|
|
102
|
+
): Promise<string> {
|
|
103
|
+
const blob = new Blob([Buffer.from(base64, "base64")], { type: mime });
|
|
104
|
+
const form = new FormData();
|
|
105
|
+
form.append("file", blob, filename || filenameForMime(mime));
|
|
106
|
+
form.append("purpose", "user_data");
|
|
107
|
+
form.append("expires_after[anchor]", "created_at");
|
|
108
|
+
form.append("expires_after[seconds]", String(AUTOMATIC_UPLOAD_EXPIRY_SECONDS));
|
|
109
|
+
const res = await fetch(META_FILES_URL, {
|
|
110
|
+
method: "POST",
|
|
111
|
+
headers: { Authorization: `Bearer ${apiKey}` },
|
|
112
|
+
body: form,
|
|
113
|
+
signal,
|
|
114
|
+
});
|
|
115
|
+
const text = await res.text();
|
|
116
|
+
if (!res.ok) {
|
|
117
|
+
throw new Error(
|
|
118
|
+
`Files upload for large inline media failed: ${text.slice(0, 500)}`,
|
|
119
|
+
);
|
|
120
|
+
}
|
|
121
|
+
let json: Record<string, unknown>;
|
|
122
|
+
try {
|
|
123
|
+
json = JSON.parse(text) as Record<string, unknown>;
|
|
124
|
+
} catch (error) {
|
|
125
|
+
throw new Error(`Files upload returned invalid JSON: ${String(error)}`);
|
|
126
|
+
}
|
|
127
|
+
if (typeof json.id !== "string" || !json.id) {
|
|
128
|
+
throw new Error(`Files upload returned no id: ${text.slice(0, 500)}`);
|
|
129
|
+
}
|
|
130
|
+
return json.id;
|
|
131
|
+
}
|
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
export const INLINE_LIMIT_BYTES = 50_000_000;
|
|
2
|
+
export const FILES_API_LIMIT_BYTES = 1_073_741_824;
|
|
3
|
+
// Meta Responses `store=true` (default) has ~20 MB persistence limit even when
|
|
4
|
+
// inline limit is 50 MB — a 24 MB base64 payload triggers HTTP 413
|
|
5
|
+
// "payload_too_large … with `store=true`". Keep inline only for small files
|
|
6
|
+
// and force Files API or `store:false` above this threshold.
|
|
7
|
+
export const STORE_SAFE_INLINE_BYTES = 15_000_000;
|
|
8
|
+
export const AUTOMATIC_UPLOAD_EXPIRY_SECONDS = 24 * 60 * 60;
|
|
9
|
+
export const EXPLICIT_UPLOAD_EXPIRY_SECONDS = 7 * 24 * 60 * 60;
|
|
10
|
+
export const MIN_UPLOAD_EXPIRY_SECONDS = 3_600;
|
|
11
|
+
export const MAX_UPLOAD_EXPIRY_SECONDS = 2_592_000;
|
|
12
|
+
export const MAX_ANALYSIS_SOURCES = 50;
|
|
@@ -0,0 +1,178 @@
|
|
|
1
|
+
import { extname } from "node:path";
|
|
2
|
+
|
|
3
|
+
export interface ResponsesContentBlock {
|
|
4
|
+
type: string;
|
|
5
|
+
[key: string]: unknown;
|
|
6
|
+
}
|
|
7
|
+
|
|
8
|
+
type SupportedMime =
|
|
9
|
+
| "video/mp4"
|
|
10
|
+
| "audio/mpeg"
|
|
11
|
+
| "audio/wav"
|
|
12
|
+
| "application/pdf"
|
|
13
|
+
| "image/png"
|
|
14
|
+
| "image/jpeg"
|
|
15
|
+
| "image/gif"
|
|
16
|
+
| "image/webp"
|
|
17
|
+
| "image/x-icon"
|
|
18
|
+
| "text/plain"
|
|
19
|
+
| "application/json"
|
|
20
|
+
| "application/jsonl";
|
|
21
|
+
|
|
22
|
+
const EXT_TO_MIME: Record<string, SupportedMime> = {
|
|
23
|
+
".mp4": "video/mp4",
|
|
24
|
+
".m4v": "video/mp4",
|
|
25
|
+
".mp3": "audio/mpeg",
|
|
26
|
+
".wav": "audio/wav",
|
|
27
|
+
".pdf": "application/pdf",
|
|
28
|
+
".png": "image/png",
|
|
29
|
+
".jpg": "image/jpeg",
|
|
30
|
+
".jpeg": "image/jpeg",
|
|
31
|
+
".gif": "image/gif",
|
|
32
|
+
".webp": "image/webp",
|
|
33
|
+
".ico": "image/x-icon",
|
|
34
|
+
".txt": "text/plain",
|
|
35
|
+
".json": "application/json",
|
|
36
|
+
".jsonl": "application/jsonl",
|
|
37
|
+
};
|
|
38
|
+
|
|
39
|
+
export function mimeForPath(path: string): SupportedMime | undefined {
|
|
40
|
+
const ext = extname(path).toLowerCase();
|
|
41
|
+
return EXT_TO_MIME[ext];
|
|
42
|
+
}
|
|
43
|
+
|
|
44
|
+
export function mimeForDataUrl(dataUrl: string): string | undefined {
|
|
45
|
+
const m = dataUrl.match(/^data:([^;,\s]+)[;,]/);
|
|
46
|
+
return m?.[1]?.toLowerCase();
|
|
47
|
+
}
|
|
48
|
+
|
|
49
|
+
export function isVideoMime(m: string | undefined): boolean {
|
|
50
|
+
return m === "video/mp4";
|
|
51
|
+
}
|
|
52
|
+
|
|
53
|
+
export function isAudioMime(m: string | undefined): boolean {
|
|
54
|
+
return (
|
|
55
|
+
m === "audio/mpeg" ||
|
|
56
|
+
m === "audio/mp3" ||
|
|
57
|
+
m === "audio/wav" ||
|
|
58
|
+
m === "audio/x-wav"
|
|
59
|
+
);
|
|
60
|
+
}
|
|
61
|
+
|
|
62
|
+
export function isPdfMime(m: string | undefined): boolean {
|
|
63
|
+
return m === "application/pdf";
|
|
64
|
+
}
|
|
65
|
+
|
|
66
|
+
export function isImageMime(m: string | undefined): boolean {
|
|
67
|
+
return typeof m === "string" && m.startsWith("image/");
|
|
68
|
+
}
|
|
69
|
+
|
|
70
|
+
export function filenameForMime(
|
|
71
|
+
mime: string | undefined,
|
|
72
|
+
fallback = "file.bin",
|
|
73
|
+
): string {
|
|
74
|
+
switch (mime) {
|
|
75
|
+
case "video/mp4":
|
|
76
|
+
return "video.mp4";
|
|
77
|
+
case "audio/mpeg":
|
|
78
|
+
case "audio/mp3":
|
|
79
|
+
return "audio.mp3";
|
|
80
|
+
case "audio/wav":
|
|
81
|
+
case "audio/x-wav":
|
|
82
|
+
return "audio.wav";
|
|
83
|
+
case "application/pdf":
|
|
84
|
+
return "document.pdf";
|
|
85
|
+
default:
|
|
86
|
+
return fallback;
|
|
87
|
+
}
|
|
88
|
+
}
|
|
89
|
+
|
|
90
|
+
export function audioFormatForMime(mime: string | undefined): "wav" | "mp3" {
|
|
91
|
+
return mime === "audio/wav" || mime === "audio/x-wav" ? "wav" : "mp3";
|
|
92
|
+
}
|
|
93
|
+
|
|
94
|
+
export function base64FromDataUrl(dataUrl: string): string {
|
|
95
|
+
const comma = dataUrl.indexOf(",");
|
|
96
|
+
return comma === -1 ? "" : dataUrl.slice(comma + 1);
|
|
97
|
+
}
|
|
98
|
+
|
|
99
|
+
export function base64ByteLength(base64: string): number {
|
|
100
|
+
const normalized = base64.replace(/\s/g, "");
|
|
101
|
+
if (!normalized) return 0;
|
|
102
|
+
const padding = normalized.endsWith("==")
|
|
103
|
+
? 2
|
|
104
|
+
: normalized.endsWith("=")
|
|
105
|
+
? 1
|
|
106
|
+
: 0;
|
|
107
|
+
return Math.max(0, Math.floor((normalized.length * 3) / 4) - padding);
|
|
108
|
+
}
|
|
109
|
+
|
|
110
|
+
export function isRemoteMediaSource(source: string): boolean {
|
|
111
|
+
return source.startsWith("https://") || source.startsWith("data:");
|
|
112
|
+
}
|
|
113
|
+
|
|
114
|
+
export function mediaPathFromUrl(url: string): string {
|
|
115
|
+
if (url.startsWith("data:")) return url;
|
|
116
|
+
try {
|
|
117
|
+
return new URL(url).pathname;
|
|
118
|
+
} catch {
|
|
119
|
+
const withoutHash = url.split("#")[0] ?? url;
|
|
120
|
+
return withoutHash.split("?")[0] ?? withoutHash;
|
|
121
|
+
}
|
|
122
|
+
}
|
|
123
|
+
|
|
124
|
+
export function mimeForSource(source: string): string | undefined {
|
|
125
|
+
if (source.startsWith("data:")) return mimeForDataUrl(source);
|
|
126
|
+
if (source.startsWith("https://")) {
|
|
127
|
+
try {
|
|
128
|
+
return mimeForPath(new URL(source).pathname);
|
|
129
|
+
} catch {
|
|
130
|
+
return undefined;
|
|
131
|
+
}
|
|
132
|
+
}
|
|
133
|
+
return mimeForPath(source);
|
|
134
|
+
}
|
|
135
|
+
|
|
136
|
+
/** Build a generic Meta Responses input_file block. */
|
|
137
|
+
export function inputFileFromSource(
|
|
138
|
+
source: string,
|
|
139
|
+
filename?: string,
|
|
140
|
+
): ResponsesContentBlock {
|
|
141
|
+
if (source.startsWith("https://")) {
|
|
142
|
+
return { type: "input_file", file_url: source };
|
|
143
|
+
}
|
|
144
|
+
if (source.startsWith("data:")) {
|
|
145
|
+
const mime = mimeForDataUrl(source);
|
|
146
|
+
return {
|
|
147
|
+
type: "input_file",
|
|
148
|
+
file_data: source,
|
|
149
|
+
filename: filename ?? filenameForMime(mime),
|
|
150
|
+
};
|
|
151
|
+
}
|
|
152
|
+
return { type: "input_file", file_id: source };
|
|
153
|
+
}
|
|
154
|
+
|
|
155
|
+
/** Build the typed content block required by Meta for each media kind. */
|
|
156
|
+
export function mediaInputFromSource(
|
|
157
|
+
source: string,
|
|
158
|
+
filename?: string,
|
|
159
|
+
expectedMime?: string,
|
|
160
|
+
): ResponsesContentBlock {
|
|
161
|
+
if (!source.startsWith("data:") && !source.startsWith("https://")) {
|
|
162
|
+
return { type: "input_file", file_id: source };
|
|
163
|
+
}
|
|
164
|
+
const mime = expectedMime ?? mimeForSource(source);
|
|
165
|
+
if (isVideoMime(mime)) {
|
|
166
|
+
return { type: "input_video", video_url: source };
|
|
167
|
+
}
|
|
168
|
+
if (isAudioMime(mime) && source.startsWith("data:")) {
|
|
169
|
+
return {
|
|
170
|
+
type: "input_audio",
|
|
171
|
+
input_audio: {
|
|
172
|
+
data: base64FromDataUrl(source),
|
|
173
|
+
format: audioFormatForMime(mime),
|
|
174
|
+
},
|
|
175
|
+
};
|
|
176
|
+
}
|
|
177
|
+
return inputFileFromSource(source, filename);
|
|
178
|
+
}
|
|
@@ -0,0 +1,303 @@
|
|
|
1
|
+
import { STORE_SAFE_INLINE_BYTES } from "./limits.ts";
|
|
2
|
+
import { uploadInlineMedia } from "./files.ts";
|
|
3
|
+
import {
|
|
4
|
+
base64ByteLength,
|
|
5
|
+
base64FromDataUrl,
|
|
6
|
+
filenameForMime,
|
|
7
|
+
isAudioMime,
|
|
8
|
+
isImageMime,
|
|
9
|
+
isPdfMime,
|
|
10
|
+
isVideoMime,
|
|
11
|
+
mediaInputFromSource,
|
|
12
|
+
mediaPathFromUrl,
|
|
13
|
+
mimeForDataUrl,
|
|
14
|
+
} from "./mime.ts";
|
|
15
|
+
|
|
16
|
+
function record(value: unknown): Record<string, unknown> | undefined {
|
|
17
|
+
return value !== null && typeof value === "object"
|
|
18
|
+
? (value as Record<string, unknown>)
|
|
19
|
+
: undefined;
|
|
20
|
+
}
|
|
21
|
+
|
|
22
|
+
function isInputImageBlock(b: Record<string, unknown>): boolean {
|
|
23
|
+
return b.type === "input_image" && typeof b.image_url === "string";
|
|
24
|
+
}
|
|
25
|
+
|
|
26
|
+
function rewriteBlock(
|
|
27
|
+
block: Record<string, unknown>,
|
|
28
|
+
): Record<string, unknown> | null {
|
|
29
|
+
if (!isInputImageBlock(block)) return null;
|
|
30
|
+
const url = block.image_url as string;
|
|
31
|
+
const mime = mimeForDataUrl(url);
|
|
32
|
+
const lowerPath = mediaPathFromUrl(url).toLowerCase();
|
|
33
|
+
const isHttps = url.startsWith("https://");
|
|
34
|
+
const isHttpsVideo =
|
|
35
|
+
isHttps && (lowerPath.endsWith(".mp4") || lowerPath.endsWith(".m4v"));
|
|
36
|
+
const isHttpsAudio =
|
|
37
|
+
isHttps && (lowerPath.endsWith(".mp3") || lowerPath.endsWith(".wav"));
|
|
38
|
+
const isHttpsPdf = isHttps && lowerPath.endsWith(".pdf");
|
|
39
|
+
const isHttpsImage =
|
|
40
|
+
isHttps &&
|
|
41
|
+
(lowerPath.endsWith(".png") ||
|
|
42
|
+
lowerPath.endsWith(".jpg") ||
|
|
43
|
+
lowerPath.endsWith(".jpeg") ||
|
|
44
|
+
lowerPath.endsWith(".gif") ||
|
|
45
|
+
lowerPath.endsWith(".webp"));
|
|
46
|
+
|
|
47
|
+
if (isHttpsImage || isImageMime(mime)) return null;
|
|
48
|
+
|
|
49
|
+
if (isVideoMime(mime) || isHttpsVideo) {
|
|
50
|
+
return mediaInputFromSource(url, "video.mp4", "video/mp4") as Record<
|
|
51
|
+
string,
|
|
52
|
+
unknown
|
|
53
|
+
>;
|
|
54
|
+
}
|
|
55
|
+
if (isAudioMime(mime) || isHttpsAudio) {
|
|
56
|
+
const effectiveMime =
|
|
57
|
+
typeof mime === "string"
|
|
58
|
+
? mime
|
|
59
|
+
: lowerPath.endsWith(".wav")
|
|
60
|
+
? "audio/wav"
|
|
61
|
+
: "audio/mpeg";
|
|
62
|
+
return mediaInputFromSource(
|
|
63
|
+
url,
|
|
64
|
+
filenameForMime(effectiveMime, "audio.bin"),
|
|
65
|
+
effectiveMime,
|
|
66
|
+
) as Record<string, unknown>;
|
|
67
|
+
}
|
|
68
|
+
if (isPdfMime(mime) || isHttpsPdf) {
|
|
69
|
+
return mediaInputFromSource(
|
|
70
|
+
url,
|
|
71
|
+
"document.pdf",
|
|
72
|
+
"application/pdf",
|
|
73
|
+
) as Record<string, unknown>;
|
|
74
|
+
}
|
|
75
|
+
return null;
|
|
76
|
+
}
|
|
77
|
+
|
|
78
|
+
function rewriteBlocks(
|
|
79
|
+
blocks: Record<string, unknown>[],
|
|
80
|
+
onlyImageUrls = false,
|
|
81
|
+
): { blocks: Record<string, unknown>[]; rewritten: boolean } {
|
|
82
|
+
let rewritten = false;
|
|
83
|
+
const next = blocks.map((block) => {
|
|
84
|
+
if (
|
|
85
|
+
onlyImageUrls &&
|
|
86
|
+
!(block.type === "input_image" && typeof block.image_url === "string")
|
|
87
|
+
) {
|
|
88
|
+
return block;
|
|
89
|
+
}
|
|
90
|
+
const replacement = rewriteBlock(block);
|
|
91
|
+
if (replacement) {
|
|
92
|
+
rewritten = true;
|
|
93
|
+
return replacement;
|
|
94
|
+
}
|
|
95
|
+
return block;
|
|
96
|
+
});
|
|
97
|
+
return { blocks: next, rewritten };
|
|
98
|
+
}
|
|
99
|
+
|
|
100
|
+
/** Rewrite smuggled media image_url blocks into Meta-supported typed blocks. */
|
|
101
|
+
export function rewriteResponsesPayload(payload: unknown): {
|
|
102
|
+
rewritten: boolean;
|
|
103
|
+
payload: unknown;
|
|
104
|
+
} {
|
|
105
|
+
if (!payload || typeof payload !== "object")
|
|
106
|
+
return { rewritten: false, payload };
|
|
107
|
+
const p = payload as Record<string, unknown>;
|
|
108
|
+
const input = p.input;
|
|
109
|
+
if (!Array.isArray(input)) return { rewritten: false, payload };
|
|
110
|
+
let rewritten = false;
|
|
111
|
+
const newInput = input.map((entry) => {
|
|
112
|
+
if (!entry || typeof entry !== "object") return entry;
|
|
113
|
+
const e = { ...(entry as Record<string, unknown>) };
|
|
114
|
+
if (Array.isArray(e.content)) {
|
|
115
|
+
const result = rewriteBlocks(e.content as Record<string, unknown>[]);
|
|
116
|
+
if (result.rewritten) {
|
|
117
|
+
rewritten = true;
|
|
118
|
+
e.content = result.blocks;
|
|
119
|
+
}
|
|
120
|
+
}
|
|
121
|
+
if (
|
|
122
|
+
(e.type === "function_call_output" ||
|
|
123
|
+
e.type === "custom_tool_call_output") &&
|
|
124
|
+
Array.isArray(e.output)
|
|
125
|
+
) {
|
|
126
|
+
const result = rewriteBlocks(
|
|
127
|
+
e.output as Record<string, unknown>[],
|
|
128
|
+
true,
|
|
129
|
+
);
|
|
130
|
+
if (result.rewritten) {
|
|
131
|
+
rewritten = true;
|
|
132
|
+
e.output = result.blocks;
|
|
133
|
+
}
|
|
134
|
+
}
|
|
135
|
+
return e;
|
|
136
|
+
});
|
|
137
|
+
if (!rewritten) return { rewritten: false, payload };
|
|
138
|
+
return { rewritten: true, payload: { ...p, input: newInput } };
|
|
139
|
+
}
|
|
140
|
+
|
|
141
|
+
function inlineFromBlock(
|
|
142
|
+
block: Record<string, unknown>,
|
|
143
|
+
): { base64: string; mime: string; filename: string } | undefined {
|
|
144
|
+
if (
|
|
145
|
+
block.type === "input_video" &&
|
|
146
|
+
typeof block.video_url === "string" &&
|
|
147
|
+
block.video_url.startsWith("data:")
|
|
148
|
+
) {
|
|
149
|
+
return {
|
|
150
|
+
base64: base64FromDataUrl(block.video_url),
|
|
151
|
+
mime: mimeForDataUrl(block.video_url) ?? "video/mp4",
|
|
152
|
+
filename: "video.mp4",
|
|
153
|
+
};
|
|
154
|
+
}
|
|
155
|
+
if (block.type === "input_audio") {
|
|
156
|
+
const audio = record(block.input_audio);
|
|
157
|
+
if (typeof audio?.data === "string") {
|
|
158
|
+
const format = audio.format === "wav" ? "wav" : "mp3";
|
|
159
|
+
return {
|
|
160
|
+
base64: audio.data.startsWith("data:")
|
|
161
|
+
? base64FromDataUrl(audio.data)
|
|
162
|
+
: audio.data,
|
|
163
|
+
mime: format === "wav" ? "audio/wav" : "audio/mpeg",
|
|
164
|
+
filename: `audio.${format}`,
|
|
165
|
+
};
|
|
166
|
+
}
|
|
167
|
+
}
|
|
168
|
+
if (
|
|
169
|
+
block.type === "input_file" &&
|
|
170
|
+
typeof block.file_data === "string" &&
|
|
171
|
+
block.file_data.startsWith("data:")
|
|
172
|
+
) {
|
|
173
|
+
const mime = mimeForDataUrl(block.file_data) ?? "application/octet-stream";
|
|
174
|
+
return {
|
|
175
|
+
base64: base64FromDataUrl(block.file_data),
|
|
176
|
+
mime,
|
|
177
|
+
filename:
|
|
178
|
+
typeof block.filename === "string" && block.filename
|
|
179
|
+
? block.filename
|
|
180
|
+
: filenameForMime(mime),
|
|
181
|
+
};
|
|
182
|
+
}
|
|
183
|
+
if (
|
|
184
|
+
block.type === "input_image" &&
|
|
185
|
+
typeof block.image_url === "string" &&
|
|
186
|
+
block.image_url.startsWith("data:")
|
|
187
|
+
) {
|
|
188
|
+
const mime = mimeForDataUrl(block.image_url) ?? "image/png";
|
|
189
|
+
return {
|
|
190
|
+
base64: base64FromDataUrl(block.image_url),
|
|
191
|
+
mime,
|
|
192
|
+
filename: filenameForMime(mime, "image.bin"),
|
|
193
|
+
};
|
|
194
|
+
}
|
|
195
|
+
return undefined;
|
|
196
|
+
}
|
|
197
|
+
|
|
198
|
+
function isMediaBlock(block: Record<string, unknown>): boolean {
|
|
199
|
+
return (
|
|
200
|
+
block.type === "input_file" ||
|
|
201
|
+
block.type === "input_image" ||
|
|
202
|
+
block.type === "input_video" ||
|
|
203
|
+
block.type === "input_audio"
|
|
204
|
+
);
|
|
205
|
+
}
|
|
206
|
+
|
|
207
|
+
export function payloadContainsMedia(payload: unknown): boolean {
|
|
208
|
+
if (!payload || typeof payload !== "object") return false;
|
|
209
|
+
const input = (payload as Record<string, unknown>).input;
|
|
210
|
+
if (!Array.isArray(input)) return false;
|
|
211
|
+
for (const entry of input as Record<string, unknown>[]) {
|
|
212
|
+
if (!entry || typeof entry !== "object") continue;
|
|
213
|
+
const groups = [entry.content, entry.output];
|
|
214
|
+
for (const group of groups) {
|
|
215
|
+
if (!Array.isArray(group)) continue;
|
|
216
|
+
for (const block of group as Record<string, unknown>[]) {
|
|
217
|
+
if (block && typeof block === "object" && isMediaBlock(block))
|
|
218
|
+
return true;
|
|
219
|
+
}
|
|
220
|
+
}
|
|
221
|
+
}
|
|
222
|
+
return false;
|
|
223
|
+
}
|
|
224
|
+
|
|
225
|
+
async function promoteBlocks(
|
|
226
|
+
blocks: Record<string, unknown>[],
|
|
227
|
+
apiKey: string,
|
|
228
|
+
uploadThresholdBytes: number,
|
|
229
|
+
signal?: AbortSignal,
|
|
230
|
+
): Promise<{
|
|
231
|
+
blocks: Record<string, unknown>[];
|
|
232
|
+
uploaded: number;
|
|
233
|
+
needsStoreFalse: boolean;
|
|
234
|
+
}> {
|
|
235
|
+
let uploaded = 0;
|
|
236
|
+
let needsStoreFalse = false;
|
|
237
|
+
const next: Record<string, unknown>[] = [];
|
|
238
|
+
for (const block of blocks) {
|
|
239
|
+
const inline = inlineFromBlock(block);
|
|
240
|
+
if (inline) {
|
|
241
|
+
needsStoreFalse = true;
|
|
242
|
+
if (base64ByteLength(inline.base64) > uploadThresholdBytes) {
|
|
243
|
+
const id = await uploadInlineMedia(
|
|
244
|
+
apiKey,
|
|
245
|
+
inline.base64,
|
|
246
|
+
inline.mime,
|
|
247
|
+
inline.filename,
|
|
248
|
+
signal,
|
|
249
|
+
);
|
|
250
|
+
uploaded += 1;
|
|
251
|
+
next.push({ type: "input_file", file_id: id });
|
|
252
|
+
continue;
|
|
253
|
+
}
|
|
254
|
+
} else if (isMediaBlock(block)) {
|
|
255
|
+
needsStoreFalse = true;
|
|
256
|
+
}
|
|
257
|
+
next.push(block);
|
|
258
|
+
}
|
|
259
|
+
return { blocks: next, uploaded, needsStoreFalse };
|
|
260
|
+
}
|
|
261
|
+
|
|
262
|
+
export async function maybeUploadLargeInlineBlocks(
|
|
263
|
+
payload: unknown,
|
|
264
|
+
apiKey: string | undefined,
|
|
265
|
+
uploadThresholdBytes = STORE_SAFE_INLINE_BYTES,
|
|
266
|
+
signal?: AbortSignal,
|
|
267
|
+
): Promise<{ payload: unknown; uploaded: number }> {
|
|
268
|
+
if (!apiKey) return { payload, uploaded: 0 };
|
|
269
|
+
if (!payload || typeof payload !== "object") return { payload, uploaded: 0 };
|
|
270
|
+
const p = payload as Record<string, unknown>;
|
|
271
|
+
const input = p.input;
|
|
272
|
+
if (!Array.isArray(input)) return { payload, uploaded: 0 };
|
|
273
|
+
let uploaded = 0;
|
|
274
|
+
let needsStoreFalse = false;
|
|
275
|
+
const newInput: unknown[] = [];
|
|
276
|
+
|
|
277
|
+
for (const entry of input as Record<string, unknown>[]) {
|
|
278
|
+
if (!entry || typeof entry !== "object") {
|
|
279
|
+
newInput.push(entry);
|
|
280
|
+
continue;
|
|
281
|
+
}
|
|
282
|
+
const nextEntry = { ...entry };
|
|
283
|
+
for (const key of ["content", "output"] as const) {
|
|
284
|
+
if (!Array.isArray(nextEntry[key])) continue;
|
|
285
|
+
const promoted = await promoteBlocks(
|
|
286
|
+
nextEntry[key] as Record<string, unknown>[],
|
|
287
|
+
apiKey,
|
|
288
|
+
uploadThresholdBytes,
|
|
289
|
+
signal,
|
|
290
|
+
);
|
|
291
|
+
nextEntry[key] = promoted.blocks;
|
|
292
|
+
uploaded += promoted.uploaded;
|
|
293
|
+
needsStoreFalse ||= promoted.needsStoreFalse;
|
|
294
|
+
}
|
|
295
|
+
newInput.push(nextEntry);
|
|
296
|
+
}
|
|
297
|
+
|
|
298
|
+
if (!needsStoreFalse && uploaded === 0) return { payload, uploaded: 0 };
|
|
299
|
+
return {
|
|
300
|
+
payload: { ...p, input: newInput, store: false },
|
|
301
|
+
uploaded,
|
|
302
|
+
};
|
|
303
|
+
}
|