pi-meta-oauth 0.4.2 → 0.4.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +40 -3
- package/extensions/media/files.ts +131 -0
- package/extensions/media/limits.ts +12 -0
- package/extensions/media/mime.ts +178 -0
- package/extensions/media/payload.ts +303 -0
- package/extensions/media/responses.ts +186 -0
- package/extensions/media-output.ts +127 -0
- package/extensions/media.ts +352 -1245
- package/extensions/meta.ts +181 -42
- package/extensions/voice/asr.ts +107 -0
- package/extensions/voice/helpers.ts +243 -0
- package/extensions/voice/macos-audio.swift +6 -16
- package/extensions/voice.ts +27 -333
- package/package.json +1 -1
package/extensions/media.ts
CHANGED
|
@@ -1,195 +1,59 @@
|
|
|
1
|
-
import type { Usage } from "@earendil-works/pi-ai";
|
|
2
1
|
import type {
|
|
3
2
|
ExtensionAPI,
|
|
4
3
|
ExtensionContext,
|
|
5
4
|
} from "@earendil-works/pi-coding-agent";
|
|
6
5
|
import { Type } from "typebox";
|
|
7
|
-
import {
|
|
8
|
-
createMetaProviderConfig,
|
|
9
|
-
META_FILES_URL,
|
|
10
|
-
META_API_BASE_URL,
|
|
11
|
-
META_PROVIDER_ID,
|
|
12
|
-
} from "./meta.ts";
|
|
6
|
+
import { META_PROVIDER_ID } from "./meta.ts";
|
|
13
7
|
import { existsSync, statSync, readFileSync } from "node:fs";
|
|
14
|
-
import { basename,
|
|
15
|
-
import {
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
".ico": "image/x-icon",
|
|
64
|
-
".txt": "text/plain",
|
|
65
|
-
".json": "application/json",
|
|
66
|
-
".jsonl": "application/jsonl",
|
|
67
|
-
};
|
|
68
|
-
|
|
69
|
-
function mimeForPath(path: string): SupportedMime | undefined {
|
|
70
|
-
const ext = extname(path).toLowerCase();
|
|
71
|
-
return EXT_TO_MIME[ext];
|
|
72
|
-
}
|
|
73
|
-
|
|
74
|
-
function mimeForDataUrl(dataUrl: string): string | undefined {
|
|
75
|
-
const m = dataUrl.match(/^data:([^;,\s]+)[;,]/);
|
|
76
|
-
return m?.[1]?.toLowerCase();
|
|
77
|
-
}
|
|
78
|
-
|
|
79
|
-
function isVideoMime(m: string | undefined): boolean {
|
|
80
|
-
return m === "video/mp4";
|
|
81
|
-
}
|
|
82
|
-
function isAudioMime(m: string | undefined): boolean {
|
|
83
|
-
return (
|
|
84
|
-
m === "audio/mpeg" ||
|
|
85
|
-
m === "audio/mp3" ||
|
|
86
|
-
m === "audio/wav" ||
|
|
87
|
-
m === "audio/x-wav"
|
|
88
|
-
);
|
|
89
|
-
}
|
|
90
|
-
function isPdfMime(m: string | undefined): boolean {
|
|
91
|
-
return m === "application/pdf";
|
|
92
|
-
}
|
|
93
|
-
function isImageMime(m: string | undefined): boolean {
|
|
94
|
-
return !!m && m.startsWith("image/");
|
|
95
|
-
}
|
|
96
|
-
|
|
97
|
-
function filenameForMime(
|
|
98
|
-
mime: string | undefined,
|
|
99
|
-
fallback = "file.bin",
|
|
100
|
-
): string {
|
|
101
|
-
switch (mime) {
|
|
102
|
-
case "video/mp4":
|
|
103
|
-
return "video.mp4";
|
|
104
|
-
case "audio/mpeg":
|
|
105
|
-
case "audio/mp3":
|
|
106
|
-
return "audio.mp3";
|
|
107
|
-
case "audio/wav":
|
|
108
|
-
case "audio/x-wav":
|
|
109
|
-
return "audio.wav";
|
|
110
|
-
case "application/pdf":
|
|
111
|
-
return "document.pdf";
|
|
112
|
-
default:
|
|
113
|
-
return fallback;
|
|
114
|
-
}
|
|
115
|
-
}
|
|
116
|
-
|
|
117
|
-
function audioFormatForMime(mime: string | undefined): "wav" | "mp3" {
|
|
118
|
-
return mime === "audio/wav" || mime === "audio/x-wav" ? "wav" : "mp3";
|
|
119
|
-
}
|
|
120
|
-
|
|
121
|
-
/** Build a generic Meta Responses input_file block. */
|
|
122
|
-
function inputFileFromSource(
|
|
123
|
-
source: string,
|
|
124
|
-
filename?: string,
|
|
125
|
-
): ResponsesContentBlock {
|
|
126
|
-
if (source.startsWith("https://")) {
|
|
127
|
-
return { type: "input_file", file_url: source };
|
|
128
|
-
}
|
|
129
|
-
if (source.startsWith("data:")) {
|
|
130
|
-
const mime = mimeForDataUrl(source);
|
|
131
|
-
return {
|
|
132
|
-
type: "input_file",
|
|
133
|
-
file_data: source,
|
|
134
|
-
filename: filename ?? filenameForMime(mime),
|
|
135
|
-
};
|
|
136
|
-
}
|
|
137
|
-
// Bare file_id (file-...)
|
|
138
|
-
return { type: "input_file", file_id: source };
|
|
139
|
-
}
|
|
140
|
-
|
|
141
|
-
function mimeForSource(source: string): string | undefined {
|
|
142
|
-
if (source.startsWith("data:")) return mimeForDataUrl(source);
|
|
143
|
-
if (source.startsWith("https://")) {
|
|
144
|
-
try {
|
|
145
|
-
return mimeForPath(new URL(source).pathname);
|
|
146
|
-
} catch {
|
|
147
|
-
return undefined;
|
|
148
|
-
}
|
|
149
|
-
}
|
|
150
|
-
return mimeForPath(source);
|
|
151
|
-
}
|
|
152
|
-
|
|
153
|
-
/** Build the typed content block required by Meta for each media kind. */
|
|
154
|
-
export function mediaInputFromSource(
|
|
155
|
-
source: string,
|
|
156
|
-
filename?: string,
|
|
157
|
-
expectedMime?: string,
|
|
158
|
-
): ResponsesContentBlock {
|
|
159
|
-
if (!source.startsWith("data:") && !source.startsWith("https://")) {
|
|
160
|
-
return { type: "input_file", file_id: source };
|
|
161
|
-
}
|
|
162
|
-
const mime = expectedMime ?? mimeForSource(source);
|
|
163
|
-
if (isVideoMime(mime)) {
|
|
164
|
-
return { type: "input_video", video_url: source };
|
|
165
|
-
}
|
|
166
|
-
if (isAudioMime(mime) && source.startsWith("data:")) {
|
|
167
|
-
return {
|
|
168
|
-
type: "input_audio",
|
|
169
|
-
input_audio: {
|
|
170
|
-
data: base64FromDataUrl(source),
|
|
171
|
-
format: audioFormatForMime(mime),
|
|
172
|
-
},
|
|
173
|
-
};
|
|
174
|
-
}
|
|
175
|
-
return inputFileFromSource(source, filename);
|
|
176
|
-
}
|
|
177
|
-
|
|
178
|
-
function base64FromDataUrl(dataUrl: string): string {
|
|
179
|
-
const comma = dataUrl.indexOf(",");
|
|
180
|
-
return comma === -1 ? "" : dataUrl.slice(comma + 1);
|
|
181
|
-
}
|
|
182
|
-
|
|
183
|
-
function base64ByteLength(base64: string): number {
|
|
184
|
-
const normalized = base64.replace(/\s/g, "");
|
|
185
|
-
if (!normalized) return 0;
|
|
186
|
-
const padding = normalized.endsWith("==")
|
|
187
|
-
? 2
|
|
188
|
-
: normalized.endsWith("=")
|
|
189
|
-
? 1
|
|
190
|
-
: 0;
|
|
191
|
-
return Math.max(0, Math.floor((normalized.length * 3) / 4) - padding);
|
|
192
|
-
}
|
|
8
|
+
import { basename, resolve } from "node:path";
|
|
9
|
+
import {
|
|
10
|
+
DEFAULT_MEDIA_MAX_CHARS,
|
|
11
|
+
DEFAULT_MEDIA_MAX_OUTPUT_TOKENS,
|
|
12
|
+
MAX_MEDIA_MAX_CHARS,
|
|
13
|
+
MAX_MEDIA_MAX_OUTPUT_TOKENS,
|
|
14
|
+
MIN_MEDIA_MAX_OUTPUT_TOKENS,
|
|
15
|
+
mediaMaxOutputTokens,
|
|
16
|
+
prepareMediaOutput,
|
|
17
|
+
} from "./media-output.ts";
|
|
18
|
+
import {
|
|
19
|
+
AUTOMATIC_UPLOAD_EXPIRY_SECONDS,
|
|
20
|
+
EXPLICIT_UPLOAD_EXPIRY_SECONDS,
|
|
21
|
+
FILES_API_LIMIT_BYTES,
|
|
22
|
+
INLINE_LIMIT_BYTES,
|
|
23
|
+
MAX_ANALYSIS_SOURCES,
|
|
24
|
+
MAX_UPLOAD_EXPIRY_SECONDS,
|
|
25
|
+
MIN_UPLOAD_EXPIRY_SECONDS,
|
|
26
|
+
STORE_SAFE_INLINE_BYTES,
|
|
27
|
+
} from "./media/limits.ts";
|
|
28
|
+
import {
|
|
29
|
+
base64FromDataUrl,
|
|
30
|
+
isAudioMime,
|
|
31
|
+
isRemoteMediaSource,
|
|
32
|
+
isVideoMime,
|
|
33
|
+
mediaInputFromSource,
|
|
34
|
+
mimeForPath,
|
|
35
|
+
mimeForSource,
|
|
36
|
+
type ResponsesContentBlock,
|
|
37
|
+
} from "./media/mime.ts";
|
|
38
|
+
import { uploadMetaFile } from "./media/files.ts";
|
|
39
|
+
import {
|
|
40
|
+
maybeUploadLargeInlineBlocks,
|
|
41
|
+
payloadContainsMedia,
|
|
42
|
+
rewriteResponsesPayload,
|
|
43
|
+
} from "./media/payload.ts";
|
|
44
|
+
import {
|
|
45
|
+
callMetaResponses,
|
|
46
|
+
extractMetaResponseUsage,
|
|
47
|
+
formatMetaResponseUsage,
|
|
48
|
+
} from "./media/responses.ts";
|
|
49
|
+
|
|
50
|
+
export { mediaInputFromSource } from "./media/mime.ts";
|
|
51
|
+
export { maybeUploadLargeInlineBlocks, rewriteResponsesPayload } from "./media/payload.ts";
|
|
52
|
+
export {
|
|
53
|
+
extractMetaResponseUsage,
|
|
54
|
+
extractResponsesText,
|
|
55
|
+
formatMetaResponseUsage,
|
|
56
|
+
} from "./media/responses.ts";
|
|
193
57
|
|
|
194
58
|
function safeNotify(
|
|
195
59
|
ctx:
|
|
@@ -211,391 +75,24 @@ function safeNotify(
|
|
|
211
75
|
}
|
|
212
76
|
}
|
|
213
77
|
|
|
214
|
-
async function safeGetMetaApiKey(ctx: ExtensionContext): Promise<string> {
|
|
215
|
-
try {
|
|
216
|
-
return await getMetaApiKey(ctx);
|
|
217
|
-
} catch (error) {
|
|
218
|
-
if (String(error).includes("stale after session replacement"))
|
|
219
|
-
throw new Error("Session was replaced — please retry the command");
|
|
220
|
-
throw error;
|
|
221
|
-
}
|
|
222
|
-
}
|
|
223
|
-
|
|
224
78
|
async function getMetaApiKey(ctx: ExtensionContext): Promise<string> {
|
|
225
79
|
const key = await ctx.modelRegistry.getApiKeyForProvider(META_PROVIDER_ID);
|
|
226
|
-
if (!key)
|
|
80
|
+
if (!key) {
|
|
227
81
|
throw new Error(
|
|
228
82
|
`Meta credentials unavailable — run /login ${META_PROVIDER_ID}`,
|
|
229
83
|
);
|
|
230
|
-
return key;
|
|
231
|
-
}
|
|
232
|
-
|
|
233
|
-
async function getMetaApiKeyFromRegistry(
|
|
234
|
-
_api: ExtensionAPI,
|
|
235
|
-
): Promise<string | undefined> {
|
|
236
|
-
// For tools that receive ExtensionContext we prefer ctx.modelRegistry; this is fallback for non-context paths.
|
|
237
|
-
return undefined;
|
|
238
|
-
}
|
|
239
|
-
|
|
240
|
-
interface UploadResult {
|
|
241
|
-
id: string;
|
|
242
|
-
filename: string;
|
|
243
|
-
bytes: number;
|
|
244
|
-
purpose: string;
|
|
245
|
-
status: string;
|
|
246
|
-
expires_at?: number;
|
|
247
|
-
}
|
|
248
|
-
|
|
249
|
-
async function uploadMetaFile(
|
|
250
|
-
apiKey: string,
|
|
251
|
-
filePath: string,
|
|
252
|
-
expiresAfter?: { anchor: "created_at"; seconds: number },
|
|
253
|
-
): Promise<UploadResult> {
|
|
254
|
-
const data = await readFile(filePath);
|
|
255
|
-
if (data.byteLength > FILES_API_LIMIT_BYTES) {
|
|
256
|
-
throw new Error(
|
|
257
|
-
`File too large for Files API (${data.byteLength} bytes > 1 GiB)`,
|
|
258
|
-
);
|
|
259
|
-
}
|
|
260
|
-
const mime = mimeForPath(filePath) ?? "application/octet-stream";
|
|
261
|
-
const blob = new Blob([data], { type: mime });
|
|
262
|
-
const form = new FormData();
|
|
263
|
-
// OpenAI SDK uses `file` field name; Meta docs show same.
|
|
264
|
-
form.append("file", blob, basename(filePath));
|
|
265
|
-
form.append("purpose", "user_data");
|
|
266
|
-
if (expiresAfter) {
|
|
267
|
-
form.append("expires_after[anchor]", expiresAfter.anchor);
|
|
268
|
-
form.append("expires_after[seconds]", String(expiresAfter.seconds));
|
|
269
|
-
}
|
|
270
|
-
const res = await fetch(META_FILES_URL, {
|
|
271
|
-
method: "POST",
|
|
272
|
-
headers: { Authorization: `Bearer ${apiKey}` },
|
|
273
|
-
body: form,
|
|
274
|
-
});
|
|
275
|
-
const bodyText = await res.text();
|
|
276
|
-
let body: Record<string, unknown> = {};
|
|
277
|
-
try {
|
|
278
|
-
body = bodyText ? (JSON.parse(bodyText) as Record<string, unknown>) : {};
|
|
279
|
-
} catch (error) {
|
|
280
|
-
// Non-JSON body — keep empty and use raw text for error detail
|
|
281
|
-
void error;
|
|
282
|
-
}
|
|
283
|
-
if (!res.ok) {
|
|
284
|
-
const detail =
|
|
285
|
-
typeof body.error === "string"
|
|
286
|
-
? body.error
|
|
287
|
-
: typeof (body as { message?: unknown }).message === "string"
|
|
288
|
-
? (body as { message?: string }).message
|
|
289
|
-
: typeof (body as { error_description?: unknown })
|
|
290
|
-
.error_description === "string"
|
|
291
|
-
? (body as { error_description?: string }).error_description
|
|
292
|
-
: bodyText.slice(0, 500);
|
|
293
|
-
if (
|
|
294
|
-
res.status === 400 &&
|
|
295
|
-
typeof detail === "string" &&
|
|
296
|
-
detail.includes("storage")
|
|
297
|
-
) {
|
|
298
|
-
throw new Error(
|
|
299
|
-
`Meta Files storage limit reached (100 GiB/team). Delete old files via DELETE /v1/files/{id}. Detail: ${detail}`,
|
|
300
|
-
);
|
|
301
|
-
}
|
|
302
|
-
throw new Error(
|
|
303
|
-
`Meta Files upload failed (HTTP ${res.status}): ${detail || res.statusText}`,
|
|
304
|
-
);
|
|
305
|
-
}
|
|
306
|
-
const id =
|
|
307
|
-
typeof body.id === "string"
|
|
308
|
-
? body.id
|
|
309
|
-
: typeof (body as { file_id?: unknown }).file_id === "string"
|
|
310
|
-
? (body as { file_id: string }).file_id
|
|
311
|
-
: undefined;
|
|
312
|
-
if (!id)
|
|
313
|
-
throw new Error(
|
|
314
|
-
`Meta Files upload returned no id: ${bodyText.slice(0, 500)}`,
|
|
315
|
-
);
|
|
316
|
-
return {
|
|
317
|
-
id,
|
|
318
|
-
filename: (typeof body.filename === "string"
|
|
319
|
-
? body.filename
|
|
320
|
-
: basename(filePath)) as string,
|
|
321
|
-
bytes: typeof body.bytes === "number" ? body.bytes : data.byteLength,
|
|
322
|
-
purpose: typeof body.purpose === "string" ? body.purpose : "user_data",
|
|
323
|
-
status: typeof body.status === "string" ? body.status : "uploaded",
|
|
324
|
-
expires_at:
|
|
325
|
-
typeof body.expires_at === "number" ? body.expires_at : undefined,
|
|
326
|
-
};
|
|
327
|
-
}
|
|
328
|
-
|
|
329
|
-
const ALLOWED_PURPOSES = ["user_data", "batch"] as const;
|
|
330
|
-
|
|
331
|
-
async function listMetaFiles(
|
|
332
|
-
apiKey: string,
|
|
333
|
-
purpose = "user_data",
|
|
334
|
-
): Promise<unknown> {
|
|
335
|
-
if (!(ALLOWED_PURPOSES as readonly string[]).includes(purpose)) {
|
|
336
|
-
throw new Error(`Invalid purpose: ${purpose}`);
|
|
337
|
-
}
|
|
338
|
-
const options = { headers: { Authorization: `Bearer ${apiKey}` } };
|
|
339
|
-
const res =
|
|
340
|
-
purpose === "batch"
|
|
341
|
-
? await fetch("https://api.meta.ai/v1/files?purpose=batch", options)
|
|
342
|
-
: await fetch("https://api.meta.ai/v1/files?purpose=user_data", options);
|
|
343
|
-
const text = await res.text();
|
|
344
|
-
if (!res.ok)
|
|
345
|
-
throw new Error(
|
|
346
|
-
`List files failed (HTTP ${res.status}): ${text.slice(0, 1000)}`,
|
|
347
|
-
);
|
|
348
|
-
try {
|
|
349
|
-
return JSON.parse(text);
|
|
350
|
-
} catch {
|
|
351
|
-
return text;
|
|
352
84
|
}
|
|
85
|
+
return key;
|
|
353
86
|
}
|
|
354
87
|
|
|
355
|
-
async function
|
|
356
|
-
apiKey: string,
|
|
357
|
-
fileId: string,
|
|
358
|
-
): Promise<unknown> {
|
|
359
|
-
if (!/^file-[a-zA-Z0-9_-]+$/.test(fileId))
|
|
360
|
-
throw new Error(`Invalid file ID: ${fileId}`);
|
|
361
|
-
let fileUrl: string;
|
|
88
|
+
async function safeGetMetaApiKey(ctx: ExtensionContext): Promise<string> {
|
|
362
89
|
try {
|
|
363
|
-
|
|
90
|
+
return await getMetaApiKey(ctx);
|
|
364
91
|
} catch (error) {
|
|
365
|
-
|
|
366
|
-
|
|
367
|
-
|
|
368
|
-
throw new Error(`Unexpected file delete URL: ${fileUrl}`);
|
|
369
|
-
const res = await fetch(fileUrl, {
|
|
370
|
-
method: "DELETE",
|
|
371
|
-
headers: { Authorization: `Bearer ${apiKey}` },
|
|
372
|
-
});
|
|
373
|
-
const text = await res.text();
|
|
374
|
-
if (!res.ok)
|
|
375
|
-
throw new Error(
|
|
376
|
-
`Delete ${fileId} failed (HTTP ${res.status}): ${text.slice(0, 1000)}`,
|
|
377
|
-
);
|
|
378
|
-
try {
|
|
379
|
-
return JSON.parse(text);
|
|
380
|
-
} catch {
|
|
381
|
-
return text;
|
|
382
|
-
}
|
|
383
|
-
}
|
|
384
|
-
|
|
385
|
-
function responsesUrl(): string {
|
|
386
|
-
return `${META_API_BASE_URL}/responses`;
|
|
387
|
-
}
|
|
388
|
-
|
|
389
|
-
interface ResponsesContentBlock {
|
|
390
|
-
type: string;
|
|
391
|
-
[key: string]: unknown;
|
|
392
|
-
}
|
|
393
|
-
|
|
394
|
-
function record(value: unknown): Record<string, unknown> | undefined {
|
|
395
|
-
return value !== null && typeof value === "object"
|
|
396
|
-
? (value as Record<string, unknown>)
|
|
397
|
-
: undefined;
|
|
398
|
-
}
|
|
399
|
-
|
|
400
|
-
function tokenCount(...values: unknown[]): number | undefined {
|
|
401
|
-
for (const value of values) {
|
|
402
|
-
const numeric =
|
|
403
|
-
typeof value === "number"
|
|
404
|
-
? value
|
|
405
|
-
: typeof value === "string" && value.trim()
|
|
406
|
-
? Number(value)
|
|
407
|
-
: Number.NaN;
|
|
408
|
-
if (Number.isFinite(numeric)) return Math.max(0, Math.trunc(numeric));
|
|
409
|
-
}
|
|
410
|
-
return undefined;
|
|
411
|
-
}
|
|
412
|
-
|
|
413
|
-
/** Convert a Meta Responses API usage block into Pi's nested-tool usage shape. */
|
|
414
|
-
export function extractMetaResponseUsage(
|
|
415
|
-
response: unknown,
|
|
416
|
-
modelId = "muse-spark-1.2",
|
|
417
|
-
): Usage | undefined {
|
|
418
|
-
const usage = record(record(response)?.usage);
|
|
419
|
-
if (!usage) return undefined;
|
|
420
|
-
|
|
421
|
-
const inputDetails = record(
|
|
422
|
-
usage.input_tokens_details ?? usage.inputTokensDetails,
|
|
423
|
-
);
|
|
424
|
-
const outputDetails = record(
|
|
425
|
-
usage.output_tokens_details ?? usage.outputTokensDetails,
|
|
426
|
-
);
|
|
427
|
-
const inputTotal = tokenCount(
|
|
428
|
-
usage.input_tokens,
|
|
429
|
-
usage.inputTokens,
|
|
430
|
-
usage.prompt_tokens,
|
|
431
|
-
usage.promptTokens,
|
|
432
|
-
usage.input,
|
|
433
|
-
);
|
|
434
|
-
const output = tokenCount(
|
|
435
|
-
usage.output_tokens,
|
|
436
|
-
usage.outputTokens,
|
|
437
|
-
usage.completion_tokens,
|
|
438
|
-
usage.completionTokens,
|
|
439
|
-
usage.output,
|
|
440
|
-
);
|
|
441
|
-
if (inputTotal === undefined && output === undefined) return undefined;
|
|
442
|
-
|
|
443
|
-
const reportedInput = inputTotal ?? 0;
|
|
444
|
-
const cacheRead = Math.min(
|
|
445
|
-
reportedInput,
|
|
446
|
-
tokenCount(inputDetails?.cached_tokens, inputDetails?.cachedTokens) ?? 0,
|
|
447
|
-
);
|
|
448
|
-
const cacheWrite = Math.min(
|
|
449
|
-
reportedInput - cacheRead,
|
|
450
|
-
tokenCount(
|
|
451
|
-
inputDetails?.cache_write_tokens,
|
|
452
|
-
inputDetails?.cacheWriteTokens,
|
|
453
|
-
) ?? 0,
|
|
454
|
-
);
|
|
455
|
-
const uncachedInput = Math.max(0, reportedInput - cacheRead - cacheWrite);
|
|
456
|
-
const outputTokens = output ?? 0;
|
|
457
|
-
const reasoning = Math.min(
|
|
458
|
-
outputTokens,
|
|
459
|
-
tokenCount(
|
|
460
|
-
outputDetails?.reasoning_tokens,
|
|
461
|
-
outputDetails?.reasoningTokens,
|
|
462
|
-
) ?? 0,
|
|
463
|
-
);
|
|
464
|
-
const totalTokens =
|
|
465
|
-
tokenCount(usage.total_tokens, usage.totalTokens) ??
|
|
466
|
-
uncachedInput + cacheRead + cacheWrite + outputTokens;
|
|
467
|
-
const model = createMetaProviderConfig().models?.find(
|
|
468
|
-
(candidate) => candidate.id === modelId,
|
|
469
|
-
);
|
|
470
|
-
const rates = model?.cost;
|
|
471
|
-
const cost = {
|
|
472
|
-
input: (uncachedInput * (rates?.input ?? 0)) / 1_000_000,
|
|
473
|
-
output: (outputTokens * (rates?.output ?? 0)) / 1_000_000,
|
|
474
|
-
cacheRead: (cacheRead * (rates?.cacheRead ?? 0)) / 1_000_000,
|
|
475
|
-
cacheWrite: (cacheWrite * (rates?.cacheWrite ?? 0)) / 1_000_000,
|
|
476
|
-
total: 0,
|
|
477
|
-
};
|
|
478
|
-
cost.total = cost.input + cost.output + cost.cacheRead + cost.cacheWrite;
|
|
479
|
-
|
|
480
|
-
return {
|
|
481
|
-
input: uncachedInput,
|
|
482
|
-
output: outputTokens,
|
|
483
|
-
cacheRead,
|
|
484
|
-
cacheWrite,
|
|
485
|
-
reasoning,
|
|
486
|
-
totalTokens,
|
|
487
|
-
cost,
|
|
488
|
-
};
|
|
489
|
-
}
|
|
490
|
-
|
|
491
|
-
export function formatMetaResponseUsage(usage: Usage): string {
|
|
492
|
-
const parts = [`${usage.input.toLocaleString()} input`];
|
|
493
|
-
if (usage.cacheRead)
|
|
494
|
-
parts.push(`${usage.cacheRead.toLocaleString()} cache read`);
|
|
495
|
-
if (usage.cacheWrite)
|
|
496
|
-
parts.push(`${usage.cacheWrite.toLocaleString()} cache write`);
|
|
497
|
-
parts.push(`${usage.output.toLocaleString()} output`);
|
|
498
|
-
if (usage.reasoning)
|
|
499
|
-
parts.push(`${usage.reasoning.toLocaleString()} reasoning`);
|
|
500
|
-
return `Video token usage: ${usage.totalTokens.toLocaleString()} total (${parts.join(", ")})`;
|
|
501
|
-
}
|
|
502
|
-
|
|
503
|
-
// Helper to extract output_text from Responses API response (covers streamed and non-streamed shapes)
|
|
504
|
-
export function extractResponsesText(json: unknown): string {
|
|
505
|
-
if (!json || typeof json !== "object") return "";
|
|
506
|
-
const j = json as Record<string, unknown>;
|
|
507
|
-
// Some Meta responses include an empty output_text alongside populated output blocks.
|
|
508
|
-
if (typeof j.output_text === "string" && j.output_text.trim())
|
|
509
|
-
return j.output_text;
|
|
510
|
-
const output = j.output as unknown;
|
|
511
|
-
if (Array.isArray(output)) {
|
|
512
|
-
const texts: string[] = [];
|
|
513
|
-
for (const item of output as Record<string, unknown>[]) {
|
|
514
|
-
if (!item || typeof item !== "object" || item.type !== "message")
|
|
515
|
-
continue;
|
|
516
|
-
if (typeof item.content === "string" && item.content.trim())
|
|
517
|
-
texts.push(item.content);
|
|
518
|
-
if (Array.isArray(item.content)) {
|
|
519
|
-
for (const c of item.content as Record<string, unknown>[]) {
|
|
520
|
-
if (
|
|
521
|
-
(c.type === "output_text" || c.type === "text") &&
|
|
522
|
-
typeof c.text === "string" &&
|
|
523
|
-
c.text.trim()
|
|
524
|
-
)
|
|
525
|
-
texts.push(c.text);
|
|
526
|
-
if (
|
|
527
|
-
c.type === "refusal" &&
|
|
528
|
-
typeof c.refusal === "string" &&
|
|
529
|
-
c.refusal.trim()
|
|
530
|
-
)
|
|
531
|
-
texts.push(c.refusal);
|
|
532
|
-
}
|
|
533
|
-
}
|
|
534
|
-
}
|
|
535
|
-
if (texts.length) return texts.join("\n\n");
|
|
536
|
-
}
|
|
537
|
-
if (typeof j.text === "string" && j.text.trim()) return j.text;
|
|
538
|
-
return JSON.stringify(json, null, 2).slice(0, 8000);
|
|
539
|
-
}
|
|
540
|
-
|
|
541
|
-
async function callMetaResponses(
|
|
542
|
-
apiKey: string,
|
|
543
|
-
payload: Record<string, unknown>,
|
|
544
|
-
signal?: AbortSignal,
|
|
545
|
-
): Promise<{ text: string; raw: unknown }> {
|
|
546
|
-
// Force store:false for media payloads to avoid the ~20 MB
|
|
547
|
-
// persistence limit (413 payload_too_large with store=true). Files API
|
|
548
|
-
// is the correct path for large media per /docs/file-handling.
|
|
549
|
-
payload.store = false;
|
|
550
|
-
const doFetch = async (body: Record<string, unknown>) =>
|
|
551
|
-
await fetch(responsesUrl(), {
|
|
552
|
-
method: "POST",
|
|
553
|
-
headers: {
|
|
554
|
-
Authorization: `Bearer ${apiKey}`,
|
|
555
|
-
"Content-Type": "application/json",
|
|
556
|
-
"x-api-version": "1.0.0",
|
|
557
|
-
},
|
|
558
|
-
body: JSON.stringify(body),
|
|
559
|
-
signal,
|
|
560
|
-
});
|
|
561
|
-
let res = await doFetch(payload);
|
|
562
|
-
let text = await res.text();
|
|
563
|
-
let json: unknown;
|
|
564
|
-
try {
|
|
565
|
-
json = text ? (JSON.parse(text) as unknown) : {};
|
|
566
|
-
} catch {
|
|
567
|
-
json = text;
|
|
568
|
-
}
|
|
569
|
-
if (!res.ok) {
|
|
570
|
-
const detail =
|
|
571
|
-
(json &&
|
|
572
|
-
typeof json === "object" &&
|
|
573
|
-
"error" in (json as Record<string, unknown>)
|
|
574
|
-
? JSON.stringify((json as Record<string, unknown>).error).slice(0, 1000)
|
|
575
|
-
: typeof json === "object" &&
|
|
576
|
-
json !== null &&
|
|
577
|
-
"message" in (json as Record<string, unknown>)
|
|
578
|
-
? String((json as Record<string, unknown>).message).slice(0, 1000)
|
|
579
|
-
: text.slice(0, 1000)) || res.statusText;
|
|
580
|
-
// Auto-retry once with store:false if the error is the ~20 MB
|
|
581
|
-
// persistence limit (413 with store=true) and we haven't already set it.
|
|
582
|
-
const isStoreLimit =
|
|
583
|
-
res.status === 413 &&
|
|
584
|
-
String(detail).includes("store=true") &&
|
|
585
|
-
payload.store !== false;
|
|
586
|
-
if (isStoreLimit) {
|
|
587
|
-
res = await doFetch({ ...payload, store: false });
|
|
588
|
-
text = await res.text();
|
|
589
|
-
try {
|
|
590
|
-
json = text ? (JSON.parse(text) as unknown) : {};
|
|
591
|
-
} catch {
|
|
592
|
-
json = text;
|
|
593
|
-
}
|
|
594
|
-
if (res.ok) return { text: extractResponsesText(json), raw: json };
|
|
595
|
-
}
|
|
596
|
-
throw new Error(`Meta Responses failed (HTTP ${res.status}): ${detail}`);
|
|
92
|
+
if (String(error).includes("stale after session replacement"))
|
|
93
|
+
throw new Error("Session was replaced — please retry the command");
|
|
94
|
+
throw error;
|
|
597
95
|
}
|
|
598
|
-
return { text: extractResponsesText(json), raw: json };
|
|
599
96
|
}
|
|
600
97
|
|
|
601
98
|
function dataUrlForFile(path: string): {
|
|
@@ -613,317 +110,88 @@ function dataUrlForFile(path: string): {
|
|
|
613
110
|
};
|
|
614
111
|
}
|
|
615
112
|
|
|
616
|
-
|
|
617
|
-
// Before-provider-request rewrite (Option B)
|
|
618
|
-
// Makes @video.mp4 / @audio.wav / @doc.pdf work transparently even though
|
|
619
|
-
// pi-ai's UserMessage is still string | (TextContent | ImageContent)[].
|
|
620
|
-
// We smuggle video/audio/pdf as ImageContent (data:video/..., data:audio/..., data:application/pdf...)
|
|
621
|
-
// via the input handler or via read tool results; this hook rewrites the
|
|
622
|
-
// resulting {type:"input_image", image_url:"data:video/..."} into the typed
|
|
623
|
-
// input_video/input_audio/input_file block required by Meta before the request is sent.
|
|
624
|
-
// Also handles https://... media URLs and large inline payloads via Files API.
|
|
625
|
-
// Enable/disable via PI_META_NATIVE_ATTACHMENTS env (default: on).
|
|
626
|
-
// ---------------------------------------------------------------------------
|
|
627
|
-
|
|
628
|
-
function shouldEnableNativeRewrite(): boolean {
|
|
629
|
-
const v =
|
|
630
|
-
process.env.PI_META_NATIVE_ATTACHMENTS ?? process.env.PI_META_MEDIA_NATIVE;
|
|
631
|
-
if (v === undefined) return true; // default on — set to "0"/"false" to disable
|
|
632
|
-
return v !== "0" && v.toLowerCase() !== "false" && v.toLowerCase() !== "off";
|
|
633
|
-
}
|
|
634
|
-
|
|
635
|
-
function isInputImageBlock(b: Record<string, unknown>): boolean {
|
|
636
|
-
return b.type === "input_image" && typeof b.image_url === "string";
|
|
637
|
-
}
|
|
638
|
-
|
|
639
|
-
function rewriteBlock(
|
|
640
|
-
block: Record<string, unknown>,
|
|
641
|
-
): Record<string, unknown> | null {
|
|
642
|
-
if (!isInputImageBlock(block)) return null;
|
|
643
|
-
const url = block.image_url as string;
|
|
644
|
-
const mime = mimeForDataUrl(url);
|
|
645
|
-
// Also handle https URLs ending with .mp4/.wav/.mp3/.pdf
|
|
646
|
-
const lowerUrl = url.toLowerCase();
|
|
647
|
-
const isHttpsVideo =
|
|
648
|
-
url.startsWith("https://") &&
|
|
649
|
-
(lowerUrl.endsWith(".mp4") || lowerUrl.endsWith(".m4v"));
|
|
650
|
-
const isHttpsAudio =
|
|
651
|
-
url.startsWith("https://") &&
|
|
652
|
-
(lowerUrl.endsWith(".mp3") || lowerUrl.endsWith(".wav"));
|
|
653
|
-
const isHttpsPdf = url.startsWith("https://") && lowerUrl.endsWith(".pdf");
|
|
654
|
-
const isHttpsImage =
|
|
655
|
-
url.startsWith("https://") &&
|
|
656
|
-
(lowerUrl.endsWith(".png") ||
|
|
657
|
-
lowerUrl.endsWith(".jpg") ||
|
|
658
|
-
lowerUrl.endsWith(".jpeg") ||
|
|
659
|
-
lowerUrl.endsWith(".gif") ||
|
|
660
|
-
lowerUrl.endsWith(".webp"));
|
|
113
|
+
type AnalysisSource = { source: string; label?: string };
|
|
661
114
|
|
|
662
|
-
|
|
115
|
+
const automaticExpiry = () => ({
|
|
116
|
+
anchor: "created_at" as const,
|
|
117
|
+
seconds: AUTOMATIC_UPLOAD_EXPIRY_SECONDS,
|
|
118
|
+
});
|
|
663
119
|
|
|
664
|
-
|
|
665
|
-
|
|
666
|
-
|
|
667
|
-
|
|
668
|
-
>;
|
|
669
|
-
}
|
|
670
|
-
if (isAudioMime(mime) || isHttpsAudio) {
|
|
671
|
-
const effectiveMime =
|
|
672
|
-
typeof mime === "string"
|
|
673
|
-
? mime
|
|
674
|
-
: lowerUrl.endsWith(".wav")
|
|
675
|
-
? "audio/wav"
|
|
676
|
-
: "audio/mpeg";
|
|
677
|
-
return mediaInputFromSource(
|
|
678
|
-
url,
|
|
679
|
-
filenameForMime(effectiveMime, "audio.bin"),
|
|
680
|
-
effectiveMime,
|
|
681
|
-
) as Record<string, unknown>;
|
|
682
|
-
}
|
|
683
|
-
if (isPdfMime(mime) || isHttpsPdf) {
|
|
684
|
-
return mediaInputFromSource(
|
|
685
|
-
url,
|
|
686
|
-
"document.pdf",
|
|
687
|
-
"application/pdf",
|
|
688
|
-
) as Record<string, unknown>;
|
|
689
|
-
}
|
|
690
|
-
return null;
|
|
120
|
+
function mediaToolError(tool: string, error: unknown): never {
|
|
121
|
+
if (error instanceof Error && error.name === "AbortError") throw error;
|
|
122
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
123
|
+
throw new Error(`${tool} failed: ${message}`, { cause: error });
|
|
691
124
|
}
|
|
692
125
|
|
|
693
|
-
|
|
694
|
-
|
|
695
|
-
rewritten: boolean;
|
|
696
|
-
payload: unknown;
|
|
697
|
-
} {
|
|
698
|
-
if (!payload || typeof payload !== "object")
|
|
699
|
-
return { rewritten: false, payload };
|
|
700
|
-
const p = payload as Record<string, unknown>;
|
|
701
|
-
const input = p.input;
|
|
702
|
-
if (!Array.isArray(input)) return { rewritten: false, payload };
|
|
703
|
-
let rewritten = false;
|
|
704
|
-
const newInput = input.map((entry) => {
|
|
705
|
-
if (!entry || typeof entry !== "object") return entry;
|
|
706
|
-
const e = entry as Record<string, unknown>;
|
|
707
|
-
// case 1: user message {role:"user", content:[...]}
|
|
708
|
-
if (Array.isArray(e.content)) {
|
|
709
|
-
const newContent = (e.content as Record<string, unknown>[]).map(
|
|
710
|
-
(block) => {
|
|
711
|
-
const r = rewriteBlock(block as Record<string, unknown>);
|
|
712
|
-
if (r) {
|
|
713
|
-
rewritten = true;
|
|
714
|
-
return r;
|
|
715
|
-
}
|
|
716
|
-
return block;
|
|
717
|
-
},
|
|
718
|
-
);
|
|
719
|
-
if (newContent !== e.content) return { ...e, content: newContent };
|
|
720
|
-
}
|
|
721
|
-
// case 2: function_call_output {type:"function_call_output", output:[...]} where output contains input_image blocks
|
|
722
|
-
if (e.type === "function_call_output" && Array.isArray(e.output)) {
|
|
723
|
-
const newOutput = (e.output as Record<string, unknown>[]).map((block) => {
|
|
724
|
-
// output blocks are like {type:"input_text", text} or {type:"input_image", image_url}
|
|
725
|
-
if (
|
|
726
|
-
block.type === "input_image" &&
|
|
727
|
-
typeof block.image_url === "string"
|
|
728
|
-
) {
|
|
729
|
-
const r = rewriteBlock(block as Record<string, unknown>);
|
|
730
|
-
if (r) {
|
|
731
|
-
rewritten = true;
|
|
732
|
-
// tool outputs use same types as user content; return rewritten block
|
|
733
|
-
return r;
|
|
734
|
-
}
|
|
735
|
-
}
|
|
736
|
-
return block;
|
|
737
|
-
});
|
|
738
|
-
if (newOutput !== e.output) return { ...e, output: newOutput };
|
|
739
|
-
}
|
|
740
|
-
// case 3: custom_tool_call_output similar
|
|
741
|
-
if (e.type === "custom_tool_call_output" && Array.isArray(e.output)) {
|
|
742
|
-
const newOutput = (e.output as Record<string, unknown>[]).map((block) => {
|
|
743
|
-
if (
|
|
744
|
-
block.type === "input_image" &&
|
|
745
|
-
typeof block.image_url === "string"
|
|
746
|
-
) {
|
|
747
|
-
const r = rewriteBlock(block as Record<string, unknown>);
|
|
748
|
-
if (r) {
|
|
749
|
-
rewritten = true;
|
|
750
|
-
return r;
|
|
751
|
-
}
|
|
752
|
-
}
|
|
753
|
-
return block;
|
|
754
|
-
});
|
|
755
|
-
if (newOutput !== e.output) return { ...e, output: newOutput };
|
|
756
|
-
}
|
|
757
|
-
return e;
|
|
758
|
-
});
|
|
759
|
-
if (!rewritten) return { rewritten: false, payload };
|
|
760
|
-
return { rewritten: true, payload: { ...p, input: newInput } };
|
|
126
|
+
function sourceLabel(source: AnalysisSource, index: number): string {
|
|
127
|
+
return source.label?.trim() || `Source ${index + 1}`;
|
|
761
128
|
}
|
|
762
129
|
|
|
763
|
-
async function
|
|
130
|
+
async function resolveGenericMediaSource(
|
|
764
131
|
apiKey: string,
|
|
765
|
-
|
|
766
|
-
|
|
767
|
-
|
|
768
|
-
): Promise<
|
|
769
|
-
|
|
770
|
-
|
|
771
|
-
form.append("file", blob, filename);
|
|
772
|
-
form.append("purpose", "user_data");
|
|
773
|
-
const res = await fetch(META_FILES_URL, {
|
|
774
|
-
method: "POST",
|
|
775
|
-
headers: { Authorization: `Bearer ${apiKey}` },
|
|
776
|
-
body: form,
|
|
777
|
-
});
|
|
778
|
-
const text = await res.text();
|
|
779
|
-
if (!res.ok) {
|
|
780
|
-
throw new Error(
|
|
781
|
-
`Files upload for large inline media failed: ${text.slice(0, 500)}`,
|
|
782
|
-
);
|
|
132
|
+
cwd: string,
|
|
133
|
+
source: string,
|
|
134
|
+
signal?: AbortSignal,
|
|
135
|
+
): Promise<ResponsesContentBlock> {
|
|
136
|
+
if (/^file-[a-zA-Z0-9_-]+$/.test(source)) {
|
|
137
|
+
return { type: "input_file", file_id: source };
|
|
783
138
|
}
|
|
784
|
-
|
|
785
|
-
|
|
786
|
-
|
|
787
|
-
|
|
788
|
-
|
|
139
|
+
if (isRemoteMediaSource(source)) {
|
|
140
|
+
return mediaInputFromSource(source);
|
|
141
|
+
}
|
|
142
|
+
const absolute = resolve(cwd, source.replace(/^@/, ""));
|
|
143
|
+
if (!existsSync(absolute)) throw new Error(`File not found: ${absolute}`);
|
|
144
|
+
const stat = statSync(absolute);
|
|
145
|
+
if (stat.size > FILES_API_LIMIT_BYTES) {
|
|
146
|
+
throw new Error(`File too large (1 GiB limit): ${source}`);
|
|
789
147
|
}
|
|
790
|
-
if (
|
|
791
|
-
|
|
148
|
+
if (stat.size > STORE_SAFE_INLINE_BYTES) {
|
|
149
|
+
const upload = await uploadMetaFile(
|
|
150
|
+
apiKey,
|
|
151
|
+
absolute,
|
|
152
|
+
automaticExpiry(),
|
|
153
|
+
signal,
|
|
154
|
+
);
|
|
155
|
+
return { type: "input_file", file_id: upload.id };
|
|
792
156
|
}
|
|
793
|
-
|
|
157
|
+
const { dataUrl, mime } = dataUrlForFile(absolute);
|
|
158
|
+
return mediaInputFromSource(dataUrl, basename(absolute), mime);
|
|
794
159
|
}
|
|
795
160
|
|
|
796
|
-
|
|
797
|
-
|
|
798
|
-
|
|
799
|
-
|
|
800
|
-
|
|
801
|
-
|
|
802
|
-
): Promise<{ payload: unknown; uploaded: number }> {
|
|
803
|
-
if (!apiKey) return { payload, uploaded: 0 };
|
|
804
|
-
if (!payload || typeof payload !== "object") return { payload, uploaded: 0 };
|
|
805
|
-
const p = payload as Record<string, unknown>;
|
|
806
|
-
const input = p.input;
|
|
807
|
-
if (!Array.isArray(input)) return { payload, uploaded: 0 };
|
|
808
|
-
let uploaded = 0;
|
|
809
|
-
let needsStoreFalse = false;
|
|
810
|
-
const newInput: unknown[] = [];
|
|
811
|
-
|
|
812
|
-
for (const entry of input as Record<string, unknown>[]) {
|
|
813
|
-
if (!entry || typeof entry !== "object" || !Array.isArray(entry.content)) {
|
|
814
|
-
newInput.push(entry);
|
|
815
|
-
continue;
|
|
816
|
-
}
|
|
817
|
-
const content = entry.content as Record<string, unknown>[];
|
|
818
|
-
const newContent: Record<string, unknown>[] = [];
|
|
819
|
-
for (const block of content) {
|
|
820
|
-
let inline:
|
|
821
|
-
| { base64: string; mime: string; filename: string }
|
|
822
|
-
| undefined;
|
|
823
|
-
if (
|
|
824
|
-
block.type === "input_video" &&
|
|
825
|
-
typeof block.video_url === "string" &&
|
|
826
|
-
block.video_url.startsWith("data:")
|
|
827
|
-
) {
|
|
828
|
-
inline = {
|
|
829
|
-
base64: base64FromDataUrl(block.video_url),
|
|
830
|
-
mime: mimeForDataUrl(block.video_url) ?? "video/mp4",
|
|
831
|
-
filename: "video.mp4",
|
|
832
|
-
};
|
|
833
|
-
} else if (block.type === "input_audio") {
|
|
834
|
-
const audio = record(block.input_audio);
|
|
835
|
-
if (typeof audio?.data === "string") {
|
|
836
|
-
const format = audio.format === "wav" ? "wav" : "mp3";
|
|
837
|
-
inline = {
|
|
838
|
-
base64: audio.data.startsWith("data:")
|
|
839
|
-
? base64FromDataUrl(audio.data)
|
|
840
|
-
: audio.data,
|
|
841
|
-
mime: format === "wav" ? "audio/wav" : "audio/mpeg",
|
|
842
|
-
filename: `audio.${format}`,
|
|
843
|
-
};
|
|
844
|
-
}
|
|
845
|
-
} else if (
|
|
846
|
-
block.type === "input_file" &&
|
|
847
|
-
typeof block.file_data === "string" &&
|
|
848
|
-
block.file_data.startsWith("data:")
|
|
849
|
-
) {
|
|
850
|
-
const mime =
|
|
851
|
-
mimeForDataUrl(block.file_data) ?? "application/octet-stream";
|
|
852
|
-
inline = {
|
|
853
|
-
base64: base64FromDataUrl(block.file_data),
|
|
854
|
-
mime,
|
|
855
|
-
filename:
|
|
856
|
-
typeof block.filename === "string" && block.filename
|
|
857
|
-
? block.filename
|
|
858
|
-
: filenameForMime(mime),
|
|
859
|
-
};
|
|
860
|
-
}
|
|
161
|
+
function shouldEnableNativeRewrite(): boolean {
|
|
162
|
+
const v =
|
|
163
|
+
process.env.PI_META_NATIVE_ATTACHMENTS ?? process.env.PI_META_MEDIA_NATIVE;
|
|
164
|
+
if (v === undefined) return true;
|
|
165
|
+
return v !== "0" && v.toLowerCase() !== "false" && v.toLowerCase() !== "off";
|
|
166
|
+
}
|
|
861
167
|
|
|
862
|
-
|
|
863
|
-
|
|
864
|
-
|
|
865
|
-
|
|
866
|
-
|
|
867
|
-
|
|
868
|
-
|
|
869
|
-
|
|
870
|
-
);
|
|
871
|
-
uploaded++;
|
|
872
|
-
newContent.push({ type: "input_file", file_id: id });
|
|
873
|
-
continue;
|
|
874
|
-
}
|
|
875
|
-
}
|
|
876
|
-
newContent.push(block);
|
|
877
|
-
}
|
|
878
|
-
newInput.push({ ...entry, content: newContent });
|
|
168
|
+
function rejectWrongMime(
|
|
169
|
+
kind: "video" | "audio",
|
|
170
|
+
source: string,
|
|
171
|
+
mime: string | undefined,
|
|
172
|
+
): void {
|
|
173
|
+
if (!mime) return;
|
|
174
|
+
if (kind === "video" && !isVideoMime(mime)) {
|
|
175
|
+
throw new Error(`Unsupported video type for ${source}; use MP4`);
|
|
879
176
|
}
|
|
880
|
-
|
|
881
|
-
|
|
882
|
-
return needsStoreFalse
|
|
883
|
-
? { payload: { ...p, store: false }, uploaded: 0 }
|
|
884
|
-
: { payload, uploaded: 0 };
|
|
177
|
+
if (kind === "audio" && !isAudioMime(mime)) {
|
|
178
|
+
throw new Error(`Unsupported audio type for ${source}; use MP3 or WAV`);
|
|
885
179
|
}
|
|
886
|
-
return { payload: { ...p, input: newInput, store: false }, uploaded };
|
|
887
180
|
}
|
|
888
181
|
|
|
889
|
-
|
|
890
|
-
|
|
891
|
-
|
|
182
|
+
function isFileId(source: string): boolean {
|
|
183
|
+
return source.startsWith("file-");
|
|
184
|
+
}
|
|
892
185
|
|
|
893
186
|
export default function metaMedia(pi: ExtensionAPI): void {
|
|
894
|
-
// --- Before provider request rewrite (Option B transparent path) ---
|
|
895
187
|
if (shouldEnableNativeRewrite()) {
|
|
896
188
|
pi.on("before_provider_request", async (event, ctx) => {
|
|
897
189
|
if (ctx.model?.provider !== META_PROVIDER_ID) return undefined;
|
|
898
190
|
const original = event.payload as Record<string, unknown>;
|
|
899
191
|
const { rewritten, payload: rewrittenPayload } =
|
|
900
192
|
rewriteResponsesPayload(original);
|
|
901
|
-
// Even without rewrite, large inline media in the original payload
|
|
902
|
-
// (e.g. drag-dropped video) can hit the 413 store=true limit —
|
|
903
|
-
// ensure store:false and try Files API upload.
|
|
904
193
|
const payloadToCheck = rewritten ? rewrittenPayload : original;
|
|
905
|
-
|
|
906
|
-
if (!payloadToCheck || typeof payloadToCheck !== "object") return false;
|
|
907
|
-
const input = (payloadToCheck as Record<string, unknown>).input;
|
|
908
|
-
if (!Array.isArray(input)) return false;
|
|
909
|
-
for (const e of input as Record<string, unknown>[]) {
|
|
910
|
-
if (!e || typeof e !== "object" || !Array.isArray(e.content))
|
|
911
|
-
continue;
|
|
912
|
-
for (const b of e.content as Record<string, unknown>[]) {
|
|
913
|
-
if (
|
|
914
|
-
b.type === "input_file" ||
|
|
915
|
-
b.type === "input_image" ||
|
|
916
|
-
b.type === "input_video" ||
|
|
917
|
-
b.type === "input_audio"
|
|
918
|
-
)
|
|
919
|
-
return true;
|
|
920
|
-
}
|
|
921
|
-
}
|
|
922
|
-
return false;
|
|
923
|
-
})();
|
|
924
|
-
if (!rewritten && !hasMedia) return undefined;
|
|
925
|
-
// Resolve the same stored Meta credential used by the active provider. This is
|
|
926
|
-
// essential for @file drag-drop, where the key usually is not in process.env.
|
|
194
|
+
if (!rewritten && !payloadContainsMedia(payloadToCheck)) return undefined;
|
|
927
195
|
let apiKey =
|
|
928
196
|
process.env.MODEL_API_KEY ?? process.env.META_API_KEY ?? undefined;
|
|
929
197
|
if (!apiKey) {
|
|
@@ -938,15 +206,18 @@ export default function metaMedia(pi: ExtensionAPI): void {
|
|
|
938
206
|
if (apiKey) {
|
|
939
207
|
try {
|
|
940
208
|
const { payload: uploadedPayload } =
|
|
941
|
-
await maybeUploadLargeInlineBlocks(
|
|
209
|
+
await maybeUploadLargeInlineBlocks(
|
|
210
|
+
payloadToCheck,
|
|
211
|
+
apiKey,
|
|
212
|
+
STORE_SAFE_INLINE_BYTES,
|
|
213
|
+
ctx.signal,
|
|
214
|
+
);
|
|
942
215
|
return uploadedPayload as unknown;
|
|
943
216
|
} catch {
|
|
944
|
-
// fall back without upload — at least force store:false
|
|
945
217
|
const fallback = payloadToCheck as Record<string, unknown>;
|
|
946
218
|
return { ...fallback, store: false } as unknown;
|
|
947
219
|
}
|
|
948
220
|
}
|
|
949
|
-
// No apiKey available for upload — force store:false to avoid 413
|
|
950
221
|
return {
|
|
951
222
|
...(payloadToCheck as Record<string, unknown>),
|
|
952
223
|
store: false,
|
|
@@ -954,19 +225,10 @@ export default function metaMedia(pi: ExtensionAPI): void {
|
|
|
954
225
|
});
|
|
955
226
|
}
|
|
956
227
|
|
|
957
|
-
// --- Input handler: let @video.mp4 / @audio.wav / @doc.pdf be pasted like @image.png ---
|
|
958
|
-
// Pi's TUI already turns @image.png into ImageContent; for other types event.text still
|
|
959
|
-
// contains the literal path and images is empty. We turn those paths into pseudo-ImageContent
|
|
960
|
-
// so the before_provider_request hook can rewrite them.
|
|
961
228
|
pi.on("input", async (event, ctx) => {
|
|
962
229
|
if (!shouldEnableNativeRewrite()) return undefined;
|
|
963
|
-
// Only Meta's request hook knows how to rewrite these pseudo-images into
|
|
964
|
-
// typed video/audio/file blocks. Other providers (notably Codex) reject a
|
|
965
|
-
// video/mp4 ImageContent before the Meta tools can be called, so leave the
|
|
966
|
-
// path as text and let the active model invoke the direct Meta media tool.
|
|
967
230
|
if (ctx.model?.provider !== META_PROVIDER_ID) return undefined;
|
|
968
231
|
const text = event.text ?? "";
|
|
969
|
-
// Match @/path/to/file.mp4, @./file.wav, /abs/file.pdf, ./file.mp3 — conservative.
|
|
970
232
|
const filePattern =
|
|
971
233
|
/(?:^|\s)@?((?:[~.]?\/[^\s]+\.(?:mp4|m4v|mp3|wav|pdf))|(?:[^\s]+\.(?:mp4|m4v|mp3|wav|pdf)))/gi;
|
|
972
234
|
const matches = [...text.matchAll(filePattern)];
|
|
@@ -980,7 +242,6 @@ export default function metaMedia(pi: ExtensionAPI): void {
|
|
|
980
242
|
for (const m of matches) {
|
|
981
243
|
const rawPath = m[1];
|
|
982
244
|
if (!rawPath) continue;
|
|
983
|
-
// Resolve relative to cwd, expand ~, strip @
|
|
984
245
|
const clean = rawPath.replace(/^@/, "");
|
|
985
246
|
const expanded = clean.startsWith("~/")
|
|
986
247
|
? clean.replace("~", process.env.HOME ?? "")
|
|
@@ -998,10 +259,6 @@ export default function metaMedia(pi: ExtensionAPI): void {
|
|
|
998
259
|
);
|
|
999
260
|
continue;
|
|
1000
261
|
}
|
|
1001
|
-
// Files API is required for reliability — inline base64 hits the
|
|
1002
|
-
// ~20 MB `store=true` persistence limit (24 MB → 413). We keep inline
|
|
1003
|
-
// only for small files (<15 MB); larger files will be uploaded via
|
|
1004
|
-
// Files API in before_provider_request or via /meta-video.
|
|
1005
262
|
if (stat.size > STORE_SAFE_INLINE_BYTES) {
|
|
1006
263
|
if (stat.size > INLINE_LIMIT_BYTES) {
|
|
1007
264
|
safeNotify(
|
|
@@ -1011,16 +268,11 @@ export default function metaMedia(pi: ExtensionAPI): void {
|
|
|
1011
268
|
);
|
|
1012
269
|
continue;
|
|
1013
270
|
}
|
|
1014
|
-
// 15-50 MB: still smuggle inline but before_provider_request
|
|
1015
|
-
// will promote to Files API (if apiKey available) and set store:false
|
|
1016
271
|
}
|
|
1017
272
|
try {
|
|
1018
273
|
const { dataUrl } = dataUrlForFile(absolute);
|
|
1019
274
|
const b64 = base64FromDataUrl(dataUrl);
|
|
1020
|
-
// Smuggle as ImageContent with video/audio/pdf mime — before_provider_request will rewrite.
|
|
1021
|
-
// pi-ai types allow any mimeType string, so video/mp4 is fine at runtime even though Model.input is cast.
|
|
1022
275
|
newImages.push({ type: "image", data: b64, mimeType: mime });
|
|
1023
|
-
// Remove the path from text so LLM doesn't see a stray filename; keep a placeholder for UX
|
|
1024
276
|
newText = newText.replace(rawPath, `[attached ${basename(absolute)}]`);
|
|
1025
277
|
touched = true;
|
|
1026
278
|
} catch (e) {
|
|
@@ -1039,144 +291,84 @@ export default function metaMedia(pi: ExtensionAPI): void {
|
|
|
1039
291
|
};
|
|
1040
292
|
});
|
|
1041
293
|
|
|
1042
|
-
// -----------------------------------------------------------------------
|
|
1043
|
-
// Tools — the reliable path that works even when UserMessage types are limited.
|
|
1044
|
-
// These call Meta's Responses API directly with the stored OAuth key, so they
|
|
1045
|
-
// don't depend on pi-ai's ImageContent union. Use when @file paste isn't enough
|
|
1046
|
-
// or for large/reused files.
|
|
1047
|
-
// -----------------------------------------------------------------------
|
|
1048
|
-
|
|
1049
294
|
pi.registerTool({
|
|
1050
295
|
name: "meta_describe_video",
|
|
1051
296
|
label: "Meta Video Describe",
|
|
1052
297
|
description:
|
|
1053
|
-
"Analyze an MP4 video with Muse Spark
|
|
1054
|
-
|
|
1055
|
-
|
|
1056
|
-
|
|
1057
|
-
|
|
1058
|
-
|
|
1059
|
-
|
|
1060
|
-
|
|
1061
|
-
|
|
1062
|
-
Type.String({
|
|
1063
|
-
|
|
1064
|
-
|
|
1065
|
-
}),
|
|
1066
|
-
|
|
1067
|
-
|
|
1068
|
-
|
|
1069
|
-
|
|
1070
|
-
|
|
1071
|
-
}),
|
|
1072
|
-
|
|
1073
|
-
|
|
1074
|
-
|
|
1075
|
-
|
|
1076
|
-
|
|
1077
|
-
|
|
1078
|
-
|
|
1079
|
-
|
|
1080
|
-
max_output_tokens: Type.Optional(
|
|
1081
|
-
Type.Number({
|
|
1082
|
-
description:
|
|
1083
|
-
"Default 4000 — reasoning shares budget, lower values can return empty with finish_reason:length",
|
|
1084
|
-
}),
|
|
1085
|
-
),
|
|
1086
|
-
}),
|
|
298
|
+
"Analyze an MP4 video with Muse Spark, including visuals and embedded speech. Accepts a local path, public HTTPS/data URL, or Meta file_id. Results are capped by max_chars; when truncated, the full text is saved to a temp file for continuation with read offset/limit.",
|
|
299
|
+
promptSnippet: "Inspect video and embedded audio when the active model cannot",
|
|
300
|
+
promptGuidelines: [
|
|
301
|
+
"Use meta_describe_video when the task depends on an MP4 video the active model cannot inspect directly.",
|
|
302
|
+
"Give meta_describe_video a task-specific prompt that asks for the exact visual, transcript, timestamp, or defect evidence needed next.",
|
|
303
|
+
"When a Meta media result is truncated, continue from its saved temp file with read using offset/limit.",
|
|
304
|
+
],
|
|
305
|
+
parameters: Type.Object(
|
|
306
|
+
{
|
|
307
|
+
path: Type.Optional(Type.String({ description: "Local .mp4/.m4v path, relative to cwd or absolute" })),
|
|
308
|
+
url: Type.Optional(Type.String({ description: "Public HTTPS or data:video/mp4 URL" })),
|
|
309
|
+
file_id: Type.Optional(Type.String({ description: "Existing Meta Files API id" })),
|
|
310
|
+
prompt: Type.String({ description: "What evidence to extract from the video" }),
|
|
311
|
+
model: Type.Optional(Type.String({ description: "Muse model id, default muse-spark-1.2" })),
|
|
312
|
+
max_output_tokens: Type.Optional(Type.Integer({
|
|
313
|
+
minimum: MIN_MEDIA_MAX_OUTPUT_TOKENS,
|
|
314
|
+
maximum: MAX_MEDIA_MAX_OUTPUT_TOKENS,
|
|
315
|
+
description: `Muse generation budget, default ${DEFAULT_MEDIA_MAX_OUTPUT_TOKENS}`,
|
|
316
|
+
})),
|
|
317
|
+
max_chars: Type.Optional(Type.Integer({
|
|
318
|
+
minimum: 1_000,
|
|
319
|
+
maximum: MAX_MEDIA_MAX_CHARS,
|
|
320
|
+
description: `Maximum result characters returned inline, default ${DEFAULT_MEDIA_MAX_CHARS}`,
|
|
321
|
+
})),
|
|
322
|
+
},
|
|
323
|
+
{ additionalProperties: false },
|
|
324
|
+
),
|
|
1087
325
|
async execute(toolCallId, params, signal, _onUpdate, ctx) {
|
|
1088
|
-
const { path, url, file_id, prompt, model, max_output_tokens } =
|
|
1089
|
-
|
|
1090
|
-
|
|
1091
|
-
|
|
1092
|
-
|
|
1093
|
-
|
|
1094
|
-
model?: string;
|
|
1095
|
-
max_output_tokens?: number;
|
|
1096
|
-
};
|
|
1097
|
-
if (!path && !url && !file_id) {
|
|
1098
|
-
return {
|
|
1099
|
-
content: [
|
|
1100
|
-
{
|
|
1101
|
-
type: "text",
|
|
1102
|
-
text: "Provide one of: path (local .mp4), url (https://...), or file_id (file-...).",
|
|
1103
|
-
},
|
|
1104
|
-
],
|
|
1105
|
-
details: { error: "missing source" },
|
|
1106
|
-
};
|
|
326
|
+
const { path, url, file_id, prompt, model, max_output_tokens, max_chars } = params as {
|
|
327
|
+
path?: string; url?: string; file_id?: string; prompt: string; model?: string;
|
|
328
|
+
max_output_tokens?: number; max_chars?: number;
|
|
329
|
+
};
|
|
330
|
+
if ([path, url, file_id].filter(Boolean).length !== 1) {
|
|
331
|
+
throw new Error("meta_describe_video requires exactly one of path, url, or file_id");
|
|
1107
332
|
}
|
|
1108
333
|
try {
|
|
1109
|
-
const apiKey = await safeGetMetaApiKey(
|
|
1110
|
-
ctx as unknown as ExtensionContext,
|
|
1111
|
-
);
|
|
334
|
+
const apiKey = await safeGetMetaApiKey(ctx as ExtensionContext);
|
|
1112
335
|
let videoBlock: ResponsesContentBlock;
|
|
1113
|
-
if (file_id) {
|
|
1114
|
-
|
|
1115
|
-
|
|
336
|
+
if (file_id) videoBlock = { type: "input_file", file_id };
|
|
337
|
+
else if (url) {
|
|
338
|
+
rejectWrongMime("video", url, mimeForSource(url));
|
|
1116
339
|
videoBlock = mediaInputFromSource(url, "video.mp4", "video/mp4");
|
|
1117
|
-
} else
|
|
1118
|
-
const absolute = resolve(
|
|
1119
|
-
|
|
1120
|
-
path,
|
|
1121
|
-
);
|
|
1122
|
-
if (!existsSync(absolute))
|
|
1123
|
-
throw new Error(`File not found: ${absolute}`);
|
|
340
|
+
} else {
|
|
341
|
+
const absolute = resolve(ctx.cwd, path as string);
|
|
342
|
+
if (!existsSync(absolute)) throw new Error(`File not found: ${absolute}`);
|
|
1124
343
|
const stat = statSync(absolute);
|
|
1125
|
-
if (stat.size > FILES_API_LIMIT_BYTES)
|
|
1126
|
-
|
|
1127
|
-
|
|
1128
|
-
|
|
1129
|
-
|
|
1130
|
-
// Use Files API for any video above the store-safe threshold
|
|
1131
|
-
// and for all larger media; keep inline only for tiny files.
|
|
1132
|
-
const useFilesApi = stat.size > STORE_SAFE_INLINE_BYTES;
|
|
1133
|
-
if (useFilesApi) {
|
|
1134
|
-
const up = await uploadMetaFile(apiKey, absolute);
|
|
1135
|
-
videoBlock = { type: "input_file", file_id: up.id };
|
|
344
|
+
if (stat.size > FILES_API_LIMIT_BYTES) throw new Error(`File too large (1 GiB limit): ${path}`);
|
|
345
|
+
rejectWrongMime("video", path as string, mimeForPath(absolute));
|
|
346
|
+
if (stat.size > STORE_SAFE_INLINE_BYTES) {
|
|
347
|
+
const upload = await uploadMetaFile(apiKey, absolute, automaticExpiry(), signal);
|
|
348
|
+
videoBlock = { type: "input_file", file_id: upload.id };
|
|
1136
349
|
} else {
|
|
1137
|
-
|
|
1138
|
-
videoBlock = mediaInputFromSource(
|
|
1139
|
-
dataUrl,
|
|
1140
|
-
basename(absolute),
|
|
1141
|
-
"video/mp4",
|
|
1142
|
-
);
|
|
350
|
+
videoBlock = mediaInputFromSource(dataUrlForFile(absolute).dataUrl, basename(absolute), "video/mp4");
|
|
1143
351
|
}
|
|
1144
|
-
} else {
|
|
1145
|
-
throw new Error("No source");
|
|
1146
352
|
}
|
|
1147
353
|
const selectedModel = model ?? "muse-spark-1.2";
|
|
1148
|
-
const
|
|
354
|
+
const { text, raw } = await callMetaResponses(apiKey, {
|
|
1149
355
|
model: selectedModel,
|
|
1150
356
|
store: false,
|
|
1151
|
-
input: [
|
|
1152
|
-
{
|
|
1153
|
-
|
|
1154
|
-
|
|
1155
|
-
|
|
1156
|
-
|
|
1157
|
-
|
|
1158
|
-
],
|
|
1159
|
-
},
|
|
1160
|
-
],
|
|
1161
|
-
max_output_tokens: Math.max(max_output_tokens ?? 8000, 4000),
|
|
1162
|
-
};
|
|
1163
|
-
const { text, raw } = await callMetaResponses(apiKey, payload, signal);
|
|
357
|
+
input: [{ type: "message", role: "user", content: [
|
|
358
|
+
{ type: "input_text", text: prompt },
|
|
359
|
+
videoBlock as Record<string, unknown>,
|
|
360
|
+
] }],
|
|
361
|
+
max_output_tokens: mediaMaxOutputTokens(max_output_tokens),
|
|
362
|
+
}, signal);
|
|
363
|
+
const output = await prepareMediaOutput({ text, response: raw, identity: toolCallId, maxChars: max_chars });
|
|
1164
364
|
const usage = extractMetaResponseUsage(raw, selectedModel);
|
|
1165
365
|
return {
|
|
1166
|
-
content: [{ type: "text", text }],
|
|
1167
|
-
details: { raw, videoBlock,
|
|
366
|
+
content: [{ type: "text", text: output.text }],
|
|
367
|
+
details: { raw, videoBlock, ...output.details },
|
|
1168
368
|
usage,
|
|
1169
369
|
};
|
|
1170
|
-
} catch (
|
|
1171
|
-
|
|
1172
|
-
content: [
|
|
1173
|
-
{
|
|
1174
|
-
type: "text",
|
|
1175
|
-
text: `Video analysis failed: ${e instanceof Error ? e.message : String(e)}`,
|
|
1176
|
-
},
|
|
1177
|
-
],
|
|
1178
|
-
details: { error: String(e) },
|
|
1179
|
-
};
|
|
370
|
+
} catch (error) {
|
|
371
|
+
mediaToolError("Video analysis", error);
|
|
1180
372
|
}
|
|
1181
373
|
},
|
|
1182
374
|
});
|
|
@@ -1185,113 +377,80 @@ export default function metaMedia(pi: ExtensionAPI): void {
|
|
|
1185
377
|
name: "meta_transcribe_audio",
|
|
1186
378
|
label: "Meta Audio Transcribe",
|
|
1187
379
|
description:
|
|
1188
|
-
"Transcribe
|
|
1189
|
-
|
|
1190
|
-
|
|
1191
|
-
|
|
1192
|
-
|
|
1193
|
-
|
|
1194
|
-
|
|
1195
|
-
|
|
1196
|
-
Type.String({
|
|
1197
|
-
|
|
1198
|
-
|
|
1199
|
-
}),
|
|
1200
|
-
|
|
1201
|
-
|
|
1202
|
-
|
|
1203
|
-
|
|
1204
|
-
|
|
380
|
+
"Transcribe MP3 or WAV audio with Muse Spark from a local path, public URL, or Meta file_id. Results are capped by max_chars; when truncated, the full transcript is saved to a temp file for continuation with read offset/limit.",
|
|
381
|
+
promptSnippet: "Transcribe audio when the active model cannot",
|
|
382
|
+
promptGuidelines: [
|
|
383
|
+
"Use meta_transcribe_audio when the task depends on MP3 or WAV speech the active model cannot inspect directly.",
|
|
384
|
+
"When a Meta media result is truncated, continue from its saved temp file with read using offset/limit.",
|
|
385
|
+
],
|
|
386
|
+
parameters: Type.Object(
|
|
387
|
+
{
|
|
388
|
+
path: Type.Optional(Type.String({ description: "Local .wav/.mp3 path" })),
|
|
389
|
+
url: Type.Optional(Type.String({ description: "Public HTTPS or data audio URL" })),
|
|
390
|
+
file_id: Type.Optional(Type.String({ description: "Existing Meta Files API id" })),
|
|
391
|
+
prompt: Type.Optional(Type.String({ description: "Transcription or audio-analysis instruction" })),
|
|
392
|
+
model: Type.Optional(Type.String({ description: "Muse model id, default muse-spark-1.2" })),
|
|
393
|
+
max_output_tokens: Type.Optional(Type.Integer({
|
|
394
|
+
minimum: MIN_MEDIA_MAX_OUTPUT_TOKENS,
|
|
395
|
+
maximum: MAX_MEDIA_MAX_OUTPUT_TOKENS,
|
|
396
|
+
description: `Muse generation budget, default ${DEFAULT_MEDIA_MAX_OUTPUT_TOKENS}`,
|
|
397
|
+
})),
|
|
398
|
+
max_chars: Type.Optional(Type.Integer({
|
|
399
|
+
minimum: 1_000,
|
|
400
|
+
maximum: MAX_MEDIA_MAX_CHARS,
|
|
401
|
+
description: `Maximum transcript characters returned inline, default ${DEFAULT_MEDIA_MAX_CHARS}`,
|
|
402
|
+
})),
|
|
403
|
+
},
|
|
404
|
+
{ additionalProperties: false },
|
|
405
|
+
),
|
|
1205
406
|
async execute(toolCallId, params, signal, _onUpdate, ctx) {
|
|
1206
|
-
const { path, url, file_id, prompt, model } = params as {
|
|
1207
|
-
path?: string;
|
|
1208
|
-
|
|
1209
|
-
file_id?: string;
|
|
1210
|
-
prompt?: string;
|
|
1211
|
-
model?: string;
|
|
407
|
+
const { path, url, file_id, prompt, model, max_output_tokens, max_chars } = params as {
|
|
408
|
+
path?: string; url?: string; file_id?: string; prompt?: string; model?: string;
|
|
409
|
+
max_output_tokens?: number; max_chars?: number;
|
|
1212
410
|
};
|
|
1213
|
-
if (
|
|
1214
|
-
|
|
1215
|
-
content: [
|
|
1216
|
-
{ type: "text", text: "Provide path, url, or file_id for audio." },
|
|
1217
|
-
],
|
|
1218
|
-
details: { error: "missing source" },
|
|
1219
|
-
};
|
|
411
|
+
if ([path, url, file_id].filter(Boolean).length !== 1) {
|
|
412
|
+
throw new Error("meta_transcribe_audio requires exactly one of path, url, or file_id");
|
|
1220
413
|
}
|
|
1221
414
|
try {
|
|
1222
|
-
const apiKey = await safeGetMetaApiKey(
|
|
1223
|
-
ctx as unknown as ExtensionContext,
|
|
1224
|
-
);
|
|
415
|
+
const apiKey = await safeGetMetaApiKey(ctx as ExtensionContext);
|
|
1225
416
|
let audioBlock: ResponsesContentBlock;
|
|
1226
|
-
if (file_id) {
|
|
1227
|
-
|
|
1228
|
-
|
|
417
|
+
if (file_id) audioBlock = { type: "input_file", file_id };
|
|
418
|
+
else if (url) {
|
|
419
|
+
rejectWrongMime("audio", url, mimeForSource(url));
|
|
1229
420
|
audioBlock = mediaInputFromSource(url, "audio.bin");
|
|
1230
|
-
} else
|
|
1231
|
-
const absolute = resolve(
|
|
1232
|
-
|
|
1233
|
-
path,
|
|
1234
|
-
);
|
|
1235
|
-
if (!existsSync(absolute))
|
|
1236
|
-
throw new Error(`File not found: ${absolute}`);
|
|
421
|
+
} else {
|
|
422
|
+
const absolute = resolve(ctx.cwd, path as string);
|
|
423
|
+
if (!existsSync(absolute)) throw new Error(`File not found: ${absolute}`);
|
|
1237
424
|
const stat = statSync(absolute);
|
|
1238
|
-
if (stat.size > FILES_API_LIMIT_BYTES)
|
|
1239
|
-
throw new Error(`Audio too large (1 GiB limit)`);
|
|
425
|
+
if (stat.size > FILES_API_LIMIT_BYTES) throw new Error("Audio too large (1 GiB limit)");
|
|
1240
426
|
const mime = mimeForPath(absolute);
|
|
1241
|
-
|
|
1242
|
-
throw new Error(
|
|
1243
|
-
`Unsupported audio mime ${mime}, need audio/wav or audio/mpeg`,
|
|
1244
|
-
);
|
|
1245
|
-
}
|
|
427
|
+
rejectWrongMime("audio", path as string, mime);
|
|
1246
428
|
if (stat.size > STORE_SAFE_INLINE_BYTES) {
|
|
1247
|
-
const
|
|
1248
|
-
audioBlock = { type: "input_file", file_id:
|
|
429
|
+
const upload = await uploadMetaFile(apiKey, absolute, automaticExpiry(), signal);
|
|
430
|
+
audioBlock = { type: "input_file", file_id: upload.id };
|
|
1249
431
|
} else {
|
|
1250
|
-
|
|
1251
|
-
audioBlock = mediaInputFromSource(
|
|
1252
|
-
dataUrl,
|
|
1253
|
-
basename(absolute),
|
|
1254
|
-
mime,
|
|
1255
|
-
);
|
|
432
|
+
audioBlock = mediaInputFromSource(dataUrlForFile(absolute).dataUrl, basename(absolute), mime);
|
|
1256
433
|
}
|
|
1257
|
-
} else {
|
|
1258
|
-
throw new Error("No audio source");
|
|
1259
434
|
}
|
|
1260
|
-
const
|
|
1261
|
-
|
|
435
|
+
const selectedModel = model ?? "muse-spark-1.2";
|
|
436
|
+
const { text, raw } = await callMetaResponses(apiKey, {
|
|
437
|
+
model: selectedModel,
|
|
1262
438
|
store: false,
|
|
1263
|
-
input: [
|
|
1264
|
-
{
|
|
1265
|
-
|
|
1266
|
-
|
|
1267
|
-
|
|
1268
|
-
|
|
1269
|
-
|
|
1270
|
-
|
|
1271
|
-
prompt ??
|
|
1272
|
-
"Transcribe this audio. Return only the transcript.",
|
|
1273
|
-
},
|
|
1274
|
-
audioBlock as unknown as Record<string, unknown>,
|
|
1275
|
-
],
|
|
1276
|
-
},
|
|
1277
|
-
],
|
|
1278
|
-
max_output_tokens: 4000,
|
|
1279
|
-
};
|
|
1280
|
-
const { text, raw } = await callMetaResponses(apiKey, payload, signal);
|
|
1281
|
-
return {
|
|
1282
|
-
content: [{ type: "text", text }],
|
|
1283
|
-
details: { raw, audioBlock },
|
|
1284
|
-
};
|
|
1285
|
-
} catch (e) {
|
|
439
|
+
input: [{ type: "message", role: "user", content: [
|
|
440
|
+
{ type: "input_text", text: prompt ?? "Transcribe this audio. Return only the transcript." },
|
|
441
|
+
audioBlock as Record<string, unknown>,
|
|
442
|
+
] }],
|
|
443
|
+
max_output_tokens: mediaMaxOutputTokens(max_output_tokens),
|
|
444
|
+
}, signal);
|
|
445
|
+
const output = await prepareMediaOutput({ text, response: raw, identity: toolCallId, maxChars: max_chars });
|
|
446
|
+
const usage = extractMetaResponseUsage(raw, selectedModel);
|
|
1286
447
|
return {
|
|
1287
|
-
content: [
|
|
1288
|
-
|
|
1289
|
-
|
|
1290
|
-
text: `Audio transcription failed: ${e instanceof Error ? e.message : String(e)}`,
|
|
1291
|
-
},
|
|
1292
|
-
],
|
|
1293
|
-
details: { error: String(e) },
|
|
448
|
+
content: [{ type: "text", text: output.text }],
|
|
449
|
+
details: { raw, audioBlock, ...output.details },
|
|
450
|
+
usage,
|
|
1294
451
|
};
|
|
452
|
+
} catch (error) {
|
|
453
|
+
mediaToolError("Audio transcription", error);
|
|
1295
454
|
}
|
|
1296
455
|
},
|
|
1297
456
|
});
|
|
@@ -1300,63 +459,46 @@ export default function metaMedia(pi: ExtensionAPI): void {
|
|
|
1300
459
|
name: "meta_upload_file",
|
|
1301
460
|
label: "Meta Files Upload",
|
|
1302
461
|
description:
|
|
1303
|
-
"Upload a file to Meta Files API
|
|
1304
|
-
parameters: Type.Object(
|
|
1305
|
-
|
|
1306
|
-
description:
|
|
1307
|
-
|
|
1308
|
-
|
|
1309
|
-
|
|
1310
|
-
|
|
1311
|
-
|
|
1312
|
-
|
|
1313
|
-
|
|
1314
|
-
|
|
1315
|
-
|
|
1316
|
-
async execute(
|
|
1317
|
-
const { path, expires_after_seconds } = params as {
|
|
1318
|
-
path: string;
|
|
1319
|
-
expires_after_seconds?: number;
|
|
462
|
+
"Upload a local file to Meta Files API for reuse. Uploads expire after seven days by default. Set expires_after_seconds to choose 3600..2592000 seconds, or retain=true to opt into no expiry.",
|
|
463
|
+
parameters: Type.Object(
|
|
464
|
+
{
|
|
465
|
+
path: Type.String({ description: "Local file path to upload" }),
|
|
466
|
+
expires_after_seconds: Type.Optional(Type.Integer({
|
|
467
|
+
minimum: MIN_UPLOAD_EXPIRY_SECONDS,
|
|
468
|
+
maximum: MAX_UPLOAD_EXPIRY_SECONDS,
|
|
469
|
+
description: `Expiry in seconds, default ${EXPLICIT_UPLOAD_EXPIRY_SECONDS} (7 days)`,
|
|
470
|
+
})),
|
|
471
|
+
retain: Type.Optional(Type.Boolean({ description: "Keep indefinitely instead of applying an expiry" })),
|
|
472
|
+
},
|
|
473
|
+
{ additionalProperties: false },
|
|
474
|
+
),
|
|
475
|
+
async execute(_toolCallId, params, signal, _onUpdate, ctx) {
|
|
476
|
+
const { path, expires_after_seconds, retain } = params as {
|
|
477
|
+
path: string; expires_after_seconds?: number; retain?: boolean;
|
|
1320
478
|
};
|
|
479
|
+
if (retain && expires_after_seconds !== undefined) {
|
|
480
|
+
throw new Error("meta_upload_file accepts either retain=true or expires_after_seconds, not both");
|
|
481
|
+
}
|
|
1321
482
|
try {
|
|
1322
|
-
const apiKey = await safeGetMetaApiKey(
|
|
1323
|
-
|
|
1324
|
-
);
|
|
1325
|
-
const
|
|
1326
|
-
|
|
1327
|
-
|
|
1328
|
-
);
|
|
1329
|
-
if (!existsSync(absolute))
|
|
1330
|
-
throw new Error(`File not found: ${absolute}`);
|
|
1331
|
-
let expiresAfter: { anchor: "created_at"; seconds: number } | undefined;
|
|
1332
|
-
if (expires_after_seconds !== undefined) {
|
|
1333
|
-
if (expires_after_seconds < 3600 || expires_after_seconds > 2592000)
|
|
1334
|
-
throw new Error("expires_after_seconds must be 3600..2592000");
|
|
1335
|
-
expiresAfter = {
|
|
1336
|
-
anchor: "created_at",
|
|
1337
|
-
seconds: expires_after_seconds,
|
|
1338
|
-
};
|
|
1339
|
-
}
|
|
1340
|
-
const result = await uploadMetaFile(apiKey, absolute, expiresAfter);
|
|
1341
|
-
return {
|
|
1342
|
-
content: [
|
|
1343
|
-
{
|
|
1344
|
-
type: "text",
|
|
1345
|
-
text: `Uploaded ${basename(absolute)} → ${result.id} (${result.bytes} bytes, status: ${result.status}${result.expires_at ? `, expires_at: ${result.expires_at}` : ""}). Use with {type:"input_file", file_id:"${result.id}"}.`,
|
|
1346
|
-
},
|
|
1347
|
-
],
|
|
1348
|
-
details: result,
|
|
483
|
+
const apiKey = await safeGetMetaApiKey(ctx as ExtensionContext);
|
|
484
|
+
const absolute = resolve(ctx.cwd, path);
|
|
485
|
+
if (!existsSync(absolute)) throw new Error(`File not found: ${absolute}`);
|
|
486
|
+
const expiresAfter = retain ? undefined : {
|
|
487
|
+
anchor: "created_at" as const,
|
|
488
|
+
seconds: expires_after_seconds ?? EXPLICIT_UPLOAD_EXPIRY_SECONDS,
|
|
1349
489
|
};
|
|
1350
|
-
|
|
490
|
+
const result = await uploadMetaFile(apiKey, absolute, expiresAfter, signal);
|
|
491
|
+
const retention = retain
|
|
492
|
+
? "retained without expiry"
|
|
493
|
+
: result.expires_at
|
|
494
|
+
? `expires_at ${result.expires_at}`
|
|
495
|
+
: `expires after ${expiresAfter?.seconds} seconds`;
|
|
1351
496
|
return {
|
|
1352
|
-
content: [
|
|
1353
|
-
|
|
1354
|
-
type: "text",
|
|
1355
|
-
text: `Upload failed: ${e instanceof Error ? e.message : String(e)}`,
|
|
1356
|
-
},
|
|
1357
|
-
],
|
|
1358
|
-
details: { error: String(e) },
|
|
497
|
+
content: [{ type: "text", text: `Uploaded ${basename(absolute)} to ${result.id} (${result.bytes} bytes, ${retention}).` }],
|
|
498
|
+
details: result,
|
|
1359
499
|
};
|
|
500
|
+
} catch (error) {
|
|
501
|
+
mediaToolError("File upload", error);
|
|
1360
502
|
}
|
|
1361
503
|
},
|
|
1362
504
|
});
|
|
@@ -1365,93 +507,83 @@ export default function metaMedia(pi: ExtensionAPI): void {
|
|
|
1365
507
|
name: "meta_analyze_file",
|
|
1366
508
|
label: "Meta File Analyze",
|
|
1367
509
|
description:
|
|
1368
|
-
"Analyze
|
|
1369
|
-
|
|
1370
|
-
|
|
1371
|
-
|
|
1372
|
-
|
|
1373
|
-
|
|
1374
|
-
|
|
1375
|
-
|
|
1376
|
-
|
|
1377
|
-
|
|
510
|
+
"Analyze one or more images, PDFs, videos, or audio files with Muse Spark. Use path/url/file_id for one source, or ordered sources for comparisons. Images and PDF pages share Meta's 50-image budget; PDFs include text from the first 100 pages and images from the first 50 pages. Truncated text is saved for continuation with read offset/limit.",
|
|
511
|
+
promptSnippet: "Inspect images, PDFs, or mixed media when the active model cannot",
|
|
512
|
+
promptGuidelines: [
|
|
513
|
+
"Use meta_analyze_file when the task depends on an image, PDF, or generic media file the active model cannot inspect directly.",
|
|
514
|
+
"Give meta_analyze_file a task-specific prompt that asks for the exact visible text, layout, comparison, defect, or action evidence needed next.",
|
|
515
|
+
"When a Meta media result is truncated, continue from its saved temp file with read using offset/limit.",
|
|
516
|
+
],
|
|
517
|
+
parameters: Type.Object(
|
|
518
|
+
{
|
|
519
|
+
path: Type.Optional(Type.String({ description: "Single local file path" })),
|
|
520
|
+
url: Type.Optional(Type.String({ description: "Single public HTTPS or data URL" })),
|
|
521
|
+
file_id: Type.Optional(Type.String({ description: "Single existing Meta file_id" })),
|
|
522
|
+
sources: Type.Optional(Type.Array(Type.Object({
|
|
523
|
+
source: Type.String({ description: "Local path, HTTPS/data URL, or Meta file_id" }),
|
|
524
|
+
label: Type.Optional(Type.String({ description: "Human-readable label such as Before or After" })),
|
|
525
|
+
}, { additionalProperties: false }), {
|
|
526
|
+
minItems: 1,
|
|
527
|
+
maxItems: MAX_ANALYSIS_SOURCES,
|
|
528
|
+
description: "Ordered sources for comparison or combined analysis",
|
|
529
|
+
})),
|
|
530
|
+
prompt: Type.String({ description: "What evidence to extract or compare" }),
|
|
531
|
+
model: Type.Optional(Type.String({ description: "Muse model id, default muse-spark-1.2" })),
|
|
532
|
+
max_output_tokens: Type.Optional(Type.Integer({
|
|
533
|
+
minimum: MIN_MEDIA_MAX_OUTPUT_TOKENS,
|
|
534
|
+
maximum: MAX_MEDIA_MAX_OUTPUT_TOKENS,
|
|
535
|
+
description: `Muse generation budget, default ${DEFAULT_MEDIA_MAX_OUTPUT_TOKENS}`,
|
|
536
|
+
})),
|
|
537
|
+
max_chars: Type.Optional(Type.Integer({
|
|
538
|
+
minimum: 1_000,
|
|
539
|
+
maximum: MAX_MEDIA_MAX_CHARS,
|
|
540
|
+
description: `Maximum result characters returned inline, default ${DEFAULT_MEDIA_MAX_CHARS}`,
|
|
541
|
+
})),
|
|
542
|
+
},
|
|
543
|
+
{ additionalProperties: false },
|
|
544
|
+
),
|
|
1378
545
|
async execute(toolCallId, params, signal, _onUpdate, ctx) {
|
|
1379
|
-
const { path, url, file_id, prompt, model } = params as {
|
|
1380
|
-
path?: string;
|
|
1381
|
-
|
|
1382
|
-
file_id?: string;
|
|
1383
|
-
prompt: string;
|
|
1384
|
-
model?: string;
|
|
546
|
+
const { path, url, file_id, sources, prompt, model, max_output_tokens, max_chars } = params as {
|
|
547
|
+
path?: string; url?: string; file_id?: string; sources?: AnalysisSource[];
|
|
548
|
+
prompt: string; model?: string; max_output_tokens?: number; max_chars?: number;
|
|
1385
549
|
};
|
|
1386
|
-
|
|
1387
|
-
|
|
1388
|
-
|
|
1389
|
-
|
|
1390
|
-
|
|
550
|
+
const legacySources = [path, url, file_id].filter((value): value is string => Boolean(value));
|
|
551
|
+
if (legacySources.length > 1 || (legacySources.length === 1 && sources) || (legacySources.length === 0 && !sources)) {
|
|
552
|
+
throw new Error("meta_analyze_file requires exactly one path/url/file_id or a sources array");
|
|
553
|
+
}
|
|
554
|
+
if (sources && sources.length > MAX_ANALYSIS_SOURCES) {
|
|
555
|
+
throw new Error(`meta_analyze_file supports at most ${MAX_ANALYSIS_SOURCES} sources`);
|
|
1391
556
|
}
|
|
1392
557
|
try {
|
|
1393
|
-
const apiKey = await safeGetMetaApiKey(
|
|
1394
|
-
|
|
1395
|
-
|
|
1396
|
-
|
|
1397
|
-
|
|
1398
|
-
|
|
1399
|
-
else if (path) {
|
|
1400
|
-
const absolute = resolve(
|
|
1401
|
-
(ctx as unknown as { cwd?: string }).cwd ?? process.cwd(),
|
|
1402
|
-
path,
|
|
1403
|
-
);
|
|
1404
|
-
if (!existsSync(absolute))
|
|
1405
|
-
throw new Error(`File not found: ${absolute}`);
|
|
1406
|
-
const stat = statSync(absolute);
|
|
1407
|
-
if (stat.size > FILES_API_LIMIT_BYTES)
|
|
1408
|
-
throw new Error(`File too large (1 GiB limit)`);
|
|
1409
|
-
if (stat.size > STORE_SAFE_INLINE_BYTES) {
|
|
1410
|
-
const up = await uploadMetaFile(apiKey, absolute);
|
|
1411
|
-
fileBlock = { type: "input_file", file_id: up.id };
|
|
1412
|
-
} else {
|
|
1413
|
-
const { dataUrl, mime } = dataUrlForFile(absolute);
|
|
1414
|
-
fileBlock = mediaInputFromSource(dataUrl, basename(absolute), mime);
|
|
558
|
+
const apiKey = await safeGetMetaApiKey(ctx as ExtensionContext);
|
|
559
|
+
const requestedSources: AnalysisSource[] = sources ?? [{ source: legacySources[0] as string }];
|
|
560
|
+
const content: ResponsesContentBlock[] = [{ type: "input_text", text: prompt }];
|
|
561
|
+
for (const [index, item] of requestedSources.entries()) {
|
|
562
|
+
if (requestedSources.length > 1 || item.label) {
|
|
563
|
+
content.push({ type: "input_text", text: `${sourceLabel(item, index)}:` });
|
|
1415
564
|
}
|
|
1416
|
-
|
|
1417
|
-
|
|
1418
|
-
|
|
565
|
+
content.push(await resolveGenericMediaSource(apiKey, ctx.cwd, item.source, signal));
|
|
566
|
+
}
|
|
567
|
+
const selectedModel = model ?? "muse-spark-1.2";
|
|
568
|
+
const { text, raw } = await callMetaResponses(apiKey, {
|
|
569
|
+
model: selectedModel,
|
|
1419
570
|
store: false,
|
|
1420
|
-
input: [
|
|
1421
|
-
|
|
1422
|
-
|
|
1423
|
-
|
|
1424
|
-
|
|
1425
|
-
{ type: "input_text", text: prompt },
|
|
1426
|
-
fileBlock as unknown as Record<string, unknown>,
|
|
1427
|
-
],
|
|
1428
|
-
},
|
|
1429
|
-
],
|
|
1430
|
-
max_output_tokens: 4000,
|
|
1431
|
-
};
|
|
1432
|
-
const { text, raw } = await callMetaResponses(apiKey, payload, signal);
|
|
1433
|
-
return {
|
|
1434
|
-
content: [{ type: "text", text }],
|
|
1435
|
-
details: { raw, fileBlock },
|
|
1436
|
-
};
|
|
1437
|
-
} catch (e) {
|
|
571
|
+
input: [{ type: "message", role: "user", content: content as Record<string, unknown>[] }],
|
|
572
|
+
max_output_tokens: mediaMaxOutputTokens(max_output_tokens),
|
|
573
|
+
}, signal);
|
|
574
|
+
const output = await prepareMediaOutput({ text, response: raw, identity: toolCallId, maxChars: max_chars });
|
|
575
|
+
const usage = extractMetaResponseUsage(raw, selectedModel);
|
|
1438
576
|
return {
|
|
1439
|
-
content: [
|
|
1440
|
-
|
|
1441
|
-
|
|
1442
|
-
text: `File analysis failed: ${e instanceof Error ? e.message : String(e)}`,
|
|
1443
|
-
},
|
|
1444
|
-
],
|
|
1445
|
-
details: { error: String(e) },
|
|
577
|
+
content: [{ type: "text", text: output.text }],
|
|
578
|
+
details: { raw, sourceCount: requestedSources.length, ...output.details },
|
|
579
|
+
usage,
|
|
1446
580
|
};
|
|
581
|
+
} catch (error) {
|
|
582
|
+
mediaToolError("File analysis", error);
|
|
1447
583
|
}
|
|
1448
584
|
},
|
|
1449
585
|
});
|
|
1450
586
|
|
|
1451
|
-
// -----------------------------------------------------------------------
|
|
1452
|
-
// Slash commands (human-in-the-loop)
|
|
1453
|
-
// -----------------------------------------------------------------------
|
|
1454
|
-
|
|
1455
587
|
pi.registerCommand("meta-video", {
|
|
1456
588
|
description:
|
|
1457
589
|
"Analyze a video with Muse Spark (mp4). Usage: /meta-video <path|url|file_id> [prompt]",
|
|
@@ -1473,29 +605,18 @@ export default function metaMedia(pi: ExtensionAPI): void {
|
|
|
1473
605
|
? "Summarize what happens in this video in detail. Transcribe any speech verbatim. List key visual moments in 5 bullet points."
|
|
1474
606
|
: trimmed.slice(firstSpace + 1).trim() ||
|
|
1475
607
|
"Summarize what happens in this video in detail. Transcribe any speech verbatim.";
|
|
1476
|
-
const isFileId = source.startsWith("file-");
|
|
1477
|
-
const isUrl = source.startsWith("https://") || source.startsWith("data:");
|
|
1478
|
-
const toolArgs: Record<string, unknown> = { prompt };
|
|
1479
|
-
if (isFileId) toolArgs.file_id = source;
|
|
1480
|
-
else if (isUrl) toolArgs.url = source;
|
|
1481
|
-
else toolArgs.path = source;
|
|
1482
|
-
safeNotify(ctx, `Analyzing video: ${source}`, "info");
|
|
1483
|
-
// Dispatch via LLM tool path by sending a follow-up that triggers the tool
|
|
1484
|
-
// For direct command path, call the API directly:
|
|
1485
608
|
try {
|
|
1486
|
-
const apiKey = await safeGetMetaApiKey(
|
|
1487
|
-
ctx as unknown as ExtensionContext,
|
|
1488
|
-
);
|
|
609
|
+
const apiKey = await safeGetMetaApiKey(ctx as unknown as ExtensionContext);
|
|
1489
610
|
let block: ResponsesContentBlock;
|
|
1490
|
-
if (isFileId) block = { type: "input_file", file_id: source };
|
|
1491
|
-
else if (
|
|
611
|
+
if (isFileId(source)) block = { type: "input_file", file_id: source };
|
|
612
|
+
else if (isRemoteMediaSource(source))
|
|
1492
613
|
block = mediaInputFromSource(source, "video.mp4", "video/mp4");
|
|
1493
614
|
else {
|
|
1494
615
|
const abs = resolve(cwd, source);
|
|
1495
616
|
if (!existsSync(abs)) throw new Error(`File not found: ${abs}`);
|
|
1496
617
|
const stat = statSync(abs);
|
|
1497
618
|
if (stat.size > STORE_SAFE_INLINE_BYTES) {
|
|
1498
|
-
const up = await uploadMetaFile(apiKey, abs);
|
|
619
|
+
const up = await uploadMetaFile(apiKey, abs, automaticExpiry());
|
|
1499
620
|
block = { type: "input_file", file_id: up.id };
|
|
1500
621
|
} else {
|
|
1501
622
|
const { dataUrl } = dataUrlForFile(abs);
|
|
@@ -1541,11 +662,7 @@ export default function metaMedia(pi: ExtensionAPI): void {
|
|
|
1541
662
|
const cwd = ctx.cwd;
|
|
1542
663
|
const trimmed = (args ?? "").trim();
|
|
1543
664
|
if (!trimmed) {
|
|
1544
|
-
safeNotify(
|
|
1545
|
-
ctx,
|
|
1546
|
-
"Usage: /meta-audio <path|url|file_id> [prompt]",
|
|
1547
|
-
"info",
|
|
1548
|
-
);
|
|
665
|
+
safeNotify(ctx, "Usage: /meta-audio <path|url|file_id> [prompt]", "info");
|
|
1549
666
|
return;
|
|
1550
667
|
}
|
|
1551
668
|
const firstSpace = trimmed.indexOf(" ");
|
|
@@ -1555,21 +672,18 @@ export default function metaMedia(pi: ExtensionAPI): void {
|
|
|
1555
672
|
? "Transcribe this audio. Return only the transcript."
|
|
1556
673
|
: trimmed.slice(firstSpace + 1).trim() ||
|
|
1557
674
|
"Transcribe this audio. Return only the transcript.";
|
|
1558
|
-
const isFileId = source.startsWith("file-");
|
|
1559
|
-
const isUrl = source.startsWith("https://");
|
|
1560
675
|
try {
|
|
1561
|
-
const apiKey = await safeGetMetaApiKey(
|
|
1562
|
-
ctx as unknown as ExtensionContext,
|
|
1563
|
-
);
|
|
676
|
+
const apiKey = await safeGetMetaApiKey(ctx as unknown as ExtensionContext);
|
|
1564
677
|
let block: ResponsesContentBlock;
|
|
1565
|
-
if (isFileId) block = { type: "input_file", file_id: source };
|
|
1566
|
-
else if (
|
|
678
|
+
if (isFileId(source)) block = { type: "input_file", file_id: source };
|
|
679
|
+
else if (isRemoteMediaSource(source))
|
|
680
|
+
block = mediaInputFromSource(source, "audio.bin");
|
|
1567
681
|
else {
|
|
1568
682
|
const abs = resolve(cwd, source);
|
|
1569
683
|
if (!existsSync(abs)) throw new Error(`File not found: ${abs}`);
|
|
1570
684
|
const stat = statSync(abs);
|
|
1571
685
|
if (stat.size > STORE_SAFE_INLINE_BYTES) {
|
|
1572
|
-
const up = await uploadMetaFile(apiKey, abs);
|
|
686
|
+
const up = await uploadMetaFile(apiKey, abs, automaticExpiry());
|
|
1573
687
|
block = { type: "input_file", file_id: up.id };
|
|
1574
688
|
} else {
|
|
1575
689
|
const { dataUrl, mime } = dataUrlForFile(abs);
|
|
@@ -1609,11 +723,7 @@ export default function metaMedia(pi: ExtensionAPI): void {
|
|
|
1609
723
|
const cwd = ctx.cwd;
|
|
1610
724
|
const trimmed = (args ?? "").trim();
|
|
1611
725
|
if (!trimmed) {
|
|
1612
|
-
safeNotify(
|
|
1613
|
-
ctx,
|
|
1614
|
-
"Usage: /meta-file <path|url|file_id> <prompt>",
|
|
1615
|
-
"info",
|
|
1616
|
-
);
|
|
726
|
+
safeNotify(ctx, "Usage: /meta-file <path|url|file_id> <prompt>", "info");
|
|
1617
727
|
return;
|
|
1618
728
|
}
|
|
1619
729
|
const firstSpace = trimmed.indexOf(" ");
|
|
@@ -1623,21 +733,18 @@ export default function metaMedia(pi: ExtensionAPI): void {
|
|
|
1623
733
|
}
|
|
1624
734
|
const source = trimmed.slice(0, firstSpace);
|
|
1625
735
|
const prompt = trimmed.slice(firstSpace + 1).trim();
|
|
1626
|
-
const isFileId = source.startsWith("file-");
|
|
1627
|
-
const isUrl = source.startsWith("https://");
|
|
1628
736
|
try {
|
|
1629
|
-
const apiKey = await safeGetMetaApiKey(
|
|
1630
|
-
ctx as unknown as ExtensionContext,
|
|
1631
|
-
);
|
|
737
|
+
const apiKey = await safeGetMetaApiKey(ctx as unknown as ExtensionContext);
|
|
1632
738
|
let block: ResponsesContentBlock;
|
|
1633
|
-
if (isFileId) block = { type: "input_file", file_id: source };
|
|
1634
|
-
else if (
|
|
739
|
+
if (isFileId(source)) block = { type: "input_file", file_id: source };
|
|
740
|
+
else if (isRemoteMediaSource(source))
|
|
741
|
+
block = mediaInputFromSource(source);
|
|
1635
742
|
else {
|
|
1636
743
|
const abs = resolve(cwd, source);
|
|
1637
744
|
if (!existsSync(abs)) throw new Error(`File not found: ${abs}`);
|
|
1638
745
|
const stat = statSync(abs);
|
|
1639
746
|
if (stat.size > STORE_SAFE_INLINE_BYTES) {
|
|
1640
|
-
const up = await uploadMetaFile(apiKey, abs);
|
|
747
|
+
const up = await uploadMetaFile(apiKey, abs, automaticExpiry());
|
|
1641
748
|
block = { type: "input_file", file_id: up.id };
|
|
1642
749
|
} else {
|
|
1643
750
|
const { dataUrl, mime } = dataUrlForFile(abs);
|