pi-meta-oauth 0.4.2 → 0.4.4

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -1,195 +1,59 @@
1
- import type { Usage } from "@earendil-works/pi-ai";
2
1
  import type {
3
2
  ExtensionAPI,
4
3
  ExtensionContext,
5
4
  } from "@earendil-works/pi-coding-agent";
6
5
  import { Type } from "typebox";
7
- import {
8
- createMetaProviderConfig,
9
- META_FILES_URL,
10
- META_API_BASE_URL,
11
- META_PROVIDER_ID,
12
- } from "./meta.ts";
6
+ import { META_PROVIDER_ID } from "./meta.ts";
13
7
  import { existsSync, statSync, readFileSync } from "node:fs";
14
- import { basename, extname, resolve } from "node:path";
15
- import { readFile } from "node:fs/promises";
16
-
17
- // ---------------------------------------------------------------------------
18
- // Meta Files API + Responses API
19
- // Docs: https://dev.meta.ai/docs/file-handling , /video-understanding
20
- // - Inline file_data / file_url / video_url / input_audio : 50 MB limit
21
- // - Files API POST /v1/files purpose=user_data : 1 GiB, 100 GiB/team storage, no expiry by default
22
- // - Uploaded media uses input_file {file_id}; inline/public video uses
23
- // input_video {video_url}; inline audio uses input_audio {data,format}.
24
- // - Video: video/mp4 only, reads frames + embedded audio together
25
- // - Audio: audio/mpeg (.mp3), audio/wav (.wav)
26
- // - Image: image/png, image/jpeg, image/gif, image/webp, image/x-icon, up to 50/request
27
- // - PDF: application/pdf -> text first 100p + images first 50p (counts to 50-image budget)
28
- // ---------------------------------------------------------------------------
29
-
30
- const INLINE_LIMIT_BYTES = 50_000_000;
31
- const FILES_API_LIMIT_BYTES = 1_073_741_824;
32
- // Meta Responses `store=true` (default) has ~20 MB persistence limit even when
33
- // inline limit is 50 MB — a 24 MB base64 payload triggers HTTP 413
34
- // "payload_too_large … with `store=true`". Keep inline only for small files
35
- // and force Files API or `store:false` above this threshold.
36
- const STORE_SAFE_INLINE_BYTES = 15_000_000;
37
-
38
- type SupportedMime =
39
- | "video/mp4"
40
- | "audio/mpeg"
41
- | "audio/wav"
42
- | "application/pdf"
43
- | "image/png"
44
- | "image/jpeg"
45
- | "image/gif"
46
- | "image/webp"
47
- | "image/x-icon"
48
- | "text/plain"
49
- | "application/json"
50
- | "application/jsonl";
51
-
52
- const EXT_TO_MIME: Record<string, SupportedMime> = {
53
- ".mp4": "video/mp4",
54
- ".m4v": "video/mp4",
55
- ".mp3": "audio/mpeg",
56
- ".wav": "audio/wav",
57
- ".pdf": "application/pdf",
58
- ".png": "image/png",
59
- ".jpg": "image/jpeg",
60
- ".jpeg": "image/jpeg",
61
- ".gif": "image/gif",
62
- ".webp": "image/webp",
63
- ".ico": "image/x-icon",
64
- ".txt": "text/plain",
65
- ".json": "application/json",
66
- ".jsonl": "application/jsonl",
67
- };
68
-
69
- function mimeForPath(path: string): SupportedMime | undefined {
70
- const ext = extname(path).toLowerCase();
71
- return EXT_TO_MIME[ext];
72
- }
73
-
74
- function mimeForDataUrl(dataUrl: string): string | undefined {
75
- const m = dataUrl.match(/^data:([^;,\s]+)[;,]/);
76
- return m?.[1]?.toLowerCase();
77
- }
78
-
79
- function isVideoMime(m: string | undefined): boolean {
80
- return m === "video/mp4";
81
- }
82
- function isAudioMime(m: string | undefined): boolean {
83
- return (
84
- m === "audio/mpeg" ||
85
- m === "audio/mp3" ||
86
- m === "audio/wav" ||
87
- m === "audio/x-wav"
88
- );
89
- }
90
- function isPdfMime(m: string | undefined): boolean {
91
- return m === "application/pdf";
92
- }
93
- function isImageMime(m: string | undefined): boolean {
94
- return !!m && m.startsWith("image/");
95
- }
96
-
97
- function filenameForMime(
98
- mime: string | undefined,
99
- fallback = "file.bin",
100
- ): string {
101
- switch (mime) {
102
- case "video/mp4":
103
- return "video.mp4";
104
- case "audio/mpeg":
105
- case "audio/mp3":
106
- return "audio.mp3";
107
- case "audio/wav":
108
- case "audio/x-wav":
109
- return "audio.wav";
110
- case "application/pdf":
111
- return "document.pdf";
112
- default:
113
- return fallback;
114
- }
115
- }
116
-
117
- function audioFormatForMime(mime: string | undefined): "wav" | "mp3" {
118
- return mime === "audio/wav" || mime === "audio/x-wav" ? "wav" : "mp3";
119
- }
120
-
121
- /** Build a generic Meta Responses input_file block. */
122
- function inputFileFromSource(
123
- source: string,
124
- filename?: string,
125
- ): ResponsesContentBlock {
126
- if (source.startsWith("https://")) {
127
- return { type: "input_file", file_url: source };
128
- }
129
- if (source.startsWith("data:")) {
130
- const mime = mimeForDataUrl(source);
131
- return {
132
- type: "input_file",
133
- file_data: source,
134
- filename: filename ?? filenameForMime(mime),
135
- };
136
- }
137
- // Bare file_id (file-...)
138
- return { type: "input_file", file_id: source };
139
- }
140
-
141
- function mimeForSource(source: string): string | undefined {
142
- if (source.startsWith("data:")) return mimeForDataUrl(source);
143
- if (source.startsWith("https://")) {
144
- try {
145
- return mimeForPath(new URL(source).pathname);
146
- } catch {
147
- return undefined;
148
- }
149
- }
150
- return mimeForPath(source);
151
- }
152
-
153
- /** Build the typed content block required by Meta for each media kind. */
154
- export function mediaInputFromSource(
155
- source: string,
156
- filename?: string,
157
- expectedMime?: string,
158
- ): ResponsesContentBlock {
159
- if (!source.startsWith("data:") && !source.startsWith("https://")) {
160
- return { type: "input_file", file_id: source };
161
- }
162
- const mime = expectedMime ?? mimeForSource(source);
163
- if (isVideoMime(mime)) {
164
- return { type: "input_video", video_url: source };
165
- }
166
- if (isAudioMime(mime) && source.startsWith("data:")) {
167
- return {
168
- type: "input_audio",
169
- input_audio: {
170
- data: base64FromDataUrl(source),
171
- format: audioFormatForMime(mime),
172
- },
173
- };
174
- }
175
- return inputFileFromSource(source, filename);
176
- }
177
-
178
- function base64FromDataUrl(dataUrl: string): string {
179
- const comma = dataUrl.indexOf(",");
180
- return comma === -1 ? "" : dataUrl.slice(comma + 1);
181
- }
182
-
183
- function base64ByteLength(base64: string): number {
184
- const normalized = base64.replace(/\s/g, "");
185
- if (!normalized) return 0;
186
- const padding = normalized.endsWith("==")
187
- ? 2
188
- : normalized.endsWith("=")
189
- ? 1
190
- : 0;
191
- return Math.max(0, Math.floor((normalized.length * 3) / 4) - padding);
192
- }
8
+ import { basename, resolve } from "node:path";
9
+ import {
10
+ DEFAULT_MEDIA_MAX_CHARS,
11
+ DEFAULT_MEDIA_MAX_OUTPUT_TOKENS,
12
+ MAX_MEDIA_MAX_CHARS,
13
+ MAX_MEDIA_MAX_OUTPUT_TOKENS,
14
+ MIN_MEDIA_MAX_OUTPUT_TOKENS,
15
+ mediaMaxOutputTokens,
16
+ prepareMediaOutput,
17
+ } from "./media-output.ts";
18
+ import {
19
+ AUTOMATIC_UPLOAD_EXPIRY_SECONDS,
20
+ EXPLICIT_UPLOAD_EXPIRY_SECONDS,
21
+ FILES_API_LIMIT_BYTES,
22
+ INLINE_LIMIT_BYTES,
23
+ MAX_ANALYSIS_SOURCES,
24
+ MAX_UPLOAD_EXPIRY_SECONDS,
25
+ MIN_UPLOAD_EXPIRY_SECONDS,
26
+ STORE_SAFE_INLINE_BYTES,
27
+ } from "./media/limits.ts";
28
+ import {
29
+ base64FromDataUrl,
30
+ isAudioMime,
31
+ isRemoteMediaSource,
32
+ isVideoMime,
33
+ mediaInputFromSource,
34
+ mimeForPath,
35
+ mimeForSource,
36
+ type ResponsesContentBlock,
37
+ } from "./media/mime.ts";
38
+ import { uploadMetaFile } from "./media/files.ts";
39
+ import {
40
+ maybeUploadLargeInlineBlocks,
41
+ payloadContainsMedia,
42
+ rewriteResponsesPayload,
43
+ } from "./media/payload.ts";
44
+ import {
45
+ callMetaResponses,
46
+ extractMetaResponseUsage,
47
+ formatMetaResponseUsage,
48
+ } from "./media/responses.ts";
49
+
50
+ export { mediaInputFromSource } from "./media/mime.ts";
51
+ export { maybeUploadLargeInlineBlocks, rewriteResponsesPayload } from "./media/payload.ts";
52
+ export {
53
+ extractMetaResponseUsage,
54
+ extractResponsesText,
55
+ formatMetaResponseUsage,
56
+ } from "./media/responses.ts";
193
57
 
194
58
  function safeNotify(
195
59
  ctx:
@@ -211,391 +75,24 @@ function safeNotify(
211
75
  }
212
76
  }
213
77
 
214
- async function safeGetMetaApiKey(ctx: ExtensionContext): Promise<string> {
215
- try {
216
- return await getMetaApiKey(ctx);
217
- } catch (error) {
218
- if (String(error).includes("stale after session replacement"))
219
- throw new Error("Session was replaced — please retry the command");
220
- throw error;
221
- }
222
- }
223
-
224
78
  async function getMetaApiKey(ctx: ExtensionContext): Promise<string> {
225
79
  const key = await ctx.modelRegistry.getApiKeyForProvider(META_PROVIDER_ID);
226
- if (!key)
80
+ if (!key) {
227
81
  throw new Error(
228
82
  `Meta credentials unavailable — run /login ${META_PROVIDER_ID}`,
229
83
  );
230
- return key;
231
- }
232
-
233
- async function getMetaApiKeyFromRegistry(
234
- _api: ExtensionAPI,
235
- ): Promise<string | undefined> {
236
- // For tools that receive ExtensionContext we prefer ctx.modelRegistry; this is fallback for non-context paths.
237
- return undefined;
238
- }
239
-
240
- interface UploadResult {
241
- id: string;
242
- filename: string;
243
- bytes: number;
244
- purpose: string;
245
- status: string;
246
- expires_at?: number;
247
- }
248
-
249
- async function uploadMetaFile(
250
- apiKey: string,
251
- filePath: string,
252
- expiresAfter?: { anchor: "created_at"; seconds: number },
253
- ): Promise<UploadResult> {
254
- const data = await readFile(filePath);
255
- if (data.byteLength > FILES_API_LIMIT_BYTES) {
256
- throw new Error(
257
- `File too large for Files API (${data.byteLength} bytes > 1 GiB)`,
258
- );
259
- }
260
- const mime = mimeForPath(filePath) ?? "application/octet-stream";
261
- const blob = new Blob([data], { type: mime });
262
- const form = new FormData();
263
- // OpenAI SDK uses `file` field name; Meta docs show same.
264
- form.append("file", blob, basename(filePath));
265
- form.append("purpose", "user_data");
266
- if (expiresAfter) {
267
- form.append("expires_after[anchor]", expiresAfter.anchor);
268
- form.append("expires_after[seconds]", String(expiresAfter.seconds));
269
- }
270
- const res = await fetch(META_FILES_URL, {
271
- method: "POST",
272
- headers: { Authorization: `Bearer ${apiKey}` },
273
- body: form,
274
- });
275
- const bodyText = await res.text();
276
- let body: Record<string, unknown> = {};
277
- try {
278
- body = bodyText ? (JSON.parse(bodyText) as Record<string, unknown>) : {};
279
- } catch (error) {
280
- // Non-JSON body — keep empty and use raw text for error detail
281
- void error;
282
- }
283
- if (!res.ok) {
284
- const detail =
285
- typeof body.error === "string"
286
- ? body.error
287
- : typeof (body as { message?: unknown }).message === "string"
288
- ? (body as { message?: string }).message
289
- : typeof (body as { error_description?: unknown })
290
- .error_description === "string"
291
- ? (body as { error_description?: string }).error_description
292
- : bodyText.slice(0, 500);
293
- if (
294
- res.status === 400 &&
295
- typeof detail === "string" &&
296
- detail.includes("storage")
297
- ) {
298
- throw new Error(
299
- `Meta Files storage limit reached (100 GiB/team). Delete old files via DELETE /v1/files/{id}. Detail: ${detail}`,
300
- );
301
- }
302
- throw new Error(
303
- `Meta Files upload failed (HTTP ${res.status}): ${detail || res.statusText}`,
304
- );
305
- }
306
- const id =
307
- typeof body.id === "string"
308
- ? body.id
309
- : typeof (body as { file_id?: unknown }).file_id === "string"
310
- ? (body as { file_id: string }).file_id
311
- : undefined;
312
- if (!id)
313
- throw new Error(
314
- `Meta Files upload returned no id: ${bodyText.slice(0, 500)}`,
315
- );
316
- return {
317
- id,
318
- filename: (typeof body.filename === "string"
319
- ? body.filename
320
- : basename(filePath)) as string,
321
- bytes: typeof body.bytes === "number" ? body.bytes : data.byteLength,
322
- purpose: typeof body.purpose === "string" ? body.purpose : "user_data",
323
- status: typeof body.status === "string" ? body.status : "uploaded",
324
- expires_at:
325
- typeof body.expires_at === "number" ? body.expires_at : undefined,
326
- };
327
- }
328
-
329
- const ALLOWED_PURPOSES = ["user_data", "batch"] as const;
330
-
331
- async function listMetaFiles(
332
- apiKey: string,
333
- purpose = "user_data",
334
- ): Promise<unknown> {
335
- if (!(ALLOWED_PURPOSES as readonly string[]).includes(purpose)) {
336
- throw new Error(`Invalid purpose: ${purpose}`);
337
- }
338
- const options = { headers: { Authorization: `Bearer ${apiKey}` } };
339
- const res =
340
- purpose === "batch"
341
- ? await fetch("https://api.meta.ai/v1/files?purpose=batch", options)
342
- : await fetch("https://api.meta.ai/v1/files?purpose=user_data", options);
343
- const text = await res.text();
344
- if (!res.ok)
345
- throw new Error(
346
- `List files failed (HTTP ${res.status}): ${text.slice(0, 1000)}`,
347
- );
348
- try {
349
- return JSON.parse(text);
350
- } catch {
351
- return text;
352
84
  }
85
+ return key;
353
86
  }
354
87
 
355
- async function deleteMetaFile(
356
- apiKey: string,
357
- fileId: string,
358
- ): Promise<unknown> {
359
- if (!/^file-[a-zA-Z0-9_-]+$/.test(fileId))
360
- throw new Error(`Invalid file ID: ${fileId}`);
361
- let fileUrl: string;
88
+ async function safeGetMetaApiKey(ctx: ExtensionContext): Promise<string> {
362
89
  try {
363
- fileUrl = `${new URL(META_FILES_URL).toString()}/${encodeURIComponent(fileId)}`;
90
+ return await getMetaApiKey(ctx);
364
91
  } catch (error) {
365
- throw new Error(`Invalid file URL: ${String(error)}`);
366
- }
367
- if (!fileUrl.startsWith("https://api.meta.ai/"))
368
- throw new Error(`Unexpected file delete URL: ${fileUrl}`);
369
- const res = await fetch(fileUrl, {
370
- method: "DELETE",
371
- headers: { Authorization: `Bearer ${apiKey}` },
372
- });
373
- const text = await res.text();
374
- if (!res.ok)
375
- throw new Error(
376
- `Delete ${fileId} failed (HTTP ${res.status}): ${text.slice(0, 1000)}`,
377
- );
378
- try {
379
- return JSON.parse(text);
380
- } catch {
381
- return text;
382
- }
383
- }
384
-
385
- function responsesUrl(): string {
386
- return `${META_API_BASE_URL}/responses`;
387
- }
388
-
389
- interface ResponsesContentBlock {
390
- type: string;
391
- [key: string]: unknown;
392
- }
393
-
394
- function record(value: unknown): Record<string, unknown> | undefined {
395
- return value !== null && typeof value === "object"
396
- ? (value as Record<string, unknown>)
397
- : undefined;
398
- }
399
-
400
- function tokenCount(...values: unknown[]): number | undefined {
401
- for (const value of values) {
402
- const numeric =
403
- typeof value === "number"
404
- ? value
405
- : typeof value === "string" && value.trim()
406
- ? Number(value)
407
- : Number.NaN;
408
- if (Number.isFinite(numeric)) return Math.max(0, Math.trunc(numeric));
409
- }
410
- return undefined;
411
- }
412
-
413
- /** Convert a Meta Responses API usage block into Pi's nested-tool usage shape. */
414
- export function extractMetaResponseUsage(
415
- response: unknown,
416
- modelId = "muse-spark-1.2",
417
- ): Usage | undefined {
418
- const usage = record(record(response)?.usage);
419
- if (!usage) return undefined;
420
-
421
- const inputDetails = record(
422
- usage.input_tokens_details ?? usage.inputTokensDetails,
423
- );
424
- const outputDetails = record(
425
- usage.output_tokens_details ?? usage.outputTokensDetails,
426
- );
427
- const inputTotal = tokenCount(
428
- usage.input_tokens,
429
- usage.inputTokens,
430
- usage.prompt_tokens,
431
- usage.promptTokens,
432
- usage.input,
433
- );
434
- const output = tokenCount(
435
- usage.output_tokens,
436
- usage.outputTokens,
437
- usage.completion_tokens,
438
- usage.completionTokens,
439
- usage.output,
440
- );
441
- if (inputTotal === undefined && output === undefined) return undefined;
442
-
443
- const reportedInput = inputTotal ?? 0;
444
- const cacheRead = Math.min(
445
- reportedInput,
446
- tokenCount(inputDetails?.cached_tokens, inputDetails?.cachedTokens) ?? 0,
447
- );
448
- const cacheWrite = Math.min(
449
- reportedInput - cacheRead,
450
- tokenCount(
451
- inputDetails?.cache_write_tokens,
452
- inputDetails?.cacheWriteTokens,
453
- ) ?? 0,
454
- );
455
- const uncachedInput = Math.max(0, reportedInput - cacheRead - cacheWrite);
456
- const outputTokens = output ?? 0;
457
- const reasoning = Math.min(
458
- outputTokens,
459
- tokenCount(
460
- outputDetails?.reasoning_tokens,
461
- outputDetails?.reasoningTokens,
462
- ) ?? 0,
463
- );
464
- const totalTokens =
465
- tokenCount(usage.total_tokens, usage.totalTokens) ??
466
- uncachedInput + cacheRead + cacheWrite + outputTokens;
467
- const model = createMetaProviderConfig().models?.find(
468
- (candidate) => candidate.id === modelId,
469
- );
470
- const rates = model?.cost;
471
- const cost = {
472
- input: (uncachedInput * (rates?.input ?? 0)) / 1_000_000,
473
- output: (outputTokens * (rates?.output ?? 0)) / 1_000_000,
474
- cacheRead: (cacheRead * (rates?.cacheRead ?? 0)) / 1_000_000,
475
- cacheWrite: (cacheWrite * (rates?.cacheWrite ?? 0)) / 1_000_000,
476
- total: 0,
477
- };
478
- cost.total = cost.input + cost.output + cost.cacheRead + cost.cacheWrite;
479
-
480
- return {
481
- input: uncachedInput,
482
- output: outputTokens,
483
- cacheRead,
484
- cacheWrite,
485
- reasoning,
486
- totalTokens,
487
- cost,
488
- };
489
- }
490
-
491
- export function formatMetaResponseUsage(usage: Usage): string {
492
- const parts = [`${usage.input.toLocaleString()} input`];
493
- if (usage.cacheRead)
494
- parts.push(`${usage.cacheRead.toLocaleString()} cache read`);
495
- if (usage.cacheWrite)
496
- parts.push(`${usage.cacheWrite.toLocaleString()} cache write`);
497
- parts.push(`${usage.output.toLocaleString()} output`);
498
- if (usage.reasoning)
499
- parts.push(`${usage.reasoning.toLocaleString()} reasoning`);
500
- return `Video token usage: ${usage.totalTokens.toLocaleString()} total (${parts.join(", ")})`;
501
- }
502
-
503
- // Helper to extract output_text from Responses API response (covers streamed and non-streamed shapes)
504
- export function extractResponsesText(json: unknown): string {
505
- if (!json || typeof json !== "object") return "";
506
- const j = json as Record<string, unknown>;
507
- // Some Meta responses include an empty output_text alongside populated output blocks.
508
- if (typeof j.output_text === "string" && j.output_text.trim())
509
- return j.output_text;
510
- const output = j.output as unknown;
511
- if (Array.isArray(output)) {
512
- const texts: string[] = [];
513
- for (const item of output as Record<string, unknown>[]) {
514
- if (!item || typeof item !== "object" || item.type !== "message")
515
- continue;
516
- if (typeof item.content === "string" && item.content.trim())
517
- texts.push(item.content);
518
- if (Array.isArray(item.content)) {
519
- for (const c of item.content as Record<string, unknown>[]) {
520
- if (
521
- (c.type === "output_text" || c.type === "text") &&
522
- typeof c.text === "string" &&
523
- c.text.trim()
524
- )
525
- texts.push(c.text);
526
- if (
527
- c.type === "refusal" &&
528
- typeof c.refusal === "string" &&
529
- c.refusal.trim()
530
- )
531
- texts.push(c.refusal);
532
- }
533
- }
534
- }
535
- if (texts.length) return texts.join("\n\n");
536
- }
537
- if (typeof j.text === "string" && j.text.trim()) return j.text;
538
- return JSON.stringify(json, null, 2).slice(0, 8000);
539
- }
540
-
541
- async function callMetaResponses(
542
- apiKey: string,
543
- payload: Record<string, unknown>,
544
- signal?: AbortSignal,
545
- ): Promise<{ text: string; raw: unknown }> {
546
- // Force store:false for media payloads to avoid the ~20 MB
547
- // persistence limit (413 payload_too_large with store=true). Files API
548
- // is the correct path for large media per /docs/file-handling.
549
- payload.store = false;
550
- const doFetch = async (body: Record<string, unknown>) =>
551
- await fetch(responsesUrl(), {
552
- method: "POST",
553
- headers: {
554
- Authorization: `Bearer ${apiKey}`,
555
- "Content-Type": "application/json",
556
- "x-api-version": "1.0.0",
557
- },
558
- body: JSON.stringify(body),
559
- signal,
560
- });
561
- let res = await doFetch(payload);
562
- let text = await res.text();
563
- let json: unknown;
564
- try {
565
- json = text ? (JSON.parse(text) as unknown) : {};
566
- } catch {
567
- json = text;
568
- }
569
- if (!res.ok) {
570
- const detail =
571
- (json &&
572
- typeof json === "object" &&
573
- "error" in (json as Record<string, unknown>)
574
- ? JSON.stringify((json as Record<string, unknown>).error).slice(0, 1000)
575
- : typeof json === "object" &&
576
- json !== null &&
577
- "message" in (json as Record<string, unknown>)
578
- ? String((json as Record<string, unknown>).message).slice(0, 1000)
579
- : text.slice(0, 1000)) || res.statusText;
580
- // Auto-retry once with store:false if the error is the ~20 MB
581
- // persistence limit (413 with store=true) and we haven't already set it.
582
- const isStoreLimit =
583
- res.status === 413 &&
584
- String(detail).includes("store=true") &&
585
- payload.store !== false;
586
- if (isStoreLimit) {
587
- res = await doFetch({ ...payload, store: false });
588
- text = await res.text();
589
- try {
590
- json = text ? (JSON.parse(text) as unknown) : {};
591
- } catch {
592
- json = text;
593
- }
594
- if (res.ok) return { text: extractResponsesText(json), raw: json };
595
- }
596
- throw new Error(`Meta Responses failed (HTTP ${res.status}): ${detail}`);
92
+ if (String(error).includes("stale after session replacement"))
93
+ throw new Error("Session was replaced — please retry the command");
94
+ throw error;
597
95
  }
598
- return { text: extractResponsesText(json), raw: json };
599
96
  }
600
97
 
601
98
  function dataUrlForFile(path: string): {
@@ -613,317 +110,88 @@ function dataUrlForFile(path: string): {
613
110
  };
614
111
  }
615
112
 
616
- // ---------------------------------------------------------------------------
617
- // Before-provider-request rewrite (Option B)
618
- // Makes @video.mp4 / @audio.wav / @doc.pdf work transparently even though
619
- // pi-ai's UserMessage is still string | (TextContent | ImageContent)[].
620
- // We smuggle video/audio/pdf as ImageContent (data:video/..., data:audio/..., data:application/pdf...)
621
- // via the input handler or via read tool results; this hook rewrites the
622
- // resulting {type:"input_image", image_url:"data:video/..."} into the typed
623
- // input_video/input_audio/input_file block required by Meta before the request is sent.
624
- // Also handles https://... media URLs and large inline payloads via Files API.
625
- // Enable/disable via PI_META_NATIVE_ATTACHMENTS env (default: on).
626
- // ---------------------------------------------------------------------------
627
-
628
- function shouldEnableNativeRewrite(): boolean {
629
- const v =
630
- process.env.PI_META_NATIVE_ATTACHMENTS ?? process.env.PI_META_MEDIA_NATIVE;
631
- if (v === undefined) return true; // default on — set to "0"/"false" to disable
632
- return v !== "0" && v.toLowerCase() !== "false" && v.toLowerCase() !== "off";
633
- }
634
-
635
- function isInputImageBlock(b: Record<string, unknown>): boolean {
636
- return b.type === "input_image" && typeof b.image_url === "string";
637
- }
638
-
639
- function rewriteBlock(
640
- block: Record<string, unknown>,
641
- ): Record<string, unknown> | null {
642
- if (!isInputImageBlock(block)) return null;
643
- const url = block.image_url as string;
644
- const mime = mimeForDataUrl(url);
645
- // Also handle https URLs ending with .mp4/.wav/.mp3/.pdf
646
- const lowerUrl = url.toLowerCase();
647
- const isHttpsVideo =
648
- url.startsWith("https://") &&
649
- (lowerUrl.endsWith(".mp4") || lowerUrl.endsWith(".m4v"));
650
- const isHttpsAudio =
651
- url.startsWith("https://") &&
652
- (lowerUrl.endsWith(".mp3") || lowerUrl.endsWith(".wav"));
653
- const isHttpsPdf = url.startsWith("https://") && lowerUrl.endsWith(".pdf");
654
- const isHttpsImage =
655
- url.startsWith("https://") &&
656
- (lowerUrl.endsWith(".png") ||
657
- lowerUrl.endsWith(".jpg") ||
658
- lowerUrl.endsWith(".jpeg") ||
659
- lowerUrl.endsWith(".gif") ||
660
- lowerUrl.endsWith(".webp"));
113
+ type AnalysisSource = { source: string; label?: string };
661
114
 
662
- if (isHttpsImage || isImageMime(mime)) return null; // keep as image
115
+ const automaticExpiry = () => ({
116
+ anchor: "created_at" as const,
117
+ seconds: AUTOMATIC_UPLOAD_EXPIRY_SECONDS,
118
+ });
663
119
 
664
- if (isVideoMime(mime) || isHttpsVideo) {
665
- return mediaInputFromSource(url, "video.mp4", "video/mp4") as Record<
666
- string,
667
- unknown
668
- >;
669
- }
670
- if (isAudioMime(mime) || isHttpsAudio) {
671
- const effectiveMime =
672
- typeof mime === "string"
673
- ? mime
674
- : lowerUrl.endsWith(".wav")
675
- ? "audio/wav"
676
- : "audio/mpeg";
677
- return mediaInputFromSource(
678
- url,
679
- filenameForMime(effectiveMime, "audio.bin"),
680
- effectiveMime,
681
- ) as Record<string, unknown>;
682
- }
683
- if (isPdfMime(mime) || isHttpsPdf) {
684
- return mediaInputFromSource(
685
- url,
686
- "document.pdf",
687
- "application/pdf",
688
- ) as Record<string, unknown>;
689
- }
690
- return null;
120
+ function mediaToolError(tool: string, error: unknown): never {
121
+ if (error instanceof Error && error.name === "AbortError") throw error;
122
+ const message = error instanceof Error ? error.message : String(error);
123
+ throw new Error(`${tool} failed: ${message}`, { cause: error });
691
124
  }
692
125
 
693
- /** Rewrite smuggled media image_url blocks into Meta-supported typed blocks. */
694
- export function rewriteResponsesPayload(payload: unknown): {
695
- rewritten: boolean;
696
- payload: unknown;
697
- } {
698
- if (!payload || typeof payload !== "object")
699
- return { rewritten: false, payload };
700
- const p = payload as Record<string, unknown>;
701
- const input = p.input;
702
- if (!Array.isArray(input)) return { rewritten: false, payload };
703
- let rewritten = false;
704
- const newInput = input.map((entry) => {
705
- if (!entry || typeof entry !== "object") return entry;
706
- const e = entry as Record<string, unknown>;
707
- // case 1: user message {role:"user", content:[...]}
708
- if (Array.isArray(e.content)) {
709
- const newContent = (e.content as Record<string, unknown>[]).map(
710
- (block) => {
711
- const r = rewriteBlock(block as Record<string, unknown>);
712
- if (r) {
713
- rewritten = true;
714
- return r;
715
- }
716
- return block;
717
- },
718
- );
719
- if (newContent !== e.content) return { ...e, content: newContent };
720
- }
721
- // case 2: function_call_output {type:"function_call_output", output:[...]} where output contains input_image blocks
722
- if (e.type === "function_call_output" && Array.isArray(e.output)) {
723
- const newOutput = (e.output as Record<string, unknown>[]).map((block) => {
724
- // output blocks are like {type:"input_text", text} or {type:"input_image", image_url}
725
- if (
726
- block.type === "input_image" &&
727
- typeof block.image_url === "string"
728
- ) {
729
- const r = rewriteBlock(block as Record<string, unknown>);
730
- if (r) {
731
- rewritten = true;
732
- // tool outputs use same types as user content; return rewritten block
733
- return r;
734
- }
735
- }
736
- return block;
737
- });
738
- if (newOutput !== e.output) return { ...e, output: newOutput };
739
- }
740
- // case 3: custom_tool_call_output similar
741
- if (e.type === "custom_tool_call_output" && Array.isArray(e.output)) {
742
- const newOutput = (e.output as Record<string, unknown>[]).map((block) => {
743
- if (
744
- block.type === "input_image" &&
745
- typeof block.image_url === "string"
746
- ) {
747
- const r = rewriteBlock(block as Record<string, unknown>);
748
- if (r) {
749
- rewritten = true;
750
- return r;
751
- }
752
- }
753
- return block;
754
- });
755
- if (newOutput !== e.output) return { ...e, output: newOutput };
756
- }
757
- return e;
758
- });
759
- if (!rewritten) return { rewritten: false, payload };
760
- return { rewritten: true, payload: { ...p, input: newInput } };
126
+ function sourceLabel(source: AnalysisSource, index: number): string {
127
+ return source.label?.trim() || `Source ${index + 1}`;
761
128
  }
762
129
 
763
- async function uploadInlineMedia(
130
+ async function resolveGenericMediaSource(
764
131
  apiKey: string,
765
- base64: string,
766
- mime: string,
767
- filename: string,
768
- ): Promise<string> {
769
- const blob = new Blob([Buffer.from(base64, "base64")], { type: mime });
770
- const form = new FormData();
771
- form.append("file", blob, filename);
772
- form.append("purpose", "user_data");
773
- const res = await fetch(META_FILES_URL, {
774
- method: "POST",
775
- headers: { Authorization: `Bearer ${apiKey}` },
776
- body: form,
777
- });
778
- const text = await res.text();
779
- if (!res.ok) {
780
- throw new Error(
781
- `Files upload for large inline media failed: ${text.slice(0, 500)}`,
782
- );
132
+ cwd: string,
133
+ source: string,
134
+ signal?: AbortSignal,
135
+ ): Promise<ResponsesContentBlock> {
136
+ if (/^file-[a-zA-Z0-9_-]+$/.test(source)) {
137
+ return { type: "input_file", file_id: source };
783
138
  }
784
- let json: Record<string, unknown>;
785
- try {
786
- json = JSON.parse(text) as Record<string, unknown>;
787
- } catch (error) {
788
- throw new Error(`Files upload returned invalid JSON: ${String(error)}`);
139
+ if (isRemoteMediaSource(source)) {
140
+ return mediaInputFromSource(source);
141
+ }
142
+ const absolute = resolve(cwd, source.replace(/^@/, ""));
143
+ if (!existsSync(absolute)) throw new Error(`File not found: ${absolute}`);
144
+ const stat = statSync(absolute);
145
+ if (stat.size > FILES_API_LIMIT_BYTES) {
146
+ throw new Error(`File too large (1 GiB limit): ${source}`);
789
147
  }
790
- if (typeof json.id !== "string" || !json.id) {
791
- throw new Error(`Files upload returned no id: ${text.slice(0, 500)}`);
148
+ if (stat.size > STORE_SAFE_INLINE_BYTES) {
149
+ const upload = await uploadMetaFile(
150
+ apiKey,
151
+ absolute,
152
+ automaticExpiry(),
153
+ signal,
154
+ );
155
+ return { type: "input_file", file_id: upload.id };
792
156
  }
793
- return json.id;
157
+ const { dataUrl, mime } = dataUrlForFile(absolute);
158
+ return mediaInputFromSource(dataUrl, basename(absolute), mime);
794
159
  }
795
160
 
796
- // Promote inline media above the store-safe threshold to the Files API. Typed
797
- // input_video/input_audio blocks stay inline below the threshold.
798
- export async function maybeUploadLargeInlineBlocks(
799
- payload: unknown,
800
- apiKey: string | undefined,
801
- uploadThresholdBytes = STORE_SAFE_INLINE_BYTES,
802
- ): Promise<{ payload: unknown; uploaded: number }> {
803
- if (!apiKey) return { payload, uploaded: 0 };
804
- if (!payload || typeof payload !== "object") return { payload, uploaded: 0 };
805
- const p = payload as Record<string, unknown>;
806
- const input = p.input;
807
- if (!Array.isArray(input)) return { payload, uploaded: 0 };
808
- let uploaded = 0;
809
- let needsStoreFalse = false;
810
- const newInput: unknown[] = [];
811
-
812
- for (const entry of input as Record<string, unknown>[]) {
813
- if (!entry || typeof entry !== "object" || !Array.isArray(entry.content)) {
814
- newInput.push(entry);
815
- continue;
816
- }
817
- const content = entry.content as Record<string, unknown>[];
818
- const newContent: Record<string, unknown>[] = [];
819
- for (const block of content) {
820
- let inline:
821
- | { base64: string; mime: string; filename: string }
822
- | undefined;
823
- if (
824
- block.type === "input_video" &&
825
- typeof block.video_url === "string" &&
826
- block.video_url.startsWith("data:")
827
- ) {
828
- inline = {
829
- base64: base64FromDataUrl(block.video_url),
830
- mime: mimeForDataUrl(block.video_url) ?? "video/mp4",
831
- filename: "video.mp4",
832
- };
833
- } else if (block.type === "input_audio") {
834
- const audio = record(block.input_audio);
835
- if (typeof audio?.data === "string") {
836
- const format = audio.format === "wav" ? "wav" : "mp3";
837
- inline = {
838
- base64: audio.data.startsWith("data:")
839
- ? base64FromDataUrl(audio.data)
840
- : audio.data,
841
- mime: format === "wav" ? "audio/wav" : "audio/mpeg",
842
- filename: `audio.${format}`,
843
- };
844
- }
845
- } else if (
846
- block.type === "input_file" &&
847
- typeof block.file_data === "string" &&
848
- block.file_data.startsWith("data:")
849
- ) {
850
- const mime =
851
- mimeForDataUrl(block.file_data) ?? "application/octet-stream";
852
- inline = {
853
- base64: base64FromDataUrl(block.file_data),
854
- mime,
855
- filename:
856
- typeof block.filename === "string" && block.filename
857
- ? block.filename
858
- : filenameForMime(mime),
859
- };
860
- }
161
+ function shouldEnableNativeRewrite(): boolean {
162
+ const v =
163
+ process.env.PI_META_NATIVE_ATTACHMENTS ?? process.env.PI_META_MEDIA_NATIVE;
164
+ if (v === undefined) return true;
165
+ return v !== "0" && v.toLowerCase() !== "false" && v.toLowerCase() !== "off";
166
+ }
861
167
 
862
- if (inline) {
863
- needsStoreFalse = true;
864
- if (base64ByteLength(inline.base64) > uploadThresholdBytes) {
865
- const id = await uploadInlineMedia(
866
- apiKey,
867
- inline.base64,
868
- inline.mime,
869
- inline.filename,
870
- );
871
- uploaded++;
872
- newContent.push({ type: "input_file", file_id: id });
873
- continue;
874
- }
875
- }
876
- newContent.push(block);
877
- }
878
- newInput.push({ ...entry, content: newContent });
168
+ function rejectWrongMime(
169
+ kind: "video" | "audio",
170
+ source: string,
171
+ mime: string | undefined,
172
+ ): void {
173
+ if (!mime) return;
174
+ if (kind === "video" && !isVideoMime(mime)) {
175
+ throw new Error(`Unsupported video type for ${source}; use MP4`);
879
176
  }
880
-
881
- if (uploaded === 0) {
882
- return needsStoreFalse
883
- ? { payload: { ...p, store: false }, uploaded: 0 }
884
- : { payload, uploaded: 0 };
177
+ if (kind === "audio" && !isAudioMime(mime)) {
178
+ throw new Error(`Unsupported audio type for ${source}; use MP3 or WAV`);
885
179
  }
886
- return { payload: { ...p, input: newInput, store: false }, uploaded };
887
180
  }
888
181
 
889
- // ---------------------------------------------------------------------------
890
- // Main extension
891
- // ---------------------------------------------------------------------------
182
+ function isFileId(source: string): boolean {
183
+ return source.startsWith("file-");
184
+ }
892
185
 
893
186
  export default function metaMedia(pi: ExtensionAPI): void {
894
- // --- Before provider request rewrite (Option B transparent path) ---
895
187
  if (shouldEnableNativeRewrite()) {
896
188
  pi.on("before_provider_request", async (event, ctx) => {
897
189
  if (ctx.model?.provider !== META_PROVIDER_ID) return undefined;
898
190
  const original = event.payload as Record<string, unknown>;
899
191
  const { rewritten, payload: rewrittenPayload } =
900
192
  rewriteResponsesPayload(original);
901
- // Even without rewrite, large inline media in the original payload
902
- // (e.g. drag-dropped video) can hit the 413 store=true limit —
903
- // ensure store:false and try Files API upload.
904
193
  const payloadToCheck = rewritten ? rewrittenPayload : original;
905
- const hasMedia = (() => {
906
- if (!payloadToCheck || typeof payloadToCheck !== "object") return false;
907
- const input = (payloadToCheck as Record<string, unknown>).input;
908
- if (!Array.isArray(input)) return false;
909
- for (const e of input as Record<string, unknown>[]) {
910
- if (!e || typeof e !== "object" || !Array.isArray(e.content))
911
- continue;
912
- for (const b of e.content as Record<string, unknown>[]) {
913
- if (
914
- b.type === "input_file" ||
915
- b.type === "input_image" ||
916
- b.type === "input_video" ||
917
- b.type === "input_audio"
918
- )
919
- return true;
920
- }
921
- }
922
- return false;
923
- })();
924
- if (!rewritten && !hasMedia) return undefined;
925
- // Resolve the same stored Meta credential used by the active provider. This is
926
- // essential for @file drag-drop, where the key usually is not in process.env.
194
+ if (!rewritten && !payloadContainsMedia(payloadToCheck)) return undefined;
927
195
  let apiKey =
928
196
  process.env.MODEL_API_KEY ?? process.env.META_API_KEY ?? undefined;
929
197
  if (!apiKey) {
@@ -938,15 +206,18 @@ export default function metaMedia(pi: ExtensionAPI): void {
938
206
  if (apiKey) {
939
207
  try {
940
208
  const { payload: uploadedPayload } =
941
- await maybeUploadLargeInlineBlocks(payloadToCheck, apiKey);
209
+ await maybeUploadLargeInlineBlocks(
210
+ payloadToCheck,
211
+ apiKey,
212
+ STORE_SAFE_INLINE_BYTES,
213
+ ctx.signal,
214
+ );
942
215
  return uploadedPayload as unknown;
943
216
  } catch {
944
- // fall back without upload — at least force store:false
945
217
  const fallback = payloadToCheck as Record<string, unknown>;
946
218
  return { ...fallback, store: false } as unknown;
947
219
  }
948
220
  }
949
- // No apiKey available for upload — force store:false to avoid 413
950
221
  return {
951
222
  ...(payloadToCheck as Record<string, unknown>),
952
223
  store: false,
@@ -954,19 +225,10 @@ export default function metaMedia(pi: ExtensionAPI): void {
954
225
  });
955
226
  }
956
227
 
957
- // --- Input handler: let @video.mp4 / @audio.wav / @doc.pdf be pasted like @image.png ---
958
- // Pi's TUI already turns @image.png into ImageContent; for other types event.text still
959
- // contains the literal path and images is empty. We turn those paths into pseudo-ImageContent
960
- // so the before_provider_request hook can rewrite them.
961
228
  pi.on("input", async (event, ctx) => {
962
229
  if (!shouldEnableNativeRewrite()) return undefined;
963
- // Only Meta's request hook knows how to rewrite these pseudo-images into
964
- // typed video/audio/file blocks. Other providers (notably Codex) reject a
965
- // video/mp4 ImageContent before the Meta tools can be called, so leave the
966
- // path as text and let the active model invoke the direct Meta media tool.
967
230
  if (ctx.model?.provider !== META_PROVIDER_ID) return undefined;
968
231
  const text = event.text ?? "";
969
- // Match @/path/to/file.mp4, @./file.wav, /abs/file.pdf, ./file.mp3 — conservative.
970
232
  const filePattern =
971
233
  /(?:^|\s)@?((?:[~.]?\/[^\s]+\.(?:mp4|m4v|mp3|wav|pdf))|(?:[^\s]+\.(?:mp4|m4v|mp3|wav|pdf)))/gi;
972
234
  const matches = [...text.matchAll(filePattern)];
@@ -980,7 +242,6 @@ export default function metaMedia(pi: ExtensionAPI): void {
980
242
  for (const m of matches) {
981
243
  const rawPath = m[1];
982
244
  if (!rawPath) continue;
983
- // Resolve relative to cwd, expand ~, strip @
984
245
  const clean = rawPath.replace(/^@/, "");
985
246
  const expanded = clean.startsWith("~/")
986
247
  ? clean.replace("~", process.env.HOME ?? "")
@@ -998,10 +259,6 @@ export default function metaMedia(pi: ExtensionAPI): void {
998
259
  );
999
260
  continue;
1000
261
  }
1001
- // Files API is required for reliability — inline base64 hits the
1002
- // ~20 MB `store=true` persistence limit (24 MB → 413). We keep inline
1003
- // only for small files (<15 MB); larger files will be uploaded via
1004
- // Files API in before_provider_request or via /meta-video.
1005
262
  if (stat.size > STORE_SAFE_INLINE_BYTES) {
1006
263
  if (stat.size > INLINE_LIMIT_BYTES) {
1007
264
  safeNotify(
@@ -1011,16 +268,11 @@ export default function metaMedia(pi: ExtensionAPI): void {
1011
268
  );
1012
269
  continue;
1013
270
  }
1014
- // 15-50 MB: still smuggle inline but before_provider_request
1015
- // will promote to Files API (if apiKey available) and set store:false
1016
271
  }
1017
272
  try {
1018
273
  const { dataUrl } = dataUrlForFile(absolute);
1019
274
  const b64 = base64FromDataUrl(dataUrl);
1020
- // Smuggle as ImageContent with video/audio/pdf mime — before_provider_request will rewrite.
1021
- // pi-ai types allow any mimeType string, so video/mp4 is fine at runtime even though Model.input is cast.
1022
275
  newImages.push({ type: "image", data: b64, mimeType: mime });
1023
- // Remove the path from text so LLM doesn't see a stray filename; keep a placeholder for UX
1024
276
  newText = newText.replace(rawPath, `[attached ${basename(absolute)}]`);
1025
277
  touched = true;
1026
278
  } catch (e) {
@@ -1039,144 +291,84 @@ export default function metaMedia(pi: ExtensionAPI): void {
1039
291
  };
1040
292
  });
1041
293
 
1042
- // -----------------------------------------------------------------------
1043
- // Tools — the reliable path that works even when UserMessage types are limited.
1044
- // These call Meta's Responses API directly with the stored OAuth key, so they
1045
- // don't depend on pi-ai's ImageContent union. Use when @file paste isn't enough
1046
- // or for large/reused files.
1047
- // -----------------------------------------------------------------------
1048
-
1049
294
  pi.registerTool({
1050
295
  name: "meta_describe_video",
1051
296
  label: "Meta Video Describe",
1052
297
  description:
1053
- "Analyze an MP4 video with Muse Spark — summarizes visuals and transcribes embedded speech in one call. Provide a local path, public https URL, or existing file_id (from Files API). Handles Files API upload for large files and inline data URLs for small files. Use for: describe what happens in a clip, answer questions about footage, extract structured details. Video without audio (screen recordings) is also valid. Supports 50MB inline, 1GiB via Files API.",
1054
- parameters: Type.Object({
1055
- path: Type.Optional(
1056
- Type.String({
1057
- description:
1058
- "Local .mp4/.m4v file path (relative to cwd or absolute)",
1059
- }),
1060
- ),
1061
- url: Type.Optional(
1062
- Type.String({
1063
- description:
1064
- "Public https:// URL or data:video/mp4;base64,... URL (skips upload)",
1065
- }),
1066
- ),
1067
- file_id: Type.Optional(
1068
- Type.String({
1069
- description:
1070
- "Existing Files API id like file-842549258569145 (skips upload)",
1071
- }),
1072
- ),
1073
- prompt: Type.String({
1074
- description:
1075
- "What to ask about the video (e.g. 'Describe what happens', 'Transcribe speech and list action items')",
1076
- }),
1077
- model: Type.Optional(
1078
- Type.String({ description: "Muse model id, default muse-spark-1.2" }),
1079
- ),
1080
- max_output_tokens: Type.Optional(
1081
- Type.Number({
1082
- description:
1083
- "Default 4000 — reasoning shares budget, lower values can return empty with finish_reason:length",
1084
- }),
1085
- ),
1086
- }),
298
+ "Analyze an MP4 video with Muse Spark, including visuals and embedded speech. Accepts a local path, public HTTPS/data URL, or Meta file_id. Results are capped by max_chars; when truncated, the full text is saved to a temp file for continuation with read offset/limit.",
299
+ promptSnippet: "Inspect video and embedded audio when the active model cannot",
300
+ promptGuidelines: [
301
+ "Use meta_describe_video when the task depends on an MP4 video the active model cannot inspect directly.",
302
+ "Give meta_describe_video a task-specific prompt that asks for the exact visual, transcript, timestamp, or defect evidence needed next.",
303
+ "When a Meta media result is truncated, continue from its saved temp file with read using offset/limit.",
304
+ ],
305
+ parameters: Type.Object(
306
+ {
307
+ path: Type.Optional(Type.String({ description: "Local .mp4/.m4v path, relative to cwd or absolute" })),
308
+ url: Type.Optional(Type.String({ description: "Public HTTPS or data:video/mp4 URL" })),
309
+ file_id: Type.Optional(Type.String({ description: "Existing Meta Files API id" })),
310
+ prompt: Type.String({ description: "What evidence to extract from the video" }),
311
+ model: Type.Optional(Type.String({ description: "Muse model id, default muse-spark-1.2" })),
312
+ max_output_tokens: Type.Optional(Type.Integer({
313
+ minimum: MIN_MEDIA_MAX_OUTPUT_TOKENS,
314
+ maximum: MAX_MEDIA_MAX_OUTPUT_TOKENS,
315
+ description: `Muse generation budget, default ${DEFAULT_MEDIA_MAX_OUTPUT_TOKENS}`,
316
+ })),
317
+ max_chars: Type.Optional(Type.Integer({
318
+ minimum: 1_000,
319
+ maximum: MAX_MEDIA_MAX_CHARS,
320
+ description: `Maximum result characters returned inline, default ${DEFAULT_MEDIA_MAX_CHARS}`,
321
+ })),
322
+ },
323
+ { additionalProperties: false },
324
+ ),
1087
325
  async execute(toolCallId, params, signal, _onUpdate, ctx) {
1088
- const { path, url, file_id, prompt, model, max_output_tokens } =
1089
- params as {
1090
- path?: string;
1091
- url?: string;
1092
- file_id?: string;
1093
- prompt: string;
1094
- model?: string;
1095
- max_output_tokens?: number;
1096
- };
1097
- if (!path && !url && !file_id) {
1098
- return {
1099
- content: [
1100
- {
1101
- type: "text",
1102
- text: "Provide one of: path (local .mp4), url (https://...), or file_id (file-...).",
1103
- },
1104
- ],
1105
- details: { error: "missing source" },
1106
- };
326
+ const { path, url, file_id, prompt, model, max_output_tokens, max_chars } = params as {
327
+ path?: string; url?: string; file_id?: string; prompt: string; model?: string;
328
+ max_output_tokens?: number; max_chars?: number;
329
+ };
330
+ if ([path, url, file_id].filter(Boolean).length !== 1) {
331
+ throw new Error("meta_describe_video requires exactly one of path, url, or file_id");
1107
332
  }
1108
333
  try {
1109
- const apiKey = await safeGetMetaApiKey(
1110
- ctx as unknown as ExtensionContext,
1111
- );
334
+ const apiKey = await safeGetMetaApiKey(ctx as ExtensionContext);
1112
335
  let videoBlock: ResponsesContentBlock;
1113
- if (file_id) {
1114
- videoBlock = { type: "input_file", file_id };
1115
- } else if (url) {
336
+ if (file_id) videoBlock = { type: "input_file", file_id };
337
+ else if (url) {
338
+ rejectWrongMime("video", url, mimeForSource(url));
1116
339
  videoBlock = mediaInputFromSource(url, "video.mp4", "video/mp4");
1117
- } else if (path) {
1118
- const absolute = resolve(
1119
- (ctx as unknown as { cwd?: string }).cwd ?? process.cwd(),
1120
- path,
1121
- );
1122
- if (!existsSync(absolute))
1123
- throw new Error(`File not found: ${absolute}`);
340
+ } else {
341
+ const absolute = resolve(ctx.cwd, path as string);
342
+ if (!existsSync(absolute)) throw new Error(`File not found: ${absolute}`);
1124
343
  const stat = statSync(absolute);
1125
- if (stat.size > FILES_API_LIMIT_BYTES)
1126
- throw new Error(`File too large (1 GiB limit): ${path}`);
1127
- // Docs recommend Files API (upload → file_id) for video. Inline
1128
- // base64 also hits the ~20 MB `store=true` persistence limit
1129
- // (24 MB → HTTP 413) even when under the 50 MB inline cap.
1130
- // Use Files API for any video above the store-safe threshold
1131
- // and for all larger media; keep inline only for tiny files.
1132
- const useFilesApi = stat.size > STORE_SAFE_INLINE_BYTES;
1133
- if (useFilesApi) {
1134
- const up = await uploadMetaFile(apiKey, absolute);
1135
- videoBlock = { type: "input_file", file_id: up.id };
344
+ if (stat.size > FILES_API_LIMIT_BYTES) throw new Error(`File too large (1 GiB limit): ${path}`);
345
+ rejectWrongMime("video", path as string, mimeForPath(absolute));
346
+ if (stat.size > STORE_SAFE_INLINE_BYTES) {
347
+ const upload = await uploadMetaFile(apiKey, absolute, automaticExpiry(), signal);
348
+ videoBlock = { type: "input_file", file_id: upload.id };
1136
349
  } else {
1137
- const { dataUrl } = dataUrlForFile(absolute);
1138
- videoBlock = mediaInputFromSource(
1139
- dataUrl,
1140
- basename(absolute),
1141
- "video/mp4",
1142
- );
350
+ videoBlock = mediaInputFromSource(dataUrlForFile(absolute).dataUrl, basename(absolute), "video/mp4");
1143
351
  }
1144
- } else {
1145
- throw new Error("No source");
1146
352
  }
1147
353
  const selectedModel = model ?? "muse-spark-1.2";
1148
- const payload: Record<string, unknown> = {
354
+ const { text, raw } = await callMetaResponses(apiKey, {
1149
355
  model: selectedModel,
1150
356
  store: false,
1151
- input: [
1152
- {
1153
- type: "message",
1154
- role: "user",
1155
- content: [
1156
- { type: "input_text", text: prompt },
1157
- videoBlock as unknown as Record<string, unknown>,
1158
- ],
1159
- },
1160
- ],
1161
- max_output_tokens: Math.max(max_output_tokens ?? 8000, 4000),
1162
- };
1163
- const { text, raw } = await callMetaResponses(apiKey, payload, signal);
357
+ input: [{ type: "message", role: "user", content: [
358
+ { type: "input_text", text: prompt },
359
+ videoBlock as Record<string, unknown>,
360
+ ] }],
361
+ max_output_tokens: mediaMaxOutputTokens(max_output_tokens),
362
+ }, signal);
363
+ const output = await prepareMediaOutput({ text, response: raw, identity: toolCallId, maxChars: max_chars });
1164
364
  const usage = extractMetaResponseUsage(raw, selectedModel);
1165
365
  return {
1166
- content: [{ type: "text", text }],
1167
- details: { raw, videoBlock, usage },
366
+ content: [{ type: "text", text: output.text }],
367
+ details: { raw, videoBlock, ...output.details },
1168
368
  usage,
1169
369
  };
1170
- } catch (e) {
1171
- return {
1172
- content: [
1173
- {
1174
- type: "text",
1175
- text: `Video analysis failed: ${e instanceof Error ? e.message : String(e)}`,
1176
- },
1177
- ],
1178
- details: { error: String(e) },
1179
- };
370
+ } catch (error) {
371
+ mediaToolError("Video analysis", error);
1180
372
  }
1181
373
  },
1182
374
  });
@@ -1185,113 +377,80 @@ export default function metaMedia(pi: ExtensionAPI): void {
1185
377
  name: "meta_transcribe_audio",
1186
378
  label: "Meta Audio Transcribe",
1187
379
  description:
1188
- "Transcribe speech from a standalone audio file (wav, mp3) via Muse Spark. Provide local path, public URL, or file_id. Handles Files API for large files, inline base64 for small. Tip: always use max_output_tokens ≥4000 and stream for long clips to avoid empty length finish.",
1189
- parameters: Type.Object({
1190
- path: Type.Optional(Type.String({ description: "Local .wav/.mp3 path" })),
1191
- url: Type.Optional(
1192
- Type.String({ description: "Public https:// URL to audio" }),
1193
- ),
1194
- file_id: Type.Optional(Type.String({ description: "Existing file_id" })),
1195
- prompt: Type.Optional(
1196
- Type.String({
1197
- description:
1198
- "Custom prompt, default 'Transcribe this audio. Return only the transcript.'",
1199
- }),
1200
- ),
1201
- model: Type.Optional(
1202
- Type.String({ description: "Model id, default muse-spark-1.2" }),
1203
- ),
1204
- }),
380
+ "Transcribe MP3 or WAV audio with Muse Spark from a local path, public URL, or Meta file_id. Results are capped by max_chars; when truncated, the full transcript is saved to a temp file for continuation with read offset/limit.",
381
+ promptSnippet: "Transcribe audio when the active model cannot",
382
+ promptGuidelines: [
383
+ "Use meta_transcribe_audio when the task depends on MP3 or WAV speech the active model cannot inspect directly.",
384
+ "When a Meta media result is truncated, continue from its saved temp file with read using offset/limit.",
385
+ ],
386
+ parameters: Type.Object(
387
+ {
388
+ path: Type.Optional(Type.String({ description: "Local .wav/.mp3 path" })),
389
+ url: Type.Optional(Type.String({ description: "Public HTTPS or data audio URL" })),
390
+ file_id: Type.Optional(Type.String({ description: "Existing Meta Files API id" })),
391
+ prompt: Type.Optional(Type.String({ description: "Transcription or audio-analysis instruction" })),
392
+ model: Type.Optional(Type.String({ description: "Muse model id, default muse-spark-1.2" })),
393
+ max_output_tokens: Type.Optional(Type.Integer({
394
+ minimum: MIN_MEDIA_MAX_OUTPUT_TOKENS,
395
+ maximum: MAX_MEDIA_MAX_OUTPUT_TOKENS,
396
+ description: `Muse generation budget, default ${DEFAULT_MEDIA_MAX_OUTPUT_TOKENS}`,
397
+ })),
398
+ max_chars: Type.Optional(Type.Integer({
399
+ minimum: 1_000,
400
+ maximum: MAX_MEDIA_MAX_CHARS,
401
+ description: `Maximum transcript characters returned inline, default ${DEFAULT_MEDIA_MAX_CHARS}`,
402
+ })),
403
+ },
404
+ { additionalProperties: false },
405
+ ),
1205
406
  async execute(toolCallId, params, signal, _onUpdate, ctx) {
1206
- const { path, url, file_id, prompt, model } = params as {
1207
- path?: string;
1208
- url?: string;
1209
- file_id?: string;
1210
- prompt?: string;
1211
- model?: string;
407
+ const { path, url, file_id, prompt, model, max_output_tokens, max_chars } = params as {
408
+ path?: string; url?: string; file_id?: string; prompt?: string; model?: string;
409
+ max_output_tokens?: number; max_chars?: number;
1212
410
  };
1213
- if (!path && !url && !file_id) {
1214
- return {
1215
- content: [
1216
- { type: "text", text: "Provide path, url, or file_id for audio." },
1217
- ],
1218
- details: { error: "missing source" },
1219
- };
411
+ if ([path, url, file_id].filter(Boolean).length !== 1) {
412
+ throw new Error("meta_transcribe_audio requires exactly one of path, url, or file_id");
1220
413
  }
1221
414
  try {
1222
- const apiKey = await safeGetMetaApiKey(
1223
- ctx as unknown as ExtensionContext,
1224
- );
415
+ const apiKey = await safeGetMetaApiKey(ctx as ExtensionContext);
1225
416
  let audioBlock: ResponsesContentBlock;
1226
- if (file_id) {
1227
- audioBlock = { type: "input_file", file_id };
1228
- } else if (url) {
417
+ if (file_id) audioBlock = { type: "input_file", file_id };
418
+ else if (url) {
419
+ rejectWrongMime("audio", url, mimeForSource(url));
1229
420
  audioBlock = mediaInputFromSource(url, "audio.bin");
1230
- } else if (path) {
1231
- const absolute = resolve(
1232
- (ctx as unknown as { cwd?: string }).cwd ?? process.cwd(),
1233
- path,
1234
- );
1235
- if (!existsSync(absolute))
1236
- throw new Error(`File not found: ${absolute}`);
421
+ } else {
422
+ const absolute = resolve(ctx.cwd, path as string);
423
+ if (!existsSync(absolute)) throw new Error(`File not found: ${absolute}`);
1237
424
  const stat = statSync(absolute);
1238
- if (stat.size > FILES_API_LIMIT_BYTES)
1239
- throw new Error(`Audio too large (1 GiB limit)`);
425
+ if (stat.size > FILES_API_LIMIT_BYTES) throw new Error("Audio too large (1 GiB limit)");
1240
426
  const mime = mimeForPath(absolute);
1241
- if (!isAudioMime(mime) && mime !== undefined) {
1242
- throw new Error(
1243
- `Unsupported audio mime ${mime}, need audio/wav or audio/mpeg`,
1244
- );
1245
- }
427
+ rejectWrongMime("audio", path as string, mime);
1246
428
  if (stat.size > STORE_SAFE_INLINE_BYTES) {
1247
- const up = await uploadMetaFile(apiKey, absolute);
1248
- audioBlock = { type: "input_file", file_id: up.id };
429
+ const upload = await uploadMetaFile(apiKey, absolute, automaticExpiry(), signal);
430
+ audioBlock = { type: "input_file", file_id: upload.id };
1249
431
  } else {
1250
- const { dataUrl } = dataUrlForFile(absolute);
1251
- audioBlock = mediaInputFromSource(
1252
- dataUrl,
1253
- basename(absolute),
1254
- mime,
1255
- );
432
+ audioBlock = mediaInputFromSource(dataUrlForFile(absolute).dataUrl, basename(absolute), mime);
1256
433
  }
1257
- } else {
1258
- throw new Error("No audio source");
1259
434
  }
1260
- const payload: Record<string, unknown> = {
1261
- model: model ?? "muse-spark-1.2",
435
+ const selectedModel = model ?? "muse-spark-1.2";
436
+ const { text, raw } = await callMetaResponses(apiKey, {
437
+ model: selectedModel,
1262
438
  store: false,
1263
- input: [
1264
- {
1265
- type: "message",
1266
- role: "user",
1267
- content: [
1268
- {
1269
- type: "input_text",
1270
- text:
1271
- prompt ??
1272
- "Transcribe this audio. Return only the transcript.",
1273
- },
1274
- audioBlock as unknown as Record<string, unknown>,
1275
- ],
1276
- },
1277
- ],
1278
- max_output_tokens: 4000,
1279
- };
1280
- const { text, raw } = await callMetaResponses(apiKey, payload, signal);
1281
- return {
1282
- content: [{ type: "text", text }],
1283
- details: { raw, audioBlock },
1284
- };
1285
- } catch (e) {
439
+ input: [{ type: "message", role: "user", content: [
440
+ { type: "input_text", text: prompt ?? "Transcribe this audio. Return only the transcript." },
441
+ audioBlock as Record<string, unknown>,
442
+ ] }],
443
+ max_output_tokens: mediaMaxOutputTokens(max_output_tokens),
444
+ }, signal);
445
+ const output = await prepareMediaOutput({ text, response: raw, identity: toolCallId, maxChars: max_chars });
446
+ const usage = extractMetaResponseUsage(raw, selectedModel);
1286
447
  return {
1287
- content: [
1288
- {
1289
- type: "text",
1290
- text: `Audio transcription failed: ${e instanceof Error ? e.message : String(e)}`,
1291
- },
1292
- ],
1293
- details: { error: String(e) },
448
+ content: [{ type: "text", text: output.text }],
449
+ details: { raw, audioBlock, ...output.details },
450
+ usage,
1294
451
  };
452
+ } catch (error) {
453
+ mediaToolError("Audio transcription", error);
1295
454
  }
1296
455
  },
1297
456
  });
@@ -1300,63 +459,46 @@ export default function metaMedia(pi: ExtensionAPI): void {
1300
459
  name: "meta_upload_file",
1301
460
  label: "Meta Files Upload",
1302
461
  description:
1303
- "Upload a file to Meta Files API (POST /v1/files purpose=user_data) for reuse across video/image/pdf/audio requests. Returns file_id to use with input_file/file_id. Supports 1 GiB limit, 100 GiB/team storage. Set expires_after for auto-expiry (3600..2592000 seconds). Use for large or reused files; small one-offs can use inline data URLs.",
1304
- parameters: Type.Object({
1305
- path: Type.String({
1306
- description:
1307
- "Local file path to upload (mp4, wav, mp3, pdf, png, jpg, etc.)",
1308
- }),
1309
- expires_after_seconds: Type.Optional(
1310
- Type.Number({
1311
- description:
1312
- "Auto-expire after seconds (3600..2592000), default no expiry",
1313
- }),
1314
- ),
1315
- }),
1316
- async execute(toolCallId, params, signal, _onUpdate, ctx) {
1317
- const { path, expires_after_seconds } = params as {
1318
- path: string;
1319
- expires_after_seconds?: number;
462
+ "Upload a local file to Meta Files API for reuse. Uploads expire after seven days by default. Set expires_after_seconds to choose 3600..2592000 seconds, or retain=true to opt into no expiry.",
463
+ parameters: Type.Object(
464
+ {
465
+ path: Type.String({ description: "Local file path to upload" }),
466
+ expires_after_seconds: Type.Optional(Type.Integer({
467
+ minimum: MIN_UPLOAD_EXPIRY_SECONDS,
468
+ maximum: MAX_UPLOAD_EXPIRY_SECONDS,
469
+ description: `Expiry in seconds, default ${EXPLICIT_UPLOAD_EXPIRY_SECONDS} (7 days)`,
470
+ })),
471
+ retain: Type.Optional(Type.Boolean({ description: "Keep indefinitely instead of applying an expiry" })),
472
+ },
473
+ { additionalProperties: false },
474
+ ),
475
+ async execute(_toolCallId, params, signal, _onUpdate, ctx) {
476
+ const { path, expires_after_seconds, retain } = params as {
477
+ path: string; expires_after_seconds?: number; retain?: boolean;
1320
478
  };
479
+ if (retain && expires_after_seconds !== undefined) {
480
+ throw new Error("meta_upload_file accepts either retain=true or expires_after_seconds, not both");
481
+ }
1321
482
  try {
1322
- const apiKey = await safeGetMetaApiKey(
1323
- ctx as unknown as ExtensionContext,
1324
- );
1325
- const absolute = resolve(
1326
- (ctx as unknown as { cwd?: string }).cwd ?? process.cwd(),
1327
- path,
1328
- );
1329
- if (!existsSync(absolute))
1330
- throw new Error(`File not found: ${absolute}`);
1331
- let expiresAfter: { anchor: "created_at"; seconds: number } | undefined;
1332
- if (expires_after_seconds !== undefined) {
1333
- if (expires_after_seconds < 3600 || expires_after_seconds > 2592000)
1334
- throw new Error("expires_after_seconds must be 3600..2592000");
1335
- expiresAfter = {
1336
- anchor: "created_at",
1337
- seconds: expires_after_seconds,
1338
- };
1339
- }
1340
- const result = await uploadMetaFile(apiKey, absolute, expiresAfter);
1341
- return {
1342
- content: [
1343
- {
1344
- type: "text",
1345
- text: `Uploaded ${basename(absolute)} → ${result.id} (${result.bytes} bytes, status: ${result.status}${result.expires_at ? `, expires_at: ${result.expires_at}` : ""}). Use with {type:"input_file", file_id:"${result.id}"}.`,
1346
- },
1347
- ],
1348
- details: result,
483
+ const apiKey = await safeGetMetaApiKey(ctx as ExtensionContext);
484
+ const absolute = resolve(ctx.cwd, path);
485
+ if (!existsSync(absolute)) throw new Error(`File not found: ${absolute}`);
486
+ const expiresAfter = retain ? undefined : {
487
+ anchor: "created_at" as const,
488
+ seconds: expires_after_seconds ?? EXPLICIT_UPLOAD_EXPIRY_SECONDS,
1349
489
  };
1350
- } catch (e) {
490
+ const result = await uploadMetaFile(apiKey, absolute, expiresAfter, signal);
491
+ const retention = retain
492
+ ? "retained without expiry"
493
+ : result.expires_at
494
+ ? `expires_at ${result.expires_at}`
495
+ : `expires after ${expiresAfter?.seconds} seconds`;
1351
496
  return {
1352
- content: [
1353
- {
1354
- type: "text",
1355
- text: `Upload failed: ${e instanceof Error ? e.message : String(e)}`,
1356
- },
1357
- ],
1358
- details: { error: String(e) },
497
+ content: [{ type: "text", text: `Uploaded ${basename(absolute)} to ${result.id} (${result.bytes} bytes, ${retention}).` }],
498
+ details: result,
1359
499
  };
500
+ } catch (error) {
501
+ mediaToolError("File upload", error);
1360
502
  }
1361
503
  },
1362
504
  });
@@ -1365,93 +507,83 @@ export default function metaMedia(pi: ExtensionAPI): void {
1365
507
  name: "meta_analyze_file",
1366
508
  label: "Meta File Analyze",
1367
509
  description:
1368
- "Analyze any supported file (image, pdf, video, audio) via Muse Spark using the Files API or inline. PDFs: text first 100 pages + images first 50 pages (counts to 50-image budget). Images: up to 50/request. For generic 'summarize this document' use this; for video/audio with specific prompts use the dedicated video/audio tools.",
1369
- parameters: Type.Object({
1370
- path: Type.Optional(Type.String({ description: "Local file path" })),
1371
- url: Type.Optional(Type.String({ description: "Public https:// URL" })),
1372
- file_id: Type.Optional(Type.String({ description: "Existing file_id" })),
1373
- prompt: Type.String({ description: "What to ask about the file" }),
1374
- model: Type.Optional(
1375
- Type.String({ description: "Model id, default muse-spark-1.2" }),
1376
- ),
1377
- }),
510
+ "Analyze one or more images, PDFs, videos, or audio files with Muse Spark. Use path/url/file_id for one source, or ordered sources for comparisons. Images and PDF pages share Meta's 50-image budget; PDFs include text from the first 100 pages and images from the first 50 pages. Truncated text is saved for continuation with read offset/limit.",
511
+ promptSnippet: "Inspect images, PDFs, or mixed media when the active model cannot",
512
+ promptGuidelines: [
513
+ "Use meta_analyze_file when the task depends on an image, PDF, or generic media file the active model cannot inspect directly.",
514
+ "Give meta_analyze_file a task-specific prompt that asks for the exact visible text, layout, comparison, defect, or action evidence needed next.",
515
+ "When a Meta media result is truncated, continue from its saved temp file with read using offset/limit.",
516
+ ],
517
+ parameters: Type.Object(
518
+ {
519
+ path: Type.Optional(Type.String({ description: "Single local file path" })),
520
+ url: Type.Optional(Type.String({ description: "Single public HTTPS or data URL" })),
521
+ file_id: Type.Optional(Type.String({ description: "Single existing Meta file_id" })),
522
+ sources: Type.Optional(Type.Array(Type.Object({
523
+ source: Type.String({ description: "Local path, HTTPS/data URL, or Meta file_id" }),
524
+ label: Type.Optional(Type.String({ description: "Human-readable label such as Before or After" })),
525
+ }, { additionalProperties: false }), {
526
+ minItems: 1,
527
+ maxItems: MAX_ANALYSIS_SOURCES,
528
+ description: "Ordered sources for comparison or combined analysis",
529
+ })),
530
+ prompt: Type.String({ description: "What evidence to extract or compare" }),
531
+ model: Type.Optional(Type.String({ description: "Muse model id, default muse-spark-1.2" })),
532
+ max_output_tokens: Type.Optional(Type.Integer({
533
+ minimum: MIN_MEDIA_MAX_OUTPUT_TOKENS,
534
+ maximum: MAX_MEDIA_MAX_OUTPUT_TOKENS,
535
+ description: `Muse generation budget, default ${DEFAULT_MEDIA_MAX_OUTPUT_TOKENS}`,
536
+ })),
537
+ max_chars: Type.Optional(Type.Integer({
538
+ minimum: 1_000,
539
+ maximum: MAX_MEDIA_MAX_CHARS,
540
+ description: `Maximum result characters returned inline, default ${DEFAULT_MEDIA_MAX_CHARS}`,
541
+ })),
542
+ },
543
+ { additionalProperties: false },
544
+ ),
1378
545
  async execute(toolCallId, params, signal, _onUpdate, ctx) {
1379
- const { path, url, file_id, prompt, model } = params as {
1380
- path?: string;
1381
- url?: string;
1382
- file_id?: string;
1383
- prompt: string;
1384
- model?: string;
546
+ const { path, url, file_id, sources, prompt, model, max_output_tokens, max_chars } = params as {
547
+ path?: string; url?: string; file_id?: string; sources?: AnalysisSource[];
548
+ prompt: string; model?: string; max_output_tokens?: number; max_chars?: number;
1385
549
  };
1386
- if (!path && !url && !file_id) {
1387
- return {
1388
- content: [{ type: "text", text: "Provide path, url, or file_id." }],
1389
- details: { error: "missing source" },
1390
- };
550
+ const legacySources = [path, url, file_id].filter((value): value is string => Boolean(value));
551
+ if (legacySources.length > 1 || (legacySources.length === 1 && sources) || (legacySources.length === 0 && !sources)) {
552
+ throw new Error("meta_analyze_file requires exactly one path/url/file_id or a sources array");
553
+ }
554
+ if (sources && sources.length > MAX_ANALYSIS_SOURCES) {
555
+ throw new Error(`meta_analyze_file supports at most ${MAX_ANALYSIS_SOURCES} sources`);
1391
556
  }
1392
557
  try {
1393
- const apiKey = await safeGetMetaApiKey(
1394
- ctx as unknown as ExtensionContext,
1395
- );
1396
- let fileBlock: ResponsesContentBlock;
1397
- if (file_id) fileBlock = { type: "input_file", file_id };
1398
- else if (url) fileBlock = mediaInputFromSource(url);
1399
- else if (path) {
1400
- const absolute = resolve(
1401
- (ctx as unknown as { cwd?: string }).cwd ?? process.cwd(),
1402
- path,
1403
- );
1404
- if (!existsSync(absolute))
1405
- throw new Error(`File not found: ${absolute}`);
1406
- const stat = statSync(absolute);
1407
- if (stat.size > FILES_API_LIMIT_BYTES)
1408
- throw new Error(`File too large (1 GiB limit)`);
1409
- if (stat.size > STORE_SAFE_INLINE_BYTES) {
1410
- const up = await uploadMetaFile(apiKey, absolute);
1411
- fileBlock = { type: "input_file", file_id: up.id };
1412
- } else {
1413
- const { dataUrl, mime } = dataUrlForFile(absolute);
1414
- fileBlock = mediaInputFromSource(dataUrl, basename(absolute), mime);
558
+ const apiKey = await safeGetMetaApiKey(ctx as ExtensionContext);
559
+ const requestedSources: AnalysisSource[] = sources ?? [{ source: legacySources[0] as string }];
560
+ const content: ResponsesContentBlock[] = [{ type: "input_text", text: prompt }];
561
+ for (const [index, item] of requestedSources.entries()) {
562
+ if (requestedSources.length > 1 || item.label) {
563
+ content.push({ type: "input_text", text: `${sourceLabel(item, index)}:` });
1415
564
  }
1416
- } else throw new Error("No source");
1417
- const payload: Record<string, unknown> = {
1418
- model: model ?? "muse-spark-1.2",
565
+ content.push(await resolveGenericMediaSource(apiKey, ctx.cwd, item.source, signal));
566
+ }
567
+ const selectedModel = model ?? "muse-spark-1.2";
568
+ const { text, raw } = await callMetaResponses(apiKey, {
569
+ model: selectedModel,
1419
570
  store: false,
1420
- input: [
1421
- {
1422
- type: "message",
1423
- role: "user",
1424
- content: [
1425
- { type: "input_text", text: prompt },
1426
- fileBlock as unknown as Record<string, unknown>,
1427
- ],
1428
- },
1429
- ],
1430
- max_output_tokens: 4000,
1431
- };
1432
- const { text, raw } = await callMetaResponses(apiKey, payload, signal);
1433
- return {
1434
- content: [{ type: "text", text }],
1435
- details: { raw, fileBlock },
1436
- };
1437
- } catch (e) {
571
+ input: [{ type: "message", role: "user", content: content as Record<string, unknown>[] }],
572
+ max_output_tokens: mediaMaxOutputTokens(max_output_tokens),
573
+ }, signal);
574
+ const output = await prepareMediaOutput({ text, response: raw, identity: toolCallId, maxChars: max_chars });
575
+ const usage = extractMetaResponseUsage(raw, selectedModel);
1438
576
  return {
1439
- content: [
1440
- {
1441
- type: "text",
1442
- text: `File analysis failed: ${e instanceof Error ? e.message : String(e)}`,
1443
- },
1444
- ],
1445
- details: { error: String(e) },
577
+ content: [{ type: "text", text: output.text }],
578
+ details: { raw, sourceCount: requestedSources.length, ...output.details },
579
+ usage,
1446
580
  };
581
+ } catch (error) {
582
+ mediaToolError("File analysis", error);
1447
583
  }
1448
584
  },
1449
585
  });
1450
586
 
1451
- // -----------------------------------------------------------------------
1452
- // Slash commands (human-in-the-loop)
1453
- // -----------------------------------------------------------------------
1454
-
1455
587
  pi.registerCommand("meta-video", {
1456
588
  description:
1457
589
  "Analyze a video with Muse Spark (mp4). Usage: /meta-video <path|url|file_id> [prompt]",
@@ -1473,29 +605,18 @@ export default function metaMedia(pi: ExtensionAPI): void {
1473
605
  ? "Summarize what happens in this video in detail. Transcribe any speech verbatim. List key visual moments in 5 bullet points."
1474
606
  : trimmed.slice(firstSpace + 1).trim() ||
1475
607
  "Summarize what happens in this video in detail. Transcribe any speech verbatim.";
1476
- const isFileId = source.startsWith("file-");
1477
- const isUrl = source.startsWith("https://") || source.startsWith("data:");
1478
- const toolArgs: Record<string, unknown> = { prompt };
1479
- if (isFileId) toolArgs.file_id = source;
1480
- else if (isUrl) toolArgs.url = source;
1481
- else toolArgs.path = source;
1482
- safeNotify(ctx, `Analyzing video: ${source}`, "info");
1483
- // Dispatch via LLM tool path by sending a follow-up that triggers the tool
1484
- // For direct command path, call the API directly:
1485
608
  try {
1486
- const apiKey = await safeGetMetaApiKey(
1487
- ctx as unknown as ExtensionContext,
1488
- );
609
+ const apiKey = await safeGetMetaApiKey(ctx as unknown as ExtensionContext);
1489
610
  let block: ResponsesContentBlock;
1490
- if (isFileId) block = { type: "input_file", file_id: source };
1491
- else if (isUrl)
611
+ if (isFileId(source)) block = { type: "input_file", file_id: source };
612
+ else if (isRemoteMediaSource(source))
1492
613
  block = mediaInputFromSource(source, "video.mp4", "video/mp4");
1493
614
  else {
1494
615
  const abs = resolve(cwd, source);
1495
616
  if (!existsSync(abs)) throw new Error(`File not found: ${abs}`);
1496
617
  const stat = statSync(abs);
1497
618
  if (stat.size > STORE_SAFE_INLINE_BYTES) {
1498
- const up = await uploadMetaFile(apiKey, abs);
619
+ const up = await uploadMetaFile(apiKey, abs, automaticExpiry());
1499
620
  block = { type: "input_file", file_id: up.id };
1500
621
  } else {
1501
622
  const { dataUrl } = dataUrlForFile(abs);
@@ -1541,11 +662,7 @@ export default function metaMedia(pi: ExtensionAPI): void {
1541
662
  const cwd = ctx.cwd;
1542
663
  const trimmed = (args ?? "").trim();
1543
664
  if (!trimmed) {
1544
- safeNotify(
1545
- ctx,
1546
- "Usage: /meta-audio <path|url|file_id> [prompt]",
1547
- "info",
1548
- );
665
+ safeNotify(ctx, "Usage: /meta-audio <path|url|file_id> [prompt]", "info");
1549
666
  return;
1550
667
  }
1551
668
  const firstSpace = trimmed.indexOf(" ");
@@ -1555,21 +672,18 @@ export default function metaMedia(pi: ExtensionAPI): void {
1555
672
  ? "Transcribe this audio. Return only the transcript."
1556
673
  : trimmed.slice(firstSpace + 1).trim() ||
1557
674
  "Transcribe this audio. Return only the transcript.";
1558
- const isFileId = source.startsWith("file-");
1559
- const isUrl = source.startsWith("https://");
1560
675
  try {
1561
- const apiKey = await safeGetMetaApiKey(
1562
- ctx as unknown as ExtensionContext,
1563
- );
676
+ const apiKey = await safeGetMetaApiKey(ctx as unknown as ExtensionContext);
1564
677
  let block: ResponsesContentBlock;
1565
- if (isFileId) block = { type: "input_file", file_id: source };
1566
- else if (isUrl) block = mediaInputFromSource(source, "audio.bin");
678
+ if (isFileId(source)) block = { type: "input_file", file_id: source };
679
+ else if (isRemoteMediaSource(source))
680
+ block = mediaInputFromSource(source, "audio.bin");
1567
681
  else {
1568
682
  const abs = resolve(cwd, source);
1569
683
  if (!existsSync(abs)) throw new Error(`File not found: ${abs}`);
1570
684
  const stat = statSync(abs);
1571
685
  if (stat.size > STORE_SAFE_INLINE_BYTES) {
1572
- const up = await uploadMetaFile(apiKey, abs);
686
+ const up = await uploadMetaFile(apiKey, abs, automaticExpiry());
1573
687
  block = { type: "input_file", file_id: up.id };
1574
688
  } else {
1575
689
  const { dataUrl, mime } = dataUrlForFile(abs);
@@ -1609,11 +723,7 @@ export default function metaMedia(pi: ExtensionAPI): void {
1609
723
  const cwd = ctx.cwd;
1610
724
  const trimmed = (args ?? "").trim();
1611
725
  if (!trimmed) {
1612
- safeNotify(
1613
- ctx,
1614
- "Usage: /meta-file <path|url|file_id> <prompt>",
1615
- "info",
1616
- );
726
+ safeNotify(ctx, "Usage: /meta-file <path|url|file_id> <prompt>", "info");
1617
727
  return;
1618
728
  }
1619
729
  const firstSpace = trimmed.indexOf(" ");
@@ -1623,21 +733,18 @@ export default function metaMedia(pi: ExtensionAPI): void {
1623
733
  }
1624
734
  const source = trimmed.slice(0, firstSpace);
1625
735
  const prompt = trimmed.slice(firstSpace + 1).trim();
1626
- const isFileId = source.startsWith("file-");
1627
- const isUrl = source.startsWith("https://");
1628
736
  try {
1629
- const apiKey = await safeGetMetaApiKey(
1630
- ctx as unknown as ExtensionContext,
1631
- );
737
+ const apiKey = await safeGetMetaApiKey(ctx as unknown as ExtensionContext);
1632
738
  let block: ResponsesContentBlock;
1633
- if (isFileId) block = { type: "input_file", file_id: source };
1634
- else if (isUrl) block = mediaInputFromSource(source);
739
+ if (isFileId(source)) block = { type: "input_file", file_id: source };
740
+ else if (isRemoteMediaSource(source))
741
+ block = mediaInputFromSource(source);
1635
742
  else {
1636
743
  const abs = resolve(cwd, source);
1637
744
  if (!existsSync(abs)) throw new Error(`File not found: ${abs}`);
1638
745
  const stat = statSync(abs);
1639
746
  if (stat.size > STORE_SAFE_INLINE_BYTES) {
1640
- const up = await uploadMetaFile(apiKey, abs);
747
+ const up = await uploadMetaFile(apiKey, abs, automaticExpiry());
1641
748
  block = { type: "input_file", file_id: up.id };
1642
749
  } else {
1643
750
  const { dataUrl, mime } = dataUrlForFile(abs);