pi-meta-oauth 0.4.2 → 0.4.4

package/README.md

@@ -3,6 +3,7 @@
 # pi-meta-oauth
 
 <!-- markdownlint-disable-next-line MD013 -->
+![X (formerly Twitter) Follow](https://img.shields.io/twitter/follow/blockedpaths?style=flat&link=https%3A%2F%2Fx.com%2FBlockedPaths)
 [![npm version](https://img.shields.io/npm/v/pi-meta-oauth)](https://www.npmjs.com/package/pi-meta-oauth) [![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](https://opensource.org/licenses/MIT) [![GitHub stars](https://img.shields.io/github/stars/BlockedPath/pi-meta-oauth?style=social)](https://github.com/BlockedPath/pi-meta-oauth/stargazers) [![Last Commit](https://img.shields.io/github/last-commit/BlockedPath/pi-meta-oauth)](https://github.com/BlockedPath/pi-meta-oauth/commits/main) [![Issues](https://img.shields.io/github/issues/BlockedPath/pi-meta-oauth)](https://github.com/BlockedPath/pi-meta-oauth/issues) [![PRs Welcome](https://img.shields.io/badge/PRs-welcome-brightgreen.svg)](https://github.com/BlockedPath/pi-meta-oauth/pulls) [![Conventional Commits](https://img.shields.io/badge/Conventional%20Commits-1.0.0-yellow.svg)](https://www.conventionalcommits.org/en/v1.0.0/) [![TypeScript](https://img.shields.io/badge/TypeScript-Ready-blue)](https://www.typescriptlang.org/) [![CI](https://github.com/BlockedPath/pi-meta-oauth/actions/workflows/ci.yml/badge.svg)](https://github.com/BlockedPath/pi-meta-oauth/actions/workflows/ci.yml) [![Pi compatible](https://img.shields.io/badge/pi-Compatible-blueviolet)](https://pi.dev)
 
 Meta Model API OAuth and Muse-style voice input for [pi](https://pi.dev).
@@ -60,10 +61,25 @@ Audio is captured only between the two Alt+V presses and streamed as 16 kHz mono
 
 **Windows:** uses `winmm.dll` `waveIn*` via a C# helper. On first run pi tries to compile `extensions/voice/windows-audio.cs` with `csc.exe` (in-box .NET Framework) to `~/.pi/agent/bin/pi-meta-oauth-voice-v1.exe` for best performance. If no compiler is found it falls back to `extensions/voice/windows-audio.ps1` executed with `powershell.exe -ExecutionPolicy Bypass` (PowerShell 5.1 inbox, or `pwsh` 7 if present) which compiles the same capture code at runtime via `Add-Type`. No extra install is required. If recording fails, check **Settings → Privacy & security → Microphone** that access is allowed for desktop apps.
 
-Optional overrides:
+Optional overrides (the `PI_META_*` name takes precedence):
 
-- `PI_META_VOICE_ASR_ENDPOINT`
-- `PI_META_VOICE_ASR_MODEL`
+- `PI_META_VOICE_ASR_ENDPOINT` (legacy alias: `MUSE_VOICE_ASR_ENDPOINT`)
+- `PI_META_VOICE_ASR_MODEL` (legacy alias: `MUSE_VOICE_ASR_MODEL`)
+
+## Media tools
+
+Text-only Pi models can delegate media inspection to Muse Spark without switching models:
+
+- `meta_analyze_file` inspects one or more images, PDFs, audio files, or videos. Use ordered `sources` with optional labels for comparisons such as before and after screenshots.
+- `meta_describe_video` analyzes MP4 visuals and embedded audio.
+- `meta_transcribe_audio` transcribes MP3 or WAV speech.
+- `meta_upload_file` uploads a large or reusable file and returns a Meta `file_id`.
+
+Give each analysis tool a task-specific prompt that asks for the evidence the calling model needs next. Analysis tools default to an 8,000-token Muse generation budget and accept `max_output_tokens` from 4,000 to 32,000. Inline tool output defaults to 20,000 characters and accepts `max_chars` up to 50,000. If a result is truncated, the complete text is saved to a temporary file so the agent can continue with Pi's `read` tool using `offset` and `limit`.
+
+Automatic uploads made during analysis expire after 24 hours. Explicit `meta_upload_file` uploads expire after seven days by default; set `expires_after_seconds` to choose another supported duration or `retain: true` to keep a file without expiry.
+
+Media tool failures are reported as failed Pi tool calls, allowing the calling model to retry or report the blocker. Media analysis is model-generated observation, so verify consequential details when another source is available.
 
 ## Models
 
@@ -75,12 +91,33 @@ Fallback models use a 1,048,576-token context window, up to 256K output tokens,
 | `muse-spark-1.2-contributor` | 0.10 / 0.20 / 0.002 |
 | `muse-spark-1.1` | 1.25 / 4.25 / 0.15 |
 
+> **Contributor-model privacy:** the discounted contributor model allows Meta to use prompts and completions for product improvement, including training future Meta models. Use the standard `muse-spark-1.2` model if you do not want the contributor terms. See [Meta's model documentation](https://dev.meta.ai/docs/models).
+
 To scope Pi's model picker to Meta models:
 
 ```json
 { "enabledModels": ["meta/*"] }
 ```
 
+### Making context windows visible to external tools
+
+After a successful network model refresh, the extension persists the Meta
+catalog to `~/.pi/agent/models-store.json`. External usage tools such as
+[herdr-agent-usage](https://github.com/senna-lang/herdr-agent-usage) can then
+show a percentage (for example, `⛁ 2% (24k)`) instead of only an absolute token
+count. Pi writes the cache during interactive or RPC startup, and again after
+`/login meta`. `pi --list-models meta` lists currently available models but does
+not itself trigger a network catalog refresh. The cached catalog is also used
+when Pi starts without network access.
+
+The bundled fallback uses Meta's nominal `1,048,576`-token context window. A
+cached Muse Code 0.1.0/R708.1 catalog observed on 2026-08-06 reported a lower
+effective limit of `1,007,997` for `muse-spark-1.2` and
+`muse-spark-1.2-contributor`. If you need percentages to match that specific
+Muse snapshot, you can still set `contextWindow: 1007997` for those models in
+`~/.pi/agent/models.json`; model overrides take precedence over the persisted
+catalog.
+
 ## Verify
 
 ```bash

package/extensions/media/files.ts

@@ -0,0 +1,131 @@
+import { basename } from "node:path";
+import { readFile } from "node:fs/promises";
+import { statSync } from "node:fs";
+import { META_FILES_URL } from "../meta.ts";
+import {
+	AUTOMATIC_UPLOAD_EXPIRY_SECONDS,
+	FILES_API_LIMIT_BYTES,
+} from "./limits.ts";
+import { filenameForMime, mimeForPath } from "./mime.ts";
+
+export interface UploadResult {
+	id: string;
+	filename: string;
+	bytes: number;
+	purpose: string;
+	status: string;
+	expires_at?: number;
+}
+
+function uploadErrorDetail(body: Record<string, unknown>, bodyText: string): string {
+	if (typeof body.error === "string") return body.error;
+	if (typeof body.message === "string") return body.message;
+	if (typeof body.error_description === "string") return body.error_description;
+	return bodyText.slice(0, 500);
+}
+
+export async function uploadMetaFile(
+	apiKey: string,
+	filePath: string,
+	expiresAfter?: { anchor: "created_at"; seconds: number },
+	signal?: AbortSignal,
+): Promise<UploadResult> {
+	const size = statSync(filePath).size;
+	if (size > FILES_API_LIMIT_BYTES) {
+		throw new Error(
+			`File too large for Files API (${size} bytes > 1 GiB)`,
+		);
+	}
+	const data = await readFile(filePath, { signal });
+	const mime = mimeForPath(filePath) ?? "application/octet-stream";
+	const blob = new Blob([data], { type: mime });
+	const form = new FormData();
+	form.append("file", blob, basename(filePath));
+	form.append("purpose", "user_data");
+	if (expiresAfter) {
+		form.append("expires_after[anchor]", expiresAfter.anchor);
+		form.append("expires_after[seconds]", String(expiresAfter.seconds));
+	}
+	const res = await fetch(META_FILES_URL, {
+		method: "POST",
+		headers: { Authorization: `Bearer ${apiKey}` },
+		body: form,
+		signal,
+	});
+	const bodyText = await res.text();
+	let body: Record<string, unknown> = {};
+	try {
+		body = bodyText ? (JSON.parse(bodyText) as Record<string, unknown>) : {};
+	} catch {
+		// Non-JSON body — keep empty and use raw text for error detail
+	}
+	if (!res.ok) {
+		const detail = uploadErrorDetail(body, bodyText);
+		if (res.status === 400 && detail.includes("storage")) {
+			throw new Error(
+				`Meta Files storage limit reached (100 GiB/team). Delete old files via DELETE /v1/files/{id}. Detail: ${detail}`,
+			);
+		}
+		throw new Error(
+			`Meta Files upload failed (HTTP ${res.status}): ${detail || res.statusText}`,
+		);
+	}
+	const id =
+		typeof body.id === "string"
+			? body.id
+			: typeof body.file_id === "string"
+				? body.file_id
+				: undefined;
+	if (!id) {
+		throw new Error(
+			`Meta Files upload returned no id: ${bodyText.slice(0, 500)}`,
+		);
+	}
+	return {
+		id,
+		filename:
+			typeof body.filename === "string" ? body.filename : basename(filePath),
+		bytes: typeof body.bytes === "number" ? body.bytes : data.byteLength,
+		purpose: typeof body.purpose === "string" ? body.purpose : "user_data",
+		status: typeof body.status === "string" ? body.status : "uploaded",
+		expires_at:
+			typeof body.expires_at === "number" ? body.expires_at : undefined,
+	};
+}
+
+export async function uploadInlineMedia(
+	apiKey: string,
+	base64: string,
+	mime: string,
+	filename: string,
+	signal?: AbortSignal,
+): Promise<string> {
+	const blob = new Blob([Buffer.from(base64, "base64")], { type: mime });
+	const form = new FormData();
+	form.append("file", blob, filename || filenameForMime(mime));
+	form.append("purpose", "user_data");
+	form.append("expires_after[anchor]", "created_at");
+	form.append("expires_after[seconds]", String(AUTOMATIC_UPLOAD_EXPIRY_SECONDS));
+	const res = await fetch(META_FILES_URL, {
+		method: "POST",
+		headers: { Authorization: `Bearer ${apiKey}` },
+		body: form,
+		signal,
+	});
+	const text = await res.text();
+	if (!res.ok) {
+		throw new Error(
+			`Files upload for large inline media failed: ${text.slice(0, 500)}`,
+		);
+	}
+	let json: Record<string, unknown>;
+	try {
+		json = JSON.parse(text) as Record<string, unknown>;
+	} catch (error) {
+		throw new Error(`Files upload returned invalid JSON: ${String(error)}`);
+	}
+	if (typeof json.id !== "string" || !json.id) {
+		throw new Error(`Files upload returned no id: ${text.slice(0, 500)}`);
+	}
+	return json.id;
+}

package/extensions/media/limits.ts

@@ -0,0 +1,12 @@
+export const INLINE_LIMIT_BYTES = 50_000_000;
+export const FILES_API_LIMIT_BYTES = 1_073_741_824;
+// Meta Responses `store=true` (default) has ~20 MB persistence limit even when
+// inline limit is 50 MB — a 24 MB base64 payload triggers HTTP 413
+// "payload_too_large … with `store=true`". Keep inline only for small files
+// and force Files API or `store:false` above this threshold.
+export const STORE_SAFE_INLINE_BYTES = 15_000_000;
+export const AUTOMATIC_UPLOAD_EXPIRY_SECONDS = 24 * 60 * 60;
+export const EXPLICIT_UPLOAD_EXPIRY_SECONDS = 7 * 24 * 60 * 60;
+export const MIN_UPLOAD_EXPIRY_SECONDS = 3_600;
+export const MAX_UPLOAD_EXPIRY_SECONDS = 2_592_000;
+export const MAX_ANALYSIS_SOURCES = 50;

package/extensions/media/mime.ts

@@ -0,0 +1,178 @@
+import { extname } from "node:path";
+
+export interface ResponsesContentBlock {
+	type: string;
+	[key: string]: unknown;
+}
+
+type SupportedMime =
+	| "video/mp4"
+	| "audio/mpeg"
+	| "audio/wav"
+	| "application/pdf"
+	| "image/png"
+	| "image/jpeg"
+	| "image/gif"
+	| "image/webp"
+	| "image/x-icon"
+	| "text/plain"
+	| "application/json"
+	| "application/jsonl";
+
+const EXT_TO_MIME: Record<string, SupportedMime> = {
+	".mp4": "video/mp4",
+	".m4v": "video/mp4",
+	".mp3": "audio/mpeg",
+	".wav": "audio/wav",
+	".pdf": "application/pdf",
+	".png": "image/png",
+	".jpg": "image/jpeg",
+	".jpeg": "image/jpeg",
+	".gif": "image/gif",
+	".webp": "image/webp",
+	".ico": "image/x-icon",
+	".txt": "text/plain",
+	".json": "application/json",
+	".jsonl": "application/jsonl",
+};
+
+export function mimeForPath(path: string): SupportedMime | undefined {
+	const ext = extname(path).toLowerCase();
+	return EXT_TO_MIME[ext];
+}
+
+export function mimeForDataUrl(dataUrl: string): string | undefined {
+	const m = dataUrl.match(/^data:([^;,\s]+)[;,]/);
+	return m?.[1]?.toLowerCase();
+}
+
+export function isVideoMime(m: string | undefined): boolean {
+	return m === "video/mp4";
+}
+
+export function isAudioMime(m: string | undefined): boolean {
+	return (
+		m === "audio/mpeg" ||
+		m === "audio/mp3" ||
+		m === "audio/wav" ||
+		m === "audio/x-wav"
+	);
+}
+
+export function isPdfMime(m: string | undefined): boolean {
+	return m === "application/pdf";
+}
+
+export function isImageMime(m: string | undefined): boolean {
+	return typeof m === "string" && m.startsWith("image/");
+}
+
+export function filenameForMime(
+	mime: string | undefined,
+	fallback = "file.bin",
+): string {
+	switch (mime) {
+		case "video/mp4":
+			return "video.mp4";
+		case "audio/mpeg":
+		case "audio/mp3":
+			return "audio.mp3";
+		case "audio/wav":
+		case "audio/x-wav":
+			return "audio.wav";
+		case "application/pdf":
+			return "document.pdf";
+		default:
+			return fallback;
+	}
+}
+
+export function audioFormatForMime(mime: string | undefined): "wav" | "mp3" {
+	return mime === "audio/wav" || mime === "audio/x-wav" ? "wav" : "mp3";
+}
+
+export function base64FromDataUrl(dataUrl: string): string {
+	const comma = dataUrl.indexOf(",");
+	return comma === -1 ? "" : dataUrl.slice(comma + 1);
+}
+
+export function base64ByteLength(base64: string): number {
+	const normalized = base64.replace(/\s/g, "");
+	if (!normalized) return 0;
+	const padding = normalized.endsWith("==")
+		? 2
+		: normalized.endsWith("=")
+			? 1
+			: 0;
+	return Math.max(0, Math.floor((normalized.length * 3) / 4) - padding);
+}
+
+export function isRemoteMediaSource(source: string): boolean {
+	return source.startsWith("https://") || source.startsWith("data:");
+}
+
+export function mediaPathFromUrl(url: string): string {
+	if (url.startsWith("data:")) return url;
+	try {
+		return new URL(url).pathname;
+	} catch {
+		const withoutHash = url.split("#")[0] ?? url;
+		return withoutHash.split("?")[0] ?? withoutHash;
+	}
+}
+
+export function mimeForSource(source: string): string | undefined {
+	if (source.startsWith("data:")) return mimeForDataUrl(source);
+	if (source.startsWith("https://")) {
+		try {
+			return mimeForPath(new URL(source).pathname);
+		} catch {
+			return undefined;
+		}
+	}
+	return mimeForPath(source);
+}
+
+/** Build a generic Meta Responses input_file block. */
+export function inputFileFromSource(
+	source: string,
+	filename?: string,
+): ResponsesContentBlock {
+	if (source.startsWith("https://")) {
+		return { type: "input_file", file_url: source };
+	}
+	if (source.startsWith("data:")) {
+		const mime = mimeForDataUrl(source);
+		return {
+			type: "input_file",
+			file_data: source,
+			filename: filename ?? filenameForMime(mime),
+		};
+	}
+	return { type: "input_file", file_id: source };
+}
+
+/** Build the typed content block required by Meta for each media kind. */
+export function mediaInputFromSource(
+	source: string,
+	filename?: string,
+	expectedMime?: string,
+): ResponsesContentBlock {
+	if (!source.startsWith("data:") && !source.startsWith("https://")) {
+		return { type: "input_file", file_id: source };
+	}
+	const mime = expectedMime ?? mimeForSource(source);
+	if (isVideoMime(mime)) {
+		return { type: "input_video", video_url: source };
+	}
+	if (isAudioMime(mime) && source.startsWith("data:")) {
+		return {
+			type: "input_audio",
+			input_audio: {
+				data: base64FromDataUrl(source),
+				format: audioFormatForMime(mime),
+			},
+		};
+	}
+	return inputFileFromSource(source, filename);
+}

package/extensions/media/payload.ts

@@ -0,0 +1,303 @@
+import { STORE_SAFE_INLINE_BYTES } from "./limits.ts";
+import { uploadInlineMedia } from "./files.ts";
+import {
+	base64ByteLength,
+	base64FromDataUrl,
+	filenameForMime,
+	isAudioMime,
+	isImageMime,
+	isPdfMime,
+	isVideoMime,
+	mediaInputFromSource,
+	mediaPathFromUrl,
+	mimeForDataUrl,
+} from "./mime.ts";
+
+function record(value: unknown): Record<string, unknown> | undefined {
+	return value !== null && typeof value === "object"
+		? (value as Record<string, unknown>)
+		: undefined;
+}
+
+function isInputImageBlock(b: Record<string, unknown>): boolean {
+	return b.type === "input_image" && typeof b.image_url === "string";
+}
+
+function rewriteBlock(
+	block: Record<string, unknown>,
+): Record<string, unknown> | null {
+	if (!isInputImageBlock(block)) return null;
+	const url = block.image_url as string;
+	const mime = mimeForDataUrl(url);
+	const lowerPath = mediaPathFromUrl(url).toLowerCase();
+	const isHttps = url.startsWith("https://");
+	const isHttpsVideo =
+		isHttps && (lowerPath.endsWith(".mp4") || lowerPath.endsWith(".m4v"));
+	const isHttpsAudio =
+		isHttps && (lowerPath.endsWith(".mp3") || lowerPath.endsWith(".wav"));
+	const isHttpsPdf = isHttps && lowerPath.endsWith(".pdf");
+	const isHttpsImage =
+		isHttps &&
+		(lowerPath.endsWith(".png") ||
+			lowerPath.endsWith(".jpg") ||
+			lowerPath.endsWith(".jpeg") ||
+			lowerPath.endsWith(".gif") ||
+			lowerPath.endsWith(".webp"));
+
+	if (isHttpsImage || isImageMime(mime)) return null;
+
+	if (isVideoMime(mime) || isHttpsVideo) {
+		return mediaInputFromSource(url, "video.mp4", "video/mp4") as Record<
+			string,
+			unknown
+		>;
+	}
+	if (isAudioMime(mime) || isHttpsAudio) {
+		const effectiveMime =
+			typeof mime === "string"
+				? mime
+				: lowerPath.endsWith(".wav")
+					? "audio/wav"
+					: "audio/mpeg";
+		return mediaInputFromSource(
+			url,
+			filenameForMime(effectiveMime, "audio.bin"),
+			effectiveMime,
+		) as Record<string, unknown>;
+	}
+	if (isPdfMime(mime) || isHttpsPdf) {
+		return mediaInputFromSource(
+			url,
+			"document.pdf",
+			"application/pdf",
+		) as Record<string, unknown>;
+	}
+	return null;
+}
+
+function rewriteBlocks(
+	blocks: Record<string, unknown>[],
+	onlyImageUrls = false,
+): { blocks: Record<string, unknown>[]; rewritten: boolean } {
+	let rewritten = false;
+	const next = blocks.map((block) => {
+		if (
+			onlyImageUrls &&
+			!(block.type === "input_image" && typeof block.image_url === "string")
+		) {
+			return block;
+		}
+		const replacement = rewriteBlock(block);
+		if (replacement) {
+			rewritten = true;
+			return replacement;
+		}
+		return block;
+	});
+	return { blocks: next, rewritten };
+}
+
+/** Rewrite smuggled media image_url blocks into Meta-supported typed blocks. */
+export function rewriteResponsesPayload(payload: unknown): {
+	rewritten: boolean;
+	payload: unknown;
+} {
+	if (!payload || typeof payload !== "object")
+		return { rewritten: false, payload };
+	const p = payload as Record<string, unknown>;
+	const input = p.input;
+	if (!Array.isArray(input)) return { rewritten: false, payload };
+	let rewritten = false;
+	const newInput = input.map((entry) => {
+		if (!entry || typeof entry !== "object") return entry;
+		const e = { ...(entry as Record<string, unknown>) };
+		if (Array.isArray(e.content)) {
+			const result = rewriteBlocks(e.content as Record<string, unknown>[]);
+			if (result.rewritten) {
+				rewritten = true;
+				e.content = result.blocks;
+			}
+		}
+		if (
+			(e.type === "function_call_output" ||
+				e.type === "custom_tool_call_output") &&
+			Array.isArray(e.output)
+		) {
+			const result = rewriteBlocks(
+				e.output as Record<string, unknown>[],
+				true,
+			);
+			if (result.rewritten) {
+				rewritten = true;
+				e.output = result.blocks;
+			}
+		}
+		return e;
+	});
+	if (!rewritten) return { rewritten: false, payload };
+	return { rewritten: true, payload: { ...p, input: newInput } };
+}
+
+function inlineFromBlock(
+	block: Record<string, unknown>,
+): { base64: string; mime: string; filename: string } | undefined {
+	if (
+		block.type === "input_video" &&
+		typeof block.video_url === "string" &&
+		block.video_url.startsWith("data:")
+	) {
+		return {
+			base64: base64FromDataUrl(block.video_url),
+			mime: mimeForDataUrl(block.video_url) ?? "video/mp4",
+			filename: "video.mp4",
+		};
+	}
+	if (block.type === "input_audio") {
+		const audio = record(block.input_audio);
+		if (typeof audio?.data === "string") {
+			const format = audio.format === "wav" ? "wav" : "mp3";
+			return {
+				base64: audio.data.startsWith("data:")
+					? base64FromDataUrl(audio.data)
+					: audio.data,
+				mime: format === "wav" ? "audio/wav" : "audio/mpeg",
+				filename: `audio.${format}`,
+			};
+		}
+	}
+	if (
+		block.type === "input_file" &&
+		typeof block.file_data === "string" &&
+		block.file_data.startsWith("data:")
+	) {
+		const mime = mimeForDataUrl(block.file_data) ?? "application/octet-stream";
+		return {
+			base64: base64FromDataUrl(block.file_data),
+			mime,
+			filename:
+				typeof block.filename === "string" && block.filename
+					? block.filename
+					: filenameForMime(mime),
+		};
+	}
+	if (
+		block.type === "input_image" &&
+		typeof block.image_url === "string" &&
+		block.image_url.startsWith("data:")
+	) {
+		const mime = mimeForDataUrl(block.image_url) ?? "image/png";
+		return {
+			base64: base64FromDataUrl(block.image_url),
+			mime,
+			filename: filenameForMime(mime, "image.bin"),
+		};
+	}
+	return undefined;
+}
+
+function isMediaBlock(block: Record<string, unknown>): boolean {
+	return (
+		block.type === "input_file" ||
+		block.type === "input_image" ||
+		block.type === "input_video" ||
+		block.type === "input_audio"
+	);
+}
+
+export function payloadContainsMedia(payload: unknown): boolean {
+	if (!payload || typeof payload !== "object") return false;
+	const input = (payload as Record<string, unknown>).input;
+	if (!Array.isArray(input)) return false;
+	for (const entry of input as Record<string, unknown>[]) {
+		if (!entry || typeof entry !== "object") continue;
+		const groups = [entry.content, entry.output];
+		for (const group of groups) {
+			if (!Array.isArray(group)) continue;
+			for (const block of group as Record<string, unknown>[]) {
+				if (block && typeof block === "object" && isMediaBlock(block))
+					return true;
+			}
+		}
+	}
+	return false;
+}
+
+async function promoteBlocks(
+	blocks: Record<string, unknown>[],
+	apiKey: string,
+	uploadThresholdBytes: number,
+	signal?: AbortSignal,
+): Promise<{
+	blocks: Record<string, unknown>[];
+	uploaded: number;
+	needsStoreFalse: boolean;
+}> {
+	let uploaded = 0;
+	let needsStoreFalse = false;
+	const next: Record<string, unknown>[] = [];
+	for (const block of blocks) {
+		const inline = inlineFromBlock(block);
+		if (inline) {
+			needsStoreFalse = true;
+			if (base64ByteLength(inline.base64) > uploadThresholdBytes) {
+				const id = await uploadInlineMedia(
+					apiKey,
+					inline.base64,
+					inline.mime,
+					inline.filename,
+					signal,
+				);
+				uploaded += 1;
+				next.push({ type: "input_file", file_id: id });
+				continue;
+			}
+		} else if (isMediaBlock(block)) {
+			needsStoreFalse = true;
+		}
+		next.push(block);
+	}
+	return { blocks: next, uploaded, needsStoreFalse };
+}
+
+export async function maybeUploadLargeInlineBlocks(
+	payload: unknown,
+	apiKey: string | undefined,
+	uploadThresholdBytes = STORE_SAFE_INLINE_BYTES,
+	signal?: AbortSignal,
+): Promise<{ payload: unknown; uploaded: number }> {
+	if (!apiKey) return { payload, uploaded: 0 };
+	if (!payload || typeof payload !== "object") return { payload, uploaded: 0 };
+	const p = payload as Record<string, unknown>;
+	const input = p.input;
+	if (!Array.isArray(input)) return { payload, uploaded: 0 };
+	let uploaded = 0;
+	let needsStoreFalse = false;
+	const newInput: unknown[] = [];
+
+	for (const entry of input as Record<string, unknown>[]) {
+		if (!entry || typeof entry !== "object") {
+			newInput.push(entry);
+			continue;
+		}
+		const nextEntry = { ...entry };
+		for (const key of ["content", "output"] as const) {
+			if (!Array.isArray(nextEntry[key])) continue;
+			const promoted = await promoteBlocks(
+				nextEntry[key] as Record<string, unknown>[],
+				apiKey,
+				uploadThresholdBytes,
+				signal,
+			);
+			nextEntry[key] = promoted.blocks;
+			uploaded += promoted.uploaded;
+			needsStoreFalse ||= promoted.needsStoreFalse;
+		}
+		newInput.push(nextEntry);
+	}
+
+	if (!needsStoreFalse && uploaded === 0) return { payload, uploaded: 0 };
+	return {
+		payload: { ...p, input: newInput, store: false },
+		uploaded,
+	};
+}