pi-banana 2.4.0 → 2.5.0

package/README.md

@@ -72,6 +72,23 @@ The model calls `banana_image` or `banana_vision` automatically.
 
 ## Configuration
 
+### OpenAI-compatible providers (mantice and friends)
+
+Run `/banana-setup` to point the tools at any OpenAI-compatible image provider. The wizard asks for a base URL and API key (prefilled from pi credentials when available), probes the endpoint for image generation, image editing, vision, and video support, and lets you pick a model per quality tier. Config is saved in settings.json under `"banana"`:
+
+```json
+"banana": {
+  "baseUrl": "https://llm.fornace.net/v1",
+  "apiKey": "sk-...",
+  "models": { "lite": "fornace-image-lite", "fast": "fornace-image", "high": "fornace-image-max" },
+  "visionModel": "fornace-vision"
+}
+```
+
+Delete the `"banana"` section to return to the built-in Google path.
+
+### Environment
+
 Two env vars adjust defaults without touching tool parameters:
 
 | Env var | Default | Effect |

package/extensions/index.ts

@@ -19,6 +19,8 @@
  */
 
 import type { ExtensionAPI, ExtensionContext } from "@earendil-works/pi-coding-agent";
+import { resolveProvider, openaiGenerateImage, openaiEditImage, openaiVision, sizeFor } from "./providers.ts";
+import { runBananaSetup } from "./wizard.ts";
 import { StringEnum } from "@earendil-works/pi-ai";
 import {
 	Box,
@@ -241,6 +243,13 @@ async function resolveOutputPath(
 // ─── Extension ─────────────────────────────────────────────────────────────
 
 export default function (pi: ExtensionAPI) {
+	pi.registerCommand("banana-setup", {
+		description: "Interactive setup: point banana at any OpenAI-compatible image provider (URL + key, capability probe, model selection)",
+		handler: async (_args, ctx) => {
+			await runBananaSetup(ctx);
+		},
+	});
+
 	pi.registerTool({
 		name: "banana_image",
 		label: "Banana Image",
@@ -326,6 +335,41 @@ export default function (pi: ExtensionAPI) {
 				);
 			}
 
+			// OpenAI-compatible provider (mantice etc.) when configured via /banana-setup.
+			const provider = await resolveProvider(ctx);
+			if (provider.kind === "openai-compat") {
+				const { config } = provider;
+				const oaiModel = config.models[quality] ?? config.models.fast;
+				if (!oaiModel) {
+					throw new Error(`banana: no model mapped for quality "${quality}". Run /banana-setup.`);
+				}
+				const size = sizeFor(aspectRatio, imageSize);
+				const refs: { mimeType: string; data: string }[] = [];
+				if (params.referenceImages) {
+					for (const refPath of params.referenceImages) {
+						refs.push(await loadReferenceImage(cwd, refPath));
+					}
+				}
+				const editing = refs.length > 0;
+				onUpdate?.({
+					content: [{ type: "text", text: `🎨 ${editing ? "Editing" : "Generating"} ${aspectRatio} ${imageSize} image with ${oaiModel}…` }],
+					details: { model: oaiModel, aspectRatio, imageSize, quality, editing },
+				});
+				const buf = editing
+					? await openaiEditImage(config, { model: oaiModel, prompt: params.prompt, images: refs, size, signal })
+					: await openaiGenerateImage(config, { model: oaiModel, prompt: params.prompt, size, signal });
+				const outPath = await resolveOutputPath(cwd, params.prompt, "image/png", params.outputPath);
+				await writeFile(outPath, buf);
+				return {
+					content: [{ type: "text", text: editing ? `Edited → ${outPath}` : `Generated → ${outPath}` }],
+					details: {
+						prompt: params.prompt, model: oaiModel, quality, aspectRatio, imageSize,
+						mimeType: "image/png", outputPath: outPath, editing,
+						referenceImages: params.referenceImages, imageBase64: buf.toString("base64"),
+					},
+				};
+			}
+
 			const client = await buildClient(ctx.modelRegistry);
 
 			// Build content parts — text + (optional) reference image.
@@ -594,6 +638,25 @@ export default function (pi: ExtensionAPI) {
 				throw new Error("At least one image path must be provided in imagePaths.");
 			}
 
+			// OpenAI-compatible provider (mantice etc.) when configured via /banana-setup.
+			const provider = await resolveProvider(ctx);
+			if (provider.kind === "openai-compat" && provider.config.visionModel) {
+				const { config } = provider;
+				const images: { mimeType: string; data: string }[] = [];
+				for (const imgPath of params.imagePaths) {
+					images.push(await loadReferenceImage(cwd, imgPath));
+				}
+				onUpdate?.({
+					content: [{ type: "text", text: `👁️ Analyzing image(s) with ${config.visionModel}…` }],
+					details: { model: config.visionModel, quality, imagePaths: params.imagePaths },
+				});
+				const textOut = await openaiVision(config, { model: config.visionModel!, prompt: params.prompt, images, signal });
+				return {
+					content: [{ type: "text", text: textOut }],
+					details: { prompt: params.prompt, model: config.visionModel, quality, imagePaths: params.imagePaths },
+				};
+			}
+
 			for (const imgPath of params.imagePaths) {
 				const ref = await loadReferenceImage(cwd, imgPath);
 				parts.push({ inlineData: ref });

package/extensions/providers.ts

@@ -0,0 +1,212 @@
+// OpenAI-compatible provider layer for pi-banana.
+// Lets banana_image / banana_vision run against any OpenAI-shaped gateway
+// (mantice/llm.fornace.net included) instead of the Google API. Configured
+// via the "banana" section in settings.json (written by /banana-setup):
+//
+//   "banana": {
+//     "baseUrl": "https://llm.fornace.net/v1",
+//     "apiKey": "sk-...",
+//     "models": { "lite": "...", "fast": "...", "high": "..." },
+//     "visionModel": "..."
+//   }
+//
+// When no baseUrl is configured, the built-in Google path is used.
+import { readFileSync } from "fs";
+import { homedir } from "os";
+import { resolve } from "path";
+
+export interface OpenAiCompatConfig {
+	baseUrl: string;
+	apiKey: string;
+	models: { lite?: string; fast?: string; high?: string };
+	visionModel?: string;
+	/** Optional provider id whose key lives in pi's auth.json (e.g. "mantice"). */
+	providerName?: string;
+}
+
+export type Provider =
+	| { kind: "openai-compat"; config: OpenAiCompatConfig }
+	| { kind: "google" };
+
+export function loadBananaSettings(): Partial<OpenAiCompatConfig> {
+	try {
+		const raw = JSON.parse(
+			readFileSync(resolve(homedir(), ".pi", "agent", "settings.json"), "utf8"),
+		);
+		return raw?.banana ?? {};
+	} catch {
+		return {};
+	}
+}
+
+export async function resolveProvider(ctx: any): Promise<Provider> {
+	const s = loadBananaSettings();
+	if (!s.baseUrl) return { kind: "google" };
+	let apiKey = s.apiKey;
+	if (!apiKey && s.providerName && ctx?.modelRegistry) {
+		try {
+			apiKey = await ctx.modelRegistry.getApiKeyForProvider(s.providerName);
+		} catch { /* fall through */ }
+	}
+	if (!apiKey) apiKey = process.env.MANTICE_API_KEY;
+	if (!apiKey) {
+		throw new Error(
+			"banana: baseUrl is configured but no API key was found. " +
+				"Run /banana-setup again, or set banana.apiKey in settings.json.",
+		);
+	}
+	return {
+		kind: "openai-compat",
+		config: {
+			baseUrl: s.baseUrl.replace(/\/+$/, ""),
+			apiKey,
+			models: s.models ?? {},
+			visionModel: s.visionModel,
+			providerName: s.providerName,
+		},
+	};
+}
+
+// ─── OpenAI-compat calls ─────────────────────────────────────────────
+
+/** Map aspect ratio + size tier to a generic WxH string. */
+export function sizeFor(aspectRatio: string, imageSize: string): string {
+	const base = imageSize === "4K" ? 4096 : imageSize === "2K" ? 2048 : 1024;
+	const ratio = (() => {
+		const [w, h] = aspectRatio.split(":").map(Number);
+		return w && h ? w / h : 1;
+	})();
+	if (Math.abs(ratio - 1) < 0.01) return `${base}x${base}`;
+	if (ratio > 1) return `${base}x${Math.round(base / ratio)}`;
+	return `${Math.round(base * ratio)}x${base}`;
+}
+
+async function asBuffer(data: any): Promise<Buffer> {
+	const first = data?.data?.[0] ?? {};
+	if (first.b64_json) return Buffer.from(first.b64_json, "base64");
+	if (first.url) {
+		const res = await fetch(first.url);
+		if (!res.ok) throw new Error(`Image download failed: ${res.statusText}`);
+		return Buffer.from(await res.arrayBuffer());
+	}
+	throw new Error(`No image payload in response: ${JSON.stringify(data).slice(0, 300)}`);
+}
+
+async function apiPost(config: OpenAiCompatConfig, path: string, body: unknown, signal?: AbortSignal): Promise<any> {
+	const res = await fetch(`${config.baseUrl}${path}`, {
+		method: "POST",
+		headers: { Authorization: `Bearer ${config.apiKey}`, "Content-Type": "application/json" },
+		body: JSON.stringify(body),
+		signal,
+	});
+	const text = await res.text();
+	if (!res.ok) throw new Error(`API error (${res.status}) on ${path}: ${text.slice(0, 400)}`);
+	try { return JSON.parse(text); } catch { return text; }
+}
+
+export async function openaiGenerateImage(
+	config: OpenAiCompatConfig,
+	opts: { model: string; prompt: string; size: string; signal?: AbortSignal },
+): Promise<Buffer> {
+	return asBuffer(await apiPost(config, "/images/generations", {
+		model: opts.model, prompt: opts.prompt, n: 1, size: opts.size,
+	}, opts.signal));
+}
+
+export async function openaiEditImage(
+	config: OpenAiCompatConfig,
+	opts: { model: string; prompt: string; images: { mimeType: string; data: string }[]; size: string; signal?: AbortSignal },
+): Promise<Buffer> {
+	const form = new FormData();
+	form.append("model", opts.model);
+	form.append("prompt", opts.prompt);
+	form.append("size", opts.size);
+	for (const img of opts.images) {
+		const bytes = Buffer.from(img.data, "base64");
+		const ext = img.mimeType.split("/")[1] ?? "png";
+		form.append("image", new Blob([bytes], { type: img.mimeType }), `ref.${ext}`);
+	}
+	const res = await fetch(`${config.baseUrl}/images/edits`, {
+		method: "POST",
+		headers: { Authorization: `Bearer ${config.apiKey}` },
+		body: form,
+		signal: opts.signal,
+	});
+	const text = await res.text();
+	if (!res.ok) throw new Error(`API error (${res.status}) on /images/edits: ${text.slice(0, 400)}`);
+	return asBuffer(JSON.parse(text));
+}
+
+export async function openaiVision(
+	config: OpenAiCompatConfig,
+	opts: { model: string; prompt: string; images: { mimeType: string; data: string }[]; signal?: AbortSignal },
+): Promise<string> {
+	const content: any[] = [{ type: "text", text: opts.prompt }];
+	for (const img of opts.images) {
+		content.push({ type: "image_url", image_url: { url: `data:${img.mimeType};base64,${img.data}` } });
+	}
+	const data = await apiPost(config, "/chat/completions", {
+		model: opts.model,
+		messages: [{ role: "user", content }],
+	}, opts.signal);
+	const text = data?.choices?.[0]?.message?.content;
+	if (!text) throw new Error(`Empty vision response: ${JSON.stringify(data).slice(0, 300)}`);
+	return typeof text === "string" ? text : text.map((p: any) => p.text ?? "").join("");
+}
+
+// ─── Capability probing (used by /banana-setup) ──────────────────────
+
+export interface ProbeResult {
+	models: string[];
+	imageModels: string[];
+	visionModels: string[];
+	videoModels: string[];
+	supportsImageGen: boolean;
+	supportsImageEdit: boolean;
+	supportsVideo: boolean;
+}
+
+const IMAGE_RE = /image|banana|seedream|imagen|gpt-image|dall-?e|flux|kolors/i;
+const VISION_RE = /vision|omni|-vl\b|vl-|multimodal/i;
+const VIDEO_RE = /video|sora|veo|wan|seedance|kling|hailuo|minimax-h/i;
+
+export function classifyModels(ids: string[]): Pick<ProbeResult, "imageModels" | "visionModels" | "videoModels"> {
+	return {
+		imageModels: ids.filter((m) => IMAGE_RE.test(m)),
+		visionModels: ids.filter((m) => VISION_RE.test(m) && !IMAGE_RE.test(m)),
+		videoModels: ids.filter((m) => VIDEO_RE.test(m)),
+	};
+}
+
+async function endpointReachable(baseUrl: string, apiKey: string, path: string): Promise<boolean> {
+	// A missing capability answers 404; an existing endpoint answers anything else
+	// (400 for a junk payload, 401 without a key).
+	try {
+		const res = await fetch(`${baseUrl}${path}`, {
+			method: "POST",
+			headers: { Authorization: `Bearer ${apiKey}`, "Content-Type": "application/json" },
+			body: JSON.stringify({ model: "__banana_probe__", prompt: "probe" }),
+		});
+		return res.status !== 404;
+	} catch {
+		return false;
+	}
+}
+
+export async function probeCapabilities(baseUrl: string, apiKey: string): Promise<ProbeResult> {
+	const base = baseUrl.replace(/\/+$/, "");
+	let models: string[] = [];
+	const res = await fetch(`${base}/models`, { headers: { Authorization: `Bearer ${apiKey}` } });
+	if (!res.ok) {
+		throw new Error(`GET ${base}/models failed (${res.status}): ${(await res.text()).slice(0, 300)}`);
+	}
+	const data: any = await res.json();
+	models = (data?.data ?? []).map((m: any) => m.id).filter(Boolean);
+	const { imageModels, visionModels, videoModels } = classifyModels(models);
+	const [supportsImageGen, supportsImageEdit, supportsVideo] = await Promise.all([
+		endpointReachable(base, apiKey, "/images/generations"),
+		endpointReachable(base, apiKey, "/images/edits"),
+		endpointReachable(base, apiKey, "/videos/generations"),
+	]);
+	return { models, imageModels, visionModels, videoModels, supportsImageGen, supportsImageEdit, supportsVideo };
+}

package/extensions/wizard.ts

@@ -0,0 +1,98 @@
+// /banana-setup: interactive provider wizard.
+// Asks for a base URL + API key, probes the endpoint for image and vision
+// capabilities, lets the user confirm model assignments per quality tier,
+// and persists the result to settings.json under the "banana" key.
+import { writeFileSync, readFileSync } from "fs";
+import { homedir } from "os";
+import { resolve } from "path";
+import { probeCapabilities, classifyModels } from "./providers.ts";
+
+const SETTINGS_PATH = resolve(homedir(), ".pi", "agent", "settings.json");
+
+function persistBananaConfig(cfg: Record<string, unknown>): void {
+	let raw: Record<string, unknown> = {};
+	try { raw = JSON.parse(readFileSync(SETTINGS_PATH, "utf8")); } catch { /* new file */ }
+	raw.banana = cfg;
+	writeFileSync(SETTINGS_PATH, JSON.stringify(raw, null, 2));
+}
+
+function pickDefault(candidates: string[], hint: RegExp): string | undefined {
+	return candidates.find((m) => hint.test(m)) ?? candidates[0];
+}
+
+export async function runBananaSetup(ctx: any): Promise<void> {
+	const ui = ctx.ui;
+	ui.notify("Banana setup: configure an OpenAI-compatible image provider (mantice works).", "info");
+
+	// Prefill base URL and key from the pi credential store when available.
+	let preKey: string | undefined;
+	for (const p of ["mantice", "google"]) {
+		try {
+			preKey = await ctx.modelRegistry?.getApiKeyForProvider(p);
+			if (preKey) break;
+		} catch { /* next */ }
+	}
+	const baseUrl = (await ui.input("Provider base URL (OpenAI-compatible)", "https://llm.fornace.net/v1"))?.trim();
+	if (!baseUrl) return;
+	const apiKey = (await ui.input("API key", preKey ?? "sk-..."))?.trim();
+	if (!apiKey) return;
+
+	ui.notify("Probing capabilities…", "info");
+	let probe;
+	try {
+		probe = await probeCapabilities(baseUrl, apiKey);
+	} catch (err: any) {
+		ui.notify(`Probe failed: ${err.message}`, "error");
+		return;
+	}
+
+	if (probe.models.length === 0) {
+		ui.notify("No models listed on this endpoint.", "error");
+		return;
+	}
+
+	const report = [
+		`Models: ${probe.models.length} total`,
+		`Image models: ${probe.imageModels.join(", ") || "none detected"}`,
+		`Vision models: ${probe.visionModels.join(", ") || "none detected"}`,
+		`Video models: ${probe.videoModels.join(", ") || "none detected"}`,
+		`Endpoints: image gen ${probe.supportsImageGen ? "✓" : "✗"}, image edit ${probe.supportsImageEdit ? "✓" : "✗"}, video ${probe.supportsVideo ? "✓" : "✗"}`,
+	].join("\n");
+	ui.notify(report, "info");
+
+	if (!probe.supportsImageGen || probe.imageModels.length === 0) {
+		ui.notify("This endpoint has no image generation models. Setup aborted, Google stays the default.", "error");
+		return;
+	}
+
+	// Auto-assign tiers, then let the user confirm each.
+	const tiers = [
+		{ key: "lite", hint: /lite|fast|turbo|mini/i, label: "lite tier (cheapest)" },
+		{ key: "fast", hint: /^(?!.*(lite|high|max|pro)).*$/i, label: "fast tier (default)" },
+		{ key: "high", hint: /max|high|pro|ultra/i, label: "high tier (top quality)" },
+	] as const;
+	const models: Record<string, string> = {};
+	for (const t of tiers) {
+		const suggestion = pickDefault(probe.imageModels, t.hint) ?? probe.imageModels[0];
+		const options = [suggestion, ...probe.imageModels.filter((m) => m !== suggestion)];
+		const choice = await ui.select(`Model for ${t.label}`, options);
+		if (!choice) return;
+		models[t.key] = choice;
+	}
+
+	let visionModel: string | undefined;
+	if (probe.visionModels.length > 0) {
+		const v = await ui.select("Model for banana_vision (image analysis)", [
+			...probe.visionModels,
+			"(keep Google default)",
+		]);
+		visionModel = v && !v.startsWith("(") ? v : undefined;
+	}
+
+	persistBananaConfig({ baseUrl, apiKey, models, ...(visionModel ? { visionModel } : {}) });
+	ui.notify(
+		`Saved. banana_image now calls ${baseUrl} (${models.fast}/${models.lite}/${models.high}). ` +
+			"Run /banana-setup again to change, or delete \"banana\" from settings.json to return to Google.",
+		"info",
+	);
+}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "pi-banana",
-  "version": "2.4.0",
+  "version": "2.5.0",
   "description": "Generate, edit, and analyze images in pi using Google Nano Banana (image gen) and Gemini Vision (analysis). Inline terminal preview, reference-image editing, auto-save.",
   "author": "Francesco Frapporti <effedue@gmail.com>",
   "license": "MIT",