pi-multimodal-proxy 1.19.0 → 1.19.1

package/CHANGELOG.md

@@ -4,6 +4,16 @@ All notable changes to this project will be documented in this file.
 
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/).
 
+## [1.19.1] - 2026-10-02
+
+### Changed
+
+- **Leaner `analyze_image` description** — the tool description shrinks ~40% (1.7 KB → ~1.0 KB) while keeping every actionable detail (when to use, the three crop forms with shapes, recall ids, fence attributes, `crop_origin` math). Pi 1.0's codemode lists tool declarations under a 3000-token inline budget and drops what doesn't fit, so a long description both costs tokens on every request and risks pushing other tools out of the listing.
+- **Recovery-style errors (Pi 1.0 codemode philosophy: errors say how to recover)** — `analyze_image`'s model-not-found error now names close registry matches (`anthropic/claude-sont-5` → suggests `anthropic/claude-sonnet-5`, `…-sonnet-4-5`), the recall-id-not-found error lists close ids still retained in the session cache, and zero-area crop errors explain the expected shape (fraction/absolute bounds) instead of just "zero area".
+- **Dynamic implicit-vision-model fallback (Pi ≥ 1.0, feature-detected)** — when the curated fallback chain (`zai/glm-5.3-flash` → `deepseek/…-flash-vision-exp` → `anthropic/claude-sonnet-5/4-5`) is entirely unkeyed but another keyed vision-capable chat model exists (openrouter, google, …), `ModelRegistry.getAvailableOfType("chat")` resolves the implicit default to the first one (cheap-tier ids — flash/mini/fast/haiku/express/lite — preferred so an implicit default never lands on a frontier-priced model). Probe cached per session; older runtimes and the "nothing keyed anywhere" notice path are unchanged.
+- **`#` recall autocomplete regression tests** mirroring pi #10218 (autocomplete must trigger when input starts with whitespace) — the `(?:^|\s)#` token regex already handled it; the tests pin it.
+- Six new tests: `closeMatches` (near-typo ranking, substring bonus, relevance floor, deterministic ties), `pickDynamicVisionCandidate` (cheap-tier preference, first-capable fallback, input filtering), zero-area crop recovery hints (507 total, all green).
+
 ## [1.19.0] - 2026-10-01
 
 ### Added

package/extensions/__tests__/internal.test.ts

@@ -79,6 +79,8 @@ import {
 	bufferToPiAiImage,
 	shouldStripImages,
 	selectVisionModels,
+	closeMatches,
+	pickDynamicVisionCandidate,
 	splitSubcommand,
 	stripImagePaths,
 	stripMediaPaths,
@@ -374,6 +376,66 @@ describe("selectVisionModels", () => {
 	});
 });
 
+describe("closeMatches", () => {
+	const catalog = [
+		"anthropic/claude-sonnet-5",
+		"anthropic/claude-sonnet-4-5",
+		"deepseek/deepseek-v4-flash-vision-exp",
+		"zai/glm-5.3-flash",
+		"openai/gpt-5.6",
+	];
+
+	it("ranks a near-typo above unrelated models", () => {
+		assert.deepEqual(closeMatches("anthropic/claude-sont-5", catalog, 2), [
+			"anthropic/claude-sonnet-5",
+			"anthropic/claude-sonnet-4-5",
+		]);
+	});
+
+	it("gives substring matches a strong bonus", () => {
+		assert.deepEqual(closeMatches("glm", catalog, 1), ["zai/glm-5.3-flash"]);
+	});
+
+	it("returns nothing for queries with no reasonable match", () => {
+		assert.deepEqual(closeMatches("banana-pudding", catalog), []);
+	});
+
+	it("is deterministic on ties (alphabetical)", () => {
+		assert.deepEqual(closeMatches("mod", ["mod-b", "mod-a", "mod-c"], 3), ["mod-a", "mod-b", "mod-c"]);
+	});
+
+	it("handles empty input", () => {
+		assert.deepEqual(closeMatches("", catalog), []);
+		assert.deepEqual(closeMatches("x", []), []);
+	});
+});
+
+describe("pickDynamicVisionCandidate", () => {
+	const m = (provider: string, modelId: string, input: readonly string[] = ["text", "image"]) => ({
+		provider,
+		modelId,
+		input,
+	});
+
+	it("prefers a cheap-tier model over registry order", () => {
+		const out = pickDynamicVisionCandidate([
+			m("anthropic", "claude-opus-6"),
+			m("zai", "glm-5.3-flash"),
+		]);
+		assert.equal(out?.modelId, "glm-5.3-flash");
+	});
+
+	it("falls back to the first image-capable model when none is cheap-tier", () => {
+		const out = pickDynamicVisionCandidate([m("anthropic", "claude-opus-6"), m("x", "y")]);
+		assert.equal(out?.modelId, "claude-opus-6");
+	});
+
+	it("skips models without image input and empty lists", () => {
+		assert.equal(pickDynamicVisionCandidate([m("a", "b", ["text"])]), undefined);
+		assert.equal(pickDynamicVisionCandidate([]), undefined);
+	});
+});
+
 describe("sanitizeProviderHeaders", () => {
 	it("drops null deletion markers", () => {
 		assert.deepEqual(
@@ -1844,6 +1906,21 @@ describe("resolveCropEntry", () => {
 			/zero area/,
 		);
 	});
+
+	it("zero-area normalized errors carry a recovery hint (pi 1.0 style)", () => {
+		assert.throws(
+			() =>
+				resolveCropEntry({ image_index: 0, normalized: { x: 1.0, y: 1.0, width: 0, height: 0 } }, 100, 100),
+			/fractions of the image/,
+		);
+	});
+
+	it("zero-area pixel errors carry a recovery hint (pi 1.0 style)", () => {
+		assert.throws(
+			() => resolveCropEntry({ image_index: 0, pixels: { x: 200, y: 200, width: 10, height: 10 } }, 100, 100),
+			/inside the image/,
+		);
+	});
 });
 
 describe("cropSignature", () => {

package/extensions/__tests__/recall-autocomplete.test.ts

@@ -16,6 +16,16 @@ describe("extractRecallToken", () => {
 		assert.deepEqual(extractRecallToken(["#"], 0, 1), { query: "", prefix: "#" });
 	});
 
+	it("matches a # token after leading whitespace (pi #10218 mirror)", () => {
+		// Pi 1.0 fixed slash-command autocomplete not triggering when the input
+		// starts with whitespace; the # recall provider must behave the same.
+		assert.deepEqual(extractRecallToken(["  #shot"], 0, 7), {
+			query: "shot",
+			prefix: "#shot",
+		});
+		assert.deepEqual(extractRecallToken(["   #"], 0, 4), { query: "", prefix: "#" });
+	});
+
 	it("matches a # token after whitespace with a partial query", () => {
 		assert.deepEqual(extractRecallToken(["zoom into #shot"], 0, 15), {
 			query: "shot",

package/extensions/internal.ts

@@ -2373,6 +2373,68 @@ export function selectVisionModels<T extends VisionModelLike>(
 	return all.filter((m) => m.input.includes("image"));
 }
 
+/** Lowercased bigram set of a string, for Dice-coefficient similarity. */
+function bigrams(s: string): Set<string> {
+	const t = s.toLowerCase();
+	const out = new Set<string>();
+	for (let i = 0; i + 1 < t.length; i++) out.add(t.slice(i, i + 2));
+	return out;
+}
+
+/**
+ * Close-match suggestions for a mistyped identifier, in the spirit of Pi
+ * 1.0's codemode recovery errors ("tools.Bash suggests tools.bash").
+ * Ranks by Dice bigram similarity with a strong exact/substring bonus;
+ * returns at most `limit` candidates above the relevance floor, best first,
+ * ties broken alphabetically so the output is deterministic.
+ */
+export function closeMatches(query: string, candidates: readonly string[], limit = 3): string[] {
+	if (!query || candidates.length === 0) return [];
+	const q = query.toLowerCase();
+	const scored = candidates.map((candidate) => {
+		const c = candidate.toLowerCase();
+		let score: number;
+		if (c === q) score = 2;
+		else if (c.includes(q) || q.includes(c)) score = 1.5;
+		else {
+			const bq = bigrams(q);
+			const bc = bigrams(c);
+			let shared = 0;
+			for (const g of bq) if (bc.has(g)) shared++;
+			score = bq.size === 0 || bc.size === 0 ? 0 : (2 * shared) / (bq.size + bc.size);
+		}
+		return { candidate, score };
+	});
+	return scored
+		.filter((s) => s.score >= 0.3)
+		.sort((a, b) => b.score - a.score || (a.candidate < b.candidate ? -1 : a.candidate > b.candidate ? 1 : 0))
+		.slice(0, limit)
+		.map((s) => s.candidate);
+}
+
+/** Minimal shape for dynamic (registry-provided) vision candidates. */
+export interface DynamicCandidateLike {
+	provider: string;
+	modelId: string;
+	input?: readonly string[];
+}
+
+/** Cheap-tier heuristic for implicit default resolution (Pi 1.0 last resort). */
+const CHEAP_MODEL_RE = /flash|mini|fast|haiku|express|lite/i;
+
+/**
+ * First keyed registry model with image input — the last-resort implicit
+ * vision default when the curated fallback chain is entirely unkeyed
+ * (Pi ≥ 1.0 `getAvailableOfType("chat")`). Cheap-tier ids are preferred so
+ * an implicit default never lands on a frontier-priced model; registry
+ * order breaks remaining ties.
+ */
+export function pickDynamicVisionCandidate<T extends DynamicCandidateLike>(models: readonly T[]): T | undefined {
+	const vision = models.filter((m) => m.input?.includes("image"));
+	if (vision.length === 0) return undefined;
+	return vision.find((m) => CHEAP_MODEL_RE.test(m.modelId)) ?? vision[0];
+}
+
 export function shouldStripImages(config: VisionConfig, modelInput: readonly string[] | undefined): boolean {
 	if (config.mode === "off") return false;
 	if (config.mode === "always") return true;
@@ -2524,12 +2586,20 @@ export function resolveCropEntry(crop: CropEntry, imgWidth: number, imgHeight: n
 	}
 	if ("normalized" in crop) {
 		const result = normalizedToPixels(crop.normalized, imgWidth, imgHeight);
-		if (!result) throw new Error(`Normalized crop has zero area after clamping (image: ${imgWidth}x${imgHeight})`);
+		if (!result)
+			throw new Error(
+				`Normalized crop has zero area after clamping (image: ${imgWidth}x${imgHeight}). ` +
+					`Use fractions of the image: x ≥ 0, y ≥ 0, x + width ≤ 1, y + height ≤ 1.`,
+			);
 		return result;
 	}
 	if ("pixels" in crop) {
 		const result = clampPixels(crop.pixels, imgWidth, imgHeight);
-		if (!result) throw new Error(`Pixel crop has zero area after clamping (image: ${imgWidth}x${imgHeight})`);
+		if (!result)
+			throw new Error(
+				`Pixel crop has zero area after clamping (image: ${imgWidth}x${imgHeight}). ` +
+					`Keep the rectangle inside the image: x ≥ 0, y ≥ 0, x + width ≤ ${imgWidth}, y + height ≤ ${imgHeight}.`,
+			);
 		return result;
 	}
 	throw new Error("Invalid CropEntry: must have exactly one of region, normalized, or pixels");

package/extensions/vision-proxy.ts

@@ -282,6 +282,8 @@ import {
 	canonicalYouTubeUrl,
 	youTubeVideoId,
 	applyDefaultModelFallback,
+	closeMatches,
+	pickDynamicVisionCandidate,
 	DEFAULT_MODEL_FALLBACKS,
 	applyRecallCompletion,
 	buildRecallItems,
@@ -450,22 +452,18 @@ const AnalyzeImageOutput = Type.Object({
 });
 
 const TOOL_DESCRIPTION = [
-	"Use `analyze_image` when (a) the cached description of an image lacks a detail you need,",
-	"(b) you need to compare or cross-reference multiple images, or (c) you need to focus on a specific region.",
+	"Targeted image analysis with cropping and model-native grounding support.",
 	"",
-	"**Cropping.** Three forms, in order of preference:",
+	"Use it when (a) a cached image description lacks a detail you need, (b) you must compare or cross-reference images, or (c) you must focus on a region. The result is authoritative for the question asked; the cached generic description stays the default for everything else.",
 	"",
-	"- **`region`** - coarse cut by name. Use when you don't have exact dimensions: `{ image_index: 0, region: \"bottom-right\" }`.",
-	"- **`normalized`** - fractional coordinates 0.0-1.0. Default choice for precise crops without knowing image dimensions: `{ image_index: 0, normalized: { x: 0.5, y: 0.5, width: 0.4, height: 0.4 } }`.",
-	"- **`pixels`** - absolute pixels. Use only when you have authoritative coordinates from a prior `<vision_proxy_description>` or `<vision_proxy_analysis>` (which carry `width` and `height` attributes) or from a previous grounded response. Example: `{ image_index: 0, pixels: { x: 1840, y: 120, width: 840, height: 360 } }`.",
+	"Image references are file paths OR the `image=\"<id>\"` id carried by every `<vision_proxy_description>`/`<vision_proxy_analysis>`/`<vision_proxy_joint_description>` fence — pass an id to re-examine or crop an image the user shared earlier, even if it is no longer attached.",
 	"",
-	"Image dimensions and filenames are available in the `width`, `height`, and `filename` attributes of `<vision_proxy_description>`, `<vision_proxy_analysis>`, and `<vision_proxy_joint_description>` blocks in your context.",
+	"Cropping, in order of preference (one crop per image, keyed by `image_index`):",
+	"- `region` — named cut (`top-left`, `bottom-right`, `center`, `right-half`, …) when you don't know the dimensions: `{ image_index: 0, region: \"bottom-right\" }`.",
+	"- `normalized` — `{ x, y, width, height }` as 0.0-1.0 fractions; the default for precise crops without dimensions.",
+	"- `pixels` — `{ x, y, width, height }` absolute; only with authoritative coordinates from a prior fence's `width`/`height` attributes or a grounded response.",
 	"",
-	"**Recalling an earlier image.** Every such block also carries an `image=\"...\"` id. To re-examine or crop an image the user shared earlier in the session — even if it is no longer attached to the current message (e.g. \"zoom into that screenshot from before\") — pass that id as the image reference instead of a file path. No re-attachment is required.",
-	"",
-	"When a crop is applied, the response fence carries a `crop_origin` attribute (e.g. `crop_origin=\"1840,120\"`). Add the origin's x to any returned x-coordinate and the origin's y to any returned y-coordinate to map coordinates back to the original full image.",
-	"",
-	"The tool result is authoritative for the specific question asked; the cached generic description remains the default for everything else.",
+	"Fences carry `width`, `height`, and `filename`, and a crop adds `crop_origin=\"x,y\"` — add the origin to returned coordinates to map them back to the full image.",
 ].join("\n");
 
 /** Maximum analyze_image tool calls per agent turn. Prevents cost runaway. */
@@ -508,6 +506,8 @@ interface SessionState {
 	compaction?: { reason?: "manual" | "threshold" | "overflow"; willRetry?: boolean };
 	/** Cached "provider/model has an API key" probes (cleared per session). */
 	keyProbe?: Map<string, boolean>;
+	/** Cached dynamic (registry-keyed) vision candidate; null = none or runtime without the API (Pi ≥ 1.0). */
+	dynamicVisionProbe?: { provider: string; modelId: string } | null;
 	/** Last implicit-default substitution already notified this session. */
 	notifiedDefaultSub?: string;
 	/**
@@ -762,7 +762,56 @@ async function withModelFallback(config: VisionConfig, ctx: ExtensionContext): P
 		keys.set(`${pair.provider}/${pair.modelId}`, await probe(pair.provider, pair.modelId));
 	}
 	const hasKey = (p: string, m: string) => keys.get(`${p}/${m}`) === true;
-	return applyDefaultModelFallback(config, hasModel, false, hasKey);
+	const resolved = applyDefaultModelFallback(config, hasModel, false, hasKey);
+	// Pi ≥ 1.0 last resort: the curated chain can be entirely unkeyed while
+	// the user has another keyed vision-capable model (openrouter, google, …).
+	// getAvailableOfType lists exactly those — land the implicit default on
+	// the first one instead of dead-ending on the "No API key" notice.
+	// (Explicit choices returned early above; a keyed chain resolution already
+	// changed provider/modelId, which the equality check skips.)
+	const anyKeyed = [...keys.values()].some(Boolean);
+	if (!anyKeyed && resolved.provider === config.provider && resolved.modelId === config.modelId) {
+		const dyn = await dynamicVisionCandidate(ctx);
+		if (dyn) return { ...resolved, provider: dyn.provider, modelId: dyn.modelId };
+	}
+	return resolved;
+}
+
+/**
+ * Pi ≥ 1.0 dynamic vision-candidate probe (ModelRegistry.getAvailableOfType):
+ * the first keyed chat model with image input, cheap tier preferred
+ * (pickDynamicVisionCandidate). Feature-detected — older runtimes and probe
+ * failures cache as null; the probe runs once per session.
+ */
+async function dynamicVisionCandidate(ctx: ExtensionContext): Promise<{ provider: string; modelId: string } | null> {
+	const state = getSessionState(ctx);
+	if (state.dynamicVisionProbe !== undefined) return state.dynamicVisionProbe;
+	let out: { provider: string; modelId: string } | null = null;
+	const reg = ctx.modelRegistry as ModelRegistry & {
+		getAvailableOfType?: (type: string) => Promise<ReadonlyArray<unknown>>;
+	};
+	if (typeof reg.getAvailableOfType === "function") {
+		try {
+			const available = await reg.getAvailableOfType("chat");
+			const candidates = (
+				available as ReadonlyArray<{ provider?: unknown; id?: unknown; input?: unknown }>
+			)
+				.filter(
+					(m): m is { provider: string; id: string; input: readonly string[] } =>
+						typeof m.provider === "string" &&
+						typeof m.id === "string" &&
+						Array.isArray(m.input) &&
+						m.input.includes("image"),
+				)
+				.map((m) => ({ provider: m.provider, modelId: m.id, input: m.input }));
+			const pick = pickDynamicVisionCandidate(candidates);
+			if (pick) out = { provider: pick.provider, modelId: pick.modelId };
+		} catch {
+			out = null;
+		}
+	}
+	state.dynamicVisionProbe = out;
+	return out;
 }
 
 /**
@@ -1646,6 +1695,52 @@ function analysisError(message: string): AnalyzeImageOutcome {
 	return { text: message, isError: true, structured: { ok: false, error: message } };
 }
 
+/**
+ * Close-match vision-model suggestions for recovery-style tool errors, in
+ * the spirit of Pi 1.0's codemode errors ("tools.Bash suggests
+ * tools.bash"). Labels every registry model with image input; no key
+ * probing — error paths must stay cheap.
+ */
+function registryVisionSuggestions(ctx: ExtensionContext, query: string, limit = 3): string[] {
+	try {
+		const models = ctx.modelRegistry.getAll() as unknown as ReadonlyArray<{
+			provider?: unknown;
+			id?: unknown;
+			input?: unknown;
+		}>;
+		const labels = models
+			.filter(
+				(m): m is { provider: string; id: string; input: readonly string[] } =>
+					typeof m.provider === "string" &&
+					typeof m.id === "string" &&
+					Array.isArray(m.input) &&
+					m.input.includes("image"),
+			)
+			.map((m) => `${m.provider}/${m.id}`);
+		return closeMatches(query, labels, limit);
+	} catch {
+		return [];
+	}
+}
+
+/**
+ * Close-match recall-id suggestions: matches the mistyped id against every
+ * retained image's hash (and filename, when known) and returns full ids the
+ * agent can pass verbatim.
+ */
+function recallSuggestions(imageData: ImageDataStore, imageMeta: ImageMetaStore, query: string, limit = 3): string[] {
+	const candidates: string[] = [];
+	try {
+		for (const hash of imageData.map.keys()) {
+			const meta = imageMeta.get(hash);
+			candidates.push(meta?.filename ? `${hash} (${meta.filename})` : hash);
+		}
+	} catch {
+		return [];
+	}
+	return closeMatches(query, candidates, limit);
+}
+
 /**
  * Answer a targeted analyze_image question about one or more images: resolve
  * references (session recall ids or file paths), validate and apply crops,
@@ -1709,7 +1804,12 @@ async function handleAnalyzeImage(
 	// Verify model exists and supports images
 	const visionModel = ctx.modelRegistry.find(visionProvider, visionModelId);
 	if (!visionModel) {
-		return analysisError(`Error: model "${visionProvider}/${visionModelId}" not found in registry. Use /multimodal-proxy pick to choose a vision model.`);
+		const suggestions = registryVisionSuggestions(ctx, `${visionProvider}/${visionModelId}`);
+		return analysisError(
+			`Error: model "${visionProvider}/${visionModelId}" not found in registry.` +
+				(suggestions.length > 0 ? ` Close matches: ${suggestions.join(", ")}.` : "") +
+				" Use /multimodal-proxy pick to choose a vision model.",
+		);
 	}
 	if (!visionModel.input.includes("image")) {
 		return analysisError(`Error: model "${visionModel.name ?? visionModelId}" does not support image input.`);
@@ -1731,7 +1831,12 @@ async function handleAnalyzeImage(
 		if (recallHash) {
 			const stored = getImageData(imageData, recallHash);
 			if (!stored) {
-				return analysisError(`Error: image "${recallHash}" is not available for recall — it may have expired from the session cache or was never analyzed. Ask the user to re-attach it, or pass a file path.`);
+				const suggestions = recallSuggestions(imageData, imageMeta, recallHash);
+				return analysisError(
+					`Error: image "${recallHash}" is not available for recall — it may have expired from the session cache or was never analyzed.` +
+						(suggestions.length > 0 ? ` Close ids still retained: ${suggestions.join(", ")}.` : "") +
+						" Ask the user to re-attach it, or pass a file path.",
+				);
 			}
 			const image: PiAiImage = { type: "image", data: stored.data, mimeType: stored.mimeType };
 			// Backfill dimensions if metadata was evicted, so crops still work on recall.

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "pi-multimodal-proxy",
-  "version": "1.19.0",
+  "version": "1.19.1",
   "description": "Automatic image, video and audio description for any model in Pi. Routes media to a multimodal model and injects descriptions into context.",
   "keywords": [
     "pi-package"