pi-multimodal-proxy 1.21.0 → 1.22.0

package/CHANGELOG.md

@@ -4,6 +4,15 @@ All notable changes to this project will be documented in this file.
 
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/).
 
+## [1.22.0] - 2026-10-10
+
+### Added
+
+- **Claude Haiku 5.5 in the implicit-model candidate chain** — `DEFAULT_MODEL_FALLBACKS` now reads `zai/glm-5.3-flash` → `deepseek/deepseek-v4-flash-vision-exp` → `anthropic/claude-haiku-5-5` (Pi ≥ 1.1.0 catalog) → `anthropic/claude-sonnet-5` → `anthropic/claude-sonnet-4-5`. An anthropic-keyed setup whose cheaper candidates are unkeyed lands on Haiku 5.5 ($0.1/M in, $0.5/M out, vision-capable, 1M context; above 100K input tokens $0.5/$2.5 — output still 4× cheaper than Sonnet's flat rate) instead of Sonnet 5 ($2/M in, $10/M out) — 20× cheaper output at typical description-prompt sizes, same API key. Description prompts (a few images + instructions) sit far below the tier threshold. Haiku's catalog `inputLimits.images.resize` (2000 px / q80) feeds the existing model-aware upload downscale. Pre-1.1.0 catalogs simply don't contain Haiku, so the registry check skips it and behavior is unchanged; explicit model choices are never rewritten.
+- **`durationMs` fallback in the `analyze_image` result row (Pi ≥ 1.1.0, feature-detected)** — the pi 1.1.0 tool render context carries `durationMs`, the monotonic-clock `execute()` time recorded with the final result. `formatAnalyzeImageResult` accepts it as a display fallback when `details` carry no `latencyMs`, so rows replayed from foreign sessions, HTML exports, and results recorded before `details` existed still render `✓ … · 1.2s`. `details.latencyMs` (the vision call itself, also reported to codemode scripts via `structuredContent`) always wins — the host duration covers consent checks and crop encoding too, so the two measure different things. Older hosts pass no `durationMs` and rendering is unchanged.
+- **Per-invocation tool toggling documented** — pi 1.1.0's `--tools +name/-name` entries change the default tool selection instead of replacing it. README now shows `pi -t -analyze_image` (one-shot session without the re-query tool) and `pi -t +analyze_image` (add it back to a trimmed selection) alongside the existing persistent toggles (`/multimodal-proxy tool off`, `PI_VISION_PROXY_TOOL=off`). Privacy section updated to spell out the full current default-model chain.
+- Six new tests (546 total, all green): Haiku placement/ordering in the chain, key-aware preference of Haiku over Sonnet, catalog-without-Haiku skip, `durationMs` fallback/preference/finite-guard in the result formatter. Syntax check clean (8 files).
+
 ## [1.21.0] - 2026-10-03
 
 ### Added

package/README.md

@@ -10,6 +10,14 @@ When **video or audio files** are detected, they are routed to a **multimodal mo
 
 **Compact tool rows (Pi ≥ 1.0.1)** — `analyze_image` and `generate_image` calls render as one tidy line each (`analyze_image  "What does the legend say?" · 2 images (image="deadbeef…", shot.png) · crop bottom-right@0`) with themed result summaries (`✓ zai/glm-5.3-flash · cached · 1.2s`, `✓ 1 image via google/imagen-4 · $0.0311`), instead of the default JSON-args dump. Generated images still render inline from the result blocks; expand a row to see the full analysis fence.
 
+## What's new in 1.22.0
+
+Borrowing from Pi 1.1.0:
+
+- **Claude Haiku 5.5 joins the implicit-model candidate chain** — `zai/glm-5.3-flash` → `deepseek/deepseek-v4-flash-vision-exp` → `anthropic/claude-haiku-5-5` (new, Pi ≥ 1.1.0) → `anthropic/claude-sonnet-5` → `anthropic/claude-sonnet-4-5`. When your cheaper candidates have no API key, an anthropic-keyed setup now lands on Haiku 5.5 instead of Sonnet 5 for image descriptions — $0.1/M in, $0.5/M out vs $2/M, $10/M (20× cheaper output at typical description-prompt sizes; above 100K input tokens Haiku is $0.5/$2.5, still 4× cheaper output). Same key, still vision-capable, and its catalog `inputLimits.images.resize` feeds the model-aware upload downscale. Catalogs without Haiku (pi < 1.1.0) skip it and behave exactly as before; explicit choices are never rewritten.
+- **Result rows show a duration even without `details`** — pi 1.1.0's render context carries `durationMs` (the recorded `execute()` time of a final tool result). The `analyze_image` result line now falls back to it when its `details` have no `latencyMs`, so rows replayed from foreign/HTML-exported sessions still show `✓ … · 1.2s`. Our own `latencyMs` (the vision call itself, reported to codemode scripts via `structuredContent`) always wins.
+- **Per-invocation tool toggling documented** — pi 1.1.0's `--tools +name/-name` entries modify the default tool selection instead of replacing it, so `pi -t -analyze_image` runs a one-shot session without the re-query tool while keeping everything else (see [Slash commands](#slash-commands)).
+
 ## What's new in 1.21.0
 
 Adopting Pi 1.0.1's new `pi.registerToolRenderer()` API:
@@ -249,6 +257,8 @@ Legacy alias: /vision-proxy <args> works identically.
 
 When an env var is set, the matching `/multimodal-proxy` subcommand is locked.
 
+For one-shot sessions you can also toggle tools per invocation (Pi ≥ 1.1.0): `pi -t -analyze_image` removes the re-query tool from the default selection, `pi -t +analyze_image` adds it back to a trimmed one. This complements the persistent `/multimodal-proxy tool off` and `PI_VISION_PROXY_TOOL=off`, which survive across sessions.
+
 ## How it works — Images
 
 ```
@@ -375,7 +385,7 @@ When a model is in the grounding registry, a format-specific instruction is appe
 
 ## Privacy & security
 
-This extension **sends data to a third-party provider**. By default that is `zai/glm-5.3-flash` for images (`anthropic/claude-sonnet-5` on older Pi versions without GLM 5.3 Flash in the catalog, then `anthropic/claude-sonnet-4-5`) and `xai/grok-4.3` for video/audio. Be aware:
+This extension **sends data to a third-party provider**. By default that is `zai/glm-5.3-flash` for images, falling back through `deepseek/deepseek-v4-flash-vision-exp` → `anthropic/claude-haiku-5-5` (Pi ≥ 1.1.0) → `anthropic/claude-sonnet-5` → `anthropic/claude-sonnet-4-5` to the first candidate in your catalog **and keyed** (`xai/grok-4.3` for video/audio). Be aware:
 
 1. **Image and video data is uploaded** to the configured provider on every proxied request. Crop coordinates are applied locally before upload — only the cropped region is sent.
 2. **Recent conversation context** (last 8 messages, truncated) is uploaded with the image unless you set `/multimodal-proxy context off` or `PI_VISION_PROXY_INCLUDE_CONTEXT=false`. Disable it for sensitive sessions.

package/extensions/__tests__/internal.test.ts

@@ -3312,6 +3312,41 @@ describe("1.18.0 key-aware default model resolution", () => {
 		assert.equal(DEFAULT_MODEL_FALLBACKS[0]!.modelId, "deepseek-v4-flash-vision-exp");
 	});
 
+	it("places Claude Haiku 5.5 between DeepSeek and Sonnet 5 (pi ≥ 1.1.0 catalogs)", () => {
+		const haiku = DEFAULT_MODEL_FALLBACKS.find((m) => m.modelId === "claude-haiku-5-5")!;
+		assert.equal(haiku.provider, "anthropic");
+		const deepseekIdx = DEFAULT_MODEL_FALLBACKS.findIndex((m) => m.provider === "deepseek");
+		const sonnetIdx = DEFAULT_MODEL_FALLBACKS.findIndex((m) => m.modelId === "claude-sonnet-5");
+		assert.ok(deepseekIdx < DEFAULT_MODEL_FALLBACKS.indexOf(haiku));
+		assert.ok(DEFAULT_MODEL_FALLBACKS.indexOf(haiku) < sonnetIdx);
+	});
+
+	it("prefers Haiku 5.5 over Sonnet 5 when both are keyed anthropic candidates", () => {
+		const out = applyDefaultModelFallback(
+			builtin,
+			registryWith(
+				`${builtin.provider}/${builtin.modelId}`,
+			"anthropic/claude-haiku-5-5",
+				"anthropic/claude-sonnet-5",
+			),
+			false,
+			keyWith("anthropic/claude-haiku-5-5", "anthropic/claude-sonnet-5"),
+		);
+		assert.equal(out.provider, "anthropic");
+		assert.equal(out.modelId, "claude-haiku-5-5");
+	});
+
+	it("skips Haiku 5.5 on pre-1.1.0 catalogs and still lands on Sonnet 5", () => {
+		const out = applyDefaultModelFallback(
+			builtin,
+			registryWith(`${builtin.provider}/${builtin.modelId}`, "anthropic/claude-sonnet-5"),
+			false,
+			keyWith("anthropic/claude-haiku-5-5", "anthropic/claude-sonnet-5"),
+		);
+		assert.equal(out.provider, "anthropic");
+		assert.equal(out.modelId, "claude-sonnet-5");
+	});
+
 	it("keeps the default when it is in the catalog and keyed", () => {
 		const out = applyDefaultModelFallback(
 			builtin,

package/extensions/__tests__/tool-render.test.ts

@@ -196,6 +196,30 @@ describe("formatAnalyzeImageResult", () => {
 		assert.equal(segs[0]!.style, "success");
 		assert.equal(segs[1]!.style, "muted");
 	});
+
+	describe("durationMs fallback (pi ≥ 1.1.0 render context)", () => {
+		it("uses the host durationMs when details carry no latencyMs", () => {
+			assert.equal(
+				plain(formatAnalyzeImageResult({ ok: true, provider: "zai", model: "m" }, false, 2500)),
+				"✓ zai/m · 2.5s",
+			);
+			// rows with no details at all (replayed foreign sessions, HTML exports)
+			assert.equal(plain(formatAnalyzeImageResult(undefined, false, 987)), "✓ · 1.0s");
+		});
+
+		it("prefers details.latencyMs over the host durationMs", () => {
+			const segs = formatAnalyzeImageResult(
+				{ ok: true, provider: "zai", model: "glm-5.3-flash", latencyMs: 1234 },
+				false,
+				99999,
+			);
+			assert.equal(plain(segs), "✓ zai/glm-5.3-flash · 1.2s");
+		});
+
+		it("ignores non-finite durationMs", () => {
+			assert.equal(plain(formatAnalyzeImageResult({ ok: true }, false, Number.NaN)), "✓");
+		});
+	});
 });
 
 describe("formatGenerateImageResult", () => {

package/extensions/internal.ts

@@ -1318,6 +1318,7 @@ export function resolveConfig(
  */
 export const DEFAULT_MODEL_FALLBACKS: ReadonlyArray<{ provider: string; modelId: string }> = [
 	{ provider: "deepseek", modelId: "deepseek-v4-flash-vision-exp" }, // Pi ≥ 0.84.4, cheap, common key
+	{ provider: "anthropic", modelId: "claude-haiku-5-5" }, // Pi ≥ 1.1.0, vision-capable, $0.1/$0.5 per M
 	{ provider: "anthropic", modelId: "claude-sonnet-5" },
 	{ provider: "anthropic", modelId: "claude-sonnet-4-5" },
 ];
@@ -3401,8 +3402,19 @@ function recordOf(details: unknown): Record<string, unknown> | undefined {
 		: undefined;
 }
 
-/** One-line transcript summary of an analyze_image result from its `details`. */
-export function formatAnalyzeImageResult(details: unknown, isError: boolean): TranscriptSegment[] {
+/**
+ * One-line transcript summary of an analyze_image result from its `details`.
+ * `durationMs` (pi ≥ 1.1.0 host render context) is a display fallback for rows
+ * whose `details` carry no `latencyMs` — replayed sessions, foreign/HTML
+ * exports, results recorded before `details` existed. Details always win:
+ * `latencyMs` measures the vision call itself, while the host's `durationMs`
+ * covers the whole `execute()` (consent checks, crop encoding included).
+ */
+export function formatAnalyzeImageResult(
+	details: unknown,
+	isError: boolean,
+	durationMs?: number,
+): TranscriptSegment[] {
 	const d = recordOf(details);
 	if (isError || d?.ok === false) {
 		const msg = typeof d?.error === "string" && d.error ? d.error : "analysis failed";
@@ -3414,8 +3426,9 @@ export function formatAnalyzeImageResult(details: unknown, isError: boolean): Tr
 	if (provider && modelId) segments.push({ text: ` ${summarizeForTranscript(`${provider}/${modelId}`, 80)}`, style: "muted" });
 	else if (modelId) segments.push({ text: ` ${summarizeForTranscript(modelId, 80)}`, style: "muted" });
 	if (d?.cached === true) segments.push({ text: " · cached", style: "dim" });
-	if (isFiniteNumber(d?.latencyMs)) {
-		segments.push({ text: ` · ${(Math.max(0, d.latencyMs) / 1000).toFixed(1)}s`, style: "dim" });
+	const latency = isFiniteNumber(d?.latencyMs) ? d.latencyMs : isFiniteNumber(durationMs) ? durationMs : undefined;
+	if (latency !== undefined) {
+		segments.push({ text: ` · ${(Math.max(0, latency) / 1000).toFixed(1)}s`, style: "dim" });
 	}
 	return segments;
 }

package/extensions/vision-proxy.ts

@@ -2407,6 +2407,8 @@ export default function (pi: ExtensionAPI) {
 		isPartial?: boolean;
 		expanded?: boolean;
 		showImages?: boolean;
+		/** pi ≥ 1.1.0: recorded execute() duration from the final result. */
+		durationMs?: number;
 	}
 
 	/** AgentToolResult subset the result renderer receives. */
@@ -2580,7 +2582,7 @@ export default function (pi: ExtensionAPI) {
 					renderResult: (result, options, theme, context) => {
 						if (options.isPartial) return textComponent(theme.fg("warning", "Analyzing…"));
 						return resultComponent(
-							formatAnalyzeImageResult(result?.details, context?.isError === true),
+							formatAnalyzeImageResult(result?.details, context?.isError === true, context?.durationMs),
 							result ?? {},
 							options,
 							theme,

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "pi-multimodal-proxy",
-  "version": "1.21.0",
+  "version": "1.22.0",
   "description": "Automatic image, video and audio description for any model in Pi. Routes media to a multimodal model and injects descriptions into context.",
   "keywords": [
     "pi-package"