pi-vision-handoff 0.7.1 → 0.8.1

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "pi-vision-handoff",
-  "version": "0.7.1",
+  "version": "0.8.1",
   "description": "Give text-only pi models vision — describe images with a vision model you pick via an interactive picker, then hand off the text description to non-vision models",
   "type": "module",
   "author": "Tom X Nguyen",

package/src/dataloader.ts

@@ -66,6 +66,9 @@ export interface LoaderDeps extends DescriberDeps {
   getConfig(): VisionHandoffConfig;
   /** Resolve the configured vision model against a registry. */
   resolveVisionModel: VisionModelResolver;
+  /** Backoff (ms) before retrying a totally-failed describer batch. Defaults
+   *  to {@link DESCRIBE_RETRY_BACKOFF_MS} when unset; 0 skips the wait (tests). */
+  retryBackoffMs?: number;
 }
 
 /** Defer `fn` to the next check phase (`setImmediate`), so every
@@ -85,6 +88,31 @@ function enqueuePostPromiseJob(fn: () => void): void {
   setImmediate(fn);
 }
 
+/** Default backoff (ms) before retrying a totally-failed describer batch. A
+ *  short backoff covers a transient provider blip (network hiccup, momentary
+ *  429) without adding meaningful latency to the tool-result phase (free
+ *  time). Generous enough to let a rate-limited provider recover, short enough
+ *  that a genuinely broken vision model doesn't stall a turn. */
+const DESCRIBE_RETRY_BACKOFF_MS = 500;
+
+/** Resolve after `ms`, or immediately if `signal` is already aborted or aborts
+ *  during the wait. Used by the retry backoff so a user cancel (ESC) doesn't
+ *  wait the full backoff before the retry is skipped. */
+function sleep(ms: number, signal: AbortSignal): Promise<void> {
+  return new Promise((resolve) => {
+    if (ms <= 0 || signal.aborted) return resolve();
+    const onAbort = () => {
+      clearTimeout(timer);
+      resolve();
+    };
+    const timer = setTimeout(() => {
+      signal.removeEventListener("abort", onAbort);
+      resolve();
+    }, ms);
+    signal.addEventListener("abort", onAbort, { once: true });
+  });
+}
+
 export class DescriptionLoader implements Disposable {
   private readonly cache = new Map<string, Promise<string>>();
   private batch: DescriptionBatch | null = null;
@@ -253,7 +281,7 @@ export class DescriptionLoader implements Disposable {
     this.deps.setLastError(null); // clear before a fresh attempt
     const misses = batch.keys.map((k, i) => ({ hash: k, img: batch.imgs[i] }));
     const cfg = this.deps.getConfig();
-    const parsed = await runBatch(
+    let parsed = await runBatch(
       misses,
       this.pendingTurnPrompt,
       this.turnVisionModel,
@@ -262,6 +290,30 @@ export class DescriptionLoader implements Disposable {
       this.deps,
       this.turnAbortController.signal,
     );
+    // Retry once on a TOTAL batch failure (the batched call itself failed:
+    // auth, network, timeout, empty, or stopReason "error"). A transient blip
+    // would otherwise cost the agent a full turn — UNAVAILABLE this turn, not
+    // cached, re-attempted next turn. Retrying the whole batch recovers it
+    // within the same tool-result phase (free time). Skip when the turn was
+    // cancelled (ESC); an auth failure re-fails cheaply (runBatch re-checks the
+    // API key before the vision call), so no vision call is wasted on a
+    // permanent auth error. Partial failures (some images described) are left
+    // as-is — only a totally-empty result retries.
+    if (parsed.size === 0 && misses.length > 0 && !this.turnAbortController.signal.aborted) {
+      await sleep(this.deps.retryBackoffMs ?? DESCRIBE_RETRY_BACKOFF_MS, this.turnAbortController.signal);
+      if (!this.turnAbortController.signal.aborted) {
+        this.deps.setLastError(null);
+        parsed = await runBatch(
+          misses,
+          this.pendingTurnPrompt,
+          this.turnVisionModel,
+          this.turnModelRegistry,
+          cfg,
+          this.deps,
+          this.turnAbortController.signal,
+        );
+      }
+    }
     // Build per-hash results, then fan them out to every callback. This reaches
     // duplicate-hash callbacks that a key-indexed loop would have skipped (and
     // left hanging).

package/src/image.ts

@@ -18,7 +18,7 @@
  * agrees with pi on whether a file is an image at all.
  */
 
-import { readFileSync } from "node:fs";
+import { readFileSync, statSync } from "node:fs";
 import { tmpdir } from "node:os";
 import { isAbsolute, join, sep } from "node:path";
 import crypto from "node:crypto";
@@ -248,6 +248,40 @@ export function readImageBuffer(filePath: string): { buf: Buffer; mimeType: stri
   }
 }
 
+/** Max raw file size the omitted-image recovery will re-read. pi's `read`
+ *  already read the file; this bounds the handoff's re-read so a pathologically
+ *  huge file doesn't double the memory. 20MB covers the most generous vision
+ *  model inline-image limit; a larger file would be rejected by the vision
+ *  model anyway. */
+export const MAX_RECOVER_IMAGE_BYTES = 20 * 1024 * 1024;
+
+/** pi core's `read` tool emits this text note (with NO image block) when it
+ *  detected an image but `processImage` failed — Photon/WASM unavailable,
+ *  decode failure, convert-to-PNG failure, or couldn't resize below the inline
+ *  size limit. The handoff's image-block path never sees these (no image
+ *  block), so the image goes undescribed and the model is told the image was
+ *  "omitted". This detects that note so the tool_result handler can re-read the
+ *  raw file and describe its bytes directly (the vision model decodes them —
+ *  no Photon needed). */
+export function isOmittedImageNote(text: string): boolean {
+  return text.includes("Read image file [") && text.includes("[Image omitted:");
+}
+
+/** Read an image file for the omitted-image recovery, bounded by
+ *  {@link MAX_RECOVER_IMAGE_BYTES}. Returns null if the file can't be read, is
+ *  too large, or isn't a supported image (aligned with pi's sniff — APNG and
+ *  unsupported formats are rejected, since the vision model can't decode them
+ *  either). */
+export function readImageBufferBounded(filePath: string): { buf: Buffer; mimeType: string } | null {
+  try {
+    const stat = statSync(filePath);
+    if (stat.size > MAX_RECOVER_IMAGE_BYTES) return null;
+  } catch {
+    return null;
+  }
+  return readImageBuffer(filePath);
+}
+
 /** Regex matching pi's pasted-clipboard temp image file paths anywhere in the
  *  prompt text. pi writes pasted clipboard images to
  *  `<tmpdir>/pi-clipboard-<uuid>.<ext>` and inserts the path as text at the

package/vision-handoff.ts

@@ -30,6 +30,7 @@
 
 import type { ExtensionAPI, ExtensionCommandContext, ExtensionContext, ModelRegistry } from "@earendil-works/pi-coding-agent";
 import type { Api, ImageContent, Model, TextContent } from "@earendil-works/pi-ai";
+import { isAbsolute, resolve } from "node:path";
 import {
   extractImageFromBlock,
   formatModelRef,
@@ -50,7 +51,7 @@ import {
   type VisionHandoffUsageRecord,
 } from "./src/usage.js";
 import { DescriptionLoader, UNAVAILABLE, type LoaderDeps } from "./src/dataloader.js";
-import { imageHash, findClipboardImagePaths, readImageBuffer, resolvePrewarmImage } from "./src/image.js";
+import { imageHash, findClipboardImagePaths, readImageBuffer, readImageBufferBounded, resolvePrewarmImage, isOmittedImageNote } from "./src/image.js";
 import { appendVisionError } from "./src/error-log.js";
 import { resizeImage } from "@earendil-works/pi-coding-agent";
 import { VisionModelSelectorComponent, type VisionModelSelectorResult } from "./src/vision-model-selector.js";
@@ -368,13 +369,51 @@ export default function (pi: ExtensionAPI) {
 
     // Collect image blocks in this read result. (The read tool emits image
     // blocks even for non-vision models — they reach here untouched.)
+    if (!isHandoffTarget(ctx.model, config)) return;
     const imgs: ExtractedImage[] = [];
+    let hasImageBlock = false;
     for (const block of content) {
       const img = extractImageFromBlock(block);
-      if (img) imgs.push(img);
+      if (img) {
+        imgs.push(img);
+        hasImageBlock = true;
+      }
     }
+
+    // Fallback: `read` detected an image but `processImage` failed (Photon
+    // unavailable / decode fail / convert fail / couldn't resize below the
+    // inline limit), so it emitted a "[Image omitted: …]" text note with NO
+    // image block. The image-block path above never sees it, so the image
+    // would go undescribed and the model would be told the image was "omitted".
+    // Re-read the raw file and describe its bytes directly — the vision model
+    // decodes them itself (no Photon needed). This recovers the Photon-
+    // unavailable and under-vision-model-limit cases; APNG/unsupported are
+    // rejected by the sniff (the vision model can't decode them either).
+    let omittedNoteIndices: number[] = [];
+    if (!hasImageBlock) {
+      for (let i = 0; i < content.length; i++) {
+        const block = content[i];
+        if (
+          block &&
+          typeof block === "object" &&
+          (block as { type: string }).type === "text" &&
+          typeof (block as { text: string }).text === "string" &&
+          isOmittedImageNote((block as { text: string }).text)
+        ) {
+          omittedNoteIndices.push(i);
+        }
+      }
+      if (omittedNoteIndices.length > 0) {
+        const inputPath = event.input.path;
+        if (typeof inputPath === "string") {
+          const resolved = isAbsolute(inputPath) ? inputPath : resolve(ctx.cwd, inputPath);
+          const read = readImageBufferBounded(resolved);
+          if (read) imgs.push({ data: read.buf.toString("base64"), mimeType: read.mimeType });
+        }
+      }
+    }
+
     if (imgs.length === 0) return;
-    if (!isHandoffTarget(ctx.model, config)) return;
     loader.bindTurnContext(ctx);
     if (!resolveVisionModel(ctx.modelRegistry, config.visionModel!)) {
       notifyUnresolvedVisionModel(ctx, config.visionModel!);
@@ -398,6 +437,18 @@ export default function (pi: ExtensionAPI) {
 
     warnFailedImages(ctx, imgs, descs, lastDescriberError ?? "unknown error");
 
+    // Recovery path: no image block was emitted, so there's nothing for the
+    // `context` hook to swap. Replace each "[Image omitted: …]" note directly
+    // with its description (the recovered raw-bytes image is the only img here,
+    // so descs aligns 1:1 with omittedNoteIndices).
+    if (!hasImageBlock) {
+      const recovered = content.slice();
+      for (let i = 0; i < omittedNoteIndices.length && i < descs.length; i++) {
+        recovered[omittedNoteIndices[i]] = { type: "text", text: descs[i] };
+      }
+      return { content: recovered as (TextContent | ImageContent)[] };
+    }
+
     // Warm the cache (done above) and strip pi's
     // `[Current model does not support images…]` note from text blocks —
     // since the handoff replaces the image with a description in the
@@ -765,6 +816,29 @@ async function showSelector(ctx: ExtensionCommandContext): Promise<void> {
     .getAll()
     .map((m) => ({ provider: m.provider, id: m.id, name: m.name, input: m.input, reasoning: m.reasoning }));
 
+  if (ctx.mode !== "tui") {
+    const modelItems = ["None", ...allModels.map((m) => `${m.provider}/${m.id}`)];
+    const modelPick = await ctx.ui.select("Vision model", modelItems);
+    if (modelPick === undefined) return;
+    const ref = modelPick === "None" ? null : modelPick;
+    const thinkingPick = await ctx.ui.select("Thinking", ["on", "off"]);
+    if (thinkingPick === undefined) return;
+    const thinking = thinkingPick === "on";
+    const thinkingLevel = config.thinkingLevel;
+    const thinkingNote = thinking
+      ? `thinking on (${thinkingLevel})${ref ? " \u2014 applies only if the vision model supports reasoning" : ""}`
+      : "thinking off";
+    updateConfig(
+      ctx,
+      (c) => ({ ...c, visionModel: ref, thinking, thinkingLevel }),
+      ref ? `Vision model set to ${ref} \u00b7 ${thinkingNote}` : `Vision model cleared \u00b7 ${thinkingNote}`,
+    );
+    if (!ref) {
+      ctx.ui.notify("Handoff is inactive until you pick a vision model.", "warning");
+    }
+    return;
+  }
+
   const result = await ctx.ui.custom<VisionModelSelectorResult>((tui, theme, _kb, done) => {
     const selector = new VisionModelSelectorComponent(
       theme,