pi-vision-handoff 0.7.0 → 0.8.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "pi-vision-handoff",
-  "version": "0.7.0",
+  "version": "0.8.0",
   "description": "Give text-only pi models vision — describe images with a vision model you pick via an interactive picker, then hand off the text description to non-vision models",
   "type": "module",
   "author": "Tom X Nguyen",

package/src/dataloader.ts

@@ -66,6 +66,9 @@ export interface LoaderDeps extends DescriberDeps {
   getConfig(): VisionHandoffConfig;
   /** Resolve the configured vision model against a registry. */
   resolveVisionModel: VisionModelResolver;
+  /** Backoff (ms) before retrying a totally-failed describer batch. Defaults
+   *  to {@link DESCRIBE_RETRY_BACKOFF_MS} when unset; 0 skips the wait (tests). */
+  retryBackoffMs?: number;
 }
 
 /** Defer `fn` to the next check phase (`setImmediate`), so every
@@ -85,6 +88,31 @@ function enqueuePostPromiseJob(fn: () => void): void {
   setImmediate(fn);
 }
 
+/** Default backoff (ms) before retrying a totally-failed describer batch. A
+ *  short backoff covers a transient provider blip (network hiccup, momentary
+ *  429) without adding meaningful latency to the tool-result phase (free
+ *  time). Generous enough to let a rate-limited provider recover, short enough
+ *  that a genuinely broken vision model doesn't stall a turn. */
+const DESCRIBE_RETRY_BACKOFF_MS = 500;
+
+/** Resolve after `ms`, or immediately if `signal` is already aborted or aborts
+ *  during the wait. Used by the retry backoff so a user cancel (ESC) doesn't
+ *  wait the full backoff before the retry is skipped. */
+function sleep(ms: number, signal: AbortSignal): Promise<void> {
+  return new Promise((resolve) => {
+    if (ms <= 0 || signal.aborted) return resolve();
+    const onAbort = () => {
+      clearTimeout(timer);
+      resolve();
+    };
+    const timer = setTimeout(() => {
+      signal.removeEventListener("abort", onAbort);
+      resolve();
+    }, ms);
+    signal.addEventListener("abort", onAbort, { once: true });
+  });
+}
+
 export class DescriptionLoader implements Disposable {
   private readonly cache = new Map<string, Promise<string>>();
   private batch: DescriptionBatch | null = null;
@@ -253,7 +281,7 @@ export class DescriptionLoader implements Disposable {
     this.deps.setLastError(null); // clear before a fresh attempt
     const misses = batch.keys.map((k, i) => ({ hash: k, img: batch.imgs[i] }));
     const cfg = this.deps.getConfig();
-    const parsed = await runBatch(
+    let parsed = await runBatch(
       misses,
       this.pendingTurnPrompt,
       this.turnVisionModel,
@@ -262,6 +290,30 @@ export class DescriptionLoader implements Disposable {
       this.deps,
       this.turnAbortController.signal,
     );
+    // Retry once on a TOTAL batch failure (the batched call itself failed:
+    // auth, network, timeout, empty, or stopReason "error"). A transient blip
+    // would otherwise cost the agent a full turn — UNAVAILABLE this turn, not
+    // cached, re-attempted next turn. Retrying the whole batch recovers it
+    // within the same tool-result phase (free time). Skip when the turn was
+    // cancelled (ESC); an auth failure re-fails cheaply (runBatch re-checks the
+    // API key before the vision call), so no vision call is wasted on a
+    // permanent auth error. Partial failures (some images described) are left
+    // as-is — only a totally-empty result retries.
+    if (parsed.size === 0 && misses.length > 0 && !this.turnAbortController.signal.aborted) {
+      await sleep(this.deps.retryBackoffMs ?? DESCRIBE_RETRY_BACKOFF_MS, this.turnAbortController.signal);
+      if (!this.turnAbortController.signal.aborted) {
+        this.deps.setLastError(null);
+        parsed = await runBatch(
+          misses,
+          this.pendingTurnPrompt,
+          this.turnVisionModel,
+          this.turnModelRegistry,
+          cfg,
+          this.deps,
+          this.turnAbortController.signal,
+        );
+      }
+    }
     // Build per-hash results, then fan them out to every callback. This reaches
     // duplicate-hash callbacks that a key-indexed loop would have skipped (and
     // left hanging).

package/src/image.ts

@@ -18,7 +18,7 @@
  * agrees with pi on whether a file is an image at all.
  */
 
-import { readFileSync } from "node:fs";
+import { readFileSync, statSync } from "node:fs";
 import { tmpdir } from "node:os";
 import { isAbsolute, join, sep } from "node:path";
 import crypto from "node:crypto";
@@ -248,6 +248,40 @@ export function readImageBuffer(filePath: string): { buf: Buffer; mimeType: stri
   }
 }
 
+/** Max raw file size the omitted-image recovery will re-read. pi's `read`
+ *  already read the file; this bounds the handoff's re-read so a pathologically
+ *  huge file doesn't double the memory. 20MB covers the most generous vision
+ *  model inline-image limit; a larger file would be rejected by the vision
+ *  model anyway. */
+export const MAX_RECOVER_IMAGE_BYTES = 20 * 1024 * 1024;
+
+/** pi core's `read` tool emits this text note (with NO image block) when it
+ *  detected an image but `processImage` failed — Photon/WASM unavailable,
+ *  decode failure, convert-to-PNG failure, or couldn't resize below the inline
+ *  size limit. The handoff's image-block path never sees these (no image
+ *  block), so the image goes undescribed and the model is told the image was
+ *  "omitted". This detects that note so the tool_result handler can re-read the
+ *  raw file and describe its bytes directly (the vision model decodes them —
+ *  no Photon needed). */
+export function isOmittedImageNote(text: string): boolean {
+  return text.includes("Read image file [") && text.includes("[Image omitted:");
+}
+
+/** Read an image file for the omitted-image recovery, bounded by
+ *  {@link MAX_RECOVER_IMAGE_BYTES}. Returns null if the file can't be read, is
+ *  too large, or isn't a supported image (aligned with pi's sniff — APNG and
+ *  unsupported formats are rejected, since the vision model can't decode them
+ *  either). */
+export function readImageBufferBounded(filePath: string): { buf: Buffer; mimeType: string } | null {
+  try {
+    const stat = statSync(filePath);
+    if (stat.size > MAX_RECOVER_IMAGE_BYTES) return null;
+  } catch {
+    return null;
+  }
+  return readImageBuffer(filePath);
+}
+
 /** Regex matching pi's pasted-clipboard temp image file paths anywhere in the
  *  prompt text. pi writes pasted clipboard images to
  *  `<tmpdir>/pi-clipboard-<uuid>.<ext>` and inserts the path as text at the

package/vision-handoff.ts

@@ -30,6 +30,7 @@
 
 import type { ExtensionAPI, ExtensionCommandContext, ExtensionContext, ModelRegistry } from "@earendil-works/pi-coding-agent";
 import type { Api, ImageContent, Model, TextContent } from "@earendil-works/pi-ai";
+import { isAbsolute, resolve } from "node:path";
 import {
   extractImageFromBlock,
   formatModelRef,
@@ -50,24 +51,12 @@ import {
   type VisionHandoffUsageRecord,
 } from "./src/usage.js";
 import { DescriptionLoader, UNAVAILABLE, type LoaderDeps } from "./src/dataloader.js";
-import { imageHash, findClipboardImagePaths, readImageBuffer, resolvePrewarmImage } from "./src/image.js";
+import { imageHash, findClipboardImagePaths, readImageBuffer, readImageBufferBounded, resolvePrewarmImage, isOmittedImageNote } from "./src/image.js";
 import { appendVisionError } from "./src/error-log.js";
 import { resizeImage } from "@earendil-works/pi-coding-agent";
 import { VisionModelSelectorComponent, type VisionModelSelectorResult } from "./src/vision-model-selector.js";
 import { PrewarmEditor } from "./src/prewarm-editor.js";
 
-/**
- * pi-fabric prepends this prefix to every nested tool-call id it generates
- * inside a fabric_exec run (one per pi.* invocation in full-code mode). The
- * LLM's own tool-call ids (e.g. openai "call_…", anthropic "toolu_…") never use
- * it, so a tool_result whose toolCallId starts with this prefix is a nested
- * fabric call whose result returns to the sandbox program — not the agent's
- * message context — so the context hook will never see its image blocks. We
- * therefore swap image→description inline in the tool_result handler for
- * nested calls. Mirror of pi-fabric's NESTED_TOOL_CALL_ID_PREFIX.
- */
-const FABRIC_NESTED_TOOL_CALL_ID_PREFIX = "fabric_";
-
 let config: VisionHandoffConfig = readConfig();
 
 /** Most recent describer failure message (auth error, network error, abort,
@@ -363,21 +352,15 @@ export default function (pi: ExtensionAPI) {
   // iteration, so reads completing together land in ONE batch and all resolve
   // together.
   //
-  // Two shapes of caller reach this handler:
-  //  • Normal flow — the agent called `read` directly, so this tool result IS
-  //    the agent's tool result. The `context` hook will swap image→text on the
-  //    LLM-bound clone, so here we only WARM the cache (during the free
-  //    tool-result phase) and strip pi's misleading non-vision note, KEEPING
-  //    the image block so kitty renders it inline and /resume retains it.
-  //  • pi-fabric full-code mode — `pi.read` runs NESTED inside fabric_exec, so
-  //    the result returns to the sandbox program, not the agent's message
-  //    context. The `context` hook never sees these image blocks, and the
-  //    nested result is transient (never rendered by kitty or stored in
-  //    /resume). So we SWAP image→description inline here, mirroring the
-  //    `context` handler, so the description becomes pi.read's return value:
-  //    pi-fabric's normalizeResult extracts the text and the text-only agent
-  //    receives the description instead of a dropped image block. Nested
-  //    fabric calls are tagged with a `fabric_`-prefixed toolCallId.
+  // Both a direct agent `read` and a NESTED `pi.read` inside fabric_exec reach
+  // this handler and take the SAME path: WARM the cache (during the free
+  // tool-result phase), strip pi's misleading non-vision note, and KEEP the
+  // image block so kitty renders it inline and /resume retains it. The
+  // `context` hook swaps image→description on the LLM-bound clone before the
+  // next turn, so the text-only agent receives the description as text. This
+  // holds for fabric too: pi-fabric re-attaches the nested read's image to the
+  // fabric_exec tool-result content, which IS the agent's message context — so
+  // the `context` hook sees and swaps it, exactly like a direct read.
   pi.on("tool_result", async (event, ctx) => {
     if (!isConfigured(config)) return;
     if (event.toolName !== "read") return;
@@ -386,13 +369,51 @@ export default function (pi: ExtensionAPI) {
 
     // Collect image blocks in this read result. (The read tool emits image
     // blocks even for non-vision models — they reach here untouched.)
+    if (!isHandoffTarget(ctx.model, config)) return;
     const imgs: ExtractedImage[] = [];
+    let hasImageBlock = false;
     for (const block of content) {
       const img = extractImageFromBlock(block);
-      if (img) imgs.push(img);
+      if (img) {
+        imgs.push(img);
+        hasImageBlock = true;
+      }
+    }
+
+    // Fallback: `read` detected an image but `processImage` failed (Photon
+    // unavailable / decode fail / convert fail / couldn't resize below the
+    // inline limit), so it emitted a "[Image omitted: …]" text note with NO
+    // image block. The image-block path above never sees it, so the image
+    // would go undescribed and the model would be told the image was "omitted".
+    // Re-read the raw file and describe its bytes directly — the vision model
+    // decodes them itself (no Photon needed). This recovers the Photon-
+    // unavailable and under-vision-model-limit cases; APNG/unsupported are
+    // rejected by the sniff (the vision model can't decode them either).
+    let omittedNoteIndices: number[] = [];
+    if (!hasImageBlock) {
+      for (let i = 0; i < content.length; i++) {
+        const block = content[i];
+        if (
+          block &&
+          typeof block === "object" &&
+          (block as { type: string }).type === "text" &&
+          typeof (block as { text: string }).text === "string" &&
+          isOmittedImageNote((block as { text: string }).text)
+        ) {
+          omittedNoteIndices.push(i);
+        }
+      }
+      if (omittedNoteIndices.length > 0) {
+        const inputPath = event.input.path;
+        if (typeof inputPath === "string") {
+          const resolved = isAbsolute(inputPath) ? inputPath : resolve(ctx.cwd, inputPath);
+          const read = readImageBufferBounded(resolved);
+          if (read) imgs.push({ data: read.buf.toString("base64"), mimeType: read.mimeType });
+        }
+      }
     }
+
     if (imgs.length === 0) return;
-    if (!isHandoffTarget(ctx.model, config)) return;
     loader.bindTurnContext(ctx);
     if (!resolveVisionModel(ctx.modelRegistry, config.visionModel!)) {
       notifyUnresolvedVisionModel(ctx, config.visionModel!);
@@ -416,47 +437,19 @@ export default function (pi: ExtensionAPI) {
 
     warnFailedImages(ctx, imgs, descs, lastDescriberError ?? "unknown error");
 
-    // pi-fabric full-code mode: swap image→description inline (see the header
-    // comment). The descriptions are already wrapped ([Image: …]) by the
-    // dataloader, so push them directly; strip pi's non-vision note from any
-    // surviving text blocks since the description replaces the omitted image.
-    if (
-      typeof event.toolCallId === "string" &&
-      event.toolCallId.startsWith(FABRIC_NESTED_TOOL_CALL_ID_PREFIX)
-    ) {
-      const descByHash = new Map<string, string>();
-      for (let i = 0; i < imgs.length; i++) {
-        descByHash.set(imageHash(imgs[i].mimeType, imgs[i].data), descs[i]);
-      }
-      const next: (TextContent | ImageContent)[] = [];
-      for (const block of content) {
-        const img = extractImageFromBlock(block);
-        if (img) {
-          next.push({
-            type: "text",
-            text: descByHash.get(imageHash(img.mimeType, img.data)) ?? UNAVAILABLE,
-          });
-          continue;
-        }
-        if (
-          block &&
-          typeof block === "object" &&
-          (block as { type: string }).type === "text" &&
-          typeof (block as { text: string }).text === "string" &&
-          (block as { text: string }).text.includes(NON_VISION_IMAGE_NOTE)
-        ) {
-          next.push({
-            type: "text",
-            text: stripNonVisionImageNote((block as { text: string }).text),
-          });
-        } else {
-          next.push(block as TextContent | ImageContent);
-        }
+    // Recovery path: no image block was emitted, so there's nothing for the
+    // `context` hook to swap. Replace each "[Image omitted: …]" note directly
+    // with its description (the recovered raw-bytes image is the only img here,
+    // so descs aligns 1:1 with omittedNoteIndices).
+    if (!hasImageBlock) {
+      const recovered = content.slice();
+      for (let i = 0; i < omittedNoteIndices.length && i < descs.length; i++) {
+        recovered[omittedNoteIndices[i]] = { type: "text", text: descs[i] };
       }
-      return { content: next };
+      return { content: recovered as (TextContent | ImageContent)[] };
     }
 
-    // Normal flow: warm the cache (done above) and strip pi's
+    // Warm the cache (done above) and strip pi's
     // `[Current model does not support images…]` note from text blocks —
     // since the handoff replaces the image with a description in the
     // `context` hook, that note is misleading (the agent WILL receive the