pi-vision-handoff 0.6.1 → 0.7.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "pi-vision-handoff",
-  "version": "0.6.1",
+  "version": "0.7.0",
   "description": "Give text-only pi models vision — describe images with a vision model you pick via an interactive picker, then hand off the text description to non-vision models",
   "type": "module",
   "author": "Tom X Nguyen",

package/vision-handoff.ts

@@ -56,6 +56,18 @@ import { resizeImage } from "@earendil-works/pi-coding-agent";
 import { VisionModelSelectorComponent, type VisionModelSelectorResult } from "./src/vision-model-selector.js";
 import { PrewarmEditor } from "./src/prewarm-editor.js";
 
+/**
+ * pi-fabric prepends this prefix to every nested tool-call id it generates
+ * inside a fabric_exec run (one per pi.* invocation in full-code mode). The
+ * LLM's own tool-call ids (e.g. openai "call_…", anthropic "toolu_…") never use
+ * it, so a tool_result whose toolCallId starts with this prefix is a nested
+ * fabric call whose result returns to the sandbox program — not the agent's
+ * message context — so the context hook will never see its image blocks. We
+ * therefore swap image→description inline in the tool_result handler for
+ * nested calls. Mirror of pi-fabric's NESTED_TOOL_CALL_ID_PREFIX.
+ */
+const FABRIC_NESTED_TOOL_CALL_ID_PREFIX = "fabric_";
+
 let config: VisionHandoffConfig = readConfig();
 
 /** Most recent describer failure message (auth error, network error, abort,
@@ -349,15 +361,23 @@ export default function (pi: ExtensionAPI) {
   // `tool_result` as its I/O completes (poll phase), and the loader's
   // `setImmediate` dispatch defers to the check phase AFTER the whole poll
   // iteration, so reads completing together land in ONE batch and all resolve
-  // together. The descriptions replace the image blocks in the returned
-  // `content`, so by the time the agent's next turn starts the tool results
-  // already carry text — the agent never sees raw image blocks it can't
-  // process.
+  // together.
   //
-  // Why block here and not in `context`: the tool-result phase is free time
-  // (the agent is just waiting for tool results), so running the describer
-  // here adds zero latency to the critical path. `context` then becomes a
-  // cache-hit no-op for read images.
+  // Two shapes of caller reach this handler:
+  //  • Normal flow — the agent called `read` directly, so this tool result IS
+  //    the agent's tool result. The `context` hook will swap image→text on the
+  //    LLM-bound clone, so here we only WARM the cache (during the free
+  //    tool-result phase) and strip pi's misleading non-vision note, KEEPING
+  //    the image block so kitty renders it inline and /resume retains it.
+  //  • pi-fabric full-code mode — `pi.read` runs NESTED inside fabric_exec, so
+  //    the result returns to the sandbox program, not the agent's message
+  //    context. The `context` hook never sees these image blocks, and the
+  //    nested result is transient (never rendered by kitty or stored in
+  //    /resume). So we SWAP image→description inline here, mirroring the
+  //    `context` handler, so the description becomes pi.read's return value:
+  //    pi-fabric's normalizeResult extracts the text and the text-only agent
+  //    receives the description instead of a dropped image block. Nested
+  //    fabric calls are tagged with a `fabric_`-prefixed toolCallId.
   pi.on("tool_result", async (event, ctx) => {
     if (!isConfigured(config)) return;
     if (event.toolName !== "read") return;
@@ -385,19 +405,9 @@ export default function (pi: ExtensionAPI) {
     // loader's `setImmediate` dispatch defers to the check phase, AFTER the
     // whole poll iteration, so reads completing together (the common case for
     // cached local files) land in ONE batch — ONE vision call for the whole
-    // read set, not N. Reads completing in separate iterations get separate
-    // calls, but always in parallel, never sequential. Awaiting here runs the
-    // describer during the tool-result phase (free time — the agent is just
-    // waiting for tool results), so the batch is COMPLETE before `context`
-    // fires, making `context` a non-blocking cache hit instead of a cold miss
-    // on the critical path.
-    //
-    // We do NOT mutate the result content here for the image blocks: returning
-    // undefined keeps the image block in storage so kitty renders it inline
-    // and `/resume` retains it. The actual image→text swap happens in the
-    // `context` hook (on the cloned LLM-bound payload only), by which point
-    // these are cache hits. We DO strip pi's misleading non-vision note (below)
-    // since the handoff will replace the image with a description.
+    // read set, not N. Awaiting here runs the describer during the tool-result
+    // phase (free time), so the batch is COMPLETE before `context` fires,
+    // making `context` a non-blocking cache hit instead of a cold miss.
     const descs = await Promise.all(imgs.map((img) => loader.loadDescription(img)));
 
     // On user abort, leave the result untouched — pi is tearing the turn
@@ -406,12 +416,53 @@ export default function (pi: ExtensionAPI) {
 
     warnFailedImages(ctx, imgs, descs, lastDescriberError ?? "unknown error");
 
-    // Strip pi's `[Current model does not support images…]` note from text
-    // blocks — since the handoff replaces the image with a description in the
+    // pi-fabric full-code mode: swap image→description inline (see the header
+    // comment). The descriptions are already wrapped ([Image: …]) by the
+    // dataloader, so push them directly; strip pi's non-vision note from any
+    // surviving text blocks since the description replaces the omitted image.
+    if (
+      typeof event.toolCallId === "string" &&
+      event.toolCallId.startsWith(FABRIC_NESTED_TOOL_CALL_ID_PREFIX)
+    ) {
+      const descByHash = new Map<string, string>();
+      for (let i = 0; i < imgs.length; i++) {
+        descByHash.set(imageHash(imgs[i].mimeType, imgs[i].data), descs[i]);
+      }
+      const next: (TextContent | ImageContent)[] = [];
+      for (const block of content) {
+        const img = extractImageFromBlock(block);
+        if (img) {
+          next.push({
+            type: "text",
+            text: descByHash.get(imageHash(img.mimeType, img.data)) ?? UNAVAILABLE,
+          });
+          continue;
+        }
+        if (
+          block &&
+          typeof block === "object" &&
+          (block as { type: string }).type === "text" &&
+          typeof (block as { text: string }).text === "string" &&
+          (block as { text: string }).text.includes(NON_VISION_IMAGE_NOTE)
+        ) {
+          next.push({
+            type: "text",
+            text: stripNonVisionImageNote((block as { text: string }).text),
+          });
+        } else {
+          next.push(block as TextContent | ImageContent);
+        }
+      }
+      return { content: next };
+    }
+
+    // Normal flow: warm the cache (done above) and strip pi's
+    // `[Current model does not support images…]` note from text blocks —
+    // since the handoff replaces the image with a description in the
     // `context` hook, that note is misleading (the agent WILL receive the
     // image's content, as text). Keep the image block itself so kitty still
-    // renders it inline and `/resume` retains it; the `context` hook swaps the
-    // image for its description in the LLM-bound clone before the next turn.
+    // renders it inline and `/resume` retains it; the `context` hook swaps
+    // the image for its description in the LLM-bound clone before the next turn.
     let stripped = false;
     const next = content.slice();
     for (let i = 0; i < next.length; i++) {