pi-vision-handoff 0.1.0 → 0.2.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "pi-vision-handoff",
-  "version": "0.1.0",
+  "version": "0.2.0",
   "description": "Give text-only pi models vision — describe images with a vision model you pick via an interactive picker, then hand off the text description to non-vision models",
   "type": "module",
   "author": "Tom X Nguyen",

package/src/index.ts

@@ -5,6 +5,7 @@
  * convention pi-model-sort uses for picker-backed extensions.
  */
 
+import type { ImageContent, TextContent } from "@earendil-works/pi-ai";
 import { getAgentDir } from "@earendil-works/pi-coding-agent";
 import { existsSync, mkdirSync, readFileSync, writeFileSync } from "node:fs";
 import { join } from "node:path";
@@ -44,6 +45,18 @@ export const DEFAULT_CACHE_MAX = 50;
 /** Per-description request timeout. */
 export const DESCRIBE_TIMEOUT_MS = 30_000;
 
+/**
+ * Default cap on the number of lines kept from a description before truncation.
+ *
+ * Mirrors pi core's read-result truncation: tool output is bounded for both the
+ * TUI render and the model context. The stored `result.content` is the single
+ * source for both surfaces (no decouple point exists at `tool_result` — it's the
+ * only hook that still has the raw image before pi-ai strips it), so the cap
+ * applies uniformly. 0 = unbounded (default): the read tool's native collapse
+ * handles compactness and `ctrl+o` expands to the full description.
+ */
+export const DEFAULT_MAX_DESCRIPTION_LINES = 0;
+
 export interface VisionHandoffConfig {
   /** Master switch. When false, no handoff occurs even if a vision model is configured. */
   enabled: boolean;
@@ -57,6 +70,12 @@ export interface VisionHandoffConfig {
   maxTokens: number;
   /** Max images kept in the in-memory description cache. */
   cacheMax: number;
+  /**
+   * Max lines kept from a description before truncation (0 = unbounded).
+   * Bounds both the TUI render and the model context, like pi core's tool-output
+   * truncation.
+   */
+  maxDescriptionLines: number;
   /** Override the describer system prompt (defaults to DEFAULT_VISION_PROMPT). */
   prompt?: string;
   /** Override the user-prompt prefix (defaults to DEFAULT_USER_PROMPT_PREFIX). */
@@ -70,6 +89,7 @@ export const DEFAULT_CONFIG: VisionHandoffConfig = {
   handoffModels: [],
   maxTokens: DEFAULT_MAX_TOKENS,
   cacheMax: DEFAULT_CACHE_MAX,
+  maxDescriptionLines: DEFAULT_MAX_DESCRIPTION_LINES,
 };
 
 /** Parse a "provider/id" reference. Returns null if malformed. */
@@ -120,6 +140,14 @@ export function normalizeConfig(raw: unknown): VisionHandoffConfig {
   if (typeof obj.cacheMax === "number" && Number.isFinite(obj.cacheMax) && obj.cacheMax > 0) {
     base.cacheMax = Math.floor(obj.cacheMax);
   }
+  // maxDescriptionLines: any non-negative finite integer. 0 = unbounded.
+  if (
+    typeof obj.maxDescriptionLines === "number" &&
+    Number.isFinite(obj.maxDescriptionLines) &&
+    obj.maxDescriptionLines >= 0
+  ) {
+    base.maxDescriptionLines = Math.floor(obj.maxDescriptionLines);
+  }
   if (typeof obj.prompt === "string" && obj.prompt.trim()) base.prompt = obj.prompt;
   if (typeof obj.userPromptPrefix === "string") base.userPromptPrefix = obj.userPromptPrefix;
 
@@ -181,10 +209,20 @@ export function extractImageFromBlock(block: unknown): ExtractedImage | null {
     return null;
   }
 
+  // anthropic-messages image block (wrapped in `source`).
   if (b.type === "image" && b.source?.type === "base64" && typeof b.source.data === "string") {
     return { data: b.source.data, mimeType: b.source.media_type || "image/png" };
   }
 
+  // pi-ai internal image block — emitted by the `read` tool and carried by
+  // ToolResultEvent.content / user-message content blocks (regardless of whether
+  // the active model declares image input):
+  //   { type: "image", data: "<base64>", mimeType }
+  // Distinct from the anthropic shape above: no `source` wrapper, direct fields.
+  if (b.type === "image" && typeof b.data === "string") {
+    return { data: b.data, mimeType: b.mimeType || "image/png" };
+  }
+
   return null;
 }
 
@@ -196,3 +234,86 @@ export function makeReplacementText(block: unknown, description: string): Record
   }
   return { type: "text", text: description };
 }
+
+/** Outcome of {@link truncateDescription}. */
+export interface TruncatedDescription {
+  text: string;
+  /** True iff lines were dropped. */
+  truncated: boolean;
+  /** Number of trailing lines removed. */
+  hidden: number;
+}
+
+/**
+ * Head-truncate a description to its first `maxLines` lines with a
+ * "... (N more lines)" footer, mirroring pi core's read-result truncation
+ * notice (which itself surfaces "... (N more lines, ctrl+o to expand)").
+ *
+ * Returns the text unchanged when it is at or below the limit, or when
+ * `maxLines` is non-positive (0 = unbounded). Head-truncation keeps the leading
+ * layout / text-content / structure sections, which are typically the most
+ * actionable for a coding agent.
+ */
+export function truncateDescription(text: string, maxLines: number): TruncatedDescription {
+  if (!maxLines || maxLines <= 0) {
+    return { text, truncated: false, hidden: 0 };
+  }
+  const lines = text.split("\n");
+  if (lines.length <= maxLines) {
+    return { text, truncated: false, hidden: 0 };
+  }
+  const hidden = lines.length - maxLines;
+  const kept = lines.slice(0, maxLines).join("\n");
+  return { text: `${kept}\n... (${hidden} more lines)`, truncated: true, hidden };
+}
+
+/** Outcome of {@link insertImageDescriptions}. */
+export interface ReplacedContent {
+  content: (TextContent | ImageContent)[];
+  /** True iff at least one image block had a description inserted before it. */
+  changed: boolean;
+}
+
+/**
+ * Insert a description text block before each image block in a tool-result /
+ * message content array, KEEPING the image block in place.
+ *
+ * Why insert (not replace): the stored tool-result content is the single
+ * source for both the TUI render (kitty inline images read it via
+ * `result.content.filter(c => c.type === "image")`) and the provider payload.
+ * Replacing the image would strip it from the terminal render. Keeping the
+ * image preserves kitty rendering; the inserted description still reaches
+ * non-vision models because pi-ai's `downgradeUnsupportedImages` only rewrites
+ * `type: "image"` blocks for non-vision models — text blocks (including this
+ * description) pass through untouched to the provider.
+ *
+ * `describe` is injected (rather than calling the vision model directly) so the
+ * extract → describe → insert pipeline is unit-testable without standing up a
+ * provider, registry, and API call. The extension wires its real `describeImage`
+ * into this helper in its `tool_result` handler.
+ *
+ * Returns a new array and `changed: false` when there were no images, so
+ * callers can short-circuit and avoid mutating pi's stored result unnecessarily.
+ */
+export async function insertImageDescriptions(
+  content: readonly (TextContent | ImageContent)[] | undefined,
+  describe: (img: ExtractedImage) => Promise<string>,
+): Promise<ReplacedContent> {
+  if (!Array.isArray(content)) {
+    return { content: [], changed: false };
+  }
+  const next: (TextContent | ImageContent)[] = [];
+  let changed = false;
+  for (const block of content) {
+    const img = extractImageFromBlock(block);
+    if (!img) {
+      next.push(block);
+      continue;
+    }
+    const description = await describe(img);
+    next.push({ type: "text", text: description } satisfies TextContent);
+    next.push(block);
+    changed = true;
+  }
+  return { content: next, changed };
+}

package/vision-handoff.ts

@@ -9,12 +9,18 @@
  *
  * Pipeline (provider-agnostic via @earendil-works/pi-ai's complete()):
  *   before_agent_start → warm the description cache for attached images
- *   before_provider_request → swap image blocks in the payload for text
+ *   tool_result (read) → describe read-tool images and INSERT the description
+ *     as a text block before each image, keeping the image so the TUI still
+ *     renders it (kitty). pi-ai later strips the image for non-vision models,
+ *     leaving the description text for the model.
+ *   before_provider_request → swap remaining image blocks in the payload for
+ *     text (catches user-attached images for vision-capable handoff targets)
  *
- * Image blocks are detected by shape across the three request formats pi uses:
+ * Image blocks are detected by shape across the four formats pi uses:
  *   openai-completions: { type: "image_url",  image_url: { url: "data:..." } }
  *   openai-responses:   { type: "input_image", image_url: "data:..." }
  *   anthropic-messages: { type: "image", source: { type: "base64", media_type, data } }
+ *   pi-ai internal:     { type: "image", data, mimeType }   ← read tool / ToolResultEvent
  *
  * Descriptions are cached per image hash (LRU, size = config.cacheMax) so the
  * swap is instant by the time before_provider_request fires.
@@ -33,10 +39,12 @@ import {
   IMAGE_PLACEHOLDER_SUFFIX,
   extractImageFromBlock,
   formatModelRef,
+  insertImageDescriptions,
   isVisionModel,
   makeReplacementText,
   parseModelRef,
   readConfig,
+  truncateDescription,
   writeConfig,
   type VisionHandoffConfig,
 } from "./src/index.js";
@@ -46,6 +54,9 @@ const UNAVAILABLE = `${IMAGE_PLACEHOLDER_PREFIX}description unavailable${IMAGE_P
 
 let config: VisionHandoffConfig = readConfig();
 
+/** User prompt for the current agent turn, captured from before_agent_start. */
+let pendingTurnPrompt: string | null = null;
+
 const visionCache = new Map<string, Promise<string>>();
 let visionModelCache: { ref: string; model: Model<Api> } | null = null;
 let visionModelUnresolvedRef: string | null = null;
@@ -134,7 +145,17 @@ async function describeImage(
         .join("\n")
         .trim();
       if (!description) return UNAVAILABLE;
-      return `${IMAGE_PLACEHOLDER_PREFIX}${description}${IMAGE_PLACEHOLDER_SUFFIX}`;
+      // The full description lives in the stored tool-result content; the read
+      // tool's native collapse (ctrl+o) handles compactness — collapsed shows
+      // the image only, expanded shows the full description + image. Only apply
+      // a hard line cap when the user opts in (maxDescriptionLines > 0); by
+      // default (0) the description is unbounded so ctrl+o can expand to all
+      // of it and the model receives the complete context.
+      const { text: final } =
+        cfg.maxDescriptionLines && cfg.maxDescriptionLines > 0
+          ? truncateDescription(description, cfg.maxDescriptionLines)
+          : { text: description };
+      return `${IMAGE_PLACEHOLDER_PREFIX}${final}${IMAGE_PLACEHOLDER_SUFFIX}`;
     } catch {
       return UNAVAILABLE;
     } finally {
@@ -196,11 +217,17 @@ export default function (pi: ExtensionAPI) {
     // Reload in case the user edited the config on disk from another session.
     config = readConfig();
     visionModelCache = null;
+    pendingTurnPrompt = null;
   });
 
   pi.on("before_agent_start", async (event, ctx) => {
     if (!isConfigured(config)) return;
     if (!isHandoffTarget(ctx.model, config)) return;
+
+    // Capture this turn's user prompt so read-tool images are described in
+    // the same context. Fires before any tool_result for the turn.
+    pendingTurnPrompt = event.prompt || "";
+
     const images = event.images;
     if (!images || images.length === 0) return;
 
@@ -235,6 +262,29 @@ export default function (pi: ExtensionAPI) {
     return payload;
   });
 
+  pi.on("tool_result", async (event, ctx) => {
+    if (!isConfigured(config)) return;
+    if (!isHandoffTarget(ctx.model, config)) return;
+    if (event.toolName !== "read") return;
+
+    const content = event.content;
+    if (!Array.isArray(content)) return;
+    // Skip the async work entirely when there is nothing to describe.
+    if (!content.some((c) => extractImageFromBlock(c))) return;
+
+    const visionModel = resolveVisionModel(ctx.modelRegistry, config.visionModel!);
+    if (!visionModel) {
+      notifyUnresolvedVisionModel(ctx, config.visionModel!);
+      return;
+    }
+
+    const { content: next, changed } = await insertImageDescriptions(content, (img) =>
+      describeImage(img.data, img.mimeType, pendingTurnPrompt ?? "", visionModel, ctx.modelRegistry, config),
+    );
+    if (!changed) return;
+    return { content: next };
+  });
+
   pi.on("model_select", (event, ctx) => {
     if (!ctx.hasUI) return;
     if (!isConfigured(config)) return;
@@ -464,7 +514,7 @@ function showStatus(ctx: ExtensionCommandContext): void {
   lines.push(`Vision model: ${config.visionModel ?? "(none — pick one with /vision-handoff)"}`);
   lines.push(`Auto handoff (non-vision models): ${config.autoHandoff ? "on" : "off"}`);
   lines.push(`Handoff targets (explicit): ${config.handoffModels.length ? config.handoffModels.join(", ") : "(none)"}`);
-  lines.push(`maxTokens: ${config.maxTokens} · cacheMax: ${config.cacheMax}`);
+  lines.push(`maxTokens: ${config.maxTokens} · cacheMax: ${config.cacheMax} · maxDescriptionLines: ${config.maxDescriptionLines === 0 ? "unbounded" : config.maxDescriptionLines}`);
 
   const model = ctx.model;
   let active = false;