pi-vision-handoff 0.5.1 → 0.6.0

package/README.md

@@ -45,6 +45,7 @@ No `settings.json` touched. No per-provider glue. Pick a describer once and ever
 - **⚡ Pre-warmed at paste-enter** — the moment you press enter, `before_agent_start` scans the prompt for pasted clipboard image temp-file paths (pi writes pasted images to `<tmpdir>/pi-clipboard-<uuid>.<ext>` and inserts the path as text), reads them, and kicks off the ONE batched vision call concurrent with the agent's first response — so by the time the agent reads the files, they're already cache hits.
 - **🚀 Paste-time prewarm (opt-in)** — `prewarmPastedImages` (off by default) wraps the editor so pasted clipboard images are described the *instant* their path lands in the prompt — before you hit enter — not at submit. The vision call starts concurrent with you typing your question. Tradeoff: the description is generated without your typed question as context (the question usually isn't entered yet at paste time), and a paste-then-abandon wastes one vision call. TUI only; inactive if another extension replaces the editor. Toggle with `/vision-handoff prewarm on`.
 - **🛡️ Graceful degradation** — no API key? Describer unreachable? Aborted? The image is replaced with a clean `[Image: description unavailable]` placeholder instead of crashing your turn.
+- **📋 Error logging** — every describer failure and every `image description failed` warning is appended as a JSONL line to `~/.pi/agent/logs/pi-vision-handoff/errors.log`, so a warning like `image description failed — unknown error` is troubleshootable: the detailed reason (exception stack, `stopReason`, config snapshot) is captured at the describer's failure source even when the in-memory error string was already reset by the time the warning fired. Size-capped with a single `.1` backup.
 - **📊 Usage reporting** — every real describer call reports model + tokens (and Neuralwatt energy/cost when the vision model is a Neuralwatt model), via `pi.appendEntry` + a `vision-handoff:usage` event for live consumers.
 - **🔧 Tunable** — cap description length (`maxDescriptionLines`; unbounded by default, so `read`'s native `ctrl+o` collapse handles compactness) and cache size, in the config file.
 
@@ -100,6 +101,20 @@ Created automatically at `~/.pi/agent/extensions/pi-vision-handoff.json` on firs
 
 > The config path uses pi's `getAgentDir()` — set `PI_CODING_AGENT_DIR` to relocate it.
 
+### Troubleshooting
+
+When handoff fails, pi shows a short warning like `pi-vision-handoff: image description failed — <reason>. Vision model: <model>`. The full detail is in the error log:
+
+```
+~/.pi/agent/logs/pi-vision-handoff/errors.log
+```
+
+(one JSON object per line; relocate via `PI_CODING_AGENT_DIR`). Each entry records the `phase` (`batch` / `single` = describer failure source, `warn` = the user-facing warning), the `reason`, the `visionModel`, the failing `imageHashes`, and — when applicable — the `stopReason`, `errorMessage`, `errorStack`, and a `config` snapshot (`maxTokens` / `thinking` / `thinkingLevel`).
+
+**`reason: "unknown error"`** in a `warn` entry means the engine's shared last-error string was already reset (typically a concurrent batch cleared it) by the time the warning fired — so the real cause isn't in the warning. Correlate it with the matching `batch`/`single` entry: match on the `imageHashes` (and the timestamp) to recover the actual exception/`stopReason`. Deliberate user cancels (ESC) are not logged — they aren't errors.
+
+The log is append-only and size-capped (10 MB, rotating to `errors.log.1`), so a runaway broken vision model can't fill the disk; truncate it any time.
+
 ## Installation
 
 **With `pi install`** (recommended):
@@ -372,6 +387,7 @@ pnpm lint:dead     # Dead code detection (knip)
 │   ├── image.ts                  # Image hashing, MIME sniffing, clipboard-path reading
 │   ├── prewarm-editor.ts        # Opt-in paste-time prewarm CustomEditor wrapper (chains onChange)
 │   ├── dispose.ts                 # `Disposable` guard factories for `using` (fetch interceptor, timer, abort wire)
+│   ├── error-log.ts              # Best-effort JSONL error log → ~/.pi/agent/logs/pi-vision-handoff/errors.log
 │   ├── usage.ts                  # Describer usage + Neuralwatt energy capture, fetch interceptor
 │   └── vision-model-selector.ts  # Interactive picker TUI component
 ├── __tests__/unit/
@@ -379,8 +395,9 @@ pnpm lint:dead     # Dead code detection (knip)
 │   ├── usage.test.ts             # Energy parsing, usage records, concurrency-safe fetch routing
 │   ├── vision-handoff.test.ts    # Config, refs, image-block extraction, insertion, truncation, round-trip
 │   ├── dataloader.test.ts        # Batch coalescing, memoization, failure eviction, Disposable reset
-│   ├── describer.test.ts        # stopReason handling (length → truncation marker, aborted/error)
+│   ├── describer.test.ts        # stopReason handling (length → truncation marker, aborted/error), error-log wiring
 │   ├── image.test.ts             # MIME sniffing, clipboard-path confinement, file reading, diffPrewarmPaths
+│   ├── error-log.test.ts         # JSONL error log: path resolution, append, size-capped rotation, never-throws
 │   └── dispose.test.ts           # `using` guards: fetch refcount, timeout, abort-wire propagation
 ├── package.json
 ├── tsconfig.json

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "pi-vision-handoff",
-  "version": "0.5.1",
+  "version": "0.6.0",
   "description": "Give text-only pi models vision — describe images with a vision model you pick via an interactive picker, then hand off the text description to non-vision models",
   "type": "module",
   "author": "Tom X Nguyen",

package/src/describer.ts

@@ -34,11 +34,13 @@ import {
   DEFAULT_USER_PROMPT_PREFIX,
   DEFAULT_VISION_PROMPT,
   describeTimeoutMs,
+  formatModelRef,
   markDescriptionTruncated,
   parseBatchedDescriptions,
   type ExtractedImage,
   type VisionHandoffConfig,
 } from "./index.js";
+import { appendVisionError } from "./error-log.js";
 import {
   buildUsageRecord,
   describeAls,
@@ -104,6 +106,17 @@ export function resolveReasoning(
   return cfg.thinkingLevel;
 }
 
+/** Small config snapshot embedded in every error-log entry — the fields most
+ *  relevant to troubleshooting a describer failure (a bad maxTokens clamp, a
+ *  reasoning level the model rejects, etc.). */
+function configSnapshot(cfg: VisionHandoffConfig): {
+  maxTokens?: number;
+  thinking: boolean;
+  thinkingLevel: string;
+} {
+  return { maxTokens: cfg.maxTokens, thinking: cfg.thinking, thinkingLevel: cfg.thinkingLevel };
+}
+
 /** Dependencies the describer can't own itself (held by the engine). */
 export interface DescriberDeps {
   /** Report a usage+energy record for one real describer call (cache hits emit none). */
@@ -138,9 +151,18 @@ export async function runBatch(
   const out: BatchResult = new Map();
   const auth = await modelRegistry.getApiKeyAndHeaders(visionModel);
   if (!auth.ok || !auth.apiKey) {
-    deps.setLastError(
-      !auth.ok ? auth.error : `No API key for vision model "${visionModel.provider}/${visionModel.id}"`,
-    );
+    const reason = !auth.ok
+      ? auth.error
+      : `No API key for vision model "${formatModelRef(visionModel.provider, visionModel.id)}"`;
+    deps.setLastError(reason);
+    appendVisionError({
+      phase: "batch",
+      reason,
+      visionModel: cfg.visionModel,
+      imageHashes: misses.map((m) => m.hash),
+      imageCount: misses.length,
+      config: configSnapshot(cfg),
+    });
     return out;
   }
 
@@ -178,7 +200,28 @@ export async function runBatch(
     const record = buildUsageRecord(response, capture, visionModel, hashes[0], hashes.length > 1 ? hashes : undefined);
     if (record) deps.reportUsage(record);
     if (response.stopReason === "aborted" || response.stopReason === "error") {
-      setStopReasonError(deps, response.stopReason, response.errorMessage, abortWire, timedOut, timeoutMs);
+      const reason = setStopReasonError(
+        deps,
+        response.stopReason,
+        response.errorMessage,
+        abortWire,
+        timedOut,
+        timeoutMs,
+      );
+      if (reason) {
+        appendVisionError({
+          phase: "batch",
+          reason,
+          visionModel: cfg.visionModel,
+          imageHashes: hashes,
+          imageCount: misses.length,
+          stopReason: response.stopReason,
+          timedOut,
+          timeoutMs,
+          errorMessage: response.errorMessage,
+          config: configSnapshot(cfg),
+        });
+      }
       return out;
     }
     const text = response.content
@@ -188,6 +231,14 @@ export async function runBatch(
       .trim();
     if (!text) {
       deps.setLastError("vision model returned an empty description");
+      appendVisionError({
+        phase: "batch",
+        reason: "vision model returned an empty description",
+        visionModel: cfg.visionModel,
+        imageHashes: hashes,
+        imageCount: misses.length,
+        config: configSnapshot(cfg),
+      });
       return out;
     }
     const parsed = parseBatchedDescriptions(text, misses.length);
@@ -227,9 +278,28 @@ export async function runBatch(
     }
     return out;
   } catch (err) {
-    deps.setLastError(
-      timedOut ? `describer timed out after ${timeoutMs / 1000}s` : err instanceof Error ? err.message : String(err),
-    );
+    const userAborted = abortWire.userAborted();
+    const reason = timedOut
+      ? `describer timed out after ${timeoutMs / 1000}s`
+      : err instanceof Error
+        ? err.message
+        : String(err);
+    deps.setLastError(reason);
+    // A deliberate user cancel isn't a troubleshooting-worthy error — skip
+    // logging it (mirroring the no-warn-on-user-abort contract).
+    if (!userAborted) {
+      appendVisionError({
+        phase: "batch",
+        reason,
+        visionModel: cfg.visionModel,
+        imageHashes: misses.map((m) => m.hash),
+        imageCount: misses.length,
+        timedOut,
+        timeoutMs,
+        errorStack: err instanceof Error ? err.stack : undefined,
+        config: configSnapshot(cfg),
+      });
+    }
     return out;
   } finally {
     // The `using` guards above already released the fetch interceptor, timer,
@@ -254,9 +324,18 @@ export async function describeSingle(
 ): Promise<string | null> {
   const auth = await modelRegistry.getApiKeyAndHeaders(visionModel);
   if (!auth.ok || !auth.apiKey) {
-    deps.setLastError(
-      !auth.ok ? auth.error : `No API key for vision model "${visionModel.provider}/${visionModel.id}"`,
-    );
+    const reason = !auth.ok
+      ? auth.error
+      : `No API key for vision model "${formatModelRef(visionModel.provider, visionModel.id)}"`;
+    deps.setLastError(reason);
+    appendVisionError({
+      phase: "single",
+      reason,
+      visionModel: cfg.visionModel,
+      imageHashes: [imageHash(img.mimeType, img.data)],
+      imageCount: 1,
+      config: configSnapshot(cfg),
+    });
     return null;
   }
   const prefix = cfg.userPromptPrefix ?? DEFAULT_USER_PROMPT_PREFIX;
@@ -292,7 +371,28 @@ export async function describeSingle(
     const record = buildUsageRecord(response, capture, visionModel, hash);
     if (record) deps.reportUsage(record);
     if (response.stopReason === "aborted" || response.stopReason === "error") {
-      setStopReasonError(deps, response.stopReason, response.errorMessage, abortWire, timedOut, timeoutMs);
+      const reason = setStopReasonError(
+        deps,
+        response.stopReason,
+        response.errorMessage,
+        abortWire,
+        timedOut,
+        timeoutMs,
+      );
+      if (reason) {
+        appendVisionError({
+          phase: "single",
+          reason,
+          visionModel: cfg.visionModel,
+          imageHashes: [hash],
+          imageCount: 1,
+          stopReason: response.stopReason,
+          timedOut,
+          timeoutMs,
+          errorMessage: response.errorMessage,
+          config: configSnapshot(cfg),
+        });
+      }
       return null;
     }
     const text = response.content
@@ -302,6 +402,14 @@ export async function describeSingle(
       .trim();
     if (!text) {
       deps.setLastError("vision model returned an empty description");
+      appendVisionError({
+        phase: "single",
+        reason: "vision model returned an empty description",
+        visionModel: cfg.visionModel,
+        imageHashes: [hash],
+        imageCount: 1,
+        config: configSnapshot(cfg),
+      });
       return null;
     }
     // stopReason "length" = the model hit a token limit (configured maxTokens or
@@ -309,9 +417,28 @@ export async function describeSingle(
     // useful, but it must not pass as complete — mark it so the agent/user know.
     return response.stopReason === "length" ? markDescriptionTruncated(text) : text;
   } catch (err) {
-    deps.setLastError(
-      timedOut ? `describer timed out after ${timeoutMs / 1000}s` : err instanceof Error ? err.message : String(err),
-    );
+    const userAborted = abortWire.userAborted();
+    const reason = timedOut
+      ? `describer timed out after ${timeoutMs / 1000}s`
+      : err instanceof Error
+        ? err.message
+        : String(err);
+    deps.setLastError(reason);
+    // A deliberate user cancel isn't a troubleshooting-worthy error — skip
+    // logging it (mirroring the no-warn-on-user-abort contract).
+    if (!userAborted) {
+      appendVisionError({
+        phase: "single",
+        reason,
+        visionModel: cfg.visionModel,
+        imageHashes: [imageHash(img.mimeType, img.data)],
+        imageCount: 1,
+        timedOut,
+        timeoutMs,
+        errorStack: err instanceof Error ? err.stack : undefined,
+        config: configSnapshot(cfg),
+      });
+    }
     return null;
   } finally {
     describeCtx.energyReader?.catch(() => {});
@@ -332,7 +459,8 @@ async function readCapture(describeCtx: DescribeContext): Promise<VisionHandoffE
 
 /** Translate a non-OK stopReason into a user-facing failure message, unless the
  *  abort came from the user cancelling the turn (then stay silent — no warning
- *  for a deliberate cancel). */
+ *  for a deliberate cancel). Returns the message that was set (so the caller
+ *  can log it), or null when the abort was a user cancel / nothing was set. */
 function setStopReasonError(
   deps: DescriberDeps,
   stopReason: string,
@@ -340,11 +468,14 @@ function setStopReasonError(
   abortWire: AbortWire,
   timedOut: boolean,
   timeoutMs: number,
-): void {
-  if (abortWire.userAborted()) return; // user cancelled the turn — no warning
+): string | null {
+  if (abortWire.userAborted()) return null; // user cancelled the turn — no warning
   if (timedOut) {
-    deps.setLastError(`describer timed out after ${timeoutMs / 1000}s`);
-    return;
+    const reason = `describer timed out after ${timeoutMs / 1000}s`;
+    deps.setLastError(reason);
+    return reason;
   }
-  deps.setLastError(`vision model returned stopReason "${stopReason}"${errorMessage ? ": " + errorMessage : ""}`);
+  const reason = `vision model returned stopReason "${stopReason}"${errorMessage ? ": " + errorMessage : ""}`;
+  deps.setLastError(reason);
+  return reason;
 }

package/src/error-log.ts

@@ -0,0 +1,131 @@
+/**
+ * Best-effort structured error logging for pi-vision-handoff.
+ *
+ * Every describer failure (auth error, network error, abort, empty response,
+ * stopReason error/aborted, timeout) and every user-facing "image description
+ * failed" warning is appended as one JSONL line to
+ *   ~/.pi/agent/logs/pi-vision-handoff/errors.log
+ * (resolvable via $PI_CODING_AGENT_DIR, like every other pi path).
+ *
+ * This is the troubleshooting surface for the "image description failed —
+ * unknown error" warning. That warning fires with reason "unknown error" when
+ * the engine's shared last-error string was already null — typically because a
+ * concurrent batch reset it between the failure and the warn — so the detailed
+ * reason lives only here, captured at the describer's failure source where the
+ * real exception/stopReason is still in hand. A `warn` entry with reason
+ * "unknown error" carries the failing image hashes; correlate them (and the
+ * timestamp) with the matching `batch`/`single` entry to recover the real
+ * cause.
+ *
+ * Best-effort: logging never throws and never breaks a describer turn. The
+ * directory is created lazily; a single size-based rotation (errors.log →
+ * errors.log.1) bounds growth to ~2× the cap.
+ */
+
+import { getAgentDir } from "@earendil-works/pi-coding-agent";
+import { appendFileSync, existsSync, mkdirSync, renameSync, statSync } from "node:fs";
+import { join } from "node:path";
+
+/** Subdirectory under the pi agent dir holding this extension's logs. */
+const LOG_SUBDIR = "logs/pi-vision-handoff";
+/** Active log file name. */
+const LOG_FILENAME = "errors.log";
+
+/** A single error log is capped at this many bytes before it rotates to the
+ *  single `.1` backup, bounding total on-disk log to ~2× this. Generous enough
+ *  to retain a long troubleshooting history; small enough that a runaway
+ *  broken vision model (failures aren't cached, so they re-fire per turn)
+ *  can't fill the disk. */
+export const MAX_LOG_BYTES = 10 * 1024 * 1024;
+
+/** One structured error-log record (one JSONL line). */
+export interface VisionErrorLogEntry {
+  /** ISO 8601 timestamp (filled by {@link appendVisionError}). */
+  timestamp: string;
+  /** Where the entry originated: "batch"/"single" = describer failure source
+   *  (rich detail), "warn" = the user-facing warning (may carry reason
+   *  "unknown error" when the engine's shared error was already reset). */
+  phase: "batch" | "single" | "warn";
+  /** Human-readable failure reason (what `setLastError` received, or the warn
+   *  reason). "unknown error" in a `warn` entry means no describer error was
+   *  captured — correlate via {@link imageHashes} + {@link timestamp} with the
+   *  matching batch/single entry. */
+  reason: string;
+  /** Configured vision model ref ("provider/id"), or null if unset. */
+  visionModel: string | null;
+  /** Image hashes the failure covered (batch/single) or the warning named. */
+  imageHashes: string[];
+  /** Number of images involved. */
+  imageCount: number;
+  /** Describer stopReason, when the failure came from a completed response. */
+  stopReason?: string;
+  /** True when the describer's timeout fired. */
+  timedOut?: boolean;
+  /** The timeout budget in ms, when timedOut applies. */
+  timeoutMs?: number;
+  /** Provider error message from the response (response.errorMessage). */
+  errorMessage?: string;
+  /** Stack trace of a thrown error, when available. */
+  errorStack?: string;
+  /** Small config snapshot relevant to troubleshooting. */
+  config?: { maxTokens?: number; thinking: boolean; thinkingLevel: string };
+  /** The model being handed off to ("provider/id"), at the warn point. */
+  activeModel?: string;
+}
+
+/** Directory holding this extension's logs:
+ *  ~/.pi/agent/logs/pi-vision-handoff (overridable via $PI_CODING_AGENT_DIR). */
+export function getErrorLogDir(): string {
+  return join(getAgentDir(), LOG_SUBDIR);
+}
+
+/** Path to the active error log file. */
+export function getErrorLogPath(): string {
+  return join(getErrorLogDir(), LOG_FILENAME);
+}
+
+/** Serialize an entry to one JSONL line (JSON + terminating newline). Pure and
+ *  unit-testable independent of the disk. */
+export function formatErrorLogLine(entry: VisionErrorLogEntry): string {
+  return JSON.stringify(entry) + "\n";
+}
+
+/** Rotate the log when it has grown to at least `maxSize`: the current file
+ *  becomes the `.1` backup (overwriting any prior backup) and the active path
+ *  is cleared for a fresh log. Bounds total on-disk log to ~2× `maxSize`.
+ *  No-op when the file doesn't exist or is under the cap. Never throws. */
+export function rotateLogIfNeeded(path: string = getErrorLogPath(), maxSize: number = MAX_LOG_BYTES): void {
+  let size: number;
+  try {
+    size = statSync(path).size;
+  } catch {
+    return; // missing file — nothing to rotate
+  }
+  if (size < maxSize) return;
+  try {
+    renameSync(path, path + ".1");
+  } catch {
+    // rename failed (permissions, etc.) — leave the file; it'll keep growing
+    // until the condition clears. Logging must not throw.
+  }
+}
+
+/**
+ * Append one structured error entry to the log. Fills `timestamp`, ensures the
+ * log directory exists, rotates on size, and writes a JSONL line. Best-effort:
+ * swallows every error so a logging failure can NEVER break a describer turn
+ * (the describer calls this from its failure paths, where throwing would mask
+ * the real error and abort the batch).
+ */
+export function appendVisionError(input: Omit<VisionErrorLogEntry, "timestamp">): void {
+  try {
+    const dir = getErrorLogDir();
+    if (!existsSync(dir)) mkdirSync(dir, { recursive: true });
+    const path = getErrorLogPath();
+    rotateLogIfNeeded(path);
+    const entry: VisionErrorLogEntry = { timestamp: new Date().toISOString(), ...input };
+    appendFileSync(path, formatErrorLogLine(entry), "utf8");
+  } catch {
+    // never break the describer on a logging failure
+  }
+}

package/vision-handoff.ts

@@ -51,6 +51,7 @@ import {
 } from "./src/usage.js";
 import { DescriptionLoader, UNAVAILABLE, type LoaderDeps } from "./src/dataloader.js";
 import { imageHash, findClipboardImagePaths, readImageBuffer, resolvePrewarmImage } from "./src/image.js";
+import { appendVisionError } from "./src/error-log.js";
 import { resizeImage } from "@earendil-works/pi-coding-agent";
 import { VisionModelSelectorComponent, type VisionModelSelectorResult } from "./src/vision-model-selector.js";
 import { PrewarmEditor } from "./src/prewarm-editor.js";
@@ -194,14 +195,17 @@ function notifyUnresolvedVisionModel(ctx: ExtensionContext, ref: string): void {
   }
 }
 
-/** Notify once per failing image per session (dedup via warnedHashes). */
+/** Notify once per failing image per session (dedup via warnedHashes), and log
+ *  EVERY failure (even in headless/SDK mode with no UI) to the error log so the
+ *  user can troubleshoot. The log entry's `phase: "warn"` carries the failing
+ *  image hashes and the surfaced reason — when the reason is "unknown error",
+ *  the real cause lives in the matching `batch`/`single` entry for those hashes. */
 function warnFailedImages(
   ctx: ExtensionContext,
   imgs: ExtractedImage[],
   descs: string[],
   reason: string,
 ): void {
-  if (!ctx.hasUI) return;
   const newlyFailed: string[] = [];
   for (let i = 0; i < imgs.length; i++) {
     if (descs[i] === UNAVAILABLE) {
@@ -211,6 +215,17 @@ function warnFailedImages(
   }
   if (newlyFailed.length === 0) return;
   for (const h of newlyFailed) warnedHashes.add(h);
+  // Always log: troubleshooting must work in headless/SDK mode too, where the
+  // notify below never fires.
+  appendVisionError({
+    phase: "warn",
+    reason,
+    visionModel: config.visionModel,
+    imageHashes: newlyFailed,
+    imageCount: newlyFailed.length,
+    activeModel: ctx.model ? formatModelRef(ctx.model.provider, ctx.model.id) : undefined,
+  });
+  if (!ctx.hasUI) return;
   ctx.ui.notify(
     `pi-vision-handoff: image description failed — ${reason}. Vision model: ${config.visionModel}`,
     "warning",