pi-vision-handoff 0.8.2 → 0.9.1

package/README.md

@@ -42,8 +42,9 @@ No `settings.json` touched. No per-provider glue. Pick a describer once and ever
 - **🔌 Provider-agnostic** — uses pi's own model execution machinery (`@earendil-works/pi-ai`'s `completeSimple()`), so it works with any provider/configured model, including custom provider extensions.
 - **🧠 Automatic targets** — by default, handoff applies to *every* model that lacks native vision. Opt out with `/vision-handoff auto off`.
 - **🗂️ Explicit overrides** — force handoff for specific models (e.g. a weak vision model) with `/vision-handoff add`.
-- **⚡ Pre-warmed at paste-enter** — the moment you press enter, `before_agent_start` scans the prompt for pasted clipboard image temp-file paths (pi writes pasted images to `<tmpdir>/pi-clipboard-<uuid>.<ext>` and inserts the path as text), reads them, and kicks off the ONE batched vision call concurrent with the agent's first response — so by the time the agent reads the files, they're already cache hits.
+- **⚡ Pre-warmed at paste-enter** — the moment you press enter, `before_agent_start` scans the prompt for pasted image temp-file paths — pi-clipboard, localterm-paste, and any other paste mechanism that writes a pasted image to a temp file and inserts the path as text — reads them, and kicks off the ONE batched vision call concurrent with the agent's first response — so by the time the agent reads the files, they're already cache hits.
 - **🚀 Paste-time prewarm (opt-in)** — `prewarmPastedImages` (off by default) wraps the editor so pasted clipboard images are described the *instant* their path lands in the prompt — before you hit enter — not at submit. The vision call starts concurrent with you typing your question. Tradeoff: the description is generated without your typed question as context (the question usually isn't entered yet at paste time), and a paste-then-abandon wastes one vision call. TUI only; inactive if another extension replaces the editor. Toggle with `/vision-handoff prewarm on`.
+- **🏁 Async pasted-path fallback (opt-in)** — `asyncClipboardHandoff` races the normal read path against asynchronous delivery. A direct `read` or nested Fabric `pi.read` of the pasted path cancels the fallback and uses the normal tool-result/context flow; otherwise the completed description is queued as a steering message. Its transcript row is collapsed to one line by default and expands with `Ctrl+O`. Toggle it in the picker with `Ctrl+A` or run `/vision-handoff fallback on`.
 - **🛡️ Graceful degradation** — no API key? Describer unreachable? Aborted? The image is replaced with a clean `[Image: description unavailable]` placeholder instead of crashing your turn.
 - **📋 Error logging** — every describer failure and every `image description failed` warning is appended as a JSONL line to `~/.pi/agent/logs/pi-vision-handoff/errors.log`, so a warning like `image description failed — unknown error` is troubleshootable: the detailed reason (exception stack, `stopReason`, config snapshot) is captured at the describer's failure source even when the in-memory error string was already reset by the time the warning fired. Size-capped with a single `.1` backup.
 - **📊 Usage reporting** — every real describer call reports model + tokens (and Neuralwatt energy/cost when the vision model is a Neuralwatt model), via `pi.appendEntry` + a `vision-handoff:usage` event for live consumers.
@@ -62,6 +63,7 @@ No `settings.json` touched. No per-provider glue. Pick a describer once and ever
 | `/vision-handoff enable` / `disable` | Master switch (keeps your configured model) |
 | `/vision-handoff auto on` / `off` | Toggle automatic handoff for all non-vision models |
 | `/vision-handoff prewarm on` / `off` | Toggle paste-time prewarm (opt-in, off by default) |
+| `/vision-handoff fallback on` / `off` | Race matching reads against async pasted-path description injection (opt-in) |
 | `/vision-handoff add ollama/llava:13b` | Force handoff for an extra model |
 | `/vision-handoff remove ollama/llava:13b` | Stop forcing handoff for a model |
 | `/vision-handoff clear` | Clear the configured vision model |
@@ -78,6 +80,7 @@ Created automatically at `~/.pi/agent/extensions/pi-vision-handoff.json` on firs
   "autoHandoff": true,
   "handoffModels": ["ollama/llava:13b"],
   "prewarmPastedImages": false,
+  "asyncClipboardHandoff": false,
   "maxTokens": null,
   "cacheMax": 50,
   "maxDescriptionLines": 0,
@@ -93,6 +96,7 @@ Created automatically at `~/.pi/agent/extensions/pi-vision-handoff.json` on firs
 | `autoHandoff` | `true` | Apply handoff to every model whose `input` does not include `image`. |
 | `handoffModels` | `[]` | Extra `provider/id` refs that should also receive handoff. |
 | `prewarmPastedImages` | `false` | **Opt-in paste-time prewarm.** When `true`, a custom editor wrapper describes pasted clipboard images the instant their temp-file path lands in the prompt (pre-submit), instead of at submit. Trades a bit of description quality (the description is generated without your typed question as context, since the question usually isn't entered yet at paste time) and a speculative vision call on paste-then-abandon, for earlier prewarm. TUI mode only; inactive if another extension has replaced the editor. |
+| `asyncClipboardHandoff` | `false` | **Opt-in async fallback.** Starts from the existing submit-time prewarm. A matching direct `read` or Fabric-nested `pi.read` cancels message delivery; if no matching read wins, the description is queued asynchronously as a collapsed custom message and triggers/continues the agent turn. |
 | `maxTokens` | _(unset = model max, clamped to context window)_ | Cap on a single description's output. `null`/unset = use the vision model's declared max output (`model.maxTokens`), clamped so `maxTokens + 8192 <= contextWindow` (a model whose declared max equals its full context window would otherwise be rejected with a 400). Set a number only to cap cost/latency. A truncation is always surfaced via a `[... description truncated …]` marker when the model hits the limit, so a cut-off description is never mistaken for complete. |
 | `cacheMax` | `50` | Max described images kept in the in-memory cache per session. |
 | `maxDescriptionLines` | `0` | Cap on description lines (`0` = unbounded). Default keeps the full description so the `read` tool's native collapse (`ctrl+o`) handles compactness and the model gets complete context; setting `> 0` applies a lossy head-cap to both the TUI render and the model. |
@@ -155,11 +159,7 @@ pi -e ./vision-handoff.ts
 
 ## How It Works
 
-The extension implements the **Facebook DataLoader pattern** for image
-descriptions. The `read` tools are the `load()` callers; a per-image cache
-memoizes promises; all `load()` calls in the same execution frame coalesce
-into ONE batched vision call, dispatched via `setImmediate` after the poll
-phase (so reads completing together batch instead of splitting into N calls).
+The extension implements the **Facebook DataLoader pattern** for image descriptions. The `read` tools are the `load()` callers; a per-image cache memoizes promises; all `load()` calls in the same execution frame coalesce into ONE batched vision call, dispatched via `setImmediate` after the poll phase (so reads completing together batch instead of splitting into N calls).
 
 With `prewarmPastedImages` on (opt-in), an even earlier stage runs:
 
@@ -179,6 +179,8 @@ With `prewarmPastedImages` on (opt-in), an even earlier stage runs:
           replaced the editor (installing over one would break paste, since
           pi wires paste-image to the outermost editor only)
 
+With `asyncClipboardHandoff` on, submit-time prewarm also schedules a non-blocking delivery race. A matching `read` tool call for the pasted path cancels only the custom-message injection (the in-flight description remains available to that read); if description delivery wins, Pi queues it as a steering message. The full description remains model-visible while the transcript shows a one-line row until `Ctrl+O` expands it.
+
 The default submit-time pipeline below runs regardless:
 
     → before_agent_start
@@ -187,9 +189,9 @@ The default submit-time pipeline below runs regardless:
         • PRE-WARMS at paste-enter via the loader, two sources coalescing into
           ONE batch:
           1. attached image blocks (event.images) — vision-capable targets
-          2. pasted clipboard image FILE PATHS in the prompt text — the common
-             non-vision flow: pi writes each pasted image to
-             `<tmpdir>/pi-clipboard-<uuid>.<ext>` and inserts the PATH as text
+          2. pasted image FILE PATHS in the prompt text — the common non-vision
+             flow: pi-clipboard, localterm-paste, and other paste mechanisms
+             write each pasted image to a temp file and insert the PATH as text
              at the cursor, so on a non-vision model they arrive as path tokens
              in `event.prompt`, NOT as `event.images`. The extension scans the
              prompt for those temp paths (confined to the OS temp dir — a
@@ -224,93 +226,25 @@ The default submit-time pipeline below runs regardless:
           payload (`emitContext` does a `structuredClone`), leaving storage
           intact for kitty inline rendering and `/resume`
 
-Because the describer runs during the **tool-result phase** (before the
-agent's next turn), not during the `context` transform (the critical path
-right before the LLM call), the agent gets described text immediately when
-its turn starts — it never waits on the describer inline.
+Because the describer runs during the **tool-result phase** (before the agent's next turn), not during the `context` transform (the critical path right before the LLM call), the agent gets described text immediately when its turn starts — it never waits on the describer inline.
 
 ### Batching: the DataLoader
 
-A single frame's image set is described with **one** vision-model call, not
-one per image. `loadDescription(img)` is synchronous: on a cache miss it
-pushes the image's key (hash) into the current batch and returns a memoized
-promise; on a cache hit it returns the existing (in-flight or resolved)
-promise. `enqueuePostPromiseJob` schedules dispatch via `setImmediate` (the
-check phase, after the whole poll phase AND after the microtask queue
-drains), so every `load()` caller — whether from sync code, a `.then`
-cascade, or a separate I/O callback in the same poll iteration — registers
-its key before the single vision call fires. This is why N parallel reads
-coalesce into one call rather than splitting into N single-image calls:
-`setImmediate` defers past the entire poll phase, whereas DataLoader's
-classic `process.nextTick` would drain between the reads' I/O callbacks and
-fire a dispatch after the first read but before the second. The batched call
-sends every uncached image in a single user message with per-image
-`<<<IMAGE k>>> … <<<END>>>` delimiters; the response is parsed back into
-per-image descriptions (keyed by `sha256(mime + base64)` in the per-image
-cache). If the vision model ignores the delimiter format and the batched
-response can't be split, each unparsed image falls back to its own
-single-image `completeSimple()` call **in parallel** (no delimiters to cooperate
-with) — descriptions still arrive together. Only when a per-image call
-itself genuinely fails (auth, timeout, empty) does that image degrade to
-`[Image: description unavailable]`; one bad image never voids the rest.
-
-Because pi runs parallel `read` tool calls via `Promise.all` and fires
-each read's `tool_result` event as that read's I/O completes (poll phase) —
-via `agent.afterToolCall` → `finalizeExecutedToolCall` — the loader's
-`setImmediate` dispatch defers to the check phase AFTER the whole poll
-iteration, so reads completing together share ONE batch → ONE vision call,
-all resolving together. Reads completing in separate poll iterations get
-separate calls, but always in parallel, never sequential. (The per-image
-cache also dedupes a duplicate `load()` of the same image in one frame:
-dispatch resolves every callback by hash, so a second load whose first cache
-entry was evicted mid-frame still resolves — it never hangs.)
+A single frame's image set is described with **one** vision-model call, not one per image. `loadDescription(img)` is synchronous: on a cache miss it pushes the image's key (hash) into the current batch and returns a memoized promise; on a cache hit it returns the existing (in-flight or resolved) promise. `enqueuePostPromiseJob` schedules dispatch via `setImmediate` (the check phase, after the whole poll phase AND after the microtask queue drains), so every `load()` caller — whether from sync code, a `.then` cascade, or a separate I/O callback in the same poll iteration — registers its key before the single vision call fires. This is why N parallel reads coalesce into one call rather than splitting into N single-image calls: `setImmediate` defers past the entire poll phase, whereas DataLoader's classic `process.nextTick` would drain between the reads' I/O callbacks and fire a dispatch after the first read but before the second. The batched call sends every uncached image in a single user message with per-image `<<<IMAGE k>>> … <<<END>>>` delimiters; the response is parsed back into per-image descriptions (keyed by `sha256(mime + base64)` in the per-image cache). If the vision model ignores the delimiter format and the batched response can't be split, each unparsed image falls back to its own single-image `completeSimple()` call **in parallel** (no delimiters to cooperate with) — descriptions still arrive together. Only when a per-image call itself genuinely fails (auth, timeout, empty) does that image degrade to `[Image: description unavailable]`; one bad image never voids the rest.
+
+Because pi runs parallel `read` tool calls via `Promise.all` and fires each read's `tool_result` event as that read's I/O completes (poll phase) — via `agent.afterToolCall` → `finalizeExecutedToolCall` — the loader's `setImmediate` dispatch defers to the check phase AFTER the whole poll iteration, so reads completing together share ONE batch → ONE vision call, all resolving together. Reads completing in separate poll iterations get separate calls, but always in parallel, never sequential. (The per-image cache also dedupes a duplicate `load()` of the same image in one frame: dispatch resolves every callback by hash, so a second load whose first cache entry was evicted mid-frame still resolves — it never hangs.)
 
 **Failures are never cached.**
 
-**Failures are never cached.** A genuine describer failure returns
-`[Image: description unavailable]` for that turn but is NOT written to the
-per-image cache, so the next turn re-attempts (and surfaces the real error
-via a `ctx.ui.notify` warning instead of serving a stuck placeholder). This
-avoids a transient failure poisoning the cache for the rest of the session.
-
-**Per-image-set warning.** The `context` hook fires before every LLM turn,
-and describer failures aren't cached — so without dedup the same broken
-vision model would re-warn every turn. Each distinct image is warned about
-at most once per session (tracked by image hash, reset on `/resume`/new
-session); subsequent turns for the same failing image degrade silently to
-the placeholder instead of spamming.
-
-**Timeout scales with batch size.** The describer generates an exhaustive
-multi-paragraph description per image, and the call is batched (N images →
-1 request). The timeout is `DESCRIBE_TIMEOUT_MS` (120s) plus a per-image
-budget (45s × (imageCount − 1)), so a 5-image batch isn't held to the same
-wall-clock budget as one image. A timeout surfaces as `describer timed out
-after <N>s` rather than a misleading `stopReason "aborted"`.
-
-**No silent truncation.** The describer prompt says "be exhaustive", so the
-default `maxTokens` is **unset** — the describer uses the vision model's
-declared max output (`model.maxTokens`) as the cap, rather than relying on a
-provider's small omitted-default. That value is clamped so `maxTokens + 8192
-<= contextWindow`: a model whose declared `maxTokens` equals its full
-`contextWindow` (e.g. a custom provider that sets both to the same number)
-would otherwise be rejected by the provider with a 400 (you can't request
-output tokens equal to the entire context window when you also have input),
-so the clamp subtracts a small input reserve. If the model still hits a token
-limit (a cap you set, or the provider's hard output maximum), `stopReason`
-becomes `"length"`; the describer appends a visible
-`[... description truncated …]` marker to the (still useful) partial text
-rather than letting a cut-off description pass as complete. For a batched call
-the marker lands on the last image being emitted when the cap hit (the one cut
-off mid-stream); earlier sections had `<<<END>>>` delimiters and are complete.
-
-**Aborts propagate.** The `context` hook runs in the foreground (pi awaits
-it before the LLM call), so a slow describer would also make aborting a turn
-slow — pi has to wait for the transform to return. The hook therefore wires
-the turn's abort signal (`ctx.signal`, the active run's `AbortController`)
-into the describer's `completeSimple()` call, so a user cancel kills the in-flight
-vision request immediately. A deliberate abort is not warned about (it's
-not a vision-model failure) and the LLM-bound payload is left untouched since
-the turn is being torn down anyway.
+**Failures are never cached.** A genuine describer failure returns `[Image: description unavailable]` for that turn but is NOT written to the per-image cache, so the next turn re-attempts (and surfaces the real error via a `ctx.ui.notify` warning instead of serving a stuck placeholder). This avoids a transient failure poisoning the cache for the rest of the session.
+
+**Per-image-set warning.** The `context` hook fires before every LLM turn, and describer failures aren't cached — so without dedup the same broken vision model would re-warn every turn. Each distinct image is warned about at most once per session (tracked by image hash, reset on `/resume`/new session); subsequent turns for the same failing image degrade silently to the placeholder instead of spamming.
+
+**Timeout scales with batch size.** The describer generates an exhaustive multi-paragraph description per image, and the call is batched (N images → 1 request). The timeout is `DESCRIBE_TIMEOUT_MS` (120s) plus a per-image budget (45s × (imageCount − 1)), so a 5-image batch isn't held to the same wall-clock budget as one image. A timeout surfaces as `describer timed out after <N>s` rather than a misleading `stopReason "aborted"`.
+
+**No silent truncation.** The describer prompt says "be exhaustive", so the default `maxTokens` is **unset** — the describer uses the vision model's declared max output (`model.maxTokens`) as the cap, rather than relying on a provider's small omitted-default. That value is clamped so `maxTokens + 8192 <= contextWindow`: a model whose declared `maxTokens` equals its full `contextWindow` (e.g. a custom provider that sets both to the same number) would otherwise be rejected by the provider with a 400 (you can't request output tokens equal to the entire context window when you also have input), so the clamp subtracts a small input reserve. If the model still hits a token limit (a cap you set, or the provider's hard output maximum), `stopReason` becomes `"length"`; the describer appends a visible `[... description truncated …]` marker to the (still useful) partial text rather than letting a cut-off description pass as complete. For a batched call the marker lands on the last image being emitted when the cap hit (the one cut off mid-stream); earlier sections had `<<<END>>>` delimiters and are complete.
+
+**Aborts propagate.** The `context` hook runs in the foreground (pi awaits it before the LLM call), so a slow describer would also make aborting a turn slow — pi has to wait for the transform to return. The hook therefore wires the turn's abort signal (`ctx.signal`, the active run's `AbortController`) into the describer's `completeSimple()` call, so a user cancel kills the in-flight vision request immediately. A deliberate abort is not warned about (it's not a vision-model failure) and the LLM-bound payload is left untouched since the turn is being torn down anyway.
 
 ### Image-block formats handled
 
@@ -321,10 +255,7 @@ the turn is being torn down anyway.
 | `context` (all messages) | `{ type: "input_image", image_url: "data:…" }` (OpenAI Responses) | detected by shape → replaced with `{ type: "input_text", text }` |
 | `context` (all messages) | `{ type: "image", source: { type: "base64", media_type, data } }` (Anthropic Messages) | detected by shape → replaced with `{ type: "text", text }` |
 
-The describer call itself goes through pi's normal model machinery (`completeSimple()`),
-**not** the agent event loop — so it never re-triggers `context` (no recursion).
-The `read` tool result keeps its image block untouched (kitty inline + `/resume`);
-only the `context`-cloned LLM-bound payload has images swapped for text.
+The describer call itself goes through pi's normal model machinery (`completeSimple()`), **not** the agent event loop — so it never re-triggers `context` (no recursion). The `read` tool result keeps its image block untouched (kitty inline + `/resume`); only the `context`-cloned LLM-bound payload has images swapped for text.
 
 ### Usage reporting
 
@@ -370,7 +301,7 @@ One record is emitted per **real** describer call (a batched call describing sev
 
 ```bash
 pnpm install
-pnpm test          # Vitest unit tests (77 passing)
+pnpm test          # Vitest unit tests
 pnpm typecheck     # TypeScript validation
 pnpm lint:dead     # Dead code detection (knip)
 ```
@@ -407,11 +338,7 @@ pnpm lint:dead     # Dead code detection (knip)
 
 ## Acknowledgements
 
-The vision handoff concept and the exhaustive describer prompt originate from
-the [pi-umans-provider](https://github.com/monotykamary/pi-umans-provider) GLM 5.1
-pipeline. The picker TUI builds on the patterns from
-[pi-hide-providers](https://github.com/monotykamary/pi-hide-providers), which in
-turn mirror pi core's built-in selectors.
+The vision handoff concept and the exhaustive describer prompt originate from the [pi-umans-provider](https://github.com/monotykamary/pi-umans-provider) GLM 5.1 pipeline. The picker TUI builds on the patterns from [pi-hide-providers](https://github.com/monotykamary/pi-hide-providers), which in turn mirror pi core's built-in selectors.
 
 ## License
 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "pi-vision-handoff",
-  "version": "0.8.2",
+  "version": "0.9.1",
   "description": "Give text-only pi models vision — describe images with a vision model you pick via an interactive picker, then hand off the text description to non-vision models",
   "type": "module",
   "author": "Tom X Nguyen",

package/src/image.ts

@@ -282,24 +282,30 @@ export function readImageBufferBounded(filePath: string): { buf: Buffer; mimeTyp
   return readImageBuffer(filePath);
 }
 
-/** Regex matching pi's pasted-clipboard temp image file paths anywhere in the
- *  prompt text. pi writes pasted clipboard images to
- *  `<tmpdir>/pi-clipboard-<uuid>.<ext>` and inserts the path as text at the
- *  cursor, so on a non-vision model these arrive as path tokens in the user
- *  prompt — NOT as `event.images`. Matching them lets the extension pre-warm
- *  the describer at paste-enter (concurrent with the agent's first response)
- *  instead of waiting for the agent to `read` them. */
-const CLIPBOARD_IMAGE_PATH_RE = /(\S*pi-clipboard-[^\s]+\.(?:png|jpe?g|gif|webp))/gi;
+/** Regex matching any pasted temp-image file path in the prompt text,
+ *  regardless of which paste mechanism wrote it (pi-clipboard,
+ *  localterm-paste, and others). We match any non-whitespace token ending in
+ *  a supported image extension, then confine to the OS temp directory below.
+ *  The confinement, not the filename pattern, is the security boundary. */
+const PASTED_IMAGE_PATH_RE = /(\S+\.(?:png|jpe?g|gif|webp))/gi;
+const URL_RE = new RegExp('^[a-z][a-z0-9+.-]*://', 'i');
+const LEADING_WRAP_RE = new RegExp('^[^A-Za-z0-9_/@.~-]+');
 
-/** Extract pasted clipboard image file paths from a prompt. Confined to the
- *  OS temp directory so an attacker-crafted prompt can't trick the extension
- *  into reading arbitrary files — only pi's own clipboard temp files qualify. */
-export function findClipboardImagePaths(prompt: string): string[] {
+/** Extract pasted image file paths from a prompt. Confined to the OS temp
+ *  directory so an attacker-crafted prompt can't trick the extension into
+ *  reading arbitrary files — only temp-dir image files qualify (this covers
+ *  every paste mechanism: pi-clipboard, localterm-paste, and others). URLs
+ *  (http(s)://, file://, ...) are skipped so they aren't joined into the temp
+ *  dir and mistaken for local files; leading wrapping chars (parentheses,
+ *  quotes) are stripped so quoted/parenthesized paths still resolve. */
+export function findPastedImagePaths(prompt: string): string[] {
   const tmp = tmpdir();
   const paths = new Set<string>();
-  for (const m of prompt.matchAll(CLIPBOARD_IMAGE_PATH_RE)) {
-    const p = m[1];
-    if (!p) continue;
+  for (const m of prompt.matchAll(PASTED_IMAGE_PATH_RE)) {
+    const token = m[1];
+    if (!token) continue;
+    if (URL_RE.test(token)) continue;
+    const p = token.replace(LEADING_WRAP_RE, "");
     const abs = isAbsolute(p) ? p : join(tmp, p);
     // Ensure the resolved candidate stays inside the temp directory.
     if (abs.startsWith(tmp + sep) || abs === tmp) paths.add(abs);
@@ -307,16 +313,16 @@ export function findClipboardImagePaths(prompt: string): string[] {
   return [...paths];
 }
 
-/** Diff the clipboard image paths in `text` against `known`, returning those
+/** Diff the pasted image paths in `text` against `known`, returning those
  *  that are newly appeared (not yet seen). Used by the paste-time prewarm
  *  editor to prewarm only newly-pasted paths on each text change, without
- *  re-reading already-seen ones. `findClipboardImagePaths` already dedups
+ *  re-reading already-seen ones. `findPastedImagePaths` already dedups
  *  within a single text, so each returned path is unique and seen at most
- *  once. Returns [] when `text` holds no clipboard paths (e.g. ordinary
+ *  once. Returns [] when `text` holds no pasted paths (e.g. ordinary
  *  typing) — so the editor's per-keystroke cost when the opt-in is on is one
  *  regex scan that almost always yields nothing. */
 export function diffPrewarmPaths(text: string, known: Set<string>): string[] {
-  const paths = findClipboardImagePaths(text);
+  const paths = findPastedImagePaths(text);
   const newPaths: string[] = [];
   for (const p of paths) {
     if (!known.has(p)) newPaths.push(p);

package/src/index.ts

@@ -149,6 +149,11 @@ export interface VisionHandoffConfig {
    *  a custom editor would break clipboard paste, since pi wires paste-image
    *  to the outermost editor only). */
   prewarmPastedImages: boolean;
+  /** Opt-in: asynchronously inject descriptions for pasted clipboard paths when
+   *  the target model does not read them. The normal read/tool_result path wins
+   *  the race: a matching read cancels the queued fallback injection while
+   *  reusing the same in-flight description. */
+  asyncClipboardHandoff: boolean;
   /** Max output tokens for a single description. `undefined` (default) = use the
    *  vision model's declared max output (`model.maxTokens`) as the cap — the
    *  highest the model supports — so the "be exhaustive" prompt isn't defeated
@@ -186,6 +191,7 @@ export const DEFAULT_CONFIG: VisionHandoffConfig = {
   autoHandoff: true,
   handoffModels: [],
   prewarmPastedImages: false,
+  asyncClipboardHandoff: false,
   maxTokens: undefined,
   cacheMax: DEFAULT_CACHE_MAX,
   maxDescriptionLines: DEFAULT_MAX_DESCRIPTION_LINES,
@@ -236,6 +242,9 @@ export function normalizeConfig(raw: unknown): VisionHandoffConfig {
       .filter((m) => m && parseModelRef(m));
   }
   if (typeof obj.prewarmPastedImages === "boolean") base.prewarmPastedImages = obj.prewarmPastedImages;
+  if (typeof obj.asyncClipboardHandoff === "boolean") {
+    base.asyncClipboardHandoff = obj.asyncClipboardHandoff;
+  }
   // maxTokens: optional. undefined (default) = no artificial cap. Only set when
   // a valid positive finite number is given; any other value leaves it unset.
   if (typeof obj.maxTokens === "number" && Number.isFinite(obj.maxTokens) && obj.maxTokens > 0) {

package/src/vision-model-selector.ts

@@ -21,6 +21,8 @@ import {
   matchesKey,
   Spacer,
   Text,
+  truncateToWidth,
+  wrapTextWithAnsi,
 } from "@earendil-works/pi-tui";
 import type { Theme } from "@earendil-works/pi-coding-agent";
 import type { ThinkingLevel } from "@earendil-works/pi-ai";
@@ -48,6 +50,8 @@ export interface VisionModelSelectorResult {
   thinking: boolean;
   /** Thinking effort chosen in the picker. */
   thinkingLevel: ThinkingLevel;
+  /** Whether pasted paths should be injected if no matching read wins. */
+  asyncClipboardHandoff: boolean;
 }
 
 export class VisionModelSelectorComponent implements Component {
@@ -65,6 +69,7 @@ export class VisionModelSelectorComponent implements Component {
   private currentRef: string | null;
   private thinking: boolean;
   private thinkingLevel: ThinkingLevel;
+  private asyncClipboardHandoff: boolean;
 
   private _focused = false;
   get focused(): boolean {
@@ -87,6 +92,7 @@ export class VisionModelSelectorComponent implements Component {
     currentRef: string | null,
     currentThinking: boolean,
     currentThinkingLevel: ThinkingLevel,
+    currentAsyncClipboardHandoff: boolean,
     done: (result: VisionModelSelectorResult) => void,
   ) {
     this.theme = theme;
@@ -94,6 +100,7 @@ export class VisionModelSelectorComponent implements Component {
     this.currentRef = currentRef;
     this.thinking = currentThinking;
     this.thinkingLevel = currentThinkingLevel;
+    this.asyncClipboardHandoff = currentAsyncClipboardHandoff;
     this.allItems = this.buildItems(allModels);
     this.filteredItems = this.allItems;
 
@@ -116,9 +123,21 @@ export class VisionModelSelectorComponent implements Component {
     const lines: string[] = [];
     lines.push(...new DynamicBorder((s) => this.theme.fg("accent", s)).render(width));
     lines.push("");
-    lines.push(this.theme.fg("accent", this.theme.bold("Vision Handoff")));
     lines.push(
-      this.theme.fg("muted", "Pick a vision-capable model to describe images for text-only models."),
+      truncateToWidth(
+        this.theme.fg("accent", this.theme.bold("Vision Handoff")),
+        width,
+        "",
+      ),
+    );
+    lines.push(
+      ...wrapTextWithAnsi(
+        this.theme.fg(
+          "muted",
+          "Pick a vision-capable model to describe images for text-only models.",
+        ),
+        width,
+      ),
     );
     lines.push("");
     lines.push(...this.searchInput.render(width));
@@ -127,7 +146,7 @@ export class VisionModelSelectorComponent implements Component {
     lines.push("");
     lines.push(...this.footerText.render(width));
     lines.push(...new DynamicBorder((s) => this.theme.fg("accent", s)).render(width));
-    return lines;
+    return lines.map((line) => truncateToWidth(line, width, ""));
   }
 
   handleInput(data: string): void {
@@ -180,6 +199,12 @@ export class VisionModelSelectorComponent implements Component {
       return;
     }
 
+    if (matchesKey(data, Key.ctrl("a"))) {
+      this.asyncClipboardHandoff = !this.asyncClipboardHandoff;
+      this.updateList();
+      return;
+    }
+
     // Thinking controls — reuse pi's own app.thinking.* keybindings so the
     // hints and behaviour match the rest of pi: ctrl+t toggles thinking
     // on/off, shift+tab cycles the effort level. Intercepted before the
@@ -263,6 +288,7 @@ export class VisionModelSelectorComponent implements Component {
       `ctrl+s done`,
       `${keyText("app.thinking.toggle")} thinking`,
       `${keyText("app.thinking.cycle")} effort`,
+      `ctrl+a async fallback`,
       `esc cancel`,
       this.searchInput.getValue() ? `${this.filteredItems.length - 1} match` : `${totalCount} models · ${visionCount} vision`,
     ];
@@ -365,6 +391,12 @@ export class VisionModelSelectorComponent implements Component {
         );
       }
       this.renderThinkingDetail(selected);
+      const fallback = this.asyncClipboardHandoff
+        ? this.theme.fg("success", "on")
+        : this.theme.fg("muted", "off");
+      this.listContainer.addChild(
+        new Text(this.theme.fg("dim", `  Async pasted-path fallback: ${fallback}`), 0, 0),
+      );
     }
 
     this.footerText.setText(this.getFooterText());
@@ -394,11 +426,23 @@ export class VisionModelSelectorComponent implements Component {
   }
 
   private confirm(item: DisplayItem): void {
-    this.done({ ref: item.ref, cancelled: false, thinking: this.thinking, thinkingLevel: this.thinkingLevel });
+    this.done({
+      ref: item.ref,
+      cancelled: false,
+      thinking: this.thinking,
+      thinkingLevel: this.thinkingLevel,
+      asyncClipboardHandoff: this.asyncClipboardHandoff,
+    });
   }
 
   private finish(cancelled: boolean): void {
-    this.done({ ref: null, cancelled, thinking: this.thinking, thinkingLevel: this.thinkingLevel });
+    this.done({
+      ref: null,
+      cancelled,
+      thinking: this.thinking,
+      thinkingLevel: this.thinkingLevel,
+      asyncClipboardHandoff: this.asyncClipboardHandoff,
+    });
   }
 
   /** Cycle the thinking effort forward through {@link THINKING_LEVELS},

package/vision-handoff.ts

@@ -11,6 +11,8 @@
  *   before_agent_start → warm the description cache for attached images AND
  *     pasted clipboard image file paths found in the prompt text (pre-warm at
  *     paste-enter, concurrent with the agent's first response).
+ *   optional async clipboard fallback → race matching read tool calls against a
+ *     non-blocking steering-message injection; matching reads cancel delivery.
  *   tool_result (read) → loadDescription() each read-tool image and AWAIT the
  *     shared batch, so N parallel reads coalesce into ONE vision call. The
  *     descriptions land in the tool results before the agent's next turn.
@@ -51,11 +53,12 @@ import {
   type VisionHandoffUsageRecord,
 } from "./src/usage.js";
 import { DescriptionLoader, UNAVAILABLE, type LoaderDeps } from "./src/dataloader.js";
-import { imageHash, findClipboardImagePaths, readImageBuffer, readImageBufferBounded, resolvePrewarmImage, isOmittedImageNote } from "./src/image.js";
+import { imageHash, findPastedImagePaths, readImageBuffer, readImageBufferBounded, resolvePrewarmImage, isOmittedImageNote } from "./src/image.js";
 import { appendVisionError } from "./src/error-log.js";
 import { resizeImage } from "@earendil-works/pi-coding-agent";
 import { VisionModelSelectorComponent, type VisionModelSelectorResult } from "./src/vision-model-selector.js";
 import { PrewarmEditor } from "./src/prewarm-editor.js";
+import { Text } from "@earendil-works/pi-tui";
 
 let config: VisionHandoffConfig = readConfig();
 
@@ -85,6 +88,110 @@ let editorInstalled = false;
 let visionModelCache: { ref: string; model: Model<Api> } | null = null;
 let visionModelUnresolvedRef: string | null = null;
 
+interface PendingAsyncClipboardHandoff {
+  token: symbol;
+  paths: Set<string>;
+  cancelled: boolean;
+}
+
+interface PreparedClipboardImage {
+  path: string;
+  image: ExtractedImage;
+  description: Promise<string>;
+}
+
+let pendingAsyncClipboardHandoff: PendingAsyncClipboardHandoff | null = null;
+
+function cancelAsyncClipboardHandoff(): void {
+  if (!pendingAsyncClipboardHandoff) return;
+  pendingAsyncClipboardHandoff.cancelled = true;
+  pendingAsyncClipboardHandoff = null;
+}
+
+function isPendingClipboardRead(input: unknown, cwd: string): boolean {
+  const pending = pendingAsyncClipboardHandoff;
+  if (!pending || !input || typeof input !== "object") return false;
+  const raw = (input as { path?: unknown }).path;
+  if (typeof raw !== "string") return false;
+  const path = raw.startsWith("@") ? raw.slice(1) : raw;
+  const absolute = isAbsolute(path) ? path : resolve(cwd, path);
+  return pending.paths.has(absolute);
+}
+
+function scheduleAsyncClipboardInjection(
+  pi: ExtensionAPI,
+  ctx: ExtensionContext,
+  paths: string[],
+  prepared: Promise<PreparedClipboardImage | null>[],
+): void {
+  cancelAsyncClipboardHandoff();
+  const pending: PendingAsyncClipboardHandoff = {
+    token: Symbol("async-clipboard-handoff"),
+    paths: new Set(paths),
+    cancelled: false,
+  };
+  pendingAsyncClipboardHandoff = pending;
+
+  void (async () => {
+    const entries = (await Promise.all(prepared)).filter(
+      (entry): entry is PreparedClipboardImage => entry !== null,
+    );
+    if (entries.length === 0) {
+      if (pendingAsyncClipboardHandoff?.token === pending.token) {
+        pendingAsyncClipboardHandoff = null;
+      }
+      return;
+    }
+    const descriptions = await Promise.all(entries.map((entry) => entry.description));
+
+    // Always yield out of before_agent_start, even when paste-time prewarm made
+    // every description an immediate cache hit. This lets Pi enter the active
+    // agent run before sendMessage queues the fallback as a steering message.
+    await new Promise<void>((resolveImmediate) => setImmediate(resolveImmediate));
+    if (
+      pending.cancelled ||
+      pendingAsyncClipboardHandoff?.token !== pending.token ||
+      !config.asyncClipboardHandoff ||
+      !isConfigured(config)
+    ) {
+      return;
+    }
+
+    warnFailedImages(
+      ctx,
+      entries.map((entry) => entry.image),
+      descriptions,
+      lastDescriberError ?? "unknown error",
+    );
+    const content = [
+      "Asynchronous vision handoff for pasted image path(s):",
+      ...entries.map(
+        (entry, index) => `${entry.path}\n${descriptions[index] ?? UNAVAILABLE}`,
+      ),
+    ].join("\n\n");
+
+    pendingAsyncClipboardHandoff = null;
+    try {
+      pi.sendMessage(
+        {
+          customType: "vision-handoff-async",
+          content,
+          display: true,
+          details: { imageCount: entries.length },
+        },
+        { deliverAs: "steer", triggerTurn: true },
+      );
+    } catch {
+      // The session may have been replaced/reloaded while the description was
+      // in flight. The stale extension instance must not affect the new one.
+    }
+  })().catch(() => {
+    if (pendingAsyncClipboardHandoff?.token === pending.token) {
+      pendingAsyncClipboardHandoff = null;
+    }
+  });
+}
+
 // Usage reporter; wired to pi.appendEntry + pi.events.emit in the default
 // export. No-op until then so the describer is safe to call before wiring.
 let reportUsage: (record: VisionHandoffUsageRecord) => void = () => {};
@@ -236,6 +343,24 @@ function warnFailedImages(
 export default function (pi: ExtensionAPI) {
   config = readConfig();
 
+  pi.registerMessageRenderer("vision-handoff-async", (message, { expanded }, theme) => {
+    const details = message.details as { imageCount?: number } | undefined;
+    const count = details?.imageCount ?? 1;
+    const label = `Vision handoff · ${count} pasted image${count === 1 ? "" : "s"}`;
+    const hint = expanded ? "Ctrl+O to collapse" : "Ctrl+O to expand";
+    const summary = theme.fg("dim", `👁 ${label} · ${hint}`);
+    if (!expanded) return new Text(summary, 0, 0);
+    // Apply the dim (grey) style per line: the TUI appends a full SGR reset at
+    // the end of each rendered line, so a single style wrapper would only tint
+    // the first line. Text wraps with wrapTextWithAnsi (ANSI-preserving), and
+    // each line here carries its own dim code so wrapped continuations stay grey.
+    const contentStr = typeof message.content === "string" ? message.content : "";
+    const body = contentStr.length > 0
+      ? contentStr.split("\n").map((line) => theme.fg("dim", line)).join("\n")
+      : "";
+    return new Text(body ? `${summary}\n${body}` : summary, 0, 0);
+  });
+
   // Wire the usage reporter to pi's persistence + event bus. appendEntry
   // persists the record so it replays on session resume/branch, and the event
   // lets live consumers filter on one channel for tokens AND energy. Each call
@@ -257,6 +382,7 @@ export default function (pi: ExtensionAPI) {
   };
 
   pi.on("session_start", async (_event, ctx) => {
+    cancelAsyncClipboardHandoff();
     // Reload in case the user edited the config on disk from another session.
     config = readConfig();
     visionModelCache = null;
@@ -268,6 +394,7 @@ export default function (pi: ExtensionAPI) {
   });
 
   pi.on("before_agent_start", async (event, ctx) => {
+    cancelAsyncClipboardHandoff();
     if (!isConfigured(config)) return;
     if (!isHandoffTarget(ctx.model, config)) return;
 
@@ -298,9 +425,10 @@ export default function (pi: ExtensionAPI) {
     // 1. Attached image blocks (event.images) — vision-capable targets where
     //    the user message itself carries image blocks (e.g. `pi --image`).
     //
-    // 2. Pasted clipboard image FILE PATHS in the prompt text — the common
-    //    non-vision flow. pi's `handleClipboardImagePaste` writes each pasted
-    //    image to `<tmpdir>/pi-clipboard-<uuid>.<ext>` and inserts the PATH as
+    // 2. Pasted image FILE PATHS in the prompt text — the common non-vision
+    //    flow. pi's `handleClipboardImagePaste` (and other paste mechanisms like
+    //    localterm-paste) write each pasted image to a temp file and insert the
+    //    PATH as
     //    text at the cursor; on a non-vision model these arrive as path tokens
     //    in `event.prompt`, NOT as `event.images`. We scan the prompt for those
     //    temp paths, read the files, and `loadDescription()` them so the ONE
@@ -331,17 +459,49 @@ export default function (pi: ExtensionAPI) {
     // describes an image the agent will see). A file that can't be read, isn't
     // a supported image, or fails to resize is skipped (the agent's `read`
     // will still describe it via `tool_result` if it emits an image block).
-    for (const p of findClipboardImagePaths(event.prompt || "")) {
-      const read = readImageBuffer(p);
-      if (!read) continue;
-      resolvePrewarmImage(read.buf, read.mimeType, resizeImage)
-        .then((img) => {
-          if (img) loader.loadDescription(img).catch(() => {});
-        })
-        .catch(() => {});
+    const clipboardPaths = findPastedImagePaths(event.prompt || "");
+    const preparedClipboardImages = clipboardPaths.map(
+      async (path): Promise<PreparedClipboardImage | null> => {
+        try {
+          const read = readImageBuffer(path);
+          if (!read) return null;
+          const image = await resolvePrewarmImage(read.buf, read.mimeType, resizeImage);
+          if (!image) return null;
+          const description = loader.loadDescription(image);
+          description.catch(() => {});
+          return { path, image, description };
+        } catch {
+          return null;
+        }
+      },
+    );
+
+    if (config.asyncClipboardHandoff && clipboardPaths.length > 0) {
+      scheduleAsyncClipboardInjection(pi, ctx, clipboardPaths, preparedClipboardImages);
+    } else {
+      // Start every preparation task even when no consumer awaits it. Each task
+      // calls loadDescription as soon as resize completes, preserving the
+      // original fire-and-forget submit-time prewarm behavior.
+      for (const prepared of preparedClipboardImages) prepared.catch(() => {});
     }
   });
 
+  // A direct read and a nested pi.read both emit a read tool_call. If it targets
+  // one of this turn's pasted clipboard paths before the async fallback is
+  // injected, the normal tool_result/context path wins and the queued custom
+  // message is cancelled. The in-flight description is deliberately retained:
+  // the read result reuses it as a cache/in-flight hit.
+  pi.on("tool_call", (event, ctx) => {
+    if (event.toolName !== "read" && event.toolName !== "pi.read") return;
+    if (isPendingClipboardRead(event.input, ctx.cwd)) {
+      cancelAsyncClipboardHandoff();
+    }
+  });
+
+  pi.on("session_shutdown", () => {
+    cancelAsyncClipboardHandoff();
+  });
+
   // The PRIMARY injection point: the `read` tool's `tool_result` handler.
   // When the agent reads image files, this fires for each read result. It
   // calls the loader's `loadDescription(img)` for every image block and
@@ -565,7 +725,7 @@ export default function (pi: ExtensionAPI) {
   pi.registerCommand("vision-handoff", {
     description: HANDOFF_COMMAND_DESCRIPTION,
     getArgumentCompletions(prefix: string) {
-      const subcommands = ["select", "model", "status", "enable", "disable", "auto", "thinking", "prewarm", "add", "remove", "clear", "help"];
+      const subcommands = ["select", "model", "status", "enable", "disable", "auto", "thinking", "prewarm", "fallback", "add", "remove", "clear", "help"];
       const matches = subcommands.filter((s) => s.startsWith(prefix));
       return matches.length > 0 ? matches.map((s) => ({ value: s, label: s })) : null;
     },
@@ -601,6 +761,8 @@ async function handleHandoffCommand(ctx: ExtensionCommandContext, args: string):
         "                               Set the vision describer's thinking effort (off = disabled)",
         "  /vision-handoff prewarm <on|off>",
         "                               Toggle describing pasted images at paste-time (opt-in, off by default)",
+        "  /vision-handoff fallback <on|off>",
+        "                               Inject pasted-image descriptions asynchronously when no matching read wins",
         "  /vision-handoff add <p/id>     Force handoff for an extra model",
         "  /vision-handoff remove <p/id>  Stop forcing handoff for a model",
         "  /vision-handoff clear          Clear the configured vision model",
@@ -611,6 +773,7 @@ async function handleHandoffCommand(ctx: ExtensionCommandContext, args: string):
         "  read images through a dataloader (one batched vision call); context swaps",
         "  image blocks in the payload for the cached text description.",
         "  prewarm on wraps the editor to describe pasted images at paste-time.",
+        "  fallback on asynchronously injects a collapsed description unless a matching read wins.",
       ].join("\n"),
       "info",
     );
@@ -657,6 +820,11 @@ async function handleHandoffCommand(ctx: ExtensionCommandContext, args: string):
     return;
   }
 
+  if (subcommand === "fallback") {
+    handleFallbackSubcommand(ctx, rest);
+    return;
+  }
+
   if (subcommand === "clear") {
     updateConfig(
       ctx,
@@ -743,6 +911,7 @@ function updateConfig(
   message: string,
 ): void {
   const next = transform(config);
+  if (!next.asyncClipboardHandoff) cancelAsyncClipboardHandoff();
   const path = writeConfig(next);
   config = next;
   visionModelCache = null;
@@ -806,6 +975,31 @@ function handlePrewarmSubcommand(ctx: ExtensionCommandContext, rest: string): vo
   updateConfig(ctx, (c) => ({ ...c, prewarmPastedImages: on }), note);
 }
 
+/** Handle /vision-handoff fallback <on|off>. */
+function handleFallbackSubcommand(ctx: ExtensionCommandContext, rest: string): void {
+  const value = rest.trim().toLowerCase();
+  if (!value) {
+    ctx.ui.notify(
+      `Async pasted-path fallback: ${config.asyncClipboardHandoff ? "on" : "off"}.\n` +
+        "Usage: /vision-handoff fallback <on|off>",
+      "info",
+    );
+    return;
+  }
+  if (value !== "on" && value !== "off") {
+    ctx.ui.notify("Usage: /vision-handoff fallback <on|off>", "warning");
+    return;
+  }
+  const on = value === "on";
+  updateConfig(
+    ctx,
+    (c) => ({ ...c, asyncClipboardHandoff: on }),
+    on
+      ? "Async pasted-path fallback on — a matching read wins the race; otherwise the description is injected as a collapsed message."
+      : "Async pasted-path fallback off.",
+  );
+}
+
 async function showSelector(ctx: ExtensionCommandContext): Promise<void> {
   if (!ctx.hasUI) {
     ctx.ui.notify("/vision-handoff requires interactive mode.", "error");
@@ -825,12 +1019,15 @@ async function showSelector(ctx: ExtensionCommandContext): Promise<void> {
     if (thinkingPick === undefined) return;
     const thinking = thinkingPick === "on";
     const thinkingLevel = config.thinkingLevel;
+    const fallbackPick = await ctx.ui.select("Async pasted-path fallback", ["on", "off"]);
+    if (fallbackPick === undefined) return;
+    const asyncClipboardHandoff = fallbackPick === "on";
     const thinkingNote = thinking
       ? `thinking on (${thinkingLevel})${ref ? " \u2014 applies only if the vision model supports reasoning" : ""}`
       : "thinking off";
     updateConfig(
       ctx,
-      (c) => ({ ...c, visionModel: ref, thinking, thinkingLevel }),
+      (c) => ({ ...c, visionModel: ref, thinking, thinkingLevel, asyncClipboardHandoff }),
       ref ? `Vision model set to ${ref} \u00b7 ${thinkingNote}` : `Vision model cleared \u00b7 ${thinkingNote}`,
     );
     if (!ref) {
@@ -846,6 +1043,7 @@ async function showSelector(ctx: ExtensionCommandContext): Promise<void> {
       config.visionModel,
       config.thinking,
       config.thinkingLevel,
+      config.asyncClipboardHandoff,
       (r) => done(r),
     );
     return {
@@ -870,6 +1068,7 @@ async function showSelector(ctx: ExtensionCommandContext): Promise<void> {
   const ref = result.ref;
   const thinking = result.thinking;
   const thinkingLevel = result.thinkingLevel;
+  const asyncClipboardHandoff = result.asyncClipboardHandoff;
   // Fold the thinking state into the single updateConfig notify so the
   // "thinking off" message can't overwrite the model-change message.
   const thinkingNote = thinking
@@ -877,7 +1076,7 @@ async function showSelector(ctx: ExtensionCommandContext): Promise<void> {
     : "thinking off";
   updateConfig(
     ctx,
-    (c) => ({ ...c, visionModel: ref, thinking, thinkingLevel }),
+    (c) => ({ ...c, visionModel: ref, thinking, thinkingLevel, asyncClipboardHandoff }),
     ref ? `Vision model set to ${ref} · ${thinkingNote}` : `Vision model cleared · ${thinkingNote}`,
   );
   if (!ref) {
@@ -893,6 +1092,7 @@ function showStatus(ctx: ExtensionCommandContext): void {
   lines.push(`Handoff targets (explicit): ${config.handoffModels.length ? config.handoffModels.join(", ") : "(none)"}`);
   lines.push(`Thinking: ${config.thinking ? `on (${config.thinkingLevel})` : "off"}`);
   lines.push(`Paste-time prewarm: ${config.prewarmPastedImages ? `on${editorInstalled ? "" : " (inactive — another custom editor is active)"}` : "off"}`);
+  lines.push(`Async pasted-path fallback: ${config.asyncClipboardHandoff ? "on" : "off"}`);
   lines.push(`maxTokens: ${config.maxTokens ?? "unbounded"} · cacheMax: ${config.cacheMax} · maxDescriptionLines: ${config.maxDescriptionLines === 0 ? "unbounded" : config.maxDescriptionLines}`);
 
   const model = ctx.model;