pi-vision-handoff 0.9.8 → 0.10.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/README.md CHANGED
@@ -45,6 +45,8 @@ No `settings.json` touched. No per-provider glue. Pick a describer once and ever
45
45
  - **⚡ Pre-warmed at paste-enter** — the moment you press enter, `before_agent_start` scans the prompt for pasted image temp-file paths — pi-clipboard, localterm-paste, and any other paste mechanism that writes a pasted image to a temp file and inserts the path as text — reads them, and kicks off the ONE batched vision call concurrent with the agent's first response — so by the time the agent reads the files, they're already cache hits.
46
46
  - **🚀 Paste-time prewarm (opt-in)** — `prewarmPastedImages` (off by default) wraps the editor so pasted clipboard images are described the *instant* their path lands in the prompt — before you hit enter — not at submit. The vision call starts concurrent with you typing your question. Tradeoff: the description is generated without your typed question as context (the question usually isn't entered yet at paste time), and a paste-then-abandon wastes one vision call. TUI only; inactive if another extension replaces the editor. Toggle with `/vision-handoff prewarm on`.
47
47
  - **🏁 Async pasted-path fallback (opt-in)** — `asyncClipboardHandoff` races the normal read path against asynchronous delivery. A direct `read` or nested Fabric `pi.read` of the pasted path cancels the fallback and uses the normal tool-result/context flow; otherwise the completed description is queued as a steering message. Its transcript row is collapsed to one line by default and expands with `Ctrl+O`. Toggle it in the picker with `Ctrl+A` or run `/vision-handoff fallback on`.
48
+ - **🧠 Aware prompt (opt-in)** — `awarePrompt` (off by default) injects a system-prompt note on handoff targets telling the text-only model it CAN read images via the read tool (the handoff swaps image blocks for descriptions). Some models otherwise refuse image-reading requests from self-knowledge even though the description is delivered as text. Toggle with `/vision-handoff aware on`.
49
+ - **💾 Session persistence (opt-in)** — `persistDescriptions` writes each generated description into the session file as a `[Image described: <hash>]` marker + text block next to the read-result blob (the blob stays for kitty rendering). On `/resume` — a new process with an empty in-memory cache — the `context` hook matches the marker to the blob by hash and reuses the persisted description, so a resumed session **never re-calls the vision model** for already-described images (no 30-40s re-describe on the first resumed "ciao"). The persisted description is also seeded into the session cache, so later re-reads of the same image hit too. Toggle with `/vision-handoff persist on`.
48
50
  - **🛡️ Graceful degradation** — no API key? Describer unreachable? Aborted? The image is replaced with a clean `[Image: description unavailable]` placeholder instead of crashing your turn.
49
51
  - **📋 Error logging** — every describer failure and every `image description failed` warning is appended as a JSONL line to `~/.pi/agent/logs/pi-vision-handoff/errors.log`, so a warning like `image description failed — unknown error` is troubleshootable: the detailed reason (exception stack, `stopReason`, config snapshot) is captured at the describer's failure source even when the in-memory error string was already reset by the time the warning fired. Size-capped with a single `.1` backup.
50
52
  - **📊 Usage reporting** — every real describer call reports model + tokens (and Neuralwatt energy/cost when the vision model is a Neuralwatt model), via `pi.appendEntry` + a `vision-handoff:usage` event for live consumers.
@@ -64,6 +66,8 @@ No `settings.json` touched. No per-provider glue. Pick a describer once and ever
64
66
  | `/vision-handoff auto on` / `off` | Toggle automatic handoff for all non-vision models |
65
67
  | `/vision-handoff prewarm on` / `off` | Toggle paste-time prewarm (opt-in, off by default) |
66
68
  | `/vision-handoff fallback on` / `off` | Race matching reads against async pasted-path description injection (opt-in) |
69
+ | `/vision-handoff aware on` / `off` | Tell handoff targets they can read images via the read tool (opt-in, off by default) |
70
+ | `/vision-handoff persist on` / `off` | Persist descriptions to the session file; resumed sessions reuse them with no new vision call (opt-in) |
67
71
  | `/vision-handoff add ollama/llava:13b` | Force handoff for an extra model |
68
72
  | `/vision-handoff remove ollama/llava:13b` | Stop forcing handoff for a model |
69
73
  | `/vision-handoff clear` | Clear the configured vision model |
@@ -81,6 +85,8 @@ Created automatically at `~/.pi/agent/extensions/pi-vision-handoff.json` on firs
81
85
  "handoffModels": ["ollama/llava:13b"],
82
86
  "prewarmPastedImages": false,
83
87
  "asyncClipboardHandoff": false,
88
+ "awarePrompt": false,
89
+ "persistDescriptions": false,
84
90
  "maxTokens": null,
85
91
  "cacheMax": 50,
86
92
  "maxDescriptionLines": 0,
@@ -97,6 +103,8 @@ Created automatically at `~/.pi/agent/extensions/pi-vision-handoff.json` on firs
97
103
  | `handoffModels` | `[]` | Extra `provider/id` refs that should also receive handoff. |
98
104
  | `prewarmPastedImages` | `false` | **Opt-in paste-time prewarm.** When `true`, a custom editor wrapper describes pasted clipboard images the instant their temp-file path lands in the prompt (pre-submit), instead of at submit. Trades a bit of description quality (the description is generated without your typed question as context, since the question usually isn't entered yet at paste time) and a speculative vision call on paste-then-abandon, for earlier prewarm. TUI mode only; inactive if another extension has replaced the editor. |
99
105
  | `asyncClipboardHandoff` | `false` | **Opt-in async fallback.** Starts from the existing submit-time prewarm. A matching direct `read` or Fabric-nested `pi.read` cancels message delivery; if no matching read wins, the description is queued asynchronously as a collapsed custom message and triggers/continues the agent turn. |
106
+ | `awarePrompt` | `false` | **Opt-in aware prompt.** When `true`, injects a system-prompt note on handoff targets telling the text-only model it CAN read images via the read tool (the handoff swaps image blocks for descriptions). Some models otherwise refuse image-reading requests from self-knowledge. |
107
+ | `persistDescriptions` | `false` | **Opt-in session persistence.** When `true`, every generated description is written into the session file as a `[Image described: <hash>]` marker + description text block in the read tool-result content, next to the blob (which stays for kitty rendering). On `/resume`, the `context` hook matches the marker to the blob by hash and reuses the persisted description — no vision call, no wait. Only real descriptions are persisted (a failed `[Image: description unavailable]` is not replayed; the next turn re-attempts). |
100
108
  | `maxTokens` | _(unset = model max, clamped to context window)_ | Cap on a single description's output. `null`/unset = use the vision model's declared max output (`model.maxTokens`), clamped so `maxTokens + 8192 <= contextWindow` (a model whose declared max equals its full context window would otherwise be rejected with a 400). Set a number only to cap cost/latency. A truncation is always surfaced via a `[... description truncated …]` marker when the model hits the limit, so a cut-off description is never mistaken for complete. |
101
109
  | `cacheMax` | `50` | Max described images kept in the in-memory cache per session. |
102
110
  | `maxDescriptionLines` | `0` | Cap on description lines (`0` = unbounded). Default keeps the full description so the `read` tool's native collapse (`ctrl+o`) handles compactness and the model gets complete context; setting `> 0` applies a lossy head-cap to both the TUI render and the model. |
@@ -214,9 +222,13 @@ The default submit-time pipeline below runs regardless:
214
222
  phase (free time: the agent is just waiting for tool results), so
215
223
  the batch is COMPLETE before `context` fires → `context` is a
216
224
  non-blocking cache hit, not a cold miss on the critical path
217
- • does NOT mutate the result: the image stays in storage for kitty
218
- inline rendering and `/resume`; the image→text swap happens in the
219
- `context` hook (on the cloned LLM-bound payload only)
225
+ • does NOT mutate the result by default: the image stays in storage for
226
+ kitty inline rendering and `/resume`; the image→text swap happens in
227
+ the `context` hook (on the cloned LLM-bound payload only). With the
228
+ opt-in `persistDescriptions`, it additionally appends a
229
+ `[Image described: <hash>]` marker + description text block after
230
+ each image block, so the description survives in the session file
231
+ for `/resume` (the blob still stays for kitty rendering)
220
232
  → context (FALLBACK + swap — fires before each LLM call)
221
233
  • catches image blocks that didn't go through `read`'s tool_result —
222
234
  user-attached images, custom extension-injected messages — and
@@ -225,6 +237,12 @@ The default submit-time pipeline below runs regardless:
225
237
  loader's current batch; swaps images for text in the cloned LLM-bound
226
238
  payload (`emitContext` does a `structuredClone`), leaving storage
227
239
  intact for kitty inline rendering and `/resume`
240
+ • persistence fast-path (opt-in): on `/resume` — new process, empty
241
+ in-memory cache — images whose session content carries a
242
+ `[Image described: <hash>]` marker matching their hash are NOT sent
243
+ to the vision model at all: the persisted description is reused
244
+ verbatim (blob dropped from the clone, marker stripped) and seeded
245
+ into the session cache for later re-reads
228
246
 
229
247
  Because the describer runs during the **tool-result phase** (before the agent's next turn), not during the `context` transform (the critical path right before the LLM call), the agent gets described text immediately when its turn starts — it never waits on the describer inline.
230
248
 
@@ -250,7 +268,7 @@ Because pi runs parallel `read` tool calls via `Promise.all` and fires each read
250
268
 
251
269
  | Hook | Image block shape | Handling |
252
270
  |------|-------------------|----------|
253
- | `context` (all messages) | `{ type: "image", data, mimeType }` (pi-ai internal) | undescribed → replaced with description text; already-described → dropped |
271
+ | `context` (all messages) | `{ type: "image", data, mimeType }` (pi-ai internal) | undescribed → replaced with description text; already-described (persisted `[Image described: <hash>]` marker present) → image dropped, marker stripped, persisted description reused — **no vision call on resume** |
254
272
  | `context` (all messages) | `{ type: "image_url", image_url: { url: "data:…" } }` (OpenAI Chat Completions) | detected by shape → replaced with `{ type: "text", text }` |
255
273
  | `context` (all messages) | `{ type: "input_image", image_url: "data:…" }` (OpenAI Responses) | detected by shape → replaced with `{ type: "input_text", text }` |
256
274
  | `context` (all messages) | `{ type: "image", source: { type: "base64", media_type, data } }` (Anthropic Messages) | detected by shape → replaced with `{ type: "text", text }` |
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "pi-vision-handoff",
3
- "version": "0.9.8",
3
+ "version": "0.10.1",
4
4
  "description": "Give text-only pi models vision — describe images with a vision model you pick via an interactive picker, then hand off the text description to non-vision models",
5
5
  "type": "module",
6
6
  "author": "Tom X Nguyen",
@@ -31,9 +31,9 @@
31
31
  "README.md"
32
32
  ],
33
33
  "devDependencies": {
34
- "@earendil-works/pi-ai": "0.84.3",
35
- "@earendil-works/pi-coding-agent": "0.84.3",
36
- "@earendil-works/pi-tui": "0.84.3",
34
+ "@earendil-works/pi-ai": "0.84.4",
35
+ "@earendil-works/pi-coding-agent": "0.84.4",
36
+ "@earendil-works/pi-tui": "0.84.4",
37
37
  "@types/node": "25.9.1",
38
38
  "@vitest/coverage-v8": "4.1.7",
39
39
  "knip": "6.14.1",
package/src/dataloader.ts CHANGED
@@ -242,6 +242,22 @@ export class DescriptionLoader implements Disposable {
242
242
  return promise;
243
243
  }
244
244
 
245
+ /** Seed the in-memory cache with a known description for a hash — used on
246
+ * session resume, where the `context` hook recovers persisted descriptions
247
+ * from the session file and wants later reads of the same image (same hash)
248
+ * in this session to hit the cache instead of re-describing. Respects
249
+ * `cacheMax` eviction like a resolved load. No-op when the hash is already
250
+ * cached (fresh value wins). */
251
+ seedDescription(hash: string, description: string): void {
252
+ if (this.cache.has(hash)) return;
253
+ const cfg = this.deps.getConfig();
254
+ if (this.cache.size >= cfg.cacheMax) {
255
+ const firstKey = this.cache.keys().next().value;
256
+ if (firstKey !== undefined) this.cache.delete(firstKey);
257
+ }
258
+ this.cache.set(hash, Promise.resolve(description));
259
+ }
260
+
245
261
  /** Abandon any in-flight batch and clear turn context. Used on session reset
246
262
  * (the batch's turn is gone) — also exposed via `[Symbol.dispose]` so a
247
263
  * `using` binding can scope a loader lifetime. The description cache is
package/src/index.ts CHANGED
@@ -185,6 +185,15 @@ export interface VisionHandoffConfig {
185
185
  prompt?: string;
186
186
  /** Override the user-prompt prefix (defaults to DEFAULT_USER_PROMPT_PREFIX). */
187
187
  userPromptPrefix?: string;
188
+ /** Opt-in: when true, generated descriptions are PERSISTED to the session
189
+ * file as `[Image described: <hash>]` marker + description text blocks in
190
+ * the read tool-result content (the image blob stays for kitty rendering).
191
+ * On session resume — when the in-memory cache is empty (new process) — the
192
+ * `context` hook matches the marker to the blob by hash and reuses the
193
+ * persisted description WITHOUT calling the vision model, eliminating the
194
+ * 30-40s re-describe latency on every resumed "ciao". Off by default:
195
+ * descriptions are generated per-session as before. */
196
+ persistDescriptions: boolean;
188
197
  /** Whether the vision describer should reason (think) before describing.
189
198
  * Off by default — describing images is a perception task, not a reasoning
190
199
  * one, and thinking adds latency + cost. When on, the level below is sent
@@ -195,6 +204,13 @@ export interface VisionHandoffConfig {
195
204
  * {@link THINKING_LEVELS}. Ignored when {@link thinking} is off or the
196
205
  * vision model lacks reasoning support. */
197
206
  thinkingLevel: ThinkingLevel;
207
+ /** Opt-in: inject a system-prompt note on handoff targets telling the
208
+ * text-only model it CAN read images via the read tool (the handoff swaps
209
+ * image blocks for descriptions). Off by default — without it, some
210
+ * models refuse image-reading requests from their own self-knowledge even
211
+ * though the description is delivered as text. Toggle with
212
+ * `/vision-handoff aware on|off`. */
213
+ awarePrompt: boolean;
198
214
  }
199
215
 
200
216
  export const DEFAULT_CONFIG: VisionHandoffConfig = {
@@ -205,11 +221,13 @@ export const DEFAULT_CONFIG: VisionHandoffConfig = {
205
221
  handoffModels: [],
206
222
  prewarmPastedImages: false,
207
223
  asyncClipboardHandoff: false,
224
+ persistDescriptions: false,
208
225
  maxTokens: undefined,
209
226
  cacheMax: DEFAULT_CACHE_MAX,
210
227
  maxDescriptionLines: DEFAULT_MAX_DESCRIPTION_LINES,
211
228
  thinking: false,
212
229
  thinkingLevel: DEFAULT_THINKING_LEVEL,
230
+ awarePrompt: false,
213
231
  };
214
232
 
215
233
  /** Parse a "provider/id" reference. Returns null if malformed. */
@@ -264,6 +282,7 @@ export function normalizeConfig(raw: unknown): VisionHandoffConfig {
264
282
  if (typeof obj.asyncClipboardHandoff === "boolean") {
265
283
  base.asyncClipboardHandoff = obj.asyncClipboardHandoff;
266
284
  }
285
+ if (typeof obj.persistDescriptions === "boolean") base.persistDescriptions = obj.persistDescriptions;
267
286
  // maxTokens: optional. undefined (default) = no artificial cap. Only set when
268
287
  // a valid positive finite number is given; any other value leaves it unset.
269
288
  if (typeof obj.maxTokens === "number" && Number.isFinite(obj.maxTokens) && obj.maxTokens > 0) {
@@ -285,6 +304,7 @@ export function normalizeConfig(raw: unknown): VisionHandoffConfig {
285
304
 
286
305
  if (typeof obj.thinking === "boolean") base.thinking = obj.thinking;
287
306
  if (isThinkingLevel(obj.thinkingLevel)) base.thinkingLevel = obj.thinkingLevel;
307
+ if (typeof obj.awarePrompt === "boolean") base.awarePrompt = obj.awarePrompt;
288
308
 
289
309
  return base;
290
310
  }
@@ -484,6 +504,50 @@ export function parseBatchedDescriptions(text: string, count: number): (string |
484
504
  return out;
485
505
  }
486
506
 
507
+ /** Marker prefix/suffix framing a PERSISTED image description in the session
508
+ * file. When {@link VisionHandoffConfig.persistDescriptions} is on, the read
509
+ * tool `tool_result` handler appends a text block of the form
510
+ * `[Image described: <hash>]\n<description>` right after the image block (the
511
+ * blob stays for kitty inline rendering). On resume the `context` hook matches
512
+ * the marker to the blob by hash and reuses the description — no vision call
513
+ * for already-described images. */
514
+ export const IMAGE_DESCRIBED_PREFIX = "[Image described: ";
515
+ export const IMAGE_DESCRIBED_SUFFIX = "]";
516
+
517
+ /** Regex matching a persisted-description text block: the marker (any 32-hex
518
+ * hash — `imageHash()`'s sha256 slice) followed by the description body, which
519
+ * may span multiple lines (the `[Image: …]` envelope is kept verbatim). */
520
+ const PERSISTED_DESCRIPTION_RE = /\[Image described: ([0-9a-f]{32})\]\r?\n?([\s\S]*)/;
521
+
522
+ /** Build the persisted-description text block for `hash`: the marker line, then
523
+ * the (already wrapped/truncated) description on the following line(s). */
524
+ export function buildPersistedDescriptionBlock(hash: string, description: string): string {
525
+ return `${IMAGE_DESCRIBED_PREFIX}${hash}${IMAGE_DESCRIBED_SUFFIX}\n${description}`;
526
+ }
527
+
528
+ /** Parse a persisted-description text block, returning the marker's image hash
529
+ * and the description body. Returns null when the text carries no
530
+ * `[Image described: <hash>]` marker or the body is empty. Callers decide what
531
+ * counts as a usable description (e.g. filter out the UNAVAILABLE placeholder
532
+ * so a persisted failure still falls back to a fresh vision call). */
533
+ export function parsePersistedDescriptionBlock(
534
+ text: string,
535
+ ): { hash: string; description: string } | null {
536
+ const m = PERSISTED_DESCRIPTION_RE.exec(text);
537
+ if (!m) return null;
538
+ const description = m[2];
539
+ if (!description) return null;
540
+ return { hash: m[1], description };
541
+ }
542
+
543
+ /** Remove a `[Image described: <hash>]` marker line from a text block, leaving
544
+ * just the description. Used by the `context` hook on the LLM-bound clone so
545
+ * the model never sees the marker itself (the stored session block keeps it).
546
+ * Returns the text unchanged when no marker is present. */
547
+ export function stripPersistedMarker(text: string): string {
548
+ return text.replace(/\[Image described: [0-9a-f]{32}\]\r?\n?/, "");
549
+ }
550
+
487
551
  /** Truncate (if configured) and wrap a raw description in the `[Image: …]`
488
552
  * placeholder envelope used by every insertion path. Centralises the wrap+
489
553
  * truncate logic shared by the read-tool and context injection paths so both
package/vision-handoff.ts CHANGED
@@ -43,6 +43,9 @@ import {
43
43
  readConfig,
44
44
  stripNonVisionImageNote,
45
45
  writeConfig,
46
+ buildPersistedDescriptionBlock,
47
+ parsePersistedDescriptionBlock,
48
+ stripPersistedMarker,
46
49
  HANDOFF_COMMAND_DESCRIPTION,
47
50
  UNAVAILABLE,
48
51
  type ExtractedImage,
@@ -61,6 +64,16 @@ import { Text } from "@earendil-works/pi-tui";
61
64
 
62
65
  let config: VisionHandoffConfig = readConfig();
63
66
 
67
+ /** Marker used to dedup the aware-prompt system-prompt note across turns. */
68
+ const AWARE_PROMPT_MARKER = "# Image reading (pi-vision-handoff active)";
69
+
70
+ /** System-prompt note injected on handoff targets when `awarePrompt` is on.
71
+ * Tells the text-only model it CAN read images via the read tool, since the
72
+ * handoff swaps image blocks for descriptions. */
73
+ const AWARE_PROMPT_NOTE = `# Image reading (pi-vision-handoff active)
74
+
75
+ This model lacks native image input, but the pi-vision-handoff extension is enabled: when you call the read tool on an image file, a configured vision model ({visionModel}) describes it and the description is injected as text in the tool result. You CAN read and act on images via the read tool — do not refuse image-reading requests or claim you cannot see images. The "[Current model does not support images…]" note in tool output is stale and is stripped by the extension.`;
76
+
64
77
  /** Most recent describer failure message (auth error, network error, abort,
65
78
  * empty response, etc.). Set by the describer via the loader deps; surfaced
66
79
  * to the user by the `context`/`tool_result` handler via ctx.ui.notify so a
@@ -422,6 +435,23 @@ export default function (pi: ExtensionAPI) {
422
435
  if (!isConfigured(config)) return;
423
436
  if (!isHandoffTarget(ctx.model, config)) return;
424
437
 
438
+ // Opt-in (`awarePrompt`): note telling text-only handoff targets they CAN
439
+ // read images via the read tool, since the handoff swaps image blocks for
440
+ // descriptions — some models otherwise refuse image-reading requests from
441
+ // self-knowledge even though the description is delivered as text. Must
442
+ // live in THIS single registration: only one before_agent_start result is
443
+ // used, and a second registration shadows this handler's fallback
444
+ // bootstrap (regression: the async pasted-path fallback never fired).
445
+ const awareSystemPrompt =
446
+ config.awarePrompt &&
447
+ config.visionModel &&
448
+ typeof event.systemPrompt === "string" &&
449
+ !event.systemPrompt.includes(AWARE_PROMPT_MARKER)
450
+ ? event.systemPrompt +
451
+ "\n\n" +
452
+ AWARE_PROMPT_NOTE.replace("{visionModel}", config.visionModel)
453
+ : undefined;
454
+
425
455
  // Fresh turn → fresh turn-abort controller. `before_agent_start` fires
426
456
  // BEFORE the agent run starts, so `ctx.signal` is undefined here (the run's
427
457
  // abort signal doesn't exist yet — it's created in `agent.prompt()` →
@@ -510,6 +540,10 @@ export default function (pi: ExtensionAPI) {
510
540
  // original fire-and-forget submit-time prewarm behavior.
511
541
  for (const prepared of preparedClipboardImages) prepared.catch(() => {});
512
542
  }
543
+
544
+ // Aware-prompt note (computed above): only the system prompt is overridden;
545
+ // the prewarm/fallback side effects above run either way.
546
+ return awareSystemPrompt !== undefined ? { systemPrompt: awareSystemPrompt } : undefined;
513
547
  });
514
548
 
515
549
  // A direct read and a nested pi.read both emit a read tool_call. If it targets
@@ -658,6 +692,37 @@ export default function (pi: ExtensionAPI) {
658
692
  }
659
693
  }
660
694
  }
695
+
696
+ // PERSISTENCE (opt-in, `persistDescriptions`): append a
697
+ // `[Image described: <hash>]` marker + description text block after each
698
+ // image block, so the session file carries the description next to the
699
+ // blob (which stays for kitty inline rendering). On `/resume` — new
700
+ // process, in-memory cache empty — the `context` hook matches the marker
701
+ // to the blob by hash and reuses the persisted description, skipping the
702
+ // vision call entirely (no 30-40s re-describe on every resumed turn).
703
+ // Only real descriptions are persisted (UNAVAILABLE is a transient
704
+ // failure, not a description worth replaying — next turn re-attempts).
705
+ // `imgs`/`descs` are aligned in content order, so walking `next` and
706
+ // consuming one desc per image block pairs them correctly.
707
+ if (config.persistDescriptions && hasImageBlock) {
708
+ const withPersisted: (TextContent | ImageContent)[] = [];
709
+ let imgIndex = 0;
710
+ for (const block of next) {
711
+ withPersisted.push(block);
712
+ const img = extractImageFromBlock(block);
713
+ if (img && imgIndex < descs.length) {
714
+ const desc = descs[imgIndex];
715
+ if (desc && desc !== UNAVAILABLE) {
716
+ withPersisted.push({
717
+ type: "text",
718
+ text: buildPersistedDescriptionBlock(imageHash(img.mimeType, img.data), desc),
719
+ });
720
+ }
721
+ imgIndex++;
722
+ }
723
+ }
724
+ return { content: withPersisted };
725
+ }
661
726
  if (stripped) return { content: next as (TextContent | ImageContent)[] };
662
727
  });
663
728
 
@@ -673,12 +738,43 @@ export default function (pi: ExtensionAPI) {
673
738
  // replaced them), so this is usually a no-op for the common paste-and-read
674
739
  // flow. For the images it does find, `loadDescription()` is a cache hit
675
740
  // (warmed by `before_agent_start`) or queues into the loader's current batch.
741
+ //
742
+ // PERSISTENCE (opt-in `persistDescriptions`): when the session was persisted
743
+ // on, a read tool result carries a `[Image described: <hash>]` marker +
744
+ // description block right next to the blob. On `/resume` (new process, empty
745
+ // in-memory cache) this hook matches the marker to the blob by hash, reuses
746
+ // the persisted description, drops the blob from the LLM-bound clone and
747
+ // strips the marker — NO vision call, no wait. Only images WITHOUT a
748
+ // persisted description fall through to `loadDescription()`.
676
749
  pi.on("context", async (event, ctx) => {
677
750
  if (!isConfigured(config)) return;
678
751
 
679
752
  const messages = event.messages as unknown as Array<Record<string, unknown>>;
680
753
  if (!Array.isArray(messages)) return;
681
754
 
755
+ // Persisted descriptions recovered from the session file (markers written
756
+ // by the read-tool `tool_result` handler when persistDescriptions was on).
757
+ // Matched by image hash; a UNAVAILABLE persisted body is ignored so a
758
+ // recorded failure still falls back to a fresh vision call (requirement:
759
+ // absent-or-failed persisted description → current behavior).
760
+ const persisted = new Map<string, string>();
761
+ for (const msg of messages) {
762
+ const content = msg.content;
763
+ if (!Array.isArray(content)) continue;
764
+ for (const block of content) {
765
+ if (!block || typeof block !== "object") continue;
766
+ const b = block as { type?: unknown; text?: unknown };
767
+ if (b.type !== "text" || typeof b.text !== "string") continue;
768
+ const parsed = parsePersistedDescriptionBlock(b.text);
769
+ if (parsed && parsed.description !== UNAVAILABLE && !persisted.has(parsed.hash)) {
770
+ persisted.set(parsed.hash, parsed.description);
771
+ }
772
+ }
773
+ }
774
+
775
+ // Collect image blocks NOT covered by a persisted description. Only these
776
+ // may need the vision model (cache hits resolve instantly; covered images
777
+ // never trigger a vision call — the resume fast-path).
682
778
  const byHash = new Map<string, ExtractedImage>();
683
779
  let anyImage = false;
684
780
  for (const msg of messages) {
@@ -688,30 +784,40 @@ export default function (pi: ExtensionAPI) {
688
784
  const img = extractImageFromBlock(block);
689
785
  if (!img) continue;
690
786
  anyImage = true;
691
- byHash.set(imageHash(img.mimeType, img.data), img);
787
+ const hash = imageHash(img.mimeType, img.data);
788
+ if (!persisted.has(hash)) byHash.set(hash, img);
692
789
  }
693
790
  }
694
791
  if (!anyImage) return;
695
792
  if (!isHandoffTarget(ctx.model, config)) return;
696
793
  const loader = await getLoader();
697
794
  loader.bindTurnContext(ctx);
698
- if (!isAnyVisionModelResolvable(ctx.modelRegistry, config)) {
699
- notifyUnresolvedVisionModel(ctx, config.visionModel!);
700
- return;
701
- }
795
+
796
+ // Seed the in-memory cache with the persisted descriptions so a later
797
+ // re-read of the same image (same hash) in this resumed session is a
798
+ // cache hit too, not a re-describe.
799
+ for (const [hash, desc] of persisted) loader.seedDescription(hash, desc);
702
800
 
703
801
  // Cache hits (warmed by before_agent_start / tool_result) resolve
704
802
  // instantly; any remaining misses queue into the loader's current batch.
803
+ // An all-persisted payload skips this block entirely: no vision call,
804
+ // no wait.
705
805
  const imgs = [...byHash.values()];
706
- const descArr = await Promise.all(imgs.map((img) => loader.loadDescription(img)));
707
806
  const descs = new Map<string, string>();
708
- for (let i = 0; i < imgs.length; i++) {
709
- descs.set(imageHash(imgs[i].mimeType, imgs[i].data), descArr[i]);
710
- }
711
-
712
- if (ctx.signal?.aborted) return;
807
+ if (imgs.length > 0) {
808
+ if (!isAnyVisionModelResolvable(ctx.modelRegistry, config)) {
809
+ notifyUnresolvedVisionModel(ctx, config.visionModel!);
810
+ return;
811
+ }
812
+ const descArr = await Promise.all(imgs.map((img) => loader.loadDescription(img)));
813
+ if (ctx.signal?.aborted) return;
713
814
 
714
- warnFailedImages(ctx, imgs, descArr, lastDescriberError ?? "unknown error");
815
+ warnFailedImages(ctx, imgs, descArr, lastDescriberError ?? "unknown error");
816
+ for (let i = 0; i < imgs.length; i++) {
817
+ descs.set(imageHash(imgs[i].mimeType, imgs[i].data), descArr[i]);
818
+ }
819
+ }
820
+ for (const [hash, desc] of persisted) descs.set(hash, desc);
715
821
 
716
822
  let changed = false;
717
823
  for (const msg of messages) {
@@ -722,12 +828,35 @@ export default function (pi: ExtensionAPI) {
722
828
  for (const block of content) {
723
829
  const img = extractImageFromBlock(block);
724
830
  if (img) {
725
- next.push({ type: "text", text: descs.get(imageHash(img.mimeType, img.data)) ?? UNAVAILABLE });
726
- touched = true;
831
+ const hash = imageHash(img.mimeType, img.data);
832
+ if (persisted.has(hash)) {
833
+ // Already described in this content: the persisted marker + text
834
+ // block is right there (marker stripped below) — drop the blob so
835
+ // the model receives the description exactly once.
836
+ touched = true;
837
+ } else {
838
+ next.push({ type: "text", text: descs.get(hash) ?? UNAVAILABLE });
839
+ touched = true;
840
+ }
727
841
  } else {
728
842
  next.push(block);
729
843
  }
730
844
  }
845
+ // Strip `[Image described: <hash>]` markers from persisted text blocks on
846
+ // the clone, so the model sees only the description itself.
847
+ for (let i = 0; i < next.length; i++) {
848
+ const block = next[i];
849
+ if (block && typeof block === "object" && (block as { type?: unknown }).type === "text") {
850
+ const text = (block as { text?: unknown }).text;
851
+ if (typeof text === "string") {
852
+ const stripped = stripPersistedMarker(text);
853
+ if (stripped !== text) {
854
+ next[i] = { type: "text", text: stripped };
855
+ touched = true;
856
+ }
857
+ }
858
+ }
859
+ }
731
860
  if (touched) {
732
861
  msg.content = next;
733
862
  changed = true;
@@ -753,7 +882,7 @@ export default function (pi: ExtensionAPI) {
753
882
  pi.registerCommand("vision-handoff", {
754
883
  description: HANDOFF_COMMAND_DESCRIPTION,
755
884
  getArgumentCompletions(prefix: string) {
756
- const subcommands = ["select", "model", "status", "enable", "disable", "auto", "thinking", "prewarm", "fallback", "add", "remove", "clear", "help"];
885
+ const subcommands = ["select", "model", "status", "enable", "disable", "auto", "thinking", "prewarm", "fallback", "aware", "persist", "add", "remove", "clear", "help"];
757
886
  const matches = subcommands.filter((s) => s.startsWith(prefix));
758
887
  return matches.length > 0 ? matches.map((s) => ({ value: s, label: s })) : null;
759
888
  },
@@ -791,6 +920,10 @@ async function handleHandoffCommand(ctx: ExtensionCommandContext, args: string):
791
920
  " Toggle describing pasted images at paste-time (opt-in, off by default)",
792
921
  " /vision-handoff fallback <on|off>",
793
922
  " Inject pasted-image descriptions asynchronously when no matching read wins",
923
+ " /vision-handoff aware <on|off>",
924
+ " Tell handoff targets they can read images via the read tool (opt-in, off by default)",
925
+ " /vision-handoff persist <on|off>",
926
+ " Persist descriptions to the session file (resume reuses them, no new vision call)",
794
927
  " /vision-handoff add <p/id> Force handoff for an extra model",
795
928
  " /vision-handoff remove <p/id> Stop forcing handoff for a model",
796
929
  " /vision-handoff clear Clear the configured vision model",
@@ -853,6 +986,16 @@ async function handleHandoffCommand(ctx: ExtensionCommandContext, args: string):
853
986
  return;
854
987
  }
855
988
 
989
+ if (subcommand === "aware") {
990
+ handleAwareSubcommand(ctx, rest);
991
+ return;
992
+ }
993
+
994
+ if (subcommand === "persist") {
995
+ handlePersistSubcommand(ctx, rest);
996
+ return;
997
+ }
998
+
856
999
  if (subcommand === "clear") {
857
1000
  updateConfig(
858
1001
  ctx,
@@ -1028,6 +1171,60 @@ function handleFallbackSubcommand(ctx: ExtensionCommandContext, rest: string): v
1028
1171
  );
1029
1172
  }
1030
1173
 
1174
+ /** Handle /vision-handoff aware <on|off>. */
1175
+ function handleAwareSubcommand(ctx: ExtensionCommandContext, rest: string): void {
1176
+ const value = rest.trim().toLowerCase();
1177
+ if (!value) {
1178
+ ctx.ui.notify(
1179
+ `Aware prompt: ${config.awarePrompt ? "on" : "off"}.\n` +
1180
+ "Usage: /vision-handoff aware <on|off>",
1181
+ "info",
1182
+ );
1183
+ return;
1184
+ }
1185
+ if (value !== "on" && value !== "off") {
1186
+ ctx.ui.notify("Usage: /vision-handoff aware <on|off>", "warning");
1187
+ return;
1188
+ }
1189
+ const on = value === "on";
1190
+ updateConfig(
1191
+ ctx,
1192
+ (c) => ({ ...c, awarePrompt: on }),
1193
+ on
1194
+ ? "Aware prompt on — handoff targets are told they can read images via the read tool."
1195
+ : "Aware prompt off.",
1196
+ );
1197
+ }
1198
+
1199
+ /** Handle /vision-handoff persist <on|off> — toggle persisting descriptions
1200
+ * to the session file. When on, generated descriptions are written as
1201
+ * `[Image described: <hash>]` marker + text blocks in the read tool-result
1202
+ * content; on `/resume` the `context` hook reuses them by hash, skipping the
1203
+ * vision call entirely (no 30-40s re-describe latency). */
1204
+ function handlePersistSubcommand(ctx: ExtensionCommandContext, rest: string): void {
1205
+ const value = rest.trim().toLowerCase();
1206
+ if (!value) {
1207
+ ctx.ui.notify(
1208
+ `Persist descriptions to the session file: ${config.persistDescriptions ? "on" : "off"}.\n` +
1209
+ "Usage: /vision-handoff persist <on|off>",
1210
+ "info",
1211
+ );
1212
+ return;
1213
+ }
1214
+ if (value !== "on" && value !== "off") {
1215
+ ctx.ui.notify("Usage: /vision-handoff persist <on|off>", "warning");
1216
+ return;
1217
+ }
1218
+ const on = value === "on";
1219
+ updateConfig(
1220
+ ctx,
1221
+ (c) => ({ ...c, persistDescriptions: on }),
1222
+ on
1223
+ ? "Persist descriptions on — generated descriptions are saved to the session file; resumed sessions reuse them without a new vision call."
1224
+ : "Persist descriptions off — descriptions are generated per-session as before.",
1225
+ );
1226
+ }
1227
+
1031
1228
  async function showSelector(ctx: ExtensionCommandContext): Promise<void> {
1032
1229
  if (!ctx.hasUI) {
1033
1230
  ctx.ui.notify("/vision-handoff requires interactive mode.", "error");
@@ -1122,6 +1319,8 @@ function showStatus(ctx: ExtensionCommandContext): void {
1122
1319
  lines.push(`Thinking: ${config.thinking ? `on (${config.thinkingLevel})` : "off"}`);
1123
1320
  lines.push(`Paste-time prewarm: ${config.prewarmPastedImages ? `on${editorInstalled ? "" : " (inactive — another custom editor is active)"}` : "off"}`);
1124
1321
  lines.push(`Async pasted-path fallback: ${config.asyncClipboardHandoff ? "on" : "off"}`);
1322
+ lines.push(`Aware prompt: ${config.awarePrompt ? "on" : "off"}`);
1323
+ lines.push(`Persist descriptions (session file): ${config.persistDescriptions ? "on" : "off"}`);
1125
1324
  lines.push(`maxTokens: ${config.maxTokens ?? "unbounded"} · cacheMax: ${config.cacheMax} · maxDescriptionLines: ${config.maxDescriptionLines === 0 ? "unbounded" : config.maxDescriptionLines}`);
1126
1325
 
1127
1326
  const model = ctx.model;