pi-vision-handoff 0.9.8 → 0.10.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +22 -4
- package/package.json +4 -4
- package/src/dataloader.ts +16 -0
- package/src/index.ts +64 -0
- package/vision-handoff.ts +214 -15
package/README.md
CHANGED
|
@@ -45,6 +45,8 @@ No `settings.json` touched. No per-provider glue. Pick a describer once and ever
|
|
|
45
45
|
- **⚡ Pre-warmed at paste-enter** — the moment you press enter, `before_agent_start` scans the prompt for pasted image temp-file paths — pi-clipboard, localterm-paste, and any other paste mechanism that writes a pasted image to a temp file and inserts the path as text — reads them, and kicks off the ONE batched vision call concurrent with the agent's first response — so by the time the agent reads the files, they're already cache hits.
|
|
46
46
|
- **🚀 Paste-time prewarm (opt-in)** — `prewarmPastedImages` (off by default) wraps the editor so pasted clipboard images are described the *instant* their path lands in the prompt — before you hit enter — not at submit. The vision call starts concurrent with you typing your question. Tradeoff: the description is generated without your typed question as context (the question usually isn't entered yet at paste time), and a paste-then-abandon wastes one vision call. TUI only; inactive if another extension replaces the editor. Toggle with `/vision-handoff prewarm on`.
|
|
47
47
|
- **🏁 Async pasted-path fallback (opt-in)** — `asyncClipboardHandoff` races the normal read path against asynchronous delivery. A direct `read` or nested Fabric `pi.read` of the pasted path cancels the fallback and uses the normal tool-result/context flow; otherwise the completed description is queued as a steering message. Its transcript row is collapsed to one line by default and expands with `Ctrl+O`. Toggle it in the picker with `Ctrl+A` or run `/vision-handoff fallback on`.
|
|
48
|
+
- **🧠 Aware prompt (opt-in)** — `awarePrompt` (off by default) injects a system-prompt note on handoff targets telling the text-only model it CAN read images via the read tool (the handoff swaps image blocks for descriptions). Some models otherwise refuse image-reading requests from self-knowledge even though the description is delivered as text. Toggle with `/vision-handoff aware on`.
|
|
49
|
+
- **💾 Session persistence (opt-in)** — `persistDescriptions` writes each generated description into the session file as a `[Image described: <hash>]` marker + text block next to the read-result blob (the blob stays for kitty rendering). On `/resume` — a new process with an empty in-memory cache — the `context` hook matches the marker to the blob by hash and reuses the persisted description, so a resumed session **never re-calls the vision model** for already-described images (no 30-40s re-describe on the first resumed "ciao"). The persisted description is also seeded into the session cache, so later re-reads of the same image hit too. Toggle with `/vision-handoff persist on`.
|
|
48
50
|
- **🛡️ Graceful degradation** — no API key? Describer unreachable? Aborted? The image is replaced with a clean `[Image: description unavailable]` placeholder instead of crashing your turn.
|
|
49
51
|
- **📋 Error logging** — every describer failure and every `image description failed` warning is appended as a JSONL line to `~/.pi/agent/logs/pi-vision-handoff/errors.log`, so a warning like `image description failed — unknown error` is troubleshootable: the detailed reason (exception stack, `stopReason`, config snapshot) is captured at the describer's failure source even when the in-memory error string was already reset by the time the warning fired. Size-capped with a single `.1` backup.
|
|
50
52
|
- **📊 Usage reporting** — every real describer call reports model + tokens (and Neuralwatt energy/cost when the vision model is a Neuralwatt model), via `pi.appendEntry` + a `vision-handoff:usage` event for live consumers.
|
|
@@ -64,6 +66,8 @@ No `settings.json` touched. No per-provider glue. Pick a describer once and ever
|
|
|
64
66
|
| `/vision-handoff auto on` / `off` | Toggle automatic handoff for all non-vision models |
|
|
65
67
|
| `/vision-handoff prewarm on` / `off` | Toggle paste-time prewarm (opt-in, off by default) |
|
|
66
68
|
| `/vision-handoff fallback on` / `off` | Race matching reads against async pasted-path description injection (opt-in) |
|
|
69
|
+
| `/vision-handoff aware on` / `off` | Tell handoff targets they can read images via the read tool (opt-in, off by default) |
|
|
70
|
+
| `/vision-handoff persist on` / `off` | Persist descriptions to the session file; resumed sessions reuse them with no new vision call (opt-in) |
|
|
67
71
|
| `/vision-handoff add ollama/llava:13b` | Force handoff for an extra model |
|
|
68
72
|
| `/vision-handoff remove ollama/llava:13b` | Stop forcing handoff for a model |
|
|
69
73
|
| `/vision-handoff clear` | Clear the configured vision model |
|
|
@@ -81,6 +85,8 @@ Created automatically at `~/.pi/agent/extensions/pi-vision-handoff.json` on firs
|
|
|
81
85
|
"handoffModels": ["ollama/llava:13b"],
|
|
82
86
|
"prewarmPastedImages": false,
|
|
83
87
|
"asyncClipboardHandoff": false,
|
|
88
|
+
"awarePrompt": false,
|
|
89
|
+
"persistDescriptions": false,
|
|
84
90
|
"maxTokens": null,
|
|
85
91
|
"cacheMax": 50,
|
|
86
92
|
"maxDescriptionLines": 0,
|
|
@@ -97,6 +103,8 @@ Created automatically at `~/.pi/agent/extensions/pi-vision-handoff.json` on firs
|
|
|
97
103
|
| `handoffModels` | `[]` | Extra `provider/id` refs that should also receive handoff. |
|
|
98
104
|
| `prewarmPastedImages` | `false` | **Opt-in paste-time prewarm.** When `true`, a custom editor wrapper describes pasted clipboard images the instant their temp-file path lands in the prompt (pre-submit), instead of at submit. Trades a bit of description quality (the description is generated without your typed question as context, since the question usually isn't entered yet at paste time) and a speculative vision call on paste-then-abandon, for earlier prewarm. TUI mode only; inactive if another extension has replaced the editor. |
|
|
99
105
|
| `asyncClipboardHandoff` | `false` | **Opt-in async fallback.** Starts from the existing submit-time prewarm. A matching direct `read` or Fabric-nested `pi.read` cancels message delivery; if no matching read wins, the description is queued asynchronously as a collapsed custom message and triggers/continues the agent turn. |
|
|
106
|
+
| `awarePrompt` | `false` | **Opt-in aware prompt.** When `true`, injects a system-prompt note on handoff targets telling the text-only model it CAN read images via the read tool (the handoff swaps image blocks for descriptions). Some models otherwise refuse image-reading requests from self-knowledge. |
|
|
107
|
+
| `persistDescriptions` | `false` | **Opt-in session persistence.** When `true`, every generated description is written into the session file as a `[Image described: <hash>]` marker + description text block in the read tool-result content, next to the blob (which stays for kitty rendering). On `/resume`, the `context` hook matches the marker to the blob by hash and reuses the persisted description — no vision call, no wait. Only real descriptions are persisted (a failed `[Image: description unavailable]` is not replayed; the next turn re-attempts). |
|
|
100
108
|
| `maxTokens` | _(unset = model max, clamped to context window)_ | Cap on a single description's output. `null`/unset = use the vision model's declared max output (`model.maxTokens`), clamped so `maxTokens + 8192 <= contextWindow` (a model whose declared max equals its full context window would otherwise be rejected with a 400). Set a number only to cap cost/latency. A truncation is always surfaced via a `[... description truncated …]` marker when the model hits the limit, so a cut-off description is never mistaken for complete. |
|
|
101
109
|
| `cacheMax` | `50` | Max described images kept in the in-memory cache per session. |
|
|
102
110
|
| `maxDescriptionLines` | `0` | Cap on description lines (`0` = unbounded). Default keeps the full description so the `read` tool's native collapse (`ctrl+o`) handles compactness and the model gets complete context; setting `> 0` applies a lossy head-cap to both the TUI render and the model. |
|
|
@@ -214,9 +222,13 @@ The default submit-time pipeline below runs regardless:
|
|
|
214
222
|
phase (free time: the agent is just waiting for tool results), so
|
|
215
223
|
the batch is COMPLETE before `context` fires → `context` is a
|
|
216
224
|
non-blocking cache hit, not a cold miss on the critical path
|
|
217
|
-
• does NOT mutate the result: the image stays in storage for
|
|
218
|
-
inline rendering and `/resume`; the image→text swap happens in
|
|
219
|
-
`context` hook (on the cloned LLM-bound payload only)
|
|
225
|
+
• does NOT mutate the result by default: the image stays in storage for
|
|
226
|
+
kitty inline rendering and `/resume`; the image→text swap happens in
|
|
227
|
+
the `context` hook (on the cloned LLM-bound payload only). With the
|
|
228
|
+
opt-in `persistDescriptions`, it additionally appends a
|
|
229
|
+
`[Image described: <hash>]` marker + description text block after
|
|
230
|
+
each image block, so the description survives in the session file
|
|
231
|
+
for `/resume` (the blob still stays for kitty rendering)
|
|
220
232
|
→ context (FALLBACK + swap — fires before each LLM call)
|
|
221
233
|
• catches image blocks that didn't go through `read`'s tool_result —
|
|
222
234
|
user-attached images, custom extension-injected messages — and
|
|
@@ -225,6 +237,12 @@ The default submit-time pipeline below runs regardless:
|
|
|
225
237
|
loader's current batch; swaps images for text in the cloned LLM-bound
|
|
226
238
|
payload (`emitContext` does a `structuredClone`), leaving storage
|
|
227
239
|
intact for kitty inline rendering and `/resume`
|
|
240
|
+
• persistence fast-path (opt-in): on `/resume` — new process, empty
|
|
241
|
+
in-memory cache — images whose session content carries a
|
|
242
|
+
`[Image described: <hash>]` marker matching their hash are NOT sent
|
|
243
|
+
to the vision model at all: the persisted description is reused
|
|
244
|
+
verbatim (blob dropped from the clone, marker stripped) and seeded
|
|
245
|
+
into the session cache for later re-reads
|
|
228
246
|
|
|
229
247
|
Because the describer runs during the **tool-result phase** (before the agent's next turn), not during the `context` transform (the critical path right before the LLM call), the agent gets described text immediately when its turn starts — it never waits on the describer inline.
|
|
230
248
|
|
|
@@ -250,7 +268,7 @@ Because pi runs parallel `read` tool calls via `Promise.all` and fires each read
|
|
|
250
268
|
|
|
251
269
|
| Hook | Image block shape | Handling |
|
|
252
270
|
|------|-------------------|----------|
|
|
253
|
-
| `context` (all messages) | `{ type: "image", data, mimeType }` (pi-ai internal) | undescribed → replaced with description text; already-described → dropped |
|
|
271
|
+
| `context` (all messages) | `{ type: "image", data, mimeType }` (pi-ai internal) | undescribed → replaced with description text; already-described (persisted `[Image described: <hash>]` marker present) → image dropped, marker stripped, persisted description reused — **no vision call on resume** |
|
|
254
272
|
| `context` (all messages) | `{ type: "image_url", image_url: { url: "data:…" } }` (OpenAI Chat Completions) | detected by shape → replaced with `{ type: "text", text }` |
|
|
255
273
|
| `context` (all messages) | `{ type: "input_image", image_url: "data:…" }` (OpenAI Responses) | detected by shape → replaced with `{ type: "input_text", text }` |
|
|
256
274
|
| `context` (all messages) | `{ type: "image", source: { type: "base64", media_type, data } }` (Anthropic Messages) | detected by shape → replaced with `{ type: "text", text }` |
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "pi-vision-handoff",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.10.1",
|
|
4
4
|
"description": "Give text-only pi models vision — describe images with a vision model you pick via an interactive picker, then hand off the text description to non-vision models",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"author": "Tom X Nguyen",
|
|
@@ -31,9 +31,9 @@
|
|
|
31
31
|
"README.md"
|
|
32
32
|
],
|
|
33
33
|
"devDependencies": {
|
|
34
|
-
"@earendil-works/pi-ai": "0.84.
|
|
35
|
-
"@earendil-works/pi-coding-agent": "0.84.
|
|
36
|
-
"@earendil-works/pi-tui": "0.84.
|
|
34
|
+
"@earendil-works/pi-ai": "0.84.4",
|
|
35
|
+
"@earendil-works/pi-coding-agent": "0.84.4",
|
|
36
|
+
"@earendil-works/pi-tui": "0.84.4",
|
|
37
37
|
"@types/node": "25.9.1",
|
|
38
38
|
"@vitest/coverage-v8": "4.1.7",
|
|
39
39
|
"knip": "6.14.1",
|
package/src/dataloader.ts
CHANGED
|
@@ -242,6 +242,22 @@ export class DescriptionLoader implements Disposable {
|
|
|
242
242
|
return promise;
|
|
243
243
|
}
|
|
244
244
|
|
|
245
|
+
/** Seed the in-memory cache with a known description for a hash — used on
|
|
246
|
+
* session resume, where the `context` hook recovers persisted descriptions
|
|
247
|
+
* from the session file and wants later reads of the same image (same hash)
|
|
248
|
+
* in this session to hit the cache instead of re-describing. Respects
|
|
249
|
+
* `cacheMax` eviction like a resolved load. No-op when the hash is already
|
|
250
|
+
* cached (fresh value wins). */
|
|
251
|
+
seedDescription(hash: string, description: string): void {
|
|
252
|
+
if (this.cache.has(hash)) return;
|
|
253
|
+
const cfg = this.deps.getConfig();
|
|
254
|
+
if (this.cache.size >= cfg.cacheMax) {
|
|
255
|
+
const firstKey = this.cache.keys().next().value;
|
|
256
|
+
if (firstKey !== undefined) this.cache.delete(firstKey);
|
|
257
|
+
}
|
|
258
|
+
this.cache.set(hash, Promise.resolve(description));
|
|
259
|
+
}
|
|
260
|
+
|
|
245
261
|
/** Abandon any in-flight batch and clear turn context. Used on session reset
|
|
246
262
|
* (the batch's turn is gone) — also exposed via `[Symbol.dispose]` so a
|
|
247
263
|
* `using` binding can scope a loader lifetime. The description cache is
|
package/src/index.ts
CHANGED
|
@@ -185,6 +185,15 @@ export interface VisionHandoffConfig {
|
|
|
185
185
|
prompt?: string;
|
|
186
186
|
/** Override the user-prompt prefix (defaults to DEFAULT_USER_PROMPT_PREFIX). */
|
|
187
187
|
userPromptPrefix?: string;
|
|
188
|
+
/** Opt-in: when true, generated descriptions are PERSISTED to the session
|
|
189
|
+
* file as `[Image described: <hash>]` marker + description text blocks in
|
|
190
|
+
* the read tool-result content (the image blob stays for kitty rendering).
|
|
191
|
+
* On session resume — when the in-memory cache is empty (new process) — the
|
|
192
|
+
* `context` hook matches the marker to the blob by hash and reuses the
|
|
193
|
+
* persisted description WITHOUT calling the vision model, eliminating the
|
|
194
|
+
* 30-40s re-describe latency on every resumed "ciao". Off by default:
|
|
195
|
+
* descriptions are generated per-session as before. */
|
|
196
|
+
persistDescriptions: boolean;
|
|
188
197
|
/** Whether the vision describer should reason (think) before describing.
|
|
189
198
|
* Off by default — describing images is a perception task, not a reasoning
|
|
190
199
|
* one, and thinking adds latency + cost. When on, the level below is sent
|
|
@@ -195,6 +204,13 @@ export interface VisionHandoffConfig {
|
|
|
195
204
|
* {@link THINKING_LEVELS}. Ignored when {@link thinking} is off or the
|
|
196
205
|
* vision model lacks reasoning support. */
|
|
197
206
|
thinkingLevel: ThinkingLevel;
|
|
207
|
+
/** Opt-in: inject a system-prompt note on handoff targets telling the
|
|
208
|
+
* text-only model it CAN read images via the read tool (the handoff swaps
|
|
209
|
+
* image blocks for descriptions). Off by default — without it, some
|
|
210
|
+
* models refuse image-reading requests from their own self-knowledge even
|
|
211
|
+
* though the description is delivered as text. Toggle with
|
|
212
|
+
* `/vision-handoff aware on|off`. */
|
|
213
|
+
awarePrompt: boolean;
|
|
198
214
|
}
|
|
199
215
|
|
|
200
216
|
export const DEFAULT_CONFIG: VisionHandoffConfig = {
|
|
@@ -205,11 +221,13 @@ export const DEFAULT_CONFIG: VisionHandoffConfig = {
|
|
|
205
221
|
handoffModels: [],
|
|
206
222
|
prewarmPastedImages: false,
|
|
207
223
|
asyncClipboardHandoff: false,
|
|
224
|
+
persistDescriptions: false,
|
|
208
225
|
maxTokens: undefined,
|
|
209
226
|
cacheMax: DEFAULT_CACHE_MAX,
|
|
210
227
|
maxDescriptionLines: DEFAULT_MAX_DESCRIPTION_LINES,
|
|
211
228
|
thinking: false,
|
|
212
229
|
thinkingLevel: DEFAULT_THINKING_LEVEL,
|
|
230
|
+
awarePrompt: false,
|
|
213
231
|
};
|
|
214
232
|
|
|
215
233
|
/** Parse a "provider/id" reference. Returns null if malformed. */
|
|
@@ -264,6 +282,7 @@ export function normalizeConfig(raw: unknown): VisionHandoffConfig {
|
|
|
264
282
|
if (typeof obj.asyncClipboardHandoff === "boolean") {
|
|
265
283
|
base.asyncClipboardHandoff = obj.asyncClipboardHandoff;
|
|
266
284
|
}
|
|
285
|
+
if (typeof obj.persistDescriptions === "boolean") base.persistDescriptions = obj.persistDescriptions;
|
|
267
286
|
// maxTokens: optional. undefined (default) = no artificial cap. Only set when
|
|
268
287
|
// a valid positive finite number is given; any other value leaves it unset.
|
|
269
288
|
if (typeof obj.maxTokens === "number" && Number.isFinite(obj.maxTokens) && obj.maxTokens > 0) {
|
|
@@ -285,6 +304,7 @@ export function normalizeConfig(raw: unknown): VisionHandoffConfig {
|
|
|
285
304
|
|
|
286
305
|
if (typeof obj.thinking === "boolean") base.thinking = obj.thinking;
|
|
287
306
|
if (isThinkingLevel(obj.thinkingLevel)) base.thinkingLevel = obj.thinkingLevel;
|
|
307
|
+
if (typeof obj.awarePrompt === "boolean") base.awarePrompt = obj.awarePrompt;
|
|
288
308
|
|
|
289
309
|
return base;
|
|
290
310
|
}
|
|
@@ -484,6 +504,50 @@ export function parseBatchedDescriptions(text: string, count: number): (string |
|
|
|
484
504
|
return out;
|
|
485
505
|
}
|
|
486
506
|
|
|
507
|
+
/** Marker prefix/suffix framing a PERSISTED image description in the session
|
|
508
|
+
* file. When {@link VisionHandoffConfig.persistDescriptions} is on, the read
|
|
509
|
+
* tool `tool_result` handler appends a text block of the form
|
|
510
|
+
* `[Image described: <hash>]\n<description>` right after the image block (the
|
|
511
|
+
* blob stays for kitty inline rendering). On resume the `context` hook matches
|
|
512
|
+
* the marker to the blob by hash and reuses the description — no vision call
|
|
513
|
+
* for already-described images. */
|
|
514
|
+
export const IMAGE_DESCRIBED_PREFIX = "[Image described: ";
|
|
515
|
+
export const IMAGE_DESCRIBED_SUFFIX = "]";
|
|
516
|
+
|
|
517
|
+
/** Regex matching a persisted-description text block: the marker (any 32-hex
|
|
518
|
+
* hash — `imageHash()`'s sha256 slice) followed by the description body, which
|
|
519
|
+
* may span multiple lines (the `[Image: …]` envelope is kept verbatim). */
|
|
520
|
+
const PERSISTED_DESCRIPTION_RE = /\[Image described: ([0-9a-f]{32})\]\r?\n?([\s\S]*)/;
|
|
521
|
+
|
|
522
|
+
/** Build the persisted-description text block for `hash`: the marker line, then
|
|
523
|
+
* the (already wrapped/truncated) description on the following line(s). */
|
|
524
|
+
export function buildPersistedDescriptionBlock(hash: string, description: string): string {
|
|
525
|
+
return `${IMAGE_DESCRIBED_PREFIX}${hash}${IMAGE_DESCRIBED_SUFFIX}\n${description}`;
|
|
526
|
+
}
|
|
527
|
+
|
|
528
|
+
/** Parse a persisted-description text block, returning the marker's image hash
|
|
529
|
+
* and the description body. Returns null when the text carries no
|
|
530
|
+
* `[Image described: <hash>]` marker or the body is empty. Callers decide what
|
|
531
|
+
* counts as a usable description (e.g. filter out the UNAVAILABLE placeholder
|
|
532
|
+
* so a persisted failure still falls back to a fresh vision call). */
|
|
533
|
+
export function parsePersistedDescriptionBlock(
|
|
534
|
+
text: string,
|
|
535
|
+
): { hash: string; description: string } | null {
|
|
536
|
+
const m = PERSISTED_DESCRIPTION_RE.exec(text);
|
|
537
|
+
if (!m) return null;
|
|
538
|
+
const description = m[2];
|
|
539
|
+
if (!description) return null;
|
|
540
|
+
return { hash: m[1], description };
|
|
541
|
+
}
|
|
542
|
+
|
|
543
|
+
/** Remove a `[Image described: <hash>]` marker line from a text block, leaving
|
|
544
|
+
* just the description. Used by the `context` hook on the LLM-bound clone so
|
|
545
|
+
* the model never sees the marker itself (the stored session block keeps it).
|
|
546
|
+
* Returns the text unchanged when no marker is present. */
|
|
547
|
+
export function stripPersistedMarker(text: string): string {
|
|
548
|
+
return text.replace(/\[Image described: [0-9a-f]{32}\]\r?\n?/, "");
|
|
549
|
+
}
|
|
550
|
+
|
|
487
551
|
/** Truncate (if configured) and wrap a raw description in the `[Image: …]`
|
|
488
552
|
* placeholder envelope used by every insertion path. Centralises the wrap+
|
|
489
553
|
* truncate logic shared by the read-tool and context injection paths so both
|
package/vision-handoff.ts
CHANGED
|
@@ -43,6 +43,9 @@ import {
|
|
|
43
43
|
readConfig,
|
|
44
44
|
stripNonVisionImageNote,
|
|
45
45
|
writeConfig,
|
|
46
|
+
buildPersistedDescriptionBlock,
|
|
47
|
+
parsePersistedDescriptionBlock,
|
|
48
|
+
stripPersistedMarker,
|
|
46
49
|
HANDOFF_COMMAND_DESCRIPTION,
|
|
47
50
|
UNAVAILABLE,
|
|
48
51
|
type ExtractedImage,
|
|
@@ -61,6 +64,16 @@ import { Text } from "@earendil-works/pi-tui";
|
|
|
61
64
|
|
|
62
65
|
let config: VisionHandoffConfig = readConfig();
|
|
63
66
|
|
|
67
|
+
/** Marker used to dedup the aware-prompt system-prompt note across turns. */
|
|
68
|
+
const AWARE_PROMPT_MARKER = "# Image reading (pi-vision-handoff active)";
|
|
69
|
+
|
|
70
|
+
/** System-prompt note injected on handoff targets when `awarePrompt` is on.
|
|
71
|
+
* Tells the text-only model it CAN read images via the read tool, since the
|
|
72
|
+
* handoff swaps image blocks for descriptions. */
|
|
73
|
+
const AWARE_PROMPT_NOTE = `# Image reading (pi-vision-handoff active)
|
|
74
|
+
|
|
75
|
+
This model lacks native image input, but the pi-vision-handoff extension is enabled: when you call the read tool on an image file, a configured vision model ({visionModel}) describes it and the description is injected as text in the tool result. You CAN read and act on images via the read tool — do not refuse image-reading requests or claim you cannot see images. The "[Current model does not support images…]" note in tool output is stale and is stripped by the extension.`;
|
|
76
|
+
|
|
64
77
|
/** Most recent describer failure message (auth error, network error, abort,
|
|
65
78
|
* empty response, etc.). Set by the describer via the loader deps; surfaced
|
|
66
79
|
* to the user by the `context`/`tool_result` handler via ctx.ui.notify so a
|
|
@@ -422,6 +435,23 @@ export default function (pi: ExtensionAPI) {
|
|
|
422
435
|
if (!isConfigured(config)) return;
|
|
423
436
|
if (!isHandoffTarget(ctx.model, config)) return;
|
|
424
437
|
|
|
438
|
+
// Opt-in (`awarePrompt`): note telling text-only handoff targets they CAN
|
|
439
|
+
// read images via the read tool, since the handoff swaps image blocks for
|
|
440
|
+
// descriptions — some models otherwise refuse image-reading requests from
|
|
441
|
+
// self-knowledge even though the description is delivered as text. Must
|
|
442
|
+
// live in THIS single registration: only one before_agent_start result is
|
|
443
|
+
// used, and a second registration shadows this handler's fallback
|
|
444
|
+
// bootstrap (regression: the async pasted-path fallback never fired).
|
|
445
|
+
const awareSystemPrompt =
|
|
446
|
+
config.awarePrompt &&
|
|
447
|
+
config.visionModel &&
|
|
448
|
+
typeof event.systemPrompt === "string" &&
|
|
449
|
+
!event.systemPrompt.includes(AWARE_PROMPT_MARKER)
|
|
450
|
+
? event.systemPrompt +
|
|
451
|
+
"\n\n" +
|
|
452
|
+
AWARE_PROMPT_NOTE.replace("{visionModel}", config.visionModel)
|
|
453
|
+
: undefined;
|
|
454
|
+
|
|
425
455
|
// Fresh turn → fresh turn-abort controller. `before_agent_start` fires
|
|
426
456
|
// BEFORE the agent run starts, so `ctx.signal` is undefined here (the run's
|
|
427
457
|
// abort signal doesn't exist yet — it's created in `agent.prompt()` →
|
|
@@ -510,6 +540,10 @@ export default function (pi: ExtensionAPI) {
|
|
|
510
540
|
// original fire-and-forget submit-time prewarm behavior.
|
|
511
541
|
for (const prepared of preparedClipboardImages) prepared.catch(() => {});
|
|
512
542
|
}
|
|
543
|
+
|
|
544
|
+
// Aware-prompt note (computed above): only the system prompt is overridden;
|
|
545
|
+
// the prewarm/fallback side effects above run either way.
|
|
546
|
+
return awareSystemPrompt !== undefined ? { systemPrompt: awareSystemPrompt } : undefined;
|
|
513
547
|
});
|
|
514
548
|
|
|
515
549
|
// A direct read and a nested pi.read both emit a read tool_call. If it targets
|
|
@@ -658,6 +692,37 @@ export default function (pi: ExtensionAPI) {
|
|
|
658
692
|
}
|
|
659
693
|
}
|
|
660
694
|
}
|
|
695
|
+
|
|
696
|
+
// PERSISTENCE (opt-in, `persistDescriptions`): append a
|
|
697
|
+
// `[Image described: <hash>]` marker + description text block after each
|
|
698
|
+
// image block, so the session file carries the description next to the
|
|
699
|
+
// blob (which stays for kitty inline rendering). On `/resume` — new
|
|
700
|
+
// process, in-memory cache empty — the `context` hook matches the marker
|
|
701
|
+
// to the blob by hash and reuses the persisted description, skipping the
|
|
702
|
+
// vision call entirely (no 30-40s re-describe on every resumed turn).
|
|
703
|
+
// Only real descriptions are persisted (UNAVAILABLE is a transient
|
|
704
|
+
// failure, not a description worth replaying — next turn re-attempts).
|
|
705
|
+
// `imgs`/`descs` are aligned in content order, so walking `next` and
|
|
706
|
+
// consuming one desc per image block pairs them correctly.
|
|
707
|
+
if (config.persistDescriptions && hasImageBlock) {
|
|
708
|
+
const withPersisted: (TextContent | ImageContent)[] = [];
|
|
709
|
+
let imgIndex = 0;
|
|
710
|
+
for (const block of next) {
|
|
711
|
+
withPersisted.push(block);
|
|
712
|
+
const img = extractImageFromBlock(block);
|
|
713
|
+
if (img && imgIndex < descs.length) {
|
|
714
|
+
const desc = descs[imgIndex];
|
|
715
|
+
if (desc && desc !== UNAVAILABLE) {
|
|
716
|
+
withPersisted.push({
|
|
717
|
+
type: "text",
|
|
718
|
+
text: buildPersistedDescriptionBlock(imageHash(img.mimeType, img.data), desc),
|
|
719
|
+
});
|
|
720
|
+
}
|
|
721
|
+
imgIndex++;
|
|
722
|
+
}
|
|
723
|
+
}
|
|
724
|
+
return { content: withPersisted };
|
|
725
|
+
}
|
|
661
726
|
if (stripped) return { content: next as (TextContent | ImageContent)[] };
|
|
662
727
|
});
|
|
663
728
|
|
|
@@ -673,12 +738,43 @@ export default function (pi: ExtensionAPI) {
|
|
|
673
738
|
// replaced them), so this is usually a no-op for the common paste-and-read
|
|
674
739
|
// flow. For the images it does find, `loadDescription()` is a cache hit
|
|
675
740
|
// (warmed by `before_agent_start`) or queues into the loader's current batch.
|
|
741
|
+
//
|
|
742
|
+
// PERSISTENCE (opt-in `persistDescriptions`): when the session was persisted
|
|
743
|
+
// on, a read tool result carries a `[Image described: <hash>]` marker +
|
|
744
|
+
// description block right next to the blob. On `/resume` (new process, empty
|
|
745
|
+
// in-memory cache) this hook matches the marker to the blob by hash, reuses
|
|
746
|
+
// the persisted description, drops the blob from the LLM-bound clone and
|
|
747
|
+
// strips the marker — NO vision call, no wait. Only images WITHOUT a
|
|
748
|
+
// persisted description fall through to `loadDescription()`.
|
|
676
749
|
pi.on("context", async (event, ctx) => {
|
|
677
750
|
if (!isConfigured(config)) return;
|
|
678
751
|
|
|
679
752
|
const messages = event.messages as unknown as Array<Record<string, unknown>>;
|
|
680
753
|
if (!Array.isArray(messages)) return;
|
|
681
754
|
|
|
755
|
+
// Persisted descriptions recovered from the session file (markers written
|
|
756
|
+
// by the read-tool `tool_result` handler when persistDescriptions was on).
|
|
757
|
+
// Matched by image hash; a UNAVAILABLE persisted body is ignored so a
|
|
758
|
+
// recorded failure still falls back to a fresh vision call (requirement:
|
|
759
|
+
// absent-or-failed persisted description → current behavior).
|
|
760
|
+
const persisted = new Map<string, string>();
|
|
761
|
+
for (const msg of messages) {
|
|
762
|
+
const content = msg.content;
|
|
763
|
+
if (!Array.isArray(content)) continue;
|
|
764
|
+
for (const block of content) {
|
|
765
|
+
if (!block || typeof block !== "object") continue;
|
|
766
|
+
const b = block as { type?: unknown; text?: unknown };
|
|
767
|
+
if (b.type !== "text" || typeof b.text !== "string") continue;
|
|
768
|
+
const parsed = parsePersistedDescriptionBlock(b.text);
|
|
769
|
+
if (parsed && parsed.description !== UNAVAILABLE && !persisted.has(parsed.hash)) {
|
|
770
|
+
persisted.set(parsed.hash, parsed.description);
|
|
771
|
+
}
|
|
772
|
+
}
|
|
773
|
+
}
|
|
774
|
+
|
|
775
|
+
// Collect image blocks NOT covered by a persisted description. Only these
|
|
776
|
+
// may need the vision model (cache hits resolve instantly; covered images
|
|
777
|
+
// never trigger a vision call — the resume fast-path).
|
|
682
778
|
const byHash = new Map<string, ExtractedImage>();
|
|
683
779
|
let anyImage = false;
|
|
684
780
|
for (const msg of messages) {
|
|
@@ -688,30 +784,40 @@ export default function (pi: ExtensionAPI) {
|
|
|
688
784
|
const img = extractImageFromBlock(block);
|
|
689
785
|
if (!img) continue;
|
|
690
786
|
anyImage = true;
|
|
691
|
-
|
|
787
|
+
const hash = imageHash(img.mimeType, img.data);
|
|
788
|
+
if (!persisted.has(hash)) byHash.set(hash, img);
|
|
692
789
|
}
|
|
693
790
|
}
|
|
694
791
|
if (!anyImage) return;
|
|
695
792
|
if (!isHandoffTarget(ctx.model, config)) return;
|
|
696
793
|
const loader = await getLoader();
|
|
697
794
|
loader.bindTurnContext(ctx);
|
|
698
|
-
|
|
699
|
-
|
|
700
|
-
|
|
701
|
-
|
|
795
|
+
|
|
796
|
+
// Seed the in-memory cache with the persisted descriptions so a later
|
|
797
|
+
// re-read of the same image (same hash) in this resumed session is a
|
|
798
|
+
// cache hit too, not a re-describe.
|
|
799
|
+
for (const [hash, desc] of persisted) loader.seedDescription(hash, desc);
|
|
702
800
|
|
|
703
801
|
// Cache hits (warmed by before_agent_start / tool_result) resolve
|
|
704
802
|
// instantly; any remaining misses queue into the loader's current batch.
|
|
803
|
+
// An all-persisted payload skips this block entirely: no vision call,
|
|
804
|
+
// no wait.
|
|
705
805
|
const imgs = [...byHash.values()];
|
|
706
|
-
const descArr = await Promise.all(imgs.map((img) => loader.loadDescription(img)));
|
|
707
806
|
const descs = new Map<string, string>();
|
|
708
|
-
|
|
709
|
-
|
|
710
|
-
|
|
711
|
-
|
|
712
|
-
|
|
807
|
+
if (imgs.length > 0) {
|
|
808
|
+
if (!isAnyVisionModelResolvable(ctx.modelRegistry, config)) {
|
|
809
|
+
notifyUnresolvedVisionModel(ctx, config.visionModel!);
|
|
810
|
+
return;
|
|
811
|
+
}
|
|
812
|
+
const descArr = await Promise.all(imgs.map((img) => loader.loadDescription(img)));
|
|
813
|
+
if (ctx.signal?.aborted) return;
|
|
713
814
|
|
|
714
|
-
|
|
815
|
+
warnFailedImages(ctx, imgs, descArr, lastDescriberError ?? "unknown error");
|
|
816
|
+
for (let i = 0; i < imgs.length; i++) {
|
|
817
|
+
descs.set(imageHash(imgs[i].mimeType, imgs[i].data), descArr[i]);
|
|
818
|
+
}
|
|
819
|
+
}
|
|
820
|
+
for (const [hash, desc] of persisted) descs.set(hash, desc);
|
|
715
821
|
|
|
716
822
|
let changed = false;
|
|
717
823
|
for (const msg of messages) {
|
|
@@ -722,12 +828,35 @@ export default function (pi: ExtensionAPI) {
|
|
|
722
828
|
for (const block of content) {
|
|
723
829
|
const img = extractImageFromBlock(block);
|
|
724
830
|
if (img) {
|
|
725
|
-
|
|
726
|
-
|
|
831
|
+
const hash = imageHash(img.mimeType, img.data);
|
|
832
|
+
if (persisted.has(hash)) {
|
|
833
|
+
// Already described in this content: the persisted marker + text
|
|
834
|
+
// block is right there (marker stripped below) — drop the blob so
|
|
835
|
+
// the model receives the description exactly once.
|
|
836
|
+
touched = true;
|
|
837
|
+
} else {
|
|
838
|
+
next.push({ type: "text", text: descs.get(hash) ?? UNAVAILABLE });
|
|
839
|
+
touched = true;
|
|
840
|
+
}
|
|
727
841
|
} else {
|
|
728
842
|
next.push(block);
|
|
729
843
|
}
|
|
730
844
|
}
|
|
845
|
+
// Strip `[Image described: <hash>]` markers from persisted text blocks on
|
|
846
|
+
// the clone, so the model sees only the description itself.
|
|
847
|
+
for (let i = 0; i < next.length; i++) {
|
|
848
|
+
const block = next[i];
|
|
849
|
+
if (block && typeof block === "object" && (block as { type?: unknown }).type === "text") {
|
|
850
|
+
const text = (block as { text?: unknown }).text;
|
|
851
|
+
if (typeof text === "string") {
|
|
852
|
+
const stripped = stripPersistedMarker(text);
|
|
853
|
+
if (stripped !== text) {
|
|
854
|
+
next[i] = { type: "text", text: stripped };
|
|
855
|
+
touched = true;
|
|
856
|
+
}
|
|
857
|
+
}
|
|
858
|
+
}
|
|
859
|
+
}
|
|
731
860
|
if (touched) {
|
|
732
861
|
msg.content = next;
|
|
733
862
|
changed = true;
|
|
@@ -753,7 +882,7 @@ export default function (pi: ExtensionAPI) {
|
|
|
753
882
|
pi.registerCommand("vision-handoff", {
|
|
754
883
|
description: HANDOFF_COMMAND_DESCRIPTION,
|
|
755
884
|
getArgumentCompletions(prefix: string) {
|
|
756
|
-
const subcommands = ["select", "model", "status", "enable", "disable", "auto", "thinking", "prewarm", "fallback", "add", "remove", "clear", "help"];
|
|
885
|
+
const subcommands = ["select", "model", "status", "enable", "disable", "auto", "thinking", "prewarm", "fallback", "aware", "persist", "add", "remove", "clear", "help"];
|
|
757
886
|
const matches = subcommands.filter((s) => s.startsWith(prefix));
|
|
758
887
|
return matches.length > 0 ? matches.map((s) => ({ value: s, label: s })) : null;
|
|
759
888
|
},
|
|
@@ -791,6 +920,10 @@ async function handleHandoffCommand(ctx: ExtensionCommandContext, args: string):
|
|
|
791
920
|
" Toggle describing pasted images at paste-time (opt-in, off by default)",
|
|
792
921
|
" /vision-handoff fallback <on|off>",
|
|
793
922
|
" Inject pasted-image descriptions asynchronously when no matching read wins",
|
|
923
|
+
" /vision-handoff aware <on|off>",
|
|
924
|
+
" Tell handoff targets they can read images via the read tool (opt-in, off by default)",
|
|
925
|
+
" /vision-handoff persist <on|off>",
|
|
926
|
+
" Persist descriptions to the session file (resume reuses them, no new vision call)",
|
|
794
927
|
" /vision-handoff add <p/id> Force handoff for an extra model",
|
|
795
928
|
" /vision-handoff remove <p/id> Stop forcing handoff for a model",
|
|
796
929
|
" /vision-handoff clear Clear the configured vision model",
|
|
@@ -853,6 +986,16 @@ async function handleHandoffCommand(ctx: ExtensionCommandContext, args: string):
|
|
|
853
986
|
return;
|
|
854
987
|
}
|
|
855
988
|
|
|
989
|
+
if (subcommand === "aware") {
|
|
990
|
+
handleAwareSubcommand(ctx, rest);
|
|
991
|
+
return;
|
|
992
|
+
}
|
|
993
|
+
|
|
994
|
+
if (subcommand === "persist") {
|
|
995
|
+
handlePersistSubcommand(ctx, rest);
|
|
996
|
+
return;
|
|
997
|
+
}
|
|
998
|
+
|
|
856
999
|
if (subcommand === "clear") {
|
|
857
1000
|
updateConfig(
|
|
858
1001
|
ctx,
|
|
@@ -1028,6 +1171,60 @@ function handleFallbackSubcommand(ctx: ExtensionCommandContext, rest: string): v
|
|
|
1028
1171
|
);
|
|
1029
1172
|
}
|
|
1030
1173
|
|
|
1174
|
+
/** Handle /vision-handoff aware <on|off>. */
|
|
1175
|
+
function handleAwareSubcommand(ctx: ExtensionCommandContext, rest: string): void {
|
|
1176
|
+
const value = rest.trim().toLowerCase();
|
|
1177
|
+
if (!value) {
|
|
1178
|
+
ctx.ui.notify(
|
|
1179
|
+
`Aware prompt: ${config.awarePrompt ? "on" : "off"}.\n` +
|
|
1180
|
+
"Usage: /vision-handoff aware <on|off>",
|
|
1181
|
+
"info",
|
|
1182
|
+
);
|
|
1183
|
+
return;
|
|
1184
|
+
}
|
|
1185
|
+
if (value !== "on" && value !== "off") {
|
|
1186
|
+
ctx.ui.notify("Usage: /vision-handoff aware <on|off>", "warning");
|
|
1187
|
+
return;
|
|
1188
|
+
}
|
|
1189
|
+
const on = value === "on";
|
|
1190
|
+
updateConfig(
|
|
1191
|
+
ctx,
|
|
1192
|
+
(c) => ({ ...c, awarePrompt: on }),
|
|
1193
|
+
on
|
|
1194
|
+
? "Aware prompt on — handoff targets are told they can read images via the read tool."
|
|
1195
|
+
: "Aware prompt off.",
|
|
1196
|
+
);
|
|
1197
|
+
}
|
|
1198
|
+
|
|
1199
|
+
/** Handle /vision-handoff persist <on|off> — toggle persisting descriptions
|
|
1200
|
+
* to the session file. When on, generated descriptions are written as
|
|
1201
|
+
* `[Image described: <hash>]` marker + text blocks in the read tool-result
|
|
1202
|
+
* content; on `/resume` the `context` hook reuses them by hash, skipping the
|
|
1203
|
+
* vision call entirely (no 30-40s re-describe latency). */
|
|
1204
|
+
function handlePersistSubcommand(ctx: ExtensionCommandContext, rest: string): void {
|
|
1205
|
+
const value = rest.trim().toLowerCase();
|
|
1206
|
+
if (!value) {
|
|
1207
|
+
ctx.ui.notify(
|
|
1208
|
+
`Persist descriptions to the session file: ${config.persistDescriptions ? "on" : "off"}.\n` +
|
|
1209
|
+
"Usage: /vision-handoff persist <on|off>",
|
|
1210
|
+
"info",
|
|
1211
|
+
);
|
|
1212
|
+
return;
|
|
1213
|
+
}
|
|
1214
|
+
if (value !== "on" && value !== "off") {
|
|
1215
|
+
ctx.ui.notify("Usage: /vision-handoff persist <on|off>", "warning");
|
|
1216
|
+
return;
|
|
1217
|
+
}
|
|
1218
|
+
const on = value === "on";
|
|
1219
|
+
updateConfig(
|
|
1220
|
+
ctx,
|
|
1221
|
+
(c) => ({ ...c, persistDescriptions: on }),
|
|
1222
|
+
on
|
|
1223
|
+
? "Persist descriptions on — generated descriptions are saved to the session file; resumed sessions reuse them without a new vision call."
|
|
1224
|
+
: "Persist descriptions off — descriptions are generated per-session as before.",
|
|
1225
|
+
);
|
|
1226
|
+
}
|
|
1227
|
+
|
|
1031
1228
|
async function showSelector(ctx: ExtensionCommandContext): Promise<void> {
|
|
1032
1229
|
if (!ctx.hasUI) {
|
|
1033
1230
|
ctx.ui.notify("/vision-handoff requires interactive mode.", "error");
|
|
@@ -1122,6 +1319,8 @@ function showStatus(ctx: ExtensionCommandContext): void {
|
|
|
1122
1319
|
lines.push(`Thinking: ${config.thinking ? `on (${config.thinkingLevel})` : "off"}`);
|
|
1123
1320
|
lines.push(`Paste-time prewarm: ${config.prewarmPastedImages ? `on${editorInstalled ? "" : " (inactive — another custom editor is active)"}` : "off"}`);
|
|
1124
1321
|
lines.push(`Async pasted-path fallback: ${config.asyncClipboardHandoff ? "on" : "off"}`);
|
|
1322
|
+
lines.push(`Aware prompt: ${config.awarePrompt ? "on" : "off"}`);
|
|
1323
|
+
lines.push(`Persist descriptions (session file): ${config.persistDescriptions ? "on" : "off"}`);
|
|
1125
1324
|
lines.push(`maxTokens: ${config.maxTokens ?? "unbounded"} · cacheMax: ${config.cacheMax} · maxDescriptionLines: ${config.maxDescriptionLines === 0 ? "unbounded" : config.maxDescriptionLines}`);
|
|
1126
1325
|
|
|
1127
1326
|
const model = ctx.model;
|