pi-vision-handoff 0.4.3 โ 0.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +26 -1
- package/package.json +1 -1
- package/src/image.ts +17 -0
- package/src/index.ts +14 -0
- package/src/prewarm-editor.ts +95 -0
- package/vision-handoff.ts +102 -2
package/README.md
CHANGED
|
@@ -43,6 +43,7 @@ No `settings.json` touched. No per-provider glue. Pick a describer once and ever
|
|
|
43
43
|
- **๐ง Automatic targets** โ by default, handoff applies to *every* model that lacks native vision. Opt out with `/vision-handoff auto off`.
|
|
44
44
|
- **๐๏ธ Explicit overrides** โ force handoff for specific models (e.g. a weak vision model) with `/vision-handoff add`.
|
|
45
45
|
- **โก Pre-warmed at paste-enter** โ the moment you press enter, `before_agent_start` scans the prompt for pasted clipboard image temp-file paths (pi writes pasted images to `<tmpdir>/pi-clipboard-<uuid>.<ext>` and inserts the path as text), reads them, and kicks off the ONE batched vision call concurrent with the agent's first response โ so by the time the agent reads the files, they're already cache hits.
|
|
46
|
+
- **๐ Paste-time prewarm (opt-in)** โ `prewarmPastedImages` (off by default) wraps the editor so pasted clipboard images are described the *instant* their path lands in the prompt โ before you hit enter โ not at submit. The vision call starts concurrent with you typing your question. Tradeoff: the description is generated without your typed question as context (the question usually isn't entered yet at paste time), and a paste-then-abandon wastes one vision call. TUI only; inactive if another extension replaces the editor. Toggle with `/vision-handoff prewarm on`.
|
|
46
47
|
- **๐ก๏ธ Graceful degradation** โ no API key? Describer unreachable? Aborted? The image is replaced with a clean `[Image: description unavailable]` placeholder instead of crashing your turn.
|
|
47
48
|
- **๐ Usage reporting** โ every real describer call reports model + tokens (and Neuralwatt energy/cost when the vision model is a Neuralwatt model), via `pi.appendEntry` + a `vision-handoff:usage` event for live consumers.
|
|
48
49
|
- **๐ง Tunable** โ cap description length (`maxDescriptionLines`; unbounded by default, so `read`'s native `ctrl+o` collapse handles compactness) and cache size, in the config file.
|
|
@@ -59,6 +60,7 @@ No `settings.json` touched. No per-provider glue. Pick a describer once and ever
|
|
|
59
60
|
| `/vision-handoff status` | Show current config + whether handoff is active for the current model |
|
|
60
61
|
| `/vision-handoff enable` / `disable` | Master switch (keeps your configured model) |
|
|
61
62
|
| `/vision-handoff auto on` / `off` | Toggle automatic handoff for all non-vision models |
|
|
63
|
+
| `/vision-handoff prewarm on` / `off` | Toggle paste-time prewarm (opt-in, off by default) |
|
|
62
64
|
| `/vision-handoff add ollama/llava:13b` | Force handoff for an extra model |
|
|
63
65
|
| `/vision-handoff remove ollama/llava:13b` | Stop forcing handoff for a model |
|
|
64
66
|
| `/vision-handoff clear` | Clear the configured vision model |
|
|
@@ -74,6 +76,7 @@ Created automatically at `~/.pi/agent/extensions/pi-vision-handoff.json` on firs
|
|
|
74
76
|
"visionModel": "openai/gpt-4o",
|
|
75
77
|
"autoHandoff": true,
|
|
76
78
|
"handoffModels": ["ollama/llava:13b"],
|
|
79
|
+
"prewarmPastedImages": false,
|
|
77
80
|
"maxTokens": null,
|
|
78
81
|
"cacheMax": 50,
|
|
79
82
|
"maxDescriptionLines": 0,
|
|
@@ -88,6 +91,7 @@ Created automatically at `~/.pi/agent/extensions/pi-vision-handoff.json` on firs
|
|
|
88
91
|
| `visionModel` | `null` | The describer, as `provider/id`. `null` = not configured (handoff inactive). |
|
|
89
92
|
| `autoHandoff` | `true` | Apply handoff to every model whose `input` does not include `image`. |
|
|
90
93
|
| `handoffModels` | `[]` | Extra `provider/id` refs that should also receive handoff. |
|
|
94
|
+
| `prewarmPastedImages` | `false` | **Opt-in paste-time prewarm.** When `true`, a custom editor wrapper describes pasted clipboard images the instant their temp-file path lands in the prompt (pre-submit), instead of at submit. Trades a bit of description quality (the description is generated without your typed question as context, since the question usually isn't entered yet at paste time) and a speculative vision call on paste-then-abandon, for earlier prewarm. TUI mode only; inactive if another extension has replaced the editor. |
|
|
91
95
|
| `maxTokens` | _(unset = model max, clamped to context window)_ | Cap on a single description's output. `null`/unset = use the vision model's declared max output (`model.maxTokens`), clamped so `maxTokens + 8192 <= contextWindow` (a model whose declared max equals its full context window would otherwise be rejected with a 400). Set a number only to cap cost/latency. A truncation is always surfaced via a `[... description truncated โฆ]` marker when the model hits the limit, so a cut-off description is never mistaken for complete. |
|
|
92
96
|
| `cacheMax` | `50` | Max described images kept in the in-memory cache per session. |
|
|
93
97
|
| `maxDescriptionLines` | `0` | Cap on description lines (`0` = unbounded). Default keeps the full description so the `read` tool's native collapse (`ctrl+o`) handles compactness and the model gets complete context; setting `> 0` applies a lossy head-cap to both the TUI render and the model. |
|
|
@@ -142,6 +146,26 @@ memoizes promises; all `load()` calls in the same execution frame coalesce
|
|
|
142
146
|
into ONE batched vision call, dispatched via `setImmediate` after the poll
|
|
143
147
|
phase (so reads completing together batch instead of splitting into N calls).
|
|
144
148
|
|
|
149
|
+
With `prewarmPastedImages` on (opt-in), an even earlier stage runs:
|
|
150
|
+
|
|
151
|
+
โ editor onChange (paste-time, pre-submit)
|
|
152
|
+
โข pi has no "image pasted" event; the earliest signal is the editor's
|
|
153
|
+
`onChange(text)`, which fires when pi's `handleClipboardImagePaste`
|
|
154
|
+
inserts the temp-file path via `insertTextAtCursor`
|
|
155
|
+
โข a wrapping CustomEditor chains `onChange` (pi assigns its own
|
|
156
|
+
`onChange` โ for bash-mode border tracking โ after construction, so
|
|
157
|
+
the wrapper captures it via an accessor and runs it alongside an
|
|
158
|
+
observer that `diffPrewarmPaths`s new `pi-clipboard-*` paths and
|
|
159
|
+
`loadDescription()`s them at paste-time โ the vision call starts
|
|
160
|
+
before you hit enter; `before_agent_start`'s same-path prewarm is
|
|
161
|
+
then a cache hit
|
|
162
|
+
โข it does NOT override `handleInput`, so keybindings + `ctrl+v` paste
|
|
163
|
+
are untouched; TUI only, and skipped when another extension has
|
|
164
|
+
replaced the editor (installing over one would break paste, since
|
|
165
|
+
pi wires paste-image to the outermost editor only)
|
|
166
|
+
|
|
167
|
+
The default submit-time pipeline below runs regardless:
|
|
168
|
+
|
|
145
169
|
โ before_agent_start
|
|
146
170
|
โข captures this turn's user prompt (shared by every image description)
|
|
147
171
|
โข binds turn context (vision model + abort signal) to the loader
|
|
@@ -346,6 +370,7 @@ pnpm lint:dead # Dead code detection (knip)
|
|
|
346
370
|
โ โโโ dataloader.ts # DescriptionLoader โ DataLoader-batched descriptions (Disposable)
|
|
347
371
|
โ โโโ describer.ts # Vision describer calls (runBatch / describeSingle) with `using` resource guards
|
|
348
372
|
โ โโโ image.ts # Image hashing, MIME sniffing, clipboard-path reading
|
|
373
|
+
โ โโโ prewarm-editor.ts # Opt-in paste-time prewarm CustomEditor wrapper (chains onChange)
|
|
349
374
|
โ โโโ dispose.ts # `Disposable` guard factories for `using` (fetch interceptor, timer, abort wire)
|
|
350
375
|
โ โโโ usage.ts # Describer usage + Neuralwatt energy capture, fetch interceptor
|
|
351
376
|
โ โโโ vision-model-selector.ts # Interactive picker TUI component
|
|
@@ -355,7 +380,7 @@ pnpm lint:dead # Dead code detection (knip)
|
|
|
355
380
|
โ โโโ vision-handoff.test.ts # Config, refs, image-block extraction, insertion, truncation, round-trip
|
|
356
381
|
โ โโโ dataloader.test.ts # Batch coalescing, memoization, failure eviction, Disposable reset
|
|
357
382
|
โ โโโ describer.test.ts # stopReason handling (length โ truncation marker, aborted/error)
|
|
358
|
-
โ โโโ image.test.ts # MIME sniffing, clipboard-path confinement, file reading
|
|
383
|
+
โ โโโ image.test.ts # MIME sniffing, clipboard-path confinement, file reading, diffPrewarmPaths
|
|
359
384
|
โ โโโ dispose.test.ts # `using` guards: fetch refcount, timeout, abort-wire propagation
|
|
360
385
|
โโโ package.json
|
|
361
386
|
โโโ tsconfig.json
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "pi-vision-handoff",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.5.0",
|
|
4
4
|
"description": "Give text-only pi models vision โ describe images with a vision model you pick via an interactive picker, then hand off the text description to non-vision models",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"author": "Tom X Nguyen",
|
package/src/image.ts
CHANGED
|
@@ -272,3 +272,20 @@ export function findClipboardImagePaths(prompt: string): string[] {
|
|
|
272
272
|
}
|
|
273
273
|
return [...paths];
|
|
274
274
|
}
|
|
275
|
+
|
|
276
|
+
/** Diff the clipboard image paths in `text` against `known`, returning those
|
|
277
|
+
* that are newly appeared (not yet seen). Used by the paste-time prewarm
|
|
278
|
+
* editor to prewarm only newly-pasted paths on each text change, without
|
|
279
|
+
* re-reading already-seen ones. `findClipboardImagePaths` already dedups
|
|
280
|
+
* within a single text, so each returned path is unique and seen at most
|
|
281
|
+
* once. Returns [] when `text` holds no clipboard paths (e.g. ordinary
|
|
282
|
+
* typing) โ so the editor's per-keystroke cost when the opt-in is on is one
|
|
283
|
+
* regex scan that almost always yields nothing. */
|
|
284
|
+
export function diffPrewarmPaths(text: string, known: Set<string>): string[] {
|
|
285
|
+
const paths = findClipboardImagePaths(text);
|
|
286
|
+
const newPaths: string[] = [];
|
|
287
|
+
for (const p of paths) {
|
|
288
|
+
if (!known.has(p)) newPaths.push(p);
|
|
289
|
+
}
|
|
290
|
+
return newPaths;
|
|
291
|
+
}
|
package/src/index.ts
CHANGED
|
@@ -136,6 +136,18 @@ export interface VisionHandoffConfig {
|
|
|
136
136
|
autoHandoff: boolean;
|
|
137
137
|
/** Extra "provider/id" refs that should ALSO receive handoff (e.g. weak vision models). */
|
|
138
138
|
handoffModels: string[];
|
|
139
|
+
/** Opt-in: when true, the editor is wrapped at session start to describe
|
|
140
|
+
* pasted clipboard images the instant their temp-file path lands in the
|
|
141
|
+
* prompt (pre-submit), instead of waiting for submit. Trades a bit of
|
|
142
|
+
* description quality (the description is generated without the user's
|
|
143
|
+
* typed question as context, since the question usually isn't entered yet
|
|
144
|
+
* at paste time) and a speculative vision call on paste-then-abandon, for
|
|
145
|
+
* earlier prewarm. Off by default โ submit-time prewarm
|
|
146
|
+
* (before_agent_start) still covers this case when off. TUI mode only, and
|
|
147
|
+
* inactive when another extension has replaced the editor (installing over
|
|
148
|
+
* a custom editor would break clipboard paste, since pi wires paste-image
|
|
149
|
+
* to the outermost editor only). */
|
|
150
|
+
prewarmPastedImages: boolean;
|
|
139
151
|
/** Max output tokens for a single description. `undefined` (default) = use the
|
|
140
152
|
* vision model's declared max output (`model.maxTokens`) as the cap โ the
|
|
141
153
|
* highest the model supports โ so the "be exhaustive" prompt isn't defeated
|
|
@@ -172,6 +184,7 @@ export const DEFAULT_CONFIG: VisionHandoffConfig = {
|
|
|
172
184
|
visionModel: null,
|
|
173
185
|
autoHandoff: true,
|
|
174
186
|
handoffModels: [],
|
|
187
|
+
prewarmPastedImages: false,
|
|
175
188
|
maxTokens: undefined,
|
|
176
189
|
cacheMax: DEFAULT_CACHE_MAX,
|
|
177
190
|
maxDescriptionLines: DEFAULT_MAX_DESCRIPTION_LINES,
|
|
@@ -221,6 +234,7 @@ export function normalizeConfig(raw: unknown): VisionHandoffConfig {
|
|
|
221
234
|
.map((m) => m.trim())
|
|
222
235
|
.filter((m) => m && parseModelRef(m));
|
|
223
236
|
}
|
|
237
|
+
if (typeof obj.prewarmPastedImages === "boolean") base.prewarmPastedImages = obj.prewarmPastedImages;
|
|
224
238
|
// maxTokens: optional. undefined (default) = no artificial cap. Only set when
|
|
225
239
|
// a valid positive finite number is given; any other value leaves it unset.
|
|
226
240
|
if (typeof obj.maxTokens === "number" && Number.isFinite(obj.maxTokens) && obj.maxTokens > 0) {
|
|
@@ -0,0 +1,95 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Paste-time prewarm editor wrapper.
|
|
3
|
+
*
|
|
4
|
+
* pi has no "image pasted into the prompt" event โ the extension event bus
|
|
5
|
+
* only surfaces input at submit time (`input` / `before_agent_start`). The
|
|
6
|
+
* earliest observable signal that a clipboard image has landed in the prompt
|
|
7
|
+
* is the editor's own `onChange(text)`, which fires when pi's
|
|
8
|
+
* `handleClipboardImagePaste` inserts the temp-file path
|
|
9
|
+
* (`<tmpdir>/pi-clipboard-<uuid>.<ext>`) via `insertTextAtCursor` โ still
|
|
10
|
+
* pre-submit. This wrapper installs a `CustomEditor` that observes `onChange`
|
|
11
|
+
* to prewarm the describer for newly-pasted clipboard image paths the instant
|
|
12
|
+
* they appear, so the vision call starts before the user hits enter (true
|
|
13
|
+
* paste-time) instead of at `before_agent_start` (submit-time).
|
|
14
|
+
*
|
|
15
|
+
* It does NOT override `handleInput`, so every keybinding and pi's clipboard
|
|
16
|
+
* paste flow (`ctrl+v` โ `onPasteImage` โ `handleClipboardImagePaste`) is
|
|
17
|
+
* untouched. We only observe the result via `onChange`.
|
|
18
|
+
*
|
|
19
|
+
* Why chain `onChange` via an accessor: pi's `setCustomEditorComponent` does
|
|
20
|
+
* `newEditor.onChange = this.defaultEditor.onChange` after construction,
|
|
21
|
+
* clobbering any `onChange` set in the constructor โ and that default
|
|
22
|
+
* `onChange` tracks bash-mode for the editor border, so it must keep running.
|
|
23
|
+
* The accessor captures pi's assignment and runs it alongside our observer.
|
|
24
|
+
* (The `Editor` base never reassigns `onChange` itself, so the captured
|
|
25
|
+
* reference is stable for the editor's lifetime.)
|
|
26
|
+
*
|
|
27
|
+
* Opt-in via `VisionHandoffConfig.prewarmPastedImages`. When off (or the
|
|
28
|
+
* active model isn't a handoff target), the observer short-circuits before
|
|
29
|
+
* the path regex runs, so overhead is one boolean check per text change.
|
|
30
|
+
*/
|
|
31
|
+
|
|
32
|
+
import { CustomEditor, type KeybindingsManager, type ModelRegistry } from "@earendil-works/pi-coding-agent";
|
|
33
|
+
import type { EditorTheme, TUI } from "@earendil-works/pi-tui";
|
|
34
|
+
import { diffPrewarmPaths } from "./image.js";
|
|
35
|
+
|
|
36
|
+
/** Dependencies the editor can't own itself (held by the engine, captured at
|
|
37
|
+
* session start). `modelRegistry` is fixed for the editor's (session)
|
|
38
|
+
* lifetime; `shouldPrewarm` and `prewarmPath` read live config/model state so
|
|
39
|
+
* toggling the opt-in or switching models takes effect without reinstalling. */
|
|
40
|
+
export interface PrewarmEditorDeps {
|
|
41
|
+
modelRegistry: ModelRegistry;
|
|
42
|
+
/** Live gate: paste-time prewarm runs only when this returns true. */
|
|
43
|
+
shouldPrewarm: () => boolean;
|
|
44
|
+
/** Prewarm one clipboard image path (read + resize-match + loadDescription). */
|
|
45
|
+
prewarmPath: (path: string, modelRegistry: ModelRegistry) => void;
|
|
46
|
+
}
|
|
47
|
+
|
|
48
|
+
/** A `CustomEditor` that observes `onChange` to prewarm pasted clipboard image
|
|
49
|
+
* paths at paste-time. See the module doc for the `onChange` chaining. */
|
|
50
|
+
export class PrewarmEditor extends CustomEditor {
|
|
51
|
+
private readonly knownPaths = new Set<string>();
|
|
52
|
+
private piOnChange: ((text: string) => void) | undefined;
|
|
53
|
+
private readonly shouldPrewarm: () => boolean;
|
|
54
|
+
private readonly prewarmPath: (path: string, modelRegistry: ModelRegistry) => void;
|
|
55
|
+
private readonly modelRegistry: ModelRegistry;
|
|
56
|
+
|
|
57
|
+
constructor(tui: TUI, theme: EditorTheme, keybindings: KeybindingsManager, deps: PrewarmEditorDeps) {
|
|
58
|
+
super(tui, theme, keybindings);
|
|
59
|
+
this.shouldPrewarm = deps.shouldPrewarm;
|
|
60
|
+
this.prewarmPath = deps.prewarmPath;
|
|
61
|
+
this.modelRegistry = deps.modelRegistry;
|
|
62
|
+
|
|
63
|
+
// Stable wrapper: calls pi's onChange (bash-mode border tracking) then our
|
|
64
|
+
// observer. Returned on every read so the Editor's `this.onChange(...)` calls
|
|
65
|
+
// all hit the same function.
|
|
66
|
+
const wrapper = (text: string): void => {
|
|
67
|
+
this.piOnChange?.(text);
|
|
68
|
+
this.handleTextChange(text);
|
|
69
|
+
};
|
|
70
|
+
// pi assigns defaultEditor.onChange to this property after construction;
|
|
71
|
+
// capture it via the setter, expose the wrapper via the getter.
|
|
72
|
+
Object.defineProperty(this, "onChange", {
|
|
73
|
+
configurable: true,
|
|
74
|
+
enumerable: true,
|
|
75
|
+
get: () => wrapper,
|
|
76
|
+
set: (fn: ((text: string) => void) | undefined) => {
|
|
77
|
+
this.piOnChange = fn;
|
|
78
|
+
},
|
|
79
|
+
});
|
|
80
|
+
}
|
|
81
|
+
|
|
82
|
+
/** Scan the editor text for clipboard image paths not yet seen this prompt
|
|
83
|
+
* and prewarm each. Clears the known set when the editor empties (after
|
|
84
|
+
* submit) so a later paste (fresh uuid) re-prewarms and the set can't grow
|
|
85
|
+
* unbounded. */
|
|
86
|
+
private handleTextChange(text: string): void {
|
|
87
|
+
if (!this.shouldPrewarm()) return;
|
|
88
|
+
const newPaths = diffPrewarmPaths(text, this.knownPaths);
|
|
89
|
+
for (const p of newPaths) {
|
|
90
|
+
this.knownPaths.add(p);
|
|
91
|
+
this.prewarmPath(p, this.modelRegistry);
|
|
92
|
+
}
|
|
93
|
+
if (text.length === 0) this.knownPaths.clear();
|
|
94
|
+
}
|
|
95
|
+
}
|
package/vision-handoff.ts
CHANGED
|
@@ -53,6 +53,7 @@ import { DescriptionLoader, UNAVAILABLE, type LoaderDeps } from "./src/dataloade
|
|
|
53
53
|
import { imageHash, findClipboardImagePaths, readImageBuffer, resolvePrewarmImage } from "./src/image.js";
|
|
54
54
|
import { resizeImage } from "@earendil-works/pi-coding-agent";
|
|
55
55
|
import { VisionModelSelectorComponent, type VisionModelSelectorResult } from "./src/vision-model-selector.js";
|
|
56
|
+
import { PrewarmEditor } from "./src/prewarm-editor.js";
|
|
56
57
|
|
|
57
58
|
let config: VisionHandoffConfig = readConfig();
|
|
58
59
|
|
|
@@ -70,6 +71,15 @@ let lastDescriberError: string | null = null;
|
|
|
70
71
|
* Cleared on `session_start`. */
|
|
71
72
|
const warnedHashes = new Set<string>();
|
|
72
73
|
|
|
74
|
+
/** Current model, tracked so the paste-time prewarm gate can skip prewarming
|
|
75
|
+
* when the active model is vision-capable (handoff won't run โ a prewarm
|
|
76
|
+
* would be a wasted vision call). Updated in session_start and model_select. */
|
|
77
|
+
let currentModel: Model<Api> | undefined | null;
|
|
78
|
+
|
|
79
|
+
/** Whether the paste-time prewarm editor is installed this session. False in
|
|
80
|
+
* non-TUI modes or when another extension already replaced the editor. */
|
|
81
|
+
let editorInstalled = false;
|
|
82
|
+
|
|
73
83
|
let visionModelCache: { ref: string; model: Model<Api> } | null = null;
|
|
74
84
|
let visionModelUnresolvedRef: string | null = null;
|
|
75
85
|
|
|
@@ -112,6 +122,60 @@ function isHandoffTarget(
|
|
|
112
122
|
return false;
|
|
113
123
|
}
|
|
114
124
|
|
|
125
|
+
/** Whether paste-time prewarm should fire for a text change right now: the
|
|
126
|
+
* opt-in flag is on, handoff is configured, and the active model is a handoff
|
|
127
|
+
* target (so the prewarmed description will actually be consumed โ a
|
|
128
|
+
* vision-capable model needs no handoff, so prewarming would waste a call). */
|
|
129
|
+
function shouldPrewarmPaste(): boolean {
|
|
130
|
+
return config.prewarmPastedImages && isConfigured(config) && isHandoffTarget(currentModel, config);
|
|
131
|
+
}
|
|
132
|
+
|
|
133
|
+
/** Prewarm one pasted clipboard image path through the dataloader at paste
|
|
134
|
+
* time. Mirrors the before_agent_start clipboard prewarm, but binds only the
|
|
135
|
+
* model registry (no turn signal โ the agent is idle at paste time) and
|
|
136
|
+
* resets the turn prompt to "" so a stale previous-turn prompt can't leak
|
|
137
|
+
* into the description. The user's question isn't typed yet at paste time, so
|
|
138
|
+
* the description is generated without question context (the documented
|
|
139
|
+
* tradeoff of the opt-in). If the user submits before this dispatch fires,
|
|
140
|
+
* before_agent_start overwrites the prompt โ a benign bonus, not a bug. */
|
|
141
|
+
function prewarmClipboardPath(path: string, modelRegistry: ModelRegistry): void {
|
|
142
|
+
const read = readImageBuffer(path);
|
|
143
|
+
if (!read) return;
|
|
144
|
+
resolvePrewarmImage(read.buf, read.mimeType, resizeImage)
|
|
145
|
+
.then((img) => {
|
|
146
|
+
if (!img) return;
|
|
147
|
+
loader.setPendingTurnPrompt("");
|
|
148
|
+
loader.bindTurnContext({ modelRegistry });
|
|
149
|
+
loader.loadDescription(img).catch(() => {});
|
|
150
|
+
})
|
|
151
|
+
.catch(() => {});
|
|
152
|
+
}
|
|
153
|
+
|
|
154
|
+
/** Install the paste-time prewarm editor wrapper for this session. TUI only,
|
|
155
|
+
* and only when no other extension has replaced the editor โ installing over
|
|
156
|
+
* a custom editor would clobber its input handling and break clipboard paste
|
|
157
|
+
* (pi wires paste-image to the outermost editor only). When a custom editor
|
|
158
|
+
* is present, paste-time prewarm is unavailable; submit-time prewarm
|
|
159
|
+
* (before_agent_start) still covers clipboard paths. */
|
|
160
|
+
function installPrewarmEditor(ctx: ExtensionContext): void {
|
|
161
|
+
if (ctx.mode !== "tui") {
|
|
162
|
+
editorInstalled = false;
|
|
163
|
+
return;
|
|
164
|
+
}
|
|
165
|
+
if (ctx.ui.getEditorComponent()) {
|
|
166
|
+
editorInstalled = false;
|
|
167
|
+
return;
|
|
168
|
+
}
|
|
169
|
+
editorInstalled = true;
|
|
170
|
+
ctx.ui.setEditorComponent((_tui, theme, keybindings) =>
|
|
171
|
+
new PrewarmEditor(_tui, theme, keybindings, {
|
|
172
|
+
modelRegistry: ctx.modelRegistry,
|
|
173
|
+
shouldPrewarm: shouldPrewarmPaste,
|
|
174
|
+
prewarmPath: prewarmClipboardPath,
|
|
175
|
+
}),
|
|
176
|
+
);
|
|
177
|
+
}
|
|
178
|
+
|
|
115
179
|
function notifyUnresolvedVisionModel(ctx: ExtensionContext, ref: string): void {
|
|
116
180
|
if (visionModelUnresolvedRef === ref) return;
|
|
117
181
|
visionModelUnresolvedRef = ref;
|
|
@@ -169,13 +233,15 @@ export default function (pi: ExtensionAPI) {
|
|
|
169
233
|
}
|
|
170
234
|
};
|
|
171
235
|
|
|
172
|
-
pi.on("session_start", async () => {
|
|
236
|
+
pi.on("session_start", async (_event, ctx) => {
|
|
173
237
|
// Reload in case the user edited the config on disk from another session.
|
|
174
238
|
config = readConfig();
|
|
175
239
|
visionModelCache = null;
|
|
176
240
|
visionModelUnresolvedRef = null;
|
|
177
241
|
warnedHashes.clear();
|
|
178
242
|
loader.reset();
|
|
243
|
+
currentModel = ctx.model;
|
|
244
|
+
installPrewarmEditor(ctx);
|
|
179
245
|
});
|
|
180
246
|
|
|
181
247
|
pi.on("before_agent_start", async (event, ctx) => {
|
|
@@ -405,6 +471,7 @@ export default function (pi: ExtensionAPI) {
|
|
|
405
471
|
});
|
|
406
472
|
|
|
407
473
|
pi.on("model_select", (event, ctx) => {
|
|
474
|
+
currentModel = event.model;
|
|
408
475
|
if (!ctx.hasUI) return;
|
|
409
476
|
if (!isConfigured(config)) return;
|
|
410
477
|
const model = event.model;
|
|
@@ -420,7 +487,7 @@ export default function (pi: ExtensionAPI) {
|
|
|
420
487
|
pi.registerCommand("vision-handoff", {
|
|
421
488
|
description: HANDOFF_COMMAND_DESCRIPTION,
|
|
422
489
|
getArgumentCompletions(prefix: string) {
|
|
423
|
-
const subcommands = ["select", "model", "status", "enable", "disable", "auto", "thinking", "add", "remove", "clear", "help"];
|
|
490
|
+
const subcommands = ["select", "model", "status", "enable", "disable", "auto", "thinking", "prewarm", "add", "remove", "clear", "help"];
|
|
424
491
|
const matches = subcommands.filter((s) => s.startsWith(prefix));
|
|
425
492
|
return matches.length > 0 ? matches.map((s) => ({ value: s, label: s })) : null;
|
|
426
493
|
},
|
|
@@ -454,6 +521,8 @@ async function handleHandoffCommand(ctx: ExtensionCommandContext, args: string):
|
|
|
454
521
|
" /vision-handoff auto <on|off> Toggle automatic handoff for all non-vision models",
|
|
455
522
|
" /vision-handoff thinking <off|minimal|low|medium|high|xhigh>",
|
|
456
523
|
" Set the vision describer's thinking effort (off = disabled)",
|
|
524
|
+
" /vision-handoff prewarm <on|off>",
|
|
525
|
+
" Toggle describing pasted images at paste-time (opt-in, off by default)",
|
|
457
526
|
" /vision-handoff add <p/id> Force handoff for an extra model",
|
|
458
527
|
" /vision-handoff remove <p/id> Stop forcing handoff for a model",
|
|
459
528
|
" /vision-handoff clear Clear the configured vision model",
|
|
@@ -463,6 +532,7 @@ async function handleHandoffCommand(ctx: ExtensionCommandContext, args: string):
|
|
|
463
532
|
"Mechanism: before_agent_start warms a description cache; tool_result loads",
|
|
464
533
|
" read images through a dataloader (one batched vision call); context swaps",
|
|
465
534
|
" image blocks in the payload for the cached text description.",
|
|
535
|
+
" prewarm on wraps the editor to describe pasted images at paste-time.",
|
|
466
536
|
].join("\n"),
|
|
467
537
|
"info",
|
|
468
538
|
);
|
|
@@ -504,6 +574,11 @@ async function handleHandoffCommand(ctx: ExtensionCommandContext, args: string):
|
|
|
504
574
|
return;
|
|
505
575
|
}
|
|
506
576
|
|
|
577
|
+
if (subcommand === "prewarm") {
|
|
578
|
+
handlePrewarmSubcommand(ctx, rest);
|
|
579
|
+
return;
|
|
580
|
+
}
|
|
581
|
+
|
|
507
582
|
if (subcommand === "clear") {
|
|
508
583
|
updateConfig(
|
|
509
584
|
ctx,
|
|
@@ -629,6 +704,30 @@ function handleThinkingSubcommand(ctx: ExtensionCommandContext, rest: string): v
|
|
|
629
704
|
);
|
|
630
705
|
}
|
|
631
706
|
|
|
707
|
+
/** Handle `/vision-handoff prewarm <on|off>` โ toggle paste-time prewarm. */
|
|
708
|
+
function handlePrewarmSubcommand(ctx: ExtensionCommandContext, rest: string): void {
|
|
709
|
+
const value = rest.trim().toLowerCase();
|
|
710
|
+
if (!value) {
|
|
711
|
+
ctx.ui.notify(
|
|
712
|
+
`Paste-time prewarm: ${config.prewarmPastedImages ? "on" : "off"}.\n` +
|
|
713
|
+
`Usage: /vision-handoff prewarm <on|off>`,
|
|
714
|
+
"info",
|
|
715
|
+
);
|
|
716
|
+
return;
|
|
717
|
+
}
|
|
718
|
+
if (value !== "on" && value !== "off") {
|
|
719
|
+
ctx.ui.notify("Usage: /vision-handoff prewarm <on|off>", "warning");
|
|
720
|
+
return;
|
|
721
|
+
}
|
|
722
|
+
const on = value === "on";
|
|
723
|
+
const note = on
|
|
724
|
+
? editorInstalled
|
|
725
|
+
? "Paste-time prewarm on โ pasted images are described the instant their path lands in the prompt (before submit)."
|
|
726
|
+
: "Paste-time prewarm on โ but another custom editor extension is active, so it's unavailable. Submit-time prewarm still works; disable the other editor extension (or /vision-handoff prewarm off) to silence this."
|
|
727
|
+
: "Paste-time prewarm off โ images are described at submit time (default).";
|
|
728
|
+
updateConfig(ctx, (c) => ({ ...c, prewarmPastedImages: on }), note);
|
|
729
|
+
}
|
|
730
|
+
|
|
632
731
|
async function showSelector(ctx: ExtensionCommandContext): Promise<void> {
|
|
633
732
|
if (!ctx.hasUI) {
|
|
634
733
|
ctx.ui.notify("/vision-handoff requires interactive mode.", "error");
|
|
@@ -692,6 +791,7 @@ function showStatus(ctx: ExtensionCommandContext): void {
|
|
|
692
791
|
lines.push(`Auto handoff (non-vision models): ${config.autoHandoff ? "on" : "off"}`);
|
|
693
792
|
lines.push(`Handoff targets (explicit): ${config.handoffModels.length ? config.handoffModels.join(", ") : "(none)"}`);
|
|
694
793
|
lines.push(`Thinking: ${config.thinking ? `on (${config.thinkingLevel})` : "off"}`);
|
|
794
|
+
lines.push(`Paste-time prewarm: ${config.prewarmPastedImages ? `on${editorInstalled ? "" : " (inactive โ another custom editor is active)"}` : "off"}`);
|
|
695
795
|
lines.push(`maxTokens: ${config.maxTokens ?? "unbounded"} ยท cacheMax: ${config.cacheMax} ยท maxDescriptionLines: ${config.maxDescriptionLines === 0 ? "unbounded" : config.maxDescriptionLines}`);
|
|
696
796
|
|
|
697
797
|
const model = ctx.model;
|