pi-vision-handoff 0.3.2 โ†’ 0.4.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/README.md CHANGED
@@ -10,6 +10,8 @@ _Describe images with a vision model you pick, then feed the text to models that
10
10
  [![npm](https://img.shields.io/npm/v/pi-vision-handoff)](https://www.npmjs.com/package/pi-vision-handoff)
11
11
  [![license](https://img.shields.io/badge/license-MIT-blue)](./LICENSE)
12
12
 
13
+ <img src="https://raw.githubusercontent.com/monotykamary/pi-vision-handoff/main/assets/vision-handoff.jpg" alt="Vision Handoff picker โ€” an interactive TUI listing every model, vision-capable ones marked with an eye, to choose the describer for text-only models" width="820">
14
+
13
15
  </div>
14
16
 
15
17
  ---
@@ -35,7 +37,7 @@ No `settings.json` touched. No per-provider glue. Pick a describer once and ever
35
37
  ## Features
36
38
 
37
39
  - **๐ŸŽฎ Interactive picker** โ€” `/vision-handoff` opens a TUI listing every model, vision-capable ones first (๐Ÿ‘), to choose your describer.
38
- - **๐Ÿ–ผ๏ธ DataLoader-batched descriptions** โ€” the `read` tools are `load()` callers: N parallel reads coalesce into ONE batched vision call (dispatched after the microtask cascade settles), awaited during the tool-result phase (free time) so the agent's next turn never blocks on the describer. Descriptions are ready before `context` fires, so the swap is a non-blocking cache hit.
40
+ - **๐Ÿ–ผ๏ธ DataLoader-batched descriptions** โ€” the `read` tools are `load()` callers: N parallel reads coalesce into ONE batched vision call (dispatched via `setImmediate` after the poll phase, so reads completing together batch instead of splitting), awaited during the tool-result phase (free time) so the agent's next turn never blocks on the describer. Descriptions are ready before `context` fires, so the swap is a non-blocking cache hit.
39
41
  - **๐Ÿงน Hides pi's "model does not support images" note** โ€” on read results the extension strips pi's `[Current model does not support imagesโ€ฆ]` note from the text block (it's misleading once the handoff delivers the image's content as text), while keeping the image block for kitty inline rendering and `/resume`.
40
42
  - **๐Ÿ”Œ Provider-agnostic** โ€” uses pi's own model execution machinery (`@earendil-works/pi-ai`'s `complete()`), so it works with any provider/configured model, including custom provider extensions.
41
43
  - **๐Ÿง  Automatic targets** โ€” by default, handoff applies to *every* model that lacks native vision. Opt out with `/vision-handoff auto off`.
@@ -137,7 +139,8 @@ pi -e ./vision-handoff.ts
137
139
  The extension implements the **Facebook DataLoader pattern** for image
138
140
  descriptions. The `read` tools are the `load()` callers; a per-image cache
139
141
  memoizes promises; all `load()` calls in the same execution frame coalesce
140
- into ONE batched vision call, dispatched after the microtask cascade settles.
142
+ into ONE batched vision call, dispatched via `setImmediate` after the poll
143
+ phase (so reads completing together batch instead of splitting into N calls).
141
144
 
142
145
  โ†’ before_agent_start
143
146
  โ€ข captures this turn's user prompt (shared by every image description)
@@ -160,10 +163,12 @@ into ONE batched vision call, dispatched after the microtask cascade settles.
160
163
  โ€ข pi runs `read` calls in parallel (Promise.all); each read's
161
164
  tool_result handler calls `loadDescription(img)` for its image
162
165
  blocks and AWAITS the shared batch
163
- โ€ข DataLoader: all `load()` calls in the same microtask frame land in
164
- ONE batch โ†’ ONE `complete()` vision call for the whole read set
165
- (dispatched after the microtask cascade via enqueuePostPromiseJob:
166
- `Promise.resolve().then(() => process.nextTick(dispatch))`)
166
+ โ€ข DataLoader: all `load()` calls in one event-loop poll iteration
167
+ land in ONE batch โ†’ ONE `complete()` vision call for the whole read
168
+ set. `enqueuePostPromiseJob` schedules dispatch via `setImmediate`
169
+ (the check phase, which runs AFTER the whole poll phase โ€” not
170
+ `process.nextTick`, which would drain between the reads' I/O
171
+ callbacks and split them into N single-image calls)
167
172
  โ€ข awaits the shared batch โ€” runs the describer during the tool-result
168
173
  phase (free time: the agent is just waiting for tool results), so
169
174
  the batch is COMPLETE before `context` fires โ†’ `context` is a
@@ -191,24 +196,35 @@ A single frame's image set is described with **one** vision-model call, not
191
196
  one per image. `loadDescription(img)` is synchronous: on a cache miss it
192
197
  pushes the image's key (hash) into the current batch and returns a memoized
193
198
  promise; on a cache hit it returns the existing (in-flight or resolved)
194
- promise. `enqueuePostPromiseJob` schedules dispatch after the microtask
195
- cascade settles, so every `load()` caller in the frame registers its key
196
- before the single vision call fires โ€” exactly DataLoader's `getCurrentBatch`
197
- + `enqueuePostPromiseJob`. The batched call sends every uncached image in a
198
- single user message with per-image `<<<IMAGE k>>> โ€ฆ <<<END>>>` delimiters; the
199
- response is parsed back into per-image descriptions (keyed by
200
- `sha256(mime + base64)` in the per-image cache). If the vision model ignores
201
- the delimiter format and the batched response can't be split, each unparsed
202
- image falls back to its own single-image `complete()` call **in parallel**
203
- (no delimiters to cooperate with) โ€” descriptions still arrive together. Only
204
- when a per-image call itself genuinely fails (auth, timeout, empty) does that
205
- image degrade to `[Image: description unavailable]`; one bad image never
206
- voids the rest.
207
-
208
- Because pi runs parallel `read` tool calls via `Promise.all` and fires the
209
- `tool_result` event inside that `Promise.all` (via `agent.afterToolCall` โ†’
210
- `finalizeExecutedToolCall`), N parallel reads' `load()` calls share ONE
211
- microtask frame โ†’ ONE batch โ†’ ONE vision call, all resolving together.
199
+ promise. `enqueuePostPromiseJob` schedules dispatch via `setImmediate` (the
200
+ check phase, after the whole poll phase AND after the microtask queue
201
+ drains), so every `load()` caller โ€” whether from sync code, a `.then`
202
+ cascade, or a separate I/O callback in the same poll iteration โ€” registers
203
+ its key before the single vision call fires. This is why N parallel reads
204
+ coalesce into one call rather than splitting into N single-image calls:
205
+ `setImmediate` defers past the entire poll phase, whereas DataLoader's
206
+ classic `process.nextTick` would drain between the reads' I/O callbacks and
207
+ fire a dispatch after the first read but before the second. The batched call
208
+ sends every uncached image in a single user message with per-image
209
+ `<<<IMAGE k>>> โ€ฆ <<<END>>>` delimiters; the response is parsed back into
210
+ per-image descriptions (keyed by `sha256(mime + base64)` in the per-image
211
+ cache). If the vision model ignores the delimiter format and the batched
212
+ response can't be split, each unparsed image falls back to its own
213
+ single-image `complete()` call **in parallel** (no delimiters to cooperate
214
+ with) โ€” descriptions still arrive together. Only when a per-image call
215
+ itself genuinely fails (auth, timeout, empty) does that image degrade to
216
+ `[Image: description unavailable]`; one bad image never voids the rest.
217
+
218
+ Because pi runs parallel `read` tool calls via `Promise.all` and fires
219
+ each read's `tool_result` event as that read's I/O completes (poll phase) โ€”
220
+ via `agent.afterToolCall` โ†’ `finalizeExecutedToolCall` โ€” the loader's
221
+ `setImmediate` dispatch defers to the check phase AFTER the whole poll
222
+ iteration, so reads completing together share ONE batch โ†’ ONE vision call,
223
+ all resolving together. Reads completing in separate poll iterations get
224
+ separate calls, but always in parallel, never sequential. (The per-image
225
+ cache also dedupes a duplicate `load()` of the same image in one frame:
226
+ dispatch resolves every callback by hash, so a second load whose first cache
227
+ entry was evicted mid-frame still resolves โ€” it never hangs.)
212
228
 
213
229
  **Failures are never cached.**
214
230
 
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "pi-vision-handoff",
3
- "version": "0.3.2",
3
+ "version": "0.4.1",
4
4
  "description": "Give text-only pi models vision โ€” describe images with a vision model you pick via an interactive picker, then hand off the text description to non-vision models",
5
5
  "type": "module",
6
6
  "author": "Tom X Nguyen",
package/src/dataloader.ts CHANGED
@@ -4,19 +4,23 @@
4
4
  * `loadDescription(img)` returns a memoized Promise for the description and
5
5
  * pushes the image's key (hash) into the CURRENT batch. All `load()` calls in
6
6
  * the same execution frame (+ its microtask cascade) coalesce into ONE batch
7
- * object. Dispatch is scheduled after the microtask queue settles (via
8
- * `enqueuePostPromiseJob`), so every load in the frame lands in the single
9
- * batch before the ONE vision call fires. Each load()'s promise then resolves
10
- * with its description.
7
+ * object. Dispatch is scheduled via `setImmediate` (see `enqueuePostPromiseJob`),
8
+ * so every load in the frame โ€” and every load from a separate I/O callback in
9
+ * the same poll iteration โ€” lands in the single batch before the ONE vision
10
+ * call fires. Each load()'s promise then resolves with its description.
11
11
  *
12
12
  * Mapping: the `read` tools are the `load()` callers. Their `tool_result`
13
- * handler awaits the shared batch โ€” so N parallel reads (pi runs `read` via
14
- * `Promise.all`) block on the SAME single vision call and all resolve together,
15
- * the descriptions landing in the tool results BEFORE the agent's next turn.
16
- * The agent's tool-result wait is free time, so this adds zero latency to the
17
- * critical path. `context` then sees text-described tool results (no image
18
- * blocks to swap); any remaining images (user-attached, custom-injected) are
19
- * cache hits.
13
+ * handler awaits the shared batch โ€” so N parallel reads coalesce into the SAME
14
+ * single vision call and all resolve together, the descriptions landing in the
15
+ * tool results BEFORE the agent's next turn. pi fires each read's `tool_result`
16
+ * as that read's I/O completes (poll phase); the loader's `setImmediate`
17
+ * dispatch defers to the check phase, AFTER the whole poll iteration, so reads
18
+ * completing together (the common case for cached local files) land in ONE
19
+ * batch โ€” and reads completing in separate iterations get separate calls, but
20
+ * always in parallel, never sequential. The agent's tool-result wait is free
21
+ * time, so this adds zero latency to the critical path. `context` then sees
22
+ * text-described tool results (no image blocks to swap); any remaining images
23
+ * (user-attached, custom-injected) are cache hits.
20
24
  *
21
25
  * All mutable state (batch, cache, turn context) lives on the instance โ€” no
22
26
  * module-level globals โ€” and the class implements `Disposable` so a `using`
@@ -43,7 +47,12 @@ export const UNAVAILABLE = `${IMAGE_PLACEHOLDER_PREFIX}description unavailable${
43
47
  interface DescriptionBatch {
44
48
  keys: string[];
45
49
  imgs: ExtractedImage[];
46
- callbacks: { resolve: (v: string) => void; reject: (e: Error) => void }[];
50
+ /** One per `loadDescription()` call. A duplicate load (same hash, but its
51
+ * first cache entry was evicted mid-frame so it couldn't short-circuit on
52
+ * the cache) pushes a second callback for an existing key โ€” so
53
+ * `callbacks.length` can exceed `keys.length`. `dispatchBatch` resolves by
54
+ * hash so every callback is reached. */
55
+ callbacks: { hash: string; resolve: (v: string) => void; reject: (e: Error) => void }[];
47
56
  }
48
57
 
49
58
  /** Engine-provided resolver for the configured vision model. */
@@ -59,16 +68,21 @@ export interface LoaderDeps extends DescriberDeps {
59
68
  resolveVisionModel: VisionModelResolver;
60
69
  }
61
70
 
62
- const resolvedMicrotask = Promise.resolve();
63
-
64
- /** Defer `fn` until after the current microtask cascade settles, so every
65
- * `loadDescription()` in the same frame coalesces into one batch before the
66
- * single vision call fires. Mirrors DataLoader's enqueuePostPromiseJob: a
67
- * Promise Job enqueues a global Job (process.nextTick), guaranteeing dispatch
68
- * runs after "PromiseJobs" ends โ€” after all `load()` callers in the frame
69
- * have registered their keys. */
71
+ /** Defer `fn` to the next check phase (`setImmediate`), so every
72
+ * `loadDescription()` โ€” whether called from sync code, a microtask cascade,
73
+ * or a separate I/O callback in the same poll iteration โ€” coalesces into one
74
+ * batch before the single vision call fires.
75
+ *
76
+ * Why `setImmediate` and not DataLoader's classic `process.nextTick`:
77
+ * nextTick drains between I/O callbacks in the poll phase, so dispatch would
78
+ * fire after the first parallel `read`'s `tool_result` but before the
79
+ * second's โ€” splitting N reads into N single-image calls. `setImmediate` runs
80
+ * in the check phase, AFTER the whole poll phase, so all `tool_result`
81
+ * handlers that fire in one poll iteration land in ONE batch. The check phase
82
+ * also runs after the microtask queue drains, so loads issued from a `.then`
83
+ * cascade (e.g. the clipboard pre-warm) still coalesce. */
70
84
  function enqueuePostPromiseJob(fn: () => void): void {
71
- resolvedMicrotask.then(() => process.nextTick(fn));
85
+ setImmediate(fn);
72
86
  }
73
87
 
74
88
  export class DescriptionLoader implements Disposable {
@@ -119,12 +133,14 @@ export class DescriptionLoader implements Disposable {
119
133
  batch.imgs.push(img);
120
134
  idx = batch.keys.length - 1;
121
135
  } else {
122
- // Same image loaded twice in the frame โ€” share one cache slot, but give
123
- // this caller its own promise.
136
+ // Same image loaded twice in the frame (the first load's cache entry was
137
+ // evicted mid-frame, else the second load would have short-circuited on
138
+ // the cache). Share the one key/image slot but give this caller its own
139
+ // promise โ€” dispatch resolves it by hash alongside the first caller's.
124
140
  batch.imgs[idx] = img;
125
141
  }
126
142
  const promise = new Promise<string>((resolve, reject) => {
127
- batch.callbacks.push({ resolve, reject });
143
+ batch.callbacks.push({ hash, resolve, reject });
128
144
  });
129
145
  const cfg = this.deps.getConfig();
130
146
  if (this.cache.size >= cfg.cacheMax) {
@@ -161,7 +177,14 @@ export class DescriptionLoader implements Disposable {
161
177
  /** Dispatch the current batch: ONE batched `runBatch` vision call for every
162
178
  * key collected this frame, then resolve each load()'s promise with its
163
179
  * description (or {@link UNAVAILABLE} on failure). Failures are evicted from
164
- * the cache so the next turn re-attempts. */
180
+ * the cache so the next turn re-attempts.
181
+ *
182
+ * Results are resolved BY HASH and fanned out to EVERY callback: a batch can
183
+ * hold more callbacks than keys when the same image was loaded twice in one
184
+ * frame (its first cache entry was evicted mid-frame, so the second load
185
+ * pushed a second callback for the same hash). Indexing callbacks by key
186
+ * would skip those duplicates and hang their promises; iterating callbacks
187
+ * and looking up each one's hash fans the one result to all of them. */
165
188
  private async dispatchBatch(): Promise<void> {
166
189
  this.dispatchScheduled = false;
167
190
  const batch = this.batch;
@@ -172,10 +195,8 @@ export class DescriptionLoader implements Disposable {
172
195
  return;
173
196
  }
174
197
  if (this.turnSignal?.aborted) {
175
- for (let i = 0; i < batch.keys.length; i++) {
176
- this.cache.delete(batch.keys[i]);
177
- batch.callbacks[i].resolve(UNAVAILABLE);
178
- }
198
+ for (const key of batch.keys) this.cache.delete(key);
199
+ for (const cb of batch.callbacks) cb.resolve(UNAVAILABLE);
179
200
  return;
180
201
  }
181
202
  this.deps.setLastError(null); // clear before a fresh attempt
@@ -190,19 +211,27 @@ export class DescriptionLoader implements Disposable {
190
211
  this.deps,
191
212
  this.turnSignal,
192
213
  );
214
+ // Build per-hash results, then fan them out to every callback. This reaches
215
+ // duplicate-hash callbacks that a key-indexed loop would have skipped (and
216
+ // left hanging).
217
+ const results = new Map<string, string>();
193
218
  for (let i = 0; i < batch.keys.length; i++) {
194
- const raw = parsed.get(batch.keys[i]);
219
+ const key = batch.keys[i];
220
+ const raw = parsed.get(key);
195
221
  if (raw) {
196
222
  const final = wrapDescription(raw, cfg);
197
- this.cache.set(batch.keys[i], Promise.resolve(final));
198
- batch.callbacks[i].resolve(final);
223
+ results.set(key, final);
224
+ // Cache the resolved value so later loads (this frame or next) hit.
225
+ this.cache.set(key, Promise.resolve(final));
199
226
  } else {
200
227
  // Genuine failure โ€” do NOT cache; next turn re-attempts (and surfaces
201
228
  // the real error). Resolve (not reject) with UNAVAILABLE to match the
202
229
  // graceful-degradation contract and avoid unhandled rejections.
203
- this.cache.delete(batch.keys[i]);
204
- batch.callbacks[i].resolve(UNAVAILABLE);
230
+ this.cache.delete(key);
205
231
  }
206
232
  }
233
+ for (const cb of batch.callbacks) {
234
+ cb.resolve(results.get(cb.hash) ?? UNAVAILABLE);
235
+ }
207
236
  }
208
237
  }
package/src/describer.ts CHANGED
@@ -18,7 +18,7 @@
18
18
  * abort listener.
19
19
  */
20
20
 
21
- import type { Api, ImageContent, Message, Model, TextContent } from "@earendil-works/pi-ai";
21
+ import type { Api, ImageContent, Message, Model, TextContent, ThinkingLevel } from "@earendil-works/pi-ai";
22
22
  import { complete } from "@earendil-works/pi-ai/compat";
23
23
  import type { ModelRegistry } from "@earendil-works/pi-coding-agent";
24
24
  import {
@@ -76,6 +76,26 @@ export function resolveMaxTokens(
76
76
  return Math.max(1, Math.min(requested, cap));
77
77
  }
78
78
 
79
+ /** Resolve the `reasoning` (thinking) level to pass to `complete()`, or
80
+ * `undefined` to leave thinking off.
81
+ *
82
+ * - Returns `undefined` when thinking is disabled in config.
83
+ * - Returns `undefined` when the vision model doesn't declare reasoning
84
+ * support (`model.reasoning === false`) โ€” sending a `reasoning` level to a
85
+ * non-reasoning model would be ignored at best and rejected at worst.
86
+ * - Otherwise returns the configured {@link VisionHandoffConfig.thinkingLevel}.
87
+ *
88
+ * Mirrors pi core's `createSummarizationOptions` guard so the describer's
89
+ * thinking behaviour matches the rest of pi. */
90
+ export function resolveReasoning(
91
+ cfg: VisionHandoffConfig,
92
+ visionModel: Model<Api>,
93
+ ): ThinkingLevel | undefined {
94
+ if (!cfg.thinking) return undefined;
95
+ if (!visionModel.reasoning) return undefined;
96
+ return cfg.thinkingLevel;
97
+ }
98
+
79
99
  /** Dependencies the describer can't own itself (held by the engine). */
80
100
  export interface DescriberDeps {
81
101
  /** Report a usage+energy record for one real describer call (cache hits emit none). */
@@ -126,6 +146,7 @@ export async function runBatch(
126
146
 
127
147
  const timeoutMs = describeTimeoutMs(misses.length);
128
148
  const maxTokens = resolveMaxTokens(cfg, visionModel);
149
+ const reasoning = resolveReasoning(cfg, visionModel);
129
150
  const controller = new AbortController();
130
151
  let timedOut = false;
131
152
  using fetchGuard = fetchInterceptorGuard();
@@ -141,7 +162,7 @@ export async function runBatch(
141
162
  complete(
142
163
  visionModel,
143
164
  { systemPrompt, messages: [userMessage] },
144
- { apiKey: auth.apiKey, headers: auth.headers, signal: controller.signal, maxTokens },
165
+ { apiKey: auth.apiKey, headers: auth.headers, signal: controller.signal, maxTokens, reasoning },
145
166
  ),
146
167
  );
147
168
  const capture = await readCapture(describeCtx);
@@ -239,6 +260,7 @@ export async function describeSingle(
239
260
  const userMessage: Message = { role: "user", content, timestamp: Date.now() };
240
261
  const timeoutMs = describeTimeoutMs(1);
241
262
  const maxTokens = resolveMaxTokens(cfg, visionModel);
263
+ const reasoning = resolveReasoning(cfg, visionModel);
242
264
  const controller = new AbortController();
243
265
  let timedOut = false;
244
266
  using fetchGuard = fetchInterceptorGuard();
@@ -254,7 +276,7 @@ export async function describeSingle(
254
276
  complete(
255
277
  visionModel,
256
278
  { systemPrompt, messages: [userMessage] },
257
- { apiKey: auth.apiKey, headers: auth.headers, signal: controller.signal, maxTokens },
279
+ { apiKey: auth.apiKey, headers: auth.headers, signal: controller.signal, maxTokens, reasoning },
258
280
  ),
259
281
  );
260
282
  const capture = await readCapture(describeCtx);
package/src/index.ts CHANGED
@@ -5,7 +5,7 @@
5
5
  * convention pi-model-sort uses for picker-backed extensions.
6
6
  */
7
7
 
8
- import type { ImageContent, TextContent } from "@earendil-works/pi-ai";
8
+ import type { ImageContent, TextContent, ThinkingLevel } from "@earendil-works/pi-ai";
9
9
  import { getAgentDir } from "@earendil-works/pi-coding-agent";
10
10
  import { existsSync, mkdirSync, readFileSync, writeFileSync } from "node:fs";
11
11
  import { join } from "node:path";
@@ -57,6 +57,26 @@ export function markDescriptionTruncated(text: string): string {
57
57
  /** Default vision cache size (number of described images kept in memory per session). */
58
58
  export const DEFAULT_CACHE_MAX = 50;
59
59
 
60
+ /** The pi thinking levels, ordered from lightest to deepest reasoning.
61
+ * Mirrors `@earendil-works/pi-ai`'s `ThinkingLevel`; "off" is handled
62
+ * separately via {@link VisionHandoffConfig.thinking} (a boolean on/off
63
+ * switch) so the level persists across toggles. */
64
+ export const THINKING_LEVELS: readonly ThinkingLevel[] = [
65
+ "minimal",
66
+ "low",
67
+ "medium",
68
+ "high",
69
+ "xhigh",
70
+ ];
71
+
72
+ /** Default thinking effort for the vision describer when thinking is enabled. */
73
+ export const DEFAULT_THINKING_LEVEL: ThinkingLevel = "medium";
74
+
75
+ /** Whether `level` is one of the supported {@link ThinkingLevel} values. */
76
+ export function isThinkingLevel(level: unknown): level is ThinkingLevel {
77
+ return typeof level === "string" && (THINKING_LEVELS as readonly string[]).includes(level);
78
+ }
79
+
60
80
  /** Per-description request timeout. Generous because the describer generates an
61
81
  * exhaustive multi-paragraph description per image; a single image commonly
62
82
  * takes ~20s, and a batched call over websocket transport scales with image
@@ -135,6 +155,16 @@ export interface VisionHandoffConfig {
135
155
  prompt?: string;
136
156
  /** Override the user-prompt prefix (defaults to DEFAULT_USER_PROMPT_PREFIX). */
137
157
  userPromptPrefix?: string;
158
+ /** Whether the vision describer should reason (think) before describing.
159
+ * Off by default โ€” describing images is a perception task, not a reasoning
160
+ * one, and thinking adds latency + cost. When on, the level below is sent
161
+ * to the vision model via pi-ai's `reasoning` option (only honoured when
162
+ * the configured vision model declares `reasoning: true`). */
163
+ thinking: boolean;
164
+ /** Thinking effort for the describer when {@link thinking} is on. One of
165
+ * {@link THINKING_LEVELS}. Ignored when {@link thinking} is off or the
166
+ * vision model lacks reasoning support. */
167
+ thinkingLevel: ThinkingLevel;
138
168
  }
139
169
 
140
170
  export const DEFAULT_CONFIG: VisionHandoffConfig = {
@@ -145,6 +175,8 @@ export const DEFAULT_CONFIG: VisionHandoffConfig = {
145
175
  maxTokens: undefined,
146
176
  cacheMax: DEFAULT_CACHE_MAX,
147
177
  maxDescriptionLines: DEFAULT_MAX_DESCRIPTION_LINES,
178
+ thinking: false,
179
+ thinkingLevel: DEFAULT_THINKING_LEVEL,
148
180
  };
149
181
 
150
182
  /** Parse a "provider/id" reference. Returns null if malformed. */
@@ -208,6 +240,9 @@ export function normalizeConfig(raw: unknown): VisionHandoffConfig {
208
240
  if (typeof obj.prompt === "string" && obj.prompt.trim()) base.prompt = obj.prompt;
209
241
  if (typeof obj.userPromptPrefix === "string") base.userPromptPrefix = obj.userPromptPrefix;
210
242
 
243
+ if (typeof obj.thinking === "boolean") base.thinking = obj.thinking;
244
+ if (isThinkingLevel(obj.thinkingLevel)) base.thinkingLevel = obj.thinkingLevel;
245
+
211
246
  return base;
212
247
  }
213
248
 
@@ -23,8 +23,9 @@ import {
23
23
  Text,
24
24
  } from "@earendil-works/pi-tui";
25
25
  import type { Theme } from "@earendil-works/pi-coding-agent";
26
+ import type { ThinkingLevel } from "@earendil-works/pi-ai";
26
27
  import { DynamicBorder, keyText } from "@earendil-works/pi-coding-agent";
27
- import { formatModelRef, isVisionModel } from "./index.js";
28
+ import { formatModelRef, isVisionModel, THINKING_LEVELS } from "./index.js";
28
29
 
29
30
  interface DisplayItem {
30
31
  /** "provider/id", or null for the synthetic "None" row. */
@@ -33,6 +34,8 @@ interface DisplayItem {
33
34
  modelId: string;
34
35
  modelName: string;
35
36
  vision: boolean;
37
+ /** Whether the model declares reasoning (thinking) support. */
38
+ reasoning: boolean;
36
39
  none?: boolean;
37
40
  }
38
41
 
@@ -41,6 +44,10 @@ export interface VisionModelSelectorResult {
41
44
  ref: string | null;
42
45
  /** True if the user cancelled (esc) โ€” config should not change. */
43
46
  cancelled: boolean;
47
+ /** Thinking on/off chosen in the picker. */
48
+ thinking: boolean;
49
+ /** Thinking effort chosen in the picker. */
50
+ thinkingLevel: ThinkingLevel;
44
51
  }
45
52
 
46
53
  export class VisionModelSelectorComponent implements Component {
@@ -56,6 +63,8 @@ export class VisionModelSelectorComponent implements Component {
56
63
  private footerText: Text;
57
64
 
58
65
  private currentRef: string | null;
66
+ private thinking: boolean;
67
+ private thinkingLevel: ThinkingLevel;
59
68
 
60
69
  private _focused = false;
61
70
  get focused(): boolean {
@@ -68,13 +77,23 @@ export class VisionModelSelectorComponent implements Component {
68
77
 
69
78
  constructor(
70
79
  theme: Theme,
71
- allModels: Array<{ provider: string; id: string; name: string; input?: ("text" | "image")[] }>,
80
+ allModels: Array<{
81
+ provider: string;
82
+ id: string;
83
+ name: string;
84
+ input?: ("text" | "image")[];
85
+ reasoning?: boolean;
86
+ }>,
72
87
  currentRef: string | null,
88
+ currentThinking: boolean,
89
+ currentThinkingLevel: ThinkingLevel,
73
90
  done: (result: VisionModelSelectorResult) => void,
74
91
  ) {
75
92
  this.theme = theme;
76
93
  this.done = done;
77
94
  this.currentRef = currentRef;
95
+ this.thinking = currentThinking;
96
+ this.thinkingLevel = currentThinkingLevel;
78
97
  this.allItems = this.buildItems(allModels);
79
98
  this.filteredItems = this.allItems;
80
99
 
@@ -161,6 +180,22 @@ export class VisionModelSelectorComponent implements Component {
161
180
  return;
162
181
  }
163
182
 
183
+ // Thinking controls โ€” reuse pi's own app.thinking.* keybindings so the
184
+ // hints and behaviour match the rest of pi: ctrl+t toggles thinking
185
+ // on/off, shift+tab cycles the effort level. Intercepted before the
186
+ // search input so they never get swallowed as filter text.
187
+ if (kb.matches(data, "app.thinking.toggle")) {
188
+ this.thinking = !this.thinking;
189
+ this.updateList();
190
+ return;
191
+ }
192
+
193
+ if (kb.matches(data, "app.thinking.cycle")) {
194
+ this.cycleThinkingLevel();
195
+ this.updateList();
196
+ return;
197
+ }
198
+
164
199
  this.searchInput.handleInput(data);
165
200
  this.refresh();
166
201
  }
@@ -174,7 +209,13 @@ export class VisionModelSelectorComponent implements Component {
174
209
  // Internal helpers
175
210
 
176
211
  private buildItems(
177
- allModels: Array<{ provider: string; id: string; name: string; input?: ("text" | "image")[] }>,
212
+ allModels: Array<{
213
+ provider: string;
214
+ id: string;
215
+ name: string;
216
+ input?: ("text" | "image")[];
217
+ reasoning?: boolean;
218
+ }>,
178
219
  ): DisplayItem[] {
179
220
  const items: DisplayItem[] = [
180
221
  {
@@ -183,6 +224,7 @@ export class VisionModelSelectorComponent implements Component {
183
224
  modelId: "none",
184
225
  modelName: "None โ€” disable vision handoff",
185
226
  vision: false,
227
+ reasoning: false,
186
228
  none: true,
187
229
  },
188
230
  ];
@@ -192,12 +234,14 @@ export class VisionModelSelectorComponent implements Component {
192
234
  id: string;
193
235
  name: string;
194
236
  input?: ("text" | "image")[];
237
+ reasoning?: boolean;
195
238
  }): DisplayItem => ({
196
239
  ref: formatModelRef(m.provider, m.id),
197
240
  provider: m.provider,
198
241
  modelId: m.id,
199
242
  modelName: m.name || m.id,
200
243
  vision: isVisionModel(m),
244
+ reasoning: !!m.reasoning,
201
245
  });
202
246
 
203
247
  // Vision-capable first (registry order), then the rest (registry order).
@@ -217,6 +261,8 @@ export class VisionModelSelectorComponent implements Component {
217
261
  const parts: string[] = [
218
262
  `${keyText("tui.select.confirm")} select`,
219
263
  `ctrl+s done`,
264
+ `${keyText("app.thinking.toggle")} thinking`,
265
+ `${keyText("app.thinking.cycle")} effort`,
220
266
  `esc cancel`,
221
267
  this.searchInput.getValue() ? `${this.filteredItems.length - 1} match` : `${totalCount} models ยท ${visionCount} vision`,
222
268
  ];
@@ -318,16 +364,52 @@ export class VisionModelSelectorComponent implements Component {
318
364
  ),
319
365
  );
320
366
  }
367
+ this.renderThinkingDetail(selected);
321
368
  }
322
369
 
323
370
  this.footerText.setText(this.getFooterText());
324
371
  }
325
372
 
373
+ /** Append the thinking on/off + effort line to the detail pane, with a
374
+ * warning when the highlighted model can't reason (so the setting would
375
+ * be silently ignored by the describer). */
376
+ private renderThinkingDetail(selected: DisplayItem): void {
377
+ const state = this.thinking
378
+ ? this.theme.fg("success", `on (${this.thinkingLevel})`)
379
+ : this.theme.fg("muted", "off");
380
+ this.listContainer.addChild(
381
+ new Text(this.theme.fg("dim", ` Thinking: ${state}`), 0, 0),
382
+ );
383
+ if (this.thinking && !selected.none && !selected.reasoning) {
384
+ this.listContainer.addChild(
385
+ new Text(
386
+ this.theme.fg(
387
+ "warning",
388
+ ` โš  ${selected.modelId} declares no reasoning โ€” thinking will be ignored`,
389
+ ),
390
+ 0, 0,
391
+ ),
392
+ );
393
+ }
394
+ }
395
+
326
396
  private confirm(item: DisplayItem): void {
327
- this.done({ ref: item.ref, cancelled: false });
397
+ this.done({ ref: item.ref, cancelled: false, thinking: this.thinking, thinkingLevel: this.thinkingLevel });
328
398
  }
329
399
 
330
400
  private finish(cancelled: boolean): void {
331
- this.done({ ref: null, cancelled });
401
+ this.done({ ref: null, cancelled, thinking: this.thinking, thinkingLevel: this.thinkingLevel });
402
+ }
403
+
404
+ /** Cycle the thinking effort forward through {@link THINKING_LEVELS},
405
+ * wrapping from the last back to the first. Cycling implicitly turns
406
+ * thinking on (you don't usually cycle a switch you want off) โ€” matching
407
+ * pi's own `app.thinking.cycle` behaviour, which is a no-op only when the
408
+ * active model has no reasoning. */
409
+ private cycleThinkingLevel(): void {
410
+ if (!this.thinking) this.thinking = true;
411
+ const idx = THINKING_LEVELS.indexOf(this.thinkingLevel);
412
+ const next = THINKING_LEVELS[(idx + 1) % THINKING_LEVELS.length]!;
413
+ this.thinkingLevel = next;
332
414
  }
333
415
  }
package/vision-handoff.ts CHANGED
@@ -33,6 +33,7 @@ import type { Api, ImageContent, Model, TextContent } from "@earendil-works/pi-a
33
33
  import {
34
34
  extractImageFromBlock,
35
35
  formatModelRef,
36
+ isThinkingLevel,
36
37
  isVisionModel,
37
38
  NON_VISION_IMAGE_NOTE,
38
39
  parseModelRef,
@@ -209,7 +210,8 @@ export default function (pi: ExtensionAPI) {
209
210
  //
210
211
  // Both sources flow through the same loader: `load()` is synchronous and
211
212
  // memoized, so all images in this frame (attached + clipboard-path)
212
- // coalesce into ONE batch dispatched after the microtask cascade.
213
+ // coalesce into ONE batch dispatched via `setImmediate` after the
214
+ // microtask cascade settles.
213
215
  for (const image of event.images ?? []) {
214
216
  if (!image || image.type !== "image" || !image.data) continue;
215
217
  loader.loadDescription({ data: image.data, mimeType: image.mimeType || "image/png" }).catch(() => {});
@@ -243,12 +245,14 @@ export default function (pi: ExtensionAPI) {
243
245
  // When the agent reads image files, this fires for each read result. It
244
246
  // calls the loader's `loadDescription(img)` for every image block and
245
247
  // AWAITS the shared batch โ€” so N parallel reads (pi runs `read` via
246
- // Promise.all) coalesce into ONE batched vision call (DataLoader: all
247
- // load() calls in the same microtask frame share one batch, dispatched
248
- // after the cascade settles) and all resolve together. The descriptions
249
- // replace the image blocks in the returned `content`, so by the time the
250
- // agent's next turn starts the tool results already carry text โ€” the
251
- // agent never sees raw image blocks it can't process.
248
+ // Promise.all) coalesce into ONE batched vision call: pi fires each read's
249
+ // `tool_result` as its I/O completes (poll phase), and the loader's
250
+ // `setImmediate` dispatch defers to the check phase AFTER the whole poll
251
+ // iteration, so reads completing together land in ONE batch and all resolve
252
+ // together. The descriptions replace the image blocks in the returned
253
+ // `content`, so by the time the agent's next turn starts the tool results
254
+ // already carry text โ€” the agent never sees raw image blocks it can't
255
+ // process.
252
256
  //
253
257
  // Why block here and not in `context`: the tool-result phase is free time
254
258
  // (the agent is just waiting for tool results), so running the describer
@@ -276,9 +280,13 @@ export default function (pi: ExtensionAPI) {
276
280
  }
277
281
 
278
282
  // load() each image โ€” synchronous calls that push into the current batch
279
- // and return memoized promises โ€” then await them all. Parallel reads'
280
- // load() calls land in the SAME batch (same microtask frame), so this is
281
- // ONE vision call for the whole read set, not N. Awaiting here runs the
283
+ // and return memoized promises โ€” then await them all. pi fires each read's
284
+ // `tool_result` event as that read's I/O completes (poll phase); the
285
+ // loader's `setImmediate` dispatch defers to the check phase, AFTER the
286
+ // whole poll iteration, so reads completing together (the common case for
287
+ // cached local files) land in ONE batch โ€” ONE vision call for the whole
288
+ // read set, not N. Reads completing in separate iterations get separate
289
+ // calls, but always in parallel, never sequential. Awaiting here runs the
282
290
  // describer during the tool-result phase (free time โ€” the agent is just
283
291
  // waiting for tool results), so the batch is COMPLETE before `context`
284
292
  // fires, making `context` a non-blocking cache hit instead of a cold miss
@@ -412,7 +420,7 @@ export default function (pi: ExtensionAPI) {
412
420
  pi.registerCommand("vision-handoff", {
413
421
  description: HANDOFF_COMMAND_DESCRIPTION,
414
422
  getArgumentCompletions(prefix: string) {
415
- const subcommands = ["select", "model", "status", "enable", "disable", "auto", "add", "remove", "clear", "help"];
423
+ const subcommands = ["select", "model", "status", "enable", "disable", "auto", "thinking", "add", "remove", "clear", "help"];
416
424
  const matches = subcommands.filter((s) => s.startsWith(prefix));
417
425
  return matches.length > 0 ? matches.map((s) => ({ value: s, label: s })) : null;
418
426
  },
@@ -444,6 +452,8 @@ async function handleHandoffCommand(ctx: ExtensionCommandContext, args: string):
444
452
  " /vision-handoff enable Enable vision handoff",
445
453
  " /vision-handoff disable Disable vision handoff (keeps configured model)",
446
454
  " /vision-handoff auto <on|off> Toggle automatic handoff for all non-vision models",
455
+ " /vision-handoff thinking <off|minimal|low|medium|high|xhigh>",
456
+ " Set the vision describer's thinking effort (off = disabled)",
447
457
  " /vision-handoff add <p/id> Force handoff for an extra model",
448
458
  " /vision-handoff remove <p/id> Stop forcing handoff for a model",
449
459
  " /vision-handoff clear Clear the configured vision model",
@@ -489,6 +499,11 @@ async function handleHandoffCommand(ctx: ExtensionCommandContext, args: string):
489
499
  return;
490
500
  }
491
501
 
502
+ if (subcommand === "thinking") {
503
+ handleThinkingSubcommand(ctx, rest);
504
+ return;
505
+ }
506
+
492
507
  if (subcommand === "clear") {
493
508
  updateConfig(
494
509
  ctx,
@@ -582,6 +597,38 @@ function updateConfig(
582
597
  ctx.ui.notify(`${message} (config: ${path})`, "info");
583
598
  }
584
599
 
600
+ /** Resolve a `/vision-handoff thinking <level>` argument into a
601
+ * `(thinking, thinkingLevel)` pair. `off` disables thinking; any of the
602
+ * {@link THINKING_LEVELS} enables it at that effort. */
603
+ function handleThinkingSubcommand(ctx: ExtensionCommandContext, rest: string): void {
604
+ const arg = rest.trim().toLowerCase();
605
+ if (!arg) {
606
+ ctx.ui.notify(
607
+ `Thinking: ${config.thinking ? `on (${config.thinkingLevel})` : "off"}.\n` +
608
+ `Usage: /vision-handoff thinking <off|minimal|low|medium|high|xhigh>`,
609
+ "info",
610
+ );
611
+ return;
612
+ }
613
+ if (arg === "off") {
614
+ updateConfig(ctx, (c) => ({ ...c, thinking: false }), "Vision describer thinking off.");
615
+ return;
616
+ }
617
+ if (!isThinkingLevel(arg)) {
618
+ ctx.ui.notify(
619
+ `Unknown thinking level: "${arg}". Use off, minimal, low, medium, high, or xhigh.`,
620
+ "error",
621
+ );
622
+ return;
623
+ }
624
+ const level = arg;
625
+ updateConfig(
626
+ ctx,
627
+ (c) => ({ ...c, thinking: true, thinkingLevel: level }),
628
+ `Vision describer thinking on (${level}).`,
629
+ );
630
+ }
631
+
585
632
  async function showSelector(ctx: ExtensionCommandContext): Promise<void> {
586
633
  if (!ctx.hasUI) {
587
634
  ctx.ui.notify("/vision-handoff requires interactive mode.", "error");
@@ -590,10 +637,17 @@ async function showSelector(ctx: ExtensionCommandContext): Promise<void> {
590
637
 
591
638
  const allModels = ctx.modelRegistry
592
639
  .getAll()
593
- .map((m) => ({ provider: m.provider, id: m.id, name: m.name, input: m.input }));
640
+ .map((m) => ({ provider: m.provider, id: m.id, name: m.name, input: m.input, reasoning: m.reasoning }));
594
641
 
595
642
  const result = await ctx.ui.custom<VisionModelSelectorResult>((tui, theme, _kb, done) => {
596
- const selector = new VisionModelSelectorComponent(theme, allModels, config.visionModel, (r) => done(r));
643
+ const selector = new VisionModelSelectorComponent(
644
+ theme,
645
+ allModels,
646
+ config.visionModel,
647
+ config.thinking,
648
+ config.thinkingLevel,
649
+ (r) => done(r),
650
+ );
597
651
  return {
598
652
  render(width: number) {
599
653
  return selector.render(width);
@@ -614,7 +668,18 @@ async function showSelector(ctx: ExtensionCommandContext): Promise<void> {
614
668
  }
615
669
 
616
670
  const ref = result.ref;
617
- updateConfig(ctx, (c) => ({ ...c, visionModel: ref }), ref ? `Vision model set to ${ref}` : "Vision model cleared");
671
+ const thinking = result.thinking;
672
+ const thinkingLevel = result.thinkingLevel;
673
+ updateConfig(
674
+ ctx,
675
+ (c) => ({ ...c, visionModel: ref, thinking, thinkingLevel }),
676
+ ref ? `Vision model set to ${ref}` : "Vision model cleared",
677
+ );
678
+ ctx.ui.notify(
679
+ `Thinking: ${thinking ? `on (${thinkingLevel})` : "off"}` +
680
+ (thinking && ref ? " โ€” applies only if the vision model supports reasoning" : ""),
681
+ "info",
682
+ );
618
683
  if (!ref) {
619
684
  ctx.ui.notify("Handoff is inactive until you pick a vision model.", "warning");
620
685
  }
@@ -626,6 +691,7 @@ function showStatus(ctx: ExtensionCommandContext): void {
626
691
  lines.push(`Vision model: ${config.visionModel ?? "(none โ€” pick one with /vision-handoff)"}`);
627
692
  lines.push(`Auto handoff (non-vision models): ${config.autoHandoff ? "on" : "off"}`);
628
693
  lines.push(`Handoff targets (explicit): ${config.handoffModels.length ? config.handoffModels.join(", ") : "(none)"}`);
694
+ lines.push(`Thinking: ${config.thinking ? `on (${config.thinkingLevel})` : "off"}`);
629
695
  lines.push(`maxTokens: ${config.maxTokens ?? "unbounded"} ยท cacheMax: ${config.cacheMax} ยท maxDescriptionLines: ${config.maxDescriptionLines === 0 ? "unbounded" : config.maxDescriptionLines}`);
630
696
 
631
697
  const model = ctx.model;