pi-vision-handoff 0.3.2 → 0.4.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "pi-vision-handoff",
-  "version": "0.3.2",
+  "version": "0.4.0",
   "description": "Give text-only pi models vision — describe images with a vision model you pick via an interactive picker, then hand off the text description to non-vision models",
   "type": "module",
   "author": "Tom X Nguyen",

package/src/describer.ts

@@ -18,7 +18,7 @@
  * abort listener.
  */
 
-import type { Api, ImageContent, Message, Model, TextContent } from "@earendil-works/pi-ai";
+import type { Api, ImageContent, Message, Model, TextContent, ThinkingLevel } from "@earendil-works/pi-ai";
 import { complete } from "@earendil-works/pi-ai/compat";
 import type { ModelRegistry } from "@earendil-works/pi-coding-agent";
 import {
@@ -76,6 +76,26 @@ export function resolveMaxTokens(
   return Math.max(1, Math.min(requested, cap));
 }
 
+/** Resolve the `reasoning` (thinking) level to pass to `complete()`, or
+ *  `undefined` to leave thinking off.
+ *
+ *  - Returns `undefined` when thinking is disabled in config.
+ *  - Returns `undefined` when the vision model doesn't declare reasoning
+ *    support (`model.reasoning === false`) — sending a `reasoning` level to a
+ *    non-reasoning model would be ignored at best and rejected at worst.
+ *  - Otherwise returns the configured {@link VisionHandoffConfig.thinkingLevel}.
+ *
+ *  Mirrors pi core's `createSummarizationOptions` guard so the describer's
+ *  thinking behaviour matches the rest of pi. */
+export function resolveReasoning(
+  cfg: VisionHandoffConfig,
+  visionModel: Model<Api>,
+): ThinkingLevel | undefined {
+  if (!cfg.thinking) return undefined;
+  if (!visionModel.reasoning) return undefined;
+  return cfg.thinkingLevel;
+}
+
 /** Dependencies the describer can't own itself (held by the engine). */
 export interface DescriberDeps {
   /** Report a usage+energy record for one real describer call (cache hits emit none). */
@@ -126,6 +146,7 @@ export async function runBatch(
 
   const timeoutMs = describeTimeoutMs(misses.length);
   const maxTokens = resolveMaxTokens(cfg, visionModel);
+  const reasoning = resolveReasoning(cfg, visionModel);
   const controller = new AbortController();
   let timedOut = false;
   using fetchGuard = fetchInterceptorGuard();
@@ -141,7 +162,7 @@ export async function runBatch(
       complete(
         visionModel,
         { systemPrompt, messages: [userMessage] },
-        { apiKey: auth.apiKey, headers: auth.headers, signal: controller.signal, maxTokens },
+        { apiKey: auth.apiKey, headers: auth.headers, signal: controller.signal, maxTokens, reasoning },
       ),
     );
     const capture = await readCapture(describeCtx);
@@ -239,6 +260,7 @@ export async function describeSingle(
   const userMessage: Message = { role: "user", content, timestamp: Date.now() };
   const timeoutMs = describeTimeoutMs(1);
   const maxTokens = resolveMaxTokens(cfg, visionModel);
+  const reasoning = resolveReasoning(cfg, visionModel);
   const controller = new AbortController();
   let timedOut = false;
   using fetchGuard = fetchInterceptorGuard();
@@ -254,7 +276,7 @@ export async function describeSingle(
       complete(
         visionModel,
         { systemPrompt, messages: [userMessage] },
-        { apiKey: auth.apiKey, headers: auth.headers, signal: controller.signal, maxTokens },
+        { apiKey: auth.apiKey, headers: auth.headers, signal: controller.signal, maxTokens, reasoning },
       ),
     );
     const capture = await readCapture(describeCtx);

package/src/index.ts

@@ -5,7 +5,7 @@
  * convention pi-model-sort uses for picker-backed extensions.
  */
 
-import type { ImageContent, TextContent } from "@earendil-works/pi-ai";
+import type { ImageContent, TextContent, ThinkingLevel } from "@earendil-works/pi-ai";
 import { getAgentDir } from "@earendil-works/pi-coding-agent";
 import { existsSync, mkdirSync, readFileSync, writeFileSync } from "node:fs";
 import { join } from "node:path";
@@ -57,6 +57,26 @@ export function markDescriptionTruncated(text: string): string {
 /** Default vision cache size (number of described images kept in memory per session). */
 export const DEFAULT_CACHE_MAX = 50;
 
+/** The pi thinking levels, ordered from lightest to deepest reasoning.
+ *  Mirrors `@earendil-works/pi-ai`'s `ThinkingLevel`; "off" is handled
+ *  separately via {@link VisionHandoffConfig.thinking} (a boolean on/off
+ *  switch) so the level persists across toggles. */
+export const THINKING_LEVELS: readonly ThinkingLevel[] = [
+  "minimal",
+  "low",
+  "medium",
+  "high",
+  "xhigh",
+];
+
+/** Default thinking effort for the vision describer when thinking is enabled. */
+export const DEFAULT_THINKING_LEVEL: ThinkingLevel = "medium";
+
+/** Whether `level` is one of the supported {@link ThinkingLevel} values. */
+export function isThinkingLevel(level: unknown): level is ThinkingLevel {
+  return typeof level === "string" && (THINKING_LEVELS as readonly string[]).includes(level);
+}
+
 /** Per-description request timeout. Generous because the describer generates an
  *  exhaustive multi-paragraph description per image; a single image commonly
  *  takes ~20s, and a batched call over websocket transport scales with image
@@ -135,6 +155,16 @@ export interface VisionHandoffConfig {
   prompt?: string;
   /** Override the user-prompt prefix (defaults to DEFAULT_USER_PROMPT_PREFIX). */
   userPromptPrefix?: string;
+  /** Whether the vision describer should reason (think) before describing.
+   *  Off by default — describing images is a perception task, not a reasoning
+   *  one, and thinking adds latency + cost. When on, the level below is sent
+   *  to the vision model via pi-ai's `reasoning` option (only honoured when
+   *  the configured vision model declares `reasoning: true`). */
+  thinking: boolean;
+  /** Thinking effort for the describer when {@link thinking} is on. One of
+   *  {@link THINKING_LEVELS}. Ignored when {@link thinking} is off or the
+   *  vision model lacks reasoning support. */
+  thinkingLevel: ThinkingLevel;
 }
 
 export const DEFAULT_CONFIG: VisionHandoffConfig = {
@@ -145,6 +175,8 @@ export const DEFAULT_CONFIG: VisionHandoffConfig = {
   maxTokens: undefined,
   cacheMax: DEFAULT_CACHE_MAX,
   maxDescriptionLines: DEFAULT_MAX_DESCRIPTION_LINES,
+  thinking: false,
+  thinkingLevel: DEFAULT_THINKING_LEVEL,
 };
 
 /** Parse a "provider/id" reference. Returns null if malformed. */
@@ -208,6 +240,9 @@ export function normalizeConfig(raw: unknown): VisionHandoffConfig {
   if (typeof obj.prompt === "string" && obj.prompt.trim()) base.prompt = obj.prompt;
   if (typeof obj.userPromptPrefix === "string") base.userPromptPrefix = obj.userPromptPrefix;
 
+  if (typeof obj.thinking === "boolean") base.thinking = obj.thinking;
+  if (isThinkingLevel(obj.thinkingLevel)) base.thinkingLevel = obj.thinkingLevel;
+
   return base;
 }
 

package/src/vision-model-selector.ts

@@ -23,8 +23,9 @@ import {
   Text,
 } from "@earendil-works/pi-tui";
 import type { Theme } from "@earendil-works/pi-coding-agent";
+import type { ThinkingLevel } from "@earendil-works/pi-ai";
 import { DynamicBorder, keyText } from "@earendil-works/pi-coding-agent";
-import { formatModelRef, isVisionModel } from "./index.js";
+import { formatModelRef, isVisionModel, THINKING_LEVELS } from "./index.js";
 
 interface DisplayItem {
   /** "provider/id", or null for the synthetic "None" row. */
@@ -33,6 +34,8 @@ interface DisplayItem {
   modelId: string;
   modelName: string;
   vision: boolean;
+  /** Whether the model declares reasoning (thinking) support. */
+  reasoning: boolean;
   none?: boolean;
 }
 
@@ -41,6 +44,10 @@ export interface VisionModelSelectorResult {
   ref: string | null;
   /** True if the user cancelled (esc) — config should not change. */
   cancelled: boolean;
+  /** Thinking on/off chosen in the picker. */
+  thinking: boolean;
+  /** Thinking effort chosen in the picker. */
+  thinkingLevel: ThinkingLevel;
 }
 
 export class VisionModelSelectorComponent implements Component {
@@ -56,6 +63,8 @@ export class VisionModelSelectorComponent implements Component {
   private footerText: Text;
 
   private currentRef: string | null;
+  private thinking: boolean;
+  private thinkingLevel: ThinkingLevel;
 
   private _focused = false;
   get focused(): boolean {
@@ -68,13 +77,23 @@ export class VisionModelSelectorComponent implements Component {
 
   constructor(
     theme: Theme,
-    allModels: Array<{ provider: string; id: string; name: string; input?: ("text" | "image")[] }>,
+    allModels: Array<{
+      provider: string;
+      id: string;
+      name: string;
+      input?: ("text" | "image")[];
+      reasoning?: boolean;
+    }>,
     currentRef: string | null,
+    currentThinking: boolean,
+    currentThinkingLevel: ThinkingLevel,
     done: (result: VisionModelSelectorResult) => void,
   ) {
     this.theme = theme;
     this.done = done;
     this.currentRef = currentRef;
+    this.thinking = currentThinking;
+    this.thinkingLevel = currentThinkingLevel;
     this.allItems = this.buildItems(allModels);
     this.filteredItems = this.allItems;
 
@@ -161,6 +180,22 @@ export class VisionModelSelectorComponent implements Component {
       return;
     }
 
+    // Thinking controls — reuse pi's own app.thinking.* keybindings so the
+    // hints and behaviour match the rest of pi: ctrl+t toggles thinking
+    // on/off, shift+tab cycles the effort level. Intercepted before the
+    // search input so they never get swallowed as filter text.
+    if (kb.matches(data, "app.thinking.toggle")) {
+      this.thinking = !this.thinking;
+      this.updateList();
+      return;
+    }
+
+    if (kb.matches(data, "app.thinking.cycle")) {
+      this.cycleThinkingLevel();
+      this.updateList();
+      return;
+    }
+
     this.searchInput.handleInput(data);
     this.refresh();
   }
@@ -174,7 +209,13 @@ export class VisionModelSelectorComponent implements Component {
   // Internal helpers
 
   private buildItems(
-    allModels: Array<{ provider: string; id: string; name: string; input?: ("text" | "image")[] }>,
+    allModels: Array<{
+      provider: string;
+      id: string;
+      name: string;
+      input?: ("text" | "image")[];
+      reasoning?: boolean;
+    }>,
   ): DisplayItem[] {
     const items: DisplayItem[] = [
       {
@@ -183,6 +224,7 @@ export class VisionModelSelectorComponent implements Component {
         modelId: "none",
         modelName: "None — disable vision handoff",
         vision: false,
+        reasoning: false,
         none: true,
       },
     ];
@@ -192,12 +234,14 @@ export class VisionModelSelectorComponent implements Component {
       id: string;
       name: string;
       input?: ("text" | "image")[];
+      reasoning?: boolean;
     }): DisplayItem => ({
       ref: formatModelRef(m.provider, m.id),
       provider: m.provider,
       modelId: m.id,
       modelName: m.name || m.id,
       vision: isVisionModel(m),
+      reasoning: !!m.reasoning,
     });
 
     // Vision-capable first (registry order), then the rest (registry order).
@@ -217,6 +261,8 @@ export class VisionModelSelectorComponent implements Component {
     const parts: string[] = [
       `${keyText("tui.select.confirm")} select`,
       `ctrl+s done`,
+      `${keyText("app.thinking.toggle")} thinking`,
+      `${keyText("app.thinking.cycle")} effort`,
       `esc cancel`,
       this.searchInput.getValue() ? `${this.filteredItems.length - 1} match` : `${totalCount} models · ${visionCount} vision`,
     ];
@@ -318,16 +364,52 @@ export class VisionModelSelectorComponent implements Component {
           ),
         );
       }
+      this.renderThinkingDetail(selected);
     }
 
     this.footerText.setText(this.getFooterText());
   }
 
+  /** Append the thinking on/off + effort line to the detail pane, with a
+   *  warning when the highlighted model can't reason (so the setting would
+   *  be silently ignored by the describer). */
+  private renderThinkingDetail(selected: DisplayItem): void {
+    const state = this.thinking
+      ? this.theme.fg("success", `on (${this.thinkingLevel})`)
+      : this.theme.fg("muted", "off");
+    this.listContainer.addChild(
+      new Text(this.theme.fg("dim", `  Thinking: ${state}`), 0, 0),
+    );
+    if (this.thinking && !selected.none && !selected.reasoning) {
+      this.listContainer.addChild(
+        new Text(
+          this.theme.fg(
+            "warning",
+            `  ⚠ ${selected.modelId} declares no reasoning — thinking will be ignored`,
+          ),
+          0, 0,
+        ),
+      );
+    }
+  }
+
   private confirm(item: DisplayItem): void {
-    this.done({ ref: item.ref, cancelled: false });
+    this.done({ ref: item.ref, cancelled: false, thinking: this.thinking, thinkingLevel: this.thinkingLevel });
   }
 
   private finish(cancelled: boolean): void {
-    this.done({ ref: null, cancelled });
+    this.done({ ref: null, cancelled, thinking: this.thinking, thinkingLevel: this.thinkingLevel });
+  }
+
+  /** Cycle the thinking effort forward through {@link THINKING_LEVELS},
+   *  wrapping from the last back to the first. Cycling implicitly turns
+   *  thinking on (you don't usually cycle a switch you want off) — matching
+   *  pi's own `app.thinking.cycle` behaviour, which is a no-op only when the
+   *  active model has no reasoning. */
+  private cycleThinkingLevel(): void {
+    if (!this.thinking) this.thinking = true;
+    const idx = THINKING_LEVELS.indexOf(this.thinkingLevel);
+    const next = THINKING_LEVELS[(idx + 1) % THINKING_LEVELS.length]!;
+    this.thinkingLevel = next;
   }
 }

package/vision-handoff.ts

@@ -33,6 +33,7 @@ import type { Api, ImageContent, Model, TextContent } from "@earendil-works/pi-a
 import {
   extractImageFromBlock,
   formatModelRef,
+  isThinkingLevel,
   isVisionModel,
   NON_VISION_IMAGE_NOTE,
   parseModelRef,
@@ -412,7 +413,7 @@ export default function (pi: ExtensionAPI) {
   pi.registerCommand("vision-handoff", {
     description: HANDOFF_COMMAND_DESCRIPTION,
     getArgumentCompletions(prefix: string) {
-      const subcommands = ["select", "model", "status", "enable", "disable", "auto", "add", "remove", "clear", "help"];
+      const subcommands = ["select", "model", "status", "enable", "disable", "auto", "thinking", "add", "remove", "clear", "help"];
       const matches = subcommands.filter((s) => s.startsWith(prefix));
       return matches.length > 0 ? matches.map((s) => ({ value: s, label: s })) : null;
     },
@@ -444,6 +445,8 @@ async function handleHandoffCommand(ctx: ExtensionCommandContext, args: string):
         "  /vision-handoff enable         Enable vision handoff",
         "  /vision-handoff disable        Disable vision handoff (keeps configured model)",
         "  /vision-handoff auto <on|off>  Toggle automatic handoff for all non-vision models",
+        "  /vision-handoff thinking <off|minimal|low|medium|high|xhigh>",
+        "                               Set the vision describer's thinking effort (off = disabled)",
         "  /vision-handoff add <p/id>     Force handoff for an extra model",
         "  /vision-handoff remove <p/id>  Stop forcing handoff for a model",
         "  /vision-handoff clear          Clear the configured vision model",
@@ -489,6 +492,11 @@ async function handleHandoffCommand(ctx: ExtensionCommandContext, args: string):
     return;
   }
 
+  if (subcommand === "thinking") {
+    handleThinkingSubcommand(ctx, rest);
+    return;
+  }
+
   if (subcommand === "clear") {
     updateConfig(
       ctx,
@@ -582,6 +590,38 @@ function updateConfig(
   ctx.ui.notify(`${message} (config: ${path})`, "info");
 }
 
+/** Resolve a `/vision-handoff thinking <level>` argument into a
+ *  `(thinking, thinkingLevel)` pair. `off` disables thinking; any of the
+ *  {@link THINKING_LEVELS} enables it at that effort. */
+function handleThinkingSubcommand(ctx: ExtensionCommandContext, rest: string): void {
+  const arg = rest.trim().toLowerCase();
+  if (!arg) {
+    ctx.ui.notify(
+      `Thinking: ${config.thinking ? `on (${config.thinkingLevel})` : "off"}.\n` +
+        `Usage: /vision-handoff thinking <off|minimal|low|medium|high|xhigh>`,
+      "info",
+    );
+    return;
+  }
+  if (arg === "off") {
+    updateConfig(ctx, (c) => ({ ...c, thinking: false }), "Vision describer thinking off.");
+    return;
+  }
+  if (!isThinkingLevel(arg)) {
+    ctx.ui.notify(
+      `Unknown thinking level: "${arg}". Use off, minimal, low, medium, high, or xhigh.`,
+      "error",
+    );
+    return;
+  }
+  const level = arg;
+  updateConfig(
+    ctx,
+    (c) => ({ ...c, thinking: true, thinkingLevel: level }),
+    `Vision describer thinking on (${level}).`,
+  );
+}
+
 async function showSelector(ctx: ExtensionCommandContext): Promise<void> {
   if (!ctx.hasUI) {
     ctx.ui.notify("/vision-handoff requires interactive mode.", "error");
@@ -590,10 +630,17 @@ async function showSelector(ctx: ExtensionCommandContext): Promise<void> {
 
   const allModels = ctx.modelRegistry
     .getAll()
-    .map((m) => ({ provider: m.provider, id: m.id, name: m.name, input: m.input }));
+    .map((m) => ({ provider: m.provider, id: m.id, name: m.name, input: m.input, reasoning: m.reasoning }));
 
   const result = await ctx.ui.custom<VisionModelSelectorResult>((tui, theme, _kb, done) => {
-    const selector = new VisionModelSelectorComponent(theme, allModels, config.visionModel, (r) => done(r));
+    const selector = new VisionModelSelectorComponent(
+      theme,
+      allModels,
+      config.visionModel,
+      config.thinking,
+      config.thinkingLevel,
+      (r) => done(r),
+    );
     return {
       render(width: number) {
         return selector.render(width);
@@ -614,7 +661,18 @@ async function showSelector(ctx: ExtensionCommandContext): Promise<void> {
   }
 
   const ref = result.ref;
-  updateConfig(ctx, (c) => ({ ...c, visionModel: ref }), ref ? `Vision model set to ${ref}` : "Vision model cleared");
+  const thinking = result.thinking;
+  const thinkingLevel = result.thinkingLevel;
+  updateConfig(
+    ctx,
+    (c) => ({ ...c, visionModel: ref, thinking, thinkingLevel }),
+    ref ? `Vision model set to ${ref}` : "Vision model cleared",
+  );
+  ctx.ui.notify(
+    `Thinking: ${thinking ? `on (${thinkingLevel})` : "off"}` +
+      (thinking && ref ? " — applies only if the vision model supports reasoning" : ""),
+    "info",
+  );
   if (!ref) {
     ctx.ui.notify("Handoff is inactive until you pick a vision model.", "warning");
   }
@@ -626,6 +684,7 @@ function showStatus(ctx: ExtensionCommandContext): void {
   lines.push(`Vision model: ${config.visionModel ?? "(none — pick one with /vision-handoff)"}`);
   lines.push(`Auto handoff (non-vision models): ${config.autoHandoff ? "on" : "off"}`);
   lines.push(`Handoff targets (explicit): ${config.handoffModels.length ? config.handoffModels.join(", ") : "(none)"}`);
+  lines.push(`Thinking: ${config.thinking ? `on (${config.thinkingLevel})` : "off"}`);
   lines.push(`maxTokens: ${config.maxTokens ?? "unbounded"} · cacheMax: ${config.cacheMax} · maxDescriptionLines: ${config.maxDescriptionLines === 0 ? "unbounded" : config.maxDescriptionLines}`);
 
   const model = ctx.model;