pi-vision-handoff 0.3.0 → 0.3.2

package/src/describer.ts

@@ -0,0 +1,320 @@
+/**
+ * The vision describer: calls a vision-capable model via pi-ai's `complete()`
+ * to produce text descriptions of images.
+ *
+ * Two entry points:
+ *   - {@link runBatch}: ONE batched call describing N images at once (the
+ *     dataloader's dispatch path). The model is asked to emit one delimited
+ *     `<<<IMAGE k>>> … <<<END>>>` section per image so the response can be
+ *     split back into per-image descriptions.
+ *   - {@link describeSingle}: one call for one image (no delimiters). The
+ *     robust per-image fallback used when a batched response couldn't be split.
+ *
+ * Resource lifetimes (fetch interceptor, timeout timer, turn-abort wire) are
+ * managed with the `using` keyword via the {@link Disposable} guards in
+ * `dispose.ts`, replacing the manual `try`/`finally` cleanup the old code
+ * carried. Disposing is lexical and exception-safe: a thrown `complete()`
+ * still tears down the timer, uninstalls the interceptor, and detaches the
+ * abort listener.
+ */
+
+import type { Api, ImageContent, Message, Model, TextContent } from "@earendil-works/pi-ai";
+import { complete } from "@earendil-works/pi-ai/compat";
+import type { ModelRegistry } from "@earendil-works/pi-coding-agent";
+import {
+  batchUserPrompt,
+  DEFAULT_USER_PROMPT_PREFIX,
+  DEFAULT_VISION_PROMPT,
+  describeTimeoutMs,
+  markDescriptionTruncated,
+  parseBatchedDescriptions,
+  type ExtractedImage,
+  type VisionHandoffConfig,
+} from "./index.js";
+import {
+  buildUsageRecord,
+  describeAls,
+  EMPTY_ENERGY_CAPTURE,
+  type DescribeContext,
+  type VisionHandoffEnergyCapture,
+  type VisionHandoffUsageRecord,
+} from "./usage.js";
+import { imageHash } from "./image.js";
+import { abortWireGuard, fetchInterceptorGuard, timeoutGuard, type AbortWire } from "./dispose.js";
+
+/** Tokens reserved for the describer's input so the requested output budget
+ *  can't exceed the model's context window. The describer's input is bounded —
+ *  one (or a few) image(s) plus a short prompt — so a fixed reserve keeps the
+ *  math simple while leaving generous room for the image + prompt tokens.
+ *  Providers reject a request when `inputTokens + maxTokens > contextWindow`
+ *  (e.g. a model whose declared maxTokens equals its full context window), so
+ *  this clamp prevents that 400 without meaningfully limiting a high-output
+ *  model (e.g. 262144 context − 8192 reserve = 253952 output budget). */
+const INPUT_RESERVE_TOKENS = 8192;
+
+/** Resolve the `maxTokens` to pass to `complete()` for a describer call.
+ *
+ *  - A configured `cfg.maxTokens` wins (explicit cost/latency cap), clamped to
+ *    fit the context window.
+ *  - Otherwise use the vision model's declared max output (`model.maxTokens`),
+ *    clamped so `maxTokens + INPUT_RESERVE_TOKENS <= contextWindow` — i.e. the
+ *    requested output can't exceed the context window minus a reserve for the
+ *    input. This is the real fix for models whose declared `maxTokens` equals
+ *    their full `contextWindow` (which providers reject with any non-empty
+ *    input): we cap at `contextWindow − reserve` instead.
+ *  - If the model declares no usable max (`maxTokens <= 0`) and no cap is
+ *    configured, return `undefined` so the provider applies its own default. */
+export function resolveMaxTokens(
+  cfg: VisionHandoffConfig,
+  visionModel: Model<Api>,
+): number | undefined {
+  const ctx = visionModel.contextWindow > 0 ? visionModel.contextWindow : Number.POSITIVE_INFINITY;
+  const cap = ctx - INPUT_RESERVE_TOKENS;
+  const requested = cfg.maxTokens ?? (visionModel.maxTokens > 0 ? visionModel.maxTokens : undefined);
+  if (requested === undefined) return undefined;
+  // Clamp to the context-window-derived ceiling; never below 1.
+  return Math.max(1, Math.min(requested, cap));
+}
+
+/** Dependencies the describer can't own itself (held by the engine). */
+export interface DescriberDeps {
+  /** Report a usage+energy record for one real describer call (cache hits emit none). */
+  reportUsage(record: VisionHandoffUsageRecord): void;
+  /** Set the most-recent describer failure message (surfaced to the user by the engine).
+   *  Pass `null` to clear before a fresh attempt. */
+  setLastError(msg: string | null): void;
+}
+
+/** The result of a batched describer call: per-image raw descriptions keyed by hash. */
+export type BatchResult = Map<string, string>;
+
+/** Describe N images with ONE batched `complete()` call and split the response
+ *  back into per-image descriptions. Returns a map keyed by image hash; an
+ *  image whose section failed to parse is omitted (the caller treats omission
+ *  as "description unavailable"). On a genuine call failure (auth, abort,
+ *  empty) the map is empty and `deps.setLastError` records the reason.
+ *
+ *  A failed delimiter-parse is a failed BATCH, not a failed description: the
+ *  unparsed images fall back to parallel single-image calls (no delimiters to
+ *  cooperate with), so the common cooperative case stays one call while an
+ *  uncooperative model still gets every image described. */
+export async function runBatch(
+  misses: { img: ExtractedImage; hash: string }[],
+  userPrompt: string,
+  visionModel: Model<Api>,
+  modelRegistry: ModelRegistry,
+  cfg: VisionHandoffConfig,
+  deps: DescriberDeps,
+  turnSignal?: AbortSignal,
+): Promise<BatchResult> {
+  const out: BatchResult = new Map();
+  const auth = await modelRegistry.getApiKeyAndHeaders(visionModel);
+  if (!auth.ok || !auth.apiKey) {
+    deps.setLastError(
+      !auth.ok ? auth.error : `No API key for vision model "${visionModel.provider}/${visionModel.id}"`,
+    );
+    return out;
+  }
+
+  const prefix = cfg.userPromptPrefix ?? DEFAULT_USER_PROMPT_PREFIX;
+  const systemPrompt = cfg.prompt ?? DEFAULT_VISION_PROMPT;
+  const content: (TextContent | ImageContent)[] = [
+    { type: "text", text: batchUserPrompt(misses.length, userPrompt, prefix) },
+    ...misses.map((m) => ({ type: "image", data: m.img.data, mimeType: m.img.mimeType } satisfies ImageContent)),
+  ];
+  const userMessage: Message = { role: "user", content, timestamp: Date.now() };
+
+  const timeoutMs = describeTimeoutMs(misses.length);
+  const maxTokens = resolveMaxTokens(cfg, visionModel);
+  const controller = new AbortController();
+  let timedOut = false;
+  using fetchGuard = fetchInterceptorGuard();
+  using timer = timeoutGuard(timeoutMs, () => {
+    timedOut = true;
+    controller.abort();
+  });
+  using abortWire = abortWireGuard(turnSignal, controller);
+
+  const describeCtx: DescribeContext = { energyReader: undefined };
+  try {
+    const response = await describeAls.run(describeCtx, async () =>
+      complete(
+        visionModel,
+        { systemPrompt, messages: [userMessage] },
+        { apiKey: auth.apiKey, headers: auth.headers, signal: controller.signal, maxTokens },
+      ),
+    );
+    const capture = await readCapture(describeCtx);
+    const hashes = misses.map((m) => m.hash);
+    const record = buildUsageRecord(response, capture, visionModel, hashes[0], hashes.length > 1 ? hashes : undefined);
+    if (record) deps.reportUsage(record);
+    if (response.stopReason === "aborted" || response.stopReason === "error") {
+      setStopReasonError(deps, response.stopReason, response.errorMessage, abortWire, timedOut, timeoutMs);
+      return out;
+    }
+    const text = response.content
+      .filter((c): c is TextContent => c.type === "text")
+      .map((c) => c.text)
+      .join("\n")
+      .trim();
+    if (!text) {
+      deps.setLastError("vision model returned an empty description");
+      return out;
+    }
+    const parsed = parseBatchedDescriptions(text, misses.length);
+    for (let i = 0; i < misses.length; i++) {
+      const d = parsed[i];
+      if (d) out.set(misses[i].hash, d);
+    }
+    // stopReason "length" = the batch response was cut off mid-stream. The last
+    // image being emitted when the cap hit is the one whose section is truncated
+    // (parseBatchedDescriptions still captures it via the end-of-text lookahead,
+    // just with partial content). Mark that one — the highest-index parsed slot —
+    // so the agent/user know it's incomplete. Earlier sections had `<<<END>>>`
+    // markers and are complete. We deliberately do NOT re-describe the truncated
+    // image: a single call for the same complex image would likely hit the same
+    // limit, so the partial text + marker is the honest, no-wasted-call result.
+    if (response.stopReason === "length") {
+      for (let i = misses.length - 1; i >= 0; i--) {
+        const h = misses[i].hash;
+        if (out.has(h)) {
+          out.set(h, markDescriptionTruncated(out.get(h)!));
+          break;
+        }
+      }
+    }
+    // Fallback: a failed delimiter-parse is NOT a failed description — it's a
+    // failed batching. Describe each unparsed image with its own single-image
+    // call (no delimiters to cooperate with). The calls run in parallel so
+    // their results still arrive together, not sequentially.
+    const unparsed = misses.filter((m) => !out.has(m.hash));
+    if (unparsed.length > 0) {
+      const fallbacks = await Promise.all(
+        unparsed.map((m) => describeSingle(m.img, userPrompt, visionModel, modelRegistry, cfg, deps, turnSignal)),
+      );
+      for (let i = 0; i < unparsed.length; i++) {
+        if (fallbacks[i]) out.set(unparsed[i].hash, fallbacks[i]!);
+      }
+    }
+    return out;
+  } catch (err) {
+    deps.setLastError(
+      timedOut ? `describer timed out after ${timeoutMs / 1000}s` : err instanceof Error ? err.message : String(err),
+    );
+    return out;
+  } finally {
+    // The `using` guards above already released the fetch interceptor, timer,
+    // and abort wire. Only the energy tee needs an explicit unhandled-rejection
+    // swallow: if the main stream aborted, the tee rejects too.
+    describeCtx.energyReader?.catch(() => {});
+  }
+}
+
+/** Describe a single image with one `complete()` call and return the RAW
+ *  description (no envelope, no truncation). Returns null on any genuine
+ *  failure (auth, abort/error, empty) so the caller only caches `UNAVAILABLE`
+ *  when a real describer attempt failed. */
+export async function describeSingle(
+  img: ExtractedImage,
+  userPrompt: string,
+  visionModel: Model<Api>,
+  modelRegistry: ModelRegistry,
+  cfg: VisionHandoffConfig,
+  deps: DescriberDeps,
+  turnSignal?: AbortSignal,
+): Promise<string | null> {
+  const auth = await modelRegistry.getApiKeyAndHeaders(visionModel);
+  if (!auth.ok || !auth.apiKey) {
+    deps.setLastError(
+      !auth.ok ? auth.error : `No API key for vision model "${visionModel.provider}/${visionModel.id}"`,
+    );
+    return null;
+  }
+  const prefix = cfg.userPromptPrefix ?? DEFAULT_USER_PROMPT_PREFIX;
+  const systemPrompt = cfg.prompt ?? DEFAULT_VISION_PROMPT;
+  const content: (TextContent | ImageContent)[] = [
+    { type: "text", text: batchUserPrompt(1, userPrompt, prefix) },
+    { type: "image", data: img.data, mimeType: img.mimeType } satisfies ImageContent,
+  ];
+  const userMessage: Message = { role: "user", content, timestamp: Date.now() };
+  const timeoutMs = describeTimeoutMs(1);
+  const maxTokens = resolveMaxTokens(cfg, visionModel);
+  const controller = new AbortController();
+  let timedOut = false;
+  using fetchGuard = fetchInterceptorGuard();
+  using timer = timeoutGuard(timeoutMs, () => {
+    timedOut = true;
+    controller.abort();
+  });
+  using abortWire = abortWireGuard(turnSignal, controller);
+
+  const describeCtx: DescribeContext = { energyReader: undefined };
+  try {
+    const response = await describeAls.run(describeCtx, async () =>
+      complete(
+        visionModel,
+        { systemPrompt, messages: [userMessage] },
+        { apiKey: auth.apiKey, headers: auth.headers, signal: controller.signal, maxTokens },
+      ),
+    );
+    const capture = await readCapture(describeCtx);
+    const hash = imageHash(img.mimeType, img.data);
+    const record = buildUsageRecord(response, capture, visionModel, hash);
+    if (record) deps.reportUsage(record);
+    if (response.stopReason === "aborted" || response.stopReason === "error") {
+      setStopReasonError(deps, response.stopReason, response.errorMessage, abortWire, timedOut, timeoutMs);
+      return null;
+    }
+    const text = response.content
+      .filter((c): c is TextContent => c.type === "text")
+      .map((c) => c.text)
+      .join("\n")
+      .trim();
+    if (!text) {
+      deps.setLastError("vision model returned an empty description");
+      return null;
+    }
+    // stopReason "length" = the model hit a token limit (configured maxTokens or
+    // the provider's hard output cap) before finishing. The partial text is still
+    // useful, but it must not pass as complete — mark it so the agent/user know.
+    return response.stopReason === "length" ? markDescriptionTruncated(text) : text;
+  } catch (err) {
+    deps.setLastError(
+      timedOut ? `describer timed out after ${timeoutMs / 1000}s` : err instanceof Error ? err.message : String(err),
+    );
+    return null;
+  } finally {
+    describeCtx.energyReader?.catch(() => {});
+  }
+}
+
+/** Read the energy tee for a describer call, if one was captured. Returns the
+ *  empty capture when there is no reader (non-Neuralwatt models) or the tee
+ *  aborted with the main stream. */
+async function readCapture(describeCtx: DescribeContext): Promise<VisionHandoffEnergyCapture> {
+  if (!describeCtx.energyReader) return EMPTY_ENERGY_CAPTURE;
+  try {
+    return await describeCtx.energyReader;
+  } catch {
+    return EMPTY_ENERGY_CAPTURE;
+  }
+}
+
+/** Translate a non-OK stopReason into a user-facing failure message, unless the
+ *  abort came from the user cancelling the turn (then stay silent — no warning
+ *  for a deliberate cancel). */
+function setStopReasonError(
+  deps: DescriberDeps,
+  stopReason: string,
+  errorMessage: string | undefined,
+  abortWire: AbortWire,
+  timedOut: boolean,
+  timeoutMs: number,
+): void {
+  if (abortWire.userAborted()) return; // user cancelled the turn — no warning
+  if (timedOut) {
+    deps.setLastError(`describer timed out after ${timeoutMs / 1000}s`);
+    return;
+  }
+  deps.setLastError(`vision model returned stopReason "${stopReason}"${errorMessage ? ": " + errorMessage : ""}`);
+}

package/src/dispose.ts

@@ -0,0 +1,61 @@
+/**
+ * `Disposable` guard factories for the `using` keyword (Explicit Resource
+ * Management). Each factory acquires a resource and returns a `Disposable`
+ * whose `[Symbol.dispose]` releases it, so a `using` binding replaces a
+ * manual acquire/`try`/`finally`/release pair.
+ *
+ * Used by the describer to bundle the fetch interceptor, the timeout timer,
+ * and the turn-abort wire into lexical scopes whose cleanup is automatic.
+ */
+
+import { installFetchInterceptor, uninstallFetchInterceptor } from "./usage.js";
+
+/** A `Disposable` that uninstalls the refcounted fetch interceptor on release. */
+export function fetchInterceptorGuard(): Disposable {
+  installFetchInterceptor();
+  return {
+    [Symbol.dispose]() {
+      uninstallFetchInterceptor();
+    },
+  };
+}
+
+/** A `Disposable` that clears a `setTimeout` handle on release. */
+export function timeoutGuard(ms: number, onTimeout: () => void): Disposable {
+  const handle = setTimeout(onTimeout, ms);
+  return {
+    [Symbol.dispose]() {
+      clearTimeout(handle);
+    },
+  };
+}
+
+/** A `Disposable` abort-wire: propagates a turn abort signal into a
+ *  describer's `AbortController`, and detaches the listener on release.
+ *  Also exposes `userAborted()` so the describer can tell a deliberate user
+ *  cancel apart from a provider/timeout abort (to suppress spurious warnings).
+ *
+ *  Always returns a `Disposable` (a no-op when there is no turn signal) so a
+ *  `using` binding never has to null-check. */
+export interface AbortWire extends Disposable {
+  /** True iff the abort originated from the turn signal (a user cancel). */
+  userAborted(): boolean;
+}
+
+export function abortWireGuard(turnSignal: AbortSignal | undefined, controller: AbortController): AbortWire {
+  if (!turnSignal) {
+    return {
+      [Symbol.dispose]() {},
+      userAborted: () => false,
+    };
+  }
+  const onAbort = () => controller.abort();
+  if (turnSignal.aborted) controller.abort();
+  else turnSignal.addEventListener("abort", onAbort, { once: true });
+  return {
+    [Symbol.dispose]() {
+      turnSignal.removeEventListener("abort", onAbort);
+    },
+    userAborted: () => turnSignal.aborted,
+  };
+}

package/src/image.ts

@@ -0,0 +1,274 @@
+/**
+ * Pure image IO utilities: MIME sniffing, dimension parsing, and resolving a
+ * pasted clipboard image file to the SAME {@link ExtractedImage} pi's `read`
+ * tool will emit — so a pre-warm's cache key matches the later `tool_result`'s
+ * key (no wasted vision call).
+ *
+ * Why this matters: pi's `read` tool runs `resizeImage` (on by default). For a
+ * small image (≤2000×2000 AND <4.5MB base64) the resize is a no-op that returns
+ * the raw input bytes unchanged — so our raw read matches. For an oversized
+ * image pi RE-ENCODES (Photon resize + possible JPEG@80), producing different
+ * bytes; pre-warming the raw file would then cache-miss at `tool_result` time
+ * and waste a vision call. {@link willBeResized} mirrors pi's threshold so we
+ * run the same {@link resolvePrewarmImage} pipeline and match the key in both
+ * cases.
+ *
+ * MIME sniffing is aligned with pi's `detectSupportedImageMimeType` (incl.
+ * rejecting animated PNG, which pi treats as a non-image → text) so our sniff
+ * agrees with pi on whether a file is an image at all.
+ */
+
+import { readFileSync } from "node:fs";
+import { tmpdir } from "node:os";
+import { isAbsolute, join, sep } from "node:path";
+import crypto from "node:crypto";
+import type { ExtractedImage } from "./index.js";
+
+// pi's resize defaults (see @earendil-works/pi-coding-agent utils/image-resize).
+// The `read` tool returns raw input bytes unchanged when the image fits BOTH
+// the dimension limit AND the base64-payload size limit; otherwise it resizes.
+const RESIZE_MAX_WIDTH = 2000;
+const RESIZE_MAX_HEIGHT = 2000;
+const RESIZE_MAX_BYTES = 4.5 * 1024 * 1024; // base64 payload size
+
+const PNG_SIGNATURE = [0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a];
+
+/** Stable hash of an image's MIME + base64 data, used as the dataloader key. */
+export function imageHash(mimeType: string, data: string): string {
+  return crypto.createHash("sha256").update(`${mimeType}\x00${data}`).digest("hex").slice(0, 32);
+}
+
+function readUint32BE(buf: Buffer, offset: number): number {
+  return (
+    (buf[offset] ?? 0) * 0x1000000 +
+    ((buf[offset + 1] ?? 0) << 16) +
+    ((buf[offset + 2] ?? 0) << 8) +
+    (buf[offset + 3] ?? 0)
+  );
+}
+
+function readUint16BE(buf: Buffer, offset: number): number {
+  return ((buf[offset] ?? 0) << 8) + (buf[offset + 1] ?? 0);
+}
+
+function readUint16LE(buf: Buffer, offset: number): number {
+  return (buf[offset] ?? 0) + ((buf[offset + 1] ?? 0) << 8);
+}
+
+function startsWithAscii(buf: Buffer, offset: number, text: string): boolean {
+  for (let i = 0; i < text.length; i++) {
+    if (buf[offset + i] !== text.charCodeAt(i)) return false;
+  }
+  return true;
+}
+
+function isPng(buf: Buffer): boolean {
+  // IHDR chunk: length == 13 at offset 8, "IHDR" at offset 12.
+  return buf.length >= 16 && readUint32BE(buf, PNG_SIGNATURE.length) === 13 && startsWithAscii(buf, 12, "IHDR");
+}
+
+function isAnimatedPng(buf: Buffer): boolean {
+  // Scan chunks for "acTL" (animation control) before "IDAT". Matches pi's
+  // detectSupportedImageMimeType so an animated PNG is treated as a non-image
+  // (pi reads it as text) and we skip pre-warming it.
+  let offset = PNG_SIGNATURE.length;
+  while (offset + 8 <= buf.length) {
+    const chunkLength = readUint32BE(buf, offset);
+    const typeOffset = offset + 4;
+    if (startsWithAscii(buf, typeOffset, "acTL")) return true;
+    if (startsWithAscii(buf, typeOffset, "IDAT")) return false;
+    const next = offset + 8 + chunkLength + 4;
+    if (next <= offset || next > buf.length) return false;
+    offset = next;
+  }
+  return false;
+}
+
+/** Sniff an image MIME type from magic bytes, aligned with pi's
+ *  detectSupportedImageMimeType. Returns null for unsupported/animated PNG. */
+export function sniffImageMime(buf: Buffer): string | null {
+  if (buf.length >= 3 && buf[0] === 0xff && buf[1] === 0xd8 && buf[2] === 0xff) {
+    return buf[3] === 0xf7 ? null : "image/jpeg";
+  }
+  if (buf.length >= 8 && PNG_SIGNATURE.every((b, i) => buf[i] === b)) {
+    return isPng(buf) && !isAnimatedPng(buf) ? "image/png" : null;
+  }
+  if (buf.length >= 3 && startsWithAscii(buf, 0, "GIF")) return "image/gif";
+  if (buf.length >= 12 && startsWithAscii(buf, 0, "RIFF") && startsWithAscii(buf, 8, "WEBP")) {
+    return "image/webp";
+  }
+  return null;
+}
+
+/** Parse an image's pixel dimensions from its headers (no decode). Returns null
+ *  for an unsupported/unparseable format. The dimension check is orientation-
+ *  invariant (`max(w,h) > 2000` is unchanged by EXIF rotation swaps), so it
+ *  agrees with pi's resize decision even for EXIF-oriented images. */
+export function imageDimensions(buf: Buffer, mimeType: string): { width: number; height: number } | null {
+  switch (mimeType) {
+    case "image/png": {
+      // IHDR: width (4 BE) at offset 16, height (4 BE) at offset 20.
+      if (buf.length < 24) return null;
+      return { width: readUint32BE(buf, 16), height: readUint32BE(buf, 20) };
+    }
+    case "image/gif": {
+      // Logical screen descriptor: width (2 LE) at 6, height (2 LE) at 8.
+      if (buf.length < 10) return null;
+      return { width: readUint16LE(buf, 6), height: readUint16LE(buf, 8) };
+    }
+    case "image/jpeg": {
+      return jpegDimensions(buf);
+    }
+    case "image/webp": {
+      return webpDimensions(buf);
+    }
+    default:
+      return null;
+  }
+}
+
+/** Scan JPEG segments for a SOF marker and read width/height. */
+function jpegDimensions(buf: Buffer): { width: number; height: number } | null {
+  if (buf.length < 4 || buf[0] !== 0xff || buf[1] !== 0xd8) return null;
+  let offset = 2;
+  while (offset + 9 <= buf.length) {
+    if (buf[offset] !== 0xff) return null;
+    // Skip fill bytes.
+    let marker = buf[offset + 1];
+    while (marker === 0xff && offset + 2 < buf.length) {
+      offset++;
+      marker = buf[offset + 1];
+    }
+    // Standalone markers (RSTn, SOI, EOI) have no length payload.
+    if (marker === 0xd8 || marker === 0xd9 || (marker >= 0xd0 && marker <= 0xd7)) {
+      offset += 2;
+      continue;
+    }
+    if (offset + 4 > buf.length) return null;
+    const segLen = readUint16BE(buf, offset + 2);
+    // SOF0–SOF15 (excluding RST/sof-defined non-SOF): C0–CF except C4 (DHT),
+    // C8 (JPG), CC (DAC).
+    if (marker >= 0xc0 && marker <= 0xcf && marker !== 0xc4 && marker !== 0xc8 && marker !== 0xcc) {
+      if (offset + 9 > buf.length) return null;
+      const height = readUint16BE(buf, offset + 5);
+      const width = readUint16BE(buf, offset + 7);
+      return { width, height };
+    }
+    // SOS: image data follows, no more SOF.
+    if (marker === 0xda) return null;
+    offset += 2 + segLen;
+  }
+  return null;
+}
+
+/** Parse WEBP dimensions across VP8 (lossy), VP8L (lossless), VP8X (extended). */
+function webpDimensions(buf: Buffer): { width: number; height: number } | null {
+  if (buf.length < 30 || !startsWithAscii(buf, 0, "RIFF") || !startsWithAscii(buf, 8, "WEBP")) return null;
+  const fourcc = buf.subarray(12, 16).toString("ascii");
+  if (fourcc === "VP8 ") {
+    // Lossy: 3-byte frame tag at 20, 3-byte start code at 23, width (2 LE &
+    // 0x3FFF) at 26, height (2 LE & 0x3FFF) at 28.
+    return { width: readUint16LE(buf, 26) & 0x3fff, height: readUint16LE(buf, 28) & 0x3fff };
+  }
+  if (fourcc === "VP8L") {
+    // Lossless: 0x2F signature at 20, then 14-bit (width-1) + 14-bit (height-1)
+    // packed LSB-first across bytes 21–24.
+    if (buf[20] !== 0x2f) return null;
+    const val = (buf[21] ?? 0) | ((buf[22] ?? 0) << 8) | ((buf[23] ?? 0) << 16) | ((buf[24] ?? 0) << 24);
+    return { width: 1 + (val & 0x3fff), height: 1 + ((val >> 14) & 0x3fff) };
+  }
+  if (fourcc === "VP8X") {
+    // Extended: canvas width-1 (24-bit LE) at 24, height-1 (24-bit LE) at 27.
+    const w = 1 + ((buf[24] ?? 0) | ((buf[25] ?? 0) << 8) | ((buf[26] ?? 0) << 16));
+    const h = 1 + ((buf[27] ?? 0) | ((buf[28] ?? 0) << 8) | ((buf[29] ?? 0) << 16));
+    return { width: w, height: h };
+  }
+  return null;
+}
+
+/** Base64-encoded size of `byteLength` raw bytes (4/3 ratio, ceil). */
+function base64Size(byteLength: number): number {
+  return Math.ceil(byteLength / 3) * 4;
+}
+
+/** Whether pi's `read` tool would RESIZE (re-encode) this image. When false,
+ *  pi returns the raw input bytes unchanged — so a raw pre-warm matches the
+ *  `tool_result` key. When true, pi re-encodes — the caller must run the same
+ *  resize pipeline to match. Mirrors pi's resize threshold exactly. Unknown
+ *  dimensions default to `true` (force the resize path, which always matches). */
+export function willBeResized(buf: Buffer, mimeType: string): boolean {
+  const dims = imageDimensions(buf, mimeType);
+  if (!dims) return true;
+  return (
+    dims.width > RESIZE_MAX_WIDTH ||
+    dims.height > RESIZE_MAX_HEIGHT ||
+    base64Size(buf.length) >= RESIZE_MAX_BYTES
+  );
+}
+
+/** A resize function shaped like pi's `resizeImage` (injected so this module
+ *  stays free of the pi-coding-agent dependency and fully unit-testable). */
+export type ResizeFn = (
+  inputBytes: Uint8Array,
+  mimeType: string,
+) => Promise<{ data: string; mimeType: string } | null>;
+
+/** Resolve a clipboard image buffer to the SAME {@link ExtractedImage} pi's
+ *  `read` tool will emit, so the pre-warm's cache key matches the later
+ *  `tool_result`'s key (no wasted vision call):
+ *  - no-resize case (≤2000² & <4.5MB base64): return the raw bytes — pi's
+ *    no-resize path returns `inputBytes` unchanged.
+ *  - resize case: run the same `resize` pipeline (pi's `resizeImage`) and
+ *    return the re-encoded data — matches pi's resize path.
+ *  Returns null if resize failed (pi then emits no image block, so there is
+ *  nothing to pre-warm) or if the bytes aren't a supported image. */
+export async function resolvePrewarmImage(
+  buf: Buffer,
+  mimeType: string,
+  resize: ResizeFn,
+): Promise<ExtractedImage | null> {
+  if (!willBeResized(buf, mimeType)) {
+    return { data: buf.toString("base64"), mimeType };
+  }
+  const resized = await resize(buf, mimeType);
+  if (!resized) return null;
+  return { data: resized.data, mimeType: resized.mimeType };
+}
+
+/** Read an image file into its raw buffer + sniffed MIME. Returns null if the
+ *  file can't be read or isn't a supported image (aligned with pi's sniff). */
+export function readImageBuffer(filePath: string): { buf: Buffer; mimeType: string } | null {
+  try {
+    const buf = readFileSync(filePath);
+    const mimeType = sniffImageMime(buf);
+    if (!mimeType) return null;
+    return { buf, mimeType };
+  } catch {
+    return null;
+  }
+}
+
+/** Regex matching pi's pasted-clipboard temp image file paths anywhere in the
+ *  prompt text. pi writes pasted clipboard images to
+ *  `<tmpdir>/pi-clipboard-<uuid>.<ext>` and inserts the path as text at the
+ *  cursor, so on a non-vision model these arrive as path tokens in the user
+ *  prompt — NOT as `event.images`. Matching them lets the extension pre-warm
+ *  the describer at paste-enter (concurrent with the agent's first response)
+ *  instead of waiting for the agent to `read` them. */
+const CLIPBOARD_IMAGE_PATH_RE = /(\S*pi-clipboard-[^\s]+\.(?:png|jpe?g|gif|webp))/gi;
+
+/** Extract pasted clipboard image file paths from a prompt. Confined to the
+ *  OS temp directory so an attacker-crafted prompt can't trick the extension
+ *  into reading arbitrary files — only pi's own clipboard temp files qualify. */
+export function findClipboardImagePaths(prompt: string): string[] {
+  const tmp = tmpdir();
+  const paths = new Set<string>();
+  for (const m of prompt.matchAll(CLIPBOARD_IMAGE_PATH_RE)) {
+    const p = m[1];
+    if (!p) continue;
+    const abs = isAbsolute(p) ? p : join(tmp, p);
+    // Ensure the resolved candidate stays inside the temp directory.
+    if (abs.startsWith(tmp + sep) || abs === tmp) paths.add(abs);
+  }
+  return [...paths];
+}