pi-paster 0.1.5 → 0.2.1

package/src/index.ts

@@ -2,13 +2,14 @@ import type { ExtensionAPI } from "@earendil-works/pi-coding-agent";
 import { readClipboardImage } from "./clipboard.ts";
 import { type PasterConfig, resolvePasterConfig } from "./config.ts";
 import { PasterEditor } from "./editor.ts";
-import { imagesForText } from "./image-utils.ts";
+import { appendImagePathContext, imagesForTextOptimized } from "./image-utils.ts";
 import { CursorImagePreviewWidget, ImagePreviewMessage } from "./preview.ts";
 import { AttachmentStore } from "./store.ts";
 import { createImagePasteTerminalInputHandler } from "./terminal-input.ts";
 import type { ImageAttachment, PasterPreviewDetails } from "./types.ts";
 
 export * from "./clipboard.ts";
+export * from "./optimize-image.ts";
 export * from "./config.ts";
 export * from "./editor.ts";
 export * from "./image-utils.ts";
@@ -28,15 +29,22 @@ export default function paster(pi: ExtensionAPI, config: PasterConfig = {}): voi
   let activeEditor: PasterEditor | undefined;
   let unsubscribeTerminalInput: (() => void) | undefined;
 
-  pi.registerMessageRenderer<PasterPreviewDetails>("paster-preview", (message, _options, theme) => {
+  pi.registerMessageRenderer<PasterPreviewDetails>("paster-preview", (message, options, theme) => {
     const placeholders = message.details?.placeholders ?? [];
     const attachments = store
       .list()
       .filter((attachment) => placeholders.includes(attachment.placeholder));
     if (attachments.length === 0) return undefined;
-    return new ImagePreviewMessage(attachments, {
-      fallbackColor: (text) => theme.fg("muted", text),
-    });
+    return new ImagePreviewMessage(
+      attachments,
+      {
+        fallbackColor: (text) => theme.fg("muted", text),
+        background: (text) => theme.bg("toolSuccessBg", text),
+        title: (text) => theme.fg("toolTitle", theme.bold(text)),
+        muted: (text) => theme.fg("muted", text),
+      },
+      { expanded: options.expanded, style: resolvedConfig.submittedPreviewStyle },
+    );
   });
 
   pi.on("session_start", (_event, ctx) => {
@@ -111,15 +119,16 @@ export default function paster(pi: ExtensionAPI, config: PasterConfig = {}): voi
   });
 
   function previewMessage(attachments: ImageAttachment[]) {
+    const placeholders = attachments.map((attachment) => attachment.placeholder);
     return {
       customType: "paster-preview",
-      content: "",
+      content: `(attachment preview: ${placeholders.join(", ")})`,
       display: true,
-      details: { placeholders: attachments.map((attachment) => attachment.placeholder) },
+      details: { placeholders },
     };
   }
 
-  pi.on("input", (event, ctx) => {
+  pi.on("input", async (event, ctx) => {
     if (event.source === "extension") return { action: "continue" as const };
     if (ctx.hasUI) {
       activeEditor?.clearCursorPreview();
@@ -136,10 +145,18 @@ export default function paster(pi: ExtensionAPI, config: PasterConfig = {}): voi
       pi.sendMessage(previewMessage(attachments), { deliverAs: "followUp" });
     }
 
+    // Optimize images on-submit so we never exceed Anthropic's 5 MB/image or
+    // 32 MB/request caps. Per-attachment caching means each image is only
+    // resized/recompressed once across the whole session.
+    const images = await imagesForTextOptimized(store, event.text, event.images);
+    const text = resolvedConfig.includeImagePathsInPrompt
+      ? appendImagePathContext(event.text, attachments)
+      : event.text;
+
     return {
       action: "transform" as const,
-      text: event.text,
-      images: imagesForText(store, event.text, event.images),
+      text,
+      images,
     };
   });
 

package/src/optimize-image.ts

@@ -0,0 +1,209 @@
+/**
+ * Image optimizer for pi-paster.
+ *
+ * Anthropic Messages API hard limits:
+ *   - 5 MB per image (base64-decoded)
+ *   - 8000 px on any dimension
+ *   - 32 MB per request total
+ *
+ * Claude internally downsamples every input image so the long edge is
+ *   - 1568 px for most models
+ *   - 2576 px for Opus 4.7
+ * which means: any client-side downscale at or above ~2600 px is lossless
+ * from the model's point of view. We resize (aspect-preserving) instead of
+ * cropping so no visual information is thrown away.
+ *
+ * Strategy applied in order:
+ *   1. If width or height > 8000 px → resize so the long edge is 8000 px.
+ *   2. If decoded bytes > 5 MB → re-encode as JPEG q=95, then 90/85/80/70/60.
+ *   3. If still > 5 MB → progressively shrink the long edge to 6000 → 4000
+ *      → 3000 → 2000 px, re-running JPEG quality ladder each step.
+ *
+ * `sharp` is loaded lazily so the extension can still ship if the native
+ * binding is unavailable (we just skip optimization with a log line).
+ */
+import type { SupportedImageMimeType } from "./types.ts";
+import { ANTHROPIC_MAX_DIMENSION, ANTHROPIC_MAX_IMAGE_BYTES } from "./types.ts";
+
+type Sharp = (input?: Buffer) => SharpInstance;
+interface SharpInstance {
+  metadata(): Promise<{ width?: number; height?: number; format?: string }>;
+  resize(opts: {
+    width?: number;
+    height?: number;
+    fit?: string;
+    withoutEnlargement?: boolean;
+  }): SharpInstance;
+  jpeg(opts: { quality: number; mozjpeg?: boolean }): SharpInstance;
+  png(opts?: { quality?: number; compressionLevel?: number }): SharpInstance;
+  toBuffer(): Promise<Buffer>;
+}
+
+let _sharp: Sharp | null | undefined;
+async function getSharp(): Promise<Sharp | null> {
+  if (_sharp !== undefined) return _sharp;
+  try {
+    // eslint-disable-next-line @typescript-eslint/no-explicit-any
+    const mod: any = await import("sharp");
+    const fn = (typeof mod === "function" ? mod : mod?.default) as Sharp | undefined;
+    _sharp = typeof fn === "function" ? fn : null;
+  } catch {
+    _sharp = null;
+  }
+  return _sharp;
+}
+
+export interface OptimizeResult {
+  data: string; // base64
+  mimeType: SupportedImageMimeType;
+  originalBytes: number;
+  finalBytes: number;
+  originalDim?: { width: number; height: number };
+  finalDim?: { width: number; height: number };
+  actions: string[];
+  changed: boolean;
+}
+
+const SHRINK_LADDER = [6000, 4000, 3000, 2000];
+const JPEG_QUALITY_LADDER = [95, 90, 85, 80, 70, 60];
+
+export async function optimizeImageBytes(
+  input: Buffer,
+  mime: SupportedImageMimeType,
+): Promise<OptimizeResult> {
+  const originalBytes = input.length;
+  const noop = (): OptimizeResult => ({
+    data: input.toString("base64"),
+    mimeType: mime,
+    originalBytes,
+    finalBytes: originalBytes,
+    actions: [],
+    changed: false,
+  });
+
+  // Fast path: already under both limits.
+  if (originalBytes <= ANTHROPIC_MAX_IMAGE_BYTES) {
+    // Still need dimension check, but cheap to skip sharp if image is small.
+    if (originalBytes <= 256 * 1024) return noop();
+  }
+
+  const sharp = await getSharp();
+  if (!sharp) return noop(); // sharp missing — fall back silently.
+
+  const meta = await sharp(input).metadata();
+  const origW = meta.width ?? 0;
+  const origH = meta.height ?? 0;
+  if (!origW || !origH) return noop();
+
+  const actions: string[] = [];
+  const originalDim = { width: origW, height: origH };
+
+  // Step 1: respect 8000 px dimension cap (resize, do not crop).
+  let workBuf = input;
+  let workW = origW;
+  let workH = origH;
+  if (workW > ANTHROPIC_MAX_DIMENSION || workH > ANTHROPIC_MAX_DIMENSION) {
+    workBuf = await sharp(workBuf)
+      .resize({
+        width: workW >= workH ? ANTHROPIC_MAX_DIMENSION : undefined,
+        height: workH > workW ? ANTHROPIC_MAX_DIMENSION : undefined,
+        fit: "inside",
+        withoutEnlargement: true,
+      })
+      .toBuffer();
+    const m = await sharp(workBuf).metadata();
+    workW = m.width ?? workW;
+    workH = m.height ?? workH;
+    actions.push(`resize to ${workW}x${workH} (8000px cap)`);
+  }
+
+  // Step 2: bytes within limit? done.
+  if (workBuf.length <= ANTHROPIC_MAX_IMAGE_BYTES && actions.length === 0) return noop();
+  if (workBuf.length <= ANTHROPIC_MAX_IMAGE_BYTES) {
+    return {
+      data: workBuf.toString("base64"),
+      mimeType: mime,
+      originalBytes,
+      finalBytes: workBuf.length,
+      originalDim,
+      finalDim: { width: workW, height: workH },
+      actions,
+      changed: true,
+    };
+  }
+
+  // Step 3: JPEG quality ladder.
+  let outMime: SupportedImageMimeType = "image/jpeg";
+  let attempt = workBuf;
+  for (const q of JPEG_QUALITY_LADDER) {
+    attempt = await sharp(workBuf).jpeg({ quality: q, mozjpeg: true }).toBuffer();
+    if (attempt.length <= ANTHROPIC_MAX_IMAGE_BYTES) {
+      actions.push(`jpeg q=${q} → ${formatBytes(attempt.length)}`);
+      return {
+        data: attempt.toString("base64"),
+        mimeType: outMime,
+        originalBytes,
+        finalBytes: attempt.length,
+        originalDim,
+        finalDim: { width: workW, height: workH },
+        actions,
+        changed: true,
+      };
+    }
+  }
+  actions.push(`jpeg q=60 still ${formatBytes(attempt.length)} — shrinking`);
+
+  // Step 4: shrink the long edge and retry quality ladder each step.
+  for (const longEdge of SHRINK_LADDER) {
+    if (Math.max(workW, workH) <= longEdge) continue;
+    const resized = await sharp(workBuf)
+      .resize({
+        width: workW >= workH ? longEdge : undefined,
+        height: workH > workW ? longEdge : undefined,
+        fit: "inside",
+        withoutEnlargement: true,
+      })
+      .toBuffer();
+    const m = await sharp(resized).metadata();
+    const newW = m.width ?? workW;
+    const newH = m.height ?? workH;
+    for (const q of JPEG_QUALITY_LADDER) {
+      attempt = await sharp(resized).jpeg({ quality: q, mozjpeg: true }).toBuffer();
+      if (attempt.length <= ANTHROPIC_MAX_IMAGE_BYTES) {
+        actions.push(`resize ${newW}x${newH} + jpeg q=${q} → ${formatBytes(attempt.length)}`);
+        return {
+          data: attempt.toString("base64"),
+          mimeType: outMime,
+          originalBytes,
+          finalBytes: attempt.length,
+          originalDim,
+          finalDim: { width: newW, height: newH },
+          actions,
+          changed: true,
+        };
+      }
+    }
+    workBuf = resized;
+    workW = newW;
+    workH = newH;
+  }
+
+  // Give up — return last attempt anyway; pi will at least try.
+  actions.push(`final ${formatBytes(attempt.length)} — over limit`);
+  return {
+    data: attempt.toString("base64"),
+    mimeType: outMime,
+    originalBytes,
+    finalBytes: attempt.length,
+    originalDim,
+    finalDim: { width: workW, height: workH },
+    actions,
+    changed: true,
+  };
+}
+
+function formatBytes(bytes: number): string {
+  if (bytes >= 1024 * 1024) return `${(bytes / 1024 / 1024).toFixed(2)}MB`;
+  if (bytes >= 1024) return `${(bytes / 1024).toFixed(0)}KB`;
+  return `${bytes}B`;
+}

package/src/preview.ts

@@ -1,8 +1,12 @@
 import {
+  Box,
+  Container,
   getCellDimensions,
   Image,
   type Component,
   type ImageTheme,
+  Spacer,
+  Text,
   truncateToWidth,
   visibleWidth,
 } from "@earendil-works/pi-tui";
@@ -18,12 +22,21 @@ function formatAttachmentLine(
   return visibleWidth(line) > maxWidth ? truncateToWidth(line, maxWidth, "") : line;
 }
 
+export type ImagePreviewMessageStyle = "raw" | "collapsible";
+
+interface ImagePreviewMessageTheme extends ImageTheme {
+  background?: (text: string) => string;
+  title?: (text: string) => string;
+  muted?: (text: string) => string;
+}
+
 export class ImagePreviewMessage implements Component {
   private readonly images: Image[];
 
   constructor(
     private readonly attachments: ImageAttachment[],
-    private readonly theme: ImageTheme,
+    private readonly theme: ImagePreviewMessageTheme,
+    private readonly options: { expanded?: boolean; style?: ImagePreviewMessageStyle } = {},
   ) {
     this.images = attachments.map(
       (attachment) =>
@@ -35,18 +48,54 @@ export class ImagePreviewMessage implements Component {
     );
   }
 
+  invalidate(): void {
+    for (const image of this.images) image.invalidate();
+  }
+
   render(width: number): string[] {
+    return this.options.style === "collapsible"
+      ? this.renderCollapsible(width)
+      : this.renderRaw(width);
+  }
+
+  private renderRaw(width: number): string[] {
     const lines: string[] = [];
+    const safeWidth = Math.max(1, width);
     for (let index = 0; index < this.attachments.length; index++) {
       const attachment = this.attachments[index]!;
-      lines.push(formatAttachmentLine(attachment, width, this.theme.fallbackColor));
-      lines.push(...this.images[index]!.render(width));
+      lines.push(formatAttachmentLine(attachment, safeWidth, this.theme.fallbackColor));
+      lines.push(...this.images[index]!.render(safeWidth));
     }
     return lines;
   }
 
-  invalidate(): void {
-    for (const image of this.images) image.invalidate();
+  private renderCollapsible(width: number): string[] {
+    const container = new Container();
+    container.addChild(new Spacer(1));
+    const box = new Box(1, 1, this.theme.background);
+    container.addChild(box);
+
+    const title = this.theme.title ?? this.theme.fallbackColor;
+    const muted = this.theme.muted ?? this.theme.fallbackColor;
+    const summary =
+      this.attachments.length === 1
+        ? `Attached ${this.attachments[0]!.placeholder}`
+        : `Attached ${this.attachments.length} images`;
+    const suffix = this.options.expanded ? " (ctrl+o to collapse)" : " (ctrl+o to expand)";
+    box.addChild(new Text(`${title(summary)}${muted(suffix)}`, 0, 0));
+
+    for (const attachment of this.attachments) {
+      box.addChild(new Text(formatAttachmentLine(attachment, width, muted), 0, 0));
+    }
+
+    const lines = container.render(width);
+    if (!this.options.expanded) return lines;
+
+    const safeWidth = Math.max(1, width);
+    for (let index = 0; index < this.attachments.length; index++) {
+      lines.push(...this.images[index]!.render(safeWidth));
+    }
+    return lines;
   }
 }
 

package/src/terminal-input.ts

@@ -1,5 +1,5 @@
 import { PASTE_END, PASTE_START } from "./editor.ts";
-import { replaceImagePathsInText } from "./image-utils.ts";
+import { describeReject, replaceImagePathsInText } from "./image-utils.ts";
 import type { AttachmentStore } from "./store.ts";
 import type { ImageAttachment, LoadImageResult } from "./types.ts";
 
@@ -19,11 +19,7 @@ export function createImagePasteTerminalInputHandler(options: {
       cwd: options.cwd,
       store: options.store,
       loadImage: options.loadImage,
-      onReject: (result) => {
-        if (result.reason === "too-large") {
-          options.notify?.(`paster: image is over 10 MB and was not attached: ${result.path}`);
-        }
-      },
+      onReject: (result) => describeReject(result, options.notify),
     });
 
   return (data: string): TerminalInputResult => {

package/src/types.ts

@@ -1,7 +1,17 @@
 import type { ImageDimensions } from "@earendil-works/pi-tui";
 
 export const EXTENSION_NAME = "paster";
-export const MAX_IMAGE_BYTES = 10 * 1024 * 1024;
+// Cap on the source file we read from disk. Larger inputs are rejected up
+// front with `too-large` so we never try to base64-encode a multi-gigabyte
+// file. Raised from the legacy 10 MB so high-resolution screenshots and raw
+// camera shots can be ingested and then shrunk by optimize-image.ts before
+// being attached.
+export const MAX_IMAGE_BYTES = 64 * 1024 * 1024;
+
+// Hard limits enforced by the Anthropic Messages API. Used by
+// optimize-image.ts to decide when an attachment needs to be shrunk.
+export const ANTHROPIC_MAX_DIMENSION = 8000;
+export const ANTHROPIC_MAX_IMAGE_BYTES = 5 * 1024 * 1024;
 
 export type SupportedImageMimeType = "image/png" | "image/jpeg" | "image/webp" | "image/gif";
 
@@ -13,6 +23,14 @@ export interface ImageAttachment {
   data: string;
   dimensions?: ImageDimensions;
   createdAt: number;
+  /** True once optimizeImageBytes has run on this attachment. */
+  optimized?: boolean;
+  /** Original (pre-optimization) base64 size in bytes — informational. */
+  originalBytes?: number;
+  /** Final (post-optimization) base64 size in bytes — informational. */
+  finalBytes?: number;
+  /** Human-readable trail of optimization actions applied, if any. */
+  optimizeActions?: string[];
 }
 
 export interface LoadedImage {