pi-paster 0.2.0 → 0.2.2

package/src/compress.ts

@@ -0,0 +1,329 @@
+import { mkdtempSync, rmSync, writeFileSync } from "node:fs";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import type { ExtensionAPI, ExtensionCommandContext } from "@earendil-works/pi-coding-agent";
+import type { ResolvedImageCompressionConfig } from "./config.ts";
+import type { ImageCompressionReportDetails } from "./types.ts";
+
+export const DEFAULT_IMAGE_SUMMARY_PROMPT =
+  "Summarize this image in 2-4 concise sentences. Include important visible text, UI elements, errors, diagrams, and details that may matter for future coding or design work.";
+
+interface TextBlock {
+  type: "text";
+  text: string;
+}
+
+interface ImageBlock {
+  type: "image";
+  data: string;
+  mimeType: string;
+}
+
+type ContentBlock = TextBlock | ImageBlock | Record<string, unknown>;
+
+type SessionEntryLike = {
+  type: string;
+  thinkingLevel?: string;
+  provider?: string;
+  modelId?: string;
+  summary?: string;
+  firstKeptEntryId?: string;
+  tokensBefore?: number;
+  details?: unknown;
+  data?: unknown;
+  name?: string;
+  fromHook?: boolean;
+  customType?: string;
+  content?: string | ContentBlock[];
+  display?: boolean;
+  message?: {
+    role?: string;
+    content?: string | ContentBlock[];
+    [key: string]: unknown;
+  };
+};
+
+interface ImageOccurrence {
+  key: string;
+  image: ImageBlock;
+}
+
+function isImageBlock(block: unknown): block is ImageBlock {
+  return (
+    !!block &&
+    typeof block === "object" &&
+    (block as { type?: unknown }).type === "image" &&
+    typeof (block as { data?: unknown }).data === "string" &&
+    typeof (block as { mimeType?: unknown }).mimeType === "string"
+  );
+}
+
+function imageKey(image: ImageBlock): string {
+  return `${image.mimeType}:${image.data.length}:${image.data.slice(0, 48)}`;
+}
+
+function collectImages(entries: SessionEntryLike[]): ImageOccurrence[] {
+  const images: ImageOccurrence[] = [];
+  for (const entry of entries) {
+    const content = entry.type === "custom_message" ? entry.content : entry.message?.content;
+    if (!Array.isArray(content)) continue;
+    for (const block of content) {
+      if (!isImageBlock(block)) continue;
+      images.push({ key: imageKey(block), image: block });
+    }
+  }
+  return images;
+}
+
+function extensionForMime(mimeType: string): string {
+  switch (mimeType) {
+    case "image/jpeg":
+      return ".jpg";
+    case "image/webp":
+      return ".webp";
+    case "image/gif":
+      return ".gif";
+    case "image/png":
+    default:
+      return ".png";
+  }
+}
+
+function normalizeSummary(stdout: string): string {
+  return stdout.trim().replace(/\n{3,}/g, "\n\n");
+}
+
+async function summarizeImage(
+  pi: ExtensionAPI,
+  ctx: ExtensionCommandContext,
+  image: ImageBlock,
+  config: ResolvedImageCompressionConfig,
+): Promise<string> {
+  const dir = mkdtempSync(join(tmpdir(), "pi-paster-image-compress-"));
+  try {
+    const imagePath = join(dir, `image${extensionForMime(image.mimeType)}`);
+    writeFileSync(imagePath, Buffer.from(image.data, "base64"));
+
+    const args = ["--no-session", "--no-extensions", "--no-tools", "--mode", "text", "-p"];
+    if (config.model) {
+      args.splice(0, 0, "--model", config.model);
+    }
+    args.push(`@${imagePath}`, config.prompt);
+
+    const result = await pi.exec(config.piCommand, args, {
+      cwd: ctx.cwd,
+      timeout: config.timeoutMs,
+    });
+    if (result.code !== 0) {
+      const detail = result.stderr.trim() || result.stdout.trim() || `exit code ${result.code}`;
+      throw new Error(`image summarization failed: ${detail}`);
+    }
+    const summary = normalizeSummary(result.stdout);
+    if (!summary) throw new Error("image summarization returned an empty response");
+    return summary;
+  } finally {
+    rmSync(dir, { recursive: true, force: true });
+  }
+}
+
+function summaryBlock(summary: string): TextBlock {
+  return { type: "text", text: `Image summary: ${summary}` };
+}
+
+function transformContent(
+  content: string | ContentBlock[] | undefined,
+  summaries: Map<string, string>,
+): string | ContentBlock[] | undefined {
+  if (!Array.isArray(content)) return content;
+  return content.map((block) => {
+    if (!isImageBlock(block)) return block;
+    return summaryBlock(summaries.get(imageKey(block)) ?? "Image summary unavailable.");
+  });
+}
+
+function buildReportDetails(
+  images: ImageOccurrence[],
+  summaries: Map<string, string>,
+): ImageCompressionReportDetails {
+  const items = [...summaries.values()].map((summary, index) => ({ index: index + 1, summary }));
+  return { imageCount: images.length, summaryCount: summaries.size, items };
+}
+
+function appendTransformedEntry(
+  sessionManager: unknown,
+  entry: SessionEntryLike,
+  summaries: Map<string, string>,
+) {
+  const manager = sessionManager as {
+    appendThinkingLevelChange(level: string): void;
+    appendModelChange(provider: string, modelId: string): void;
+    appendCustomEntry(customType: string, data?: unknown): void;
+    appendSessionInfo(name: string): void;
+    appendCustomMessageEntry(
+      customType: string,
+      content: string | ContentBlock[],
+      display: boolean,
+      details?: unknown,
+    ): void;
+    appendMessage(message: Record<string, unknown>): void;
+  };
+
+  switch (entry.type) {
+    case "thinking_level_change":
+      if (entry.thinkingLevel) manager.appendThinkingLevelChange(entry.thinkingLevel);
+      return;
+    case "model_change":
+      if (entry.provider && entry.modelId) manager.appendModelChange(entry.provider, entry.modelId);
+      return;
+    case "compaction":
+      if (entry.summary) {
+        manager.appendCustomMessageEntry(
+          "paster-image-compress-compaction-summary",
+          entry.summary,
+          false,
+          entry.details,
+        );
+      }
+      return;
+    case "branch_summary":
+      if (entry.summary) {
+        manager.appendCustomMessageEntry(
+          "paster-image-compress-branch-summary",
+          entry.summary,
+          false,
+          entry.details,
+        );
+      }
+      return;
+    case "custom":
+      if (entry.customType) manager.appendCustomEntry(entry.customType, entry.data);
+      return;
+    case "session_info":
+      if (typeof entry.name === "string") {
+        manager.appendSessionInfo(entry.name);
+      }
+      return;
+    case "custom_message":
+      if (entry.customType === "paster-preview") return;
+      if (entry.customType && entry.content !== undefined) {
+        manager.appendCustomMessageEntry(
+          entry.customType,
+          transformContent(entry.content, summaries) ?? "",
+          entry.display ?? true,
+          entry.details,
+        );
+      }
+      return;
+    case "message":
+      if (entry.message) {
+        manager.appendMessage({
+          ...entry.message,
+          content: transformContent(entry.message.content, summaries),
+        });
+      }
+      return;
+    default:
+      return;
+  }
+}
+
+function resolveCommandOptions(args: string, config: ResolvedImageCompressionConfig) {
+  const model = args.trim() || config.model;
+  return { ...config, model };
+}
+
+function setCompressionProgress(
+  ctx: ExtensionCommandContext,
+  current: number,
+  total: number,
+  model: string,
+): void {
+  if (!ctx.hasUI) return;
+  const progress = `Summarizing image ${current}/${total}`;
+  ctx.ui.setStatus("paster-image-compress", progress);
+  ctx.ui.setWidget(
+    "paster-image-compress-progress",
+    [`paster: ${progress}`, model ? `model: ${model}` : "model: pi default"],
+    { placement: "aboveEditor" },
+  );
+}
+
+function clearCompressionProgress(ctx: ExtensionCommandContext): void {
+  if (!ctx.hasUI) return;
+  ctx.ui.setStatus("paster-image-compress", undefined);
+  ctx.ui.setWidget("paster-image-compress-progress", undefined, { placement: "aboveEditor" });
+}
+
+export function registerImageCompressionCommand(
+  pi: ExtensionAPI,
+  config: ResolvedImageCompressionConfig,
+): void {
+  if (!config.enabled) return;
+
+  pi.registerCommand(config.command, {
+    description:
+      "Summarize images in the current branch and switch to a new session where image blocks are replaced with text summaries.",
+    handler: async (args, ctx): Promise<void> => {
+      await ctx.waitForIdle();
+      const commandConfig = resolveCommandOptions(args, config);
+      const branch = ctx.sessionManager.getBranch() as SessionEntryLike[];
+      const images = collectImages(branch);
+      if (images.length === 0) {
+        if (ctx.hasUI) ctx.ui.notify("No images found in the current branch", "warning");
+        return;
+      }
+
+      const uniqueImages = [...new Map(images.map((item) => [item.key, item.image])).entries()];
+      const summaries = new Map<string, string>();
+
+      if (ctx.hasUI) {
+        ctx.ui.notify(`Summarizing ${uniqueImages.length} image(s)...`, "info");
+      }
+      setCompressionProgress(ctx, 0, uniqueImages.length, commandConfig.model);
+
+      try {
+        for (let index = 0; index < uniqueImages.length; index++) {
+          const [key, image] = uniqueImages[index]!;
+          setCompressionProgress(ctx, index + 1, uniqueImages.length, commandConfig.model);
+          summaries.set(key, await summarizeImage(pi, ctx, image, commandConfig));
+        }
+      } catch (error) {
+        const reason = error instanceof Error ? error.message : String(error);
+        const modelHint = commandConfig.model || "pi's default model";
+        const message = `Image compression failed while using ${modelHint}. ${reason}\n\nTo try another model once, run /${commandConfig.command} provider/model. To change the default, configure pi-paster's imageCompression.model option in your wrapper extension. See https://github.com/beowulf11/pi-paster#configuration.`;
+        if (ctx.hasUI) ctx.ui.notify(message, "error");
+        else console.error(message);
+        return;
+      } finally {
+        clearCompressionProgress(ctx);
+      }
+
+      const parentSession = ctx.sessionManager.getSessionFile();
+      const reportDetails = buildReportDetails(images, summaries);
+      await ctx.newSession({
+        parentSession,
+        setup: async (sessionManager) => {
+          for (const entry of branch) appendTransformedEntry(sessionManager, entry, summaries);
+          if (commandConfig.includeReport) {
+            sessionManager.appendCustomMessageEntry(
+              "paster-image-compress-report",
+              "",
+              true,
+              reportDetails,
+            );
+          }
+        },
+        withSession: async (newCtx) => {
+          if (newCtx.hasUI) {
+            newCtx.ui.notify(
+              `Compressed ${images.length} image block(s) into text summaries`,
+              "info",
+            );
+          }
+        },
+      });
+
+      return;
+    },
+  });
+}

package/src/config.ts

@@ -1,4 +1,39 @@
+export type SubmittedPreviewStyle = "raw" | "collapsible";
+
+export interface ImageCompressionConfig {
+  /** Enable the image compression slash command. */
+  enabled?: boolean;
+  /** Slash command name without the leading slash. */
+  command?: string;
+  /** Model passed to pi for image summarization. Pass an empty string to use pi's default model. */
+  model?: string;
+  /** Prompt used to summarize each image. */
+  prompt?: string;
+  /** pi executable used for summarization subprocesses. */
+  piCommand?: string;
+  /** Per-image summarization timeout in milliseconds. */
+  timeoutMs?: number;
+  /** Add a visible, collapsible UI report after compression. */
+  includeReport?: boolean;
+}
+
+export interface ResolvedImageCompressionConfig {
+  enabled: boolean;
+  command: string;
+  model: string;
+  prompt: string;
+  piCommand: string;
+  timeoutMs: number;
+  includeReport: boolean;
+}
+
 export interface PasterConfig {
+  /** How submitted attachment previews render in chat history. */
+  submittedPreviewStyle?: SubmittedPreviewStyle;
+  /** Append local image paths to the submitted prompt so the agent can manipulate the source files. */
+  includeImagePathsInPrompt?: boolean;
+  /** Configure the /image-compress command. */
+  imageCompression?: ImageCompressionConfig;
   customEditor?: {
     /** Replace pi's input editor to enable inline image UX features. */
     enabled?: boolean;
@@ -10,6 +45,9 @@ export interface PasterConfig {
 }
 
 export interface ResolvedPasterConfig {
+  submittedPreviewStyle: SubmittedPreviewStyle;
+  includeImagePathsInPrompt: boolean;
+  imageCompression: ResolvedImageCompressionConfig;
   customEditor: {
     enabled: boolean;
     showImagePreview: boolean;
@@ -18,6 +56,18 @@ export interface ResolvedPasterConfig {
 }
 
 export const DEFAULT_PASTER_CONFIG: ResolvedPasterConfig = {
+  submittedPreviewStyle: "raw",
+  includeImagePathsInPrompt: true,
+  imageCompression: {
+    enabled: true,
+    command: "image-compress",
+    model: "openai-codex/gpt-5.4-mini",
+    prompt:
+      "Summarize this image in 2-4 concise sentences. Include important visible text, UI elements, errors, diagrams, and details that may matter for future coding or design work.",
+    piCommand: "pi",
+    timeoutMs: 120_000,
+    includeReport: true,
+  },
   customEditor: {
     enabled: true,
     showImagePreview: true,
@@ -27,6 +77,23 @@ export const DEFAULT_PASTER_CONFIG: ResolvedPasterConfig = {
 
 export function resolvePasterConfig(config: PasterConfig = {}): ResolvedPasterConfig {
   return {
+    submittedPreviewStyle:
+      config.submittedPreviewStyle ?? DEFAULT_PASTER_CONFIG.submittedPreviewStyle,
+    includeImagePathsInPrompt:
+      config.includeImagePathsInPrompt ?? DEFAULT_PASTER_CONFIG.includeImagePathsInPrompt,
+    imageCompression: {
+      enabled: config.imageCompression?.enabled ?? DEFAULT_PASTER_CONFIG.imageCompression.enabled,
+      command: config.imageCompression?.command ?? DEFAULT_PASTER_CONFIG.imageCompression.command,
+      model: config.imageCompression?.model ?? DEFAULT_PASTER_CONFIG.imageCompression.model,
+      prompt: config.imageCompression?.prompt ?? DEFAULT_PASTER_CONFIG.imageCompression.prompt,
+      piCommand:
+        config.imageCompression?.piCommand ?? DEFAULT_PASTER_CONFIG.imageCompression.piCommand,
+      timeoutMs:
+        config.imageCompression?.timeoutMs ?? DEFAULT_PASTER_CONFIG.imageCompression.timeoutMs,
+      includeReport:
+        config.imageCompression?.includeReport ??
+        DEFAULT_PASTER_CONFIG.imageCompression.includeReport,
+    },
     customEditor: {
       enabled: config.customEditor?.enabled ?? DEFAULT_PASTER_CONFIG.customEditor.enabled,
       showImagePreview:

package/src/editor.ts

@@ -7,6 +7,59 @@ import type { ImageAttachment } from "./types.ts";
 export const PASTE_START = "\x1b[200~";
 export const PASTE_END = "\x1b[201~";
 const PLACEHOLDER_REGEX = /\[#image \d+\]/g;
+const PASTE_MARKER_REGEX = /\[paste #(\d+)( (\+\d+ lines|\d+ chars))?\]/g;
+const baseSegmenter = new Intl.Segmenter();
+
+interface AtomicSpan {
+  start: number;
+  end: number;
+}
+
+interface EditorSegmentationAccess {
+  segment?: (text: string) => Iterable<Intl.SegmentData>;
+  pastes?: Map<number, string>;
+}
+
+function atomicSpansForText(text: string, validPasteIds: Set<number>): AtomicSpan[] {
+  const spans: AtomicSpan[] = [];
+
+  for (const match of text.matchAll(PASTE_MARKER_REGEX)) {
+    const id = Number.parseInt(match[1]!, 10);
+    if (!validPasteIds.has(id)) continue;
+    spans.push({ start: match.index, end: match.index + match[0].length });
+  }
+
+  for (const match of text.matchAll(PLACEHOLDER_REGEX)) {
+    const placeholder = match[0];
+    spans.push({ start: match.index, end: match.index + placeholder.length });
+  }
+
+  return spans.sort((a, b) => a.start - b.start || a.end - b.end);
+}
+
+export function segmentTextWithAtomicImages(
+  text: string,
+  store: AttachmentStore,
+  validPasteIds: Set<number> = new Set(),
+): Intl.SegmentData[] {
+  const spans = atomicSpansForText(text, validPasteIds);
+  if (spans.length === 0) return [...baseSegmenter.segment(text)];
+
+  const result: Intl.SegmentData[] = [];
+  let spanIndex = 0;
+  for (const segment of baseSegmenter.segment(text)) {
+    while (spanIndex < spans.length && spans[spanIndex]!.end <= segment.index) spanIndex++;
+    const span = spans[spanIndex];
+    if (span && segment.index >= span.start && segment.index < span.end) {
+      if (segment.index === span.start) {
+        result.push({ segment: text.slice(span.start, span.end), index: span.start, input: text });
+      }
+      continue;
+    }
+    result.push(segment);
+  }
+  return result;
+}
 
 interface EditorCursor {
   line: number;
@@ -14,7 +67,8 @@ interface EditorCursor {
 }
 
 interface PlaceholderAtCursor {
-  attachment: ImageAttachment;
+  attachment?: ImageAttachment;
+  placeholder: string;
   line: number;
   start: number;
   end: number;
@@ -32,16 +86,16 @@ function findPlaceholderAtCursor(
     const start = match.index;
     const end = start + placeholder.length;
     const attachment = store.get(placeholder);
-    if (!attachment) continue;
+    if (!attachment && mode !== "hover") continue;
 
     if (mode === "hover" && cursor.col >= start && cursor.col < end) {
-      return { attachment, line: cursor.line, start, end };
+      return { attachment, placeholder, line: cursor.line, start, end };
     }
     if (mode === "backspace" && cursor.col > start && cursor.col <= end) {
-      return { attachment, line: cursor.line, start, end };
+      return { attachment, placeholder, line: cursor.line, start, end };
     }
     if (mode === "delete" && cursor.col >= start && cursor.col < end) {
-      return { attachment, line: cursor.line, start, end };
+      return { attachment, placeholder, line: cursor.line, start, end };
     }
   }
   return undefined;
@@ -76,6 +130,7 @@ export class PasterEditor extends CustomEditor {
     },
   ) {
     super(tui, theme, pasterKeybindings);
+    this.installAtomicImageSegmentation();
     this.onPasteImage = () => {
       void this.handlePasteClipboardImage();
     };
@@ -89,6 +144,7 @@ export class PasterEditor extends CustomEditor {
 
   override handleInput(data: string): void {
     if (this.handleBracketedPaste(data)) return;
+    if (this.handleAtomicPlaceholderNavigation(data)) return;
     if (this.pasterOptions.deletePlaceholderAsBlock && this.handleAtomicPlaceholderDelete(data))
       return;
 
@@ -101,6 +157,16 @@ export class PasterEditor extends CustomEditor {
     this.pasterOptions.setCursorPreview(undefined);
   }
 
+  private installAtomicImageSegmentation(): void {
+    const editor = this as unknown as EditorSegmentationAccess;
+    editor.segment = (text: string) =>
+      segmentTextWithAtomicImages(
+        text,
+        this.pasterOptions.store,
+        new Set(editor.pastes?.keys() ?? []),
+      );
+  }
+
   private async handlePasteClipboardImage(): Promise<void> {
     const attachment = await this.pasterOptions.pasteClipboardImage?.();
     if (!attachment) return;
@@ -149,6 +215,29 @@ export class PasterEditor extends CustomEditor {
     return true;
   }
 
+  private handleAtomicPlaceholderNavigation(data: string): boolean {
+    const isLeft = this.pasterKeybindings.matches(data, "tui.editor.cursorLeft");
+    const isRight = this.pasterKeybindings.matches(data, "tui.editor.cursorRight");
+    if (!isLeft && !isRight) return false;
+
+    const line = this.getLines()[this.getCursor().line] ?? "";
+    const cursor = this.getCursor();
+    const matches = [...line.matchAll(PLACEHOLDER_REGEX)];
+    const target = isRight
+      ? matches.find(
+          (match) => cursor.col >= match.index && cursor.col < match.index + match[0].length,
+        )
+      : matches.find(
+          (match) => cursor.col > match.index && cursor.col <= match.index + match[0].length,
+        );
+    if (!target) return false;
+
+    this.setCursor(target.index + (isRight ? target[0].length : 0));
+    this.updateCursorPreview();
+    this.tui.requestRender();
+    return true;
+  }
+
   private handleAtomicPlaceholderDelete(data: string): boolean {
     const isBackspace = this.pasterKeybindings.matches(data, "tui.editor.deleteCharBackward");
     const isDelete = this.pasterKeybindings.matches(data, "tui.editor.deleteCharForward");
@@ -168,17 +257,22 @@ export class PasterEditor extends CustomEditor {
     return true;
   }
 
+  private setCursor(col: number): void {
+    const editor = this as unknown as EditorStateAccess;
+    if (editor.setCursorCol) {
+      editor.setCursorCol(col);
+    } else {
+      editor.state.cursorCol = col;
+    }
+  }
+
   private deleteLineRange(lineIndex: number, start: number, end: number): void {
     const editor = this as unknown as EditorStateAccess;
     editor.pushUndoSnapshot?.();
     const line = editor.state.lines[lineIndex] ?? "";
     editor.state.lines[lineIndex] = line.slice(0, start) + line.slice(end);
     editor.state.cursorLine = lineIndex;
-    if (editor.setCursorCol) {
-      editor.setCursorCol(start);
-    } else {
-      editor.state.cursorCol = start;
-    }
+    this.setCursor(start);
     editor.lastAction = null;
     editor.historyIndex = -1;
     this.onChange?.(this.getText());
@@ -200,7 +294,7 @@ export class PasterEditor extends CustomEditor {
       this.getCursor(),
       "hover",
     );
-    const nextPlaceholder = target?.attachment.placeholder;
+    const nextPlaceholder = target?.attachment?.placeholder;
     if (nextPlaceholder === this.activePreviewPlaceholder) return;
     this.activePreviewPlaceholder = nextPlaceholder;
     this.pasterOptions.setCursorPreview(target?.attachment);

package/src/image-utils.ts

@@ -357,6 +357,14 @@ export function imagesForText(
   ];
 }
 
+export function appendImagePathContext(text: string, attachments: ImageAttachment[]): string {
+  if (attachments.length === 0) return text;
+  const lines = attachments.map(
+    (attachment) => `- ${attachment.placeholder}: ${attachment.originalPath}`,
+  );
+  return `${text}\n\nAttached image paths:\n${lines.join("\n")}`;
+}
+
 /**
  * Async variant of imagesForText that runs each attachment through the
  * Anthropic-aware image optimizer (resize to 8000px cap, JPEG ladder to stay