pi-image-tools 1.2.0 → 1.3.0

package/src/inline-user-preview.ts

@@ -12,17 +12,11 @@ import { buildPreviewItems, type ImagePayload, type ImagePreviewItem } from "./i
 import { buildSixelRenderLines, isInlineImageProtocolLine } from "./sixel-protocol.js";
 import { setActiveTerminalImageSettingsCwd } from "./terminal-image-width.js";
 
-type UserMessageRenderFn = (width: number) => string[];
-
-type UserMessagePrototype = {
-  render: UserMessageRenderFn;
-  __piImageToolsInlineOriginalRender?: UserMessageRenderFn;
-  __piImageToolsInlinePatched?: boolean;
-};
+const INLINE_PREVIEW_PATCH_VERSION = "pi-image-tools-inline-preview-chat-component-v4";
+const PREPARE_SUBMITTED_PREVIEW_TIMEOUT_MS = 2_500;
 
 type UserMessageInstance = {
-  __piImageToolsInlineAssigned?: boolean;
-  __piImageToolsInlineItems?: ImagePreviewItem[];
+  render?: (width: number) => string[];
 };
 
 type InteractiveModePrototype = {
@@ -31,6 +25,7 @@ type InteractiveModePrototype = {
   __piImageToolsOriginalAddMessageToChat?: (message: unknown, options?: unknown) => void;
   __piImageToolsOriginalGetUserMessageText?: (message: unknown) => string;
   __piImageToolsPreviewPatched?: boolean;
+  __piImageToolsPreviewPatchVersion?: string;
 };
 
 interface UserImageContent {
@@ -46,8 +41,22 @@ interface UserMessageLike {
 
 interface InteractiveModeLike {
   chatContainer?: {
+    addChild?: (component: unknown) => void;
     children?: unknown[];
   };
+  ui?: {
+    requestRender?: () => void;
+  };
+}
+
+class ImagePreviewChatComponent {
+  constructor(private readonly items: readonly ImagePreviewItem[]) {}
+
+  invalidate(): void {}
+
+  render(width: number): string[] {
+    return renderPreviewLines(this.items, width);
+  }
 }
 
 function buildNativeLines(item: ImagePreviewItem, width: number): string[] {
@@ -137,18 +146,13 @@ function toImageContent(value: unknown): UserImageContent | null {
   };
 }
 
-function extractImagePayloads(message: unknown): ImagePayload[] {
-  const userMessage = toUserMessage(message);
-  if (userMessage.role !== "user") {
-    return [];
-  }
-
-  if (!Array.isArray(userMessage.content)) {
+function extractImagePayloadsFromContent(content: unknown): ImagePayload[] {
+  if (!Array.isArray(content)) {
     return [];
   }
 
   const payloads: ImagePayload[] = [];
-  for (const part of userMessage.content) {
+  for (const part of content) {
     const image = toImageContent(part);
     if (!image) {
       continue;
@@ -164,76 +168,171 @@ function extractImagePayloads(message: unknown): ImagePayload[] {
   return payloads;
 }
 
-function imagePlaceholderText(count: number): string {
-  if (count <= 1) {
-    return "[󰈟 1 image attached]";
+function extractImagePayloads(message: unknown): ImagePayload[] {
+  const userMessage = toUserMessage(message);
+  if (userMessage.role !== "user") {
+    return [];
   }
 
-  return `[󰈟 ${count} images attached]`;
+  return extractImagePayloadsFromContent(userMessage.content);
+}
+
+function getImagePayloadSignature(images: readonly ImagePayload[]): string {
+  return images
+    .map((image) => `${image.mimeType}:${image.data.length}:${image.data.slice(0, 32)}:${image.data.slice(-32)}`)
+    .join("|");
+}
+
+interface PreparedPreviewItems {
+  signature: string;
+  items: ImagePreviewItem[];
 }
 
-function patchUserMessageRender(): void {
-  const prototype = (UserMessageComponent as unknown as { prototype: UserMessagePrototype }).prototype;
-  if (typeof prototype.render !== "function") {
+const preparedPreviewItemsQueue: PreparedPreviewItems[] = [];
+const inFlightPreviewItems = new Map<string, Promise<ImagePreviewItem[]>>();
+
+function queuePreparedPreviewItems(images: readonly ImagePayload[], items: ImagePreviewItem[]): void {
+  if (images.length === 0 || items.length === 0) {
     return;
   }
 
-  if (!prototype.__piImageToolsInlineOriginalRender) {
-    prototype.__piImageToolsInlineOriginalRender = prototype.render;
+  preparedPreviewItemsQueue.push({
+    signature: getImagePayloadSignature(images),
+    items,
+  });
+
+  if (preparedPreviewItemsQueue.length > 8) {
+    preparedPreviewItemsQueue.splice(0, preparedPreviewItemsQueue.length - 8);
   }
+}
 
-  if (prototype.__piImageToolsInlinePatched) {
-    return;
+function consumePreparedPreviewItems(images: readonly ImagePayload[]): ImagePreviewItem[] | undefined {
+  if (images.length === 0 || preparedPreviewItemsQueue.length === 0) {
+    return undefined;
   }
 
-  prototype.render = function renderWithInlineImagePreview(width: number): string[] {
-    const originalRender = prototype.__piImageToolsInlineOriginalRender;
-    if (!originalRender) {
-      return [];
-    }
+  const signature = getImagePayloadSignature(images);
+  const index = preparedPreviewItemsQueue.findIndex((entry) => entry.signature === signature);
+  if (index === -1) {
+    return undefined;
+  }
 
-    const instance = this as unknown as UserMessageInstance;
-    if (!instance.__piImageToolsInlineAssigned) {
-      instance.__piImageToolsInlineAssigned = true;
-      if (!Array.isArray(instance.__piImageToolsInlineItems)) {
-        instance.__piImageToolsInlineItems = [];
-      }
-    }
+  const [entry] = preparedPreviewItemsQueue.splice(index, 1);
+  return entry?.items;
+}
 
-    const baseLines = originalRender.call(this, width);
-    const previewLines = renderPreviewLines(instance.__piImageToolsInlineItems ?? [], width);
-    if (previewLines.length === 0) {
-      return baseLines;
+function getInFlightPreviewItems(images: readonly ImagePayload[]): Promise<ImagePreviewItem[]> | undefined {
+  if (images.length === 0) {
+    return undefined;
+  }
+
+  return inFlightPreviewItems.get(getImagePayloadSignature(images));
+}
+
+function buildPreviewItemsOnce(
+  images: readonly ImagePayload[],
+  options: { cwd?: string; logger?: DebugLogger } = {},
+): Promise<ImagePreviewItem[]> {
+  const signature = getImagePayloadSignature(images);
+  const existing = inFlightPreviewItems.get(signature);
+  if (existing) {
+    return existing;
+  }
+
+  const promise = buildPreviewItems(images, options);
+  promise.then(
+    () => inFlightPreviewItems.delete(signature),
+    () => inFlightPreviewItems.delete(signature),
+  );
+  promise.catch(() => {
+    // Consumers log contextual errors; this prevents unhandled rejections when prebuild times out.
+  });
+  inFlightPreviewItems.set(signature, promise);
+  return promise;
+}
+
+async function waitForPreviewItemsDeadline(
+  previewItemsPromise: Promise<ImagePreviewItem[]>,
+): Promise<ImagePreviewItem[] | undefined> {
+  let timeout: ReturnType<typeof setTimeout> | undefined;
+
+  try {
+    return await Promise.race([
+      previewItemsPromise,
+      new Promise<undefined>((resolve) => {
+        timeout = setTimeout(() => resolve(undefined), PREPARE_SUBMITTED_PREVIEW_TIMEOUT_MS);
+        timeout.unref?.();
+      }),
+    ]);
+  } finally {
+    if (timeout) {
+      clearTimeout(timeout);
     }
+  }
+}
 
-    return [...baseLines, ...previewLines];
-  };
+function imagePlaceholderText(count: number): string {
+  if (count <= 1) {
+    return "[󰈟 1 image attached]";
+  }
 
-  prototype.__piImageToolsInlinePatched = true;
+  return `[󰈟 ${count} images attached]`;
 }
 
-function assignPreviewItemsToLatestUserMessage(
+function isUserMessageComponentLike(value: unknown): value is UserMessageInstance {
+  if (value instanceof UserMessageComponent) {
+    return true;
+  }
+
+  if (!isRecord(value) || typeof value.render !== "function") {
+    return false;
+  }
+
+  const constructorName =
+    typeof value.constructor === "function" && typeof value.constructor.name === "string"
+      ? value.constructor.name
+      : undefined;
+
+  return constructorName === "UserMessageComponent";
+}
+
+function requestInteractiveModeRender(mode: InteractiveModeLike): void {
+  try {
+    mode.ui?.requestRender?.();
+  } catch {
+    // Rendering will be retried by the next TUI update.
+  }
+}
+
+function addPreviewItemsAfterLatestUserMessage(
   mode: InteractiveModeLike,
   fromChildIndex: number,
   previewItems: ImagePreviewItem[],
-): void {
-  const children = mode.chatContainer?.children;
-  if (!Array.isArray(children) || children.length === 0) {
-    return;
+): boolean {
+  const chatContainer = mode.chatContainer;
+  const children = chatContainer?.children;
+  if (!Array.isArray(children) || children.length === 0 || previewItems.length === 0) {
+    return false;
   }
 
   const start = Math.max(0, fromChildIndex);
   for (let index = children.length - 1; index >= start; index -= 1) {
     const child = children[index];
-    if (!(child instanceof UserMessageComponent)) {
+    if (!isUserMessageComponentLike(child)) {
       continue;
     }
 
-    const instance = child as unknown as UserMessageInstance;
-    instance.__piImageToolsInlineItems = previewItems;
-    instance.__piImageToolsInlineAssigned = true;
-    return;
+    const previewComponent = new ImagePreviewChatComponent(previewItems);
+    const insertIndex = index + 1;
+    if (insertIndex >= children.length && typeof chatContainer?.addChild === "function") {
+      chatContainer.addChild(previewComponent);
+    } else {
+      children.splice(insertIndex, 0, previewComponent);
+    }
+    return true;
   }
+
+  return false;
 }
 
 function logInlinePreviewError(
@@ -262,7 +361,10 @@ function patchInteractiveMode(logger?: DebugLogger): void {
     prototype.__piImageToolsOriginalAddMessageToChat = prototype.addMessageToChat;
   }
 
-  if (prototype.__piImageToolsPreviewPatched) {
+  if (
+    prototype.__piImageToolsPreviewPatched &&
+    prototype.__piImageToolsPreviewPatchVersion === INLINE_PREVIEW_PATCH_VERSION
+  ) {
     return;
   }
 
@@ -299,13 +401,26 @@ function patchInteractiveMode(logger?: DebugLogger): void {
       return;
     }
 
-    void buildPreviewItems(imagePayloads, { logger })
+    const preparedPreviewItems = consumePreparedPreviewItems(imagePayloads);
+    if (preparedPreviewItems) {
+      if (addPreviewItemsAfterLatestUserMessage(mode, beforeCount, preparedPreviewItems)) {
+        requestInteractiveModeRender(mode);
+      }
+      return;
+    }
+
+    const previewItemsPromise = getInFlightPreviewItems(imagePayloads)
+      ?? buildPreviewItemsOnce(imagePayloads, { logger });
+
+    void previewItemsPromise
       .then((previewItems) => {
         if (previewItems.length === 0) {
           return;
         }
 
-        assignPreviewItemsToLatestUserMessage(mode, beforeCount, previewItems);
+        if (addPreviewItemsAfterLatestUserMessage(mode, beforeCount, previewItems)) {
+          requestInteractiveModeRender(mode);
+        }
       })
       .catch((error: unknown) => {
         logInlinePreviewError(logger, "inline-user-preview.build_preview_failed", error);
@@ -313,6 +428,7 @@ function patchInteractiveMode(logger?: DebugLogger): void {
   };
 
   prototype.__piImageToolsPreviewPatched = true;
+  prototype.__piImageToolsPreviewPatchVersion = INLINE_PREVIEW_PATCH_VERSION;
 }
 
 export interface RegisterInlineUserImagePreviewOptions {
@@ -323,15 +439,16 @@ export function registerInlineUserImagePreview(
   pi: ExtensionAPI,
   options: RegisterInlineUserImagePreviewOptions = {},
 ): void {
+  const applyPatch = (): void => {
+    try {
+      patchInteractiveMode(options.logger);
+    } catch (error) {
+      logInlinePreviewError(options.logger, "inline-user-preview.patch_failed", error);
+    }
+  };
+
   const runPatch = (delayMs: number): void => {
-    setTimeout(() => {
-      try {
-        patchInteractiveMode(options.logger);
-        patchUserMessageRender();
-      } catch (error) {
-        logInlinePreviewError(options.logger, "inline-user-preview.patch_failed", error);
-      }
-    }, delayMs);
+    setTimeout(applyPatch, delayMs);
   };
 
   const schedulePatch = (): void => {
@@ -342,18 +459,57 @@ export function registerInlineUserImagePreview(
   const handleSessionEvent = (eventName: string, cwd: string | undefined): void => {
     try {
       setActiveTerminalImageSettingsCwd(cwd);
+      applyPatch();
       schedulePatch();
     } catch (error) {
       logInlinePreviewError(options.logger, `inline-user-preview.${eventName}_failed`, error);
     }
   };
 
+  const prepareSubmittedImagePreview = async (
+    event: { images?: unknown },
+    cwd: string | undefined,
+  ): Promise<{ imagePayloads: ImagePayload[]; previewItems: ImagePreviewItem[] } | undefined> => {
+    try {
+      setActiveTerminalImageSettingsCwd(cwd);
+      applyPatch();
+      const imagePayloads = extractImagePayloadsFromContent(event.images);
+      if (imagePayloads.length === 0) {
+        return undefined;
+      }
+
+      const previewItemsPromise = buildPreviewItemsOnce(imagePayloads, { cwd, logger: options.logger });
+      const previewItems = await waitForPreviewItemsDeadline(previewItemsPromise);
+      if (!previewItems || previewItems.length === 0) {
+        return undefined;
+      }
+
+      return { imagePayloads, previewItems };
+    } catch (error) {
+      logInlinePreviewError(options.logger, "inline-user-preview.prepare_submitted_preview_failed", error);
+      return undefined;
+    }
+  };
+
+  applyPatch();
+
   pi.on("session_start", async (_event, ctx) => {
     handleSessionEvent("session_start", ctx.cwd);
   });
 
-  pi.on("before_agent_start", async (_event, ctx) => {
+  pi.on("before_agent_start", async (event, ctx) => {
     handleSessionEvent("before_agent_start", ctx.cwd);
+    if (!ctx.hasUI) {
+      return undefined;
+    }
+
+    const preparedPreview = await prepareSubmittedImagePreview(event, ctx.cwd);
+    if (!preparedPreview) {
+      return undefined;
+    }
+
+    queuePreparedPreviewItems(preparedPreview.imagePayloads, preparedPreview.previewItems);
+    return undefined;
   });
 
   const onSessionSwitch = pi.on as unknown as (

package/src/keybindings.ts

@@ -230,20 +230,15 @@ function removeBuiltinConflicts(shortcuts: readonly KeyId[]): KeyId[] {
   return shortcuts.filter((shortcut) => !builtinShortcuts.has(normalizeShortcutKey(shortcut)));
 }
 
-function getImagePasteShortcuts(
+export function getImagePasteShortcuts(
   config: ImageToolsConfig,
   platform: NodeJS.Platform = process.platform,
 ): KeyId[] {
   const shouldAvoidBuiltinConflicts =
     config.shortcuts.avoidBuiltinConflicts || config.shortcuts.suppressBuiltinConflictWarnings;
+  const candidates = config.shortcuts.pasteImage ?? getImagePasteShortcutCandidates(platform);
 
-  if (config.shortcuts.pasteImage !== undefined) {
-    return shouldAvoidBuiltinConflicts
-      ? removeBuiltinConflicts(config.shortcuts.pasteImage)
-      : config.shortcuts.pasteImage;
-  }
-
-  return removeBuiltinConflicts(getImagePasteShortcutCandidates(platform));
+  return shouldAvoidBuiltinConflicts ? removeBuiltinConflicts(candidates) : [...candidates];
 }
 
 export function registerImagePasteKeybindings(
@@ -251,16 +246,20 @@ export function registerImagePasteKeybindings(
   handler: PasteImageHandler,
   options: RegisterImagePasteKeybindingsOptions,
 ): void {
+  const platform = process.platform;
   const configuredShortcuts = options.config.shortcuts.pasteImage;
-  const shortcuts = getImagePasteShortcuts(options.config);
-  const skippedShortcuts = configuredShortcuts?.filter(
+  const requestedShortcuts = configuredShortcuts ?? getImagePasteShortcutCandidates(platform);
+  const shortcuts = getImagePasteShortcuts(options.config, platform);
+  const skippedShortcuts = requestedShortcuts.filter(
     (shortcut) => !shortcuts.some((registeredShortcut) => normalizeShortcutKey(registeredShortcut) === normalizeShortcutKey(shortcut)),
-  ) ?? [];
+  );
 
   options.logger.log("keybindings.register", {
     avoidBuiltinConflicts: options.config.shortcuts.avoidBuiltinConflicts,
     suppressBuiltinConflictWarnings: options.config.shortcuts.suppressBuiltinConflictWarnings,
     configured: configuredShortcuts !== undefined,
+    platform,
+    requestedShortcuts,
     registeredShortcuts: shortcuts,
     skippedShortcuts,
   });

package/src/providers/command-runner.ts

@@ -0,0 +1,68 @@
+import { spawnSync } from "node:child_process";
+
+import { isErrnoException } from "../errors.js";
+
+export const LIST_TYPES_TIMEOUT_MS = 1000;
+export const READ_TIMEOUT_MS = 5000;
+export const MAX_BUFFER_BYTES = 50 * 1024 * 1024;
+
+export interface CommandResult {
+  ok: boolean;
+  stdout: Buffer;
+  stderr: Buffer;
+  missingCommand: boolean;
+  status: number | null;
+}
+
+export interface CommandRunOptions {
+  environment?: NodeJS.ProcessEnv;
+  maxBuffer?: number;
+  timeout: number;
+  windowsHide?: boolean;
+}
+
+export type CommandRunner = (
+  command: string,
+  args: readonly string[],
+  options: CommandRunOptions,
+) => CommandResult;
+
+function toBuffer(value: unknown): Buffer {
+  if (Buffer.isBuffer(value)) {
+    return value;
+  }
+
+  if (value instanceof Uint8Array) {
+    return Buffer.from(value);
+  }
+
+  return Buffer.from(value === undefined || value === null ? "" : String(value), "utf8");
+}
+
+export const defaultCommandRunner: CommandRunner = (command, args, options) => {
+  const result = spawnSync(command, [...args], {
+    env: options.environment,
+    maxBuffer: options.maxBuffer ?? MAX_BUFFER_BYTES,
+    stdio: ["ignore", "pipe", "pipe"],
+    timeout: options.timeout,
+    windowsHide: options.windowsHide,
+  });
+
+  if (result.error) {
+    return {
+      ok: false,
+      stdout: toBuffer(result.stdout),
+      stderr: toBuffer(result.stderr),
+      missingCommand: isErrnoException(result.error) && result.error.code === "ENOENT",
+      status: result.status ?? null,
+    };
+  }
+
+  return {
+    ok: result.status === 0,
+    stdout: toBuffer(result.stdout),
+    stderr: toBuffer(result.stderr),
+    missingCommand: false,
+    status: result.status ?? null,
+  };
+};

package/src/providers/mac-osascript-pngf.ts

@@ -0,0 +1,101 @@
+import { buildNamespaceWrappedCommand, defaultCommandExists, type CommandExists } from "../shell-environment.js";
+import {
+  defaultCommandRunner,
+  MAX_BUFFER_BYTES,
+  READ_TIMEOUT_MS,
+  type CommandRunner,
+} from "./command-runner.js";
+import type { ClipboardImageProvider, ClipboardProviderContext, ClipboardReadResult } from "./types.js";
+
+const PNGF_SCRIPT = `try
+  set imageData to the clipboard as «class PNGf»
+  return imageData
+on error
+  return ""
+end try`;
+
+export interface OsascriptPngfProviderOptions {
+  priority?: number;
+  commandRunner?: CommandRunner;
+  commandExists?: CommandExists;
+}
+
+function parseAppleScriptPngfData(stdout: Buffer): Uint8Array | null {
+  const text = stdout.toString("utf8").trim();
+  if (text.length === 0) {
+    return null;
+  }
+
+  const match = text.match(/«data\s+PNGf([0-9a-fA-F\s]+)»/i);
+  if (!match) {
+    return null;
+  }
+
+  const hex = match[1]?.replace(/\s+/g, "") ?? "";
+  if (hex.length === 0 || hex.length % 2 !== 0) {
+    return null;
+  }
+
+  const bytes = Buffer.from(hex, "hex");
+  return bytes.length > 0 ? new Uint8Array(bytes) : null;
+}
+
+export class OsascriptPngfProvider implements ClipboardImageProvider {
+  readonly capabilities;
+  private readonly commandRunner: CommandRunner;
+  private readonly commandExists: CommandExists;
+
+  constructor(options: OsascriptPngfProviderOptions = {}) {
+    this.capabilities = {
+      id: "mac-osascript-pngf",
+      name: "osascript PNGf",
+      platforms: ["darwin"],
+      priority: options.priority ?? 30,
+    };
+    this.commandRunner = options.commandRunner ?? defaultCommandRunner;
+    this.commandExists = options.commandExists ?? defaultCommandExists;
+  }
+
+  isAvailable(context: ClipboardProviderContext): boolean {
+    try {
+      return this.commandExists("osascript", context);
+    } catch {
+      return false;
+    }
+  }
+
+  read(context: ClipboardProviderContext): ClipboardReadResult {
+    const wrapped = buildNamespaceWrappedCommand(
+      "osascript",
+      ["-e", PNGF_SCRIPT],
+      context,
+      this.commandExists,
+    );
+    const result = this.commandRunner(wrapped.command, wrapped.args, {
+      environment: context.environment,
+      maxBuffer: MAX_BUFFER_BYTES,
+      timeout: READ_TIMEOUT_MS,
+    });
+
+    if (result.missingCommand) {
+      return { available: false, image: null };
+    }
+
+    if (!result.ok || result.stdout.length === 0) {
+      return { available: true, image: null };
+    }
+
+    const bytes = parseAppleScriptPngfData(result.stdout);
+    if (!bytes) {
+      return { available: true, image: null };
+    }
+
+    return {
+      available: true,
+      image: {
+        bytes,
+        mimeType: "image/png",
+      },
+    };
+  }
+}