pi-image-preview 0.1.0

package/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2025 rielj
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

package/README.md

@@ -0,0 +1,110 @@
+# pi-image-preview
+
+Image preview extension for [pi coding agent](https://github.com/mariozechner/pi-coding-agent) — renders inline image thumbnails above the editor using the kitty graphics protocol with full tmux support.
+
+![Screenshot](screenshot.png)
+
+## Features
+
+- **Inline image preview** — paste an image (`Ctrl+V`) and a thumbnail renders above the editor
+- **Horizontal layout** — multiple images display side by side
+- **tmux support** — uses kitty's Unicode placeholder protocol (`U=1`) so images are pane-aware (no ghosting across panes)
+- **Auto-cleanup** — delete the image path from editor text and the preview disappears
+- **No editor conflicts** — works alongside vim mode and other editor extensions (does not use `setEditorComponent`)
+- **Image resizing** — leverages pi's built-in WASM image resizer for efficient thumbnails
+- **Screenshot integration** — automatically inlines images from screenshot tool results
+
+## Install
+
+```bash
+pi install npm:pi-image-preview
+```
+
+## How it works
+
+1. **Paste** an image with `Ctrl+V`
+2. Pi saves the clipboard to a temp file and inserts the path into the editor
+3. The extension **detects the image path**, reads the file, and renders a thumbnail above the editor
+4. The raw file path stays in the editor — the label below the thumbnail shows a truncated version
+5. On **submit**, image paths are stripped from the text and the images are attached to your message
+
+## Prerequisites
+
+### Terminal: [Kitty](https://sw.kovidgoyal.net/kitty/) (required)
+
+This extension uses the **kitty graphics protocol** to render images. It will **not** render images in other terminals (iTerm2, Alacritty, WezTerm, etc.) — it falls back to text labels instead.
+
+- **Minimum version**: Kitty 0.28+ (Unicode placeholder support)
+- **Recommended**: Kitty 0.35+ for best compatibility
+
+No special kitty config is required — the extension works with default kitty settings.
+
+### tmux (optional but supported)
+
+If you run pi inside tmux, you need **one config change** in your `~/.tmux.conf`:
+
+```tmux
+set -g allow-passthrough all
+```
+
+Then reload: `tmux source-file ~/.tmux.conf`
+
+This allows kitty graphics escape sequences to pass through tmux to the terminal. Without it, images will not render.
+
+**tmux version**: 3.3a+ required (added `allow-passthrough` support).
+
+### pi coding agent
+
+- **Version**: Latest recommended — the extension uses `setWidget`, `getEditorText`, and the `input` event transform API
+- **No additional pi configuration needed** — just install the extension
+
+## Supported image formats
+
+- PNG
+- JPEG / JPG
+- GIF (first frame)
+- WebP
+
+Maximum file size: **50 MB** (larger files are silently skipped).
+
+## Limitations
+
+- **Kitty terminal only** — other terminals get text-only labels (no image rendering)
+- **macOS / Linux only** — kitty does not run on Windows natively
+- **tmux requires `allow-passthrough all`** — without it, images won't render inside tmux (the extension still works, but shows text fallback)
+- **No image selection/navigation** — this is a simple preview, not a gallery browser
+- **Thumbnail size is fixed** — images are scaled to fit within 25 columns; not configurable yet
+- **Images are not preserved in chat history** — after submitting, the preview clears; the image is sent as an attachment to the model
+- **GIF animation** — only the first frame is displayed
+- **SSH sessions** — kitty graphics protocol does not work over SSH unless using `kitten ssh` (kitty's SSH kitten)
+- **Multiple tmux panes showing pi** — each pane renders independently; switching panes clears/restores images correctly via Unicode placeholders, but rapid switching may briefly show artifacts
+
+## How tmux support works
+
+Standard kitty graphics render pixels at absolute terminal positions. This causes images to "ghost" across tmux panes — an image rendered in pane 1 is still visible when you switch to pane 2.
+
+This extension uses kitty's **Unicode placeholder protocol** instead:
+
+1. Image data is transmitted to kitty with `U=1` flag (stored but not directly rendered)
+2. Special `U+10EEEE` characters with combining diacritics are output where the image should appear
+3. These are **regular text characters** that tmux manages per-pane
+4. Switching panes swaps the text buffer → placeholders disappear → image disappears
+5. Switching back → placeholders redrawn → image reappears
+
+## Development
+
+```bash
+# Clone
+git clone https://github.com/rielj/pi-image-preview.git
+cd pi-image-preview
+
+# Symlink into pi extensions
+ln -s "$(pwd)" ~/.pi/agent/extensions/image-preview
+
+# Reload pi
+# Inside pi, run: /reload
+```
+
+## License
+
+MIT

package/index.ts

@@ -0,0 +1,59 @@
+import { createRequire } from "node:module";
+import path from "node:path";
+import { pathToFileURL } from "node:url";
+import {
+	loadImageContentFromPath,
+	maybeResizeImage,
+	readImageContentFromPathAsync,
+	type ImageResizer,
+} from "./src/image-content.ts";
+import { registerImagePreviewExtension } from "./src/extension-runtime.ts";
+import { debugLog } from "./src/debug.ts";
+
+let cachedResizerPromise: Promise<ImageResizer | null> | undefined;
+
+async function loadPiImageResizer(): Promise<ImageResizer | null> {
+	if (cachedResizerPromise) return cachedResizerPromise;
+
+	cachedResizerPromise = (async () => {
+		try {
+			const require = createRequire(import.meta.url);
+			const piEntry = require.resolve("@mariozechner/pi-coding-agent");
+			const distDir = path.dirname(piEntry);
+			const moduleUrl = pathToFileURL(
+				path.join(distDir, "utils", "image-resize.js"),
+			).href;
+			const mod = (await import(moduleUrl)) as {
+				resizeImage?: (image: {
+					type: "image";
+					data: string;
+					mimeType: string;
+				}) => Promise<{ data: string; mimeType: string }>;
+			};
+			if (!mod.resizeImage) return null;
+			return async (image) => {
+				const resized = await mod.resizeImage!(image);
+				return {
+					type: "image",
+					data: resized.data,
+					mimeType: resized.mimeType,
+				};
+			};
+		} catch (err) {
+			debugLog("Failed to load pi image resizer", err);
+			return null;
+		}
+	})();
+
+	return cachedResizerPromise;
+}
+
+export default function (pi: any): void {
+	registerImagePreviewExtension(pi, {
+		readImageContentFromPathAsync,
+		maybeResizeImage: async (image) =>
+			maybeResizeImage(image, await loadPiImageResizer()),
+		loadImageContentFromPath: async (filePath) =>
+			loadImageContentFromPath(filePath, await loadPiImageResizer()),
+	});
+}

package/package.json

@@ -0,0 +1,35 @@
+{
+  "name": "pi-image-preview",
+  "version": "0.1.0",
+  "description": "Image preview extension for pi coding agent — renders inline image thumbnails above the editor using kitty graphics protocol with tmux support",
+  "keywords": [
+    "pi-package",
+    "kitty",
+    "image",
+    "preview",
+    "tmux",
+    "terminal",
+    "graphics"
+  ],
+  "license": "MIT",
+  "author": "rielj",
+  "type": "module",
+  "files": [
+    "index.ts",
+    "src/",
+    "README.md",
+    "LICENSE"
+  ],
+  "pi": {
+    "extensions": [
+      "./index.ts"
+    ]
+  },
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/rielj/pi-image-preview.git"
+  },
+  "peerDependencies": {
+    "@mariozechner/pi-coding-agent": "*"
+  }
+}

package/src/content.ts

@@ -0,0 +1,12 @@
+export type TextContent = {
+	type: "text";
+	text: string;
+};
+
+export type ImageContent = {
+	type: "image";
+	data: string;
+	mimeType: string;
+};
+
+export type ContentBlock = TextContent | ImageContent;

package/src/debug.ts

@@ -0,0 +1,21 @@
+/**
+ * Debug logging utility for pi-image-preview.
+ *
+ * Set the PI_IMAGE_PREVIEW_DEBUG environment variable to enable debug output.
+ * Logs are written to stderr so they don't interfere with terminal rendering.
+ *
+ * @example
+ *   PI_IMAGE_PREVIEW_DEBUG=1 pi
+ */
+const DEBUG_ENABLED = Boolean(process.env.PI_IMAGE_PREVIEW_DEBUG);
+
+export function debugLog(message: string, error?: unknown): void {
+	if (!DEBUG_ENABLED) return;
+
+	const prefix = "[pi-image-preview]";
+	if (error) {
+		console.error(prefix, message, error);
+	} else {
+		console.error(prefix, message);
+	}
+}

package/src/extension-runtime.ts

@@ -0,0 +1,332 @@
+import path from "node:path";
+import type { ImageContent, ContentBlock } from "./content.ts";
+import { ImageGallery, type GalleryImage } from "./image-gallery.ts";
+import { PREFER_INLINE_SCREENSHOT_PROMPT } from "./prompt.ts";
+import { upgradeScreenshotToolResult } from "./tool-result-upgrader.ts";
+import { debugLog } from "./debug.ts";
+
+// ── Types ──────────────────────────────────────────────────
+
+type TrackedImage = {
+	filePath: string;
+	image: ImageContent;
+	label: string;
+};
+
+export type ExtensionDeps = {
+	readImageContentFromPathAsync: (
+		filePath: string,
+	) => Promise<ImageContent | null>;
+	maybeResizeImage?: (image: ImageContent) => Promise<ImageContent>;
+	loadImageContentFromPath: (
+		filePath: string,
+	) => Promise<ImageContent | null>;
+};
+
+type PiLike = {
+	on(event: string, handler: (...args: any[]) => any): void;
+	sendUserMessage(
+		content: string | ContentBlock[],
+		options?: { deliverAs?: "steer" | "followUp" },
+	): void;
+};
+
+type CtxLike = {
+	cwd: string;
+	isIdle(): boolean;
+	ui: {
+		setWidget(
+			key: string,
+			content:
+				| string[]
+				| ((tui: any, theme: any) => any)
+				| undefined,
+			options?: { placement?: "aboveEditor" | "belowEditor" },
+		): void;
+		getEditorText(): string;
+		setEditorText(text: string): void;
+		theme: any;
+	};
+};
+
+/** Event shape for the "input" event from pi. */
+type InputEvent = {
+	text: string;
+	images?: ImageContent[];
+};
+
+/** Discriminated union for input handler return values. */
+type InputResult =
+	| { action: "continue" }
+	| { action: "handled" }
+	| { action: "transform"; text: string; images: ImageContent[] };
+
+/** Re-export for tool_result event typing. */
+type ToolResultEvent = import("./tool-result-upgrader.ts").ToolResultEventLike;
+
+// ── Constants ──────────────────────────────────────────────
+
+const WIDGET_KEY = "image-preview";
+const POLL_INTERVAL_MS = 250;
+
+// Matches image file paths:
+//   - Absolute: /path/to/image.png
+//   - Home-relative: ~/screenshots/image.png
+//   - Relative: ./images/image.png, ../images/image.png
+// Supports common path characters including spaces (escaped with \),
+// parens, #, +, and other special characters.
+const IMAGE_PATH_RE =
+	/((?:~\/|\.\.?\/|\/)[^\s:*?"<>|][^\s:*?"<>|]*\.(?:png|jpe?g|gif|webp))(?=\s|$)/gi;
+
+/** Produce a label from an image path — just the filename. */
+function trimImageLabel(filePath: string): string {
+	return path.basename(filePath);
+}
+
+// ── Extension ──────────────────────────────────────────────
+
+export function registerImagePreviewExtension(
+	pi: PiLike,
+	deps: ExtensionDeps,
+): void {
+	let tracked: Map<string, TrackedImage> = new Map();
+	let gallery: ImageGallery | null = null;
+	let pollTimer: ReturnType<typeof setInterval> | null = null;
+	let latestCtx: CtxLike | null = null;
+
+	// ── Helpers ────────────────────────────────────────────
+
+	function refreshWidget(ctx: CtxLike): void {
+		if (tracked.size === 0) {
+			if (gallery) {
+				gallery.dispose();
+				gallery = null;
+			}
+			ctx.ui.setWidget(WIDGET_KEY, undefined);
+			return;
+		}
+
+		const galleryImages: GalleryImage[] = [...tracked.values()].map((t) => ({
+			data: t.image.data,
+			mimeType: t.image.mimeType,
+			label: t.label,
+		}));
+
+		// Dispose the previous gallery to free kitty image resources before replacement
+		if (gallery) {
+			gallery.dispose();
+			gallery = null;
+		}
+
+		ctx.ui.setWidget(
+			WIDGET_KEY,
+			(_tui: any, theme: any) => {
+				const galleryTheme = {
+					accent: (s: string) => theme.fg("accent", s),
+					muted: (s: string) => theme.fg("muted", s),
+					dim: (s: string) => theme.fg("dim", s),
+					bold: (s: string) => theme.bold(s),
+				};
+
+				gallery = new ImageGallery(galleryTheme);
+				gallery.setImages(galleryImages);
+				return gallery;
+			},
+			{ placement: "aboveEditor" },
+		);
+	}
+
+	function resetDraft(ctx: CtxLike): void {
+		if (gallery) {
+			gallery.dispose();
+			gallery = null;
+		}
+		tracked = new Map();
+		ctx.ui.setWidget(WIDGET_KEY, undefined);
+	}
+
+	/**
+	 * Scan editor text for image paths.
+	 * Track new ones, remove ones that are no longer in the text.
+	 * Async to avoid blocking the event loop with file I/O.
+	 */
+	async function scanEditorText(ctx: CtxLike): Promise<void> {
+		let text: string;
+		try {
+			text = ctx.ui.getEditorText();
+		} catch (err) {
+			debugLog("Failed to get editor text", err);
+			return;
+		}
+		if (!text) {
+			if (tracked.size > 0) {
+				tracked = new Map();
+				refreshWidget(ctx);
+			}
+			return;
+		}
+
+		// Find all image paths currently in the text
+		// Create a fresh regex each time to avoid stale lastIndex from the `g` flag
+		const imagePathRe = new RegExp(IMAGE_PATH_RE.source, IMAGE_PATH_RE.flags);
+		const matches = [...text.matchAll(imagePathRe)];
+		const currentPaths = new Set<string>();
+
+		let changed = false;
+
+		for (const match of matches) {
+			const rawPath = match[1];
+			if (!rawPath) continue;
+			currentPaths.add(rawPath);
+
+			// Already tracked?
+			if (tracked.has(rawPath)) continue;
+
+			// New path — try to load it (async to avoid blocking event loop)
+			const image = await deps.readImageContentFromPathAsync(rawPath);
+			if (!image) continue;
+
+			tracked.set(rawPath, {
+				filePath: rawPath,
+				image,
+				label: trimImageLabel(rawPath),
+			});
+			changed = true;
+
+			// Async resize in background
+			if (deps.maybeResizeImage) {
+				const entry = tracked.get(rawPath)!;
+				void deps.maybeResizeImage(image).then((resized) => {
+					// Guard against the entry having been removed while resize was in-flight
+					if (tracked.has(rawPath) && tracked.get(rawPath) === entry) {
+						entry.image = resized;
+						if (latestCtx) refreshWidget(latestCtx);
+					}
+				}).catch((err) => {
+					debugLog(`Failed to resize image ${rawPath}`, err);
+				});
+			}
+		}
+
+		// Remove tracked images whose paths are no longer in the text
+		for (const trackedPath of tracked.keys()) {
+			if (!currentPaths.has(trackedPath)) {
+				tracked.delete(trackedPath);
+				changed = true;
+			}
+		}
+
+		if (changed) {
+			refreshWidget(ctx);
+		}
+	}
+
+	function startPolling(): void {
+		stopPolling();
+		pollTimer = setInterval(() => {
+			if (!latestCtx) return;
+			scanEditorText(latestCtx).catch((err) => {
+				debugLog("Error during editor text scan", err);
+			});
+		}, POLL_INTERVAL_MS);
+	}
+
+	function stopPolling(): void {
+		if (pollTimer) {
+			clearInterval(pollTimer);
+			pollTimer = null;
+		}
+	}
+
+	// ── Event handlers ─────────────────────────────────────
+
+	pi.on("before_agent_start", () => {
+		return { systemPrompt: PREFER_INLINE_SCREENSHOT_PROMPT };
+	});
+
+	// Clean up resources when the process exits
+	const cleanup = (): void => {
+		stopPolling();
+		if (gallery) {
+			gallery.dispose();
+			gallery = null;
+		}
+	};
+	process.on("exit", cleanup);
+	process.on("SIGINT", cleanup);
+	process.on("SIGTERM", cleanup);
+
+	pi.on("session_start", async (_event: unknown, ctx: CtxLike) => {
+		latestCtx = ctx;
+		resetDraft(ctx);
+		startPolling();
+	});
+
+	pi.on("session_switch", async (_event: unknown, ctx: CtxLike) => {
+		latestCtx = ctx;
+		resetDraft(ctx);
+		startPolling();
+	});
+
+	pi.on("tool_result", async (event: ToolResultEvent, ctx: CtxLike) => {
+		latestCtx = ctx;
+		return upgradeScreenshotToolResult(
+			event,
+			ctx.cwd,
+			deps.loadImageContentFromPath,
+		);
+	});
+
+	// On submit: strip image paths from text, attach actual images
+	pi.on("input", async (event: InputEvent, ctx: CtxLike): Promise<InputResult> => {
+		latestCtx = ctx;
+
+		if (tracked.size === 0) {
+			return { action: "continue" };
+		}
+
+		const fullText = (event.text || "").trim();
+
+		// Don't transform commands or shell escapes
+		if (fullText.startsWith("/") || fullText.trimStart().startsWith("!")) {
+			return { action: "continue" };
+		}
+
+		// Find which tracked paths are still in the submitted text
+		const usedImages: ImageContent[] = [];
+		let strippedText = fullText;
+
+		for (const [trackedPath, entry] of tracked) {
+			if (fullText.includes(trackedPath)) {
+				usedImages.push(entry.image);
+				// Strip the path from the text
+				strippedText = strippedText.split(trackedPath).join("");
+			}
+		}
+
+		if (usedImages.length === 0) {
+			return { action: "continue" };
+		}
+
+		// Clean up whitespace after stripping paths
+		strippedText = strippedText.replace(/\s+/g, " ").trim();
+
+		// Clear state
+		resetDraft(ctx);
+
+		if (!strippedText) {
+			// Images only, no text — send directly
+			pi.sendUserMessage(
+				usedImages,
+				ctx.isIdle() ? undefined : { deliverAs: "steer" },
+			);
+			return { action: "handled" };
+		}
+
+		return {
+			action: "transform",
+			text: strippedText,
+			images: [...(event.images ?? []), ...usedImages],
+		};
+	});
+}

package/src/image-content.ts

@@ -0,0 +1,107 @@
+import fs from "node:fs";
+import fsp from "node:fs/promises";
+import type { ImageContent } from "./content.ts";
+import {
+	inferMimeType,
+	looksLikeImagePath,
+	looksLikeImagePathAsync,
+} from "./path-utils.ts";
+import { debugLog } from "./debug.ts";
+
+export type ImageResizer = (image: ImageContent) => Promise<ImageContent>;
+
+/** Maximum image file size in bytes (50 MB). Files larger than this are skipped to prevent OOM. */
+const MAX_IMAGE_FILE_SIZE = 50 * 1024 * 1024;
+
+/**
+ * Synchronous image read — used only at startup / non-poll paths.
+ * Prefer readImageContentFromPathAsync in the poll loop.
+ */
+export function readImageContentFromPath(
+	filePath: string,
+): ImageContent | null {
+	if (!looksLikeImagePath(filePath)) return null;
+
+	try {
+		const stat = fs.statSync(filePath);
+		if (stat.size > MAX_IMAGE_FILE_SIZE) {
+			debugLog(
+				`Skipping image ${filePath}: file size ${(stat.size / 1024 / 1024).toFixed(1)}MB exceeds ${MAX_IMAGE_FILE_SIZE / 1024 / 1024}MB limit`,
+			);
+			return null;
+		}
+	} catch (err) {
+		debugLog(`Failed to stat image file ${filePath}`, err);
+		return null;
+	}
+
+	const mimeType = inferMimeType(filePath)!;
+	try {
+		const bytes = fs.readFileSync(filePath);
+		return {
+			type: "image",
+			data: bytes.toString("base64"),
+			mimeType,
+		};
+	} catch (err) {
+		debugLog(`Failed to read image file ${filePath}`, err);
+		return null;
+	}
+}
+
+/**
+ * Async image read — non-blocking, preferred in the 250ms poll loop.
+ */
+export async function readImageContentFromPathAsync(
+	filePath: string,
+): Promise<ImageContent | null> {
+	if (!(await looksLikeImagePathAsync(filePath))) return null;
+
+	try {
+		const stat = await fsp.stat(filePath);
+		if (stat.size > MAX_IMAGE_FILE_SIZE) {
+			debugLog(
+				`Skipping image ${filePath}: file size ${(stat.size / 1024 / 1024).toFixed(1)}MB exceeds ${MAX_IMAGE_FILE_SIZE / 1024 / 1024}MB limit`,
+			);
+			return null;
+		}
+	} catch (err) {
+		debugLog(`Failed to stat image file ${filePath}`, err);
+		return null;
+	}
+
+	const mimeType = inferMimeType(filePath)!;
+	try {
+		const bytes = await fsp.readFile(filePath);
+		return {
+			type: "image",
+			data: bytes.toString("base64"),
+			mimeType,
+		};
+	} catch (err) {
+		debugLog(`Failed to read image file ${filePath}`, err);
+		return null;
+	}
+}
+
+export async function maybeResizeImage(
+	image: ImageContent,
+	resizeImage?: ImageResizer | null,
+): Promise<ImageContent> {
+	if (!resizeImage) return image;
+	try {
+		return await resizeImage(image);
+	} catch (err) {
+		debugLog("Image resize failed, using original", err);
+		return image;
+	}
+}
+
+export async function loadImageContentFromPath(
+	filePath: string,
+	resizeImage?: ImageResizer | null,
+): Promise<ImageContent | null> {
+	const image = readImageContentFromPath(filePath);
+	if (!image) return null;
+	return maybeResizeImage(image, resizeImage);
+}

package/src/image-gallery.ts

@@ -0,0 +1,345 @@
+import {
+	type Component,
+	getCapabilities,
+	getImageDimensions,
+	calculateImageRows,
+	getCellDimensions,
+} from "@mariozechner/pi-tui";
+
+export interface GalleryTheme {
+	accent: (s: string) => string;
+	muted: (s: string) => string;
+	dim: (s: string) => string;
+	bold: (s: string) => string;
+}
+
+export interface GalleryImage {
+	data: string; // base64
+	mimeType: string;
+	label: string;
+}
+
+const THUMB_MAX_WIDTH = 25;
+const GAP = 2; // columns between images
+
+// Monotonic counter for kitty image IDs — avoids birthday-paradox collisions
+// that occurred with the previous Math.random() * 254 approach.
+// Uses the full 24-bit range (1–16,777,215) supported by kitty's true-color encoding.
+let nextImageId = 1;
+function allocateImageId(): number {
+	const id = nextImageId;
+	nextImageId = (nextImageId % 0xffffff) + 1; // wrap at 16M, skip 0
+	return id;
+}
+
+// ── Kitty Unicode Placeholder Protocol ─────────────────────
+// Instead of rendering pixels directly (which ghost across tmux panes),
+// we transmit the image data and then output U+10EEEE placeholder
+// characters with diacritics encoding row/col. Since these are just
+// text characters, tmux manages them per-pane — images appear/disappear
+// naturally when switching panes.
+//
+// Protocol:
+// 1. Transmit image: ESC_G a=T,U=1,f=100,i=<id>,c=<cols>,r=<rows>,q=2; base64 ESC\
+// 2. Print placeholder chars with foreground color set to image_id:
+//    ESC[38;5;<id>m  <U+10EEEE><row_diac><col_diac> ... ESC[39m
+//
+// Diacritics: row 0 = U+0305, row 1 = U+030D, row 2 = U+030E, etc.
+// See kitty docs rowcolumn-diacritics.txt
+
+// Row/column diacritics from kitty's rowcolumn-diacritics.txt
+// These are the combining characters used to encode row and column numbers
+const ROW_COL_DIACRITICS = [
+	0x0305, 0x030d, 0x030e, 0x0310, 0x0312, 0x033d, 0x033e, 0x033f,
+	0x0346, 0x034a, 0x034b, 0x034c, 0x0350, 0x0351, 0x0352, 0x0353,
+	0x0357, 0x035b, 0x0363, 0x0364, 0x0365, 0x0366, 0x0367, 0x0368,
+	0x0369, 0x036a, 0x036b, 0x036c, 0x036d, 0x036e, 0x036f, 0x0483,
+	0x0484, 0x0485, 0x0486, 0x0592, 0x0593, 0x0594, 0x0595, 0x0597,
+	0x0598, 0x0599, 0x059c, 0x059d, 0x059e, 0x059f, 0x05a0, 0x05a1,
+	0x05a8, 0x05a9, 0x05ab, 0x05ac, 0x05af, 0x05c4, 0x0610, 0x0611,
+	0x0612, 0x0613, 0x0614, 0x0615, 0x0616, 0x0617, 0x0618, 0x0619,
+	0x061a, 0x064b, 0x064c, 0x064d, 0x064e, 0x064f, 0x0650, 0x0651,
+	0x0652, 0x0653, 0x0654, 0x0655, 0x0656, 0x0657, 0x0658, 0x0659,
+	0x065a, 0x065b, 0x065c, 0x065d, 0x065e, 0x065f, 0x0670, 0x06d6,
+	0x06d7, 0x06d8, 0x06d9, 0x06da, 0x06db, 0x06dc, 0x06df, 0x06e0,
+	0x06e1, 0x06e2, 0x06e3, 0x06e4, 0x06e7, 0x06e8, 0x06ea, 0x06eb,
+	0x06ec, 0x06ed,
+];
+
+const PLACEHOLDER_CHAR = String.fromCodePoint(0x10EEEE);
+
+function diacriticFor(n: number): string {
+	if (n < ROW_COL_DIACRITICS.length) {
+		return String.fromCodePoint(ROW_COL_DIACRITICS[n]);
+	}
+	// Fallback for very large values (shouldn't happen for thumbnails)
+	return String.fromCodePoint(ROW_COL_DIACRITICS[0]);
+}
+
+function isInTmux(): boolean {
+	return Boolean(process.env.TMUX);
+}
+
+/**
+ * Wrap kitty APC sequences in DCS passthrough for tmux.
+ */
+function wrapForTmux(sequence: string): string {
+	if (!isInTmux()) return sequence;
+	return sequence.replace(
+		/\x1b_G([^\x1b]*)\x1b\\/g,
+		(_match, content) =>
+			`\x1bPtmux;\x1b\x1b_G${content}\x1b\x1b\\\x1b\\`,
+	);
+}
+
+/**
+ * Transmit image and create virtual placement using Unicode placeholder mode.
+ * The image data is sent to kitty but NOT displayed directly.
+ * Display happens via U+10EEEE placeholder characters.
+ */
+function transmitImageWithPlaceholder(
+	base64Data: string,
+	imageId: number,
+	columns: number,
+	rows: number,
+): void {
+	// Transmit image + create virtual placement in one command
+	// a=T: transmit and display, U=1: use unicode placeholders
+	// q=2: suppress all responses (important in tmux)
+	const CHUNK_SIZE = 4096;
+
+	if (base64Data.length <= CHUNK_SIZE) {
+		const seq = `\x1b_Ga=T,U=1,f=100,i=${imageId},c=${columns},r=${rows},q=2;${base64Data}\x1b\\`;
+		process.stdout.write(wrapForTmux(seq));
+	} else {
+		// Chunked transfer
+		let offset = 0;
+		let isFirst = true;
+		while (offset < base64Data.length) {
+			const chunk = base64Data.slice(offset, offset + CHUNK_SIZE);
+			const isLast = offset + CHUNK_SIZE >= base64Data.length;
+			let seq: string;
+
+			if (isFirst) {
+				seq = `\x1b_Ga=T,U=1,f=100,i=${imageId},c=${columns},r=${rows},q=2,m=1;${chunk}\x1b\\`;
+				isFirst = false;
+			} else if (isLast) {
+				seq = `\x1b_Gm=0;${chunk}\x1b\\`;
+			} else {
+				seq = `\x1b_Gm=1;${chunk}\x1b\\`;
+			}
+
+			process.stdout.write(wrapForTmux(seq));
+			offset += CHUNK_SIZE;
+		}
+	}
+}
+
+/**
+ * Delete a kitty image by ID.
+ */
+function deleteImage(imageId: number): void {
+	const seq = `\x1b_Ga=d,d=I,i=${imageId},q=2\x1b\\`;
+	process.stdout.write(wrapForTmux(seq));
+}
+
+/**
+ * Build a row of Unicode placeholder characters for the given image.
+ * Uses foreground color to encode image_id, diacritics to encode row/col.
+ */
+function buildPlaceholderRow(
+	imageId: number,
+	row: number,
+	columns: number,
+): string {
+	// Set foreground color to image_id (using 24-bit true color for large IDs)
+	const r = (imageId >> 16) & 0xff;
+	const g = (imageId >> 8) & 0xff;
+	const b = imageId & 0xff;
+	const fgStart = imageId < 256
+		? `\x1b[38;5;${imageId}m`
+		: `\x1b[38;2;${r};${g};${b}m`;
+	const fgEnd = `\x1b[39m`;
+
+	let line = fgStart;
+
+	// First cell: full diacritics (row + col)
+	line += PLACEHOLDER_CHAR + diacriticFor(row) + diacriticFor(0);
+
+	// Subsequent cells: no diacritics needed — kitty auto-increments
+	// column index from the left neighbor's col diacritic
+	for (let col = 1; col < columns; col++) {
+		line += PLACEHOLDER_CHAR;
+	}
+
+	line += fgEnd;
+	return line;
+}
+
+// ── Gallery Component ──────────────────────────────────────
+
+/**
+ * Renders image thumbnails above the editor using kitty's Unicode
+ * placeholder protocol. Images are part of the text buffer, so
+ * tmux manages them per-pane — no ghosting across panes.
+ */
+export class ImageGallery implements Component {
+	private images: GalleryImage[] = [];
+	private theme: GalleryTheme;
+	private cachedLines?: string[];
+	private cachedWidth?: number;
+	private activeImageIds: number[] = [];
+
+	constructor(theme: GalleryTheme) {
+		this.theme = theme;
+	}
+
+	setImages(images: GalleryImage[]): void {
+		this.images = images;
+		this.invalidate();
+	}
+
+	invalidate(): void {
+		this.cachedLines = undefined;
+		this.cachedWidth = undefined;
+	}
+
+	dispose(): void {
+		for (const id of this.activeImageIds) {
+			deleteImage(id);
+		}
+		this.activeImageIds = [];
+	}
+
+	render(width: number): string[] {
+		if (this.cachedLines && this.cachedWidth === width) {
+			return this.cachedLines;
+		}
+
+		// Delete previous images before re-rendering
+		for (const id of this.activeImageIds) {
+			deleteImage(id);
+		}
+		this.activeImageIds = [];
+
+		if (this.images.length === 0) {
+			this.cachedLines = [];
+			this.cachedWidth = width;
+			return this.cachedLines;
+		}
+
+		const lines: string[] = [];
+		const caps = getCapabilities();
+
+		// Header
+		const count = this.images.length;
+		const headerText =
+			count === 1
+				? " 📎 1 image attached"
+				: ` 📎 ${count} images attached`;
+		lines.push(this.theme.accent(headerText));
+
+		if (caps.images === "kitty") {
+			this.renderKittyHorizontal(lines, width);
+		} else {
+			this.renderTextFallback(lines);
+		}
+
+		this.cachedLines = lines;
+		this.cachedWidth = width;
+		return this.cachedLines;
+	}
+
+	private renderKittyHorizontal(lines: string[], width: number): void {
+		// Calculate per-image thumb width so they all fit side by side
+		const available = width - 2; // padding
+		const totalGaps = Math.max(0, this.images.length - 1) * GAP;
+		const thumbWidth = Math.min(
+			THUMB_MAX_WIDTH,
+			Math.floor((available - totalGaps) / this.images.length),
+		);
+
+		if (thumbWidth < 4) {
+			// Too narrow for horizontal, fall back to text
+			this.renderTextFallback(lines);
+			return;
+		}
+
+		// Prepare each image: transmit data, calculate rows
+		const imageInfos: { imageId: number; rows: number; cols: number }[] = [];
+
+		for (const img of this.images) {
+			// getImageDimensions returns null for corrupt or unrecognised image data;
+			// fall back to a common aspect ratio so the thumbnail still renders.
+			const dims = getImageDimensions(img.data, img.mimeType) || {
+				widthPx: 800,
+				heightPx: 600,
+			};
+
+			const rows = calculateImageRows(dims, thumbWidth, getCellDimensions());
+			const imageId = allocateImageId();
+			this.activeImageIds.push(imageId);
+
+			transmitImageWithPlaceholder(img.data, imageId, thumbWidth, rows);
+			imageInfos.push({ imageId, rows, cols: thumbWidth });
+		}
+
+		const maxRows = Math.max(...imageInfos.map((i) => i.rows));
+
+		// Build horizontal rows: each line has placeholder chars for all images side by side
+		for (let row = 0; row < maxRows; row++) {
+			let line = " ";
+			for (let i = 0; i < this.images.length; i++) {
+				const info = imageInfos[i];
+
+				if (row < info.rows) {
+					// Output placeholder chars for this image at this row
+					line += buildPlaceholderRow(info.imageId, row, info.cols);
+				} else {
+					// Image is shorter, pad with spaces
+					line += " ".repeat(info.cols);
+				}
+
+				if (i < this.images.length - 1) {
+					line += " ".repeat(GAP);
+				}
+			}
+			lines.push(line);
+		}
+
+		// Label row beneath images — middle-truncate and center
+		let labelLine = " ";
+		for (let i = 0; i < this.images.length; i++) {
+			const cols = imageInfos[i].cols;
+			let label = this.images[i].label;
+
+			// Middle-truncate: "pi-clipboard-044c...21ad4.png"
+			if (label.length > cols) {
+				const keep = cols - 1; // 1 char for …
+				const head = Math.ceil(keep / 2);
+				const tail = keep - head;
+				label = label.slice(0, head) + "…" + label.slice(-tail);
+			}
+
+			// Center the label within the column width
+			const totalPad = Math.max(0, cols - label.length);
+			const leftPad = Math.floor(totalPad / 2);
+			const rightPad = totalPad - leftPad;
+			const padded = " ".repeat(leftPad) + label + " ".repeat(rightPad);
+			labelLine += this.theme.dim(padded);
+
+			if (i < this.images.length - 1) {
+				labelLine += " ".repeat(GAP);
+			}
+		}
+		lines.push(labelLine);
+	}
+
+	private renderTextFallback(lines: string[]): void {
+		for (const img of this.images) {
+			lines.push(
+				this.theme.muted(`  ${img.label}`),
+			);
+		}
+	}
+}

package/src/path-utils.ts

@@ -0,0 +1,92 @@
+import fs from "node:fs";
+import fsp from "node:fs/promises";
+import path from "node:path";
+
+const IMAGE_MIME_BY_EXT: Record<string, string> = {
+	png: "image/png",
+	jpg: "image/jpeg",
+	jpeg: "image/jpeg",
+	gif: "image/gif",
+	webp: "image/webp",
+};
+
+export function inferMimeType(filePath: string): string | null {
+	const ext = path.extname(filePath).replace(/^\./, "").toLowerCase();
+	return IMAGE_MIME_BY_EXT[ext] ?? null;
+}
+
+export function looksLikeImagePath(filePath: string): boolean {
+	const mimeType = inferMimeType(filePath);
+	if (!mimeType) return false;
+	try {
+		return fs.statSync(filePath).isFile();
+	} catch {
+		return false;
+	}
+}
+
+/** Async version of looksLikeImagePath — preferred in the poll path to avoid blocking the event loop. */
+export async function looksLikeImagePathAsync(
+	filePath: string,
+): Promise<boolean> {
+	const mimeType = inferMimeType(filePath);
+	if (!mimeType) return false;
+	try {
+		const stat = await fsp.stat(filePath);
+		return stat.isFile();
+	} catch {
+		return false;
+	}
+}
+
+export function resolveMaybeRelativePath(
+	filePath: string,
+	cwd: string,
+): string {
+	return path.isAbsolute(filePath) ? filePath : path.resolve(cwd, filePath);
+}
+
+export function isScreenshotToolName(toolName: string): boolean {
+	return (
+		toolName === "take_screenshot" ||
+		toolName === "chrome_devtools_take_screenshot" ||
+		toolName.endsWith("_take_screenshot")
+	);
+}
+
+export function isScreenshotToolResult(event: {
+	toolName: string;
+	details?: unknown;
+}): boolean {
+	if (isScreenshotToolName(event.toolName)) return true;
+	if (!event.details || typeof event.details !== "object") return false;
+	const maybeTool = (event.details as { tool?: unknown }).tool;
+	return typeof maybeTool === "string" && isScreenshotToolName(maybeTool);
+}
+
+const SCREENSHOT_SAVE_LINE_RE = /^Saved screenshot to\s+(.+)$/gim;
+
+export function extractSavedScreenshotPaths(text: string): string[] {
+	const paths: string[] = [];
+	for (const match of text.matchAll(SCREENSHOT_SAVE_LINE_RE)) {
+		const rawPath = match[1]?.trim();
+		if (!rawPath) continue;
+		paths.push(rawPath.replace(/\.$/, ""));
+	}
+	return paths;
+}
+
+export function collectTextContent(
+	content: Array<{ type: string; text?: string }>,
+): string {
+	return content
+		.filter((item) => item.type === "text" && item.text)
+		.map((item) => item.text!)
+		.join("\n");
+}
+
+export function hasInlineImageContent(
+	content: Array<{ type: string }>,
+): boolean {
+	return content.some((item) => item.type === "image");
+}

package/src/prompt.ts

@@ -0,0 +1,2 @@
+export const PREFER_INLINE_SCREENSHOT_PROMPT =
+	"When you need to inspect a screenshot yourself, prefer screenshot tool calls that return the image inline. Avoid passing filePath to screenshot tools unless the user explicitly asked you to save a file or you do not need to inspect the image content yourself.";

package/src/tool-result-upgrader.ts

@@ -0,0 +1,52 @@
+import type { ContentBlock, ImageContent, TextContent } from "./content.ts";
+import {
+	collectTextContent,
+	extractSavedScreenshotPaths,
+	hasInlineImageContent,
+	isScreenshotToolResult,
+	resolveMaybeRelativePath,
+} from "./path-utils.ts";
+
+export type ToolResultEventLike = {
+	toolName: string;
+	content: ContentBlock[];
+	details?: unknown;
+	isError: boolean;
+};
+
+export async function upgradeScreenshotToolResult(
+	event: ToolResultEventLike,
+	cwd: string,
+	loadImageFromPath: (filePath: string) => Promise<ImageContent | null>,
+): Promise<{ content: ContentBlock[] } | undefined> {
+	if (
+		event.isError ||
+		!isScreenshotToolResult(event) ||
+		hasInlineImageContent(event.content)
+	) {
+		return undefined;
+	}
+
+	const text = collectTextContent(event.content);
+	const savedPaths = extractSavedScreenshotPaths(text);
+	if (savedPaths.length === 0) return undefined;
+
+	const images: ImageContent[] = [];
+	for (const rawPath of savedPaths) {
+		const resolvedPath = resolveMaybeRelativePath(rawPath, cwd);
+		const image = await loadImageFromPath(resolvedPath);
+		if (image) {
+			images.push(image);
+		}
+	}
+
+	if (images.length > 0) {
+		return { content: [...event.content, ...images] };
+	}
+
+	const hint: TextContent = {
+		type: "text",
+		text: "[image-preview: screenshot was saved via filePath but the image file was not readable. If you need to inspect the screenshot agentically, retry the screenshot tool without filePath so the image is returned inline.]",
+	};
+	return { content: [...event.content, hint] };
+}