pi-openui 0.0.0-stage → 0.1.0

package/src/pi/extension.ts

@@ -0,0 +1,190 @@
+// pi extension: render_widget (OpenUI Lang -> interactive terminal widget) plus a few commands.
+import type { ExtensionAPI } from "@earendil-works/pi-coding-agent";
+import { Text, truncateToWidth } from "@earendil-works/pi-tui";
+import { Type } from "typebox";
+import { readFileSync } from "node:fs";
+import { compile, LIMITS, reference, type Compiled, type El } from "../lang/library.ts";
+import { markdownImages } from "../lang/md.ts";
+import { LANG_SAMPLES } from "../lang/samples.ts";
+import { getCapabilities } from "@earendil-works/pi-tui";
+import { loadImage } from "./images.ts";
+import { active, armNewest, entryOf, handleKey, hooks, OpenUIWidget, resetAll, toggleNotesNewest } from "./widget.ts";
+
+const compiled = new Map<string, Compiled>();
+let compiledChars = 0;
+/** A source is the key, so the cache is bounded by size as well as count: 64 sources at the 100 KB limit would be 6 MB. */
+const CACHE_CHARS = 2_000_000;
+export function compileCached(source: string): Compiled {
+	if (source.length > LIMITS.source) return compile(source); // rejected at once; never kept
+	let c = compiled.get(source);
+	if (!c) {
+		c = compile(source);
+		if (compiled.size >= 64 || compiledChars + source.length > CACHE_CHARS) {
+			compiled.clear();
+			compiledChars = 0;
+		}
+		compiled.set(source, c);
+		compiledChars += source.length;
+	}
+	return c;
+}
+
+/** A picture the widget will draw: an Image component, or a `![alt](path)` line in a Markdown or an FAQ answer. */
+export interface Picture {
+	src: string;
+	line: number;
+	via: "Image" | "Markdown picture";
+}
+
+/** Every picture in a compiled tree, wherever it was written. */
+export function picturesIn(el: El | undefined, out: Picture[] = []): Picture[] {
+	if (!el || typeof el !== "object") return out;
+	if (el.type === "Image") out.push({ src: String(el.props.src ?? ""), line: el.line, via: "Image" });
+	const text = el.type === "Markdown" ? el.props.text : el.type === "FaqItem" ? el.props.answer : undefined;
+	if (text !== undefined) for (const m of markdownImages(text)) out.push({ src: m.src, line: el.line, via: "Markdown picture" });
+	for (const v of Object.values(el.props ?? {})) for (const x of Array.isArray(v) ? v : [v]) if (x && typeof x === "object") picturesIn(Array.isArray(x) ? undefined : (x as El), out);
+	return out;
+}
+
+/** Problems the model can fix: the picture must exist and be a format a terminal can show. */
+export function checkImages(root: El | undefined): string[] {
+	const errs: string[] = [];
+	for (const im of picturesIn(root)) {
+		const r = loadImage(im.src);
+		if ("error" in r) errs.push(`line ${im.line}: ${im.via} ${im.src.startsWith("data:") ? "data: URI" : `"${im.src.length > 80 ? im.src.slice(0, 77) + "..." : im.src}"`}: ${r.error}`);
+	}
+	return errs;
+}
+
+export default function (pi: ExtensionAPI) {
+	hooks.sendLive = (text) => pi.sendUserMessage(text, { deliverAs: "followUp" });
+	hooks.sendEcho = (text) => pi.sendMessage({ customType: "openui-click", content: `[widget click → assistant] ${text}`, display: true });
+
+	pi.registerTool({
+		name: "render_widget",
+		label: "render_widget",
+		description:
+			"Draw an interactive widget in the terminal: dashboards, tables, charts (bar, line, area, pie, radar), diagrams (DAG, ER, infrastructure map, sequence), maps, tabs, buttons and choices. Pass OpenUI Lang as `source`. Call with reference=true once to get the component signatures. Clicking a Button or Choice sends its text back to you as the user's next message.",
+		promptSnippet: "render_widget: draw an interactive terminal widget (dashboards, charts, diagrams) from OpenUI Lang",
+		promptGuidelines: [
+			"Before the first render_widget call in a session, call render_widget with reference=true to read the component signatures. Never guess a signature.",
+			"After render_widget succeeds, reply with at most one short sentence. Do not repeat the widget's data in prose.",
+			"If render_widget returns errors, fix exactly the errors listed and call it again with the full corrected source.",
+			"Use plain text, not a widget, for a single value or a short answer. One widget per reply: use Tabs or Card to organise sections.",
+			"A click on a widget Button or Choice arrives as your next user message: act on it, then render a NEW widget if needed.",
+		],
+		parameters: Type.Object({
+			source: Type.Optional(Type.String({ description: "OpenUI Lang source: root = Stack([...]) first, one statement per line" })),
+			reference: Type.Optional(Type.Boolean({ description: "Return the component reference instead of drawing" })),
+		}),
+		renderShell: "self",
+		async execute(_id: string, params: { source?: string; reference?: boolean }) {
+			if (params.reference || params.source === undefined) return { content: [{ type: "text" as const, text: reference() }], details: { kind: "reference" } };
+			const c = compileCached(params.source);
+			if (!c.ok) throw new Error(`The widget was not drawn: ${c.errors.length} error(s). Fix these and call render_widget again with the full corrected source.\n${c.errors.map((e) => `- ${e}`).join("\n")}`);
+			const imgErrs = checkImages(c.root);
+			if (imgErrs.length) throw new Error(`The widget was not drawn: ${imgErrs.length} error(s). Fix these and call render_widget again with the full corrected source.\n${imgErrs.map((e) => `- ${e}`).join("\n")}`);
+			const hasImage = picturesIn(c.root).length > 0;
+			const noPics = hasImage && getCapabilities().images !== "kitty";
+			const note = [...c.warnings, ...(noPics ? ["this terminal cannot show pictures, so each Image is drawn as a labelled frame with its file details"] : [])];
+			const noteText = note.length ? ` Notes: ${note.join("; ")}.` : "";
+			return { content: [{ type: "text" as const, text: `Widget drawn. The user can see and click it in the terminal; do not repeat its contents.${noteText}` }], details: { kind: "widget", source: params.source } };
+		},
+		renderCall(args: any, theme: any) {
+			const n = typeof args?.source === "string" ? args.source.split("\n").length : 0;
+			const label = args?.reference ? "reference" : n ? `${n} lines of OpenUI` : "…";
+			return new Text(theme.fg("toolTitle", theme.bold("render_widget")) + theme.fg("dim", ` ${label}`), 0, 0);
+		},
+		renderResult(result: any, _opts: any, theme: any, ctx: any) {
+			if (ctx.isError) {
+				// the model gets the whole list; show a person the same thing
+				const lines = String(result?.content?.[0]?.text ?? "failed").split("\n").slice(0, 9);
+				return new Text(lines.map((l) => theme.fg("error", l)).join("\n"), 0, 0);
+			}
+			if (result?.details?.kind !== "widget") return new Text(theme.fg("dim", "component reference loaded"), 0, 0);
+			const c = compileCached(String(result.details.source ?? ""));
+			if (!c.ok || !c.root) return new Text(theme.fg("error", "widget could not be restored"), 0, 0);
+			return new OpenUIWidget(ctx.toolCallId, c.root, theme, "live");
+		},
+	});
+
+	pi.registerMessageRenderer("openui-widget", (message: any, _o: any, theme: any) => {
+		const c = compileCached(String(message.details?.source ?? ""));
+		if (!c.ok || !c.root) return undefined;
+		return new OpenUIWidget(String(message.details.id), c.root, theme, message.details.mode === "live" ? "live" : "echo");
+	});
+
+	const draw = (source: string, title: string, ctx: any) => {
+		const c = compileCached(source);
+		if (!c.ok) return ctx.ui.notify(`${title}: ${c.errors.slice(0, 3).join(" | ")}`, "error");
+		pi.sendMessage({ customType: "openui-widget", content: `[widget: ${title}]`, display: true, details: { id: `w-${Date.now()}-${Math.random().toString(36).slice(2, 6)}`, source, mode: "echo" } });
+	};
+
+	pi.registerCommand("openui-gallery", {
+		description: "Draw one example per OpenUI component (or a key prefix: dashboard, table, tabs, line, pie, radar, image, dag, erd, infra, sequence, geo ...). Clicks echo locally.",
+		handler: async (args: string, ctx: any) => {
+			const which = args.trim() || "all";
+			const picked = LANG_SAMPLES.filter((s) => which === "all" || s.key.startsWith(which));
+			if (!picked.length) return ctx.ui.notify(`unknown example "${which}". Try: all, ${LANG_SAMPLES.map((s) => s.key).join(", ")}`, "warning");
+			for (const s of picked) draw(s.source, s.title, ctx);
+		},
+	});
+
+	pi.registerCommand("openui-file", {
+		description: "Draw a widget from an OpenUI Lang file: /openui-file path/to/widget.oui",
+		handler: async (args: string, ctx: any) => {
+			const path = args.trim();
+			if (!path) return ctx.ui.notify("usage: /openui-file path/to/widget.oui", "warning");
+			try {
+				draw(readFileSync(path, "utf8"), path.split("/").pop() ?? path, ctx);
+			} catch (e) {
+				ctx.ui.notify(`cannot read ${path}: ${(e as Error).message}`, "error");
+			}
+		},
+	});
+
+	pi.registerCommand("openui-notes", {
+		description: "Write notes on the newest widget: click an element (or Tab to it and press Enter), type, Enter to save. Then add them to your message.",
+		handler: async (_a: string, ctx: any) => {
+			if (!toggleNotesNewest()) ctx.ui.notify("no widget yet: run /openui-gallery", "warning");
+		},
+	});
+
+	pi.registerCommand("openui-focus", {
+		description: "Give the keyboard to the newest widget (Tab/arrows move, Enter activates, Esc leaves)",
+		handler: async (_a: string, ctx: any) => {
+			if (!armNewest()) ctx.ui.notify("no widget yet: run /openui-gallery", "warning");
+		},
+	});
+
+	let unsub: (() => void) | undefined;
+	pi.on("session_start", async (_e: any, ctx: any) => {
+		if (!ctx.hasUI) return;
+		unsub?.();
+		unsub = ctx.ui.onTerminalInput((data: string) => (handleKey(data) ? { consume: true } : undefined));
+		// the note editor is pi's own multi-line editor: Enter saves, Shift+Enter adds a line, Esc cancels
+		hooks.prompt = (title: string, initial: string) => ctx.ui.editor(title, initial);
+		hooks.toEditor = (text: string) => ctx.ui.pasteToEditor(text);
+		ctx.ui.setWidget("openui-hint", (tui: any, theme: any) => {
+			hooks.requestRender = () => tui.requestRender();
+			return {
+				render: (w: number) => {
+					if (!active.id) return [];
+					const e = entryOf(active.id);
+					const keys = e?.state.noteMode
+						? "notes: click or Enter writes a note  arrows/Tab move  Shift+Enter picks the parent  Esc leaves"
+						: e?.state.focus
+							? "arrows move  Tab next  Enter activate  Esc close/leave"
+							: "Tab or arrows select  Enter activate  Esc leave";
+					return [truncateToWidth(theme.fg("accent", " openui ") + theme.fg("dim", keys), w)];
+				},
+				invalidate() {},
+			};
+		});
+	});
+	pi.on("session_shutdown", async () => {
+		unsub?.();
+		unsub = undefined;
+		resetAll(); // a pi process lives for days across /new and resume: each session's widgets must not pile up
+	});
+}

package/src/pi/images.ts

@@ -0,0 +1,185 @@
+// Pictures for the Image component. pi-tui's own Image does the sizing and the Kitty encoding (so cell maths, image ids
+// and the alt-screen cropping metadata are pi's), this file loads the bytes and keeps the placements.
+import { getCapabilities, getImageDimensions, Image } from "@earendil-works/pi-tui";
+import { readFileSync, statSync } from "node:fs";
+import { homedir } from "node:os";
+import { basename, isAbsolute, resolve } from "node:path";
+import type { ImageService } from "../blocks/types.ts";
+
+export const MAX_IMAGE_BYTES = 12 * 1024 * 1024;
+
+export interface LoadedImage {
+	base64: string;
+	mime: string;
+	name: string;
+	bytes: number;
+	dims: { widthPx: number; heightPx: number } | null;
+}
+
+function sniff(buf: Buffer): string | null {
+	if (buf.length >= 8 && buf[0] === 0x89 && buf[1] === 0x50 && buf[2] === 0x4e && buf[3] === 0x47) return "image/png";
+	if (buf.length >= 3 && buf[0] === 0xff && buf[1] === 0xd8 && buf[2] === 0xff) return "image/jpeg";
+	if (buf.length >= 6 && buf.subarray(0, 4).toString("ascii") === "GIF8") return "image/gif";
+	if (buf.length >= 12 && buf.subarray(0, 4).toString("ascii") === "RIFF" && buf.subarray(8, 12).toString("ascii") === "WEBP") return "image/webp";
+	return null;
+}
+
+const cache = new Map<string, { stamp: string; value: LoadedImage | { error: string } }>();
+/**
+ * Pictures are kept as base64 (a file of 12 MB is 16 MB of string), so the cache is bounded by size, not by count: 32 large
+ * screenshots would otherwise pin half a gigabyte for as long as pi runs.
+ */
+const CACHE_BUDGET = 48 * 1024 * 1024;
+let cacheBytes = 0;
+const weigh = (v: LoadedImage | { error: string }) => ("base64" in v ? v.base64.length : 0);
+export function imageCacheStats() {
+	return { entries: cache.size, bytes: cacheBytes, budget: CACHE_BUDGET };
+}
+function remember(path: string, stamp: string, value: LoadedImage | { error: string }) {
+	const old = cache.get(path);
+	if (old) cacheBytes -= weigh(old.value);
+	cache.delete(path);
+	cache.set(path, { stamp, value });
+	cacheBytes += weigh(value);
+	// oldest first; the newest picture is always kept, whatever its size
+	for (const [k, v] of cache) {
+		if (cache.size <= 1 || (cache.size <= 32 && cacheBytes <= CACHE_BUDGET)) break;
+		cache.delete(k);
+		cacheBytes -= weigh(v.value);
+	}
+}
+
+/** Read a local file (absolute, ~/ or relative to the working directory) or a base64 data: URI. Never throws. */
+export function loadImage(src: string): LoadedImage | { error: string } {
+	const s = String(src ?? "").trim();
+	if (!s) return { error: "src is empty" };
+	if (/^https?:\/\//i.test(s)) return { error: "remote URLs are not fetched; use a local file path or a data: URI" };
+	let stamp = "data";
+	let buf: Buffer;
+	let name = "image";
+	if (s.startsWith("data:")) {
+		const m = /^data:([^;,]*)(;[^,]*)?,(.*)$/s.exec(s);
+		if (!m || !/;base64/i.test(m[2] ?? "")) return { error: "a data: URI must be base64, like data:image/png;base64,..." };
+		try {
+			buf = Buffer.from(m[3], "base64");
+		} catch {
+			return { error: "the data: URI is not valid base64" };
+		}
+		name = `inline.${(m[1] || "image/png").split("/")[1] ?? "png"}`;
+	} else {
+		const path = s.startsWith("~/") ? resolve(homedir(), s.slice(2)) : isAbsolute(s) ? s : resolve(process.cwd(), s);
+		let st;
+		try {
+			st = statSync(path);
+		} catch {
+			return { error: `file not found: ${path}` };
+		}
+		if (!st.isFile()) return { error: `not a file: ${path}` };
+		if (st.size > MAX_IMAGE_BYTES) return { error: `file is ${Math.round(st.size / 1024 / 1024)} MB; the limit is ${MAX_IMAGE_BYTES / 1024 / 1024} MB` };
+		stamp = `${st.mtimeMs}:${st.size}`;
+		const hit = cache.get(path);
+		if (hit && hit.stamp === stamp) return hit.value;
+		try {
+			buf = readFileSync(path);
+		} catch (e) {
+			return { error: `cannot read ${path}: ${(e as Error).message}` };
+		}
+		name = basename(path);
+		const value = decode(buf, name);
+		remember(path, stamp, value);
+		return value;
+	}
+	if (buf.length > MAX_IMAGE_BYTES) return { error: "the data: URI is too large" };
+	return decode(buf, name);
+}
+
+function decode(buf: Buffer, name: string): LoadedImage | { error: string } {
+	const mime = sniff(buf);
+	if (!mime) {
+		const head = buf.subarray(0, 200).toString("utf8").trimStart();
+		if (/^<(\?xml|svg)/i.test(head)) return { error: "SVG cannot be drawn in a terminal; use png, jpg, gif or webp" };
+		return { error: "not a png, jpg, gif or webp picture" };
+	}
+	const base64 = buf.toString("base64");
+	return { base64, mime, name, bytes: buf.length, dims: getImageDimensions(base64, mime) };
+}
+
+const kb = (n: number) => (n >= 1024 * 1024 ? `${(n / 1024 / 1024).toFixed(1)} MB` : `${Math.max(1, Math.round(n / 1024))} KB`);
+
+interface Placement {
+	sequence: string;
+	cols: number;
+	rows: number;
+}
+
+function hash(s: string): string {
+	let h = 2166136261;
+	for (let i = 0; i < s.length; i++) h = Math.imul(h ^ s.charCodeAt(i), 16777619);
+	return (h >>> 0).toString(36) + s.length.toString(36);
+}
+
+export interface PiImageService extends ImageService {
+	/** The terminal sequence that draws a placed image (the first line pi-tui's Image returns). */
+	sequence(key: string): string | null;
+	/** Called after a picture that could not be drawn yet becomes drawable (pi loads its JPEG/GIF/WebP converter lazily). */
+	onLate: (() => void) | null;
+	/** Forget every placement: the terminal reported another cell size, so the same picture now needs other columns and rows. */
+	reset(): void;
+}
+
+export function createImageService(): PiImageService {
+	const placements = new Map<string, Placement>();
+	const pending = new Map<string, number>();
+	// a placement holds the picture's whole transmit command: bound them by size too
+	let placedChars = 0;
+	const svc: PiImageService = {
+		onLate: null,
+		reset() {
+			placements.clear();
+			pending.clear();
+			placedChars = 0;
+		},
+		sequence: (key) => placements.get(key)?.sequence ?? null,
+		describe(src) {
+			const img = loadImage(src);
+			if ("error" in img) return img.error;
+			const fmt = img.mime.split("/")[1];
+			return [img.name, img.dims ? `${img.dims.widthPx}×${img.dims.heightPx}` : "", fmt, kb(img.bytes)].filter(Boolean).join(" · ");
+		},
+		place(src, maxCols, maxRows) {
+			if (getCapabilities().images !== "kitty") return null;
+			const img = loadImage(src);
+			if ("error" in img) return null;
+			const key = `${hash(src)}:${maxCols}x${maxRows}`;
+			let p = placements.get(key);
+			if (!p) {
+				const comp = new Image(img.base64, img.mime, { fallbackColor: (s: string) => s }, { maxWidthCells: maxCols, maxHeightCells: maxRows, filename: img.name }, img.dims ?? undefined);
+				const lines = comp.render(maxCols + 2);
+				const seq = lines[0] ?? "";
+				if (!seq.includes("\x1b_G")) {
+					// pi converts JPEG, GIF and WebP lazily after startup; look again shortly so the picture replaces the frame
+					const tries = pending.get(key) ?? 0;
+					if (tries < 8 && img.mime !== "image/png") {
+						pending.set(key, tries + 1);
+						const t = setTimeout(() => svc.onLate?.(), 350);
+						if (typeof (t as any).unref === "function") (t as any).unref();
+					}
+					return null;
+				}
+				const c = /\x1b_G[^;]*,c=(\d+)/.exec(seq);
+				p = { sequence: seq, cols: c ? Number(c[1]) : maxCols, rows: lines.length };
+				placements.set(key, p);
+				placedChars += p.sequence.length;
+				for (const [k, old] of placements) {
+					if (placements.size <= 1 || (placements.size <= 40 && placedChars <= CACHE_BUDGET)) break;
+					placements.delete(k);
+					placedChars -= old.sequence.length;
+				}
+			}
+			return { key, cols: p.cols, rows: p.rows };
+		},
+	};
+	return svc;
+}
+
+export const imageService = createImageService();