pi-agent-zh 17.1.8

package/src/tools/computer/exposure.ts

@@ -0,0 +1,38 @@
+import type { Model } from "@oh-my-pi/pi-ai";
+
+export type ComputerExposureMode = "native" | "function" | "unavailable";
+
+export interface ComputerExposureOptions {
+	azureBaseUrl?: string;
+	azureResourceName?: string;
+}
+
+function isFirstPartyAzureEndpoint(baseUrl: string): boolean {
+	try {
+		const url = new URL(baseUrl);
+		return (
+			url.protocol === "https:" &&
+			(url.hostname.endsWith(".openai.azure.com") || url.hostname === "models.inference.ai.azure.com")
+		);
+	} catch {
+		return false;
+	}
+}
+
+/** Match the provider transport's effective Computer Use tool representation. */
+export function computerExposureMode(
+	model: Model | undefined,
+	options?: ComputerExposureOptions,
+): ComputerExposureMode {
+	if (!model) return "unavailable";
+	if (model.supportsComputerUse !== true) return "function";
+	if (model.api !== "azure-openai-responses" || model.supportsComputerUseConfig !== undefined) return "native";
+	const baseUrl =
+		options?.azureBaseUrl?.trim() ||
+		process.env.AZURE_OPENAI_BASE_URL?.trim() ||
+		(options?.azureResourceName || process.env.AZURE_OPENAI_RESOURCE_NAME
+			? `https://${options?.azureResourceName || process.env.AZURE_OPENAI_RESOURCE_NAME}.openai.azure.com/openai/v1`
+			: undefined) ||
+		model.baseUrl;
+	return baseUrl && isFirstPartyAzureEndpoint(baseUrl) ? "native" : "function";
+}

package/src/tools/computer/protocol.ts

@@ -0,0 +1,28 @@
+import type { DesktopAction, DesktopCapabilities, DesktopCapture, DesktopSessionOptions } from "@oh-my-pi/pi-natives";
+
+export const COMPUTER_WORKER_ARG = "__omp_worker_computer";
+
+export type ComputerWorkerInbound =
+	| { type: "ping"; id: string }
+	| { type: "init"; options: DesktopSessionOptions }
+	| { type: "execute"; id: string; actions: DesktopAction[] }
+	| { type: "close" };
+
+export type ComputerWorkerOutbound =
+	| { type: "pong"; id: string }
+	| { type: "ready"; capabilities: DesktopCapabilities }
+	| { type: "result"; id: string; capture: DesktopCapture; capabilities: DesktopCapabilities }
+	| { type: "error"; id?: string; error: ComputerWorkerError }
+	| { type: "closed" };
+
+export interface ComputerWorkerError {
+	name: string;
+	message: string;
+	stack?: string;
+}
+
+export interface ComputerWorkerTransport {
+	send(message: ComputerWorkerOutbound, transfer?: Bun.Transferable[]): void;
+	onMessage(handler: (message: ComputerWorkerInbound) => void): () => void;
+	close(): void;
+}

package/src/tools/computer/supervisor.ts

@@ -0,0 +1,306 @@
+import type { DesktopAction, DesktopCapabilities, DesktopCapture, DesktopSessionOptions } from "@oh-my-pi/pi-natives";
+import { withTimeout } from "@oh-my-pi/pi-utils/async";
+import * as logger from "@oh-my-pi/pi-utils/logger";
+import { workerHostEntry } from "@oh-my-pi/pi-utils/worker-host";
+import { ToolAbortError, ToolError } from "../tool-errors";
+import {
+	COMPUTER_WORKER_ARG,
+	type ComputerWorkerError,
+	type ComputerWorkerInbound,
+	type ComputerWorkerOutbound,
+} from "./protocol";
+
+const START_TIMEOUT_MS = 10_000;
+const CLOSE_TIMEOUT_MS = 1_500;
+const SMOKE_TIMEOUT_MS = 5_000;
+
+export interface ComputerController {
+	readonly capabilities: DesktopCapabilities | undefined;
+	execute(actions: DesktopAction[], signal?: AbortSignal): Promise<DesktopCapture>;
+	close(): Promise<void>;
+}
+
+export interface ComputerWorkerHandle {
+	send(message: ComputerWorkerInbound): void;
+	onMessage(handler: (message: ComputerWorkerOutbound) => void): () => void;
+	onError(handler: (error: Error) => void): () => void;
+	terminate(): Promise<void>;
+}
+
+export interface ComputerSupervisorTimeouts {
+	startMs: number;
+	closeMs: number;
+}
+
+const DEFAULT_TIMEOUTS: ComputerSupervisorTimeouts = {
+	startMs: START_TIMEOUT_MS,
+	closeMs: CLOSE_TIMEOUT_MS,
+};
+export type ComputerWorkerFactory = () => ComputerWorkerHandle;
+
+function workerError(error: ComputerWorkerError): Error {
+	const result = new ToolError(error.message);
+	result.name = error.name;
+	if (error.stack) result.stack = error.stack;
+	return result;
+}
+
+function wrapWorker(worker: Worker): ComputerWorkerHandle {
+	return {
+		send(message) {
+			worker.postMessage(message);
+		},
+		onMessage(handler) {
+			const listener = (event: MessageEvent): void => handler(event.data as ComputerWorkerOutbound);
+			worker.addEventListener("message", listener);
+			return () => worker.removeEventListener("message", listener);
+		},
+		onError(handler) {
+			const listener = (event: ErrorEvent): void =>
+				handler(event.error instanceof Error ? event.error : new Error(event.message));
+			worker.addEventListener("error", listener);
+			return () => worker.removeEventListener("error", listener);
+		},
+		async terminate() {
+			worker.terminate();
+		},
+	};
+}
+
+export function spawnComputerWorker(): ComputerWorkerHandle {
+	const hostEntry = workerHostEntry();
+	const worker = hostEntry
+		? new Worker(hostEntry, { type: "module", argv: [COMPUTER_WORKER_ARG] })
+		: new Worker(new URL("./worker-entry.ts", import.meta.url).href, { type: "module" });
+	return wrapWorker(worker);
+}
+
+interface PendingRequest {
+	resolve(capture: DesktopCapture): void;
+	reject(error: unknown): void;
+}
+
+export class ComputerSupervisor implements ComputerController {
+	#worker?: ComputerWorkerHandle;
+	#startPromise?: Promise<void>;
+	#capabilities?: DesktopCapabilities;
+	#pending = new Map<string, PendingRequest>();
+	#nextId = 0;
+	#serial: Promise<void> = Promise.resolve();
+	#closed = false;
+	#unsubscribeMessage?: () => void;
+	#unsubscribeError?: () => void;
+
+	constructor(
+		private readonly options: DesktopSessionOptions,
+		private readonly createWorker: ComputerWorkerFactory = spawnComputerWorker,
+		private readonly timeouts: ComputerSupervisorTimeouts = DEFAULT_TIMEOUTS,
+	) {}
+
+	get capabilities(): DesktopCapabilities | undefined {
+		return this.#capabilities;
+	}
+
+	execute(actions: DesktopAction[], signal?: AbortSignal): Promise<DesktopCapture> {
+		if (this.#closed) return Promise.reject(new ToolError("Computer session is closed"));
+		const run = (): Promise<DesktopCapture> => this.#execute(actions, signal);
+		const result = this.#serial.then(run, run);
+		this.#serial = result.then(
+			() => undefined,
+			() => undefined,
+		);
+		return result;
+	}
+
+	async #execute(actions: DesktopAction[], signal?: AbortSignal): Promise<DesktopCapture> {
+		if (this.#closed) throw new ToolError("Computer session is closed");
+		if (signal?.aborted) throw new ToolAbortError();
+		await this.#start();
+		if (signal?.aborted) throw new ToolAbortError();
+		const id = `computer-${++this.#nextId}`;
+		const request = Promise.withResolvers<DesktopCapture>();
+		this.#pending.set(id, request);
+		this.#worker!.send({ type: "execute", id, actions });
+		if (!signal) return request.promise;
+
+		const aborted = Promise.withResolvers<never>();
+		const onAbort = (): void => aborted.reject(new ToolAbortError());
+		signal.addEventListener("abort", onAbort, { once: true });
+		try {
+			return await Promise.race([request.promise, aborted.promise]);
+		} catch (error) {
+			if (error instanceof ToolAbortError) await this.#terminate(error);
+			throw error;
+		} finally {
+			signal.removeEventListener("abort", onAbort);
+		}
+	}
+
+	#start(): Promise<void> {
+		if (this.#startPromise) return this.#startPromise;
+		const ready = Promise.withResolvers<void>();
+		try {
+			logger.debug("Starting native computer worker", {
+				backend: this.options.backend,
+				display: this.options.display,
+				maxWidth: this.options.maxWidth,
+				maxHeight: this.options.maxHeight,
+			});
+			const worker = this.createWorker();
+			this.#worker = worker;
+			this.#unsubscribeMessage = worker.onMessage(message => {
+				if (message.type === "ready") {
+					this.#capabilities = message.capabilities;
+					logger.debug("Native computer worker ready", {
+						backend: message.capabilities.backend,
+						capturePermission: message.capabilities.capturePermission,
+						inputPermission: message.capabilities.inputPermission,
+						displayCount: message.capabilities.displayCount,
+					});
+					ready.resolve();
+					return;
+				}
+				if (message.type === "result") {
+					this.#capabilities = message.capabilities;
+					const pending = this.#pending.get(message.id);
+					this.#pending.delete(message.id);
+					pending?.resolve(message.capture);
+					logger.debug("Native computer capture completed", {
+						requestId: message.id,
+						backend: message.capture.backend,
+						capturePermission: message.capture.capturePermission,
+						inputPermission: message.capture.inputPermission,
+						width: message.capture.width,
+						height: message.capture.height,
+					});
+					return;
+				}
+				if (message.type === "error") {
+					const error = workerError(message.error);
+					logger.warn("Native computer worker request failed", {
+						requestId: message.id,
+						errorName: message.error.name,
+					});
+					if (message.id) {
+						const pending = this.#pending.get(message.id);
+						this.#pending.delete(message.id);
+						pending?.reject(error);
+					} else {
+						ready.reject(error);
+					}
+				}
+			});
+			this.#unsubscribeError = worker.onError(error => {
+				ready.reject(error);
+				void this.#terminate(error);
+			});
+			worker.send({ type: "init", options: this.options });
+		} catch (error) {
+			ready.reject(error);
+		}
+		this.#startPromise = withTimeout(
+			ready.promise,
+			this.timeouts.startMs,
+			"Timed out starting native computer worker",
+		).catch(async error => {
+			await this.#terminate(error);
+			throw error;
+		});
+		return this.#startPromise;
+	}
+
+	async #terminate(reason: unknown): Promise<void> {
+		logger.debug("Terminating native computer worker", {
+			reason: reason instanceof Error ? reason.name : typeof reason,
+		});
+		const worker = this.#worker;
+		this.#worker = undefined;
+		this.#startPromise = undefined;
+		this.#capabilities = undefined;
+		this.#unsubscribeMessage?.();
+		this.#unsubscribeMessage = undefined;
+		this.#unsubscribeError?.();
+		this.#unsubscribeError = undefined;
+		for (const pending of this.#pending.values()) pending.reject(reason);
+		this.#pending.clear();
+		await worker?.terminate();
+	}
+
+	async close(): Promise<void> {
+		if (this.#closed) return;
+		this.#closed = true;
+		const worker = this.#worker;
+		if (!worker) return;
+		const closed = Promise.withResolvers<void>();
+		const unsubscribe = worker.onMessage(message => {
+			if (message.type === "closed") closed.resolve();
+		});
+		try {
+			worker.send({ type: "close" });
+			await withTimeout(closed.promise, this.timeouts.closeMs, "Timed out closing native computer worker");
+		} catch {
+			// Forced termination below is the bounded close fallback.
+		} finally {
+			unsubscribe();
+			await this.#terminate(new ToolError("Computer session closed"));
+		}
+	}
+}
+
+const ownedSupervisors = new Map<string, Set<ComputerController>>();
+
+export function registerComputerController(ownerId: string | undefined, controller: ComputerController): () => void {
+	if (!ownerId) return () => {};
+	const controllers = ownedSupervisors.get(ownerId) ?? new Set<ComputerController>();
+	controllers.add(controller);
+	ownedSupervisors.set(ownerId, controllers);
+	return () => {
+		controllers.delete(controller);
+		if (controllers.size === 0) ownedSupervisors.delete(ownerId);
+	};
+}
+
+export async function releaseComputerSessionsForOwner(ownerId: string | undefined): Promise<void> {
+	if (!ownerId) return;
+	const controllers = ownedSupervisors.get(ownerId);
+	if (!controllers) return;
+	ownedSupervisors.delete(ownerId);
+	await Promise.allSettled(Array.from(controllers, controller => controller.close()));
+}
+
+export async function smokeTestComputerWorker(
+	timeoutMs = SMOKE_TIMEOUT_MS,
+	createWorker: ComputerWorkerFactory = spawnComputerWorker,
+): Promise<void> {
+	const worker = createWorker();
+	const exchange = async (
+		message: ComputerWorkerInbound,
+		expected: ComputerWorkerOutbound["type"],
+		failureMessage: string,
+	): Promise<void> => {
+		const response = Promise.withResolvers<void>();
+		const unsubscribeMessage = worker.onMessage(received => {
+			if (received.type === expected) response.resolve();
+			else if (received.type === "error") response.reject(workerError(received.error));
+		});
+		const unsubscribeError = worker.onError(error => response.reject(error));
+		try {
+			worker.send(message);
+			await withTimeout(response.promise, timeoutMs, failureMessage);
+		} finally {
+			unsubscribeMessage();
+			unsubscribeError();
+		}
+	};
+
+	try {
+		await exchange(
+			{ type: "init", options: { backend: "auto", display: "all", maxWidth: 1920, maxHeight: 1200 } },
+			"ready",
+			"Computer worker smoke initialization timed out",
+		);
+		await exchange({ type: "close" }, "closed", "Computer worker smoke close timed out");
+	} finally {
+		await worker.terminate();
+	}
+}

package/src/tools/computer/worker-entry.ts

@@ -0,0 +1,34 @@
+import { parentPort } from "node:worker_threads";
+import { consumeWorkerInbox, isWorkerHostSelector } from "@oh-my-pi/pi-utils/worker-host";
+import type { ComputerWorkerInbound, ComputerWorkerTransport } from "./protocol";
+import { ComputerWorkerCore } from "./worker";
+
+export function startComputerWorker(): void {
+	if (!parentPort) throw new Error("computer-worker-entry: missing parentPort");
+
+	const port = parentPort;
+	const inbox = consumeWorkerInbox();
+	const transport: ComputerWorkerTransport = {
+		send(message, transfer) {
+			port.postMessage(message, transfer ?? []);
+		},
+		onMessage(handler) {
+			if (inbox) return inbox.bind(message => handler(message as ComputerWorkerInbound));
+			const listener = (message: unknown): void => handler(message as ComputerWorkerInbound);
+			port.on("message", listener);
+			return () => port.off("message", listener);
+		},
+		close() {
+			port.close();
+		},
+	};
+
+	new ComputerWorkerCore(transport);
+}
+
+// Bun workers report `import.meta.main === false`. The source fallback still
+// enters this file directly, while packaged CLI workers carry the selector and
+// start the named entry only after installing its inbox.
+if (!Bun.isMainThread && !process.argv.some(isWorkerHostSelector) && import.meta.path === Bun.main) {
+	startComputerWorker();
+}

package/src/tools/computer/worker.ts

@@ -0,0 +1,131 @@
+import type { DesktopAction, DesktopCapture, DesktopSession, DesktopSessionOptions } from "@oh-my-pi/pi-natives";
+import { createDesktopSession } from "@oh-my-pi/pi-natives/desktop";
+import type { ComputerWorkerError, ComputerWorkerInbound, ComputerWorkerTransport } from "./protocol";
+
+export interface NativeDesktopSession {
+	readonly capabilities: DesktopSession["capabilities"];
+	capture(): Promise<DesktopCapture>;
+	execute(actions: DesktopAction[]): Promise<DesktopCapture>;
+	close(): Promise<void>;
+}
+
+export type NativeDesktopSessionFactory = (options: DesktopSessionOptions) => NativeDesktopSession;
+
+const COORDINATE_ACTIONS: ReadonlySet<DesktopAction["type"]> = new Set([
+	"click",
+	"double_click",
+	"drag",
+	"move",
+	"scroll",
+]);
+
+function serializeError(error: unknown): ComputerWorkerError {
+	if (error instanceof Error) {
+		return { name: error.name, message: error.message, ...(error.stack ? { stack: error.stack } : {}) };
+	}
+	return { name: "Error", message: String(error) };
+}
+
+function captureTransfer(capture: DesktopCapture): Bun.Transferable[] {
+	const buffer = capture.data.buffer;
+	return buffer instanceof ArrayBuffer ? [buffer] : [];
+}
+
+export class ComputerWorkerCore {
+	#session?: NativeDesktopSession;
+	#hasReturnedFrame = false;
+	#closed = false;
+	#tail: Promise<void> = Promise.resolve();
+	readonly #unsubscribe: () => void;
+
+	constructor(
+		private readonly transport: ComputerWorkerTransport,
+		private readonly createSession: NativeDesktopSessionFactory = createDesktopSession,
+	) {
+		this.#unsubscribe = transport.onMessage(message => this.#onMessage(message));
+	}
+
+	#onMessage(message: ComputerWorkerInbound): void {
+		if (message.type === "ping") {
+			this.transport.send({ type: "pong", id: message.id });
+			return;
+		}
+		if (message.type === "close") {
+			this.#tail = this.#tail.then(() => this.#close());
+			return;
+		}
+		if (message.type === "init") {
+			this.#tail = this.#tail.then(() => this.#init(message.options));
+			return;
+		}
+		this.#tail = this.#tail.then(() => this.#execute(message.id, message.actions));
+	}
+
+	async #init(options: DesktopSessionOptions): Promise<void> {
+		if (this.#closed) return;
+		if (this.#session) {
+			this.transport.send({
+				type: "error",
+				error: { name: "Error", message: "Computer worker already initialized" },
+			});
+			return;
+		}
+		try {
+			this.#session = this.createSession(options);
+			this.#hasReturnedFrame = false;
+			this.transport.send({ type: "ready", capabilities: this.#session.capabilities });
+		} catch (error) {
+			this.transport.send({ type: "error", error: serializeError(error) });
+		}
+	}
+
+	async #execute(id: string, actions: DesktopAction[]): Promise<void> {
+		const session = this.#session;
+		if (!session) {
+			this.transport.send({
+				type: "error",
+				id,
+				error: { name: "Error", message: "Computer worker is not initialized" },
+			});
+			return;
+		}
+		try {
+			if (!this.#hasReturnedFrame && actions.some(action => COORDINATE_ACTIONS.has(action.type))) {
+				this.transport.send({
+					type: "error",
+					id,
+					error: {
+						name: "Error",
+						message:
+							"Coordinate computer actions require a screenshot returned to the provider; request a screenshot first",
+					},
+				});
+				return;
+			}
+			const capture = await session.execute(actions);
+			this.transport.send(
+				{ type: "result", id, capture, capabilities: session.capabilities },
+				captureTransfer(capture),
+			);
+			this.#hasReturnedFrame = true;
+		} catch (error) {
+			this.transport.send({ type: "error", id, error: serializeError(error) });
+		}
+	}
+
+	async #close(): Promise<void> {
+		if (this.#closed) return;
+		this.#closed = true;
+		try {
+			await this.#session?.close();
+		} catch (error) {
+			this.transport.send({ type: "error", error: serializeError(error) });
+		} finally {
+			this.#session = undefined;
+			this.#hasReturnedFrame = false;
+			this.#unsubscribe();
+			this.transport.send({ type: "closed" });
+			this.transport.close();
+		}
+	}
+}

package/src/tools/computer-renderer.ts

@@ -0,0 +1,108 @@
+import type { Component } from "@oh-my-pi/pi-tui";
+import { Text } from "@oh-my-pi/pi-tui";
+import { sanitizeText } from "@oh-my-pi/pi-utils";
+import type { RenderResultOptions } from "../extensibility/custom-tools/types";
+import type { Theme } from "../modes/theme/theme";
+import { framedBlock, renderStatusLine } from "../tui";
+import type { ComputerToolDetails } from "./computer";
+import { replaceTabs, truncateToWidth } from "./render-utils";
+
+interface ComputerRenderArgs {
+	actions?: Array<{ type?: unknown }>;
+}
+
+interface ComputerRenderResult {
+	content: Array<{ type: string; text?: string }>;
+	details?: unknown;
+	isError?: boolean;
+}
+
+function clean(value: unknown, width = 100): string {
+	const text = typeof value === "string" ? value : JSON.stringify(value) || String(value);
+	return truncateToWidth(replaceTabs(sanitizeText(text)).replace(/[\r\n]+/g, " "), width);
+}
+
+function isComputerToolDetails(value: unknown): value is ComputerToolDetails {
+	if (!value || typeof value !== "object") return false;
+	const details = value as Partial<ComputerToolDetails>;
+	return (
+		Array.isArray(details.actions) &&
+		Array.isArray(details.displays) &&
+		typeof details.width === "number" &&
+		typeof details.height === "number"
+	);
+}
+
+function actionDescription(args: ComputerRenderArgs | undefined): string | undefined {
+	if (!Array.isArray(args?.actions) || args.actions.length === 0) return undefined;
+	return clean(args.actions.map(action => (typeof action?.type === "string" ? action.type : "action")).join(" → "));
+}
+
+function resultDescription(details: ComputerToolDetails): string {
+	return clean(`${details.actions.join(" → ")} · ${details.width}×${details.height}`, 120);
+}
+
+function errorDescription(result: ComputerRenderResult, args: ComputerRenderArgs | undefined): string | undefined {
+	const text = result.content.find(item => item.type === "text" && typeof item.text === "string")?.text;
+	return text ? clean(text, 120) : actionDescription(args);
+}
+
+export const computerToolRenderer = {
+	mergeCallAndResult: true,
+	renderCall(args: ComputerRenderArgs, _options: RenderResultOptions, theme: Theme): Component {
+		return new Text(
+			renderStatusLine({ icon: "pending", title: "Computer", description: actionDescription(args) }, theme),
+			0,
+			0,
+		);
+	},
+	renderResult(
+		result: ComputerRenderResult,
+		options: RenderResultOptions,
+		theme: Theme,
+		args?: ComputerRenderArgs,
+	): Component {
+		const details = isComputerToolDetails(result.details) ? result.details : undefined;
+		const header = renderStatusLine(
+			result.isError
+				? { icon: "error", title: "Computer", description: errorDescription(result, args) }
+				: { icon: "success", title: "Computer", description: details ? resultDescription(details) : undefined },
+			theme,
+		);
+		if (!details) return new Text(header, 0, 0);
+		return framedBlock(theme, width => {
+			const body: string[] = [
+				theme.fg(
+					"dim",
+					`backend ${clean(details.backend)}${details.displayServer ? ` · server ${clean(details.displayServer)}` : ""} · capture ${clean(details.capturePermission)} · input ${clean(details.inputPermission)} · ${details.displays.length} display(s)`,
+				),
+			];
+			const displayLimit = options.expanded ? details.displays.length : Math.min(details.displays.length, 3);
+			for (const display of details.displays.slice(0, displayLimit)) {
+				body.push(
+					theme.fg(
+						"toolOutput",
+						clean(
+							`${display.id}${display.name ? ` ${display.name}` : ""}: logical ${display.x},${display.y} ${display.width}×${display.height}; pixels ${display.pixelX},${display.pixelY} ${display.pixelWidth}×${display.pixelHeight}; scale ${display.scale}${display.isPrimary ? "; primary" : ""}`,
+							160,
+						),
+					),
+				);
+			}
+			if (displayLimit < details.displays.length) {
+				body.push(theme.fg("dim", `… ${details.displays.length - displayLimit} more display(s)`));
+			}
+			if (details.capabilities) {
+				body.push(theme.fg("dim", `capabilities ${clean(details.capabilities, 160)}`));
+			}
+			return {
+				header,
+				sections: [{ lines: body }],
+				state: result.isError ? "error" : "success",
+				borderColor: result.isError ? "error" : "borderMuted",
+				applyBg: false,
+				width,
+			};
+		});
+	},
+};