pi-openai-toolkit 0.3.0

package/src/compact-client.ts

@@ -0,0 +1,426 @@
+import { writeDebugArtifact } from "./debug";
+import { mergeProviderHeaders } from "./provider-headers";
+import type { NativeCompactionRuntime } from "./runtime";
+import type { NativeCompactionRequestBody } from "./serializer";
+import type { ArtifactContext, CompactionConfig } from "./types";
+
+const JSON_CONTENT_TYPE = "application/json";
+
+type CompactResponseEnvelope = {
+	id?: string;
+	created_at?: number | string;
+	output: unknown[];
+	[key: string]: unknown;
+};
+
+export type NativeCompactionClientFailureReason =
+	| "aborted"
+	| "network-error"
+	| "non-2xx"
+	| "empty-body"
+	| "invalid-json"
+	| "malformed-response"
+	| "empty-output";
+
+export type NativeCompactionClientSuccess = {
+	ok: true;
+	status: number;
+	compactedWindow: unknown[];
+	compactResponseId?: string;
+	createdAt?: string;
+	/** Assistant summary text extracted from the compact output, for CompactionEntry.summary. */
+	summaryText?: string;
+	response: CompactResponseEnvelope;
+};
+
+export type NativeCompactionClientFailure = {
+	ok: false;
+	reason: NativeCompactionClientFailureReason;
+	status?: number;
+	errorMessage?: string;
+	responseText?: string;
+	responseJson?: unknown;
+};
+
+export type NativeCompactionClientResult = NativeCompactionClientSuccess | NativeCompactionClientFailure;
+
+export type ExecuteNativeCompactionOptions = {
+	runtime: NativeCompactionRuntime;
+	request: NativeCompactionRequestBody;
+	signal?: AbortSignal;
+	settings?: CompactionConfig;
+	context?: ArtifactContext;
+};
+
+function isRecord(value: unknown): value is Record<string, unknown> {
+	return !!value && typeof value === "object" && !Array.isArray(value);
+}
+
+function isAbortError(error: unknown): boolean {
+	return (
+		(error instanceof DOMException && error.name === "AbortError") ||
+		(error instanceof Error && (error.name === "AbortError" || error.name === "ABORT_ERR"))
+	);
+}
+
+function normalizeResponseTimestamp(value: unknown): string | undefined {
+	if (typeof value === "number" && Number.isFinite(value)) {
+		const milliseconds = value > 1_000_000_000_000 ? value : value * 1000;
+		return new Date(milliseconds).toISOString();
+	}
+
+	if (typeof value !== "string") {
+		return undefined;
+	}
+
+	const trimmed = value.trim();
+	if (!trimmed) {
+		return undefined;
+	}
+
+	const parsed = Date.parse(trimmed);
+	return Number.isNaN(parsed) ? trimmed : new Date(parsed).toISOString();
+}
+
+function isCompactOutputItem(value: unknown): value is Record<string, unknown> {
+	return isRecord(value);
+}
+
+function isCompactResponseEnvelope(value: unknown): value is CompactResponseEnvelope {
+	return isRecord(value) && Array.isArray(value.output) && value.output.every(isCompactOutputItem);
+}
+
+/**
+ * Extract the assistant-authored summary text from the compacted window so the
+ * persisted CompactionEntry.summary carries real context. Without this, switching
+ * to a non-Responses model later would replay a meaningless placeholder.
+ */
+export function extractCompactedSummaryText(output: readonly unknown[]): string | undefined {
+	const texts: string[] = [];
+	for (const item of output) {
+		if (!isRecord(item) || item.type !== "message" || item.role !== "assistant" || !Array.isArray(item.content)) {
+			continue;
+		}
+		for (const block of item.content) {
+			if (isRecord(block) && block.type === "output_text" && typeof block.text === "string" && block.text.trim()) {
+				texts.push(block.text.trim());
+			}
+		}
+	}
+
+	const joined = texts.join("\n\n").trim();
+	return joined.length > 0 ? joined : undefined;
+}
+
+function decodeJwtPayload(token: string): Record<string, unknown> | undefined {
+	const parts = token.split(".");
+	if (parts.length !== 3) {
+		return undefined;
+	}
+
+	try {
+		const payloadText = Buffer.from(parts[1]!, "base64url").toString("utf8");
+		const payload = JSON.parse(payloadText);
+		return isRecord(payload) ? payload : undefined;
+	} catch {
+		return undefined;
+	}
+}
+
+function extractCodexAccountId(token: string): string | undefined {
+	const payload = decodeJwtPayload(token);
+	const authClaims = payload?.["https://api.openai.com/auth"];
+	if (!isRecord(authClaims)) {
+		return undefined;
+	}
+
+	const accountId = authClaims.chatgpt_account_id;
+	return typeof accountId === "string" && accountId.trim().length > 0 ? accountId.trim() : undefined;
+}
+
+function buildCodexUserAgent(): string {
+	const platform = typeof process !== "undefined" ? process.platform : "browser";
+	const arch = typeof process !== "undefined" ? process.arch : "unknown";
+	return `pi (${platform}; ${arch})`;
+}
+
+function toHeaders(runtime: NativeCompactionRuntime): Record<string, string> {
+	const headers = new Headers(mergeProviderHeaders(runtime.currentModel.headers, runtime.headers));
+	headers.set("accept", JSON_CONTENT_TYPE);
+	headers.set("content-type", JSON_CONTENT_TYPE);
+	if (!headers.has("authorization")) {
+		headers.set("authorization", `Bearer ${runtime.apiKey}`);
+	}
+
+	if (runtime.api === "openai-codex-responses") {
+		const accountId = extractCodexAccountId(runtime.apiKey);
+		if (accountId) {
+			headers.set("chatgpt-account-id", accountId);
+		}
+		headers.set("originator", "pi");
+		headers.set("user-agent", buildCodexUserAgent());
+		headers.set("openai-beta", "responses=experimental");
+	}
+
+	return Object.fromEntries(headers.entries());
+}
+
+function writeCompactArtifact(
+	data: unknown,
+	settings: CompactionConfig | undefined,
+	context: ArtifactContext | undefined,
+): void {
+	if (!settings || !context) {
+		return;
+	}
+
+	writeDebugArtifact("compact-response", data, settings, context);
+}
+
+export async function executeNativeCompaction(
+	options: ExecuteNativeCompactionOptions,
+): Promise<NativeCompactionClientResult> {
+	const { runtime, request, signal, settings, context } = options;
+	const headers = toHeaders(runtime);
+
+	if (signal?.aborted) {
+		const aborted: NativeCompactionClientFailure = {
+			ok: false,
+			reason: "aborted",
+		};
+		writeCompactArtifact(
+			{
+				request: {
+					url: runtime.compactUrl,
+					headers,
+					body: request,
+				},
+				outcome: aborted,
+			},
+			settings,
+			context,
+		);
+		return aborted;
+	}
+
+	try {
+		const response = await fetch(runtime.compactUrl, {
+			method: "POST",
+			headers,
+			body: JSON.stringify(request),
+			signal,
+		});
+		const responseText = await response.text();
+		const responseHeaders: Record<string, string> = {};
+		response.headers.forEach((value, key) => {
+			responseHeaders[key] = value;
+		});
+
+		if (!response.ok) {
+			let responseJson: unknown;
+			if (responseText.trim().length > 0) {
+				try {
+					responseJson = JSON.parse(responseText);
+				} catch {
+					responseJson = undefined;
+				}
+			}
+
+			const failure: NativeCompactionClientFailure = {
+				ok: false,
+				reason: "non-2xx",
+				status: response.status,
+				responseText: responseText || undefined,
+				responseJson,
+			};
+			writeCompactArtifact(
+				{
+					request: {
+						url: runtime.compactUrl,
+						headers,
+						body: request,
+					},
+					response: {
+						status: response.status,
+						headers: responseHeaders,
+						body: responseJson ?? responseText,
+					},
+					outcome: failure,
+				},
+				settings,
+				context,
+			);
+			return failure;
+		}
+
+		if (!responseText.trim()) {
+			const failure: NativeCompactionClientFailure = {
+				ok: false,
+				reason: "empty-body",
+				status: response.status,
+			};
+			writeCompactArtifact(
+				{
+					request: {
+						url: runtime.compactUrl,
+						headers,
+						body: request,
+					},
+					response: {
+						status: response.status,
+						headers: responseHeaders,
+						body: responseText,
+					},
+					outcome: failure,
+				},
+				settings,
+				context,
+			);
+			return failure;
+		}
+
+		let parsed: unknown;
+		try {
+			parsed = JSON.parse(responseText);
+		} catch (error) {
+			const failure: NativeCompactionClientFailure = {
+				ok: false,
+				reason: "invalid-json",
+				status: response.status,
+				errorMessage: error instanceof Error ? error.message : String(error),
+				responseText,
+			};
+			writeCompactArtifact(
+				{
+					request: {
+						url: runtime.compactUrl,
+						headers,
+						body: request,
+					},
+					response: {
+						status: response.status,
+						headers: responseHeaders,
+						body: responseText,
+					},
+					outcome: failure,
+				},
+				settings,
+				context,
+			);
+			return failure;
+		}
+
+		if (!isCompactResponseEnvelope(parsed)) {
+			const failure: NativeCompactionClientFailure = {
+				ok: false,
+				reason: "malformed-response",
+				status: response.status,
+				responseJson: parsed,
+			};
+			writeCompactArtifact(
+				{
+					request: {
+						url: runtime.compactUrl,
+						headers,
+						body: request,
+					},
+					response: {
+						status: response.status,
+						headers: responseHeaders,
+						body: parsed,
+					},
+					outcome: failure,
+				},
+				settings,
+				context,
+			);
+			return failure;
+		}
+
+		if (parsed.output.length === 0) {
+			const failure: NativeCompactionClientFailure = {
+				ok: false,
+				reason: "empty-output",
+				status: response.status,
+				responseJson: parsed,
+			};
+			writeCompactArtifact(
+				{
+					request: {
+						url: runtime.compactUrl,
+						headers,
+						body: request,
+					},
+					response: {
+						status: response.status,
+						headers: responseHeaders,
+						body: parsed,
+					},
+					outcome: failure,
+				},
+				settings,
+				context,
+			);
+			return failure;
+		}
+
+		const success: NativeCompactionClientSuccess = {
+			ok: true,
+			status: response.status,
+			compactedWindow: [...parsed.output],
+			compactResponseId: typeof parsed.id === "string" && parsed.id.trim() ? parsed.id.trim() : undefined,
+			createdAt: normalizeResponseTimestamp(parsed.created_at),
+			summaryText: extractCompactedSummaryText(parsed.output),
+			response: parsed,
+		};
+		writeCompactArtifact(
+			{
+				request: {
+					url: runtime.compactUrl,
+					headers,
+					body: request,
+				},
+				response: {
+					status: response.status,
+					headers: responseHeaders,
+					body: parsed,
+				},
+				outcome: {
+					ok: true,
+					status: success.status,
+					compactResponseId: success.compactResponseId,
+					createdAt: success.createdAt,
+					compactedItems: success.compactedWindow.length,
+				},
+			},
+			settings,
+			context,
+		);
+		return success;
+	} catch (error) {
+		const failure: NativeCompactionClientFailure = isAbortError(error)
+			? {
+				ok: false,
+				reason: "aborted",
+			}
+			: {
+				ok: false,
+				reason: "network-error",
+				errorMessage: error instanceof Error ? error.message : String(error),
+			};
+
+		writeCompactArtifact(
+			{
+				request: {
+					url: runtime.compactUrl,
+					headers,
+					body: request,
+				},
+				outcome: failure,
+			},
+			settings,
+			context,
+		);
+		return failure;
+	}
+}

package/src/config.ts

@@ -0,0 +1,273 @@
+import * as fs from "node:fs";
+import * as os from "node:os";
+import * as path from "node:path";
+import type { ThinkingLevel } from "@earendil-works/pi-agent-core";
+import {
+	DEFAULT_COMPACTION_CONFIG,
+	DEFAULT_TOOLKIT_CONFIG,
+	DEFAULT_WEB_SEARCH_CONFIG,
+	RESPONSES_COMPACT_CAPABLE_APIS,
+	THINKING_LEVELS,
+	TOOLKIT_ID,
+	type CompactionConfig,
+	type LoadedToolkitConfig,
+	type ToolkitConfig,
+	type WebSearchConfig,
+} from "./types";
+
+export const CONFIG_DIR = path.join(os.homedir(), ".pi", "agent", "extensions", TOOLKIT_ID);
+export const CONFIG_PATH = path.join(CONFIG_DIR, "config.json");
+
+const TOP_LEVEL_FIELDS = new Set(["compaction", "webSearch"]);
+const COMPACTION_FIELDS = new Set([
+	"enabled",
+	"allowCompactionContinuityBreak",
+	"model",
+	"thinkingLevel",
+	"responsesApis",
+	"notifyOnLoad",
+	"debug",
+	"logProviderPayloads",
+	"logCompactResponses",
+	"redactSensitiveData",
+	"artifactRoot",
+]);
+const WEB_SEARCH_FIELDS = new Set(["enabled", "models"]);
+
+function isRecord(value: unknown): value is Record<string, unknown> {
+	return !!value && typeof value === "object" && !Array.isArray(value);
+}
+
+function isFile(filePath: string): boolean {
+	try {
+		return fs.statSync(filePath).isFile();
+	} catch {
+		return false;
+	}
+}
+
+function readJsonObject(filePath: string, warnings: string[]): Record<string, unknown> | undefined {
+	if (!isFile(filePath)) return undefined;
+
+	try {
+		const parsed = JSON.parse(fs.readFileSync(filePath, "utf8"));
+		if (isRecord(parsed)) return parsed;
+		warnings.push(`Ignoring ${filePath}: expected a JSON object at the top level.`);
+		return undefined;
+	} catch (error) {
+		const message = error instanceof Error ? error.message : String(error);
+		warnings.push(`Ignoring ${filePath}: ${message}`);
+		return undefined;
+	}
+}
+
+function resolveConfiguredPath(rawPath: string, baseDir: string): string {
+	if (rawPath.startsWith("~/")) {
+		return path.join(os.homedir(), rawPath.slice(2));
+	}
+	if (path.isAbsolute(rawPath)) {
+		return path.resolve(rawPath);
+	}
+	return path.resolve(baseDir, rawPath);
+}
+
+function warnUnknownFields(
+	value: Record<string, unknown>,
+	knownFields: ReadonlySet<string>,
+	fieldPath: string,
+	warnings: string[],
+): void {
+	for (const key of Object.keys(value)) {
+		if (!knownFields.has(key)) {
+			warnings.push(`Ignoring ${fieldPath ? `${fieldPath}.` : ""}${key}: unknown field.`);
+		}
+	}
+}
+
+function toBoolean(value: unknown, fieldPath: string, warnings: string[]): boolean | undefined {
+	if (value === undefined) return undefined;
+	if (typeof value === "boolean") return value;
+	warnings.push(`Ignoring ${fieldPath}: expected a boolean.`);
+	return undefined;
+}
+
+function toModelSpec(value: unknown, fieldPath: string, warnings: string[]): string | null | undefined {
+	if (value === undefined) return undefined;
+	if (value === null) return null;
+	if (typeof value === "string" && value.trim().length > 0) {
+		return value.trim();
+	}
+	warnings.push(`Ignoring ${fieldPath}: expected "provider/model-id" or null.`);
+	return undefined;
+}
+
+function toThinkingLevel(value: unknown, fieldPath: string, warnings: string[]): ThinkingLevel | undefined {
+	if (value === undefined) return undefined;
+	if (typeof value === "string" && (THINKING_LEVELS as readonly string[]).includes(value)) {
+		return value as ThinkingLevel;
+	}
+	warnings.push(`Ignoring ${fieldPath}: expected one of ${THINKING_LEVELS.join(", ")}.`);
+	return undefined;
+}
+
+function toSupportedApis(
+	value: unknown,
+	fieldPath: string,
+	capableApis: readonly string[],
+	warnings: string[],
+): string[] | undefined {
+	if (value === undefined) return undefined;
+	if (!Array.isArray(value) || value.some((item) => typeof item !== "string")) {
+		warnings.push(`Ignoring ${fieldPath}: expected a string array.`);
+		return undefined;
+	}
+
+	const capable = new Set(capableApis);
+	const accepted: string[] = [];
+	for (const item of new Set(value.map((entry) => entry.trim()).filter(Boolean))) {
+		if (capable.has(item)) {
+			accepted.push(item);
+		} else {
+			warnings.push(
+				`Ignoring ${fieldPath} entry "${item}": only ${capableApis.join(", ")} are supported.`,
+			);
+		}
+	}
+
+	return accepted;
+}
+
+function toModelAllowlist(value: unknown, fieldPath: string, warnings: string[]): string[] | undefined {
+	if (value === undefined) return undefined;
+	if (!Array.isArray(value) || value.some((item) => typeof item !== "string")) {
+		warnings.push(`Ignoring ${fieldPath}: expected a string array.`);
+		return undefined;
+	}
+
+	return [...new Set(value.map((entry) => entry.trim()).filter(Boolean))];
+}
+
+function cloneDefaults(): ToolkitConfig {
+	return {
+		compaction: {
+			...DEFAULT_COMPACTION_CONFIG,
+			responsesApis: [...DEFAULT_COMPACTION_CONFIG.responsesApis],
+		},
+		webSearch: {
+			...DEFAULT_WEB_SEARCH_CONFIG,
+			models: [...DEFAULT_WEB_SEARCH_CONFIG.models],
+		},
+	};
+}
+
+function applyCompactionConfig(
+	raw: Record<string, unknown>,
+	resolved: CompactionConfig,
+	warnings: string[],
+): void {
+	warnUnknownFields(raw, COMPACTION_FIELDS, "compaction", warnings);
+
+	resolved.enabled = toBoolean(raw.enabled, "compaction.enabled", warnings) ?? resolved.enabled;
+	resolved.allowCompactionContinuityBreak =
+		toBoolean(
+			raw.allowCompactionContinuityBreak,
+			"compaction.allowCompactionContinuityBreak",
+			warnings,
+		) ?? resolved.allowCompactionContinuityBreak;
+	resolved.notifyOnLoad =
+		toBoolean(raw.notifyOnLoad, "compaction.notifyOnLoad", warnings) ?? resolved.notifyOnLoad;
+	resolved.debug = toBoolean(raw.debug, "compaction.debug", warnings) ?? resolved.debug;
+	resolved.logProviderPayloads =
+		toBoolean(raw.logProviderPayloads, "compaction.logProviderPayloads", warnings) ??
+		resolved.logProviderPayloads;
+	resolved.logCompactResponses =
+		toBoolean(raw.logCompactResponses, "compaction.logCompactResponses", warnings) ??
+		resolved.logCompactResponses;
+	resolved.redactSensitiveData =
+		toBoolean(raw.redactSensitiveData, "compaction.redactSensitiveData", warnings) ??
+		resolved.redactSensitiveData;
+
+	const modelSpec = toModelSpec(raw.model, "compaction.model", warnings);
+	if (modelSpec !== undefined) {
+		resolved.model = modelSpec === null ? undefined : modelSpec;
+	}
+
+	resolved.thinkingLevel =
+		toThinkingLevel(raw.thinkingLevel, "compaction.thinkingLevel", warnings) ?? resolved.thinkingLevel;
+
+	const apis = toSupportedApis(
+		raw.responsesApis,
+		"compaction.responsesApis",
+		RESPONSES_COMPACT_CAPABLE_APIS,
+		warnings,
+	);
+	if (apis !== undefined) {
+		resolved.responsesApis = apis;
+	}
+
+	if (typeof raw.artifactRoot === "string" && raw.artifactRoot.trim().length > 0) {
+		resolved.artifactRoot = raw.artifactRoot.trim();
+	} else if (raw.artifactRoot !== undefined) {
+		warnings.push("Ignoring compaction.artifactRoot: expected a non-empty string.");
+	}
+}
+
+function applyWebSearchConfig(
+	raw: Record<string, unknown>,
+	resolved: WebSearchConfig,
+	warnings: string[],
+): void {
+	warnUnknownFields(raw, WEB_SEARCH_FIELDS, "webSearch", warnings);
+	resolved.enabled = toBoolean(raw.enabled, "webSearch.enabled", warnings) ?? resolved.enabled;
+
+	const models = toModelAllowlist(raw.models, "webSearch.models", warnings);
+	if (models !== undefined) {
+		resolved.models = models;
+	}
+}
+
+/**
+ * Load the canonical toolkit config from
+ * `~/.pi/agent/extensions/pi-openai-toolkit/config.json`.
+ * A missing file silently yields defaults; legacy branded paths are never read.
+ */
+export function loadToolkitConfig(configPath: string = CONFIG_PATH): LoadedToolkitConfig {
+	const warnings: string[] = [];
+	const resolved = cloneDefaults();
+	let source: string | undefined;
+
+	const raw = readJsonObject(configPath, warnings);
+	if (raw) {
+		source = configPath;
+		warnUnknownFields(raw, TOP_LEVEL_FIELDS, "", warnings);
+
+		if (raw.compaction !== undefined) {
+			if (isRecord(raw.compaction)) {
+				applyCompactionConfig(raw.compaction, resolved.compaction, warnings);
+			} else {
+				warnings.push("Ignoring compaction: expected a JSON object.");
+			}
+		}
+
+		if (raw.webSearch !== undefined) {
+			if (isRecord(raw.webSearch)) {
+				applyWebSearchConfig(raw.webSearch, resolved.webSearch, warnings);
+			} else {
+				warnings.push("Ignoring webSearch: expected a JSON object.");
+			}
+		}
+	}
+
+	resolved.compaction.artifactRoot = resolveConfiguredPath(
+		resolved.compaction.artifactRoot,
+		path.dirname(configPath),
+	);
+
+	return {
+		config: resolved,
+		source,
+		warnings,
+	};
+}
+
+export { DEFAULT_TOOLKIT_CONFIG };