pi-openai-toolkit 0.3.0 → 0.4.0

package/src/config.ts

@@ -1,273 +1,283 @@
-import * as fs from "node:fs";
-import * as os from "node:os";
-import * as path from "node:path";
-import type { ThinkingLevel } from "@earendil-works/pi-agent-core";
-import {
-	DEFAULT_COMPACTION_CONFIG,
-	DEFAULT_TOOLKIT_CONFIG,
-	DEFAULT_WEB_SEARCH_CONFIG,
-	RESPONSES_COMPACT_CAPABLE_APIS,
-	THINKING_LEVELS,
-	TOOLKIT_ID,
-	type CompactionConfig,
-	type LoadedToolkitConfig,
-	type ToolkitConfig,
-	type WebSearchConfig,
-} from "./types";
-
-export const CONFIG_DIR = path.join(os.homedir(), ".pi", "agent", "extensions", TOOLKIT_ID);
-export const CONFIG_PATH = path.join(CONFIG_DIR, "config.json");
-
-const TOP_LEVEL_FIELDS = new Set(["compaction", "webSearch"]);
-const COMPACTION_FIELDS = new Set([
-	"enabled",
-	"allowCompactionContinuityBreak",
-	"model",
-	"thinkingLevel",
-	"responsesApis",
-	"notifyOnLoad",
-	"debug",
-	"logProviderPayloads",
-	"logCompactResponses",
-	"redactSensitiveData",
-	"artifactRoot",
-]);
-const WEB_SEARCH_FIELDS = new Set(["enabled", "models"]);
-
-function isRecord(value: unknown): value is Record<string, unknown> {
-	return !!value && typeof value === "object" && !Array.isArray(value);
-}
-
-function isFile(filePath: string): boolean {
-	try {
-		return fs.statSync(filePath).isFile();
-	} catch {
-		return false;
-	}
-}
-
-function readJsonObject(filePath: string, warnings: string[]): Record<string, unknown> | undefined {
-	if (!isFile(filePath)) return undefined;
-
-	try {
-		const parsed = JSON.parse(fs.readFileSync(filePath, "utf8"));
-		if (isRecord(parsed)) return parsed;
-		warnings.push(`Ignoring ${filePath}: expected a JSON object at the top level.`);
-		return undefined;
-	} catch (error) {
-		const message = error instanceof Error ? error.message : String(error);
-		warnings.push(`Ignoring ${filePath}: ${message}`);
-		return undefined;
-	}
-}
-
-function resolveConfiguredPath(rawPath: string, baseDir: string): string {
-	if (rawPath.startsWith("~/")) {
-		return path.join(os.homedir(), rawPath.slice(2));
-	}
-	if (path.isAbsolute(rawPath)) {
-		return path.resolve(rawPath);
-	}
-	return path.resolve(baseDir, rawPath);
-}
-
-function warnUnknownFields(
-	value: Record<string, unknown>,
-	knownFields: ReadonlySet<string>,
-	fieldPath: string,
-	warnings: string[],
-): void {
-	for (const key of Object.keys(value)) {
-		if (!knownFields.has(key)) {
-			warnings.push(`Ignoring ${fieldPath ? `${fieldPath}.` : ""}${key}: unknown field.`);
-		}
-	}
-}
-
-function toBoolean(value: unknown, fieldPath: string, warnings: string[]): boolean | undefined {
-	if (value === undefined) return undefined;
-	if (typeof value === "boolean") return value;
-	warnings.push(`Ignoring ${fieldPath}: expected a boolean.`);
-	return undefined;
-}
-
-function toModelSpec(value: unknown, fieldPath: string, warnings: string[]): string | null | undefined {
-	if (value === undefined) return undefined;
-	if (value === null) return null;
-	if (typeof value === "string" && value.trim().length > 0) {
-		return value.trim();
-	}
-	warnings.push(`Ignoring ${fieldPath}: expected "provider/model-id" or null.`);
-	return undefined;
-}
-
-function toThinkingLevel(value: unknown, fieldPath: string, warnings: string[]): ThinkingLevel | undefined {
-	if (value === undefined) return undefined;
-	if (typeof value === "string" && (THINKING_LEVELS as readonly string[]).includes(value)) {
-		return value as ThinkingLevel;
-	}
-	warnings.push(`Ignoring ${fieldPath}: expected one of ${THINKING_LEVELS.join(", ")}.`);
-	return undefined;
-}
-
-function toSupportedApis(
-	value: unknown,
-	fieldPath: string,
-	capableApis: readonly string[],
-	warnings: string[],
-): string[] | undefined {
-	if (value === undefined) return undefined;
-	if (!Array.isArray(value) || value.some((item) => typeof item !== "string")) {
-		warnings.push(`Ignoring ${fieldPath}: expected a string array.`);
-		return undefined;
-	}
-
-	const capable = new Set(capableApis);
-	const accepted: string[] = [];
-	for (const item of new Set(value.map((entry) => entry.trim()).filter(Boolean))) {
-		if (capable.has(item)) {
-			accepted.push(item);
-		} else {
-			warnings.push(
-				`Ignoring ${fieldPath} entry "${item}": only ${capableApis.join(", ")} are supported.`,
-			);
-		}
-	}
-
-	return accepted;
-}
-
-function toModelAllowlist(value: unknown, fieldPath: string, warnings: string[]): string[] | undefined {
-	if (value === undefined) return undefined;
-	if (!Array.isArray(value) || value.some((item) => typeof item !== "string")) {
-		warnings.push(`Ignoring ${fieldPath}: expected a string array.`);
-		return undefined;
-	}
-
-	return [...new Set(value.map((entry) => entry.trim()).filter(Boolean))];
-}
-
-function cloneDefaults(): ToolkitConfig {
-	return {
-		compaction: {
-			...DEFAULT_COMPACTION_CONFIG,
-			responsesApis: [...DEFAULT_COMPACTION_CONFIG.responsesApis],
-		},
-		webSearch: {
-			...DEFAULT_WEB_SEARCH_CONFIG,
-			models: [...DEFAULT_WEB_SEARCH_CONFIG.models],
-		},
-	};
-}
-
-function applyCompactionConfig(
-	raw: Record<string, unknown>,
-	resolved: CompactionConfig,
-	warnings: string[],
-): void {
-	warnUnknownFields(raw, COMPACTION_FIELDS, "compaction", warnings);
-
-	resolved.enabled = toBoolean(raw.enabled, "compaction.enabled", warnings) ?? resolved.enabled;
-	resolved.allowCompactionContinuityBreak =
-		toBoolean(
-			raw.allowCompactionContinuityBreak,
-			"compaction.allowCompactionContinuityBreak",
-			warnings,
-		) ?? resolved.allowCompactionContinuityBreak;
-	resolved.notifyOnLoad =
-		toBoolean(raw.notifyOnLoad, "compaction.notifyOnLoad", warnings) ?? resolved.notifyOnLoad;
-	resolved.debug = toBoolean(raw.debug, "compaction.debug", warnings) ?? resolved.debug;
-	resolved.logProviderPayloads =
-		toBoolean(raw.logProviderPayloads, "compaction.logProviderPayloads", warnings) ??
-		resolved.logProviderPayloads;
-	resolved.logCompactResponses =
-		toBoolean(raw.logCompactResponses, "compaction.logCompactResponses", warnings) ??
-		resolved.logCompactResponses;
-	resolved.redactSensitiveData =
-		toBoolean(raw.redactSensitiveData, "compaction.redactSensitiveData", warnings) ??
-		resolved.redactSensitiveData;
-
-	const modelSpec = toModelSpec(raw.model, "compaction.model", warnings);
-	if (modelSpec !== undefined) {
-		resolved.model = modelSpec === null ? undefined : modelSpec;
-	}
-
-	resolved.thinkingLevel =
-		toThinkingLevel(raw.thinkingLevel, "compaction.thinkingLevel", warnings) ?? resolved.thinkingLevel;
-
-	const apis = toSupportedApis(
-		raw.responsesApis,
-		"compaction.responsesApis",
-		RESPONSES_COMPACT_CAPABLE_APIS,
-		warnings,
-	);
-	if (apis !== undefined) {
-		resolved.responsesApis = apis;
-	}
-
-	if (typeof raw.artifactRoot === "string" && raw.artifactRoot.trim().length > 0) {
-		resolved.artifactRoot = raw.artifactRoot.trim();
-	} else if (raw.artifactRoot !== undefined) {
-		warnings.push("Ignoring compaction.artifactRoot: expected a non-empty string.");
-	}
-}
-
-function applyWebSearchConfig(
-	raw: Record<string, unknown>,
-	resolved: WebSearchConfig,
-	warnings: string[],
-): void {
-	warnUnknownFields(raw, WEB_SEARCH_FIELDS, "webSearch", warnings);
-	resolved.enabled = toBoolean(raw.enabled, "webSearch.enabled", warnings) ?? resolved.enabled;
-
-	const models = toModelAllowlist(raw.models, "webSearch.models", warnings);
-	if (models !== undefined) {
-		resolved.models = models;
-	}
-}
-
-/**
- * Load the canonical toolkit config from
- * `~/.pi/agent/extensions/pi-openai-toolkit/config.json`.
- * A missing file silently yields defaults; legacy branded paths are never read.
- */
-export function loadToolkitConfig(configPath: string = CONFIG_PATH): LoadedToolkitConfig {
-	const warnings: string[] = [];
-	const resolved = cloneDefaults();
-	let source: string | undefined;
-
-	const raw = readJsonObject(configPath, warnings);
-	if (raw) {
-		source = configPath;
-		warnUnknownFields(raw, TOP_LEVEL_FIELDS, "", warnings);
-
-		if (raw.compaction !== undefined) {
-			if (isRecord(raw.compaction)) {
-				applyCompactionConfig(raw.compaction, resolved.compaction, warnings);
-			} else {
-				warnings.push("Ignoring compaction: expected a JSON object.");
-			}
-		}
-
-		if (raw.webSearch !== undefined) {
-			if (isRecord(raw.webSearch)) {
-				applyWebSearchConfig(raw.webSearch, resolved.webSearch, warnings);
-			} else {
-				warnings.push("Ignoring webSearch: expected a JSON object.");
-			}
-		}
-	}
-
-	resolved.compaction.artifactRoot = resolveConfiguredPath(
-		resolved.compaction.artifactRoot,
-		path.dirname(configPath),
-	);
-
-	return {
-		config: resolved,
-		source,
-		warnings,
-	};
-}
-
-export { DEFAULT_TOOLKIT_CONFIG };
+import * as fs from "node:fs";
+import * as os from "node:os";
+import * as path from "node:path";
+import type { ThinkingLevel } from "@earendil-works/pi-agent-core";
+import {
+	DEFAULT_COMPACTION_CONFIG,
+	DEFAULT_TOOLKIT_CONFIG,
+	DEFAULT_WEB_SEARCH_CONFIG,
+	RESPONSES_COMPACT_CAPABLE_APIS,
+	THINKING_LEVELS,
+	TOOLKIT_ID,
+	type CompactionConfig,
+	type LoadedToolkitConfig,
+	type ToolkitConfig,
+	type WebSearchConfig,
+} from "./types";
+
+export const CONFIG_DIR = path.join(os.homedir(), ".pi", "agent", "extensions", TOOLKIT_ID);
+export const CONFIG_PATH = path.join(CONFIG_DIR, "config.json");
+
+const TOP_LEVEL_FIELDS = new Set(["compaction", "webSearch"]);
+const COMPACTION_FIELDS = new Set([
+	"enabled",
+	"allowCompactionContinuityBreak",
+	"remoteCompactModel",
+	"model",
+	"thinkingLevel",
+	"responsesApis",
+	"notifyOnLoad",
+	"debug",
+	"logProviderPayloads",
+	"logCompactResponses",
+	"redactSensitiveData",
+	"artifactRoot",
+]);
+const WEB_SEARCH_FIELDS = new Set(["enabled", "models"]);
+
+function isRecord(value: unknown): value is Record<string, unknown> {
+	return !!value && typeof value === "object" && !Array.isArray(value);
+}
+
+function isFile(filePath: string): boolean {
+	try {
+		return fs.statSync(filePath).isFile();
+	} catch {
+		return false;
+	}
+}
+
+function readJsonObject(filePath: string, warnings: string[]): Record<string, unknown> | undefined {
+	if (!isFile(filePath)) return undefined;
+
+	try {
+		const parsed = JSON.parse(fs.readFileSync(filePath, "utf8"));
+		if (isRecord(parsed)) return parsed;
+		warnings.push(`Ignoring ${filePath}: expected a JSON object at the top level.`);
+		return undefined;
+	} catch (error) {
+		const message = error instanceof Error ? error.message : String(error);
+		warnings.push(`Ignoring ${filePath}: ${message}`);
+		return undefined;
+	}
+}
+
+function resolveConfiguredPath(rawPath: string, baseDir: string): string {
+	if (rawPath.startsWith("~/")) {
+		return path.join(os.homedir(), rawPath.slice(2));
+	}
+	if (path.isAbsolute(rawPath)) {
+		return path.resolve(rawPath);
+	}
+	return path.resolve(baseDir, rawPath);
+}
+
+function warnUnknownFields(
+	value: Record<string, unknown>,
+	knownFields: ReadonlySet<string>,
+	fieldPath: string,
+	warnings: string[],
+): void {
+	for (const key of Object.keys(value)) {
+		if (!knownFields.has(key)) {
+			warnings.push(`Ignoring ${fieldPath ? `${fieldPath}.` : ""}${key}: unknown field.`);
+		}
+	}
+}
+
+function toBoolean(value: unknown, fieldPath: string, warnings: string[]): boolean | undefined {
+	if (value === undefined) return undefined;
+	if (typeof value === "boolean") return value;
+	warnings.push(`Ignoring ${fieldPath}: expected a boolean.`);
+	return undefined;
+}
+
+function toModelSpec(value: unknown, fieldPath: string, warnings: string[]): string | null | undefined {
+	if (value === undefined) return undefined;
+	if (value === null) return null;
+	if (typeof value === "string" && value.trim().length > 0) {
+		return value.trim();
+	}
+	warnings.push(`Ignoring ${fieldPath}: expected "provider/model-id" or null.`);
+	return undefined;
+}
+
+function toThinkingLevel(value: unknown, fieldPath: string, warnings: string[]): ThinkingLevel | undefined {
+	if (value === undefined) return undefined;
+	if (typeof value === "string" && (THINKING_LEVELS as readonly string[]).includes(value)) {
+		return value as ThinkingLevel;
+	}
+	warnings.push(`Ignoring ${fieldPath}: expected one of ${THINKING_LEVELS.join(", ")}.`);
+	return undefined;
+}
+
+function toSupportedApis(
+	value: unknown,
+	fieldPath: string,
+	capableApis: readonly string[],
+	warnings: string[],
+): string[] | undefined {
+	if (value === undefined) return undefined;
+	if (!Array.isArray(value) || value.some((item) => typeof item !== "string")) {
+		warnings.push(`Ignoring ${fieldPath}: expected a string array.`);
+		return undefined;
+	}
+
+	const capable = new Set(capableApis);
+	const accepted: string[] = [];
+	for (const item of new Set(value.map((entry) => entry.trim()).filter(Boolean))) {
+		if (capable.has(item)) {
+			accepted.push(item);
+		} else {
+			warnings.push(
+				`Ignoring ${fieldPath} entry "${item}": only ${capableApis.join(", ")} are supported.`,
+			);
+		}
+	}
+
+	return accepted;
+}
+
+function toModelAllowlist(value: unknown, fieldPath: string, warnings: string[]): string[] | undefined {
+	if (value === undefined) return undefined;
+	if (!Array.isArray(value) || value.some((item) => typeof item !== "string")) {
+		warnings.push(`Ignoring ${fieldPath}: expected a string array.`);
+		return undefined;
+	}
+
+	return [...new Set(value.map((entry) => entry.trim()).filter(Boolean))];
+}
+
+function cloneDefaults(): ToolkitConfig {
+	return {
+		compaction: {
+			...DEFAULT_COMPACTION_CONFIG,
+			responsesApis: [...DEFAULT_COMPACTION_CONFIG.responsesApis],
+		},
+		webSearch: {
+			...DEFAULT_WEB_SEARCH_CONFIG,
+			models: [...DEFAULT_WEB_SEARCH_CONFIG.models],
+		},
+	};
+}
+
+function applyCompactionConfig(
+	raw: Record<string, unknown>,
+	resolved: CompactionConfig,
+	warnings: string[],
+): void {
+	warnUnknownFields(raw, COMPACTION_FIELDS, "compaction", warnings);
+
+	resolved.enabled = toBoolean(raw.enabled, "compaction.enabled", warnings) ?? resolved.enabled;
+	resolved.allowCompactionContinuityBreak =
+		toBoolean(
+			raw.allowCompactionContinuityBreak,
+			"compaction.allowCompactionContinuityBreak",
+			warnings,
+		) ?? resolved.allowCompactionContinuityBreak;
+	resolved.notifyOnLoad =
+		toBoolean(raw.notifyOnLoad, "compaction.notifyOnLoad", warnings) ?? resolved.notifyOnLoad;
+	resolved.debug = toBoolean(raw.debug, "compaction.debug", warnings) ?? resolved.debug;
+	resolved.logProviderPayloads =
+		toBoolean(raw.logProviderPayloads, "compaction.logProviderPayloads", warnings) ??
+		resolved.logProviderPayloads;
+	resolved.logCompactResponses =
+		toBoolean(raw.logCompactResponses, "compaction.logCompactResponses", warnings) ??
+		resolved.logCompactResponses;
+	resolved.redactSensitiveData =
+		toBoolean(raw.redactSensitiveData, "compaction.redactSensitiveData", warnings) ??
+		resolved.redactSensitiveData;
+
+	const remoteCompactModelSpec = toModelSpec(
+		raw.remoteCompactModel,
+		"compaction.remoteCompactModel",
+		warnings,
+	);
+	if (remoteCompactModelSpec !== undefined) {
+		resolved.remoteCompactModel = remoteCompactModelSpec === null ? undefined : remoteCompactModelSpec;
+	}
+
+	const modelSpec = toModelSpec(raw.model, "compaction.model", warnings);
+	if (modelSpec !== undefined) {
+		resolved.model = modelSpec === null ? undefined : modelSpec;
+	}
+
+	resolved.thinkingLevel =
+		toThinkingLevel(raw.thinkingLevel, "compaction.thinkingLevel", warnings) ?? resolved.thinkingLevel;
+
+	const apis = toSupportedApis(
+		raw.responsesApis,
+		"compaction.responsesApis",
+		RESPONSES_COMPACT_CAPABLE_APIS,
+		warnings,
+	);
+	if (apis !== undefined) {
+		resolved.responsesApis = apis;
+	}
+
+	if (typeof raw.artifactRoot === "string" && raw.artifactRoot.trim().length > 0) {
+		resolved.artifactRoot = raw.artifactRoot.trim();
+	} else if (raw.artifactRoot !== undefined) {
+		warnings.push("Ignoring compaction.artifactRoot: expected a non-empty string.");
+	}
+}
+
+function applyWebSearchConfig(
+	raw: Record<string, unknown>,
+	resolved: WebSearchConfig,
+	warnings: string[],
+): void {
+	warnUnknownFields(raw, WEB_SEARCH_FIELDS, "webSearch", warnings);
+	resolved.enabled = toBoolean(raw.enabled, "webSearch.enabled", warnings) ?? resolved.enabled;
+
+	const models = toModelAllowlist(raw.models, "webSearch.models", warnings);
+	if (models !== undefined) {
+		resolved.models = models;
+	}
+}
+
+/**
+ * Load the canonical toolkit config from
+ * `~/.pi/agent/extensions/pi-openai-toolkit/config.json`.
+ * A missing file silently yields defaults; legacy branded paths are never read.
+ */
+export function loadToolkitConfig(configPath: string = CONFIG_PATH): LoadedToolkitConfig {
+	const warnings: string[] = [];
+	const resolved = cloneDefaults();
+	let source: string | undefined;
+
+	const raw = readJsonObject(configPath, warnings);
+	if (raw) {
+		source = configPath;
+		warnUnknownFields(raw, TOP_LEVEL_FIELDS, "", warnings);
+
+		if (raw.compaction !== undefined) {
+			if (isRecord(raw.compaction)) {
+				applyCompactionConfig(raw.compaction, resolved.compaction, warnings);
+			} else {
+				warnings.push("Ignoring compaction: expected a JSON object.");
+			}
+		}
+
+		if (raw.webSearch !== undefined) {
+			if (isRecord(raw.webSearch)) {
+				applyWebSearchConfig(raw.webSearch, resolved.webSearch, warnings);
+			} else {
+				warnings.push("Ignoring webSearch: expected a JSON object.");
+			}
+		}
+	}
+
+	resolved.compaction.artifactRoot = resolveConfiguredPath(
+		resolved.compaction.artifactRoot,
+		path.dirname(configPath),
+	);
+
+	return {
+		config: resolved,
+		source,
+		warnings,
+	};
+}
+
+export { DEFAULT_TOOLKIT_CONFIG };

package/src/debug.ts

@@ -14,6 +14,7 @@ import {
 	type RedactOptions,
 } from "./types";
 
+const CRITICAL_KEY_RE = /(authorization|api[-_]?key|token|credential|oauth|auth|encrypted[_-]?content)/i;
 const SENSITIVE_KEY_RE = /(authorization|api[-_]?key|token|secret|password|cookie|set-cookie|signature|credential|oauth|auth|encrypted[_-]?content)/i;
 const BEARER_RE = /\bBearer\s+[A-Za-z0-9._\-+/=]+/gi;
 const OPENAI_KEY_RE = /\bsk-[A-Za-z0-9\-_]+\b/g;
@@ -52,11 +53,11 @@ function redactInlineSecrets(value: string, placeholder: string): string {
 		.replace(SENSITIVE_QUERY_RE, (_match, prefix: string) => `${prefix}${placeholder}`);
 }
 
-function shouldRedactKey(key: string): boolean {
-	return SENSITIVE_KEY_RE.test(key);
-}
-
-export function redactValue(value: unknown, options: RedactOptions = {}): unknown {
+function redactWithKeyPattern(
+	value: unknown,
+	keyPattern: RegExp,
+	options: RedactOptions = {},
+): unknown {
 	const placeholder = options.placeholder ?? REDACTED_VALUE;
 	const seen = new WeakSet<object>();
 
@@ -78,7 +79,7 @@ export function redactValue(value: unknown, options: RedactOptions = {}): unknow
 
 		const result: Record<string, unknown> = {};
 		for (const [key, item] of Object.entries(input)) {
-			result[key] = shouldRedactKey(key) ? placeholder : visit(item);
+			result[key] = keyPattern.test(key) ? placeholder : visit(item);
 		}
 		return result;
 	};
@@ -86,6 +87,15 @@ export function redactValue(value: unknown, options: RedactOptions = {}): unknow
 	return visit(value);
 }
 
+/** Credentials and opaque checkpoints are never written to artifacts, even when optional redaction is disabled. */
+export function redactCriticalValue(value: unknown, options: RedactOptions = {}): unknown {
+	return redactWithKeyPattern(value, CRITICAL_KEY_RE, options);
+}
+
+export function redactValue(value: unknown, options: RedactOptions = {}): unknown {
+	return redactWithKeyPattern(value, SENSITIVE_KEY_RE, options);
+}
+
 export function resolveArtifactPaths(settings: CompactionConfig, context: ArtifactContext): ArtifactPaths {
 	const sessionInfo = toSessionInfo(context);
 	const rootDir = settings.artifactRoot.startsWith("~/")
@@ -149,6 +159,7 @@ export function writeDebugArtifact(
 	const timestamp = new Date().toISOString();
 	const fileName = `${timestamp.replace(/[.:]/g, "-")}-${kind}.json`;
 	const filePath = path.join(targetDir, fileName);
+	const criticallyRedactedData = redactCriticalValue(data);
 	const envelope: DebugArtifactEnvelope = {
 		extension: COMPACTION_EXTENSION_ID,
 		kind,
@@ -160,7 +171,7 @@ export function writeDebugArtifact(
 		redaction: {
 			enabled: settings.redactSensitiveData,
 		},
-		data: settings.redactSensitiveData ? redactValue(data) : data,
+		data: settings.redactSensitiveData ? redactValue(criticallyRedactedData) : criticallyRedactedData,
 	};
 
 	fs.writeFileSync(filePath, `${JSON.stringify(envelope, null, 2)}\n`, "utf8");