pi-openai-toolkit 0.8.0 → 0.10.0

package/src/image-generation/eligibility.ts

@@ -1,10 +1,11 @@
-import { isExactModelAllowed } from "../model-scope";
 import type { ImageGenerationConfig } from "../types";
-import {
-	IMAGE_GENERATION_CAPABLE_APIS,
-	type ImageGenerationModel,
-} from "./types";
+import { IMAGE_GENERATION_CAPABLE_APIS, type ImageGenerationModel } from "./types";
 
+/**
+ * The hosted `image_generation` tool is a capability of the Responses API rather than of a
+ * specific model, so support is decided by API alone. `enabled` is the single opt-in switch
+ * and defaults to off because every successful provider request may incur a charge.
+ */
 export function isImageGenerationEnabledForModel(
 	model: ImageGenerationModel | undefined,
 	config: ImageGenerationConfig,
@@ -12,7 +13,6 @@ export function isImageGenerationEnabledForModel(
 	return (
 		config.enabled &&
 		typeof model?.api === "string" &&
-		(IMAGE_GENERATION_CAPABLE_APIS as readonly string[]).includes(model.api) &&
-		isExactModelAllowed(model, config.models)
+		(IMAGE_GENERATION_CAPABLE_APIS as readonly string[]).includes(model.api)
 	);
 }

package/src/image-generation/extension.ts

@@ -1,160 +1,160 @@
-import { StringEnum, Type, type Static } from "@earendil-works/pi-ai";
-import type { ExtensionAPI } from "@earendil-works/pi-coding-agent";
-import { loadToolkitConfig } from "../config";
-import { executeImageGeneration } from "./service";
-import { isImageGenerationEnabledForModel } from "./eligibility";
-import { renderImageGenerationResult } from "./render";
-import {
-	IMAGE_GENERATION_QUALITIES,
-	IMAGE_GENERATION_SIZES,
-	IMAGE_GENERATION_TOOL_NAME,
-	MAX_IMAGE_PATH_CHARS,
-	MAX_IMAGE_PROMPT_CHARS,
-	MAX_REFERENCE_IMAGE_COUNT,
-	ImageGenerationError,
-	sanitizeImageDiagnostic,
-	type ImageGenerationDetails,
-} from "./types";
-
-const registeredApis = new WeakSet<object>();
-
-const GenerateImageParameters = Type.Object(
-	{
-		prompt: Type.String({
-			minLength: 1,
-			maxLength: MAX_IMAGE_PROMPT_CHARS,
-			description: "Image generation or edit prompt. Preserve the user's requested subject and constraints.",
-		}),
-		referenceImagePaths: Type.Optional(
-			Type.Union(
-				[
-					Type.Null(),
-					Type.Array(Type.String({ minLength: 1, maxLength: MAX_IMAGE_PATH_CHARS }), {
-						minItems: 1,
-						maxItems: MAX_REFERENCE_IMAGE_COUNT,
-					}),
-				],
-				{
-					description:
-						"Use null when the user did not explicitly identify reference images; otherwise provide one to five user-identified local image paths.",
-				},
-			),
-		),
-		outputPath: Type.Optional(
-			Type.Union(
-				[
-					Type.Null(),
-					Type.String({
-						minLength: 1,
-						maxLength: MAX_IMAGE_PATH_CHARS,
-					}),
-				],
-				{
-					description:
-						"Use null when the user did not explicitly request a destination; otherwise provide the requested explicit .png file path.",
-				},
-			),
-		),
-		size: Type.Optional(
-			StringEnum(IMAGE_GENERATION_SIZES, {
-				description: "Requested PNG dimensions, or auto.",
-			}),
-		),
-		quality: Type.Optional(
-			StringEnum(IMAGE_GENERATION_QUALITIES, {
-				description: "Requested image quality, or auto.",
-			}),
-		),
-	},
-	{ additionalProperties: false },
-);
-
-type GenerateImageToolParams = Static<typeof GenerateImageParameters>;
-
-function syncImageGenerationTool(
-	pi: ExtensionAPI,
-	model: Parameters<typeof isImageGenerationEnabledForModel>[0],
-	config: Parameters<typeof isImageGenerationEnabledForModel>[1],
-): void {
-	const eligible = isImageGenerationEnabledForModel(model, config);
-	const activeTools = pi.getActiveTools();
-	const active = activeTools.includes(IMAGE_GENERATION_TOOL_NAME);
-	if (eligible && !active) {
-		pi.setActiveTools([...activeTools, IMAGE_GENERATION_TOOL_NAME]);
-	} else if (!eligible && active) {
-		pi.setActiveTools(activeTools.filter((name) => name !== IMAGE_GENERATION_TOOL_NAME));
-	}
-}
-
-export function registerImageGenerationExtension(
-	pi: ExtensionAPI,
-	loadConfig: typeof loadToolkitConfig = loadToolkitConfig,
-	executeImage: typeof executeImageGeneration = executeImageGeneration,
-): void {
-	if (registeredApis.has(pi)) return;
-	registeredApis.add(pi);
-
-	pi.registerTool<typeof GenerateImageParameters, ImageGenerationDetails, Record<string, never>>({
-		name: IMAGE_GENERATION_TOOL_NAME,
-		label: "OpenAI Generate Image",
-		description:
-			"Generate a PNG image, or edit from one to five user-approved local reference images, through the current allowlisted Responses model and the hosted gpt-image-2 image_generation tool. This is a paid provider operation.",
-		promptSnippet: "Generate or edit PNG images through the current allowlisted Responses model and gpt-image-2.",
-		promptGuidelines: [
-			"Use openai_generate_image when the user explicitly asks to create, draw, render, or edit a raster image.",
-			"Do not call openai_generate_image speculatively: it consumes the user's provider or gateway image quota.",
-			"Keep the image prompt faithful to the user's requested subject and constraints; do not invent unrequested style details.",
-			"Set referenceImagePaths to null unless the user explicitly identified local files; never invent paths or placeholder strings, and remember upload requires user approval.",
-			"Set outputPath to null unless the user explicitly asks for a destination; never invent a destination or placeholder string, and otherwise use the default Pi agent artifact.",
-			"Do not substitute Python, browser automation, shell scripts, or unrelated image tools for an eligible image request.",
-		],
-		parameters: GenerateImageParameters,
-		executionMode: "sequential",
-		async execute(toolCallId, params: GenerateImageToolParams, signal, _onUpdate, ctx) {
-			try {
-				const result = await executeImage({
-					params,
-					toolCallId,
-					signal,
-					ctx,
-				});
-				return {
-					content: [{ type: "text", text: result.text }],
-					details: result.details,
-				};
-			} catch (error) {
-				if (error instanceof ImageGenerationError) {
-					throw new Error(error.message);
-				}
-				throw new Error(
-					sanitizeImageDiagnostic(error instanceof Error ? error.message : error, "Image generation failed."),
-				);
-			}
-		},
-		renderResult: renderImageGenerationResult,
-	});
-
-	pi.on("session_start", (_event, ctx) => {
-		const { config } = loadConfig();
-		syncImageGenerationTool(pi, ctx.model, config.imageGeneration);
-	});
-
-	pi.on("model_select", (event) => {
-		const { config } = loadConfig();
-		syncImageGenerationTool(pi, event.model, config.imageGeneration);
-	});
-
-	pi.on("before_agent_start", (_event, ctx) => {
-		const { config } = loadConfig();
-		syncImageGenerationTool(pi, ctx.model, config.imageGeneration);
-	});
-}
-
-export default function imageGenerationExtension(pi: ExtensionAPI): void {
-	registerImageGenerationExtension(pi);
-}
-
-export const _extensionTest = {
-	GenerateImageParameters,
-	syncImageGenerationTool,
-};
+import { StringEnum, Type, type Static } from "@earendil-works/pi-ai";
+import type { ExtensionAPI } from "@earendil-works/pi-coding-agent";
+import { loadToolkitConfig } from "../config";
+import { executeImageGeneration } from "./service";
+import { isImageGenerationEnabledForModel } from "./eligibility";
+import { renderImageGenerationResult } from "./render";
+import {
+	IMAGE_GENERATION_QUALITIES,
+	IMAGE_GENERATION_SIZES,
+	IMAGE_GENERATION_TOOL_NAME,
+	MAX_IMAGE_PATH_CHARS,
+	MAX_IMAGE_PROMPT_CHARS,
+	MAX_REFERENCE_IMAGE_COUNT,
+	ImageGenerationError,
+	sanitizeImageDiagnostic,
+	type ImageGenerationDetails,
+} from "./types";
+
+const registeredApis = new WeakSet<object>();
+
+const GenerateImageParameters = Type.Object(
+	{
+		prompt: Type.String({
+			minLength: 1,
+			maxLength: MAX_IMAGE_PROMPT_CHARS,
+			description: "Image generation or edit prompt. Preserve the user's requested subject and constraints.",
+		}),
+		referenceImagePaths: Type.Optional(
+			Type.Union(
+				[
+					Type.Null(),
+					Type.Array(Type.String({ minLength: 1, maxLength: MAX_IMAGE_PATH_CHARS }), {
+						minItems: 1,
+						maxItems: MAX_REFERENCE_IMAGE_COUNT,
+					}),
+				],
+				{
+					description:
+						"Use null when the user did not explicitly identify reference images; otherwise provide one to five user-identified local image paths.",
+				},
+			),
+		),
+		outputPath: Type.Optional(
+			Type.Union(
+				[
+					Type.Null(),
+					Type.String({
+						minLength: 1,
+						maxLength: MAX_IMAGE_PATH_CHARS,
+					}),
+				],
+				{
+					description:
+						"Use null when the user did not explicitly request a destination; otherwise provide the requested explicit .png file path.",
+				},
+			),
+		),
+		size: Type.Optional(
+			StringEnum(IMAGE_GENERATION_SIZES, {
+				description: "Requested PNG dimensions, or auto.",
+			}),
+		),
+		quality: Type.Optional(
+			StringEnum(IMAGE_GENERATION_QUALITIES, {
+				description: "Requested image quality, or auto.",
+			}),
+		),
+	},
+	{ additionalProperties: false },
+);
+
+type GenerateImageToolParams = Static<typeof GenerateImageParameters>;
+
+function syncImageGenerationTool(
+	pi: ExtensionAPI,
+	model: Parameters<typeof isImageGenerationEnabledForModel>[0],
+	config: Parameters<typeof isImageGenerationEnabledForModel>[1],
+): void {
+	const eligible = isImageGenerationEnabledForModel(model, config);
+	const activeTools = pi.getActiveTools();
+	const active = activeTools.includes(IMAGE_GENERATION_TOOL_NAME);
+	if (eligible && !active) {
+		pi.setActiveTools([...activeTools, IMAGE_GENERATION_TOOL_NAME]);
+	} else if (!eligible && active) {
+		pi.setActiveTools(activeTools.filter((name) => name !== IMAGE_GENERATION_TOOL_NAME));
+	}
+}
+
+export function registerImageGenerationExtension(
+	pi: ExtensionAPI,
+	loadConfig: typeof loadToolkitConfig = loadToolkitConfig,
+	executeImage: typeof executeImageGeneration = executeImageGeneration,
+): void {
+	if (registeredApis.has(pi)) return;
+	registeredApis.add(pi);
+
+	pi.registerTool<typeof GenerateImageParameters, ImageGenerationDetails, Record<string, never>>({
+		name: IMAGE_GENERATION_TOOL_NAME,
+		label: "OpenAI Generate Image",
+		description:
+			"Generate a PNG image, or edit from one to five user-approved local reference images, through the current Responses-capable model and the hosted gpt-image-2 image_generation tool. This is a paid provider operation.",
+		promptSnippet: "Generate or edit PNG images through the current Responses-capable model and gpt-image-2.",
+		promptGuidelines: [
+			"Use openai_generate_image when the user explicitly asks to create, draw, render, or edit a raster image and the active model speaks a Responses API.",
+			"Do not call openai_generate_image speculatively: it consumes the user's provider or gateway image quota.",
+			"Keep the image prompt faithful to the user's requested subject and constraints; do not invent unrequested style details.",
+			"Set referenceImagePaths to null unless the user explicitly identified local files; never invent paths or placeholder strings, and remember upload requires user approval.",
+			"Set outputPath to null unless the user explicitly asks for a destination; never invent a destination or placeholder string, and otherwise use the default Pi agent artifact.",
+			"Do not substitute Python, browser automation, shell scripts, or unrelated image tools for an eligible image request.",
+		],
+		parameters: GenerateImageParameters,
+		executionMode: "sequential",
+		async execute(toolCallId, params: GenerateImageToolParams, signal, _onUpdate, ctx) {
+			try {
+				const result = await executeImage({
+					params,
+					toolCallId,
+					signal,
+					ctx,
+				});
+				return {
+					content: [{ type: "text", text: result.text }],
+					details: result.details,
+				};
+			} catch (error) {
+				if (error instanceof ImageGenerationError) {
+					throw new Error(error.message);
+				}
+				throw new Error(
+					sanitizeImageDiagnostic(error instanceof Error ? error.message : error, "Image generation failed."),
+				);
+			}
+		},
+		renderResult: renderImageGenerationResult,
+	});
+
+	pi.on("session_start", (_event, ctx) => {
+		const { config } = loadConfig();
+		syncImageGenerationTool(pi, ctx.model, config.imageGeneration);
+	});
+
+	pi.on("model_select", (event) => {
+		const { config } = loadConfig();
+		syncImageGenerationTool(pi, event.model, config.imageGeneration);
+	});
+
+	pi.on("before_agent_start", (_event, ctx) => {
+		const { config } = loadConfig();
+		syncImageGenerationTool(pi, ctx.model, config.imageGeneration);
+	});
+}
+
+export default function imageGenerationExtension(pi: ExtensionAPI): void {
+	registerImageGenerationExtension(pi);
+}
+
+export const _extensionTest = {
+	GenerateImageParameters,
+	syncImageGenerationTool,
+};

package/src/native-fallback.ts

@@ -12,6 +12,7 @@ import type { CompactionConfig } from "./types";
 export { parseModelSpec } from "./runtime";
 
 export type NativeFallbackFailureReason =
+	| "disabled"
 	| "no-model-configured"
 	| "invalid-model-spec"
 	| "model-not-found"
@@ -53,23 +54,35 @@ function toErrorMessage(error: unknown): string {
 }
 
 /**
- * Run pi's native compaction method with the user-configured compaction model.
+ * Run pi's native compaction method with a caller-resolved summary model.
  *
- * Only handles the "configured model differs from the current one" case: when no model
- * is configured (or it equals the current model), the caller should return undefined from
- * session_before_compact so pi runs the same native path itself, keeping its internal
+ * The two compaction chains stay separate: the caller passes `remoteCompactModel` only when a
+ * remote v2 request was actually attempted and failed, and `nativeFallback.model` only when the
+ * active model cannot use remote v2 at all. This function never inspects `remoteCompactModel`
+ * itself, so a model that cannot compact remotely can never inherit the remote producer.
+ *
+ * Only the "resolved model differs from the current one" case is handled here: when the feature
+ * is disabled, nothing is resolved, or it equals the current model, the caller returns undefined
+ * from session_before_compact so pi runs the same native path itself, keeping its internal
  * streamFn/thinkingLevel wiring.
  */
 export async function runNativeFallbackCompaction(args: {
 	ctx: ExtensionContext;
 	event: SessionBeforeCompactEvent;
 	config: CompactionConfig;
+	/** Model spec supplied by the caller for this specific fallback reason. */
+	modelSpec?: string;
 	compactFn?: NativeCompactFn;
 }): Promise<NativeFallbackResult> {
 	const { ctx, event, config } = args;
 	const compactFn = args.compactFn ?? compact;
+	const nativeFallback = config.nativeFallback;
+
+	if (!nativeFallback.enabled) {
+		return { ok: false, reason: "disabled" };
+	}
 
-	const spec = config.model?.trim();
+	const spec = args.modelSpec?.trim() ?? "";
 	if (!spec) {
 		return { ok: false, reason: "no-model-configured" };
 	}
@@ -106,7 +119,7 @@ export async function runNativeFallbackCompaction(args: {
 			mergeProviderHeaders(auth.headers),
 			event.customInstructions,
 			event.signal,
-			config.thinkingLevel,
+			nativeFallback.thinkingLevel,
 			undefined,
 			auth.env,
 		);

package/src/types.ts

@@ -5,6 +5,7 @@ export const TOOLKIT_ID = "pi-openai-toolkit";
 export const COMPACTION_EXTENSION_ID = `${TOOLKIT_ID}:compaction`;
 export const WEB_SEARCH_EXTENSION_ID = `${TOOLKIT_ID}:web-search`;
 export const IMAGE_GENERATION_EXTENSION_ID = `${TOOLKIT_ID}:image-generation`;
+export const AUTO_MODE_EXTENSION_ID = `${TOOLKIT_ID}:auto-mode`;
 export const DEFAULT_ARTIFACT_ROOT = "~/.pi/agent/artifacts/pi-openai-toolkit/compaction";
 export const REDACTED_VALUE = "[REDACTED]";
 /**
@@ -39,6 +40,20 @@ export type DebugArtifactKind =
 
 export type CompactionContinuationMode = "inline" | "followUp" | "off";
 
+/** Native-method fallback compaction: which model runs pi's compact() and how deeply. */
+export type NativeFallbackConfig = {
+	enabled: boolean;
+	/**
+	 * "provider/model-id" used for native-method fallback compaction when the active model
+	 * cannot use remote v2 at all. A failed remote request is instead summarized by
+	 * `remoteCompactModel`; the two sources never substitute for each other. Unset means the
+	 * active model via pi's default path.
+	 */
+	model?: string;
+	/** Thinking level passed to pi's native compact() when the fallback model runs. */
+	thinkingLevel: ThinkingLevel;
+};
+
 export type AutoCompactionConfig = {
 	enabled: boolean;
 	continuation: CompactionContinuationMode;
@@ -58,13 +73,8 @@ export type CompactionConfig = {
 	 * remains the checkpoint consumer and continues handling normal requests.
 	 */
 	remoteCompactModel?: string;
-	/**
-	 * "provider/model-id" used for native-method fallback compaction (non-Responses APIs,
-	 * or when the compact endpoint fails). Unset = current model via pi's default path.
-	 */
-	model?: string;
-	/** Thinking level passed to pi's native compact() when the fallback model runs. */
-	thinkingLevel: ThinkingLevel;
+	/** Native-method fallback compaction policy (non-Responses APIs, or when the compact endpoint fails). */
+	nativeFallback: NativeFallbackConfig;
 	/** Threshold-triggered compaction and same-tool-loop continuation policy. */
 	autoCompaction: AutoCompactionConfig;
 	/** Subset of RESPONSES_COMPACT_CAPABLE_APIS that should use remote compaction. */
@@ -85,14 +95,73 @@ export type WebSearchConfig = {
 
 export type ImageGenerationConfig = {
 	enabled: boolean;
-	/** Exact provider/model keys allowed to use toolkit-native image generation. */
+};
+
+/** `side-effect` reviews bash/write/edit plus extras; `all` reviews every tool call. */
+export type AutoModeGate = "side-effect" | "all";
+
+export type AutoModeConfig = {
+	enabled: boolean;
+	/** Exact provider/model keys whose sessions may engage auto mode. */
 	models: string[];
+	/** "provider/model-id" that approves gated tool calls while auto mode is engaged. */
+	reviewerModel?: string;
+	gate: AutoModeGate;
+	/** Extra tool names reviewed on top of the side-effect default set. */
+	extraTools: string[];
+	/** Reviewer deadline in ms; expiry degrades to human confirmation or fail-closed. */
+	timeoutMs: number;
+	/** Give the reviewer a bounded transcript so it can judge user authorization. */
+	transcript: boolean;
+	/** Let the reviewer spend read-only tool calls establishing facts before deciding. */
+	evidenceTools: boolean;
+	maxEvidenceRounds: number;
+	/** Non-blocking trajectory pre-scorer that can satisfy ordinary gated calls. */
+	classifier: AutoModeClassifierConfig;
+	/** End a turn that keeps getting denied instead of negotiating forever. */
+	circuitBreaker: AutoModeCircuitBreakerConfig;
 };
 
+export type AutoModeClassifierConfig = {
+	enabled: boolean;
+	/** "provider/model-id"; falls back to `autoMode.reviewerModel` when unset. */
+	model?: string;
+	timeoutMs: number;
+	/** How many tool calls may elapse between scoring and use before a score is stale. */
+	maxLag: number;
+};
+
+export type AutoModeCircuitBreakerConfig = {
+	/** Consecutive reviewer denials in one turn that end the turn. 0 disables. */
+	consecutiveDenials: number;
+	/** Denials inside `windowSize` reviewed calls that end the turn. 0 disables. */
+	recentDenials: number;
+	windowSize: number;
+};
+
+export const REVIEWER_TIMEOUT_MIN_MS = 1_000;
+export const REVIEWER_TIMEOUT_MAX_MS = 120_000;
+
+/** Read-only investigation rounds the reviewer may spend before it must answer. */
+export const EVIDENCE_ROUNDS_MIN = 0;
+export const EVIDENCE_ROUNDS_MAX = 8;
+
+export const CLASSIFIER_MAX_LAG_MIN = 0;
+export const CLASSIFIER_MAX_LAG_MAX = 20;
+
+export const DEFAULT_CLASSIFIER_TIMEOUT_MS = 15_000;
+
+/** Denial counts are small integers; 0 disables the matching trigger. */
+export const BREAKER_LIMIT_MIN = 0;
+export const BREAKER_LIMIT_MAX = 100;
+export const BREAKER_WINDOW_MIN = 1;
+export const BREAKER_WINDOW_MAX = 200;
+
 export type ToolkitConfig = {
 	compaction: CompactionConfig;
 	webSearch: WebSearchConfig;
 	imageGeneration: ImageGenerationConfig;
+	autoMode: AutoModeConfig;
 };
 
 export type LoadedToolkitConfig = {
@@ -425,12 +494,17 @@ export const DEFAULT_AUTO_COMPACTION_CONFIG: AutoCompactionConfig = {
 	reserveTokens: undefined,
 };
 
+export const DEFAULT_NATIVE_FALLBACK_CONFIG: NativeFallbackConfig = {
+	enabled: true,
+	model: undefined,
+	thinkingLevel: "off",
+};
+
 export const DEFAULT_COMPACTION_CONFIG: CompactionConfig = {
 	enabled: true,
 	allowCompactionContinuityBreak: false,
 	remoteCompactModel: undefined,
-	model: undefined,
-	thinkingLevel: "off",
+	nativeFallback: { ...DEFAULT_NATIVE_FALLBACK_CONFIG },
 	autoCompaction: { ...DEFAULT_AUTO_COMPACTION_CONFIG },
 	responsesApis: [...RESPONSES_COMPACT_CAPABLE_APIS],
 	notifyOnLoad: false,
@@ -447,12 +521,35 @@ export const DEFAULT_WEB_SEARCH_CONFIG: WebSearchConfig = {
 };
 
 export const DEFAULT_IMAGE_GENERATION_CONFIG: ImageGenerationConfig = {
+	enabled: false,
+};
+
+export const DEFAULT_AUTO_MODE_CONFIG: AutoModeConfig = {
 	enabled: true,
 	models: [],
+	reviewerModel: undefined,
+	gate: "side-effect",
+	extraTools: [],
+	timeoutMs: 30_000,
+	transcript: true,
+	evidenceTools: true,
+	maxEvidenceRounds: 3,
+	classifier: {
+		enabled: false,
+		model: undefined,
+		timeoutMs: DEFAULT_CLASSIFIER_TIMEOUT_MS,
+		maxLag: 2,
+	},
+	circuitBreaker: {
+		consecutiveDenials: 3,
+		recentDenials: 10,
+		windowSize: 50,
+	},
 };
 
 export const DEFAULT_TOOLKIT_CONFIG: ToolkitConfig = {
 	compaction: DEFAULT_COMPACTION_CONFIG,
 	webSearch: DEFAULT_WEB_SEARCH_CONFIG,
 	imageGeneration: DEFAULT_IMAGE_GENERATION_CONFIG,
+	autoMode: DEFAULT_AUTO_MODE_CONFIG,
 };

package/tsconfig.check.json

@@ -15,6 +15,7 @@
     "extensions/compaction.ts",
     "extensions/web-search.ts",
     "extensions/image-generation.ts",
+    "extensions/auto-mode.ts",
     "src/compact-client.ts",
     "src/config.ts",
     "src/debug.ts",
@@ -31,6 +32,10 @@
     "src/image-generation/render.ts",
     "src/image-generation/service.ts",
     "src/image-generation/types.ts",
+    "src/auto-mode/extension.ts",
+    "src/auto-mode/policy.ts",
+    "src/auto-mode/reviewer.ts",
+    "src/auto-mode/types.ts",
     "src/model-scope.ts",
     "src/native-fallback.ts",
     "src/payload-rewrite.ts",