promptfoo 0.121.2 → 0.121.4

package/dist/src/{agents-DbRtpYxR.cjs → agents-BahDpe5G.cjs}

@@ -1,10 +1,43 @@
-const require_logger = require("./logger-Cp1GPUjj.cjs");
-const require_esm = require("./esm-CnNt7sI4.cjs");
-const require_fetch = require("./fetch-BPkYtG8K.cjs");
-const require_openai = require("./openai-Cuif0GEt.cjs");
+const require_logger = require("./logger-COuQb2xB.cjs");
+const require_esm = require("./esm-CtEPLdAj.cjs");
+const require_fetch = require("./fetch-CJU5ELPa.cjs");
+const require_openai = require("./openai-URNyItar.cjs");
 let path = require("path");
 path = require_logger.__toESM(path);
 let _openai_agents = require("@openai/agents");
+//#region src/providers/openai/agents-model-settings.ts
+function resolveModelSettings(modelSettings) {
+	if (!modelSettings) return;
+	const retry = modelSettings.retry ? {
+		...modelSettings.retry,
+		policy: resolveRetryPolicy(modelSettings.retry.policy)
+	} : void 0;
+	return {
+		...modelSettings,
+		retry
+	};
+}
+function resolveRetryPolicy(policy) {
+	if (!policy) return;
+	if (typeof policy === "function") return policy;
+	if (typeof policy === "string") switch (policy) {
+		case "never": return _openai_agents.retryPolicies.never();
+		case "providerSuggested": return _openai_agents.retryPolicies.providerSuggested();
+		case "networkError": return _openai_agents.retryPolicies.networkError();
+		case "retryAfter": return _openai_agents.retryPolicies.retryAfter();
+		default: return;
+	}
+	if ("httpStatus" in policy) return _openai_agents.retryPolicies.httpStatus(policy.httpStatus);
+	if ("any" in policy) {
+		const policies = policy.any.map((nestedPolicy) => resolveRetryPolicy(nestedPolicy)).filter((nestedPolicy) => !!nestedPolicy);
+		return policies.length ? _openai_agents.retryPolicies.any(...policies) : void 0;
+	}
+	if ("all" in policy) {
+		const policies = policy.all.map((nestedPolicy) => resolveRetryPolicy(nestedPolicy)).filter((nestedPolicy) => !!nestedPolicy);
+		return policies.length ? _openai_agents.retryPolicies.all(...policies) : void 0;
+	}
+}
+//#endregion
 //#region src/providers/openai/agents-loader.ts
 /**
 * Load agent definition from file path or return inline definition
@@ -18,7 +51,7 @@ async function loadAgentDefinition(agentConfig) {
 		require_logger.logger.debug("[AgentsLoader] Loading agent from file", { path: agentConfig });
 		return await loadAgentFromFile(agentConfig);
 	}
-	if (typeof agentConfig === "object") {
+	if (agentConfig !== null && typeof agentConfig === "object" && !Array.isArray(agentConfig)) {
 		require_logger.logger.debug("[AgentsLoader] Creating agent from inline definition");
 		return await createAgentFromDefinition(agentConfig);
 	}
@@ -29,6 +62,82 @@ async function loadAgentDefinition(agentConfig) {
 	throw new Error("Invalid agent configuration: expected Agent instance, file:// URL, or inline definition");
 }
 /**
+* Load tools from file path or return inline definitions
+*/
+async function loadTools(toolsConfig) {
+	if (!toolsConfig) return;
+	if (typeof toolsConfig === "string" && toolsConfig.startsWith("file://")) {
+		require_logger.logger.debug("[AgentsLoader] Loading tools from file", { path: toolsConfig });
+		return await loadToolsFromFile(toolsConfig);
+	}
+	if (Array.isArray(toolsConfig)) {
+		require_logger.logger.debug("[AgentsLoader] Using inline tool definitions");
+		return normalizeTools(toolsConfig);
+	}
+	require_logger.logger.debug("[AgentsLoader] Invalid tools configuration", {
+		type: typeof toolsConfig,
+		isArray: Array.isArray(toolsConfig)
+	});
+	throw new Error("Invalid tools configuration: expected file:// URL or array");
+}
+/**
+* Load handoffs from file path or return inline definitions
+*/
+async function loadHandoffs(handoffsConfig) {
+	if (!handoffsConfig) return;
+	if (typeof handoffsConfig === "string" && handoffsConfig.startsWith("file://")) {
+		require_logger.logger.debug("[AgentsLoader] Loading handoffs from file", { path: handoffsConfig });
+		return await loadHandoffsFromFile(handoffsConfig);
+	}
+	if (Array.isArray(handoffsConfig)) {
+		require_logger.logger.debug("[AgentsLoader] Using inline handoff definitions");
+		return normalizeHandoffs(handoffsConfig);
+	}
+	require_logger.logger.debug("[AgentsLoader] Invalid handoffs configuration", {
+		type: typeof handoffsConfig,
+		isArray: Array.isArray(handoffsConfig)
+	});
+	throw new Error("Invalid handoffs configuration: expected file:// URL or array");
+}
+/**
+* Load input guardrails from file path or return inline definitions
+*/
+async function loadInputGuardrails(guardrailsConfig) {
+	if (!guardrailsConfig) return;
+	if (typeof guardrailsConfig === "string" && guardrailsConfig.startsWith("file://")) {
+		require_logger.logger.debug("[AgentsLoader] Loading input guardrails from file", { path: guardrailsConfig });
+		return await loadArrayFromFile(guardrailsConfig, "input guardrails");
+	}
+	if (Array.isArray(guardrailsConfig)) {
+		require_logger.logger.debug("[AgentsLoader] Using inline input guardrails");
+		return guardrailsConfig;
+	}
+	require_logger.logger.debug("[AgentsLoader] Invalid input guardrails configuration", {
+		type: typeof guardrailsConfig,
+		isArray: Array.isArray(guardrailsConfig)
+	});
+	throw new Error("Invalid input guardrails configuration: expected file:// URL or array");
+}
+/**
+* Load output guardrails from file path or return inline definitions
+*/
+async function loadOutputGuardrails(guardrailsConfig) {
+	if (!guardrailsConfig) return;
+	if (typeof guardrailsConfig === "string" && guardrailsConfig.startsWith("file://")) {
+		require_logger.logger.debug("[AgentsLoader] Loading output guardrails from file", { path: guardrailsConfig });
+		return await loadArrayFromFile(guardrailsConfig, "output guardrails");
+	}
+	if (Array.isArray(guardrailsConfig)) {
+		require_logger.logger.debug("[AgentsLoader] Using inline output guardrails");
+		return guardrailsConfig;
+	}
+	require_logger.logger.debug("[AgentsLoader] Invalid output guardrails configuration", {
+		type: typeof guardrailsConfig,
+		isArray: Array.isArray(guardrailsConfig)
+	});
+	throw new Error("Invalid output guardrails configuration: expected file:// URL or array");
+}
+/**
 * Load agent from file
 */
 async function loadAgentFromFile(filePath) {
@@ -48,19 +157,39 @@ async function loadAgentFromFile(filePath) {
 	}
 }
 /**
+* Load tools from file
+*/
+async function loadToolsFromFile(filePath) {
+	return await normalizeTools(await loadArrayFromFile(filePath, "tools")) ?? [];
+}
+/**
+* Load handoffs from file
+*/
+async function loadHandoffsFromFile(filePath) {
+	return await normalizeHandoffs(await loadArrayFromFile(filePath, "handoffs")) ?? [];
+}
+/**
 * Create an Agent instance from an inline definition
 */
 async function createAgentFromDefinition(definition) {
 	try {
-		const { Agent } = await import("@openai/agents");
-		return new Agent({
+		const tools = await normalizeTools(definition.tools);
+		const handoffs = await normalizeHandoffs(definition.handoffs);
+		return new _openai_agents.Agent({
 			name: definition.name,
 			instructions: definition.instructions,
+			prompt: definition.prompt,
 			model: definition.model,
+			modelSettings: resolveModelSettings(definition.modelSettings),
 			handoffDescription: definition.handoffDescription,
 			outputType: definition.outputType,
-			tools: definition.tools,
-			handoffs: definition.handoffs
+			tools,
+			handoffs,
+			inputGuardrails: definition.inputGuardrails,
+			outputGuardrails: definition.outputGuardrails,
+			mcpServers: definition.mcpServers,
+			toolUseBehavior: definition.toolUseBehavior,
+			resetToolChoice: definition.resetToolChoice
 		});
 	} catch (error) {
 		require_logger.logger.error("[AgentsLoader] Failed to create agent from definition", {
@@ -77,7 +206,52 @@ async function createAgentFromDefinition(definition) {
 * Check if a value is an Agent instance
 */
 function isAgentInstance(value) {
-	return value && typeof value === "object" && "name" in value && "instructions" in value && typeof value.name === "string";
+	return value instanceof _openai_agents.Agent;
+}
+function isToolInstance(value) {
+	return !!value && typeof value === "object" && "type" in value && typeof value.type === "string";
+}
+function isHandoffInstance(value) {
+	return !!value && typeof value === "object" && "agent" in value && "getHandoffAsFunctionTool" in value && typeof value.getHandoffAsFunctionTool === "function";
+}
+async function normalizeTools(definitions) {
+	if (!definitions?.length) return;
+	return definitions.map((definition) => {
+		if (isToolInstance(definition)) return definition;
+		return (0, _openai_agents.tool)({
+			name: definition.name,
+			description: definition.description,
+			parameters: definition.parameters,
+			strict: definition.strict ?? true,
+			deferLoading: definition.deferLoading,
+			execute: typeof definition.execute === "function" ? definition.execute : async () => definition.execute
+		});
+	});
+}
+async function normalizeHandoffs(definitions) {
+	if (!definitions?.length) return;
+	return Promise.all(definitions.map(async (definition) => {
+		if (isAgentInstance(definition) || isHandoffInstance(definition)) return definition;
+		const agent = await loadAgentDefinition(definition.agent);
+		if (!definition.description) return agent;
+		return (0, _openai_agents.handoff)(agent, { toolDescriptionOverride: definition.description });
+	}));
+}
+async function loadArrayFromFile(filePath, label) {
+	const resolvedPath = resolveFilePath(filePath);
+	require_logger.logger.debug(`[AgentsLoader] Loading ${label} from resolved path`, { path: resolvedPath });
+	try {
+		const module = await require_esm.importModule(resolvedPath);
+		const exported = module.default || module;
+		if (!Array.isArray(exported)) throw new Error(`File ${resolvedPath} does not export an array of ${label}`);
+		return exported;
+	} catch (error) {
+		require_logger.logger.error(`[AgentsLoader] Failed to load ${label} from file`, {
+			path: resolvedPath,
+			error
+		});
+		throw new Error(`Failed to load ${label} from ${resolvedPath}: ${error}`);
+	}
 }
 /**
 * Resolve file:// path to absolute file system path
@@ -329,8 +503,27 @@ var OpenAiAgentsProvider = class extends require_openai.OpenAiGenericProvider {
 		if (!this.agentConfig.agent) throw new Error("No agent configuration provided");
 		try {
 			const agent = await loadAgentDefinition(this.agentConfig.agent);
-			require_logger.logger.debug("[AgentsProvider] Agent initialized successfully", { name: agent.name });
-			return agent;
+			const [tools, handoffs, inputGuardrails, outputGuardrails] = await Promise.all([
+				loadTools(this.agentConfig.tools),
+				loadHandoffs(this.agentConfig.handoffs),
+				loadInputGuardrails(this.agentConfig.inputGuardrails),
+				loadOutputGuardrails(this.agentConfig.outputGuardrails)
+			]);
+			const configuredAgent = agent.clone({
+				tools: mergeArrays(agent.tools, tools),
+				handoffs: mergeArrays(agent.handoffs, handoffs),
+				inputGuardrails: mergeArrays(agent.inputGuardrails, inputGuardrails),
+				outputGuardrails: mergeArrays(agent.outputGuardrails, outputGuardrails)
+			});
+			const mockAwareAgent = this.wrapToolsIfNeeded(configuredAgent);
+			require_logger.logger.debug("[AgentsProvider] Agent initialized successfully", {
+				name: mockAwareAgent.name,
+				toolCount: mockAwareAgent.tools.length,
+				handoffCount: mockAwareAgent.handoffs.length,
+				inputGuardrailCount: mockAwareAgent.inputGuardrails.length,
+				outputGuardrailCount: mockAwareAgent.outputGuardrails.length
+			});
+			return mockAwareAgent;
 		} catch (error) {
 			require_logger.logger.error("[AgentsProvider] Failed to initialize agent", { error });
 			throw new Error(`Failed to initialize agent: ${error}`);
@@ -381,17 +574,17 @@ var OpenAiAgentsProvider = class extends require_openai.OpenAiGenericProvider {
 		try {
 			require_logger.logger.debug("[AgentsProvider] Running agent", {
 				agentName: this.agent?.name,
-				maxTurns: this.agentConfig.maxTurns || 10
+				maxTurns: this.agentConfig.maxTurns ?? 10
 			});
 			const runOptions = {
 				context: context?.vars,
-				maxTurns: this.agentConfig.maxTurns || 10,
+				maxTurns: this.agentConfig.maxTurns ?? 10,
 				signal: callApiOptions?.abortSignal
 			};
 			if (this.agentConfig.model || this.modelName) runOptions.model = this.agentConfig.model || this.modelName;
-			if (this.agentConfig.modelSettings) runOptions.modelSettings = this.agentConfig.modelSettings;
+			if (this.agentConfig.modelSettings) runOptions.modelSettings = resolveModelSettings(this.agentConfig.modelSettings);
 			const result = await (0, _openai_agents.getOrCreateTrace)(async () => {
-				return await (0, _openai_agents.run)(this.agent, prompt, runOptions);
+				return await (0, _openai_agents.run)(this.agent, this.parsePromptInput(prompt), runOptions);
 			});
 			require_logger.logger.debug("[AgentsProvider] Agent run result", {
 				hasOutput: !!result.finalOutput,
@@ -415,17 +608,59 @@ var OpenAiAgentsProvider = class extends require_openai.OpenAiGenericProvider {
 		if (!result.usage) return {};
 		const usage = result.usage;
 		return {
-			total: usage.totalTokens || void 0,
-			prompt: usage.promptTokens || void 0,
-			completion: usage.completionTokens || void 0
+			total: usage.totalTokens ?? void 0,
+			prompt: usage.promptTokens ?? void 0,
+			completion: usage.completionTokens ?? void 0
 		};
 	}
 	/**
 	* Calculate cost from agent result
 	*/
 	calculateCost(_result) {}
+	wrapToolsIfNeeded(agent) {
+		if (this.agentConfig.executeTools !== false && this.agentConfig.executeTools !== "mock") return agent;
+		const toolMocks = this.agentConfig.toolMocks ?? {};
+		const tools = agent.tools.map((tool) => {
+			if (tool.type !== "function") return tool;
+			const mockValue = toolMocks[tool.name];
+			return {
+				...tool,
+				invoke: async () => mockValue ?? {
+					mocked: true,
+					tool: tool.name
+				}
+			};
+		});
+		return agent.clone({ tools });
+	}
+	parsePromptInput(prompt) {
+		try {
+			const parsedInput = parseAgentInputItems(JSON.parse(prompt));
+			if (parsedInput) return parsedInput;
+		} catch {}
+		return prompt;
+	}
 };
+function mergeArrays(existing, additions) {
+	if (!existing?.length && !additions?.length) return;
+	return [...existing ?? [], ...additions ?? []];
+}
+function parseAgentInputItems(value) {
+	if (Array.isArray(value)) {
+		if (value.length === 0) return;
+		const parsedItems = [];
+		for (const item of value) {
+			const parsedItem = _openai_agents.protocol.ModelItem.safeParse(item);
+			if (!parsedItem.success) return;
+			parsedItems.push(parsedItem.data);
+		}
+		return parsedItems;
+	}
+	const parsedItem = _openai_agents.protocol.ModelItem.safeParse(value);
+	if (!parsedItem.success) return;
+	return [parsedItem.data];
+}
 //#endregion
 exports.OpenAiAgentsProvider = OpenAiAgentsProvider;
 
-//# sourceMappingURL=agents-DbRtpYxR.cjs.map
+//# sourceMappingURL=agents-BahDpe5G.cjs.map