promptfoo 0.125.0 → 0.125.1

package/dist/src/functionCallbackUtils-B_cYp7P5.js

@@ -0,0 +1,245 @@
+import { o as logger } from "./logger-CNHLbG5Y.js";
+import { S as isCallerAbortError, h as throwIfAborted, y as waitForPromiseWithAbort } from "./shared-BnN4RWgk.js";
+import { h as loadCallbackFromFileUrl, m as CallbackPathTraversalError, y as wrapError } from "./file-BPc_9IQ5.js";
+import { g as withGenAIToolSpan } from "./genaiTracer-QTyx3Q-T.js";
+import "./tracing-B8p6SMbW.js";
+import { c as normalizeMcpToolContent, i as getMcpErrorMessage, o as isMcpErrorResult } from "./util-CePiY2xZ.js";
+//#region src/providers/functionCallbackUtils.ts
+/**
+* Resolve a `file://` callback reference through the shared path-traversal guard,
+* preserving the underlying error on `cause` so callers can classify it.
+*/
+async function loadProviderCallbackFromFileUrl(fileRef, logPrefix) {
+	try {
+		return await loadCallbackFromFileUrl(fileRef, logPrefix ? { logPrefix } : void 0);
+	} catch (error) {
+		if (error instanceof CallbackPathTraversalError) throw error;
+		throw wrapError(`Error loading function from ${fileRef}: ${error.message}`, error);
+	}
+}
+/**
+* Load, cache, and invoke one `functionToolCallbacks` entry, returning the string a
+* tool-result message expects.
+*
+* Shared by the two providers that implement function-tool callbacks directly on the
+* provider class — Bedrock Converse and OpenAI Chat — whose copies were identical apart
+* from log prefixes. Keeping one implementation is what stops them drifting: the
+* path-traversal guard is a worked example of a fix that reached one copy of this logic
+* and not the others.
+*
+* The remaining providers with a `loadedFunctionCallbacks` cache (Azure Foundry, Google
+* base and live) have genuinely different loading and caching behaviour — Azure preloads,
+* Google Live gates on a shared-cache flag — so they deliberately keep their own.
+*/
+async function executeProviderFunctionCallback({ functionName, args, callId, callbacks, cache, logPrefix, abortSignal }) {
+	const prefix = logPrefix ? `${logPrefix} ` : "";
+	try {
+		throwIfAborted(abortSignal);
+		let callback = cache[functionName];
+		if (!callback) {
+			const callbackRef = callbacks?.[functionName];
+			if (callbackRef && typeof callbackRef === "string") {
+				if (callbackRef.startsWith("file://")) {
+					const loading = loadProviderCallbackFromFileUrl(callbackRef, logPrefix).then((loadedCallback) => {
+						cache[functionName] = loadedCallback;
+						return loadedCallback;
+					});
+					callback = await waitForPromiseWithAbort(loading, abortSignal);
+				} else {
+					callback = new Function("return " + callbackRef)();
+					cache[functionName] = callback;
+				}
+			} else if (typeof callbackRef === "function") {
+				callback = callbackRef;
+				cache[functionName] = callback;
+			}
+		}
+		throwIfAborted(abortSignal);
+		if (!callback) throw new Error(`No callback found for function '${functionName}'`);
+		logger.debug(`${prefix}Executing function '${functionName}' with args: ${args}`);
+		const result = await withGenAIToolSpan({
+			name: functionName,
+			arguments: args,
+			callId
+		}, () => {
+			throwIfAborted(abortSignal);
+			return waitForPromiseWithAbort(Promise.resolve(callback(args)), abortSignal);
+		});
+		throwIfAborted(abortSignal);
+		if (result === void 0 || result === null) return "";
+		if (typeof result === "object") try {
+			return JSON.stringify(result);
+		} catch (error) {
+			logger.warn(`Error stringifying result from function '${functionName}': ${error}`);
+			return String(result);
+		}
+		return String(result);
+	} catch (error) {
+		if (isCallerAbortError(error, abortSignal, { requireReasonMatch: true })) throwIfAborted(abortSignal);
+		logger.error(`${prefix}Error executing function '${functionName}': ${error.message || String(error)}`);
+		throw error;
+	}
+}
+/**
+* Handles function callback execution for AI providers.
+* Provides a unified way to execute function callbacks across different provider formats.
+*/
+var FunctionCallbackHandler = class {
+	mcpClient;
+	loadedCallbacks = {};
+	mcpToolNames = null;
+	constructor(mcpClient) {
+		this.mcpClient = mcpClient;
+	}
+	/**
+	* Processes a function call by executing its callback or returning the original call
+	* @param call The function call to process (can be various formats)
+	* @param callbacks Configuration mapping function names to callbacks
+	* @param context Optional context to pass to the callback
+	* @returns The result of processing
+	*/
+	async processCall(call, callbacks, context) {
+		const functionInfo = this.extractFunctionInfo(call);
+		if (this.mcpClient && functionInfo) {
+			if (this.mcpToolNames === null) {
+				const mcpTools = this.mcpClient.getAllTools();
+				this.mcpToolNames = new Set(mcpTools.map((tool) => tool.name));
+			}
+			if (this.mcpToolNames.has(functionInfo.name)) return await this.executeMcpTool(functionInfo.name, functionInfo.arguments);
+		}
+		if (!functionInfo || !callbacks || !callbacks[functionInfo.name]) return {
+			output: typeof call === "string" ? call : JSON.stringify(call),
+			isError: false
+		};
+		try {
+			return {
+				output: await this.executeCallback(functionInfo.name, functionInfo.arguments || "{}", callbacks, context, typeof call?.call_id === "string" ? call.call_id : typeof call?.id === "string" ? call.id : void 0),
+				isError: false
+			};
+		} catch (error) {
+			const cause = error?.cause;
+			if (error instanceof CallbackPathTraversalError || cause instanceof CallbackPathTraversalError) logger.warn(`[FunctionCallback] Rejected callback '${functionInfo.name}': ${error.message}`);
+			else logger.debug(`Function callback failed for ${functionInfo.name}: ${error}`);
+			return {
+				output: typeof call === "string" ? call : JSON.stringify(call),
+				isError: true
+			};
+		}
+	}
+	/**
+	* Processes multiple function calls
+	* @param calls Array of calls or a single call
+	* @param callbacks Configuration mapping function names to callbacks
+	* @param context Optional context to pass to callbacks
+	* @param options Processing options
+	* @returns Processed output in appropriate format
+	*/
+	async processCalls(calls, callbacks, context, _options) {
+		if (!calls) return calls;
+		const isArray = Array.isArray(calls);
+		const callsArray = isArray ? calls : [calls];
+		const results = await Promise.all(callsArray.map((call) => this.processCall(call, callbacks, context)));
+		if (results.some((r, index) => !r.isError && r.output !== JSON.stringify(callsArray[index]))) {
+			const outputs = results.map((r) => r.output);
+			if (!isArray && outputs.length === 1) return outputs[0];
+			return outputs.every((o) => typeof o === "string") ? outputs.join("\n") : outputs;
+		}
+		if (!isArray && results.length === 1) return results[0].output;
+		return calls;
+	}
+	/**
+	* Extracts function name and arguments from various call formats
+	*/
+	extractFunctionInfo(call) {
+		if (!call || typeof call !== "object") return null;
+		if (call.name && typeof call.name === "string") return {
+			name: call.name,
+			arguments: call.arguments
+		};
+		if (call.type === "function" && call.function?.name) return {
+			name: call.function.name,
+			arguments: call.function.arguments
+		};
+		return null;
+	}
+	/**
+	* Executes a function callback
+	*/
+	async executeCallback(functionName, args, callbacks, context, callId) {
+		return await withGenAIToolSpan({
+			name: functionName,
+			arguments: args,
+			callId
+		}, async () => {
+			let callback = this.loadedCallbacks[functionName];
+			if (!callback) {
+				const callbackConfig = callbacks[functionName];
+				if (typeof callbackConfig === "string") {
+					if (callbackConfig.startsWith("file://")) callback = await this.loadExternalFunction(callbackConfig);
+					else callback = new Function("return " + callbackConfig)();
+				} else if (typeof callbackConfig === "function") callback = callbackConfig;
+				else throw new Error(`Invalid callback configuration for ${functionName}`);
+				this.loadedCallbacks[functionName] = callback;
+			}
+			const result = await callback(args, context);
+			return typeof result === "string" ? result : JSON.stringify(result);
+		});
+	}
+	/**
+	* Loads a function from an external file.
+	*
+	* Uses `lenient: true` so a missing named export silently falls back to the
+	* module's default export, preserving long-standing behavior. The six
+	* provider-specific loaders use strict mode and throw on missing named
+	* exports — see {@link loadCallbackFromFileUrl}.
+	*/
+	async loadExternalFunction(fileRef) {
+		try {
+			return await loadCallbackFromFileUrl(fileRef, { lenient: true });
+		} catch (error) {
+			throw wrapError(`Failed to load function from ${fileRef}: ${error.message}`, error);
+		}
+	}
+	/**
+	* Executes an MCP tool
+	*/
+	async executeMcpTool(toolName, args) {
+		try {
+			if (!this.mcpClient) throw new Error("MCP client not available");
+			const parsedArgs = args == null || args === "" ? {} : typeof args === "string" ? JSON.parse(args) : args;
+			const result = await this.mcpClient.callTool(toolName, parsedArgs);
+			if (isMcpErrorResult(result)) return {
+				output: `MCP Tool Error (${toolName}): ${getMcpErrorMessage(result)}`,
+				isError: true
+			};
+			return {
+				output: `MCP Tool Result (${toolName}): ${normalizeMcpToolContent(result?.content)}`,
+				isError: false
+			};
+		} catch (error) {
+			const errorMessage = error instanceof Error ? error.message : String(error);
+			logger.debug(`MCP tool execution failed for ${toolName}: ${errorMessage}`);
+			return {
+				output: `MCP Tool Error (${toolName}): ${errorMessage}`,
+				isError: true
+			};
+		}
+	}
+	/**
+	* Sets the MCP client, preserving any loaded callbacks
+	*/
+	setMcpClient(client) {
+		this.mcpClient = client;
+		this.mcpToolNames = null;
+	}
+	/**
+	* Clears the cached callbacks
+	*/
+	clearCache() {
+		this.loadedCallbacks = {};
+	}
+};
+//#endregion
+export { executeProviderFunctionCallback as n, loadProviderCallbackFromFileUrl as r, FunctionCallbackHandler as t };
+
+//# sourceMappingURL=functionCallbackUtils-B_cYp7P5.js.map

package/dist/src/functionCallbackUtils-BfXnPo0o.js

@@ -0,0 +1,245 @@
+import { o as logger } from "./logger-Gff7RMNF.js";
+import { S as isCallerAbortError, h as throwIfAborted, y as waitForPromiseWithAbort } from "./shared-BBI-NXjW.js";
+import { g as withGenAIToolSpan } from "./genaiTracer-CuSjHi4t.js";
+import { h as loadCallbackFromFileUrl, m as CallbackPathTraversalError, y as wrapError } from "./file-Di0ypTgN.js";
+import "./tracing-CfUes1dH.js";
+import { c as normalizeMcpToolContent, i as getMcpErrorMessage, o as isMcpErrorResult } from "./util-DMNFhk47.js";
+//#region src/providers/functionCallbackUtils.ts
+/**
+* Resolve a `file://` callback reference through the shared path-traversal guard,
+* preserving the underlying error on `cause` so callers can classify it.
+*/
+async function loadProviderCallbackFromFileUrl(fileRef, logPrefix) {
+	try {
+		return await loadCallbackFromFileUrl(fileRef, logPrefix ? { logPrefix } : void 0);
+	} catch (error) {
+		if (error instanceof CallbackPathTraversalError) throw error;
+		throw wrapError(`Error loading function from ${fileRef}: ${error.message}`, error);
+	}
+}
+/**
+* Load, cache, and invoke one `functionToolCallbacks` entry, returning the string a
+* tool-result message expects.
+*
+* Shared by the two providers that implement function-tool callbacks directly on the
+* provider class — Bedrock Converse and OpenAI Chat — whose copies were identical apart
+* from log prefixes. Keeping one implementation is what stops them drifting: the
+* path-traversal guard is a worked example of a fix that reached one copy of this logic
+* and not the others.
+*
+* The remaining providers with a `loadedFunctionCallbacks` cache (Azure Foundry, Google
+* base and live) have genuinely different loading and caching behaviour — Azure preloads,
+* Google Live gates on a shared-cache flag — so they deliberately keep their own.
+*/
+async function executeProviderFunctionCallback({ functionName, args, callId, callbacks, cache, logPrefix, abortSignal }) {
+	const prefix = logPrefix ? `${logPrefix} ` : "";
+	try {
+		throwIfAborted(abortSignal);
+		let callback = cache[functionName];
+		if (!callback) {
+			const callbackRef = callbacks?.[functionName];
+			if (callbackRef && typeof callbackRef === "string") {
+				if (callbackRef.startsWith("file://")) {
+					const loading = loadProviderCallbackFromFileUrl(callbackRef, logPrefix).then((loadedCallback) => {
+						cache[functionName] = loadedCallback;
+						return loadedCallback;
+					});
+					callback = await waitForPromiseWithAbort(loading, abortSignal);
+				} else {
+					callback = new Function("return " + callbackRef)();
+					cache[functionName] = callback;
+				}
+			} else if (typeof callbackRef === "function") {
+				callback = callbackRef;
+				cache[functionName] = callback;
+			}
+		}
+		throwIfAborted(abortSignal);
+		if (!callback) throw new Error(`No callback found for function '${functionName}'`);
+		logger.debug(`${prefix}Executing function '${functionName}' with args: ${args}`);
+		const result = await withGenAIToolSpan({
+			name: functionName,
+			arguments: args,
+			callId
+		}, () => {
+			throwIfAborted(abortSignal);
+			return waitForPromiseWithAbort(Promise.resolve(callback(args)), abortSignal);
+		});
+		throwIfAborted(abortSignal);
+		if (result === void 0 || result === null) return "";
+		if (typeof result === "object") try {
+			return JSON.stringify(result);
+		} catch (error) {
+			logger.warn(`Error stringifying result from function '${functionName}': ${error}`);
+			return String(result);
+		}
+		return String(result);
+	} catch (error) {
+		if (isCallerAbortError(error, abortSignal, { requireReasonMatch: true })) throwIfAborted(abortSignal);
+		logger.error(`${prefix}Error executing function '${functionName}': ${error.message || String(error)}`);
+		throw error;
+	}
+}
+/**
+* Handles function callback execution for AI providers.
+* Provides a unified way to execute function callbacks across different provider formats.
+*/
+var FunctionCallbackHandler = class {
+	mcpClient;
+	loadedCallbacks = {};
+	mcpToolNames = null;
+	constructor(mcpClient) {
+		this.mcpClient = mcpClient;
+	}
+	/**
+	* Processes a function call by executing its callback or returning the original call
+	* @param call The function call to process (can be various formats)
+	* @param callbacks Configuration mapping function names to callbacks
+	* @param context Optional context to pass to the callback
+	* @returns The result of processing
+	*/
+	async processCall(call, callbacks, context) {
+		const functionInfo = this.extractFunctionInfo(call);
+		if (this.mcpClient && functionInfo) {
+			if (this.mcpToolNames === null) {
+				const mcpTools = this.mcpClient.getAllTools();
+				this.mcpToolNames = new Set(mcpTools.map((tool) => tool.name));
+			}
+			if (this.mcpToolNames.has(functionInfo.name)) return await this.executeMcpTool(functionInfo.name, functionInfo.arguments);
+		}
+		if (!functionInfo || !callbacks || !callbacks[functionInfo.name]) return {
+			output: typeof call === "string" ? call : JSON.stringify(call),
+			isError: false
+		};
+		try {
+			return {
+				output: await this.executeCallback(functionInfo.name, functionInfo.arguments || "{}", callbacks, context, typeof call?.call_id === "string" ? call.call_id : typeof call?.id === "string" ? call.id : void 0),
+				isError: false
+			};
+		} catch (error) {
+			const cause = error?.cause;
+			if (error instanceof CallbackPathTraversalError || cause instanceof CallbackPathTraversalError) logger.warn(`[FunctionCallback] Rejected callback '${functionInfo.name}': ${error.message}`);
+			else logger.debug(`Function callback failed for ${functionInfo.name}: ${error}`);
+			return {
+				output: typeof call === "string" ? call : JSON.stringify(call),
+				isError: true
+			};
+		}
+	}
+	/**
+	* Processes multiple function calls
+	* @param calls Array of calls or a single call
+	* @param callbacks Configuration mapping function names to callbacks
+	* @param context Optional context to pass to callbacks
+	* @param options Processing options
+	* @returns Processed output in appropriate format
+	*/
+	async processCalls(calls, callbacks, context, _options) {
+		if (!calls) return calls;
+		const isArray = Array.isArray(calls);
+		const callsArray = isArray ? calls : [calls];
+		const results = await Promise.all(callsArray.map((call) => this.processCall(call, callbacks, context)));
+		if (results.some((r, index) => !r.isError && r.output !== JSON.stringify(callsArray[index]))) {
+			const outputs = results.map((r) => r.output);
+			if (!isArray && outputs.length === 1) return outputs[0];
+			return outputs.every((o) => typeof o === "string") ? outputs.join("\n") : outputs;
+		}
+		if (!isArray && results.length === 1) return results[0].output;
+		return calls;
+	}
+	/**
+	* Extracts function name and arguments from various call formats
+	*/
+	extractFunctionInfo(call) {
+		if (!call || typeof call !== "object") return null;
+		if (call.name && typeof call.name === "string") return {
+			name: call.name,
+			arguments: call.arguments
+		};
+		if (call.type === "function" && call.function?.name) return {
+			name: call.function.name,
+			arguments: call.function.arguments
+		};
+		return null;
+	}
+	/**
+	* Executes a function callback
+	*/
+	async executeCallback(functionName, args, callbacks, context, callId) {
+		return await withGenAIToolSpan({
+			name: functionName,
+			arguments: args,
+			callId
+		}, async () => {
+			let callback = this.loadedCallbacks[functionName];
+			if (!callback) {
+				const callbackConfig = callbacks[functionName];
+				if (typeof callbackConfig === "string") {
+					if (callbackConfig.startsWith("file://")) callback = await this.loadExternalFunction(callbackConfig);
+					else callback = new Function("return " + callbackConfig)();
+				} else if (typeof callbackConfig === "function") callback = callbackConfig;
+				else throw new Error(`Invalid callback configuration for ${functionName}`);
+				this.loadedCallbacks[functionName] = callback;
+			}
+			const result = await callback(args, context);
+			return typeof result === "string" ? result : JSON.stringify(result);
+		});
+	}
+	/**
+	* Loads a function from an external file.
+	*
+	* Uses `lenient: true` so a missing named export silently falls back to the
+	* module's default export, preserving long-standing behavior. The six
+	* provider-specific loaders use strict mode and throw on missing named
+	* exports — see {@link loadCallbackFromFileUrl}.
+	*/
+	async loadExternalFunction(fileRef) {
+		try {
+			return await loadCallbackFromFileUrl(fileRef, { lenient: true });
+		} catch (error) {
+			throw wrapError(`Failed to load function from ${fileRef}: ${error.message}`, error);
+		}
+	}
+	/**
+	* Executes an MCP tool
+	*/
+	async executeMcpTool(toolName, args) {
+		try {
+			if (!this.mcpClient) throw new Error("MCP client not available");
+			const parsedArgs = args == null || args === "" ? {} : typeof args === "string" ? JSON.parse(args) : args;
+			const result = await this.mcpClient.callTool(toolName, parsedArgs);
+			if (isMcpErrorResult(result)) return {
+				output: `MCP Tool Error (${toolName}): ${getMcpErrorMessage(result)}`,
+				isError: true
+			};
+			return {
+				output: `MCP Tool Result (${toolName}): ${normalizeMcpToolContent(result?.content)}`,
+				isError: false
+			};
+		} catch (error) {
+			const errorMessage = error instanceof Error ? error.message : String(error);
+			logger.debug(`MCP tool execution failed for ${toolName}: ${errorMessage}`);
+			return {
+				output: `MCP Tool Error (${toolName}): ${errorMessage}`,
+				isError: true
+			};
+		}
+	}
+	/**
+	* Sets the MCP client, preserving any loaded callbacks
+	*/
+	setMcpClient(client) {
+		this.mcpClient = client;
+		this.mcpToolNames = null;
+	}
+	/**
+	* Clears the cached callbacks
+	*/
+	clearCache() {
+		this.loadedCallbacks = {};
+	}
+};
+//#endregion
+export { executeProviderFunctionCallback as n, loadProviderCallbackFromFileUrl as r, FunctionCallbackHandler as t };
+
+//# sourceMappingURL=functionCallbackUtils-BfXnPo0o.js.map