promptfoo 0.120.26 → 0.121.1

package/dist/src/{responses-zOtKtnY_.cjs → responses-tD4Bd4dc.cjs}

@@ -1,14 +1,13 @@
-const require_logger = require('./logger-BdZ-IqBc.cjs');
-const require_esm = require('./esm-BQkx5roy.cjs');
-const require_fileExtensions = require('./fileExtensions-DkJYkWUy.cjs');
-const require_util = require('./util-DUYOvxAy.cjs');
-const require_fetch = require('./fetch-ZMn_oemb.cjs');
-const require_cache = require('./cache-7xULbvt3.cjs');
-const require_openai = require('./openai-Bigwjgo1.cjs');
-const require_util$1 = require('./util-C08Kns6-.cjs');
+const require_logger = require("./logger-Cp1GPUjj.cjs");
+const require_esm = require("./esm-CnNt7sI4.cjs");
+const require_fileExtensions = require("./fileExtensions-bYh77CN8.cjs");
+const require_util = require("./util-Yz-1aEhW.cjs");
+const require_fetch = require("./fetch-BxUk8odA.cjs");
+const require_cache = require("./cache-COish3-W.cjs");
+const require_openai = require("./openai-Cuif0GEt.cjs");
+const require_util$1 = require("./util-Db0a0AFH.cjs");
 let path = require("path");
 path = require_logger.__toESM(path);
-
 //#region src/providers/functionCallbackUtils.ts
 /**
 * Handles function callback execution for AI providers.
@@ -46,7 +45,7 @@ var FunctionCallbackHandler = class {
 				isError: false
 			};
 		} catch (error) {
-			require_logger.logger_default.debug(`Function callback failed for ${functionInfo.name}: ${error}`);
+			require_logger.logger.debug(`Function callback failed for ${functionInfo.name}: ${error}`);
 			return {
 				output: typeof call === "string" ? call : JSON.stringify(call),
 				isError: true
@@ -116,8 +115,8 @@ var FunctionCallbackHandler = class {
 			if (splits[0] && require_fileExtensions.isJavascriptFile(splits[0])) [filePath, functionName] = splits;
 		}
 		try {
-			const resolvedPath = path.default.resolve(require_logger.cliState_default.basePath || "", filePath);
-			require_logger.logger_default.debug(`Loading function from ${resolvedPath}${functionName ? `:${functionName}` : ""}`);
+			const resolvedPath = path.default.resolve(require_logger.state.basePath || "", filePath);
+			require_logger.logger.debug(`Loading function from ${resolvedPath}${functionName ? `:${functionName}` : ""}`);
 			const mod = await require_esm.importModule(resolvedPath);
 			const func = functionName && mod[functionName] ? mod[functionName] : mod.default || mod;
 			if (typeof func !== "function") throw new Error(`Expected ${resolvedPath}${functionName ? `:${functionName}` : ""} to export a function, got ${typeof func}`);
@@ -159,7 +158,7 @@ var FunctionCallbackHandler = class {
 			};
 		} catch (error) {
 			const errorMessage = error instanceof Error ? error.message : String(error);
-			require_logger.logger_default.debug(`MCP tool execution failed for ${toolName}: ${errorMessage}`);
+			require_logger.logger.debug(`MCP tool execution failed for ${toolName}: ${errorMessage}`);
 			return {
 				output: `MCP Tool Error (${toolName}): ${errorMessage}`,
 				isError: true
@@ -180,7 +179,6 @@ var FunctionCallbackHandler = class {
 		this.loadedCallbacks = {};
 	}
 };
-
 //#endregion
 //#region src/providers/responses/processor.ts
 /**
@@ -232,7 +230,7 @@ var ResponsesProcessor = class {
 		this.config = config;
 	}
 	async processResponseOutput(data, requestConfig, cached) {
-		require_logger.logger_default.debug(`Processing ${this.config.providerType} responses output`, {
+		require_logger.logger.debug(`Processing ${this.config.providerType} responses output`, {
 			responseId: data.id,
 			model: data.model
 		});
@@ -257,7 +255,7 @@ var ResponsesProcessor = class {
 			if (requestConfig.response_format?.type === "json_schema" && typeof finalOutput === "string") try {
 				finalOutput = JSON.parse(finalOutput);
 			} catch (error) {
-				require_logger.logger_default.error(`Failed to parse JSON output: ${error}`);
+				require_logger.logger.error(`Failed to parse JSON output: ${error}`);
 			}
 			const result = {
 				output: finalOutput,
@@ -277,7 +275,7 @@ var ResponsesProcessor = class {
 		}
 	}
 	async processOutput(output, context) {
-		if (this.config.modelName.includes("deep-research")) require_logger.logger_default.debug(`Deep research response structure: ${JSON.stringify(context.data, null, 2)}`);
+		if (this.config.modelName.includes("deep-research")) require_logger.logger.debug(`Deep research response structure: ${JSON.stringify(context.data, null, 2)}`);
 		if (!output || !Array.isArray(output) || output.length === 0) throw new Error("Invalid response format: Missing output array");
 		let result = "";
 		let refusal = "";
@@ -285,7 +283,7 @@ var ResponsesProcessor = class {
 		const annotations = [];
 		for (const item of output) {
 			if (!item || typeof item !== "object") {
-				require_logger.logger_default.warn(`Skipping invalid output item: ${JSON.stringify(item)}`);
+				require_logger.logger.warn(`Skipping invalid output item: ${JSON.stringify(item)}`);
 				continue;
 			}
 			const processed = await this.processOutputItem(item, context);
@@ -315,7 +313,7 @@ var ResponsesProcessor = class {
 			case "mcp_call": return this.processMcpCall(item);
 			case "mcp_approval_request": return this.processMcpApprovalRequest(item);
 			default:
-				require_logger.logger_default.debug(`Unknown output item type: ${item.type}`);
+				require_logger.logger.debug(`Unknown output item type: ${item.type}`);
 				return {};
 		}
 	}
@@ -338,7 +336,7 @@ var ResponsesProcessor = class {
 		const annotations = [];
 		if (item.content) for (const contentItem of item.content) {
 			if (!contentItem || typeof contentItem !== "object") {
-				require_logger.logger_default.warn(`Skipping invalid content item: ${JSON.stringify(contentItem)}`);
+				require_logger.logger.warn(`Skipping invalid content item: ${JSON.stringify(contentItem)}`);
 				continue;
 			}
 			if (contentItem.type === "output_text") {
@@ -399,7 +397,6 @@ var ResponsesProcessor = class {
 		return Promise.resolve({ content });
 	}
 };
-
 //#endregion
 //#region src/providers/openai/responses.ts
 var OpenAiResponsesProvider = class extends require_openai.OpenAiGenericProvider {
@@ -438,8 +435,17 @@ var OpenAiResponsesProvider = class extends require_openai.OpenAiGenericProvider
 		"gpt-5.1-chat-latest",
 		"gpt-5.2",
 		"gpt-5.2-2025-12-11",
+		"gpt-5.2-chat-latest",
+		"gpt-5.2-codex",
+		"gpt-5.2-pro",
+		"gpt-5.2-pro-2025-12-11",
+		"gpt-5.3-chat-latest",
 		"gpt-5.3-codex",
 		"gpt-5.3-codex-spark",
+		"gpt-5.4",
+		"gpt-5.4-2026-03-05",
+		"gpt-5.4-pro",
+		"gpt-5.4-pro-2026-03-05",
 		"gpt-audio",
 		"gpt-audio-2025-08-28",
 		"gpt-audio-mini",
@@ -532,7 +538,7 @@ var OpenAiResponsesProvider = class extends require_openai.OpenAiGenericProvider
 			...reasoningEffort ? { reasoning: { effort: reasoningEffort } } : {},
 			...temperature !== void 0 ? { temperature } : {},
 			...instructions ? { instructions } : {},
-			...config.top_p !== void 0 || require_logger.getEnvString("OPENAI_TOP_P") ? { top_p: config.top_p ?? require_logger.getEnvFloat("OPENAI_TOP_P", 1) } : {},
+			...(!reasoningEffort || reasoningEffort === "none") && (config.top_p !== void 0 || require_logger.getEnvString("OPENAI_TOP_P")) ? { top_p: config.top_p ?? require_logger.getEnvFloat("OPENAI_TOP_P", 1) } : {},
 			...loadedTools ? { tools: loadedTools } : {},
 			...config.tool_choice ? { tool_choice: config.tool_choice } : {},
 			...config.max_tool_calls ? { max_tool_calls: config.max_tool_calls } : {},
@@ -568,10 +574,11 @@ var OpenAiResponsesProvider = class extends require_openai.OpenAiGenericProvider
 			for (const mcpTool of mcpTools) if (mcpTool.require_approval !== "never") return { error: `Deep research model ${this.modelName} requires MCP tools to have require_approval: 'never'. Update your MCP tool configuration:\ntools:\n  - type: mcp\n    require_approval: never` };
 		}
 		let timeout = require_fetch.REQUEST_TIMEOUT_MS;
-		if (isDeepResearchModel || this.modelName.includes("gpt-5-pro")) {
+		const isGpt5ProModel = /(^|\/)gpt-5(?:\.\d+)?-pro(?:-|$)/.test(this.modelName);
+		if (isDeepResearchModel || isGpt5ProModel) {
 			const evalTimeout = require_logger.getEnvInt("PROMPTFOO_EVAL_TIMEOUT_MS", 0);
 			timeout = evalTimeout > 0 ? evalTimeout : require_fetch.LONG_RUNNING_MODEL_TIMEOUT_MS;
-			require_logger.logger_default.debug(`Using timeout of ${timeout}ms for long-running model ${this.modelName}`);
+			require_logger.logger.debug(`Using timeout of ${timeout}ms for long-running model ${this.modelName}`);
 		}
 		let data;
 		let status;
@@ -612,7 +619,7 @@ var OpenAiResponsesProvider = class extends require_openai.OpenAiGenericProvider
 				};
 			}
 		} catch (err) {
-			require_logger.logger_default.error(`API call error: ${String(err)}`);
+			require_logger.logger.error(`API call error: ${String(err)}`);
 			await deleteFromCache?.();
 			return {
 				error: `API call error: ${String(err)}`,
@@ -648,24 +655,24 @@ var OpenAiResponsesProvider = class extends require_openai.OpenAiGenericProvider
 		};
 	}
 };
-
 //#endregion
-Object.defineProperty(exports, 'FunctionCallbackHandler', {
-  enumerable: true,
-  get: function () {
-    return FunctionCallbackHandler;
-  }
+Object.defineProperty(exports, "FunctionCallbackHandler", {
+	enumerable: true,
+	get: function() {
+		return FunctionCallbackHandler;
+	}
 });
-Object.defineProperty(exports, 'OpenAiResponsesProvider', {
-  enumerable: true,
-  get: function () {
-    return OpenAiResponsesProvider;
-  }
+Object.defineProperty(exports, "OpenAiResponsesProvider", {
+	enumerable: true,
+	get: function() {
+		return OpenAiResponsesProvider;
+	}
 });
-Object.defineProperty(exports, 'ResponsesProcessor', {
-  enumerable: true,
-  get: function () {
-    return ResponsesProcessor;
-  }
+Object.defineProperty(exports, "ResponsesProcessor", {
+	enumerable: true,
+	get: function() {
+		return ResponsesProcessor;
+	}
 });
-//# sourceMappingURL=responses-zOtKtnY_.cjs.map
+
+//# sourceMappingURL=responses-tD4Bd4dc.cjs.map

package/dist/src/rubyUtils-BUHu6PhO.js

@@ -0,0 +1,5 @@
+#!/usr/bin/env node
+import "./logger-KkObSCzq.js";
+import "./esm-C03C-mv3.js";
+import { n as runRuby } from "./rubyUtils-Boc4HZzX.js";
+export { runRuby };

package/dist/src/{rubyUtils-Cs35SDYa.js → rubyUtils-BUVePouc.js}

@@ -1,12 +1,19 @@
-import { b as getEnvString, i as logger_default, m as safeJsonStringify } from "./logger-w8Ozp0Td.js";
-import { n as getWrapperDir } from "./esm-rDtG_2rg.js";
+import { t as __exportAll } from "./chunk-DEq-mXcV.js";
+import { b as getEnvString, i as logger, m as safeJsonStringify } from "./logger-CT3IKMKA.js";
+import { n as getWrapperDir } from "./esm-Cd1AjG1D.js";
 import fs from "fs";
 import path from "path";
 import os from "os";
 import { execFile } from "child_process";
 import { promisify } from "util";
-
 //#region src/ruby/rubyUtils.ts
+var rubyUtils_exports = /* @__PURE__ */ __exportAll({
+	getSysExecutable: () => getSysExecutable,
+	runRuby: () => runRuby,
+	state: () => state,
+	tryPath: () => tryPath,
+	validateRubyPath: () => validateRubyPath
+});
 const execFileAsync = promisify(execFile);
 /**
 * Global state for Ruby executable path caching.
@@ -26,7 +33,7 @@ async function tryWindowsWhere() {
 	try {
 		const output = (await execFileAsync("where", ["ruby"])).stdout.trim();
 		if (!output) {
-			logger_default.debug("Windows 'where ruby' returned empty output");
+			logger.debug("Windows 'where ruby' returned empty output");
 			return null;
 		}
 		const paths = output.split("\n").filter((path) => path.trim());
@@ -38,8 +45,8 @@ async function tryWindowsWhere() {
 		}
 	} catch (error) {
 		const errorMsg = error instanceof Error ? error.message : String(error);
-		logger_default.debug(`Windows 'where ruby' failed: ${errorMsg}`);
-		if (errorMsg.includes("Access is denied") || errorMsg.includes("EACCES")) logger_default.warn(`Permission denied when searching for Ruby: ${errorMsg}`);
+		logger.debug(`Windows 'where ruby' failed: ${errorMsg}`);
+		if (errorMsg.includes("Access is denied") || errorMsg.includes("EACCES")) logger.warn(`Permission denied when searching for Ruby: ${errorMsg}`);
 	}
 	return null;
 }
@@ -55,8 +62,8 @@ async function tryRubyCommands(commands) {
 		if (executablePath && executablePath !== "None") return executablePath;
 	} catch (error) {
 		const errorMsg = error instanceof Error ? error.message : String(error);
-		logger_default.debug(`Ruby command "${cmd}" failed: ${errorMsg}`);
-		if (errorMsg.includes("Access is denied") || errorMsg.includes("EACCES") || errorMsg.includes("EPERM")) logger_default.warn(`Permission denied when trying Ruby command "${cmd}": ${errorMsg}`);
+		logger.debug(`Ruby command "${cmd}" failed: ${errorMsg}`);
+		if (errorMsg.includes("Access is denied") || errorMsg.includes("EACCES") || errorMsg.includes("EPERM")) logger.warn(`Permission denied when trying Ruby command "${cmd}": ${errorMsg}`);
 	}
 	return null;
 }
@@ -72,8 +79,8 @@ async function tryDirectCommands(commands) {
 		if (validated) return validated;
 	} catch (error) {
 		const errorMsg = error instanceof Error ? error.message : String(error);
-		logger_default.debug(`Direct command "${cmd}" failed: ${errorMsg}`);
-		if (errorMsg.includes("Access is denied") || errorMsg.includes("EACCES") || errorMsg.includes("EPERM")) logger_default.warn(`Permission denied when trying Ruby command "${cmd}": ${errorMsg}`);
+		logger.debug(`Direct command "${cmd}" failed: ${errorMsg}`);
+		if (errorMsg.includes("Access is denied") || errorMsg.includes("EACCES") || errorMsg.includes("EPERM")) logger.warn(`Permission denied when trying Ruby command "${cmd}": ${errorMsg}`);
 	}
 	return null;
 }
@@ -175,7 +182,7 @@ async function runRuby(scriptPath, method, args, options = {}) {
 	const wrapperPath = path.join(getWrapperDir("ruby"), "wrapper.rb");
 	try {
 		fs.writeFileSync(tempJsonPath, safeJsonStringify(args), "utf-8");
-		logger_default.debug(`Running Ruby wrapper with args: ${safeJsonStringify(args)}`);
+		logger.debug(`Running Ruby wrapper with args: ${safeJsonStringify(args)}`);
 		const { stdout, stderr } = await execFileAsync(rubyPath, [
 			wrapperPath,
 			absPath,
@@ -183,33 +190,33 @@ async function runRuby(scriptPath, method, args, options = {}) {
 			tempJsonPath,
 			outputPath
 		]);
-		if (stdout) logger_default.debug(stdout.trim());
-		if (stderr) logger_default.error(stderr.trim());
+		if (stdout) logger.debug(stdout.trim());
+		if (stderr) logger.error(stderr.trim());
 		const output = fs.readFileSync(outputPath, "utf-8");
-		logger_default.debug(`Ruby script ${absPath} returned: ${output}`);
+		logger.debug(`Ruby script ${absPath} returned: ${output}`);
 		let result;
 		try {
 			result = JSON.parse(output);
-			logger_default.debug(`Ruby script ${absPath} parsed output type: ${typeof result}, structure: ${result ? JSON.stringify(Object.keys(result)) : "undefined"}`);
+			logger.debug(`Ruby script ${absPath} parsed output type: ${typeof result}, structure: ${result ? JSON.stringify(Object.keys(result)) : "undefined"}`);
 		} catch (error) {
 			throw new Error(`Invalid JSON: ${error.message} when parsing result: ${output}\nStack Trace: ${error.stack}`);
 		}
 		if (result?.type !== "final_result") throw new Error("The Ruby script `call_api` function must return a hash with an `output`");
 		return result.data;
 	} catch (error) {
-		logger_default.error(`Error running Ruby script: ${error.message}\nStack Trace: ${error.stack || "No Ruby traceback available"}`);
+		logger.error(`Error running Ruby script: ${error.message}\nStack Trace: ${error.stack || "No Ruby traceback available"}`);
 		throw new Error(`Error running Ruby script: ${error.message}\nStack Trace: ${error.stack || "No Ruby traceback available"}`);
 	} finally {
 		[tempJsonPath, outputPath].forEach((file) => {
 			try {
 				fs.unlinkSync(file);
 			} catch (error) {
-				logger_default.error(`Error removing ${file}: ${error}`);
+				logger.error(`Error removing ${file}: ${error}`);
 			}
 		});
 	}
 }
-
 //#endregion
-export { validateRubyPath as a, tryPath as i, runRuby as n, state as r, getSysExecutable as t };
-//# sourceMappingURL=rubyUtils-Cs35SDYa.js.map
+export { runRuby as n, rubyUtils_exports as t };
+
+//# sourceMappingURL=rubyUtils-BUVePouc.js.map

package/dist/src/rubyUtils-BcuGX77l.js

@@ -0,0 +1,222 @@
+import { t as __exportAll } from "./chunk-DEq-mXcV.js";
+import { b as getEnvString, i as logger, m as safeJsonStringify } from "./logger-DLcq4dWf.js";
+import { n as getWrapperDir } from "./esm-CaIwzWR5.js";
+import fs from "fs";
+import path from "path";
+import os from "os";
+import { promisify } from "util";
+import { execFile } from "child_process";
+//#region src/ruby/rubyUtils.ts
+var rubyUtils_exports = /* @__PURE__ */ __exportAll({
+	getSysExecutable: () => getSysExecutable,
+	runRuby: () => runRuby,
+	state: () => state,
+	tryPath: () => tryPath,
+	validateRubyPath: () => validateRubyPath
+});
+const execFileAsync = promisify(execFile);
+/**
+* Global state for Ruby executable path caching.
+* Ensures consistent Ruby executable usage across multiple provider instances.
+*/
+const state = {
+	cachedRubyPath: null,
+	validationPromise: null,
+	validatingPath: null
+};
+/**
+* Attempts to find Ruby using Windows 'where' command.
+* Only applicable on Windows platforms.
+* @returns The validated Ruby executable path, or null if not found
+*/
+async function tryWindowsWhere() {
+	try {
+		const output = (await execFileAsync("where", ["ruby"])).stdout.trim();
+		if (!output) {
+			logger.debug("Windows 'where ruby' returned empty output");
+			return null;
+		}
+		const paths = output.split("\n").filter((path) => path.trim());
+		for (const rubyPath of paths) {
+			const trimmedPath = rubyPath.trim();
+			if (!trimmedPath.endsWith(".exe")) continue;
+			const validated = await tryPath(trimmedPath);
+			if (validated) return validated;
+		}
+	} catch (error) {
+		const errorMsg = error instanceof Error ? error.message : String(error);
+		logger.debug(`Windows 'where ruby' failed: ${errorMsg}`);
+		if (errorMsg.includes("Access is denied") || errorMsg.includes("EACCES")) logger.warn(`Permission denied when searching for Ruby: ${errorMsg}`);
+	}
+	return null;
+}
+/**
+* Attempts to get Ruby executable path by running Ruby commands.
+* Uses RbConfig.ruby to get the actual Ruby executable path.
+* @param commands - Array of Ruby command names to try (e.g., ['ruby'])
+* @returns The Ruby executable path, or null if all commands fail
+*/
+async function tryRubyCommands(commands) {
+	for (const cmd of commands) try {
+		const executablePath = (await execFileAsync(cmd, ["-e", "puts RbConfig.ruby"])).stdout.trim();
+		if (executablePath && executablePath !== "None") return executablePath;
+	} catch (error) {
+		const errorMsg = error instanceof Error ? error.message : String(error);
+		logger.debug(`Ruby command "${cmd}" failed: ${errorMsg}`);
+		if (errorMsg.includes("Access is denied") || errorMsg.includes("EACCES") || errorMsg.includes("EPERM")) logger.warn(`Permission denied when trying Ruby command "${cmd}": ${errorMsg}`);
+	}
+	return null;
+}
+/**
+* Attempts to validate Ruby commands directly as a final fallback.
+* Validates each command by running it with --version.
+* @param commands - Array of Ruby command names to try (e.g., ['ruby'])
+* @returns The validated Ruby executable path, or null if all commands fail
+*/
+async function tryDirectCommands(commands) {
+	for (const cmd of commands) try {
+		const validated = await tryPath(cmd);
+		if (validated) return validated;
+	} catch (error) {
+		const errorMsg = error instanceof Error ? error.message : String(error);
+		logger.debug(`Direct command "${cmd}" failed: ${errorMsg}`);
+		if (errorMsg.includes("Access is denied") || errorMsg.includes("EACCES") || errorMsg.includes("EPERM")) logger.warn(`Permission denied when trying Ruby command "${cmd}": ${errorMsg}`);
+	}
+	return null;
+}
+/**
+* Attempts to get the Ruby executable path using platform-appropriate strategies.
+* @returns The Ruby executable path if successful, or null if failed.
+*/
+async function getSysExecutable() {
+	if (process.platform === "win32") {
+		const whereResult = await tryWindowsWhere();
+		if (whereResult) return whereResult;
+		const sysResult = await tryRubyCommands(["ruby"]);
+		if (sysResult) return sysResult;
+		return await tryDirectCommands(["ruby"]);
+	} else return await tryRubyCommands(["ruby"]);
+}
+/**
+* Attempts to validate a Ruby executable path.
+* @param path - The path to the Ruby executable to test.
+* @returns The validated path if successful, or null if invalid.
+*/
+async function tryPath(path) {
+	let timeoutId;
+	try {
+		const timeoutPromise = new Promise((_, reject) => {
+			timeoutId = setTimeout(() => reject(/* @__PURE__ */ new Error("Command timed out")), 2500);
+		});
+		const result = await Promise.race([execFileAsync(path, ["--version"]), timeoutPromise]);
+		if (timeoutId) clearTimeout(timeoutId);
+		if (result.stdout.trim().toLowerCase().includes("ruby")) return path;
+		return null;
+	} catch {
+		if (timeoutId) clearTimeout(timeoutId);
+		return null;
+	}
+}
+/**
+* Validates and caches the Ruby executable path.
+*
+* @param rubyPath - Path to the Ruby executable.
+* @param isExplicit - If true, only tries the provided path.
+* @returns Validated Ruby executable path.
+* @throws {Error} If no valid Ruby executable is found.
+*/
+async function validateRubyPath(rubyPath, isExplicit) {
+	if (state.cachedRubyPath && state.validatingPath === rubyPath) return state.cachedRubyPath;
+	if (state.validatingPath !== rubyPath) {
+		state.cachedRubyPath = null;
+		state.validationPromise = null;
+		state.validatingPath = rubyPath;
+	}
+	if (!state.validationPromise) state.validationPromise = (async () => {
+		try {
+			const primaryPath = await tryPath(rubyPath);
+			if (primaryPath) {
+				state.cachedRubyPath = primaryPath;
+				state.validationPromise = null;
+				return primaryPath;
+			}
+			if (isExplicit) {
+				const error = /* @__PURE__ */ new Error(`Ruby not found. Tried "${rubyPath}" Please ensure Ruby is installed and set the PROMPTFOO_RUBY environment variable to your Ruby executable path (e.g., '${process.platform === "win32" ? "C:\\Ruby32\\bin\\ruby.exe" : "/usr/bin/ruby"}').`);
+				state.validationPromise = null;
+				throw error;
+			}
+			const detectedPath = await getSysExecutable();
+			if (detectedPath) {
+				state.cachedRubyPath = detectedPath;
+				state.validationPromise = null;
+				return detectedPath;
+			}
+			const error = /* @__PURE__ */ new Error(`Ruby not found. Tried "${rubyPath}", ruby executable detection, and fallback commands. Please ensure Ruby is installed and set the PROMPTFOO_RUBY environment variable to your Ruby executable path (e.g., '${process.platform === "win32" ? "C:\\Ruby32\\bin\\ruby.exe" : "/usr/bin/ruby"}').`);
+			state.validationPromise = null;
+			throw error;
+		} catch (error) {
+			state.validationPromise = null;
+			throw error;
+		}
+	})();
+	return state.validationPromise;
+}
+/**
+* Runs a Ruby script with the specified method and arguments.
+*
+* @param scriptPath - The path to the Ruby script to run.
+* @param method - The name of the method to call in the Ruby script.
+* @param args - An array of arguments to pass to the Ruby script.
+* @param options - Optional settings for running the Ruby script.
+* @param options.rubyExecutable - Optional path to the Ruby executable.
+* @returns A promise that resolves to the output of the Ruby script.
+* @throws An error if there's an issue running the Ruby script or parsing its output.
+*/
+async function runRuby(scriptPath, method, args, options = {}) {
+	const absPath = path.resolve(scriptPath);
+	const tempJsonPath = path.join(os.tmpdir(), `promptfoo-ruby-input-json-${Date.now()}-${Math.random().toString(16).slice(2)}.json`);
+	const outputPath = path.join(os.tmpdir(), `promptfoo-ruby-output-json-${Date.now()}-${Math.random().toString(16).slice(2)}.json`);
+	const customPath = options.rubyExecutable || getEnvString("PROMPTFOO_RUBY");
+	let rubyPath = customPath || "ruby";
+	rubyPath = await validateRubyPath(rubyPath, typeof customPath === "string");
+	const wrapperPath = path.join(getWrapperDir("ruby"), "wrapper.rb");
+	try {
+		fs.writeFileSync(tempJsonPath, safeJsonStringify(args), "utf-8");
+		logger.debug(`Running Ruby wrapper with args: ${safeJsonStringify(args)}`);
+		const { stdout, stderr } = await execFileAsync(rubyPath, [
+			wrapperPath,
+			absPath,
+			method,
+			tempJsonPath,
+			outputPath
+		]);
+		if (stdout) logger.debug(stdout.trim());
+		if (stderr) logger.error(stderr.trim());
+		const output = fs.readFileSync(outputPath, "utf-8");
+		logger.debug(`Ruby script ${absPath} returned: ${output}`);
+		let result;
+		try {
+			result = JSON.parse(output);
+			logger.debug(`Ruby script ${absPath} parsed output type: ${typeof result}, structure: ${result ? JSON.stringify(Object.keys(result)) : "undefined"}`);
+		} catch (error) {
+			throw new Error(`Invalid JSON: ${error.message} when parsing result: ${output}\nStack Trace: ${error.stack}`);
+		}
+		if (result?.type !== "final_result") throw new Error("The Ruby script `call_api` function must return a hash with an `output`");
+		return result.data;
+	} catch (error) {
+		logger.error(`Error running Ruby script: ${error.message}\nStack Trace: ${error.stack || "No Ruby traceback available"}`);
+		throw new Error(`Error running Ruby script: ${error.message}\nStack Trace: ${error.stack || "No Ruby traceback available"}`);
+	} finally {
+		[tempJsonPath, outputPath].forEach((file) => {
+			try {
+				fs.unlinkSync(file);
+			} catch (error) {
+				logger.error(`Error removing ${file}: ${error}`);
+			}
+		});
+	}
+}
+//#endregion
+export { runRuby as n, rubyUtils_exports as t };
+
+//# sourceMappingURL=rubyUtils-BcuGX77l.js.map

package/dist/src/{rubyUtils-BtjlqyXt.js → rubyUtils-Boc4HZzX.js}

@@ -1,12 +1,11 @@
 #!/usr/bin/env node
-import { C as getEnvString, g as safeJsonStringify, o as logger_default } from "./logger-BotXmWKW.js";
-import { n as getWrapperDir } from "./esm-CYhseqj4.js";
+import { C as getEnvString, g as safeJsonStringify, o as logger } from "./logger-KkObSCzq.js";
+import { n as getWrapperDir } from "./esm-C03C-mv3.js";
 import fs from "fs";
 import path from "path";
 import os from "os";
 import { promisify } from "util";
 import { execFile } from "child_process";
-
 //#region src/ruby/rubyUtils.ts
 const execFileAsync = promisify(execFile);
 /**
@@ -27,7 +26,7 @@ async function tryWindowsWhere() {
 	try {
 		const output = (await execFileAsync("where", ["ruby"])).stdout.trim();
 		if (!output) {
-			logger_default.debug("Windows 'where ruby' returned empty output");
+			logger.debug("Windows 'where ruby' returned empty output");
 			return null;
 		}
 		const paths = output.split("\n").filter((path) => path.trim());
@@ -39,8 +38,8 @@ async function tryWindowsWhere() {
 		}
 	} catch (error) {
 		const errorMsg = error instanceof Error ? error.message : String(error);
-		logger_default.debug(`Windows 'where ruby' failed: ${errorMsg}`);
-		if (errorMsg.includes("Access is denied") || errorMsg.includes("EACCES")) logger_default.warn(`Permission denied when searching for Ruby: ${errorMsg}`);
+		logger.debug(`Windows 'where ruby' failed: ${errorMsg}`);
+		if (errorMsg.includes("Access is denied") || errorMsg.includes("EACCES")) logger.warn(`Permission denied when searching for Ruby: ${errorMsg}`);
 	}
 	return null;
 }
@@ -56,8 +55,8 @@ async function tryRubyCommands(commands) {
 		if (executablePath && executablePath !== "None") return executablePath;
 	} catch (error) {
 		const errorMsg = error instanceof Error ? error.message : String(error);
-		logger_default.debug(`Ruby command "${cmd}" failed: ${errorMsg}`);
-		if (errorMsg.includes("Access is denied") || errorMsg.includes("EACCES") || errorMsg.includes("EPERM")) logger_default.warn(`Permission denied when trying Ruby command "${cmd}": ${errorMsg}`);
+		logger.debug(`Ruby command "${cmd}" failed: ${errorMsg}`);
+		if (errorMsg.includes("Access is denied") || errorMsg.includes("EACCES") || errorMsg.includes("EPERM")) logger.warn(`Permission denied when trying Ruby command "${cmd}": ${errorMsg}`);
 	}
 	return null;
 }
@@ -73,8 +72,8 @@ async function tryDirectCommands(commands) {
 		if (validated) return validated;
 	} catch (error) {
 		const errorMsg = error instanceof Error ? error.message : String(error);
-		logger_default.debug(`Direct command "${cmd}" failed: ${errorMsg}`);
-		if (errorMsg.includes("Access is denied") || errorMsg.includes("EACCES") || errorMsg.includes("EPERM")) logger_default.warn(`Permission denied when trying Ruby command "${cmd}": ${errorMsg}`);
+		logger.debug(`Direct command "${cmd}" failed: ${errorMsg}`);
+		if (errorMsg.includes("Access is denied") || errorMsg.includes("EACCES") || errorMsg.includes("EPERM")) logger.warn(`Permission denied when trying Ruby command "${cmd}": ${errorMsg}`);
 	}
 	return null;
 }
@@ -176,7 +175,7 @@ async function runRuby(scriptPath, method, args, options = {}) {
 	const wrapperPath = path.join(getWrapperDir("ruby"), "wrapper.rb");
 	try {
 		fs.writeFileSync(tempJsonPath, safeJsonStringify(args), "utf-8");
-		logger_default.debug(`Running Ruby wrapper with args: ${safeJsonStringify(args)}`);
+		logger.debug(`Running Ruby wrapper with args: ${safeJsonStringify(args)}`);
 		const { stdout, stderr } = await execFileAsync(rubyPath, [
 			wrapperPath,
 			absPath,
@@ -184,33 +183,33 @@ async function runRuby(scriptPath, method, args, options = {}) {
 			tempJsonPath,
 			outputPath
 		]);
-		if (stdout) logger_default.debug(stdout.trim());
-		if (stderr) logger_default.error(stderr.trim());
+		if (stdout) logger.debug(stdout.trim());
+		if (stderr) logger.error(stderr.trim());
 		const output = fs.readFileSync(outputPath, "utf-8");
-		logger_default.debug(`Ruby script ${absPath} returned: ${output}`);
+		logger.debug(`Ruby script ${absPath} returned: ${output}`);
 		let result;
 		try {
 			result = JSON.parse(output);
-			logger_default.debug(`Ruby script ${absPath} parsed output type: ${typeof result}, structure: ${result ? JSON.stringify(Object.keys(result)) : "undefined"}`);
+			logger.debug(`Ruby script ${absPath} parsed output type: ${typeof result}, structure: ${result ? JSON.stringify(Object.keys(result)) : "undefined"}`);
 		} catch (error) {
 			throw new Error(`Invalid JSON: ${error.message} when parsing result: ${output}\nStack Trace: ${error.stack}`);
 		}
 		if (result?.type !== "final_result") throw new Error("The Ruby script `call_api` function must return a hash with an `output`");
 		return result.data;
 	} catch (error) {
-		logger_default.error(`Error running Ruby script: ${error.message}\nStack Trace: ${error.stack || "No Ruby traceback available"}`);
+		logger.error(`Error running Ruby script: ${error.message}\nStack Trace: ${error.stack || "No Ruby traceback available"}`);
 		throw new Error(`Error running Ruby script: ${error.message}\nStack Trace: ${error.stack || "No Ruby traceback available"}`);
 	} finally {
 		[tempJsonPath, outputPath].forEach((file) => {
 			try {
 				fs.unlinkSync(file);
 			} catch (error) {
-				logger_default.error(`Error removing ${file}: ${error}`);
+				logger.error(`Error removing ${file}: ${error}`);
 			}
 		});
 	}
 }
-
 //#endregion
 export { validateRubyPath as a, tryPath as i, runRuby as n, state as r, getSysExecutable as t };
-//# sourceMappingURL=rubyUtils-BtjlqyXt.js.map
+
+//# sourceMappingURL=rubyUtils-Boc4HZzX.js.map

package/dist/src/rubyUtils-CP42kMvq.cjs

@@ -0,0 +1,4 @@
+require("./logger-Cp1GPUjj.cjs");
+require("./esm-CnNt7sI4.cjs");
+const require_rubyUtils = require("./rubyUtils-DhCAlxZr.cjs");
+exports.runRuby = require_rubyUtils.runRuby;