promptfoo 0.120.27 → 0.121.2

package/dist/src/pythonUtils-D6fwaDSg.js

@@ -0,0 +1,249 @@
+import { _ as getEnvBool, b as getEnvString, i as logger, m as safeJsonStringify } from "./logger-DLcq4dWf.js";
+import { n as getWrapperDir } from "./esm-CaIwzWR5.js";
+import fs from "fs";
+import path from "path";
+import os from "os";
+import { promisify } from "util";
+import { execFile } from "child_process";
+import { PythonShell } from "python-shell";
+//#region src/python/pythonUtils.ts
+const execFileAsync = promisify(execFile);
+/**
+* Gets an integer value from an environment variable.
+* @param key - The environment variable name
+* @returns The parsed integer value, or undefined if not set or not a valid integer
+*/
+function getEnvInt(key) {
+	const value = process.env[key];
+	if (value === void 0) return;
+	const parsed = parseInt(value, 10);
+	return isNaN(parsed) ? void 0 : parsed;
+}
+/**
+* Resolves the Python executable path from explicit config and environment.
+* This centralizes the fallback logic: configPath > PROMPTFOO_PYTHON env var.
+*
+* Note: Does NOT apply the final 'python' default - that's handled by
+* validatePythonPath. This preserves the distinction between "explicitly
+* configured" (should fail if invalid) and "using system default" (should
+* try fallback detection).
+*
+* @param configPath - Explicitly configured Python path from provider config
+* @returns The configured path, or undefined if neither config nor env var is set
+*/
+function getConfiguredPythonPath(configPath) {
+	if (configPath) return configPath;
+	return getEnvString("PROMPTFOO_PYTHON") || void 0;
+}
+const state = {
+	cachedPythonPath: null,
+	validationPromise: null
+};
+/**
+* Try to find Python using Windows 'where' command, filtering out Microsoft Store stubs.
+*/
+async function tryWindowsWhere() {
+	try {
+		const output = (await execFileAsync("where", ["python"])).stdout.trim();
+		if (!output) {
+			logger.debug("Windows 'where python' returned empty output");
+			return null;
+		}
+		const paths = output.split("\n").filter((path) => path.trim());
+		for (const pythonPath of paths) {
+			const trimmedPath = pythonPath.trim();
+			if (trimmedPath.includes("WindowsApps") || !trimmedPath.endsWith(".exe")) continue;
+			const validated = await tryPath(trimmedPath);
+			if (validated) return validated;
+		}
+	} catch (error) {
+		const errorMsg = error instanceof Error ? error.message : String(error);
+		logger.debug(`Windows 'where python' failed: ${errorMsg}`);
+		if (errorMsg.includes("Access is denied") || errorMsg.includes("EACCES")) logger.warn(`Permission denied when searching for Python: ${errorMsg}`);
+	}
+	return null;
+}
+/**
+* Try Python commands to get sys.executable path.
+*/
+async function tryPythonCommands(commands) {
+	for (const cmd of commands) try {
+		const executablePath = (await execFileAsync(cmd, ["-c", "import sys; print(sys.executable)"])).stdout.trim();
+		if (executablePath && executablePath !== "None") {
+			if (process.platform === "win32" && !executablePath.toLowerCase().endsWith(".exe")) {
+				if (executablePath.includes("\\") || /^[A-Za-z]:/.test(executablePath)) return executablePath + ".exe";
+			}
+			return executablePath;
+		}
+	} catch (error) {
+		const errorMsg = error instanceof Error ? error.message : String(error);
+		logger.debug(`Python command "${cmd}" failed: ${errorMsg}`);
+		if (errorMsg.includes("Access is denied") || errorMsg.includes("EACCES") || errorMsg.includes("EPERM")) logger.warn(`Permission denied when trying Python command "${cmd}": ${errorMsg}`);
+	}
+	return null;
+}
+/**
+* Try direct command validation as final fallback.
+*/
+async function tryDirectCommands(commands) {
+	for (const cmd of commands) try {
+		const validated = await tryPath(cmd);
+		if (validated) return validated;
+	} catch (error) {
+		const errorMsg = error instanceof Error ? error.message : String(error);
+		logger.debug(`Direct command "${cmd}" failed: ${errorMsg}`);
+		if (errorMsg.includes("Access is denied") || errorMsg.includes("EACCES") || errorMsg.includes("EPERM")) logger.warn(`Permission denied when trying Python command "${cmd}": ${errorMsg}`);
+	}
+	return null;
+}
+/**
+* Attempts to get the Python executable path using platform-appropriate strategies.
+* @returns The Python executable path if successful, or null if failed.
+*/
+async function getSysExecutable() {
+	if (process.platform === "win32") {
+		const whereResult = await tryWindowsWhere();
+		if (whereResult) return whereResult;
+		const sysResult = await tryPythonCommands(["py", "py -3"]);
+		if (sysResult) return sysResult;
+		return await tryDirectCommands(["python"]);
+	} else return await tryPythonCommands(["python3", "python"]);
+}
+/**
+* Attempts to validate a Python executable path.
+* @param path - The path to the Python executable to test.
+* @returns The validated path if successful, or null if invalid.
+*/
+async function tryPath(path) {
+	let timeoutId;
+	try {
+		const timeoutPromise = new Promise((_, reject) => {
+			timeoutId = setTimeout(() => reject(/* @__PURE__ */ new Error("Command timed out")), 2500);
+		});
+		const result = await Promise.race([execFileAsync(path, ["--version"]), timeoutPromise]);
+		if (timeoutId) clearTimeout(timeoutId);
+		if (result.stdout.trim().startsWith("Python")) return path;
+		return null;
+	} catch {
+		if (timeoutId) clearTimeout(timeoutId);
+		return null;
+	}
+}
+/**
+* Validates and caches the Python executable path.
+*
+* @param pythonPath - Path to the Python executable.
+* @param isExplicit - If true, only tries the provided path.
+* @returns Validated Python executable path.
+* @throws {Error} If no valid Python executable is found.
+*/
+async function validatePythonPath(pythonPath, isExplicit) {
+	if (state.cachedPythonPath) return state.cachedPythonPath;
+	if (!state.validationPromise) state.validationPromise = (async () => {
+		try {
+			const primaryPath = await tryPath(pythonPath);
+			if (primaryPath) {
+				state.cachedPythonPath = primaryPath;
+				state.validationPromise = null;
+				return primaryPath;
+			}
+			if (isExplicit) {
+				const error = /* @__PURE__ */ new Error(`Python 3 not found. Tried "${pythonPath}" Please ensure Python 3 is installed and set the PROMPTFOO_PYTHON environment variable to your Python 3 executable path (e.g., '${process.platform === "win32" ? "C:\\Python39\\python.exe" : "/usr/bin/python3"}').`);
+				state.validationPromise = null;
+				throw error;
+			}
+			const detectedPath = await getSysExecutable();
+			if (detectedPath) {
+				state.cachedPythonPath = detectedPath;
+				state.validationPromise = null;
+				return detectedPath;
+			}
+			const error = /* @__PURE__ */ new Error(`Python 3 not found. Tried "${pythonPath}", sys.executable detection, and fallback commands. Please ensure Python 3 is installed and set the PROMPTFOO_PYTHON environment variable to your Python 3 executable path (e.g., '${process.platform === "win32" ? "C:\\Python39\\python.exe" : "/usr/bin/python3"}').`);
+			state.validationPromise = null;
+			throw error;
+		} catch (error) {
+			state.validationPromise = null;
+			throw error;
+		}
+	})();
+	return state.validationPromise;
+}
+/**
+* Runs a Python script with the specified method and arguments.
+*
+* @param scriptPath - The path to the Python script to run.
+* @param method - The name of the method to call in the Python script.
+* @param args - An array of arguments to pass to the Python script.
+* @param options - Optional settings for running the Python script.
+* @param options.pythonExecutable - Optional path to the Python executable.
+* @returns A promise that resolves to the output of the Python script.
+* @throws An error if there's an issue running the Python script or parsing its output.
+*/
+async function runPython(scriptPath, method, args, options = {}) {
+	const absPath = path.resolve(scriptPath);
+	const tempJsonPath = path.join(os.tmpdir(), `promptfoo-python-input-json-${Date.now()}-${Math.random().toString(16).slice(2)}.json`);
+	const outputPath = path.join(os.tmpdir(), `promptfoo-python-output-json-${Date.now()}-${Math.random().toString(16).slice(2)}.json`);
+	const customPath = getConfiguredPythonPath(options.pythonExecutable);
+	let pythonPath = customPath || "python";
+	pythonPath = await validatePythonPath(pythonPath, typeof customPath === "string");
+	const pythonOptions = {
+		args: [
+			absPath,
+			method,
+			tempJsonPath,
+			outputPath
+		],
+		env: process.env,
+		mode: "binary",
+		pythonPath,
+		scriptPath: getWrapperDir("python"),
+		...getEnvBool("PROMPTFOO_PYTHON_DEBUG_ENABLED") && { stdio: "inherit" }
+	};
+	try {
+		fs.writeFileSync(tempJsonPath, safeJsonStringify(args), "utf-8");
+		logger.debug(`Running Python wrapper with args: ${safeJsonStringify(args)}`);
+		await new Promise((resolve, reject) => {
+			try {
+				const pyshell = new PythonShell("wrapper.py", pythonOptions);
+				pyshell.stdout?.on("data", (chunk) => {
+					logger.debug(chunk.toString("utf-8").trim());
+				});
+				pyshell.stderr?.on("data", (chunk) => {
+					logger.error(chunk.toString("utf-8").trim());
+				});
+				pyshell.end((err) => {
+					if (err) reject(err);
+					else resolve();
+				});
+			} catch (error) {
+				reject(error);
+			}
+		});
+		const output = fs.readFileSync(outputPath, "utf-8");
+		logger.debug(`Python script ${absPath} returned: ${output}`);
+		let result;
+		try {
+			result = JSON.parse(output);
+			logger.debug(`Python script ${absPath} parsed output type: ${typeof result}, structure: ${result ? JSON.stringify(Object.keys(result)) : "undefined"}`);
+		} catch (error) {
+			throw new Error(`Invalid JSON: ${error.message} when parsing result: ${output}\nStack Trace: ${error.stack}`);
+		}
+		if (result?.type !== "final_result") throw new Error("The Python script `call_api` function must return a dict with an `output`");
+		return result.data;
+	} catch (error) {
+		logger.error(`Error running Python script: ${error.message}\nStack Trace: ${error.stack?.replace("--- Python Traceback ---", "Python Traceback: ") || "No Python traceback available"}`);
+		throw new Error(`Error running Python script: ${error.message}\nStack Trace: ${error.stack?.replace("--- Python Traceback ---", "Python Traceback: ") || "No Python traceback available"}`);
+	} finally {
+		[tempJsonPath, outputPath].forEach((file) => {
+			try {
+				fs.unlinkSync(file);
+			} catch (error) {
+				logger.error(`Error removing ${file}: ${error}`);
+			}
+		});
+	}
+}
+//#endregion
+export { validatePythonPath as i, getEnvInt as n, runPython as r, getConfiguredPythonPath as t };
+
+//# sourceMappingURL=pythonUtils-D6fwaDSg.js.map

package/dist/src/{quiverai-BuI0tE39.js → quiverai-BbOUOn2L.js}

@@ -1,5 +1,6 @@
-import { A as fetchWithProxy, M as REQUEST_TIMEOUT_MS, P as logger_default, y as fetchWithCache, z as getEnvString } from "./server/index.js";
-
+import { b as getEnvString, i as logger } from "./logger-DLcq4dWf.js";
+import { p as REQUEST_TIMEOUT_MS, t as fetchWithProxy } from "./fetch-HaqdX7U1.js";
+import { r as fetchWithCache } from "./cache-i1P6crbO.js";
 //#region src/providers/quiverai.ts
 const QUIVERAI_API_BASE_URL = "https://api.quiver.ai/v1";
 const QUIVERAI_DEFAULT_MODEL = "arrow-preview";
@@ -61,7 +62,7 @@ var QuiverAiProvider = class {
 			if (useStream) return await this.callApiStreaming(body);
 			return await this.callApiNonStreaming(body, context);
 		} catch (error) {
-			logger_default.error(`QuiverAI API call error: ${error}`);
+			logger.error(`QuiverAI API call error: ${error}`);
 			return { error: `QuiverAI API call error: ${error}` };
 		}
 	}
@@ -98,7 +99,7 @@ var QuiverAiProvider = class {
 				});
 				if (lastResp.status === 429 && attempt < maxRetries) {
 					const waitMs = getRetryAfterMs(lastResp.headers, attempt);
-					logger_default.debug(`QuiverAI: rate limited, retry ${attempt + 1}/${maxRetries} in ${waitMs}ms`);
+					logger.debug(`QuiverAI: rate limited, retry ${attempt + 1}/${maxRetries} in ${waitMs}ms`);
 					await lastResp.body?.cancel();
 					await new Promise((resolve) => setTimeout(resolve, waitMs));
 					continue;
@@ -187,7 +188,7 @@ function parseSSELine(line) {
 			usage: event.usage
 		};
 	} catch {
-		logger_default.debug(`QuiverAI: failed to parse SSE data: ${payload}`);
+		logger.debug(`QuiverAI: failed to parse SSE data: ${payload}`);
 		return {};
 	}
 }
@@ -206,7 +207,7 @@ function createQuiverAiProvider(providerPath, providerOptions = {}, env) {
 		env: providerOptions.env ?? env
 	});
 }
-
 //#endregion
 export { createQuiverAiProvider };
-//# sourceMappingURL=quiverai-BuI0tE39.js.map
+
+//# sourceMappingURL=quiverai-BbOUOn2L.js.map

package/dist/src/{quiverai-DCGSZt4U.js → quiverai-CIaELU_m.js}

@@ -1,8 +1,6 @@
-import "./chunk-FhC4c-0y.js";
-import { b as getEnvString, i as logger_default } from "./logger-CG1uZPbQ.js";
-import { h as REQUEST_TIMEOUT_MS, t as fetchWithProxy } from "./fetch-D3OHf-lV.js";
-import { a as fetchWithCache } from "./cache-CIpsoBZR.js";
-
+import { b as getEnvString, i as logger } from "./logger-CT3IKMKA.js";
+import { p as REQUEST_TIMEOUT_MS, t as fetchWithProxy } from "./fetch-Cwxnd8zz.js";
+import { r as fetchWithCache } from "./cache-C4Nxf52C.js";
 //#region src/providers/quiverai.ts
 const QUIVERAI_API_BASE_URL = "https://api.quiver.ai/v1";
 const QUIVERAI_DEFAULT_MODEL = "arrow-preview";
@@ -64,7 +62,7 @@ var QuiverAiProvider = class {
 			if (useStream) return await this.callApiStreaming(body);
 			return await this.callApiNonStreaming(body, context);
 		} catch (error) {
-			logger_default.error(`QuiverAI API call error: ${error}`);
+			logger.error(`QuiverAI API call error: ${error}`);
 			return { error: `QuiverAI API call error: ${error}` };
 		}
 	}
@@ -101,7 +99,7 @@ var QuiverAiProvider = class {
 				});
 				if (lastResp.status === 429 && attempt < maxRetries) {
 					const waitMs = getRetryAfterMs(lastResp.headers, attempt);
-					logger_default.debug(`QuiverAI: rate limited, retry ${attempt + 1}/${maxRetries} in ${waitMs}ms`);
+					logger.debug(`QuiverAI: rate limited, retry ${attempt + 1}/${maxRetries} in ${waitMs}ms`);
 					await lastResp.body?.cancel();
 					await new Promise((resolve) => setTimeout(resolve, waitMs));
 					continue;
@@ -190,7 +188,7 @@ function parseSSELine(line) {
 			usage: event.usage
 		};
 	} catch {
-		logger_default.debug(`QuiverAI: failed to parse SSE data: ${payload}`);
+		logger.debug(`QuiverAI: failed to parse SSE data: ${payload}`);
 		return {};
 	}
 }
@@ -209,7 +207,7 @@ function createQuiverAiProvider(providerPath, providerOptions = {}, env) {
 		env: providerOptions.env ?? env
 	});
 }
-
 //#endregion
 export { createQuiverAiProvider };
-//# sourceMappingURL=quiverai-DCGSZt4U.js.map
+
+//# sourceMappingURL=quiverai-CIaELU_m.js.map

package/dist/src/{quiverai-DiMVJQDz.cjs → quiverai-PdShCPox.cjs}

@@ -1,7 +1,6 @@
-const require_logger = require('./logger-B7sBeGa0.cjs');
-const require_fetch = require('./fetch-BmbD-v1L.cjs');
-const require_cache = require('./cache-BTVYfbka.cjs');
-
+const require_logger = require("./logger-Cp1GPUjj.cjs");
+const require_fetch = require("./fetch-BPkYtG8K.cjs");
+const require_cache = require("./cache-Dh5WtQps.cjs");
 //#region src/providers/quiverai.ts
 const QUIVERAI_API_BASE_URL = "https://api.quiver.ai/v1";
 const QUIVERAI_DEFAULT_MODEL = "arrow-preview";
@@ -63,7 +62,7 @@ var QuiverAiProvider = class {
 			if (useStream) return await this.callApiStreaming(body);
 			return await this.callApiNonStreaming(body, context);
 		} catch (error) {
-			require_logger.logger_default.error(`QuiverAI API call error: ${error}`);
+			require_logger.logger.error(`QuiverAI API call error: ${error}`);
 			return { error: `QuiverAI API call error: ${error}` };
 		}
 	}
@@ -100,7 +99,7 @@ var QuiverAiProvider = class {
 				});
 				if (lastResp.status === 429 && attempt < maxRetries) {
 					const waitMs = getRetryAfterMs(lastResp.headers, attempt);
-					require_logger.logger_default.debug(`QuiverAI: rate limited, retry ${attempt + 1}/${maxRetries} in ${waitMs}ms`);
+					require_logger.logger.debug(`QuiverAI: rate limited, retry ${attempt + 1}/${maxRetries} in ${waitMs}ms`);
 					await lastResp.body?.cancel();
 					await new Promise((resolve) => setTimeout(resolve, waitMs));
 					continue;
@@ -189,7 +188,7 @@ function parseSSELine(line) {
 			usage: event.usage
 		};
 	} catch {
-		require_logger.logger_default.debug(`QuiverAI: failed to parse SSE data: ${payload}`);
+		require_logger.logger.debug(`QuiverAI: failed to parse SSE data: ${payload}`);
 		return {};
 	}
 }
@@ -208,7 +207,7 @@ function createQuiverAiProvider(providerPath, providerOptions = {}, env) {
 		env: providerOptions.env ?? env
 	});
 }
-
 //#endregion
 exports.createQuiverAiProvider = createQuiverAiProvider;
-//# sourceMappingURL=quiverai-DiMVJQDz.cjs.map
+
+//# sourceMappingURL=quiverai-PdShCPox.cjs.map

package/dist/src/{quiverai-fQNkExW4.js → quiverai-uH-dcTIr.js}

@@ -1,10 +1,8 @@
 #!/usr/bin/env node
-import "./chunk-DHDDz29n.js";
-import { C as getEnvString, o as logger_default } from "./logger-LSBxlt7a.js";
-import { f as REQUEST_TIMEOUT_MS, n as fetchWithProxy } from "./fetch-CXZI9RRr.js";
-import "./cloud-CZ-q9Ier.js";
-import { a as fetchWithCache } from "./cache-jsiwsAJv.js";
-
+import { C as getEnvString, o as logger } from "./logger-KkObSCzq.js";
+import { f as REQUEST_TIMEOUT_MS, n as fetchWithProxy } from "./fetch-Dxpd4_sr.js";
+import "./cloud-C0dlstV_.js";
+import { a as fetchWithCache } from "./cache-BVeDlD87.js";
 //#region src/providers/quiverai.ts
 const QUIVERAI_API_BASE_URL = "https://api.quiver.ai/v1";
 const QUIVERAI_DEFAULT_MODEL = "arrow-preview";
@@ -66,7 +64,7 @@ var QuiverAiProvider = class {
 			if (useStream) return await this.callApiStreaming(body);
 			return await this.callApiNonStreaming(body, context);
 		} catch (error) {
-			logger_default.error(`QuiverAI API call error: ${error}`);
+			logger.error(`QuiverAI API call error: ${error}`);
 			return { error: `QuiverAI API call error: ${error}` };
 		}
 	}
@@ -103,7 +101,7 @@ var QuiverAiProvider = class {
 				});
 				if (lastResp.status === 429 && attempt < maxRetries) {
 					const waitMs = getRetryAfterMs(lastResp.headers, attempt);
-					logger_default.debug(`QuiverAI: rate limited, retry ${attempt + 1}/${maxRetries} in ${waitMs}ms`);
+					logger.debug(`QuiverAI: rate limited, retry ${attempt + 1}/${maxRetries} in ${waitMs}ms`);
 					await lastResp.body?.cancel();
 					await new Promise((resolve) => setTimeout(resolve, waitMs));
 					continue;
@@ -192,7 +190,7 @@ function parseSSELine(line) {
 			usage: event.usage
 		};
 	} catch {
-		logger_default.debug(`QuiverAI: failed to parse SSE data: ${payload}`);
+		logger.debug(`QuiverAI: failed to parse SSE data: ${payload}`);
 		return {};
 	}
 }
@@ -211,7 +209,7 @@ function createQuiverAiProvider(providerPath, providerOptions = {}, env) {
 		env: providerOptions.env ?? env
 	});
 }
-
 //#endregion
 export { createQuiverAiProvider };
-//# sourceMappingURL=quiverai-fQNkExW4.js.map
+
+//# sourceMappingURL=quiverai-uH-dcTIr.js.map

package/dist/src/{render-Dj1smHEb.js → render-Drod8m7K.js}

@@ -1,7 +1,6 @@
 #!/usr/bin/env node
-import "./logger-LSBxlt7a.js";
-import { t as canUseInteractiveUI } from "./interactiveCheck-Bxj1Swex.js";
-
+import "./logger-KkObSCzq.js";
+import { t as canUseInteractiveUI } from "./interactiveCheck-BgLZUIt3.js";
 //#region src/ui/render.ts
 /**
 * Process exit codes for signal handling.
@@ -112,7 +111,7 @@ function supportsColor() {
 	if (process.env.FORCE_COLOR) return true;
 	return process.stdout.isTTY && process.stdout.hasColors?.() !== false;
 }
-
 //#endregion
 export { canUseInteractiveUI, getTerminalSize, renderInteractive, runInteractive, supportsColor };
-//# sourceMappingURL=render-Dj1smHEb.js.map
+
+//# sourceMappingURL=render-Drod8m7K.js.map