promptfoo 0.121.4 → 0.121.7

package/dist/src/{util-Bxn8emtE.cjs → util-SPsvFONY.cjs}

@@ -1,697 +1,829 @@
-const require_logger = require("./logger-COuQb2xB.cjs");
-const require_invariant = require("./invariant-kfQ8Bu82.cjs");
-const require_esm = require("./esm-CtEPLdAj.cjs");
-const require_pythonUtils = require("./pythonUtils-tAJvvpS-.cjs");
-const require_fileExtensions = require("./fileExtensions-bYh77CN8.cjs");
-const require_types = require("./types-CeaeaZdP.cjs");
-const require_render = require("./render-DfQSFxGE.cjs");
-const require_fetch = require("./fetch-CJU5ELPa.cjs");
+const require_rolldown_runtime = require("./rolldown-runtime-D_mwlA32.cjs");
+const require_logger = require("./logger-cfNpzI4o.cjs");
+const require_invariant = require("./invariant-QtnLD03y.cjs");
+const require_types = require("./types-CxJvaY2S.cjs");
+const require_fetch = require("./fetch-Dw4XZHjj.cjs");
+const require_version = require("./version-BWCSaByA.cjs");
+const require_fileExtensions = require("./fileExtensions-BhdwzYaD.cjs");
+const require_esm = require("./esm-BIKakvNa.cjs");
+const require_render = require("./render-BNTrbmBw.cjs");
+const require_pythonUtils = require("./pythonUtils-Cokhluq3.cjs");
 let fs = require("fs");
-fs = require_logger.__toESM(fs);
+fs = require_rolldown_runtime.__toESM(fs, 1);
 let path = require("path");
-path = require_logger.__toESM(path);
+path = require_rolldown_runtime.__toESM(path, 1);
 let js_yaml = require("js-yaml");
-js_yaml = require_logger.__toESM(js_yaml);
+js_yaml = require_rolldown_runtime.__toESM(js_yaml, 1);
 let dotenv = require("dotenv");
-dotenv = require_logger.__toESM(dotenv);
+dotenv = require_rolldown_runtime.__toESM(dotenv, 1);
 let os = require("os");
-os = require_logger.__toESM(os);
+os = require_rolldown_runtime.__toESM(os, 1);
 let dedent = require("dedent");
-dedent = require_logger.__toESM(dedent);
+dedent = require_rolldown_runtime.__toESM(dedent, 1);
 let fs_promises = require("fs/promises");
-fs_promises = require_logger.__toESM(fs_promises);
+fs_promises = require_rolldown_runtime.__toESM(fs_promises, 1);
+let nunjucks = require("nunjucks");
+nunjucks = require_rolldown_runtime.__toESM(nunjucks, 1);
+let csv_parse_sync = require("csv-parse/sync");
 let glob = require("glob");
 let fast_deep_equal = require("fast-deep-equal");
-fast_deep_equal = require_logger.__toESM(fast_deep_equal);
-let csv_parse_sync = require("csv-parse/sync");
-let nunjucks = require("nunjucks");
-nunjucks = require_logger.__toESM(nunjucks);
+fast_deep_equal = require_rolldown_runtime.__toESM(fast_deep_equal, 1);
 let fast_xml_parser = require("fast-xml-parser");
 let csv_stringify_sync = require("csv-stringify/sync");
-//#region src/util/provider.ts
-function canonicalizeProviderId(id) {
-	if (id.startsWith("file://")) {
-		const filePath = id.slice(7);
-		return path.isAbsolute(filePath) ? id : `file://${path.resolve(filePath)}`;
-	}
-	for (const prefix of [
-		"exec:",
-		"python:",
-		"golang:"
-	]) if (id.startsWith(prefix)) {
-		const filePath = id.slice(prefix.length);
-		if (filePath.includes("/") || filePath.includes("\\")) return `${prefix}${path.resolve(filePath)}`;
-		return id;
+//#region src/util/functions/loadFunction.ts
+const functionCache = {};
+/**
+* Loads a function from a JavaScript or Python file
+* @param options Options for loading the function
+* @returns The loaded function
+*/
+async function loadFunction({ filePath, functionName, defaultFunctionName = "func", basePath = require_logger.state.basePath, useCache = true }) {
+	const cacheKey = `${filePath}${functionName ? `:${functionName}` : ""}`;
+	if (useCache && functionCache[cacheKey]) return functionCache[cacheKey];
+	const resolvedPath = basePath ? path.default.resolve(basePath, filePath) : filePath;
+	if (!require_fileExtensions.isJavascriptFile(resolvedPath) && !resolvedPath.endsWith(".py")) throw new Error(`File must be a JavaScript (${require_fileExtensions.JAVASCRIPT_EXTENSIONS.join(", ")}) or Python (.py) file`);
+	try {
+		let func;
+		if (require_fileExtensions.isJavascriptFile(resolvedPath)) {
+			const module = await require_esm.importModule(resolvedPath, functionName);
+			let moduleFunc;
+			if (functionName) moduleFunc = module;
+			else moduleFunc = typeof module === "function" ? module : module?.default?.default || module?.default || module?.[defaultFunctionName] || module;
+			if (typeof moduleFunc !== "function") throw new Error(functionName ? `JavaScript file must export a "${functionName}" function` : `JavaScript file must export a function (as default export or named export "${defaultFunctionName}")`);
+			func = moduleFunc;
+		} else {
+			const result = (...args) => require_pythonUtils.runPython(resolvedPath, functionName || defaultFunctionName, args);
+			func = result;
+		}
+		if (useCache) functionCache[cacheKey] = func;
+		return func;
+	} catch (err) {
+		require_logger.logger.error(`Failed to load function: ${err.message}`);
+		throw err;
 	}
-	if ((id.endsWith(".js") || id.endsWith(".ts") || id.endsWith(".mjs")) && (id.includes("/") || id.includes("\\"))) return `file://${path.resolve(id)}`;
-	return id;
-}
-function getProviderLabel(provider) {
-	return provider?.label && typeof provider.label === "string" ? provider.label : void 0;
 }
-function providerToIdentifier(provider) {
-	if (!provider) return;
-	if (typeof provider === "string") return canonicalizeProviderId(provider);
-	const label = getProviderLabel(provider);
-	if (label) return label;
-	if (require_types.isApiProvider(provider)) return canonicalizeProviderId(provider.id());
-	if (require_types.isProviderOptions(provider)) {
-		if (provider.id) return canonicalizeProviderId(provider.id);
-		return;
-	}
-	if (typeof provider === "object" && "id" in provider && typeof provider.id === "string") return canonicalizeProviderId(provider.id);
+/**
+* Extracts the file path and function name from a file:// URL
+* @param fileUrl The file:// URL (e.g., "file://path/to/file.js:functionName")
+* @returns The file path and optional function name
+*/
+function parseFileUrl(fileUrl) {
+	if (!fileUrl.startsWith("file://")) throw new Error("URL must start with file://");
+	const urlWithoutProtocol = fileUrl.slice(7);
+	const lastColonIndex = urlWithoutProtocol.lastIndexOf(":");
+	if (lastColonIndex > 1) return {
+		filePath: urlWithoutProtocol.slice(0, lastColonIndex),
+		functionName: urlWithoutProtocol.slice(lastColonIndex + 1)
+	};
+	return { filePath: urlWithoutProtocol };
 }
+//#endregion
+//#region src/util/file.ts
 /**
-* Gets a descriptive identifier string for a provider, showing both label and ID when both exist.
-* Useful for error messages to help users debug provider reference issues.
+* Simple Nunjucks engine specifically for file paths
+* This function is separate from the main getNunjucksEngine to avoid circular dependencies
 */
-function getProviderDescription(provider) {
-	const label = provider.label;
-	const id = provider.id();
-	if (label && label !== id) return `${label} (${id})`;
-	return id;
+function getNunjucksEngineForFilePath() {
+	const env = nunjucks.default.configure({ autoescape: false });
+	env.addGlobal("env", {
+		...process.env,
+		...require_logger.state.config?.env
+	});
+	return env;
 }
 /**
-* Checks if a provider reference matches a given provider.
-* Supports exact matching and wildcard patterns.
+* Loads content from an external file if the input is a file path, otherwise
+* returns the input as-is. Supports Nunjucks templating for file paths.
+*
+* @param filePath - The input to process. Can be a file path string starting with "file://",
+* an array of file paths, or any other type of data.
+* @param context - Optional context to control file loading behavior. 'assertion' context
+* preserves Python/JS file references instead of loading their content.
+* @returns The loaded content if the input was a file path, otherwise the original input.
+* For JSON and YAML files, the content is parsed into an object.
+* For other file types, the raw file content is returned as a string.
+*
+* @throws {Error} If the specified file does not exist.
 */
-function doesProviderRefMatch(ref, provider) {
-	const label = provider.label;
-	const id = provider.id();
-	const canonicalRef = canonicalizeProviderId(ref);
-	const canonicalId = canonicalizeProviderId(id);
-	if (label && label === ref) return true;
-	if (id === ref || canonicalId === canonicalRef) return true;
-	if (ref.endsWith("*")) {
-		const prefix = ref.slice(0, -1);
-		if (label?.startsWith(prefix) || id.startsWith(prefix) || canonicalId.startsWith(prefix)) return true;
+function maybeLoadFromExternalFile(filePath, context) {
+	if (Array.isArray(filePath)) return filePath.map((path$3) => {
+		return maybeLoadFromExternalFile(path$3, context);
+	});
+	if (typeof filePath !== "string") return filePath;
+	if (!filePath.startsWith("file://")) return filePath;
+	const renderedFilePath = getNunjucksEngineForFilePath().renderString(filePath, {});
+	const { filePath: cleanPath, functionName } = parseFileUrl(renderedFilePath);
+	if (context === "assertion" && (cleanPath.endsWith(".py") || require_fileExtensions.isJavascriptFile(cleanPath))) {
+		require_logger.logger.debug(`Preserving Python/JS file reference in assertion context: ${renderedFilePath}`);
+		return renderedFilePath;
 	}
-	if (label?.startsWith(`${ref}:`) || id.startsWith(`${ref}:`) || canonicalId.startsWith(`${ref}:`)) return true;
-	return false;
+	if (context === "vars") {
+		require_logger.logger.debug(`Preserving file reference in vars context: ${renderedFilePath}`);
+		return renderedFilePath;
+	}
+	if (functionName && (cleanPath.endsWith(".py") || require_fileExtensions.isJavascriptFile(cleanPath))) return renderedFilePath;
+	const pathToUse = functionName && !(cleanPath.endsWith(".py") || require_fileExtensions.isJavascriptFile(cleanPath)) ? renderedFilePath.slice(7) : cleanPath;
+	const resolvedPath = path.resolve(require_logger.state.basePath || "", pathToUse);
+	if ((0, glob.hasMagic)(pathToUse)) {
+		const matchedFiles = (0, glob.globSync)(resolvedPath, { windowsPathsNoEscape: true });
+		if (matchedFiles.length === 0) throw new Error(`No files found matching pattern: ${resolvedPath}`);
+		const allContents = [];
+		for (const matchedFile of matchedFiles) {
+			let contents;
+			try {
+				contents = fs.readFileSync(matchedFile, "utf8");
+			} catch (error) {
+				if (error.code === "ENOENT") {
+					require_logger.logger.debug(`File disappeared during glob expansion: ${matchedFile}`);
+					continue;
+				}
+				throw error;
+			}
+			if (matchedFile.endsWith(".json")) {
+				const parsed = JSON.parse(contents);
+				if (Array.isArray(parsed)) allContents.push(...parsed);
+				else allContents.push(parsed);
+			} else if (matchedFile.endsWith(".yaml") || matchedFile.endsWith(".yml")) {
+				const parsed = js_yaml.default.load(contents);
+				if (parsed === null || parsed === void 0) continue;
+				if (Array.isArray(parsed)) allContents.push(...parsed);
+				else allContents.push(parsed);
+			} else if (matchedFile.endsWith(".csv")) {
+				const records = (0, csv_parse_sync.parse)(contents, { columns: true });
+				if (records.length > 0 && Object.keys(records[0]).length === 1) allContents.push(...records.map((record) => Object.values(record)[0]));
+				else allContents.push(...records);
+			} else allContents.push(contents);
+		}
+		return allContents;
+	}
+	const finalPath = resolvedPath;
+	let contents;
+	try {
+		contents = fs.readFileSync(finalPath, "utf8");
+	} catch (error) {
+		if (error.code === "ENOENT") throw new Error(`File does not exist: ${finalPath}`);
+		throw new Error(`Failed to read file ${finalPath}: ${error}`);
+	}
+	if (finalPath.endsWith(".json")) try {
+		return JSON.parse(contents);
+	} catch (error) {
+		throw new Error(`Failed to parse JSON file ${finalPath}: ${error}`);
+	}
+	if (finalPath.endsWith(".yaml") || finalPath.endsWith(".yml")) try {
+		return js_yaml.default.load(contents);
+	} catch (error) {
+		throw new Error(`Failed to parse YAML file ${finalPath}: ${error}`);
+	}
+	if (finalPath.endsWith(".csv")) {
+		const records = (0, csv_parse_sync.parse)(contents, { columns: true });
+		if (records.length > 0 && Object.keys(records[0]).length === 1) return records.map((record) => Object.values(record)[0]);
+		return records;
+	}
+	return contents;
 }
 /**
-* Checks if a provider is allowed based on a list of allowed references.
+* Resolves a relative file path with respect to a base path, handling cloud configuration appropriately.
+* When using a cloud configuration, the current working directory is always used instead of the context's base path.
+*
+* @param filePath - The relative or absolute file path to resolve.
+* @param isCloudConfig - Whether this is a cloud configuration.
+* @returns The resolved absolute file path.
 */
-function isProviderAllowed(provider, allowedProviders) {
-	if (!Array.isArray(allowedProviders)) return true;
-	if (allowedProviders.length === 0) return false;
-	return allowedProviders.some((ref) => doesProviderRefMatch(ref, provider));
+function getResolvedRelativePath(filePath, isCloudConfig) {
+	if (path.isAbsolute(filePath) || !isCloudConfig) return filePath;
+	return path.join(process.cwd(), filePath);
 }
 /**
-* Detects if a provider uses OpenAI models.
-* This includes direct OpenAI providers and Azure OpenAI.
+* Recursively loads external file references from a configuration object.
+*
+* @param config - The configuration object to process
+* @param context - Optional context to control file loading behavior
+* @returns The configuration with external file references resolved
 */
-function isOpenAiProvider(providerId) {
-	const lowerProviderId = providerId.toLowerCase();
-	if (lowerProviderId.startsWith("openai:")) return true;
-	if (lowerProviderId.startsWith("azureopenai:")) return true;
-	if (lowerProviderId.startsWith("azure:")) {
-		if ([
-			"gpt",
-			"openai",
-			"davinci",
-			"curie",
-			"babbage",
-			"ada",
-			"text-embedding",
-			"whisper",
-			"dall-e",
-			"tts"
-		].some((indicator) => lowerProviderId.includes(indicator))) return true;
+function maybeLoadConfigFromExternalFile(config, context) {
+	if (Array.isArray(config)) return config.map((item) => maybeLoadConfigFromExternalFile(item, context));
+	if (typeof config === "object" && config !== null) {
+		const result = {};
+		for (const key of Object.keys(config)) {
+			const childContext = key === "value" && "type" in config && typeof config.type === "string" && (config.type === "python" || config.type === "javascript") ? "assertion" : key === "vars" ? "vars" : context;
+			const value = maybeLoadConfigFromExternalFile(config[key], childContext);
+			if (key === "__proto__") Object.defineProperty(result, key, {
+				value,
+				enumerable: true,
+				configurable: true,
+				writable: true
+			});
+			else result[key] = value;
+		}
+		return result;
 	}
-	return false;
+	return maybeLoadFromExternalFile(config, context);
 }
 /**
-* Detects if a provider uses Anthropic/Claude models.
-* This includes direct Anthropic providers, Bedrock with Claude, and Vertex with Claude.
+* Parses a file path or glob pattern to extract function names and file extensions.
+* Function names can be specified in the filename like this:
+* prompt.py:myFunction or prompts.js:myFunction.
+* @param basePath - The base path for file resolution.
+* @param promptPath - The path or glob pattern.
+* @returns Parsed details including function name, file extension, and directory status.
 */
-function isAnthropicProvider(providerId) {
-	const lowerProviderId = providerId.toLowerCase();
-	if (lowerProviderId.startsWith("anthropic:")) return true;
-	if (lowerProviderId.startsWith("bedrock:")) {
-		if (lowerProviderId.includes("claude") || lowerProviderId.includes("anthropic")) return true;
-	}
-	if (lowerProviderId.startsWith("vertex:")) {
-		if (lowerProviderId.includes("claude")) return true;
-	}
-	return false;
-}
-const KNOWN_ENV_VARS = {
-	openai: "OPENAI_API_KEY",
-	anthropic: "ANTHROPIC_API_KEY",
-	google: "GOOGLE_API_KEY",
-	mistral: "MISTRAL_API_KEY",
-	cohere: "COHERE_API_KEY",
-	replicate: "REPLICATE_API_TOKEN",
-	voyage: "VOYAGE_API_KEY",
-	ai21: "AI21_API_KEY",
-	xai: "XAI_API_KEY",
-	groq: "GROQ_API_KEY",
-	deepseek: "DEEPSEEK_API_KEY",
-	perplexity: "PERPLEXITY_API_KEY",
-	hyperbolic: "HYPERBOLIC_API_KEY",
-	cerebras: "CEREBRAS_API_KEY",
-	togetherai: "TOGETHER_API_KEY",
-	fal: "FAL_KEY",
-	huggingface: "HF_TOKEN",
-	"cloudflare-ai": "CLOUDFLARE_API_KEY"
-};
-function getDefaultEnvVar(providerId) {
-	const prefix = providerId.split(":")[0];
-	return KNOWN_ENV_VARS[prefix] || `${prefix.toUpperCase()}_API_KEY`;
-}
-/**
-* Pre-checks providers for missing API keys before evaluation starts.
-* Assumes getApiKey() is side-effect free (no network calls or token refresh).
-*/
-function checkProviderApiKeys(providers) {
-	const missingApiKeys = /* @__PURE__ */ new Map();
-	for (const provider of providers) {
-		const p = provider;
-		if (typeof p.getApiKey !== "function") continue;
-		if (provider.id().startsWith("azure:")) continue;
-		const requiresKey = typeof p.requiresApiKey === "function" ? p.requiresApiKey() : p.config?.apiKeyRequired !== false;
-		let apiKey;
-		try {
-			apiKey = p.getApiKey();
-		} catch {
-			apiKey = void 0;
-		}
-		if (requiresKey && !apiKey) {
-			const envVar = p.config?.apiKeyEnvar || getDefaultEnvVar(provider.id());
-			if (!missingApiKeys.has(envVar)) missingApiKeys.set(envVar, []);
-			missingApiKeys.get(envVar).push(provider.id());
+function parsePathOrGlob(basePath, promptPath) {
+	if (promptPath.startsWith("file://")) promptPath = promptPath.slice(7);
+	const filePath = path.resolve(basePath, promptPath);
+	let filename = path.relative(basePath, filePath);
+	let functionName;
+	if (filename.includes(":")) {
+		const lastColonIndex = filename.lastIndexOf(":");
+		if (lastColonIndex > 1) {
+			const pathWithoutFunction = filename.slice(0, lastColonIndex);
+			if (require_fileExtensions.isJavascriptFile(pathWithoutFunction) || pathWithoutFunction.endsWith(".py") || pathWithoutFunction.endsWith(".go") || pathWithoutFunction.endsWith(".rb")) {
+				functionName = filename.slice(lastColonIndex + 1);
+				filename = pathWithoutFunction;
+			}
 		}
 	}
-	return missingApiKeys;
+	let stats;
+	try {
+		stats = fs.statSync(path.join(basePath, filename));
+	} catch (err) {
+		if (require_logger.getEnvBool("PROMPTFOO_STRICT_FILES")) throw err;
+	}
+	const normalizedFilePath = filePath.replace(/\\/g, "/");
+	const isPathPattern = stats?.isDirectory() || (0, glob.hasMagic)(promptPath) || (0, glob.hasMagic)(normalizedFilePath);
+	const safeFilename = path.relative(basePath, require_esm.safeResolve(basePath, filename));
+	return {
+		extension: isPathPattern ? void 0 : path.parse(safeFilename).ext,
+		filePath: path.join(basePath, safeFilename),
+		functionName,
+		isPathPattern
+	};
+}
+function readOutput(outputPath) {
+	const ext = path.parse(outputPath).ext.slice(1);
+	switch (ext) {
+		case "json": return JSON.parse(fs.readFileSync(outputPath, "utf-8"));
+		default: throw new Error(`Unsupported output file format: ${ext} currently only supports json`);
+	}
 }
 /**
-* Detects if a provider uses Google models.
-* This includes direct Google/Vertex providers with Gemini and other Google models.
-* Note: Vertex with Claude models is NOT counted as Google (it's Anthropic).
+* Load custom Nunjucks filters from external files.
+* Note: If a glob pattern matches multiple files, only the last file's export is used.
+* Each filter name should typically resolve to a single file.
 */
-function isGoogleProvider(providerId) {
-	const lowerProviderId = providerId.toLowerCase();
-	if (lowerProviderId.startsWith("google:")) return true;
-	if (lowerProviderId.startsWith("vertex:")) {
-		if (!lowerProviderId.includes("claude")) return true;
+async function readFilters(filters, basePath = "") {
+	const ret = {};
+	for (const [name, filterPath] of Object.entries(filters)) {
+		const filePaths = (0, glob.globSync)(path.join(basePath, filterPath), { windowsPathsNoEscape: true });
+		for (const filePath of filePaths) ret[name] = await require_esm.importModule(path.resolve(filePath));
 	}
-	return false;
+	return ret;
 }
-//#endregion
-//#region src/util/comparison.ts
 /**
-* Explicit runtime variable names that don't follow the underscore convention.
-* These are added during evaluation but aren't part of the original test definition.
+* Loads configuration from an external file with variable rendering.
+* This is a convenience wrapper that combines renderVarsInObject and maybeLoadFromExternalFile.
 *
-* - sessionId: Added by multi-turn strategy providers (GOAT, Crescendo)
+* Use this for simple config fields that:
+* - Need variable rendering ({{ vars.x }}, {{ env.X }})
+* - May reference external files (file://path.json)
+* - Don't have nested file references that need loading
 *
-* Note: Variables starting with underscore (e.g., _conversation) are automatically
-* treated as runtime variables and filtered out.
-*/
-const EXPLICIT_RUNTIME_VAR_KEYS = ["sessionId"];
-/**
-* Checks if a variable key is a runtime-only variable that should be filtered
-* when comparing test cases.
+* For fields with nested file references (like response_format.schema),
+* use maybeLoadResponseFormatFromExternalFile instead.
 *
-* Runtime variables are identified by:
-* 1. Starting with underscore (_) - convention for internal/runtime vars
-* 2. Being in the explicit runtime var list (for legacy vars like sessionId)
+* @param config - The configuration to process
+* @param vars - Variables for template rendering
+* @returns The processed configuration with variables rendered and files loaded
 */
-function isRuntimeVar(key) {
-	return key.startsWith("_") || EXPLICIT_RUNTIME_VAR_KEYS.includes(key);
+function maybeLoadFromExternalFileWithVars(config, vars) {
+	return maybeLoadFromExternalFile(require_render.renderVarsInObject(config, vars));
 }
 /**
-* Filters out runtime-only variables that are added during evaluation
-* but aren't part of the original test definition.
+* Loads response_format configuration from an external file with variable rendering.
 *
-* This is used when comparing test cases to determine if a result
-* corresponds to a particular test, regardless of runtime state.
+* This function handles the special case where response_format may contain:
+* 1. A top-level file reference (file://format.json)
+* 2. A nested schema reference for json_schema type (schema: file://schema.json)
 *
-* Runtime variables are identified by:
-* - Starting with underscore (e.g., _conversation, _metadata)
-* - Being in the explicit list (e.g., sessionId for backward compatibility)
+* Both levels need variable rendering and file loading.
+*
+* @param responseFormat - The response_format configuration
+* @param vars - Variables for template rendering
+* @returns The processed response_format with all files loaded
 */
-function filterRuntimeVars(vars) {
-	if (!vars || typeof vars !== "object" || Array.isArray(vars)) return vars;
-	const filtered = {};
-	for (const [key, value] of Object.entries(vars)) if (!isRuntimeVar(key)) filtered[key] = value;
-	return filtered;
+function maybeLoadResponseFormatFromExternalFile(responseFormat, vars) {
+	if (responseFormat === void 0 || responseFormat === null) return responseFormat;
+	const loaded = maybeLoadFromExternalFile(require_render.renderVarsInObject(responseFormat, vars));
+	if (!loaded || typeof loaded !== "object") return loaded;
+	if (loaded.type === "json_schema") {
+		const nestedSchema = loaded.schema || loaded.json_schema?.schema;
+		if (nestedSchema) {
+			const loadedSchema = maybeLoadFromExternalFile(require_render.renderVarsInObject(nestedSchema, vars));
+			if (loaded.schema !== void 0) return {
+				...loaded,
+				schema: loadedSchema
+			};
+			else if (loaded.json_schema?.schema !== void 0) return {
+				...loaded,
+				json_schema: {
+					...loaded.json_schema,
+					schema: loadedSchema
+				}
+			};
+		}
+	}
+	return loaded;
 }
 /**
-* Extracts only runtime variables from a vars object.
-* This is the inverse of filterRuntimeVars.
+* Renders variables in a tools object and loads from external file if applicable.
+* This function combines renderVarsInObject and maybeLoadFromExternalFile into a single step
+* specifically for handling tools configurations.
 *
-* Used to restore runtime state when re-running filtered tests.
+* Supports loading from JSON, YAML, Python, and JavaScript files.
+*
+* @param tools - The tools configuration object or array to process.
+* @param vars - Variables to use for rendering.
+* @returns The processed tools configuration with variables rendered and content loaded from files if needed.
+* @throws {Error} If the loaded tools are in an invalid format
 */
-function extractRuntimeVars(vars) {
-	if (!vars || typeof vars !== "object" || Array.isArray(vars)) return;
-	const extracted = {};
-	for (const [key, value] of Object.entries(vars)) if (isRuntimeVar(key)) extracted[key] = value;
-	return Object.keys(extracted).length > 0 ? extracted : void 0;
+async function maybeLoadToolsFromExternalFile(tools, vars) {
+	const rendered = require_render.renderVarsInObject(tools, vars);
+	if (typeof rendered === "string" && rendered.startsWith("file://")) {
+		const { filePath, functionName } = parseFileUrl(rendered);
+		if (functionName && (filePath.endsWith(".py") || require_fileExtensions.isJavascriptFile(filePath))) {
+			const fileType = filePath.endsWith(".py") ? "Python" : "JavaScript";
+			require_logger.logger.debug(`[maybeLoadToolsFromExternalFile] Loading tools from ${fileType} file: ${filePath}:${functionName}`);
+			try {
+				let toolDefinitions;
+				if (filePath.endsWith(".py")) {
+					const absPath = require_esm.safeResolve(require_logger.state.basePath || process.cwd(), filePath);
+					require_logger.logger.debug(`[maybeLoadToolsFromExternalFile] Resolved Python path: ${absPath}`);
+					toolDefinitions = await require_pythonUtils.runPython(absPath, functionName, []);
+				} else {
+					const absPath = require_esm.safeResolve(require_logger.state.basePath || process.cwd(), filePath);
+					require_logger.logger.debug(`[maybeLoadToolsFromExternalFile] Resolved JavaScript path: ${absPath}`);
+					const module = await require_esm.importModule(absPath);
+					const fn = module[functionName] || module.default?.[functionName];
+					if (typeof fn !== "function") {
+						const availableExports = Object.keys(module).filter((k) => k !== "default");
+						const basePath = require_logger.state.basePath || process.cwd();
+						throw new Error(`Function "${functionName}" not found in ${filePath}. Available exports: ${availableExports.length > 0 ? availableExports.join(", ") : "(none)"}\nResolved from: ${basePath}`);
+					}
+					toolDefinitions = await Promise.resolve(fn());
+				}
+				if (!toolDefinitions || typeof toolDefinitions === "string" || typeof toolDefinitions === "number" || typeof toolDefinitions === "boolean") throw new Error(`Function "${functionName}" must return an array or object of tool definitions, but returned: ${toolDefinitions === null ? "null" : typeof toolDefinitions}`);
+				require_logger.logger.debug(`[maybeLoadToolsFromExternalFile] Successfully loaded ${Array.isArray(toolDefinitions) ? toolDefinitions.length : "object"} tools`);
+				return toolDefinitions;
+			} catch (err) {
+				const errorMessage = err instanceof Error ? err.message : String(err);
+				const basePath = require_logger.state.basePath || process.cwd();
+				throw new Error(`Failed to load tools from ${rendered}:\n${errorMessage}\n\nMake sure the function "${functionName}" exists and returns a valid tool definition array.\nResolved from: ${basePath}`);
+			}
+		}
+		if (filePath.endsWith(".py") || require_fileExtensions.isJavascriptFile(filePath)) {
+			const ext = filePath.endsWith(".py") ? "Python" : "JavaScript";
+			const basePath = require_logger.state.basePath || process.cwd();
+			throw new Error(`Cannot load tools from ${rendered}\n${ext} files require a function name. Use this format:\n  tools: file://${filePath}:get_tools\n\nYour ${ext} file should export a function that returns tool definitions:\n` + (filePath.endsWith(".py") ? `  def get_tools():\n      return [{"type": "function", "function": {...}}]` : `  module.exports.get_tools = () => [{ type: "function", function: {...} }];`) + `\n\nResolved from: ${basePath}`);
+		}
+	}
+	if (Array.isArray(rendered)) {
+		const results = await Promise.all(rendered.map((item) => maybeLoadToolsFromExternalFile(item, vars)));
+		if (results.every((r) => Array.isArray(r))) return results.flat();
+		return results;
+	}
+	if (typeof rendered !== "string") return rendered;
+	const loaded = maybeLoadFromExternalFile(rendered);
+	if (loaded !== void 0 && loaded !== null && typeof loaded === "string") {
+		if (loaded.startsWith("file://")) throw new Error(`Failed to load tools from ${loaded}\nEnsure the file exists and contains valid JSON or YAML tool definitions.`);
+		if (loaded.includes("def ") || loaded.includes("import ")) throw new Error("Invalid tools configuration: file appears to contain Python code.\nPython files require a function name. Use this format:\n  tools: file://tools.py:get_tools");
+		throw new Error("Invalid tools configuration: expected an array or object, but got a string.\nIf using file://, ensure the file contains valid JSON or YAML tool definitions.");
+	}
+	return loaded;
 }
-function varsMatch(vars1, vars2) {
-	return (0, fast_deep_equal.default)(vars1, vars2);
+//#endregion
+//#region src/util/providerRef.ts
+const PROVIDER_OPTION_KEYS = new Set([
+	"id",
+	"label",
+	"config",
+	"prompts",
+	"transform",
+	"delay",
+	"env",
+	"inputs"
+]);
+/** Returns true if the value is a non-empty string suitable as a provider identifier. */
+function isValidProviderId(id) {
+	return typeof id === "string" && id !== "";
+}
+function getProviderLabel(provider) {
+	if ((typeof provider === "object" || typeof provider === "function") && provider !== null && "label" in provider && typeof provider.label === "string") return provider.label;
 }
 /**
-* Generate a unique key for a test case for deduplication purposes.
-* Excludes runtime variables and includes strategyId to distinguish tests
-* with the same prompt but different strategies.
-*
-* @param testCase - The test case to generate a key for
-* @returns A JSON string that uniquely identifies the test case
+* Resolves relative file paths in provider IDs to absolute paths for consistent matching.
+* Handles file://, exec:, python:, golang: prefixes and bare .js/.ts/.mjs paths.
 */
-function getTestCaseDeduplicationKey(testCase) {
-	const filteredVars = filterRuntimeVars(testCase.vars);
-	const strategyId = testCase.metadata?.strategyId || "none";
-	return JSON.stringify({
-		vars: filteredVars,
-		strategyId
-	});
-}
-/**
-* Deduplicates an array of test cases based on their vars and strategyId.
-* Tests with the same vars but different strategies are considered different.
-* Runtime variables (like _conversation, sessionId) are filtered out before comparison.
-*
-* @param tests - Array of test cases to deduplicate
-* @returns Deduplicated array of test cases
-*/
-function deduplicateTestCases(tests) {
-	const seen = /* @__PURE__ */ new Set();
-	return tests.filter((test) => {
-		const key = getTestCaseDeduplicationKey(test);
-		if (seen.has(key)) return false;
-		seen.add(key);
-		return true;
-	});
-}
-function resultIsForTestCase(result, testCase) {
-	const testProviderId = testCase.provider ? providerToIdentifier(testCase.provider) : void 0;
-	const resultProviderId = providerToIdentifier(result.provider);
-	const providersMatch = !testProviderId || !resultProviderId || testProviderId === resultProviderId;
-	const resultVars = filterRuntimeVars(result.vars);
-	const testVars = filterRuntimeVars(testCase.vars);
-	const doVarsMatch = varsMatch(testVars, resultVars);
-	const isMatch = doVarsMatch && providersMatch;
-	if (!isMatch) {
-		const varKeys = testVars ? Object.keys(testVars).join(", ") : "none";
-		require_logger.logger.debug(`[resultIsForTestCase] No match: vars=${doVarsMatch}, providers=${providersMatch}`, {
-			testProvider: testProviderId || "none",
-			resultProvider: resultProviderId || "none",
-			testVarKeys: varKeys
-		});
+function canonicalizeProviderId(id) {
+	if (id.startsWith("file://")) {
+		const filePath = id.slice(7);
+		return path.default.isAbsolute(filePath) ? id : `file://${path.default.resolve(filePath)}`;
 	}
-	return isMatch;
+	for (const prefix of [
+		"exec:",
+		"python:",
+		"golang:"
+	]) if (id.startsWith(prefix)) {
+		const filePath = id.slice(prefix.length);
+		if (filePath.includes("/") || filePath.includes("\\")) return `${prefix}${path.default.resolve(filePath)}`;
+		return id;
+	}
+	if ((id.endsWith(".js") || id.endsWith(".ts") || id.endsWith(".mjs")) && (id.includes("/") || id.includes("\\"))) return `file://${path.default.resolve(id)}`;
+	return id;
 }
-//#endregion
-//#region src/util/env.ts
 /**
-* Load environment variables from .env file(s).
-* @param envPath - Single path, array of paths, or undefined for default .env loading.
-*                  When paths are explicitly specified, all files must exist or an error is thrown.
-*                  When multiple files are provided, later files override values from earlier files.
+* Returns true for provider refs that should be expanded from YAML/JSON config files.
 */
-function setupEnv(envPath) {
-	if (envPath) {
-		const paths = (Array.isArray(envPath) ? envPath : [envPath]).flatMap((p) => p.includes(",") ? p.split(",").map((s) => s.trim()) : p.trim()).filter((p) => p.length > 0);
-		if (paths.length === 0) {
-			dotenv.default.config({ quiet: true });
-			return;
-		}
-		for (const p of paths) if (!fs.existsSync(p)) throw new Error(`Environment file not found: ${p}`);
-		if (paths.length === 1) require_logger.logger.info(`Loading environment variables from ${paths[0]}`);
-		else require_logger.logger.info(`Loading environment variables from: ${paths.join(", ")}`);
-		const pathArg = paths.length === 1 ? paths[0] : paths;
-		dotenv.default.config({
-			path: pathArg,
-			override: true,
-			quiet: true
-		});
-	} else dotenv.default.config({ quiet: true });
+function isProviderConfigFileReference(providerPath) {
+	return providerPath.startsWith("file://") && (providerPath.endsWith(".yaml") || providerPath.endsWith(".yml") || providerPath.endsWith(".json"));
 }
-//#endregion
-//#region src/util/functions/loadFunction.ts
-const functionCache = {};
 /**
-* Loads a function from a JavaScript or Python file
-* @param options Options for loading the function
-* @returns The loaded function
+* Reads a provider config file and normalizes single-provider and multi-provider files.
+* Returns a `wasArray` flag so callers can detect multi-provider files that require
+* `loadApiProviders` instead of `loadApiProvider`.
 */
-async function loadFunction({ filePath, functionName, defaultFunctionName = "func", basePath = require_logger.state.basePath, useCache = true }) {
-	const cacheKey = `${filePath}${functionName ? `:${functionName}` : ""}`;
-	if (useCache && functionCache[cacheKey]) return functionCache[cacheKey];
-	const resolvedPath = basePath ? path.default.resolve(basePath, filePath) : filePath;
-	if (!require_fileExtensions.isJavascriptFile(resolvedPath) && !resolvedPath.endsWith(".py")) throw new Error(`File must be a JavaScript (${require_fileExtensions.JAVASCRIPT_EXTENSIONS.join(", ")}) or Python (.py) file`);
+function readProviderConfigFile(providerPath, basePath) {
+	const relativePath = providerPath.slice(7);
+	const resolvedPath = path.default.isAbsolute(relativePath) ? relativePath : path.default.join(basePath || process.cwd(), relativePath);
+	let rawContent;
 	try {
-		let func;
-		if (require_fileExtensions.isJavascriptFile(resolvedPath)) {
-			const module = await require_esm.importModule(resolvedPath, functionName);
-			let moduleFunc;
-			if (functionName) moduleFunc = module;
-			else moduleFunc = typeof module === "function" ? module : module?.default?.default || module?.default || module?.[defaultFunctionName] || module;
-			if (typeof moduleFunc !== "function") throw new Error(functionName ? `JavaScript file must export a "${functionName}" function` : `JavaScript file must export a function (as default export or named export "${defaultFunctionName}")`);
-			func = moduleFunc;
-		} else {
-			const result = (...args) => require_pythonUtils.runPython(resolvedPath, functionName || defaultFunctionName, args);
-			func = result;
-		}
-		if (useCache) functionCache[cacheKey] = func;
-		return func;
+		rawContent = js_yaml.default.load(fs.default.readFileSync(resolvedPath, "utf8"));
 	} catch (err) {
-		require_logger.logger.error(`Failed to load function: ${err.message}`);
-		throw err;
+		throw new Error(`Failed to load provider config ${relativePath}: ${err instanceof Error ? err.message : err}`);
 	}
-}
-/**
-* Extracts the file path and function name from a file:// URL
-* @param fileUrl The file:// URL (e.g., "file://path/to/file.js:functionName")
-* @returns The file path and optional function name
-*/
-function parseFileUrl(fileUrl) {
-	if (!fileUrl.startsWith("file://")) throw new Error("URL must start with file://");
-	const urlWithoutProtocol = fileUrl.slice(7);
-	const lastColonIndex = urlWithoutProtocol.lastIndexOf(":");
-	if (lastColonIndex > 1) return {
-		filePath: urlWithoutProtocol.slice(0, lastColonIndex),
-		functionName: urlWithoutProtocol.slice(lastColonIndex + 1)
+	const fileContent = maybeLoadConfigFromExternalFile(rawContent);
+	require_invariant.invariant(fileContent, `Provider config ${relativePath} is undefined`);
+	return {
+		configs: [fileContent].flat(),
+		relativePath,
+		wasArray: Array.isArray(fileContent)
 	};
-	return { filePath: urlWithoutProtocol };
 }
-//#endregion
-//#region src/util/file.ts
 /**
-* Simple Nunjucks engine specifically for file paths
-* This function is separate from the main getNunjucksEngine to avoid circular dependencies
+* Loads provider config objects from a file-backed provider reference.
 */
-function getNunjucksEngineForFilePath() {
-	const env = nunjucks.default.configure({ autoescape: false });
-	env.addGlobal("env", {
-		...process.env,
-		...require_logger.state.config?.env
-	});
-	return env;
+function loadProviderConfigsFromFile(providerPath, basePath) {
+	return readProviderConfigFile(providerPath, basePath).configs;
 }
 /**
-* Loads content from an external file if the input is a file path, otherwise
-* returns the input as-is. Supports Nunjucks templating for file paths.
-*
-* @param filePath - The input to process. Can be a file path string starting with "file://",
-* an array of file paths, or any other type of data.
-* @param context - Optional context to control file loading behavior. 'assertion' context
-* preserves Python/JS file references instead of loading their content.
-* @returns The loaded content if the input was a file path, otherwise the original input.
-* For JSON and YAML files, the content is parsed into an object.
-* For other file types, the raw file content is returned as a string.
-*
-* @throws {Error} If the specified file does not exist.
+* Pure, synchronous classifier that converts every supported provider reference shape
+* into a discriminated descriptor. Does not read files or instantiate providers.
 */
-function maybeLoadFromExternalFile(filePath, context) {
-	if (Array.isArray(filePath)) return filePath.map((path$2) => {
-		return maybeLoadFromExternalFile(path$2, context);
-	});
-	if (typeof filePath !== "string") return filePath;
-	if (!filePath.startsWith("file://")) return filePath;
-	const renderedFilePath = getNunjucksEngineForFilePath().renderString(filePath, {});
-	const { filePath: cleanPath, functionName } = parseFileUrl(renderedFilePath);
-	if (context === "assertion" && (cleanPath.endsWith(".py") || require_fileExtensions.isJavascriptFile(cleanPath))) {
-		require_logger.logger.debug(`Preserving Python/JS file reference in assertion context: ${renderedFilePath}`);
-		return renderedFilePath;
+function normalizeProviderRef(provider, options = {}) {
+	const { index } = options;
+	if (typeof provider === "string") {
+		if (!isValidProviderId(provider)) return {
+			kind: "unknown",
+			id: index === void 0 ? "unknown" : `unknown-${index}`
+		};
+		if (isProviderConfigFileReference(provider)) return {
+			kind: "file",
+			id: provider,
+			loadProviderPath: provider
+		};
+		return {
+			kind: "named",
+			id: provider,
+			loadProviderPath: provider
+		};
 	}
-	if (context === "vars") {
-		require_logger.logger.debug(`Preserving file reference in vars context: ${renderedFilePath}`);
-		return renderedFilePath;
+	if (typeof provider === "function") {
+		const label = getProviderLabel(provider);
+		return {
+			kind: "function",
+			id: label ?? (index === void 0 ? "custom-function" : `custom-function-${index}`),
+			label
+		};
 	}
-	if (functionName && (cleanPath.endsWith(".py") || require_fileExtensions.isJavascriptFile(cleanPath))) return renderedFilePath;
-	const pathToUse = functionName && !(cleanPath.endsWith(".py") || require_fileExtensions.isJavascriptFile(cleanPath)) ? renderedFilePath.slice(7) : cleanPath;
-	const resolvedPath = path.resolve(require_logger.state.basePath || "", pathToUse);
-	if ((0, glob.hasMagic)(pathToUse)) {
-		const matchedFiles = (0, glob.globSync)(resolvedPath, { windowsPathsNoEscape: true });
-		if (matchedFiles.length === 0) throw new Error(`No files found matching pattern: ${resolvedPath}`);
-		const allContents = [];
-		for (const matchedFile of matchedFiles) {
-			let contents;
-			try {
-				contents = fs.readFileSync(matchedFile, "utf8");
-			} catch (error) {
-				if (error.code === "ENOENT") {
-					require_logger.logger.debug(`File disappeared during glob expansion: ${matchedFile}`);
-					continue;
-				}
-				throw error;
+	if (typeof provider === "object" && provider !== null && !Array.isArray(provider)) {
+		const providerId = provider.id;
+		const label = getProviderLabel(provider);
+		if (isValidProviderId(providerId)) return {
+			kind: "options",
+			id: providerId,
+			label,
+			loadOptions: provider,
+			loadProviderPath: providerId
+		};
+		const keys = Object.keys(provider);
+		if (keys.length === 1 && !PROVIDER_OPTION_KEYS.has(keys[0])) {
+			const originalId = keys[0];
+			const providerObject = provider[originalId];
+			if (typeof providerObject === "object" && providerObject !== null && !Array.isArray(providerObject) && isValidProviderId(originalId)) {
+				const id = isValidProviderId(providerObject.id) ? providerObject.id : originalId;
+				return {
+					kind: "map",
+					id,
+					label: getProviderLabel(providerObject),
+					loadOptions: {
+						...providerObject,
+						id
+					},
+					loadProviderPath: originalId
+				};
 			}
-			if (matchedFile.endsWith(".json")) {
-				const parsed = JSON.parse(contents);
-				if (Array.isArray(parsed)) allContents.push(...parsed);
-				else allContents.push(parsed);
-			} else if (matchedFile.endsWith(".yaml") || matchedFile.endsWith(".yml")) {
-				const parsed = js_yaml.default.load(contents);
-				if (parsed === null || parsed === void 0) continue;
-				if (Array.isArray(parsed)) allContents.push(...parsed);
-				else allContents.push(parsed);
-			} else if (matchedFile.endsWith(".csv")) {
-				const records = (0, csv_parse_sync.parse)(contents, { columns: true });
-				if (records.length > 0 && Object.keys(records[0]).length === 1) allContents.push(...records.map((record) => Object.values(record)[0]));
-				else allContents.push(...records);
-			} else allContents.push(contents);
 		}
-		return allContents;
-	}
-	const finalPath = resolvedPath;
-	let contents;
-	try {
-		contents = fs.readFileSync(finalPath, "utf8");
-	} catch (error) {
-		if (error.code === "ENOENT") throw new Error(`File does not exist: ${finalPath}`);
-		throw new Error(`Failed to read file ${finalPath}: ${error}`);
-	}
-	if (finalPath.endsWith(".json")) try {
-		return JSON.parse(contents);
-	} catch (error) {
-		throw new Error(`Failed to parse JSON file ${finalPath}: ${error}`);
+		if (isValidProviderId(label)) return {
+			kind: "unknown",
+			id: label,
+			label
+		};
 	}
-	if (finalPath.endsWith(".yaml") || finalPath.endsWith(".yml")) try {
-		return js_yaml.default.load(contents);
-	} catch (error) {
-		throw new Error(`Failed to parse YAML file ${finalPath}: ${error}`);
+	return {
+		kind: "unknown",
+		id: index === void 0 ? "unknown" : `unknown-${index}`
+	};
+}
+//#endregion
+//#region src/util/provider.ts
+function providerToIdentifier(provider) {
+	if (!provider) return;
+	if (typeof provider === "string") return canonicalizeProviderId(provider);
+	const { label } = normalizeProviderRef(provider);
+	if (label) return label;
+	if (require_types.isApiProvider(provider)) return canonicalizeProviderId(provider.id());
+	if (require_types.isProviderOptions(provider)) {
+		if (provider.id) return canonicalizeProviderId(provider.id);
+		return;
 	}
-	if (finalPath.endsWith(".csv")) {
-		const records = (0, csv_parse_sync.parse)(contents, { columns: true });
-		if (records.length > 0 && Object.keys(records[0]).length === 1) return records.map((record) => Object.values(record)[0]);
-		return records;
+	if (typeof provider === "object" && "id" in provider && typeof provider.id === "string") return canonicalizeProviderId(provider.id);
+}
+/**
+* Gets a descriptive identifier string for a provider, showing both label and ID when both exist.
+* Useful for error messages to help users debug provider reference issues.
+*/
+function getProviderDescription(provider) {
+	const label = provider.label;
+	const id = provider.id();
+	if (label && label !== id) return `${label} (${id})`;
+	return id;
+}
+/**
+* Checks if a provider reference matches a given provider.
+* Supports exact matching and wildcard patterns.
+*/
+function doesProviderRefMatch(ref, provider) {
+	const label = provider.label;
+	const id = provider.id();
+	const canonicalRef = canonicalizeProviderId(ref);
+	const canonicalId = canonicalizeProviderId(id);
+	if (label && label === ref) return true;
+	if (id === ref || canonicalId === canonicalRef) return true;
+	if (ref.endsWith("*")) {
+		const prefix = ref.slice(0, -1);
+		if (label?.startsWith(prefix) || id.startsWith(prefix) || canonicalId.startsWith(prefix)) return true;
 	}
-	return contents;
+	if (label?.startsWith(`${ref}:`) || id.startsWith(`${ref}:`) || canonicalId.startsWith(`${ref}:`)) return true;
+	return false;
 }
 /**
-* Resolves a relative file path with respect to a base path, handling cloud configuration appropriately.
-* When using a cloud configuration, the current working directory is always used instead of the context's base path.
-*
-* @param filePath - The relative or absolute file path to resolve.
-* @param isCloudConfig - Whether this is a cloud configuration.
-* @returns The resolved absolute file path.
+* Checks if a provider is allowed based on a list of allowed references.
 */
-function getResolvedRelativePath(filePath, isCloudConfig) {
-	if (path.isAbsolute(filePath) || !isCloudConfig) return filePath;
-	return path.join(process.cwd(), filePath);
+function isProviderAllowed(provider, allowedProviders) {
+	if (!Array.isArray(allowedProviders)) return true;
+	if (allowedProviders.length === 0) return false;
+	return allowedProviders.some((ref) => doesProviderRefMatch(ref, provider));
 }
 /**
-* Recursively loads external file references from a configuration object.
-*
-* @param config - The configuration object to process
-* @param context - Optional context to control file loading behavior
-* @returns The configuration with external file references resolved
+* Detects if a provider uses OpenAI models.
+* This includes direct OpenAI providers and Azure OpenAI.
 */
-function maybeLoadConfigFromExternalFile(config, context) {
-	if (Array.isArray(config)) return config.map((item) => maybeLoadConfigFromExternalFile(item, context));
-	if (typeof config === "object" && config !== null) {
-		const result = {};
-		for (const key of Object.keys(config)) {
-			const childContext = key === "value" && "type" in config && typeof config.type === "string" && (config.type === "python" || config.type === "javascript") ? "assertion" : key === "vars" ? "vars" : context;
-			result[key] = maybeLoadConfigFromExternalFile(config[key], childContext);
-		}
-		return result;
+function isOpenAiProvider(providerId) {
+	const lowerProviderId = providerId.toLowerCase();
+	if (lowerProviderId.startsWith("openai:")) return true;
+	if (lowerProviderId.startsWith("azureopenai:")) return true;
+	if (lowerProviderId.startsWith("azure:")) {
+		if ([
+			"gpt",
+			"openai",
+			"davinci",
+			"curie",
+			"babbage",
+			"ada",
+			"text-embedding",
+			"whisper",
+			"dall-e",
+			"tts"
+		].some((indicator) => lowerProviderId.includes(indicator))) return true;
 	}
-	return maybeLoadFromExternalFile(config, context);
+	return false;
 }
 /**
-* Parses a file path or glob pattern to extract function names and file extensions.
-* Function names can be specified in the filename like this:
-* prompt.py:myFunction or prompts.js:myFunction.
-* @param basePath - The base path for file resolution.
-* @param promptPath - The path or glob pattern.
-* @returns Parsed details including function name, file extension, and directory status.
+* Detects if a provider uses Anthropic/Claude models.
+* This includes direct Anthropic providers, Bedrock with Claude, and Vertex with Claude.
 */
-function parsePathOrGlob(basePath, promptPath) {
-	if (promptPath.startsWith("file://")) promptPath = promptPath.slice(7);
-	const filePath = path.resolve(basePath, promptPath);
-	let filename = path.relative(basePath, filePath);
-	let functionName;
-	if (filename.includes(":")) {
-		const lastColonIndex = filename.lastIndexOf(":");
-		if (lastColonIndex > 1) {
-			const pathWithoutFunction = filename.slice(0, lastColonIndex);
-			if (require_fileExtensions.isJavascriptFile(pathWithoutFunction) || pathWithoutFunction.endsWith(".py") || pathWithoutFunction.endsWith(".go") || pathWithoutFunction.endsWith(".rb")) {
-				functionName = filename.slice(lastColonIndex + 1);
-				filename = pathWithoutFunction;
-			}
-		}
+function isAnthropicProvider(providerId) {
+	const lowerProviderId = providerId.toLowerCase();
+	if (lowerProviderId.startsWith("anthropic:")) return true;
+	if (lowerProviderId.startsWith("bedrock:")) {
+		if (lowerProviderId.includes("claude") || lowerProviderId.includes("anthropic")) return true;
 	}
-	let stats;
-	try {
-		stats = fs.statSync(path.join(basePath, filename));
-	} catch (err) {
-		if (require_logger.getEnvBool("PROMPTFOO_STRICT_FILES")) throw err;
+	if (lowerProviderId.startsWith("vertex:")) {
+		if (lowerProviderId.includes("claude")) return true;
 	}
-	const normalizedFilePath = filePath.replace(/\\/g, "/");
-	const isPathPattern = stats?.isDirectory() || (0, glob.hasMagic)(promptPath) || (0, glob.hasMagic)(normalizedFilePath);
-	const safeFilename = path.relative(basePath, require_esm.safeResolve(basePath, filename));
-	return {
-		extension: isPathPattern ? void 0 : path.parse(safeFilename).ext,
-		filePath: path.join(basePath, safeFilename),
-		functionName,
-		isPathPattern
-	};
+	return false;
 }
-function readOutput(outputPath) {
-	const ext = path.parse(outputPath).ext.slice(1);
-	switch (ext) {
-		case "json": return JSON.parse(fs.readFileSync(outputPath, "utf-8"));
-		default: throw new Error(`Unsupported output file format: ${ext} currently only supports json`);
+const KNOWN_ENV_VARS = {
+	openai: "OPENAI_API_KEY",
+	anthropic: "ANTHROPIC_API_KEY",
+	google: "GOOGLE_API_KEY",
+	mistral: "MISTRAL_API_KEY",
+	cohere: "COHERE_API_KEY",
+	replicate: "REPLICATE_API_TOKEN",
+	voyage: "VOYAGE_API_KEY",
+	ai21: "AI21_API_KEY",
+	xai: "XAI_API_KEY",
+	groq: "GROQ_API_KEY",
+	deepseek: "DEEPSEEK_API_KEY",
+	perplexity: "PERPLEXITY_API_KEY",
+	hyperbolic: "HYPERBOLIC_API_KEY",
+	cerebras: "CEREBRAS_API_KEY",
+	togetherai: "TOGETHER_API_KEY",
+	fal: "FAL_KEY",
+	huggingface: "HF_TOKEN",
+	"cloudflare-ai": "CLOUDFLARE_API_KEY"
+};
+function getDefaultEnvVar(providerId) {
+	const prefix = providerId.split(":")[0];
+	return KNOWN_ENV_VARS[prefix] || `${prefix.toUpperCase()}_API_KEY`;
+}
+/**
+* Pre-checks providers for missing API keys before evaluation starts.
+* Assumes getApiKey() is side-effect free (no network calls or token refresh).
+*/
+function checkProviderApiKeys(providers) {
+	const missingApiKeys = /* @__PURE__ */ new Map();
+	for (const provider of providers) {
+		const p = provider;
+		if (typeof p.getApiKey !== "function") continue;
+		if (provider.id().startsWith("azure:")) continue;
+		const requiresKey = typeof p.requiresApiKey === "function" ? p.requiresApiKey() : p.config?.apiKeyRequired !== false;
+		let apiKey;
+		try {
+			apiKey = p.getApiKey();
+		} catch {
+			apiKey = void 0;
+		}
+		if (requiresKey && !apiKey) {
+			const envVar = p.config?.apiKeyEnvar || getDefaultEnvVar(provider.id());
+			if (!missingApiKeys.has(envVar)) missingApiKeys.set(envVar, []);
+			missingApiKeys.get(envVar).push(provider.id());
+		}
 	}
+	return missingApiKeys;
 }
 /**
-* Load custom Nunjucks filters from external files.
-* Note: If a glob pattern matches multiple files, only the last file's export is used.
-* Each filter name should typically resolve to a single file.
+* Detects if a provider uses Google models.
+* This includes direct Google/Vertex providers with Gemini and other Google models.
+* Note: Vertex with Claude models is NOT counted as Google (it's Anthropic).
 */
-async function readFilters(filters, basePath = "") {
-	const ret = {};
-	for (const [name, filterPath] of Object.entries(filters)) {
-		const filePaths = (0, glob.globSync)(path.join(basePath, filterPath), { windowsPathsNoEscape: true });
-		for (const filePath of filePaths) ret[name] = await require_esm.importModule(path.resolve(filePath));
+function isGoogleProvider(providerId) {
+	const lowerProviderId = providerId.toLowerCase();
+	if (lowerProviderId.startsWith("google:")) return true;
+	if (lowerProviderId.startsWith("vertex:")) {
+		if (!lowerProviderId.includes("claude")) return true;
 	}
-	return ret;
+	return false;
 }
+//#endregion
+//#region src/util/comparison.ts
 /**
-* Loads configuration from an external file with variable rendering.
-* This is a convenience wrapper that combines renderVarsInObject and maybeLoadFromExternalFile.
+* Explicit runtime variable names that don't follow the underscore convention.
+* These are added during evaluation but aren't part of the original test definition.
 *
-* Use this for simple config fields that:
-* - Need variable rendering ({{ vars.x }}, {{ env.X }})
-* - May reference external files (file://path.json)
-* - Don't have nested file references that need loading
+* - sessionId: Added by multi-turn strategy providers (GOAT, Crescendo)
 *
-* For fields with nested file references (like response_format.schema),
-* use maybeLoadResponseFormatFromExternalFile instead.
+* Note: Variables starting with underscore (e.g., _conversation) are automatically
+* treated as runtime variables and filtered out.
+*/
+const EXPLICIT_RUNTIME_VAR_KEYS = ["sessionId"];
+/**
+* Checks if a variable key is a runtime-only variable that should be filtered
+* when comparing test cases.
 *
-* @param config - The configuration to process
-* @param vars - Variables for template rendering
-* @returns The processed configuration with variables rendered and files loaded
+* Runtime variables are identified by:
+* 1. Starting with underscore (_) - convention for internal/runtime vars
+* 2. Being in the explicit runtime var list (for legacy vars like sessionId)
 */
-function maybeLoadFromExternalFileWithVars(config, vars) {
-	return maybeLoadFromExternalFile(require_render.renderVarsInObject(config, vars));
+function isRuntimeVar(key) {
+	return key.startsWith("_") || EXPLICIT_RUNTIME_VAR_KEYS.includes(key);
 }
 /**
-* Loads response_format configuration from an external file with variable rendering.
+* Filters out runtime-only variables that are added during evaluation
+* but aren't part of the original test definition.
 *
-* This function handles the special case where response_format may contain:
-* 1. A top-level file reference (file://format.json)
-* 2. A nested schema reference for json_schema type (schema: file://schema.json)
+* This is used when comparing test cases to determine if a result
+* corresponds to a particular test, regardless of runtime state.
 *
-* Both levels need variable rendering and file loading.
+* Runtime variables are identified by:
+* - Starting with underscore (e.g., _conversation, _metadata)
+* - Being in the explicit list (e.g., sessionId for backward compatibility)
+*/
+function filterRuntimeVars(vars) {
+	if (!vars || typeof vars !== "object" || Array.isArray(vars)) return vars;
+	const filtered = {};
+	for (const [key, value] of Object.entries(vars)) if (!isRuntimeVar(key)) filtered[key] = value;
+	return filtered;
+}
+/**
+* Extracts only runtime variables from a vars object.
+* This is the inverse of filterRuntimeVars.
 *
-* @param responseFormat - The response_format configuration
-* @param vars - Variables for template rendering
-* @returns The processed response_format with all files loaded
+* Used to restore runtime state when re-running filtered tests.
 */
-function maybeLoadResponseFormatFromExternalFile(responseFormat, vars) {
-	if (responseFormat === void 0 || responseFormat === null) return responseFormat;
-	const loaded = maybeLoadFromExternalFile(require_render.renderVarsInObject(responseFormat, vars));
-	if (!loaded || typeof loaded !== "object") return loaded;
-	if (loaded.type === "json_schema") {
-		const nestedSchema = loaded.schema || loaded.json_schema?.schema;
-		if (nestedSchema) {
-			const loadedSchema = maybeLoadFromExternalFile(require_render.renderVarsInObject(nestedSchema, vars));
-			if (loaded.schema !== void 0) return {
-				...loaded,
-				schema: loadedSchema
-			};
-			else if (loaded.json_schema?.schema !== void 0) return {
-				...loaded,
-				json_schema: {
-					...loaded.json_schema,
-					schema: loadedSchema
-				}
-			};
-		}
-	}
-	return loaded;
+function extractRuntimeVars(vars) {
+	if (!vars || typeof vars !== "object" || Array.isArray(vars)) return;
+	const extracted = {};
+	for (const [key, value] of Object.entries(vars)) if (isRuntimeVar(key)) extracted[key] = value;
+	return Object.keys(extracted).length > 0 ? extracted : void 0;
+}
+function varsMatch(vars1, vars2) {
+	return (0, fast_deep_equal.default)(vars1, vars2);
 }
 /**
-* Renders variables in a tools object and loads from external file if applicable.
-* This function combines renderVarsInObject and maybeLoadFromExternalFile into a single step
-* specifically for handling tools configurations.
+* Generate a unique key for a test case for deduplication purposes.
+* Excludes runtime variables and includes strategyId to distinguish tests
+* with the same prompt but different strategies.
 *
-* Supports loading from JSON, YAML, Python, and JavaScript files.
+* @param testCase - The test case to generate a key for
+* @returns A JSON string that uniquely identifies the test case
+*/
+function getTestCaseDeduplicationKey(testCase) {
+	const filteredVars = filterRuntimeVars(testCase.vars);
+	const strategyId = testCase.metadata?.strategyId || "none";
+	return JSON.stringify({
+		vars: filteredVars,
+		strategyId
+	});
+}
+/**
+* Deduplicates an array of test cases based on their vars and strategyId.
+* Tests with the same vars but different strategies are considered different.
+* Runtime variables (like _conversation, sessionId) are filtered out before comparison.
 *
-* @param tools - The tools configuration object or array to process.
-* @param vars - Variables to use for rendering.
-* @returns The processed tools configuration with variables rendered and content loaded from files if needed.
-* @throws {Error} If the loaded tools are in an invalid format
+* @param tests - Array of test cases to deduplicate
+* @returns Deduplicated array of test cases
 */
-async function maybeLoadToolsFromExternalFile(tools, vars) {
-	const rendered = require_render.renderVarsInObject(tools, vars);
-	if (typeof rendered === "string" && rendered.startsWith("file://")) {
-		const { filePath, functionName } = parseFileUrl(rendered);
-		if (functionName && (filePath.endsWith(".py") || require_fileExtensions.isJavascriptFile(filePath))) {
-			const fileType = filePath.endsWith(".py") ? "Python" : "JavaScript";
-			require_logger.logger.debug(`[maybeLoadToolsFromExternalFile] Loading tools from ${fileType} file: ${filePath}:${functionName}`);
-			try {
-				let toolDefinitions;
-				if (filePath.endsWith(".py")) {
-					const absPath = require_esm.safeResolve(require_logger.state.basePath || process.cwd(), filePath);
-					require_logger.logger.debug(`[maybeLoadToolsFromExternalFile] Resolved Python path: ${absPath}`);
-					toolDefinitions = await require_pythonUtils.runPython(absPath, functionName, []);
-				} else {
-					const absPath = require_esm.safeResolve(require_logger.state.basePath || process.cwd(), filePath);
-					require_logger.logger.debug(`[maybeLoadToolsFromExternalFile] Resolved JavaScript path: ${absPath}`);
-					const module = await require_esm.importModule(absPath);
-					const fn = module[functionName] || module.default?.[functionName];
-					if (typeof fn !== "function") {
-						const availableExports = Object.keys(module).filter((k) => k !== "default");
-						const basePath = require_logger.state.basePath || process.cwd();
-						throw new Error(`Function "${functionName}" not found in ${filePath}. Available exports: ${availableExports.length > 0 ? availableExports.join(", ") : "(none)"}\nResolved from: ${basePath}`);
-					}
-					toolDefinitions = await Promise.resolve(fn());
-				}
-				if (!toolDefinitions || typeof toolDefinitions === "string" || typeof toolDefinitions === "number" || typeof toolDefinitions === "boolean") throw new Error(`Function "${functionName}" must return an array or object of tool definitions, but returned: ${toolDefinitions === null ? "null" : typeof toolDefinitions}`);
-				require_logger.logger.debug(`[maybeLoadToolsFromExternalFile] Successfully loaded ${Array.isArray(toolDefinitions) ? toolDefinitions.length : "object"} tools`);
-				return toolDefinitions;
-			} catch (err) {
-				const errorMessage = err instanceof Error ? err.message : String(err);
-				const basePath = require_logger.state.basePath || process.cwd();
-				throw new Error(`Failed to load tools from ${rendered}:\n${errorMessage}\n\nMake sure the function "${functionName}" exists and returns a valid tool definition array.\nResolved from: ${basePath}`);
-			}
-		}
-		if (filePath.endsWith(".py") || require_fileExtensions.isJavascriptFile(filePath)) {
-			const ext = filePath.endsWith(".py") ? "Python" : "JavaScript";
-			const basePath = require_logger.state.basePath || process.cwd();
-			throw new Error(`Cannot load tools from ${rendered}\n${ext} files require a function name. Use this format:\n  tools: file://${filePath}:get_tools\n\nYour ${ext} file should export a function that returns tool definitions:\n` + (filePath.endsWith(".py") ? `  def get_tools():\n      return [{"type": "function", "function": {...}}]` : `  module.exports.get_tools = () => [{ type: "function", function: {...} }];`) + `\n\nResolved from: ${basePath}`);
-		}
-	}
-	if (Array.isArray(rendered)) {
-		const results = await Promise.all(rendered.map((item) => maybeLoadToolsFromExternalFile(item, vars)));
-		if (results.every((r) => Array.isArray(r))) return results.flat();
-		return results;
-	}
-	if (typeof rendered !== "string") return rendered;
-	const loaded = maybeLoadFromExternalFile(rendered);
-	if (loaded !== void 0 && loaded !== null && typeof loaded === "string") {
-		if (loaded.startsWith("file://")) throw new Error(`Failed to load tools from ${loaded}\nEnsure the file exists and contains valid JSON or YAML tool definitions.`);
-		if (loaded.includes("def ") || loaded.includes("import ")) throw new Error("Invalid tools configuration: file appears to contain Python code.\nPython files require a function name. Use this format:\n  tools: file://tools.py:get_tools");
-		throw new Error("Invalid tools configuration: expected an array or object, but got a string.\nIf using file://, ensure the file contains valid JSON or YAML tool definitions.");
+function deduplicateTestCases(tests) {
+	const seen = /* @__PURE__ */ new Set();
+	return tests.filter((test) => {
+		const key = getTestCaseDeduplicationKey(test);
+		if (seen.has(key)) return false;
+		seen.add(key);
+		return true;
+	});
+}
+function resultIsForTestCase(result, testCase) {
+	const testProviderId = testCase.provider ? providerToIdentifier(testCase.provider) : void 0;
+	const resultProviderId = providerToIdentifier(result.provider);
+	const providersMatch = !testProviderId || !resultProviderId || testProviderId === resultProviderId;
+	const resultVars = filterRuntimeVars(result.vars);
+	const testVars = filterRuntimeVars(testCase.vars);
+	const doVarsMatch = varsMatch(testVars, resultVars);
+	const isMatch = doVarsMatch && providersMatch;
+	if (!isMatch) {
+		const varKeys = testVars ? Object.keys(testVars).join(", ") : "none";
+		require_logger.logger.debug(`[resultIsForTestCase] No match: vars=${doVarsMatch}, providers=${providersMatch}`, {
+			testProvider: testProviderId || "none",
+			resultProvider: resultProviderId || "none",
+			testVarKeys: varKeys
+		});
 	}
-	return loaded;
+	return isMatch;
+}
+//#endregion
+//#region src/util/env.ts
+/**
+* Load environment variables from .env file(s).
+* @param envPath - Single path, array of paths, or undefined for default .env loading.
+*                  When paths are explicitly specified, all files must exist or an error is thrown.
+*                  When multiple files are provided, later files override values from earlier files.
+*/
+function setupEnv(envPath) {
+	if (envPath) {
+		const paths = (Array.isArray(envPath) ? envPath : [envPath]).flatMap((p) => p.includes(",") ? p.split(",").map((s) => s.trim()) : p.trim()).filter((p) => p.length > 0);
+		if (paths.length === 0) {
+			dotenv.default.config({ quiet: true });
+			return;
+		}
+		for (const p of paths) if (!fs.existsSync(p)) throw new Error(`Environment file not found: ${p}`);
+		if (paths.length === 1) require_logger.logger.info(`Loading environment variables from ${paths[0]}`);
+		else require_logger.logger.info(`Loading environment variables from: ${paths.join(", ")}`);
+		const pathArg = paths.length === 1 ? paths[0] : paths;
+		dotenv.default.config({
+			path: pathArg,
+			override: true,
+			quiet: true
+		});
+	} else dotenv.default.config({ quiet: true });
 }
 //#endregion
 //#region src/googleSheets.ts
@@ -1040,13 +1172,13 @@ function createOutputMetadata(evalRecord) {
 		evaluationCreatedAt = void 0;
 	}
 	return {
-		promptfooVersion: require_fetch.VERSION,
+		promptfooVersion: require_version.VERSION,
 		nodeVersion: process.version,
 		platform: os.platform(),
 		arch: os.arch(),
 		exportedAt: (/* @__PURE__ */ new Date()).toISOString(),
 		evaluationCreatedAt,
-		author: evalRecord.author
+		author: evalRecord.author ?? void 0
 	};
 }
 /**
@@ -1265,12 +1397,24 @@ Object.defineProperty(exports, "isProviderAllowed", {
 		return isProviderAllowed;
 	}
 });
+Object.defineProperty(exports, "isProviderConfigFileReference", {
+	enumerable: true,
+	get: function() {
+		return isProviderConfigFileReference;
+	}
+});
 Object.defineProperty(exports, "loadFunction", {
 	enumerable: true,
 	get: function() {
 		return loadFunction;
 	}
 });
+Object.defineProperty(exports, "loadProviderConfigsFromFile", {
+	enumerable: true,
+	get: function() {
+		return loadProviderConfigsFromFile;
+	}
+});
 Object.defineProperty(exports, "maybeLoadConfigFromExternalFile", {
 	enumerable: true,
 	get: function() {
@@ -1301,6 +1445,12 @@ Object.defineProperty(exports, "maybeLoadToolsFromExternalFile", {
 		return maybeLoadToolsFromExternalFile;
 	}
 });
+Object.defineProperty(exports, "normalizeProviderRef", {
+	enumerable: true,
+	get: function() {
+		return normalizeProviderRef;
+	}
+});
 Object.defineProperty(exports, "parseFileUrl", {
 	enumerable: true,
 	get: function() {
@@ -1331,6 +1481,12 @@ Object.defineProperty(exports, "readOutput", {
 		return readOutput;
 	}
 });
+Object.defineProperty(exports, "readProviderConfigFile", {
+	enumerable: true,
+	get: function() {
+		return readProviderConfigFile;
+	}
+});
 Object.defineProperty(exports, "resultIsForTestCase", {
 	enumerable: true,
 	get: function() {
@@ -1356,4 +1512,4 @@ Object.defineProperty(exports, "writeOutput", {
 	}
 });
 
-//# sourceMappingURL=util-Bxn8emtE.cjs.map
+//# sourceMappingURL=util-SPsvFONY.cjs.map