promptfoo 0.121.4 → 0.121.5

package/dist/src/{util-DxWpWjhc.js → util-BQOCAHQC.js}

@@ -1,14 +1,16 @@
 #!/usr/bin/env node
-import { N as state, T as getEnvBool, m as sanitizeObject, s as logger } from "./logger-DksKw1Qc.js";
-import { F as VERSION, k as TERMINAL_MAX_WIDTH, n as fetchWithProxy } from "./fetch-BEWnXrrG.js";
-import { t as invariant } from "./invariant-BtWWVVhl.js";
-import { m as isProviderOptions, o as OutputFileExtension, p as isApiProvider, s as ResultFailureReason } from "./types-Dm9JM6Vb.js";
-import { o as safeResolve, r as importModule, t as getDirectory } from "./esm-tVgYPY-f.js";
-import { a as getNunjucksEngine, n as renderVarsInObject } from "./render-CMEpfLaO.js";
-import { i as isJavascriptFile, t as JAVASCRIPT_EXTENSIONS } from "./fileExtensions-AWa2ZML4.js";
-import { r as runPython } from "./pythonUtils-CnndUbW-.js";
+import { N as state, T as getEnvBool, m as sanitizeObject, s as logger } from "./logger-BbY6ypFL.js";
+import { k as TERMINAL_MAX_WIDTH, n as fetchWithProxy } from "./fetch-B6ch2nU2.js";
+import { n as VERSION } from "./version-CbpiUINz.js";
+import { t as invariant } from "./invariant-B2Rf6avk.js";
+import { m as isProviderOptions, o as OutputFileExtension, p as isApiProvider, s as ResultFailureReason } from "./types-BVH9hjgW.js";
+import { o as safeResolve, r as importModule, t as getDirectory } from "./esm-BX8fwlAO.js";
+import { a as getNunjucksEngine, n as renderVarsInObject } from "./render-eui5p5mL.js";
+import { i as isJavascriptFile, t as JAVASCRIPT_EXTENSIONS } from "./fileExtensions-DysCsxNG.js";
+import { r as runPython } from "./pythonUtils-DNqbnRdx.js";
 import dotenv from "dotenv";
 import * as fs$2 from "fs";
+import fs from "fs";
 import * as path$1 from "path";
 import path from "path";
 import * as os$1 from "os";
@@ -21,670 +23,793 @@ import { parse as parse$1 } from "csv-parse/sync";
 import { globSync, hasMagic } from "glob";
 import { XMLBuilder } from "fast-xml-parser";
 import { stringify } from "csv-stringify/sync";
-//#region src/util/provider.ts
-function canonicalizeProviderId(id) {
-	if (id.startsWith("file://")) {
-		const filePath = id.slice(7);
-		return path$1.isAbsolute(filePath) ? id : `file://${path$1.resolve(filePath)}`;
-	}
-	for (const prefix of [
-		"exec:",
-		"python:",
-		"golang:"
-	]) if (id.startsWith(prefix)) {
-		const filePath = id.slice(prefix.length);
-		if (filePath.includes("/") || filePath.includes("\\")) return `${prefix}${path$1.resolve(filePath)}`;
-		return id;
-	}
-	if ((id.endsWith(".js") || id.endsWith(".ts") || id.endsWith(".mjs")) && (id.includes("/") || id.includes("\\"))) return `file://${path$1.resolve(id)}`;
-	return id;
-}
-function getProviderLabel(provider) {
-	return provider?.label && typeof provider.label === "string" ? provider.label : void 0;
-}
-function providerToIdentifier(provider) {
-	if (!provider) return;
-	if (typeof provider === "string") return canonicalizeProviderId(provider);
-	const label = getProviderLabel(provider);
-	if (label) return label;
-	if (isApiProvider(provider)) return canonicalizeProviderId(provider.id());
-	if (isProviderOptions(provider)) {
-		if (provider.id) return canonicalizeProviderId(provider.id);
-		return;
-	}
-	if (typeof provider === "object" && "id" in provider && typeof provider.id === "string") return canonicalizeProviderId(provider.id);
-}
-/**
-* Gets a descriptive identifier string for a provider, showing both label and ID when both exist.
-* Useful for error messages to help users debug provider reference issues.
-*/
-function getProviderDescription(provider) {
-	const label = provider.label;
-	const id = provider.id();
-	if (label && label !== id) return `${label} (${id})`;
-	return id;
-}
+//#region src/util/functions/loadFunction.ts
+const functionCache = {};
 /**
-* Checks if a provider reference matches a given provider.
-* Supports exact matching and wildcard patterns.
+* Loads a function from a JavaScript or Python file
+* @param options Options for loading the function
+* @returns The loaded function
 */
-function doesProviderRefMatch(ref, provider) {
-	const label = provider.label;
-	const id = provider.id();
-	const canonicalRef = canonicalizeProviderId(ref);
-	const canonicalId = canonicalizeProviderId(id);
-	if (label && label === ref) return true;
-	if (id === ref || canonicalId === canonicalRef) return true;
-	if (ref.endsWith("*")) {
-		const prefix = ref.slice(0, -1);
-		if (label?.startsWith(prefix) || id.startsWith(prefix) || canonicalId.startsWith(prefix)) return true;
+async function loadFunction({ filePath, functionName, defaultFunctionName = "func", basePath = state.basePath, useCache = true }) {
+	const cacheKey = `${filePath}${functionName ? `:${functionName}` : ""}`;
+	if (useCache && functionCache[cacheKey]) return functionCache[cacheKey];
+	const resolvedPath = basePath ? path.resolve(basePath, filePath) : filePath;
+	if (!isJavascriptFile(resolvedPath) && !resolvedPath.endsWith(".py")) throw new Error(`File must be a JavaScript (${JAVASCRIPT_EXTENSIONS.join(", ")}) or Python (.py) file`);
+	try {
+		let func;
+		if (isJavascriptFile(resolvedPath)) {
+			const module = await importModule(resolvedPath, functionName);
+			let moduleFunc;
+			if (functionName) moduleFunc = module;
+			else moduleFunc = typeof module === "function" ? module : module?.default?.default || module?.default || module?.[defaultFunctionName] || module;
+			if (typeof moduleFunc !== "function") throw new Error(functionName ? `JavaScript file must export a "${functionName}" function` : `JavaScript file must export a function (as default export or named export "${defaultFunctionName}")`);
+			func = moduleFunc;
+		} else {
+			const result = (...args) => runPython(resolvedPath, functionName || defaultFunctionName, args);
+			func = result;
+		}
+		if (useCache) functionCache[cacheKey] = func;
+		return func;
+	} catch (err) {
+		logger.error(`Failed to load function: ${err.message}`);
+		throw err;
 	}
-	if (label?.startsWith(`${ref}:`) || id.startsWith(`${ref}:`) || canonicalId.startsWith(`${ref}:`)) return true;
-	return false;
 }
 /**
-* Checks if a provider is allowed based on a list of allowed references.
+* Extracts the file path and function name from a file:// URL
+* @param fileUrl The file:// URL (e.g., "file://path/to/file.js:functionName")
+* @returns The file path and optional function name
 */
-function isProviderAllowed(provider, allowedProviders) {
-	if (!Array.isArray(allowedProviders)) return true;
-	if (allowedProviders.length === 0) return false;
-	return allowedProviders.some((ref) => doesProviderRefMatch(ref, provider));
+function parseFileUrl(fileUrl) {
+	if (!fileUrl.startsWith("file://")) throw new Error("URL must start with file://");
+	const urlWithoutProtocol = fileUrl.slice(7);
+	const lastColonIndex = urlWithoutProtocol.lastIndexOf(":");
+	if (lastColonIndex > 1) return {
+		filePath: urlWithoutProtocol.slice(0, lastColonIndex),
+		functionName: urlWithoutProtocol.slice(lastColonIndex + 1)
+	};
+	return { filePath: urlWithoutProtocol };
 }
+//#endregion
+//#region src/util/file.ts
 /**
-* Detects if a provider uses OpenAI models.
-* This includes direct OpenAI providers and Azure OpenAI.
+* Simple Nunjucks engine specifically for file paths
+* This function is separate from the main getNunjucksEngine to avoid circular dependencies
 */
-function isOpenAiProvider(providerId) {
-	const lowerProviderId = providerId.toLowerCase();
-	if (lowerProviderId.startsWith("openai:")) return true;
-	if (lowerProviderId.startsWith("azureopenai:")) return true;
-	if (lowerProviderId.startsWith("azure:")) {
-		if ([
-			"gpt",
-			"openai",
-			"davinci",
-			"curie",
-			"babbage",
-			"ada",
-			"text-embedding",
-			"whisper",
-			"dall-e",
-			"tts"
-		].some((indicator) => lowerProviderId.includes(indicator))) return true;
-	}
-	return false;
+function getNunjucksEngineForFilePath() {
+	const env = nunjucks.configure({ autoescape: false });
+	env.addGlobal("env", {
+		...process.env,
+		...state.config?.env
+	});
+	return env;
 }
 /**
-* Detects if a provider uses Anthropic/Claude models.
-* This includes direct Anthropic providers, Bedrock with Claude, and Vertex with Claude.
+* Loads content from an external file if the input is a file path, otherwise
+* returns the input as-is. Supports Nunjucks templating for file paths.
+*
+* @param filePath - The input to process. Can be a file path string starting with "file://",
+* an array of file paths, or any other type of data.
+* @param context - Optional context to control file loading behavior. 'assertion' context
+* preserves Python/JS file references instead of loading their content.
+* @returns The loaded content if the input was a file path, otherwise the original input.
+* For JSON and YAML files, the content is parsed into an object.
+* For other file types, the raw file content is returned as a string.
+*
+* @throws {Error} If the specified file does not exist.
 */
-function isAnthropicProvider(providerId) {
-	const lowerProviderId = providerId.toLowerCase();
-	if (lowerProviderId.startsWith("anthropic:")) return true;
-	if (lowerProviderId.startsWith("bedrock:")) {
-		if (lowerProviderId.includes("claude") || lowerProviderId.includes("anthropic")) return true;
+function maybeLoadFromExternalFile(filePath, context) {
+	if (Array.isArray(filePath)) return filePath.map((path) => {
+		return maybeLoadFromExternalFile(path, context);
+	});
+	if (typeof filePath !== "string") return filePath;
+	if (!filePath.startsWith("file://")) return filePath;
+	const renderedFilePath = getNunjucksEngineForFilePath().renderString(filePath, {});
+	const { filePath: cleanPath, functionName } = parseFileUrl(renderedFilePath);
+	if (context === "assertion" && (cleanPath.endsWith(".py") || isJavascriptFile(cleanPath))) {
+		logger.debug(`Preserving Python/JS file reference in assertion context: ${renderedFilePath}`);
+		return renderedFilePath;
 	}
-	if (lowerProviderId.startsWith("vertex:")) {
-		if (lowerProviderId.includes("claude")) return true;
+	if (context === "vars") {
+		logger.debug(`Preserving file reference in vars context: ${renderedFilePath}`);
+		return renderedFilePath;
 	}
-	return false;
-}
-const KNOWN_ENV_VARS = {
-	openai: "OPENAI_API_KEY",
-	anthropic: "ANTHROPIC_API_KEY",
-	google: "GOOGLE_API_KEY",
-	mistral: "MISTRAL_API_KEY",
-	cohere: "COHERE_API_KEY",
-	replicate: "REPLICATE_API_TOKEN",
-	voyage: "VOYAGE_API_KEY",
-	ai21: "AI21_API_KEY",
-	xai: "XAI_API_KEY",
-	groq: "GROQ_API_KEY",
-	deepseek: "DEEPSEEK_API_KEY",
-	perplexity: "PERPLEXITY_API_KEY",
-	hyperbolic: "HYPERBOLIC_API_KEY",
-	cerebras: "CEREBRAS_API_KEY",
-	togetherai: "TOGETHER_API_KEY",
-	fal: "FAL_KEY",
-	huggingface: "HF_TOKEN",
-	"cloudflare-ai": "CLOUDFLARE_API_KEY"
-};
-function getDefaultEnvVar(providerId) {
-	const prefix = providerId.split(":")[0];
-	return KNOWN_ENV_VARS[prefix] || `${prefix.toUpperCase()}_API_KEY`;
-}
-/**
-* Pre-checks providers for missing API keys before evaluation starts.
-* Assumes getApiKey() is side-effect free (no network calls or token refresh).
-*/
-function checkProviderApiKeys(providers) {
-	const missingApiKeys = /* @__PURE__ */ new Map();
-	for (const provider of providers) {
-		const p = provider;
-		if (typeof p.getApiKey !== "function") continue;
-		if (provider.id().startsWith("azure:")) continue;
-		const requiresKey = typeof p.requiresApiKey === "function" ? p.requiresApiKey() : p.config?.apiKeyRequired !== false;
-		let apiKey;
-		try {
-			apiKey = p.getApiKey();
-		} catch {
-			apiKey = void 0;
-		}
-		if (requiresKey && !apiKey) {
-			const envVar = p.config?.apiKeyEnvar || getDefaultEnvVar(provider.id());
-			if (!missingApiKeys.has(envVar)) missingApiKeys.set(envVar, []);
-			missingApiKeys.get(envVar).push(provider.id());
+	if (functionName && (cleanPath.endsWith(".py") || isJavascriptFile(cleanPath))) return renderedFilePath;
+	const pathToUse = functionName && !(cleanPath.endsWith(".py") || isJavascriptFile(cleanPath)) ? renderedFilePath.slice(7) : cleanPath;
+	const resolvedPath = path$1.resolve(state.basePath || "", pathToUse);
+	if (hasMagic(pathToUse)) {
+		const matchedFiles = globSync(resolvedPath, { windowsPathsNoEscape: true });
+		if (matchedFiles.length === 0) throw new Error(`No files found matching pattern: ${resolvedPath}`);
+		const allContents = [];
+		for (const matchedFile of matchedFiles) {
+			let contents;
+			try {
+				contents = fs$2.readFileSync(matchedFile, "utf8");
+			} catch (error) {
+				if (error.code === "ENOENT") {
+					logger.debug(`File disappeared during glob expansion: ${matchedFile}`);
+					continue;
+				}
+				throw error;
+			}
+			if (matchedFile.endsWith(".json")) {
+				const parsed = JSON.parse(contents);
+				if (Array.isArray(parsed)) allContents.push(...parsed);
+				else allContents.push(parsed);
+			} else if (matchedFile.endsWith(".yaml") || matchedFile.endsWith(".yml")) {
+				const parsed = yaml.load(contents);
+				if (parsed === null || parsed === void 0) continue;
+				if (Array.isArray(parsed)) allContents.push(...parsed);
+				else allContents.push(parsed);
+			} else if (matchedFile.endsWith(".csv")) {
+				const records = parse$1(contents, { columns: true });
+				if (records.length > 0 && Object.keys(records[0]).length === 1) allContents.push(...records.map((record) => Object.values(record)[0]));
+				else allContents.push(...records);
+			} else allContents.push(contents);
 		}
+		return allContents;
 	}
-	return missingApiKeys;
-}
-/**
-* Detects if a provider uses Google models.
-* This includes direct Google/Vertex providers with Gemini and other Google models.
-* Note: Vertex with Claude models is NOT counted as Google (it's Anthropic).
-*/
-function isGoogleProvider(providerId) {
-	const lowerProviderId = providerId.toLowerCase();
-	if (lowerProviderId.startsWith("google:")) return true;
-	if (lowerProviderId.startsWith("vertex:")) {
-		if (!lowerProviderId.includes("claude")) return true;
+	const finalPath = resolvedPath;
+	let contents;
+	try {
+		contents = fs$2.readFileSync(finalPath, "utf8");
+	} catch (error) {
+		if (error.code === "ENOENT") throw new Error(`File does not exist: ${finalPath}`);
+		throw new Error(`Failed to read file ${finalPath}: ${error}`);
 	}
-	return false;
+	if (finalPath.endsWith(".json")) try {
+		return JSON.parse(contents);
+	} catch (error) {
+		throw new Error(`Failed to parse JSON file ${finalPath}: ${error}`);
+	}
+	if (finalPath.endsWith(".yaml") || finalPath.endsWith(".yml")) try {
+		return yaml.load(contents);
+	} catch (error) {
+		throw new Error(`Failed to parse YAML file ${finalPath}: ${error}`);
+	}
+	if (finalPath.endsWith(".csv")) {
+		const records = parse$1(contents, { columns: true });
+		if (records.length > 0 && Object.keys(records[0]).length === 1) return records.map((record) => Object.values(record)[0]);
+		return records;
+	}
+	return contents;
 }
-//#endregion
-//#region src/util/comparison.ts
 /**
-* Explicit runtime variable names that don't follow the underscore convention.
-* These are added during evaluation but aren't part of the original test definition.
-*
-* - sessionId: Added by multi-turn strategy providers (GOAT, Crescendo)
-*
-* Note: Variables starting with underscore (e.g., _conversation) are automatically
-* treated as runtime variables and filtered out.
-*/
-const EXPLICIT_RUNTIME_VAR_KEYS = ["sessionId"];
-/**
-* Checks if a variable key is a runtime-only variable that should be filtered
-* when comparing test cases.
+* Resolves a relative file path with respect to a base path, handling cloud configuration appropriately.
+* When using a cloud configuration, the current working directory is always used instead of the context's base path.
 *
-* Runtime variables are identified by:
-* 1. Starting with underscore (_) - convention for internal/runtime vars
-* 2. Being in the explicit runtime var list (for legacy vars like sessionId)
+* @param filePath - The relative or absolute file path to resolve.
+* @param isCloudConfig - Whether this is a cloud configuration.
+* @returns The resolved absolute file path.
 */
-function isRuntimeVar(key) {
-	return key.startsWith("_") || EXPLICIT_RUNTIME_VAR_KEYS.includes(key);
+function getResolvedRelativePath(filePath, isCloudConfig) {
+	if (path$1.isAbsolute(filePath) || !isCloudConfig) return filePath;
+	return path$1.join(process.cwd(), filePath);
 }
 /**
-* Filters out runtime-only variables that are added during evaluation
-* but aren't part of the original test definition.
-*
-* This is used when comparing test cases to determine if a result
-* corresponds to a particular test, regardless of runtime state.
+* Recursively loads external file references from a configuration object.
 *
-* Runtime variables are identified by:
-* - Starting with underscore (e.g., _conversation, _metadata)
-* - Being in the explicit list (e.g., sessionId for backward compatibility)
+* @param config - The configuration object to process
+* @param context - Optional context to control file loading behavior
+* @returns The configuration with external file references resolved
 */
-function filterRuntimeVars(vars) {
-	if (!vars || typeof vars !== "object" || Array.isArray(vars)) return vars;
-	const filtered = {};
-	for (const [key, value] of Object.entries(vars)) if (!isRuntimeVar(key)) filtered[key] = value;
-	return filtered;
+function maybeLoadConfigFromExternalFile(config, context) {
+	if (Array.isArray(config)) return config.map((item) => maybeLoadConfigFromExternalFile(item, context));
+	if (typeof config === "object" && config !== null) {
+		const result = {};
+		for (const key of Object.keys(config)) {
+			const childContext = key === "value" && "type" in config && typeof config.type === "string" && (config.type === "python" || config.type === "javascript") ? "assertion" : key === "vars" ? "vars" : context;
+			result[key] = maybeLoadConfigFromExternalFile(config[key], childContext);
+		}
+		return result;
+	}
+	return maybeLoadFromExternalFile(config, context);
 }
 /**
-* Extracts only runtime variables from a vars object.
-* This is the inverse of filterRuntimeVars.
-*
-* Used to restore runtime state when re-running filtered tests.
+* Parses a file path or glob pattern to extract function names and file extensions.
+* Function names can be specified in the filename like this:
+* prompt.py:myFunction or prompts.js:myFunction.
+* @param basePath - The base path for file resolution.
+* @param promptPath - The path or glob pattern.
+* @returns Parsed details including function name, file extension, and directory status.
 */
-function extractRuntimeVars(vars) {
-	if (!vars || typeof vars !== "object" || Array.isArray(vars)) return;
-	const extracted = {};
-	for (const [key, value] of Object.entries(vars)) if (isRuntimeVar(key)) extracted[key] = value;
-	return Object.keys(extracted).length > 0 ? extracted : void 0;
+function parsePathOrGlob(basePath, promptPath) {
+	if (promptPath.startsWith("file://")) promptPath = promptPath.slice(7);
+	const filePath = path$1.resolve(basePath, promptPath);
+	let filename = path$1.relative(basePath, filePath);
+	let functionName;
+	if (filename.includes(":")) {
+		const lastColonIndex = filename.lastIndexOf(":");
+		if (lastColonIndex > 1) {
+			const pathWithoutFunction = filename.slice(0, lastColonIndex);
+			if (isJavascriptFile(pathWithoutFunction) || pathWithoutFunction.endsWith(".py") || pathWithoutFunction.endsWith(".go") || pathWithoutFunction.endsWith(".rb")) {
+				functionName = filename.slice(lastColonIndex + 1);
+				filename = pathWithoutFunction;
+			}
+		}
+	}
+	let stats;
+	try {
+		stats = fs$2.statSync(path$1.join(basePath, filename));
+	} catch (err) {
+		if (getEnvBool("PROMPTFOO_STRICT_FILES")) throw err;
+	}
+	const normalizedFilePath = filePath.replace(/\\/g, "/");
+	const isPathPattern = stats?.isDirectory() || hasMagic(promptPath) || hasMagic(normalizedFilePath);
+	const safeFilename = path$1.relative(basePath, safeResolve(basePath, filename));
+	return {
+		extension: isPathPattern ? void 0 : path$1.parse(safeFilename).ext,
+		filePath: path$1.join(basePath, safeFilename),
+		functionName,
+		isPathPattern
+	};
+}
+function readOutput(outputPath) {
+	const ext = path$1.parse(outputPath).ext.slice(1);
+	switch (ext) {
+		case "json": return JSON.parse(fs$2.readFileSync(outputPath, "utf-8"));
+		default: throw new Error(`Unsupported output file format: ${ext} currently only supports json`);
+	}
 }
-function varsMatch(vars1, vars2) {
-	return deepEqual(vars1, vars2);
+/**
+* Load custom Nunjucks filters from external files.
+* Note: If a glob pattern matches multiple files, only the last file's export is used.
+* Each filter name should typically resolve to a single file.
+*/
+async function readFilters(filters, basePath = "") {
+	const ret = {};
+	for (const [name, filterPath] of Object.entries(filters)) {
+		const filePaths = globSync(path$1.join(basePath, filterPath), { windowsPathsNoEscape: true });
+		for (const filePath of filePaths) ret[name] = await importModule(path$1.resolve(filePath));
+	}
+	return ret;
 }
 /**
-* Generate a unique key for a test case for deduplication purposes.
-* Excludes runtime variables and includes strategyId to distinguish tests
-* with the same prompt but different strategies.
+* Loads configuration from an external file with variable rendering.
+* This is a convenience wrapper that combines renderVarsInObject and maybeLoadFromExternalFile.
 *
-* @param testCase - The test case to generate a key for
-* @returns A JSON string that uniquely identifies the test case
+* Use this for simple config fields that:
+* - Need variable rendering ({{ vars.x }}, {{ env.X }})
+* - May reference external files (file://path.json)
+* - Don't have nested file references that need loading
+*
+* For fields with nested file references (like response_format.schema),
+* use maybeLoadResponseFormatFromExternalFile instead.
+*
+* @param config - The configuration to process
+* @param vars - Variables for template rendering
+* @returns The processed configuration with variables rendered and files loaded
 */
-function getTestCaseDeduplicationKey(testCase) {
-	const filteredVars = filterRuntimeVars(testCase.vars);
-	const strategyId = testCase.metadata?.strategyId || "none";
-	return JSON.stringify({
-		vars: filteredVars,
-		strategyId
-	});
+function maybeLoadFromExternalFileWithVars(config, vars) {
+	return maybeLoadFromExternalFile(renderVarsInObject(config, vars));
 }
 /**
-* Deduplicates an array of test cases based on their vars and strategyId.
-* Tests with the same vars but different strategies are considered different.
-* Runtime variables (like _conversation, sessionId) are filtered out before comparison.
+* Loads response_format configuration from an external file with variable rendering.
 *
-* @param tests - Array of test cases to deduplicate
-* @returns Deduplicated array of test cases
+* This function handles the special case where response_format may contain:
+* 1. A top-level file reference (file://format.json)
+* 2. A nested schema reference for json_schema type (schema: file://schema.json)
+*
+* Both levels need variable rendering and file loading.
+*
+* @param responseFormat - The response_format configuration
+* @param vars - Variables for template rendering
+* @returns The processed response_format with all files loaded
 */
-function deduplicateTestCases(tests) {
-	const seen = /* @__PURE__ */ new Set();
-	return tests.filter((test) => {
-		const key = getTestCaseDeduplicationKey(test);
-		if (seen.has(key)) return false;
-		seen.add(key);
-		return true;
-	});
-}
-function resultIsForTestCase(result, testCase) {
-	const testProviderId = testCase.provider ? providerToIdentifier(testCase.provider) : void 0;
-	const resultProviderId = providerToIdentifier(result.provider);
-	const providersMatch = !testProviderId || !resultProviderId || testProviderId === resultProviderId;
-	const resultVars = filterRuntimeVars(result.vars);
-	const testVars = filterRuntimeVars(testCase.vars);
-	const doVarsMatch = varsMatch(testVars, resultVars);
-	const isMatch = doVarsMatch && providersMatch;
-	if (!isMatch) {
-		const varKeys = testVars ? Object.keys(testVars).join(", ") : "none";
-		logger.debug(`[resultIsForTestCase] No match: vars=${doVarsMatch}, providers=${providersMatch}`, {
-			testProvider: testProviderId || "none",
-			resultProvider: resultProviderId || "none",
-			testVarKeys: varKeys
-		});
+function maybeLoadResponseFormatFromExternalFile(responseFormat, vars) {
+	if (responseFormat === void 0 || responseFormat === null) return responseFormat;
+	const loaded = maybeLoadFromExternalFile(renderVarsInObject(responseFormat, vars));
+	if (!loaded || typeof loaded !== "object") return loaded;
+	if (loaded.type === "json_schema") {
+		const nestedSchema = loaded.schema || loaded.json_schema?.schema;
+		if (nestedSchema) {
+			const loadedSchema = maybeLoadFromExternalFile(renderVarsInObject(nestedSchema, vars));
+			if (loaded.schema !== void 0) return {
+				...loaded,
+				schema: loadedSchema
+			};
+			else if (loaded.json_schema?.schema !== void 0) return {
+				...loaded,
+				json_schema: {
+					...loaded.json_schema,
+					schema: loadedSchema
+				}
+			};
+		}
 	}
-	return isMatch;
+	return loaded;
 }
-//#endregion
-//#region src/util/env.ts
 /**
-* Load environment variables from .env file(s).
-* @param envPath - Single path, array of paths, or undefined for default .env loading.
-*                  When paths are explicitly specified, all files must exist or an error is thrown.
-*                  When multiple files are provided, later files override values from earlier files.
+* Renders variables in a tools object and loads from external file if applicable.
+* This function combines renderVarsInObject and maybeLoadFromExternalFile into a single step
+* specifically for handling tools configurations.
+*
+* Supports loading from JSON, YAML, Python, and JavaScript files.
+*
+* @param tools - The tools configuration object or array to process.
+* @param vars - Variables to use for rendering.
+* @returns The processed tools configuration with variables rendered and content loaded from files if needed.
+* @throws {Error} If the loaded tools are in an invalid format
 */
-function setupEnv(envPath) {
-	if (envPath) {
-		const paths = (Array.isArray(envPath) ? envPath : [envPath]).flatMap((p) => p.includes(",") ? p.split(",").map((s) => s.trim()) : p.trim()).filter((p) => p.length > 0);
-		if (paths.length === 0) {
-			dotenv.config({ quiet: true });
-			return;
+async function maybeLoadToolsFromExternalFile(tools, vars) {
+	const rendered = renderVarsInObject(tools, vars);
+	if (typeof rendered === "string" && rendered.startsWith("file://")) {
+		const { filePath, functionName } = parseFileUrl(rendered);
+		if (functionName && (filePath.endsWith(".py") || isJavascriptFile(filePath))) {
+			const fileType = filePath.endsWith(".py") ? "Python" : "JavaScript";
+			logger.debug(`[maybeLoadToolsFromExternalFile] Loading tools from ${fileType} file: ${filePath}:${functionName}`);
+			try {
+				let toolDefinitions;
+				if (filePath.endsWith(".py")) {
+					const absPath = safeResolve(state.basePath || process.cwd(), filePath);
+					logger.debug(`[maybeLoadToolsFromExternalFile] Resolved Python path: ${absPath}`);
+					toolDefinitions = await runPython(absPath, functionName, []);
+				} else {
+					const absPath = safeResolve(state.basePath || process.cwd(), filePath);
+					logger.debug(`[maybeLoadToolsFromExternalFile] Resolved JavaScript path: ${absPath}`);
+					const module = await importModule(absPath);
+					const fn = module[functionName] || module.default?.[functionName];
+					if (typeof fn !== "function") {
+						const availableExports = Object.keys(module).filter((k) => k !== "default");
+						const basePath = state.basePath || process.cwd();
+						throw new Error(`Function "${functionName}" not found in ${filePath}. Available exports: ${availableExports.length > 0 ? availableExports.join(", ") : "(none)"}\nResolved from: ${basePath}`);
+					}
+					toolDefinitions = await Promise.resolve(fn());
+				}
+				if (!toolDefinitions || typeof toolDefinitions === "string" || typeof toolDefinitions === "number" || typeof toolDefinitions === "boolean") throw new Error(`Function "${functionName}" must return an array or object of tool definitions, but returned: ${toolDefinitions === null ? "null" : typeof toolDefinitions}`);
+				logger.debug(`[maybeLoadToolsFromExternalFile] Successfully loaded ${Array.isArray(toolDefinitions) ? toolDefinitions.length : "object"} tools`);
+				return toolDefinitions;
+			} catch (err) {
+				const errorMessage = err instanceof Error ? err.message : String(err);
+				const basePath = state.basePath || process.cwd();
+				throw new Error(`Failed to load tools from ${rendered}:\n${errorMessage}\n\nMake sure the function "${functionName}" exists and returns a valid tool definition array.\nResolved from: ${basePath}`);
+			}
 		}
-		for (const p of paths) if (!fs$2.existsSync(p)) throw new Error(`Environment file not found: ${p}`);
-		if (paths.length === 1) logger.info(`Loading environment variables from ${paths[0]}`);
-		else logger.info(`Loading environment variables from: ${paths.join(", ")}`);
-		const pathArg = paths.length === 1 ? paths[0] : paths;
-		dotenv.config({
-			path: pathArg,
-			override: true,
-			quiet: true
-		});
-	} else dotenv.config({ quiet: true });
+		if (filePath.endsWith(".py") || isJavascriptFile(filePath)) {
+			const ext = filePath.endsWith(".py") ? "Python" : "JavaScript";
+			const basePath = state.basePath || process.cwd();
+			throw new Error(`Cannot load tools from ${rendered}\n${ext} files require a function name. Use this format:\n  tools: file://${filePath}:get_tools\n\nYour ${ext} file should export a function that returns tool definitions:\n` + (filePath.endsWith(".py") ? `  def get_tools():\n      return [{"type": "function", "function": {...}}]` : `  module.exports.get_tools = () => [{ type: "function", function: {...} }];`) + `\n\nResolved from: ${basePath}`);
+		}
+	}
+	if (Array.isArray(rendered)) {
+		const results = await Promise.all(rendered.map((item) => maybeLoadToolsFromExternalFile(item, vars)));
+		if (results.every((r) => Array.isArray(r))) return results.flat();
+		return results;
+	}
+	if (typeof rendered !== "string") return rendered;
+	const loaded = maybeLoadFromExternalFile(rendered);
+	if (loaded !== void 0 && loaded !== null && typeof loaded === "string") {
+		if (loaded.startsWith("file://")) throw new Error(`Failed to load tools from ${loaded}\nEnsure the file exists and contains valid JSON or YAML tool definitions.`);
+		if (loaded.includes("def ") || loaded.includes("import ")) throw new Error("Invalid tools configuration: file appears to contain Python code.\nPython files require a function name. Use this format:\n  tools: file://tools.py:get_tools");
+		throw new Error("Invalid tools configuration: expected an array or object, but got a string.\nIf using file://, ensure the file contains valid JSON or YAML tool definitions.");
+	}
+	return loaded;
 }
 //#endregion
-//#region src/util/functions/loadFunction.ts
-const functionCache = {};
+//#region src/util/providerRef.ts
+const PROVIDER_OPTION_KEYS = new Set([
+	"id",
+	"label",
+	"config",
+	"prompts",
+	"transform",
+	"delay",
+	"env",
+	"inputs"
+]);
+/** Returns true if the value is a non-empty string suitable as a provider identifier. */
+function isValidProviderId(id) {
+	return typeof id === "string" && id !== "";
+}
+function getProviderLabel(provider) {
+	if ((typeof provider === "object" || typeof provider === "function") && provider !== null && "label" in provider && typeof provider.label === "string") return provider.label;
+}
 /**
-* Loads a function from a JavaScript or Python file
-* @param options Options for loading the function
-* @returns The loaded function
+* Resolves relative file paths in provider IDs to absolute paths for consistent matching.
+* Handles file://, exec:, python:, golang: prefixes and bare .js/.ts/.mjs paths.
 */
-async function loadFunction({ filePath, functionName, defaultFunctionName = "func", basePath = state.basePath, useCache = true }) {
-	const cacheKey = `${filePath}${functionName ? `:${functionName}` : ""}`;
-	if (useCache && functionCache[cacheKey]) return functionCache[cacheKey];
-	const resolvedPath = basePath ? path.resolve(basePath, filePath) : filePath;
-	if (!isJavascriptFile(resolvedPath) && !resolvedPath.endsWith(".py")) throw new Error(`File must be a JavaScript (${JAVASCRIPT_EXTENSIONS.join(", ")}) or Python (.py) file`);
-	try {
-		let func;
-		if (isJavascriptFile(resolvedPath)) {
-			const module = await importModule(resolvedPath, functionName);
-			let moduleFunc;
-			if (functionName) moduleFunc = module;
-			else moduleFunc = typeof module === "function" ? module : module?.default?.default || module?.default || module?.[defaultFunctionName] || module;
-			if (typeof moduleFunc !== "function") throw new Error(functionName ? `JavaScript file must export a "${functionName}" function` : `JavaScript file must export a function (as default export or named export "${defaultFunctionName}")`);
-			func = moduleFunc;
-		} else {
-			const result = (...args) => runPython(resolvedPath, functionName || defaultFunctionName, args);
-			func = result;
-		}
-		if (useCache) functionCache[cacheKey] = func;
-		return func;
-	} catch (err) {
-		logger.error(`Failed to load function: ${err.message}`);
-		throw err;
+function canonicalizeProviderId(id) {
+	if (id.startsWith("file://")) {
+		const filePath = id.slice(7);
+		return path.isAbsolute(filePath) ? id : `file://${path.resolve(filePath)}`;
+	}
+	for (const prefix of [
+		"exec:",
+		"python:",
+		"golang:"
+	]) if (id.startsWith(prefix)) {
+		const filePath = id.slice(prefix.length);
+		if (filePath.includes("/") || filePath.includes("\\")) return `${prefix}${path.resolve(filePath)}`;
+		return id;
 	}
+	if ((id.endsWith(".js") || id.endsWith(".ts") || id.endsWith(".mjs")) && (id.includes("/") || id.includes("\\"))) return `file://${path.resolve(id)}`;
+	return id;
 }
 /**
-* Extracts the file path and function name from a file:// URL
-* @param fileUrl The file:// URL (e.g., "file://path/to/file.js:functionName")
-* @returns The file path and optional function name
+* Returns true for provider refs that should be expanded from YAML/JSON config files.
 */
-function parseFileUrl(fileUrl) {
-	if (!fileUrl.startsWith("file://")) throw new Error("URL must start with file://");
-	const urlWithoutProtocol = fileUrl.slice(7);
-	const lastColonIndex = urlWithoutProtocol.lastIndexOf(":");
-	if (lastColonIndex > 1) return {
-		filePath: urlWithoutProtocol.slice(0, lastColonIndex),
-		functionName: urlWithoutProtocol.slice(lastColonIndex + 1)
-	};
-	return { filePath: urlWithoutProtocol };
+function isProviderConfigFileReference(providerPath) {
+	return providerPath.startsWith("file://") && (providerPath.endsWith(".yaml") || providerPath.endsWith(".yml") || providerPath.endsWith(".json"));
 }
-//#endregion
-//#region src/util/file.ts
 /**
-* Simple Nunjucks engine specifically for file paths
-* This function is separate from the main getNunjucksEngine to avoid circular dependencies
+* Reads a provider config file and normalizes single-provider and multi-provider files.
+* Returns a `wasArray` flag so callers can detect multi-provider files that require
+* `loadApiProviders` instead of `loadApiProvider`.
 */
-function getNunjucksEngineForFilePath() {
-	const env = nunjucks.configure({ autoescape: false });
-	env.addGlobal("env", {
-		...process.env,
-		...state.config?.env
-	});
-	return env;
+function readProviderConfigFile(providerPath, basePath) {
+	const relativePath = providerPath.slice(7);
+	const resolvedPath = path.isAbsolute(relativePath) ? relativePath : path.join(basePath || process.cwd(), relativePath);
+	let rawContent;
+	try {
+		rawContent = yaml.load(fs.readFileSync(resolvedPath, "utf8"));
+	} catch (err) {
+		throw new Error(`Failed to load provider config ${relativePath}: ${err instanceof Error ? err.message : err}`);
+	}
+	const fileContent = maybeLoadConfigFromExternalFile(rawContent);
+	invariant(fileContent, `Provider config ${relativePath} is undefined`);
+	return {
+		configs: [fileContent].flat(),
+		relativePath,
+		wasArray: Array.isArray(fileContent)
+	};
 }
 /**
-* Loads content from an external file if the input is a file path, otherwise
-* returns the input as-is. Supports Nunjucks templating for file paths.
-*
-* @param filePath - The input to process. Can be a file path string starting with "file://",
-* an array of file paths, or any other type of data.
-* @param context - Optional context to control file loading behavior. 'assertion' context
-* preserves Python/JS file references instead of loading their content.
-* @returns The loaded content if the input was a file path, otherwise the original input.
-* For JSON and YAML files, the content is parsed into an object.
-* For other file types, the raw file content is returned as a string.
-*
-* @throws {Error} If the specified file does not exist.
+* Loads provider config objects from a file-backed provider reference.
 */
-function maybeLoadFromExternalFile(filePath, context) {
-	if (Array.isArray(filePath)) return filePath.map((path) => {
-		return maybeLoadFromExternalFile(path, context);
-	});
-	if (typeof filePath !== "string") return filePath;
-	if (!filePath.startsWith("file://")) return filePath;
-	const renderedFilePath = getNunjucksEngineForFilePath().renderString(filePath, {});
-	const { filePath: cleanPath, functionName } = parseFileUrl(renderedFilePath);
-	if (context === "assertion" && (cleanPath.endsWith(".py") || isJavascriptFile(cleanPath))) {
-		logger.debug(`Preserving Python/JS file reference in assertion context: ${renderedFilePath}`);
-		return renderedFilePath;
+function loadProviderConfigsFromFile(providerPath, basePath) {
+	return readProviderConfigFile(providerPath, basePath).configs;
+}
+/**
+* Pure, synchronous classifier that converts every supported provider reference shape
+* into a discriminated descriptor. Does not read files or instantiate providers.
+*/
+function normalizeProviderRef(provider, options = {}) {
+	const { index } = options;
+	if (typeof provider === "string") {
+		if (!isValidProviderId(provider)) return {
+			kind: "unknown",
+			id: index === void 0 ? "unknown" : `unknown-${index}`
+		};
+		if (isProviderConfigFileReference(provider)) return {
+			kind: "file",
+			id: provider,
+			loadProviderPath: provider
+		};
+		return {
+			kind: "named",
+			id: provider,
+			loadProviderPath: provider
+		};
 	}
-	if (context === "vars") {
-		logger.debug(`Preserving file reference in vars context: ${renderedFilePath}`);
-		return renderedFilePath;
+	if (typeof provider === "function") {
+		const label = getProviderLabel(provider);
+		return {
+			kind: "function",
+			id: label ?? (index === void 0 ? "custom-function" : `custom-function-${index}`),
+			label
+		};
 	}
-	if (functionName && (cleanPath.endsWith(".py") || isJavascriptFile(cleanPath))) return renderedFilePath;
-	const pathToUse = functionName && !(cleanPath.endsWith(".py") || isJavascriptFile(cleanPath)) ? renderedFilePath.slice(7) : cleanPath;
-	const resolvedPath = path$1.resolve(state.basePath || "", pathToUse);
-	if (hasMagic(pathToUse)) {
-		const matchedFiles = globSync(resolvedPath, { windowsPathsNoEscape: true });
-		if (matchedFiles.length === 0) throw new Error(`No files found matching pattern: ${resolvedPath}`);
-		const allContents = [];
-		for (const matchedFile of matchedFiles) {
-			let contents;
-			try {
-				contents = fs$2.readFileSync(matchedFile, "utf8");
-			} catch (error) {
-				if (error.code === "ENOENT") {
-					logger.debug(`File disappeared during glob expansion: ${matchedFile}`);
-					continue;
-				}
-				throw error;
+	if (typeof provider === "object" && provider !== null && !Array.isArray(provider)) {
+		const providerId = provider.id;
+		const label = getProviderLabel(provider);
+		if (isValidProviderId(providerId)) return {
+			kind: "options",
+			id: providerId,
+			label,
+			loadOptions: provider,
+			loadProviderPath: providerId
+		};
+		const keys = Object.keys(provider);
+		if (keys.length === 1 && !PROVIDER_OPTION_KEYS.has(keys[0])) {
+			const originalId = keys[0];
+			const providerObject = provider[originalId];
+			if (typeof providerObject === "object" && providerObject !== null && !Array.isArray(providerObject) && isValidProviderId(originalId)) {
+				const id = isValidProviderId(providerObject.id) ? providerObject.id : originalId;
+				return {
+					kind: "map",
+					id,
+					label: getProviderLabel(providerObject),
+					loadOptions: {
+						...providerObject,
+						id
+					},
+					loadProviderPath: originalId
+				};
 			}
-			if (matchedFile.endsWith(".json")) {
-				const parsed = JSON.parse(contents);
-				if (Array.isArray(parsed)) allContents.push(...parsed);
-				else allContents.push(parsed);
-			} else if (matchedFile.endsWith(".yaml") || matchedFile.endsWith(".yml")) {
-				const parsed = yaml.load(contents);
-				if (parsed === null || parsed === void 0) continue;
-				if (Array.isArray(parsed)) allContents.push(...parsed);
-				else allContents.push(parsed);
-			} else if (matchedFile.endsWith(".csv")) {
-				const records = parse$1(contents, { columns: true });
-				if (records.length > 0 && Object.keys(records[0]).length === 1) allContents.push(...records.map((record) => Object.values(record)[0]));
-				else allContents.push(...records);
-			} else allContents.push(contents);
 		}
-		return allContents;
-	}
-	const finalPath = resolvedPath;
-	let contents;
-	try {
-		contents = fs$2.readFileSync(finalPath, "utf8");
-	} catch (error) {
-		if (error.code === "ENOENT") throw new Error(`File does not exist: ${finalPath}`);
-		throw new Error(`Failed to read file ${finalPath}: ${error}`);
-	}
-	if (finalPath.endsWith(".json")) try {
-		return JSON.parse(contents);
-	} catch (error) {
-		throw new Error(`Failed to parse JSON file ${finalPath}: ${error}`);
+		if (isValidProviderId(label)) return {
+			kind: "unknown",
+			id: label,
+			label
+		};
 	}
-	if (finalPath.endsWith(".yaml") || finalPath.endsWith(".yml")) try {
-		return yaml.load(contents);
-	} catch (error) {
-		throw new Error(`Failed to parse YAML file ${finalPath}: ${error}`);
+	return {
+		kind: "unknown",
+		id: index === void 0 ? "unknown" : `unknown-${index}`
+	};
+}
+//#endregion
+//#region src/util/provider.ts
+function providerToIdentifier(provider) {
+	if (!provider) return;
+	if (typeof provider === "string") return canonicalizeProviderId(provider);
+	const { label } = normalizeProviderRef(provider);
+	if (label) return label;
+	if (isApiProvider(provider)) return canonicalizeProviderId(provider.id());
+	if (isProviderOptions(provider)) {
+		if (provider.id) return canonicalizeProviderId(provider.id);
+		return;
 	}
-	if (finalPath.endsWith(".csv")) {
-		const records = parse$1(contents, { columns: true });
-		if (records.length > 0 && Object.keys(records[0]).length === 1) return records.map((record) => Object.values(record)[0]);
-		return records;
+	if (typeof provider === "object" && "id" in provider && typeof provider.id === "string") return canonicalizeProviderId(provider.id);
+}
+/**
+* Gets a descriptive identifier string for a provider, showing both label and ID when both exist.
+* Useful for error messages to help users debug provider reference issues.
+*/
+function getProviderDescription(provider) {
+	const label = provider.label;
+	const id = provider.id();
+	if (label && label !== id) return `${label} (${id})`;
+	return id;
+}
+/**
+* Checks if a provider reference matches a given provider.
+* Supports exact matching and wildcard patterns.
+*/
+function doesProviderRefMatch(ref, provider) {
+	const label = provider.label;
+	const id = provider.id();
+	const canonicalRef = canonicalizeProviderId(ref);
+	const canonicalId = canonicalizeProviderId(id);
+	if (label && label === ref) return true;
+	if (id === ref || canonicalId === canonicalRef) return true;
+	if (ref.endsWith("*")) {
+		const prefix = ref.slice(0, -1);
+		if (label?.startsWith(prefix) || id.startsWith(prefix) || canonicalId.startsWith(prefix)) return true;
 	}
-	return contents;
+	if (label?.startsWith(`${ref}:`) || id.startsWith(`${ref}:`) || canonicalId.startsWith(`${ref}:`)) return true;
+	return false;
 }
 /**
-* Resolves a relative file path with respect to a base path, handling cloud configuration appropriately.
-* When using a cloud configuration, the current working directory is always used instead of the context's base path.
-*
-* @param filePath - The relative or absolute file path to resolve.
-* @param isCloudConfig - Whether this is a cloud configuration.
-* @returns The resolved absolute file path.
+* Checks if a provider is allowed based on a list of allowed references.
 */
-function getResolvedRelativePath(filePath, isCloudConfig) {
-	if (path$1.isAbsolute(filePath) || !isCloudConfig) return filePath;
-	return path$1.join(process.cwd(), filePath);
+function isProviderAllowed(provider, allowedProviders) {
+	if (!Array.isArray(allowedProviders)) return true;
+	if (allowedProviders.length === 0) return false;
+	return allowedProviders.some((ref) => doesProviderRefMatch(ref, provider));
 }
 /**
-* Recursively loads external file references from a configuration object.
-*
-* @param config - The configuration object to process
-* @param context - Optional context to control file loading behavior
-* @returns The configuration with external file references resolved
+* Detects if a provider uses OpenAI models.
+* This includes direct OpenAI providers and Azure OpenAI.
 */
-function maybeLoadConfigFromExternalFile(config, context) {
-	if (Array.isArray(config)) return config.map((item) => maybeLoadConfigFromExternalFile(item, context));
-	if (typeof config === "object" && config !== null) {
-		const result = {};
-		for (const key of Object.keys(config)) {
-			const childContext = key === "value" && "type" in config && typeof config.type === "string" && (config.type === "python" || config.type === "javascript") ? "assertion" : key === "vars" ? "vars" : context;
-			result[key] = maybeLoadConfigFromExternalFile(config[key], childContext);
-		}
-		return result;
+function isOpenAiProvider(providerId) {
+	const lowerProviderId = providerId.toLowerCase();
+	if (lowerProviderId.startsWith("openai:")) return true;
+	if (lowerProviderId.startsWith("azureopenai:")) return true;
+	if (lowerProviderId.startsWith("azure:")) {
+		if ([
+			"gpt",
+			"openai",
+			"davinci",
+			"curie",
+			"babbage",
+			"ada",
+			"text-embedding",
+			"whisper",
+			"dall-e",
+			"tts"
+		].some((indicator) => lowerProviderId.includes(indicator))) return true;
 	}
-	return maybeLoadFromExternalFile(config, context);
+	return false;
 }
 /**
-* Parses a file path or glob pattern to extract function names and file extensions.
-* Function names can be specified in the filename like this:
-* prompt.py:myFunction or prompts.js:myFunction.
-* @param basePath - The base path for file resolution.
-* @param promptPath - The path or glob pattern.
-* @returns Parsed details including function name, file extension, and directory status.
+* Detects if a provider uses Anthropic/Claude models.
+* This includes direct Anthropic providers, Bedrock with Claude, and Vertex with Claude.
 */
-function parsePathOrGlob(basePath, promptPath) {
-	if (promptPath.startsWith("file://")) promptPath = promptPath.slice(7);
-	const filePath = path$1.resolve(basePath, promptPath);
-	let filename = path$1.relative(basePath, filePath);
-	let functionName;
-	if (filename.includes(":")) {
-		const lastColonIndex = filename.lastIndexOf(":");
-		if (lastColonIndex > 1) {
-			const pathWithoutFunction = filename.slice(0, lastColonIndex);
-			if (isJavascriptFile(pathWithoutFunction) || pathWithoutFunction.endsWith(".py") || pathWithoutFunction.endsWith(".go") || pathWithoutFunction.endsWith(".rb")) {
-				functionName = filename.slice(lastColonIndex + 1);
-				filename = pathWithoutFunction;
-			}
-		}
+function isAnthropicProvider(providerId) {
+	const lowerProviderId = providerId.toLowerCase();
+	if (lowerProviderId.startsWith("anthropic:")) return true;
+	if (lowerProviderId.startsWith("bedrock:")) {
+		if (lowerProviderId.includes("claude") || lowerProviderId.includes("anthropic")) return true;
 	}
-	let stats;
-	try {
-		stats = fs$2.statSync(path$1.join(basePath, filename));
-	} catch (err) {
-		if (getEnvBool("PROMPTFOO_STRICT_FILES")) throw err;
+	if (lowerProviderId.startsWith("vertex:")) {
+		if (lowerProviderId.includes("claude")) return true;
 	}
-	const normalizedFilePath = filePath.replace(/\\/g, "/");
-	const isPathPattern = stats?.isDirectory() || hasMagic(promptPath) || hasMagic(normalizedFilePath);
-	const safeFilename = path$1.relative(basePath, safeResolve(basePath, filename));
-	return {
-		extension: isPathPattern ? void 0 : path$1.parse(safeFilename).ext,
-		filePath: path$1.join(basePath, safeFilename),
-		functionName,
-		isPathPattern
-	};
+	return false;
 }
-function readOutput(outputPath) {
-	const ext = path$1.parse(outputPath).ext.slice(1);
-	switch (ext) {
-		case "json": return JSON.parse(fs$2.readFileSync(outputPath, "utf-8"));
-		default: throw new Error(`Unsupported output file format: ${ext} currently only supports json`);
+const KNOWN_ENV_VARS = {
+	openai: "OPENAI_API_KEY",
+	anthropic: "ANTHROPIC_API_KEY",
+	google: "GOOGLE_API_KEY",
+	mistral: "MISTRAL_API_KEY",
+	cohere: "COHERE_API_KEY",
+	replicate: "REPLICATE_API_TOKEN",
+	voyage: "VOYAGE_API_KEY",
+	ai21: "AI21_API_KEY",
+	xai: "XAI_API_KEY",
+	groq: "GROQ_API_KEY",
+	deepseek: "DEEPSEEK_API_KEY",
+	perplexity: "PERPLEXITY_API_KEY",
+	hyperbolic: "HYPERBOLIC_API_KEY",
+	cerebras: "CEREBRAS_API_KEY",
+	togetherai: "TOGETHER_API_KEY",
+	fal: "FAL_KEY",
+	huggingface: "HF_TOKEN",
+	"cloudflare-ai": "CLOUDFLARE_API_KEY"
+};
+function getDefaultEnvVar(providerId) {
+	const prefix = providerId.split(":")[0];
+	return KNOWN_ENV_VARS[prefix] || `${prefix.toUpperCase()}_API_KEY`;
+}
+/**
+* Pre-checks providers for missing API keys before evaluation starts.
+* Assumes getApiKey() is side-effect free (no network calls or token refresh).
+*/
+function checkProviderApiKeys(providers) {
+	const missingApiKeys = /* @__PURE__ */ new Map();
+	for (const provider of providers) {
+		const p = provider;
+		if (typeof p.getApiKey !== "function") continue;
+		if (provider.id().startsWith("azure:")) continue;
+		const requiresKey = typeof p.requiresApiKey === "function" ? p.requiresApiKey() : p.config?.apiKeyRequired !== false;
+		let apiKey;
+		try {
+			apiKey = p.getApiKey();
+		} catch {
+			apiKey = void 0;
+		}
+		if (requiresKey && !apiKey) {
+			const envVar = p.config?.apiKeyEnvar || getDefaultEnvVar(provider.id());
+			if (!missingApiKeys.has(envVar)) missingApiKeys.set(envVar, []);
+			missingApiKeys.get(envVar).push(provider.id());
+		}
 	}
+	return missingApiKeys;
 }
 /**
-* Load custom Nunjucks filters from external files.
-* Note: If a glob pattern matches multiple files, only the last file's export is used.
-* Each filter name should typically resolve to a single file.
+* Detects if a provider uses Google models.
+* This includes direct Google/Vertex providers with Gemini and other Google models.
+* Note: Vertex with Claude models is NOT counted as Google (it's Anthropic).
 */
-async function readFilters(filters, basePath = "") {
-	const ret = {};
-	for (const [name, filterPath] of Object.entries(filters)) {
-		const filePaths = globSync(path$1.join(basePath, filterPath), { windowsPathsNoEscape: true });
-		for (const filePath of filePaths) ret[name] = await importModule(path$1.resolve(filePath));
+function isGoogleProvider(providerId) {
+	const lowerProviderId = providerId.toLowerCase();
+	if (lowerProviderId.startsWith("google:")) return true;
+	if (lowerProviderId.startsWith("vertex:")) {
+		if (!lowerProviderId.includes("claude")) return true;
 	}
-	return ret;
+	return false;
 }
+//#endregion
+//#region src/util/comparison.ts
 /**
-* Loads configuration from an external file with variable rendering.
-* This is a convenience wrapper that combines renderVarsInObject and maybeLoadFromExternalFile.
+* Explicit runtime variable names that don't follow the underscore convention.
+* These are added during evaluation but aren't part of the original test definition.
 *
-* Use this for simple config fields that:
-* - Need variable rendering ({{ vars.x }}, {{ env.X }})
-* - May reference external files (file://path.json)
-* - Don't have nested file references that need loading
+* - sessionId: Added by multi-turn strategy providers (GOAT, Crescendo)
 *
-* For fields with nested file references (like response_format.schema),
-* use maybeLoadResponseFormatFromExternalFile instead.
+* Note: Variables starting with underscore (e.g., _conversation) are automatically
+* treated as runtime variables and filtered out.
+*/
+const EXPLICIT_RUNTIME_VAR_KEYS = ["sessionId"];
+/**
+* Checks if a variable key is a runtime-only variable that should be filtered
+* when comparing test cases.
 *
-* @param config - The configuration to process
-* @param vars - Variables for template rendering
-* @returns The processed configuration with variables rendered and files loaded
+* Runtime variables are identified by:
+* 1. Starting with underscore (_) - convention for internal/runtime vars
+* 2. Being in the explicit runtime var list (for legacy vars like sessionId)
 */
-function maybeLoadFromExternalFileWithVars(config, vars) {
-	return maybeLoadFromExternalFile(renderVarsInObject(config, vars));
+function isRuntimeVar(key) {
+	return key.startsWith("_") || EXPLICIT_RUNTIME_VAR_KEYS.includes(key);
 }
 /**
-* Loads response_format configuration from an external file with variable rendering.
+* Filters out runtime-only variables that are added during evaluation
+* but aren't part of the original test definition.
 *
-* This function handles the special case where response_format may contain:
-* 1. A top-level file reference (file://format.json)
-* 2. A nested schema reference for json_schema type (schema: file://schema.json)
+* This is used when comparing test cases to determine if a result
+* corresponds to a particular test, regardless of runtime state.
 *
-* Both levels need variable rendering and file loading.
+* Runtime variables are identified by:
+* - Starting with underscore (e.g., _conversation, _metadata)
+* - Being in the explicit list (e.g., sessionId for backward compatibility)
+*/
+function filterRuntimeVars(vars) {
+	if (!vars || typeof vars !== "object" || Array.isArray(vars)) return vars;
+	const filtered = {};
+	for (const [key, value] of Object.entries(vars)) if (!isRuntimeVar(key)) filtered[key] = value;
+	return filtered;
+}
+/**
+* Extracts only runtime variables from a vars object.
+* This is the inverse of filterRuntimeVars.
 *
-* @param responseFormat - The response_format configuration
-* @param vars - Variables for template rendering
-* @returns The processed response_format with all files loaded
+* Used to restore runtime state when re-running filtered tests.
 */
-function maybeLoadResponseFormatFromExternalFile(responseFormat, vars) {
-	if (responseFormat === void 0 || responseFormat === null) return responseFormat;
-	const loaded = maybeLoadFromExternalFile(renderVarsInObject(responseFormat, vars));
-	if (!loaded || typeof loaded !== "object") return loaded;
-	if (loaded.type === "json_schema") {
-		const nestedSchema = loaded.schema || loaded.json_schema?.schema;
-		if (nestedSchema) {
-			const loadedSchema = maybeLoadFromExternalFile(renderVarsInObject(nestedSchema, vars));
-			if (loaded.schema !== void 0) return {
-				...loaded,
-				schema: loadedSchema
-			};
-			else if (loaded.json_schema?.schema !== void 0) return {
-				...loaded,
-				json_schema: {
-					...loaded.json_schema,
-					schema: loadedSchema
-				}
-			};
-		}
-	}
-	return loaded;
+function extractRuntimeVars(vars) {
+	if (!vars || typeof vars !== "object" || Array.isArray(vars)) return;
+	const extracted = {};
+	for (const [key, value] of Object.entries(vars)) if (isRuntimeVar(key)) extracted[key] = value;
+	return Object.keys(extracted).length > 0 ? extracted : void 0;
+}
+function varsMatch(vars1, vars2) {
+	return deepEqual(vars1, vars2);
 }
 /**
-* Renders variables in a tools object and loads from external file if applicable.
-* This function combines renderVarsInObject and maybeLoadFromExternalFile into a single step
-* specifically for handling tools configurations.
+* Generate a unique key for a test case for deduplication purposes.
+* Excludes runtime variables and includes strategyId to distinguish tests
+* with the same prompt but different strategies.
 *
-* Supports loading from JSON, YAML, Python, and JavaScript files.
+* @param testCase - The test case to generate a key for
+* @returns A JSON string that uniquely identifies the test case
+*/
+function getTestCaseDeduplicationKey(testCase) {
+	const filteredVars = filterRuntimeVars(testCase.vars);
+	const strategyId = testCase.metadata?.strategyId || "none";
+	return JSON.stringify({
+		vars: filteredVars,
+		strategyId
+	});
+}
+/**
+* Deduplicates an array of test cases based on their vars and strategyId.
+* Tests with the same vars but different strategies are considered different.
+* Runtime variables (like _conversation, sessionId) are filtered out before comparison.
 *
-* @param tools - The tools configuration object or array to process.
-* @param vars - Variables to use for rendering.
-* @returns The processed tools configuration with variables rendered and content loaded from files if needed.
-* @throws {Error} If the loaded tools are in an invalid format
+* @param tests - Array of test cases to deduplicate
+* @returns Deduplicated array of test cases
 */
-async function maybeLoadToolsFromExternalFile(tools, vars) {
-	const rendered = renderVarsInObject(tools, vars);
-	if (typeof rendered === "string" && rendered.startsWith("file://")) {
-		const { filePath, functionName } = parseFileUrl(rendered);
-		if (functionName && (filePath.endsWith(".py") || isJavascriptFile(filePath))) {
-			const fileType = filePath.endsWith(".py") ? "Python" : "JavaScript";
-			logger.debug(`[maybeLoadToolsFromExternalFile] Loading tools from ${fileType} file: ${filePath}:${functionName}`);
-			try {
-				let toolDefinitions;
-				if (filePath.endsWith(".py")) {
-					const absPath = safeResolve(state.basePath || process.cwd(), filePath);
-					logger.debug(`[maybeLoadToolsFromExternalFile] Resolved Python path: ${absPath}`);
-					toolDefinitions = await runPython(absPath, functionName, []);
-				} else {
-					const absPath = safeResolve(state.basePath || process.cwd(), filePath);
-					logger.debug(`[maybeLoadToolsFromExternalFile] Resolved JavaScript path: ${absPath}`);
-					const module = await importModule(absPath);
-					const fn = module[functionName] || module.default?.[functionName];
-					if (typeof fn !== "function") {
-						const availableExports = Object.keys(module).filter((k) => k !== "default");
-						const basePath = state.basePath || process.cwd();
-						throw new Error(`Function "${functionName}" not found in ${filePath}. Available exports: ${availableExports.length > 0 ? availableExports.join(", ") : "(none)"}\nResolved from: ${basePath}`);
-					}
-					toolDefinitions = await Promise.resolve(fn());
-				}
-				if (!toolDefinitions || typeof toolDefinitions === "string" || typeof toolDefinitions === "number" || typeof toolDefinitions === "boolean") throw new Error(`Function "${functionName}" must return an array or object of tool definitions, but returned: ${toolDefinitions === null ? "null" : typeof toolDefinitions}`);
-				logger.debug(`[maybeLoadToolsFromExternalFile] Successfully loaded ${Array.isArray(toolDefinitions) ? toolDefinitions.length : "object"} tools`);
-				return toolDefinitions;
-			} catch (err) {
-				const errorMessage = err instanceof Error ? err.message : String(err);
-				const basePath = state.basePath || process.cwd();
-				throw new Error(`Failed to load tools from ${rendered}:\n${errorMessage}\n\nMake sure the function "${functionName}" exists and returns a valid tool definition array.\nResolved from: ${basePath}`);
-			}
-		}
-		if (filePath.endsWith(".py") || isJavascriptFile(filePath)) {
-			const ext = filePath.endsWith(".py") ? "Python" : "JavaScript";
-			const basePath = state.basePath || process.cwd();
-			throw new Error(`Cannot load tools from ${rendered}\n${ext} files require a function name. Use this format:\n  tools: file://${filePath}:get_tools\n\nYour ${ext} file should export a function that returns tool definitions:\n` + (filePath.endsWith(".py") ? `  def get_tools():\n      return [{"type": "function", "function": {...}}]` : `  module.exports.get_tools = () => [{ type: "function", function: {...} }];`) + `\n\nResolved from: ${basePath}`);
-		}
-	}
-	if (Array.isArray(rendered)) {
-		const results = await Promise.all(rendered.map((item) => maybeLoadToolsFromExternalFile(item, vars)));
-		if (results.every((r) => Array.isArray(r))) return results.flat();
-		return results;
-	}
-	if (typeof rendered !== "string") return rendered;
-	const loaded = maybeLoadFromExternalFile(rendered);
-	if (loaded !== void 0 && loaded !== null && typeof loaded === "string") {
-		if (loaded.startsWith("file://")) throw new Error(`Failed to load tools from ${loaded}\nEnsure the file exists and contains valid JSON or YAML tool definitions.`);
-		if (loaded.includes("def ") || loaded.includes("import ")) throw new Error("Invalid tools configuration: file appears to contain Python code.\nPython files require a function name. Use this format:\n  tools: file://tools.py:get_tools");
-		throw new Error("Invalid tools configuration: expected an array or object, but got a string.\nIf using file://, ensure the file contains valid JSON or YAML tool definitions.");
+function deduplicateTestCases(tests) {
+	const seen = /* @__PURE__ */ new Set();
+	return tests.filter((test) => {
+		const key = getTestCaseDeduplicationKey(test);
+		if (seen.has(key)) return false;
+		seen.add(key);
+		return true;
+	});
+}
+function resultIsForTestCase(result, testCase) {
+	const testProviderId = testCase.provider ? providerToIdentifier(testCase.provider) : void 0;
+	const resultProviderId = providerToIdentifier(result.provider);
+	const providersMatch = !testProviderId || !resultProviderId || testProviderId === resultProviderId;
+	const resultVars = filterRuntimeVars(result.vars);
+	const testVars = filterRuntimeVars(testCase.vars);
+	const doVarsMatch = varsMatch(testVars, resultVars);
+	const isMatch = doVarsMatch && providersMatch;
+	if (!isMatch) {
+		const varKeys = testVars ? Object.keys(testVars).join(", ") : "none";
+		logger.debug(`[resultIsForTestCase] No match: vars=${doVarsMatch}, providers=${providersMatch}`, {
+			testProvider: testProviderId || "none",
+			resultProvider: resultProviderId || "none",
+			testVarKeys: varKeys
+		});
 	}
-	return loaded;
+	return isMatch;
+}
+//#endregion
+//#region src/util/env.ts
+/**
+* Load environment variables from .env file(s).
+* @param envPath - Single path, array of paths, or undefined for default .env loading.
+*                  When paths are explicitly specified, all files must exist or an error is thrown.
+*                  When multiple files are provided, later files override values from earlier files.
+*/
+function setupEnv(envPath) {
+	if (envPath) {
+		const paths = (Array.isArray(envPath) ? envPath : [envPath]).flatMap((p) => p.includes(",") ? p.split(",").map((s) => s.trim()) : p.trim()).filter((p) => p.length > 0);
+		if (paths.length === 0) {
+			dotenv.config({ quiet: true });
+			return;
+		}
+		for (const p of paths) if (!fs$2.existsSync(p)) throw new Error(`Environment file not found: ${p}`);
+		if (paths.length === 1) logger.info(`Loading environment variables from ${paths[0]}`);
+		else logger.info(`Loading environment variables from: ${paths.join(", ")}`);
+		const pathArg = paths.length === 1 ? paths[0] : paths;
+		dotenv.config({
+			path: pathArg,
+			override: true,
+			quiet: true
+		});
+	} else dotenv.config({ quiet: true });
 }
 //#endregion
 //#region src/googleSheets.ts
@@ -1299,6 +1424,6 @@ function printBorder() {
 	logger.info(border);
 }
 //#endregion
-export { getProviderDescription as A, deduplicateTestCases as C, resultIsForTestCase as D, getTestCaseDeduplicationKey as E, isGoogleProvider as M, isOpenAiProvider as N, checkProviderApiKeys as O, isProviderAllowed as P, setupEnv as S, filterRuntimeVars as T, parsePathOrGlob as _, ComparisonEvalNotFoundError as a, loadFunction as b, mergeComparisonTables as c, getResolvedRelativePath as d, maybeLoadConfigFromExternalFile as f, maybeLoadToolsFromExternalFile as g, maybeLoadResponseFormatFromExternalFile as h, writeOutput as i, isAnthropicProvider as j, doesProviderRefMatch as k, fetchCsvFromGoogleSheet as l, maybeLoadFromExternalFileWithVars as m, createOutputMetadata as n, evalTableToJson as o, maybeLoadFromExternalFile as p, writeMultipleOutputs as r, generateEvalCsv as s, printBorder as t, getNunjucksEngineForFilePath as u, readFilters as v, extractRuntimeVars as w, parseFileUrl as x, readOutput as y };
+export { maybeLoadFromExternalFile as A, isProviderConfigFileReference as C, getNunjucksEngineForFilePath as D, readProviderConfigFile as E, readFilters as F, readOutput as I, loadFunction as L, maybeLoadResponseFormatFromExternalFile as M, maybeLoadToolsFromExternalFile as N, getResolvedRelativePath as O, parsePathOrGlob as P, parseFileUrl as R, isProviderAllowed as S, normalizeProviderRef as T, doesProviderRefMatch as _, ComparisonEvalNotFoundError as a, isGoogleProvider as b, mergeComparisonTables as c, deduplicateTestCases as d, extractRuntimeVars as f, checkProviderApiKeys as g, resultIsForTestCase as h, writeOutput as i, maybeLoadFromExternalFileWithVars as j, maybeLoadConfigFromExternalFile as k, fetchCsvFromGoogleSheet as l, getTestCaseDeduplicationKey as m, createOutputMetadata as n, evalTableToJson as o, filterRuntimeVars as p, writeMultipleOutputs as r, generateEvalCsv as s, printBorder as t, setupEnv as u, getProviderDescription as v, loadProviderConfigsFromFile as w, isOpenAiProvider as x, isAnthropicProvider as y };
 
-//# sourceMappingURL=util-DxWpWjhc.js.map
+//# sourceMappingURL=util-BQOCAHQC.js.map