promptfoo 0.121.3 → 0.121.4

package/dist/src/{util-DbVG-yZU.js → util-DvU2Pw8c.js}

@@ -1,7 +1,14 @@
-import { m as calculateCost } from "./fetch-BiYv2BZc.js";
-import { C as parseDataUrl } from "./transform-B2-jIv68.js";
+import { g as calculateCost } from "./fetch-Di00EQrc.js";
+import { w as parseDataUrl } from "./transform-CG0ehZNG.js";
 //#region src/providers/anthropic/util.ts
 const ANTHROPIC_MODELS = [
+	...["claude-mythos-preview"].map((model) => ({
+		id: model,
+		cost: {
+			input: 25 / 1e6,
+			output: 125 / 1e6
+		}
+	})),
 	...["claude-sonnet-4-6", "claude-sonnet-4-6-latest"].map((model) => ({
 		id: model,
 		cost: {
@@ -16,7 +23,11 @@ const ANTHROPIC_MODELS = [
 			output: 25 / 1e6
 		}
 	})),
-	...["claude-opus-4-5-20251101", "claude-opus-4-5-latest"].map((model) => ({
+	...[
+		"claude-opus-4-5",
+		"claude-opus-4-5-20251101",
+		"claude-opus-4-5-latest"
+	].map((model) => ({
 		id: model,
 		cost: {
 			input: 5 / 1e6,
@@ -24,6 +35,7 @@ const ANTHROPIC_MODELS = [
 		}
 	})),
 	...[
+		"claude-opus-4-1",
 		"claude-opus-4-1-20250805",
 		"claude-opus-4-20250514",
 		"claude-opus-4-0",
@@ -36,6 +48,7 @@ const ANTHROPIC_MODELS = [
 		}
 	})),
 	...[
+		"claude-sonnet-4-5",
 		"claude-sonnet-4-5-20250929",
 		"claude-sonnet-4-5-latest",
 		"claude-sonnet-4-20250514",
@@ -48,7 +61,11 @@ const ANTHROPIC_MODELS = [
 			output: 15 / 1e6
 		}
 	})),
-	...["claude-haiku-4-5-20251001", "claude-haiku-4-5-latest"].map((model) => ({
+	...[
+		"claude-haiku-4-5",
+		"claude-haiku-4-5-20251001",
+		"claude-haiku-4-5-latest"
+	].map((model) => ({
 		id: model,
 		cost: {
 			input: 1 / 1e6,
@@ -209,31 +226,72 @@ function parseMessages(messages) {
 		thinking
 	};
 }
-function calculateAnthropicCost(modelName, config, promptTokens, completionTokens) {
+/**
+* Compute input cost with Anthropic cache pricing applied.
+* Anthropic docs: input_tokens is the non-cached portion; cache_read and cache_creation are additive.
+* Cache reads cost 10% of base rate (90% discount), cache writes cost 125% of base rate (25% surcharge).
+*/
+function calculateCacheInputCost(baseInputRate, uncachedInputTokens, cacheRead, cacheCreation) {
+	return uncachedInputTokens * baseInputRate + cacheRead * baseInputRate * .1 + cacheCreation * baseInputRate * 1.25;
+}
+function calculateAnthropicCost(modelName, config, promptTokens, completionTokens, cacheReadTokens, cacheCreationTokens) {
+	if (config.cost != null) return calculateCost(modelName, config, promptTokens, completionTokens, ANTHROPIC_MODELS);
+	if (!Number.isFinite(promptTokens) || !Number.isFinite(completionTokens) || typeof promptTokens === "undefined" || typeof completionTokens === "undefined") return calculateCost(modelName, config, promptTokens, completionTokens, ANTHROPIC_MODELS);
+	const cacheRead = cacheReadTokens ?? 0;
+	const cacheCreation = cacheCreationTokens ?? 0;
+	const effectiveInputTokens = promptTokens + cacheRead + cacheCreation;
 	if ([
+		"claude-sonnet-4-5",
 		"claude-sonnet-4-5-20250929",
 		"claude-sonnet-4-5-latest",
 		"claude-sonnet-4-6",
 		"claude-sonnet-4-6-latest"
-	].includes(modelName) && Number.isFinite(promptTokens) && Number.isFinite(completionTokens) && typeof promptTokens !== "undefined" && typeof completionTokens !== "undefined") {
-		const inputCost = config.cost ?? (promptTokens > 2e5 ? 6 / 1e6 : 3 / 1e6);
-		const outputCost = config.cost ?? (promptTokens > 2e5 ? 22.5 / 1e6 : 15 / 1e6);
-		return inputCost * promptTokens + outputCost * completionTokens;
+	].includes(modelName)) {
+		const isLongContext = effectiveInputTokens > 2e5;
+		const baseInputRate = isLongContext ? 6 / 1e6 : 3 / 1e6;
+		const outputRate = isLongContext ? 22.5 / 1e6 : 15 / 1e6;
+		return calculateCacheInputCost(baseInputRate, promptTokens, cacheRead, cacheCreation) + completionTokens * outputRate;
+	}
+	if (cacheRead || cacheCreation) {
+		const modelInfo = ANTHROPIC_MODELS.find((m) => m.id === modelName);
+		if (modelInfo) return calculateCacheInputCost(modelInfo.cost.input, promptTokens, cacheRead, cacheCreation) + completionTokens * modelInfo.cost.output;
 	}
 	return calculateCost(modelName, config, promptTokens, completionTokens, ANTHROPIC_MODELS);
 }
+/**
+* Extract refusal details from the Anthropic stop_details field.
+* Returns a human-readable string if the response was refused, or undefined otherwise.
+*/
+function getRefusalDetails(data) {
+	if (data.stop_reason !== "refusal" || !data.stop_details) return;
+	const details = data.stop_details;
+	const parts = ["Content refused by Anthropic safety filters"];
+	if (details.category) parts.push(`category: ${details.category}`);
+	if (details.explanation) parts.push(`explanation: ${details.explanation}`);
+	return parts.join(" — ");
+}
 function getTokenUsage(data, cached) {
 	if (data.usage) {
-		const total_tokens = data.usage.input_tokens + data.usage.output_tokens;
+		const cacheRead = data.usage.cache_read_input_tokens ?? 0;
+		const cacheCreation = data.usage.cache_creation_input_tokens ?? 0;
+		const allInputTokens = (data.usage.input_tokens ?? 0) + cacheRead + cacheCreation;
+		const total_tokens = allInputTokens + (data.usage.output_tokens ?? 0);
 		if (cached) return {
 			cached: total_tokens,
 			total: total_tokens
 		};
-		else return {
-			total: total_tokens,
-			prompt: data.usage.input_tokens || 0,
-			completion: data.usage.output_tokens || 0
-		};
+		else {
+			const usage = {
+				total: total_tokens,
+				prompt: allInputTokens,
+				completion: data.usage.output_tokens ?? 0
+			};
+			if (data.usage.cache_read_input_tokens != null || data.usage.cache_creation_input_tokens != null) usage.completionDetails = {
+				cacheReadInputTokens: cacheRead,
+				cacheCreationInputTokens: cacheCreation
+			};
+			return usage;
+		}
 	}
 	return {};
 }
@@ -243,16 +301,20 @@ function getTokenUsage(data, cached) {
 function processAnthropicTools(tools = []) {
 	const processedTools = [];
 	const requiredBetaFeatures = [];
-	for (const tool of tools) if ("type" in tool) if (tool.type === "web_fetch_20250910") {
-		processedTools.push(transformWebFetchTool(tool));
-		if (!requiredBetaFeatures.includes("web-fetch-2025-09-10")) requiredBetaFeatures.push("web-fetch-2025-09-10");
-	} else if (tool.type === "web_search_20250305") processedTools.push(transformWebSearchTool(tool));
-	else processedTools.push(tool);
-	else {
-		processedTools.push(tool);
-		if ("strict" in tool && tool.strict === true) {
-			if (!requiredBetaFeatures.includes("structured-outputs-2025-11-13")) requiredBetaFeatures.push("structured-outputs-2025-11-13");
-		}
+	const addRequiredBetaFeature = (feature) => {
+		if (!requiredBetaFeatures.includes(feature)) requiredBetaFeatures.push(feature);
+	};
+	for (const tool of tools) {
+		if ("type" in tool) if (tool.type === "web_fetch_20250910") {
+			processedTools.push(transformWebFetchTool(tool));
+			addRequiredBetaFeature("web-fetch-2025-09-10");
+		} else if (tool.type === "web_fetch_20260209") processedTools.push(transformWebFetchTool20260209(tool));
+		else if (tool.type === "web_fetch_20260309") processedTools.push(transformWebFetchToolV2(tool));
+		else if (tool.type === "web_search_20250305") processedTools.push(transformWebSearchTool(tool));
+		else if (tool.type === "web_search_20260209") processedTools.push(transformWebSearchTool20260209(tool));
+		else processedTools.push(tool);
+		else processedTools.push(tool);
+		if ("strict" in tool && tool.strict === true) addRequiredBetaFeature("structured-outputs-2025-11-13");
 	}
 	return {
 		processedTools,
@@ -260,21 +322,54 @@ function processAnthropicTools(tools = []) {
 	};
 }
 /**
-* Transform web fetch tool config to Anthropic beta tool format
+* Apply shared web fetch tool fields from config onto the SDK tool object.
 */
+function applyWebFetchFields(tool, config) {
+	if (config.allowed_callers !== void 0) tool.allowed_callers = config.allowed_callers;
+	if (config.max_uses !== void 0) tool.max_uses = config.max_uses;
+	if (config.allowed_domains !== void 0) tool.allowed_domains = config.allowed_domains;
+	if (config.blocked_domains !== void 0) tool.blocked_domains = config.blocked_domains;
+	if (config.citations !== void 0) tool.citations = config.citations;
+	if (config.max_content_tokens !== void 0) tool.max_content_tokens = config.max_content_tokens;
+	if (config.cache_control !== void 0) tool.cache_control = config.cache_control;
+	if (config.defer_loading !== void 0) tool.defer_loading = config.defer_loading;
+	if (config.strict !== void 0) tool.strict = config.strict;
+}
 function transformWebFetchTool(config) {
 	const tool = {
 		type: "web_fetch_20250910",
 		name: "web_fetch"
 	};
-	if (config.max_uses !== void 0) tool.max_uses = config.max_uses;
-	if (config.allowed_domains) tool.allowed_domains = config.allowed_domains;
-	if (config.blocked_domains) tool.blocked_domains = config.blocked_domains;
-	if (config.citations) tool.citations = config.citations;
-	if (config.max_content_tokens !== void 0) tool.max_content_tokens = config.max_content_tokens;
-	if (config.cache_control) tool.cache_control = config.cache_control;
+	applyWebFetchFields(tool, config);
+	return tool;
+}
+function transformWebFetchTool20260209(config) {
+	const tool = {
+		type: "web_fetch_20260209",
+		name: "web_fetch"
+	};
+	applyWebFetchFields(tool, config);
+	return tool;
+}
+function transformWebFetchToolV2(config) {
+	const tool = {
+		type: "web_fetch_20260309",
+		name: "web_fetch"
+	};
+	applyWebFetchFields(tool, config);
+	if (config.use_cache !== void 0) tool.use_cache = config.use_cache;
 	return tool;
 }
+function applyWebSearchFields(tool, config) {
+	if (config.allowed_callers !== void 0) tool.allowed_callers = config.allowed_callers;
+	if (config.allowed_domains !== void 0) tool.allowed_domains = config.allowed_domains;
+	if (config.blocked_domains !== void 0) tool.blocked_domains = config.blocked_domains;
+	if (config.cache_control !== void 0) tool.cache_control = config.cache_control;
+	if (config.defer_loading !== void 0) tool.defer_loading = config.defer_loading;
+	if (config.max_uses !== void 0) tool.max_uses = config.max_uses;
+	if (config.strict !== void 0) tool.strict = config.strict;
+	if (config.user_location !== void 0) tool.user_location = config.user_location;
+}
 /**
 * Transform web search tool config to Anthropic beta tool format
 */
@@ -283,11 +378,18 @@ function transformWebSearchTool(config) {
 		type: "web_search_20250305",
 		name: "web_search"
 	};
-	if (config.max_uses !== void 0) tool.max_uses = config.max_uses;
-	if (config.cache_control) tool.cache_control = config.cache_control;
+	applyWebSearchFields(tool, config);
+	return tool;
+}
+function transformWebSearchTool20260209(config) {
+	const tool = {
+		type: "web_search_20260209",
+		name: "web_search"
+	};
+	applyWebSearchFields(tool, config);
 	return tool;
 }
 //#endregion
-export { parseMessages as a, outputFromMessage as i, calculateAnthropicCost as n, processAnthropicTools as o, getTokenUsage as r, ANTHROPIC_MODELS as t };
+export { outputFromMessage as a, getTokenUsage as i, calculateAnthropicCost as n, parseMessages as o, getRefusalDetails as r, processAnthropicTools as s, ANTHROPIC_MODELS as t };
 
-//# sourceMappingURL=util-DbVG-yZU.js.map
+//# sourceMappingURL=util-DvU2Pw8c.js.map

package/dist/src/{util-Bm3E9jpK.js → util-DxWpWjhc.js}

@@ -1,11 +1,12 @@
 #!/usr/bin/env node
-import { C as getEnvBool, f as sanitizeObject, j as state, s as logger } from "./logger-BcJBzSSA.js";
-import { D as TERMINAL_MAX_WIDTH, N as VERSION, n as fetchWithProxy } from "./fetch-DoVRJZhJ.js";
+import { N as state, T as getEnvBool, m as sanitizeObject, s as logger } from "./logger-DksKw1Qc.js";
+import { F as VERSION, k as TERMINAL_MAX_WIDTH, n as fetchWithProxy } from "./fetch-BEWnXrrG.js";
 import { t as invariant } from "./invariant-BtWWVVhl.js";
-import { m as isProviderOptions, o as OutputFileExtension, p as isApiProvider, s as ResultFailureReason } from "./types-CIhFeUC4.js";
-import { i as isJavascriptFile, t as JAVASCRIPT_EXTENSIONS } from "./fileExtensions-Ds-foDzt.js";
-import { o as safeResolve, r as importModule, t as getDirectory } from "./esm-7UIl0pPM.js";
-import { r as runPython } from "./pythonUtils-wIqk7zAf.js";
+import { m as isProviderOptions, o as OutputFileExtension, p as isApiProvider, s as ResultFailureReason } from "./types-Dm9JM6Vb.js";
+import { o as safeResolve, r as importModule, t as getDirectory } from "./esm-tVgYPY-f.js";
+import { a as getNunjucksEngine, n as renderVarsInObject } from "./render-CMEpfLaO.js";
+import { i as isJavascriptFile, t as JAVASCRIPT_EXTENSIONS } from "./fileExtensions-AWa2ZML4.js";
+import { r as runPython } from "./pythonUtils-CnndUbW-.js";
 import dotenv from "dotenv";
 import * as fs$2 from "fs";
 import * as path$1 from "path";
@@ -14,10 +15,10 @@ import * as os$1 from "os";
 import yaml from "js-yaml";
 import * as fsPromises from "fs/promises";
 import dedent from "dedent";
+import nunjucks from "nunjucks";
 import deepEqual from "fast-deep-equal";
 import { parse as parse$1 } from "csv-parse/sync";
 import { globSync, hasMagic } from "glob";
-import nunjucks from "nunjucks";
 import { XMLBuilder } from "fast-xml-parser";
 import { stringify } from "csv-stringify/sync";
 //#region src/util/provider.ts
@@ -371,130 +372,6 @@ function parseFileUrl(fileUrl) {
 	return { filePath: urlWithoutProtocol };
 }
 //#endregion
-//#region src/util/templates.ts
-/**
-* Get a Nunjucks engine instance with optional filters and configuration.
-* @param filters - Optional map of custom Nunjucks filters.
-* @param throwOnUndefined - Whether to throw an error on undefined variables.
-* @param isGrader - Whether this engine is being used in a grader context.
-* Nunjucks is always enabled in grader mode.
-* @returns A configured Nunjucks environment.
-*/
-function getNunjucksEngine(filters, throwOnUndefined = false, isGrader = false) {
-	if (!isGrader && getEnvBool("PROMPTFOO_DISABLE_TEMPLATING")) return { renderString: (template) => template };
-	const env = nunjucks.configure({
-		autoescape: false,
-		throwOnUndefined
-	});
-	const envGlobals = {
-		...getEnvBool("PROMPTFOO_DISABLE_TEMPLATE_ENV_VARS", getEnvBool("PROMPTFOO_SELF_HOSTED", false)) ? {} : process.env,
-		...state.config?.env
-	};
-	env.addGlobal("env", envGlobals);
-	env.addFilter("load", function(str) {
-		return JSON.parse(str);
-	});
-	if (filters) for (const [name, filter] of Object.entries(filters)) env.addFilter(name, filter);
-	return env;
-}
-/**
-* Parse Nunjucks template to extract variables.
-* @param template - The Nunjucks template string.
-* @returns An array of variables used in the template.
-*/
-function extractVariablesFromTemplate(template) {
-	const variableSet = /* @__PURE__ */ new Set();
-	const regex = /\{\{[\s]*([^{}\s|]+)[\s]*(?:\|[^}]+)?\}\}|\{%[\s]*(?:if|for)[\s]+([^{}\s]+)[\s]*.*?%\}/g;
-	template = template.replace(/\{#[\s\S]*?#\}/g, "");
-	let match;
-	while ((match = regex.exec(template)) !== null) {
-		const variable = match[1] || match[2];
-		if (variable) variableSet.add(variable);
-	}
-	const forLoopRegex = /\{%[\s]*for[\s]+(\w+)[\s]+in[\s]+(\w+)[\s]*%\}/g;
-	while ((match = forLoopRegex.exec(template)) !== null) {
-		variableSet.delete(match[1]);
-		variableSet.add(match[2]);
-	}
-	return Array.from(variableSet);
-}
-/**
-* Extract variables from multiple Nunjucks templates.
-* @param templates - An array of Nunjucks template strings.
-* @returns An array of variables used in the templates.
-*/
-function extractVariablesFromTemplates(templates) {
-	const variableSet = /* @__PURE__ */ new Set();
-	for (const template of templates) extractVariablesFromTemplate(template).forEach((variable) => variableSet.add(variable));
-	return Array.from(variableSet);
-}
-//#endregion
-//#region src/util/render.ts
-/**
-* Renders ONLY environment variable templates in an object, leaving all other templates untouched.
-* This allows env vars to be resolved at provider load time while preserving runtime var templates.
-*
-* Supports full Nunjucks syntax for env vars including filters and expressions:
-* - {{ env.VAR_NAME }}
-* - {{ env['VAR-NAME'] }}
-* - {{ env["VAR-NAME"] }}
-* - {{ env.VAR | default('fallback') }}
-* - {{ env.VAR | upper }}
-*
-* Preserves non-env templates for runtime rendering:
-* - {{ vars.x }} - preserved as literal
-* - {{ prompt }} - preserved as literal
-*
-* Implementation: Uses regex to find env templates, delegates to Nunjucks for rendering.
-* This ensures full Nunjucks feature support while preserving non-env templates.
-*
-* @param obj - The object to process
-* @param envOverrides - Optional env vars to merge with (or replace) the base env
-* @param replaceBase - If true, envOverrides replaces the base env entirely instead of merging
-* @returns The object with only env templates rendered
-*/
-function renderEnvOnlyInObject(obj, envOverrides, replaceBase) {
-	if (getEnvBool("PROMPTFOO_DISABLE_TEMPLATING")) return obj;
-	if (typeof obj === "string") {
-		const nunjucks = getNunjucksEngine();
-		const baseEnvGlobals = nunjucks.getGlobal("env");
-		const envGlobals = replaceBase ? envOverrides ?? {} : envOverrides ? {
-			...baseEnvGlobals,
-			...envOverrides
-		} : baseEnvGlobals;
-		return obj.replace(/\{\{(?:[^}]|\}(?!\}))*\}\}/g, (match) => {
-			if (!match.match(/\benv\.|env\[/)) return match;
-			const varMatch = match.match(/env\.(\w+)|env\[['"]([^'"]+)['"]\]/);
-			const varName = varMatch?.[1] || varMatch?.[2];
-			if (match.includes("|") || varName && varName in envGlobals && envGlobals[varName] !== void 0) try {
-				return nunjucks.renderString(match, { env: envGlobals });
-			} catch (error) {
-				logger.debug(`Failed to render env template "${match}": ${error instanceof Error ? error.message : String(error)}`);
-				return match;
-			}
-			return match;
-		});
-	}
-	if (Array.isArray(obj)) return obj.map((item) => renderEnvOnlyInObject(item, envOverrides, replaceBase));
-	if (typeof obj === "object" && obj !== null) {
-		const result = {};
-		for (const key in obj) result[key] = renderEnvOnlyInObject(obj[key], envOverrides, replaceBase);
-		return result;
-	}
-	return obj;
-}
-function renderVarsInObject(obj, vars) {
-	if (!vars || getEnvBool("PROMPTFOO_DISABLE_TEMPLATING")) return obj;
-	if (typeof obj === "string") return getNunjucksEngine().renderString(obj, vars);
-	if (Array.isArray(obj)) return obj.map((item) => renderVarsInObject(item, vars));
-	if (typeof obj === "object" && obj !== null) {
-		const result = {};
-		for (const key in obj) result[key] = renderVarsInObject(obj[key], vars);
-		return result;
-	} else if (typeof obj === "function") return renderVarsInObject(obj({ vars }));
-	return obj;
-}
-//#endregion
 //#region src/util/file.ts
 /**
 * Simple Nunjucks engine specifically for file paths
@@ -619,10 +496,10 @@ function getResolvedRelativePath(filePath, isCloudConfig) {
 */
 function maybeLoadConfigFromExternalFile(config, context) {
 	if (Array.isArray(config)) return config.map((item) => maybeLoadConfigFromExternalFile(item, context));
-	if (config && typeof config === "object" && config !== null) {
+	if (typeof config === "object" && config !== null) {
 		const result = {};
 		for (const key of Object.keys(config)) {
-			const childContext = key === "value" && typeof config === "object" && config && "type" in config && typeof config.type === "string" && (config.type === "python" || config.type === "javascript") ? "assertion" : key === "vars" ? "vars" : context;
+			const childContext = key === "value" && "type" in config && typeof config.type === "string" && (config.type === "python" || config.type === "javascript") ? "assertion" : key === "vars" ? "vars" : context;
 			result[key] = maybeLoadConfigFromExternalFile(config[key], childContext);
 		}
 		return result;
@@ -1422,6 +1299,6 @@ function printBorder() {
 	logger.info(border);
 }
 //#endregion
-export { filterRuntimeVars as A, extractVariablesFromTemplates as C, setupEnv as D, parseFileUrl as E, getProviderDescription as F, isAnthropicProvider as I, isGoogleProvider as L, resultIsForTestCase as M, checkProviderApiKeys as N, deduplicateTestCases as O, doesProviderRefMatch as P, isOpenAiProvider as R, extractVariablesFromTemplate as S, loadFunction as T, parsePathOrGlob as _, ComparisonEvalNotFoundError as a, renderEnvOnlyInObject as b, mergeComparisonTables as c, getResolvedRelativePath as d, maybeLoadConfigFromExternalFile as f, maybeLoadToolsFromExternalFile as g, maybeLoadResponseFormatFromExternalFile as h, writeOutput as i, getTestCaseDeduplicationKey as j, extractRuntimeVars as k, fetchCsvFromGoogleSheet as l, maybeLoadFromExternalFileWithVars as m, createOutputMetadata as n, evalTableToJson as o, maybeLoadFromExternalFile as p, writeMultipleOutputs as r, generateEvalCsv as s, printBorder as t, getNunjucksEngineForFilePath as u, readFilters as v, getNunjucksEngine as w, renderVarsInObject as x, readOutput as y, isProviderAllowed as z };
+export { getProviderDescription as A, deduplicateTestCases as C, resultIsForTestCase as D, getTestCaseDeduplicationKey as E, isGoogleProvider as M, isOpenAiProvider as N, checkProviderApiKeys as O, isProviderAllowed as P, setupEnv as S, filterRuntimeVars as T, parsePathOrGlob as _, ComparisonEvalNotFoundError as a, loadFunction as b, mergeComparisonTables as c, getResolvedRelativePath as d, maybeLoadConfigFromExternalFile as f, maybeLoadToolsFromExternalFile as g, maybeLoadResponseFormatFromExternalFile as h, writeOutput as i, isAnthropicProvider as j, doesProviderRefMatch as k, fetchCsvFromGoogleSheet as l, maybeLoadFromExternalFileWithVars as m, createOutputMetadata as n, evalTableToJson as o, maybeLoadFromExternalFile as p, writeMultipleOutputs as r, generateEvalCsv as s, printBorder as t, getNunjucksEngineForFilePath as u, readFilters as v, extractRuntimeVars as w, parseFileUrl as x, readOutput as y };
 
-//# sourceMappingURL=util-Bm3E9jpK.js.map
+//# sourceMappingURL=util-DxWpWjhc.js.map

package/dist/src/{util-CMMkIxfU.js → util-oGMLA7vc.js}

@@ -1,11 +1,12 @@
-import { D as state, a as logger, b as getEnvBool, u as sanitizeObject } from "./logger-DO8_zM18.js";
+import { S as getEnvBool, a as logger, f as sanitizeObject, k as state } from "./logger-Ct2S6Yx-.js";
 import { t as invariant } from "./invariant-Ddh24eXh.js";
-import { o as safeResolve, r as importModule, t as getDirectory } from "./esm-SUNIX1x3.js";
-import { r as runPython } from "./pythonUtils-Cpo0Ez1p.js";
+import { o as safeResolve, r as importModule, t as getDirectory } from "./esm-C7PnfdF8.js";
+import { r as runPython } from "./pythonUtils-C2UQ30Rz.js";
 import { i as isJavascriptFile, t as JAVASCRIPT_EXTENSIONS } from "./fileExtensions-DnqA1y9x.js";
-import { A as isApiProvider, g as ResultFailureReason, h as OutputFileExtension, j as isProviderOptions } from "./types-q8GXGF65.js";
-import { N as VERSION, O as TERMINAL_MAX_WIDTH, t as fetchWithProxy } from "./fetch-CVAtKnI3.js";
-import * as fs$1 from "fs";
+import { A as isApiProvider, g as ResultFailureReason, h as OutputFileExtension, j as isProviderOptions } from "./types-BGQDAP8i.js";
+import { a as getNunjucksEngine, n as renderVarsInObject } from "./render-CH-62LbA.js";
+import { A as TERMINAL_MAX_WIDTH, F as VERSION, t as fetchWithProxy } from "./fetch-Di00EQrc.js";
+import * as fs$2 from "fs";
 import * as path$1 from "path";
 import path from "path";
 import yaml from "js-yaml";
@@ -310,7 +311,7 @@ function setupEnv(envPath) {
 			dotenv.config({ quiet: true });
 			return;
 		}
-		for (const p of paths) if (!fs$1.existsSync(p)) throw new Error(`Environment file not found: ${p}`);
+		for (const p of paths) if (!fs$2.existsSync(p)) throw new Error(`Environment file not found: ${p}`);
 		if (paths.length === 1) logger.info(`Loading environment variables from ${paths[0]}`);
 		else logger.info(`Loading environment variables from: ${paths.join(", ")}`);
 		const pathArg = paths.length === 1 ? paths[0] : paths;
@@ -370,130 +371,6 @@ function parseFileUrl(fileUrl) {
 	return { filePath: urlWithoutProtocol };
 }
 //#endregion
-//#region src/util/templates.ts
-/**
-* Get a Nunjucks engine instance with optional filters and configuration.
-* @param filters - Optional map of custom Nunjucks filters.
-* @param throwOnUndefined - Whether to throw an error on undefined variables.
-* @param isGrader - Whether this engine is being used in a grader context.
-* Nunjucks is always enabled in grader mode.
-* @returns A configured Nunjucks environment.
-*/
-function getNunjucksEngine(filters, throwOnUndefined = false, isGrader = false) {
-	if (!isGrader && getEnvBool("PROMPTFOO_DISABLE_TEMPLATING")) return { renderString: (template) => template };
-	const env = nunjucks.configure({
-		autoescape: false,
-		throwOnUndefined
-	});
-	const envGlobals = {
-		...getEnvBool("PROMPTFOO_DISABLE_TEMPLATE_ENV_VARS", getEnvBool("PROMPTFOO_SELF_HOSTED", false)) ? {} : process.env,
-		...state.config?.env
-	};
-	env.addGlobal("env", envGlobals);
-	env.addFilter("load", function(str) {
-		return JSON.parse(str);
-	});
-	if (filters) for (const [name, filter] of Object.entries(filters)) env.addFilter(name, filter);
-	return env;
-}
-/**
-* Parse Nunjucks template to extract variables.
-* @param template - The Nunjucks template string.
-* @returns An array of variables used in the template.
-*/
-function extractVariablesFromTemplate(template) {
-	const variableSet = /* @__PURE__ */ new Set();
-	const regex = /\{\{[\s]*([^{}\s|]+)[\s]*(?:\|[^}]+)?\}\}|\{%[\s]*(?:if|for)[\s]+([^{}\s]+)[\s]*.*?%\}/g;
-	template = template.replace(/\{#[\s\S]*?#\}/g, "");
-	let match;
-	while ((match = regex.exec(template)) !== null) {
-		const variable = match[1] || match[2];
-		if (variable) variableSet.add(variable);
-	}
-	const forLoopRegex = /\{%[\s]*for[\s]+(\w+)[\s]+in[\s]+(\w+)[\s]*%\}/g;
-	while ((match = forLoopRegex.exec(template)) !== null) {
-		variableSet.delete(match[1]);
-		variableSet.add(match[2]);
-	}
-	return Array.from(variableSet);
-}
-/**
-* Extract variables from multiple Nunjucks templates.
-* @param templates - An array of Nunjucks template strings.
-* @returns An array of variables used in the templates.
-*/
-function extractVariablesFromTemplates(templates) {
-	const variableSet = /* @__PURE__ */ new Set();
-	for (const template of templates) extractVariablesFromTemplate(template).forEach((variable) => variableSet.add(variable));
-	return Array.from(variableSet);
-}
-//#endregion
-//#region src/util/render.ts
-/**
-* Renders ONLY environment variable templates in an object, leaving all other templates untouched.
-* This allows env vars to be resolved at provider load time while preserving runtime var templates.
-*
-* Supports full Nunjucks syntax for env vars including filters and expressions:
-* - {{ env.VAR_NAME }}
-* - {{ env['VAR-NAME'] }}
-* - {{ env["VAR-NAME"] }}
-* - {{ env.VAR | default('fallback') }}
-* - {{ env.VAR | upper }}
-*
-* Preserves non-env templates for runtime rendering:
-* - {{ vars.x }} - preserved as literal
-* - {{ prompt }} - preserved as literal
-*
-* Implementation: Uses regex to find env templates, delegates to Nunjucks for rendering.
-* This ensures full Nunjucks feature support while preserving non-env templates.
-*
-* @param obj - The object to process
-* @param envOverrides - Optional env vars to merge with (or replace) the base env
-* @param replaceBase - If true, envOverrides replaces the base env entirely instead of merging
-* @returns The object with only env templates rendered
-*/
-function renderEnvOnlyInObject(obj, envOverrides, replaceBase) {
-	if (getEnvBool("PROMPTFOO_DISABLE_TEMPLATING")) return obj;
-	if (typeof obj === "string") {
-		const nunjucks = getNunjucksEngine();
-		const baseEnvGlobals = nunjucks.getGlobal("env");
-		const envGlobals = replaceBase ? envOverrides ?? {} : envOverrides ? {
-			...baseEnvGlobals,
-			...envOverrides
-		} : baseEnvGlobals;
-		return obj.replace(/\{\{(?:[^}]|\}(?!\}))*\}\}/g, (match) => {
-			if (!match.match(/\benv\.|env\[/)) return match;
-			const varMatch = match.match(/env\.(\w+)|env\[['"]([^'"]+)['"]\]/);
-			const varName = varMatch?.[1] || varMatch?.[2];
-			if (match.includes("|") || varName && varName in envGlobals && envGlobals[varName] !== void 0) try {
-				return nunjucks.renderString(match, { env: envGlobals });
-			} catch (error) {
-				logger.debug(`Failed to render env template "${match}": ${error instanceof Error ? error.message : String(error)}`);
-				return match;
-			}
-			return match;
-		});
-	}
-	if (Array.isArray(obj)) return obj.map((item) => renderEnvOnlyInObject(item, envOverrides, replaceBase));
-	if (typeof obj === "object" && obj !== null) {
-		const result = {};
-		for (const key in obj) result[key] = renderEnvOnlyInObject(obj[key], envOverrides, replaceBase);
-		return result;
-	}
-	return obj;
-}
-function renderVarsInObject(obj, vars) {
-	if (!vars || getEnvBool("PROMPTFOO_DISABLE_TEMPLATING")) return obj;
-	if (typeof obj === "string") return getNunjucksEngine().renderString(obj, vars);
-	if (Array.isArray(obj)) return obj.map((item) => renderVarsInObject(item, vars));
-	if (typeof obj === "object" && obj !== null) {
-		const result = {};
-		for (const key in obj) result[key] = renderVarsInObject(obj[key], vars);
-		return result;
-	} else if (typeof obj === "function") return renderVarsInObject(obj({ vars }));
-	return obj;
-}
-//#endregion
 //#region src/util/file.ts
 /**
 * Simple Nunjucks engine specifically for file paths
@@ -547,7 +424,7 @@ function maybeLoadFromExternalFile(filePath, context) {
 		for (const matchedFile of matchedFiles) {
 			let contents;
 			try {
-				contents = fs$1.readFileSync(matchedFile, "utf8");
+				contents = fs$2.readFileSync(matchedFile, "utf8");
 			} catch (error) {
 				if (error.code === "ENOENT") {
 					logger.debug(`File disappeared during glob expansion: ${matchedFile}`);
@@ -575,7 +452,7 @@ function maybeLoadFromExternalFile(filePath, context) {
 	const finalPath = resolvedPath;
 	let contents;
 	try {
-		contents = fs$1.readFileSync(finalPath, "utf8");
+		contents = fs$2.readFileSync(finalPath, "utf8");
 	} catch (error) {
 		if (error.code === "ENOENT") throw new Error(`File does not exist: ${finalPath}`);
 		throw new Error(`Failed to read file ${finalPath}: ${error}`);
@@ -618,10 +495,10 @@ function getResolvedRelativePath(filePath, isCloudConfig) {
 */
 function maybeLoadConfigFromExternalFile(config, context) {
 	if (Array.isArray(config)) return config.map((item) => maybeLoadConfigFromExternalFile(item, context));
-	if (config && typeof config === "object" && config !== null) {
+	if (typeof config === "object" && config !== null) {
 		const result = {};
 		for (const key of Object.keys(config)) {
-			const childContext = key === "value" && typeof config === "object" && config && "type" in config && typeof config.type === "string" && (config.type === "python" || config.type === "javascript") ? "assertion" : key === "vars" ? "vars" : context;
+			const childContext = key === "value" && "type" in config && typeof config.type === "string" && (config.type === "python" || config.type === "javascript") ? "assertion" : key === "vars" ? "vars" : context;
 			result[key] = maybeLoadConfigFromExternalFile(config[key], childContext);
 		}
 		return result;
@@ -653,7 +530,7 @@ function parsePathOrGlob(basePath, promptPath) {
 	}
 	let stats;
 	try {
-		stats = fs$1.statSync(path$1.join(basePath, filename));
+		stats = fs$2.statSync(path$1.join(basePath, filename));
 	} catch (err) {
 		if (getEnvBool("PROMPTFOO_STRICT_FILES")) throw err;
 	}
@@ -670,7 +547,7 @@ function parsePathOrGlob(basePath, promptPath) {
 function readOutput(outputPath) {
 	const ext = path$1.parse(outputPath).ext.slice(1);
 	switch (ext) {
-		case "json": return JSON.parse(fs$1.readFileSync(outputPath, "utf-8"));
+		case "json": return JSON.parse(fs$2.readFileSync(outputPath, "utf-8"));
 		default: throw new Error(`Unsupported output file format: ${ext} currently only supports json`);
 	}
 }
@@ -1296,6 +1173,6 @@ function printBorder() {
 	logger.info(border);
 }
 //#endregion
-export { getProviderDescription as A, deduplicateTestCases as C, resultIsForTestCase as D, getTestCaseDeduplicationKey as E, isGoogleProvider as M, isOpenAiProvider as N, checkProviderApiKeys as O, isProviderAllowed as P, setupEnv as S, filterRuntimeVars as T, extractVariablesFromTemplate as _, getNunjucksEngineForFilePath as a, loadFunction as b, maybeLoadFromExternalFile as c, maybeLoadToolsFromExternalFile as d, parsePathOrGlob as f, renderVarsInObject as g, renderEnvOnlyInObject as h, fetchCsvFromGoogleSheet as i, isAnthropicProvider as j, doesProviderRefMatch as k, maybeLoadFromExternalFileWithVars as l, readOutput as m, writeMultipleOutputs as n, getResolvedRelativePath as o, readFilters as p, writeOutput as r, maybeLoadConfigFromExternalFile as s, printBorder as t, maybeLoadResponseFormatFromExternalFile as u, extractVariablesFromTemplates as v, extractRuntimeVars as w, parseFileUrl as x, getNunjucksEngine as y };
+export { checkProviderApiKeys as C, isGoogleProvider as D, isAnthropicProvider as E, isOpenAiProvider as O, resultIsForTestCase as S, getProviderDescription as T, setupEnv as _, getNunjucksEngineForFilePath as a, filterRuntimeVars as b, maybeLoadFromExternalFile as c, maybeLoadToolsFromExternalFile as d, parsePathOrGlob as f, parseFileUrl as g, loadFunction as h, fetchCsvFromGoogleSheet as i, isProviderAllowed as k, maybeLoadFromExternalFileWithVars as l, readOutput as m, writeMultipleOutputs as n, getResolvedRelativePath as o, readFilters as p, writeOutput as r, maybeLoadConfigFromExternalFile as s, printBorder as t, maybeLoadResponseFormatFromExternalFile as u, deduplicateTestCases as v, doesProviderRefMatch as w, getTestCaseDeduplicationKey as x, extractRuntimeVars as y };
 
-//# sourceMappingURL=util-CMMkIxfU.js.map
+//# sourceMappingURL=util-oGMLA7vc.js.map