promptfoo 0.122.1 → 0.122.2

package/dist/src/{util-L5T81aJ0.cjs → util-BPkxeLuk.cjs}

@@ -1,6 +1,6 @@
 const require_logger = require("./logger-BN3n9c1R.cjs");
 const require_invariant = require("./invariant-C2JOzzO2.cjs");
-const require_fetch = require("./fetch-DTnUr1aF.cjs");
+const require_fetch = require("./fetch-D54lzFcD.cjs");
 const require_tables = require("./tables-BaWJhXDV.cjs");
 const require_fileExtensions = require("./fileExtensions-BQZEFw03.cjs");
 const require_time = require("./time-B7AqDy5g.cjs");
@@ -8,6 +8,7 @@ const require_esm = require("./esm-DQhzWczC.cjs");
 const require_genaiTracer = require("./genaiTracer-D_3mCxrD.cjs");
 const require_createHash = require("./createHash-sm6Q6LWg.cjs");
 const require_pythonUtils = require("./pythonUtils-CqlMEmzG.cjs");
+const require_render = require("./render-C1VJjFT2.cjs");
 require("./blobs-DsXWJXf6.cjs");
 const require_extractor = require("./extractor-C6JkPnKn.cjs");
 const require_tokenUsageUtils = require("./tokenUsageUtils-BbqyaBi4.cjs");
@@ -26,12 +27,12 @@ js_yaml = require_logger.__toESM(js_yaml, 1);
 let dedent = require("dedent");
 dedent = require_logger.__toESM(dedent, 1);
 let crypto$1 = require("crypto");
-let nunjucks = require("nunjucks");
-nunjucks = require_logger.__toESM(nunjucks, 1);
 let fast_deep_equal = require("fast-deep-equal");
 fast_deep_equal = require_logger.__toESM(fast_deep_equal, 1);
 let csv_parse_sync = require("csv-parse/sync");
 let glob = require("glob");
+let nunjucks = require("nunjucks");
+nunjucks = require_logger.__toESM(nunjucks, 1);
 let fast_xml_parser = require("fast-xml-parser");
 let node_util = require("node:util");
 let drizzle_orm = require("drizzle-orm");
@@ -54,348 +55,6 @@ const FILE_METADATA_KEY = "_promptfooFileMetadata";
 */
 const HUMAN_ASSERTION_TYPE = "human";
 //#endregion
-//#region src/util/templates.ts
-function isNunjucksAstNode(value) {
-	return Boolean(value && typeof value === "object" && typeof value.typename === "string");
-}
-function getNunjucksParser() {
-	return nunjucks.default.parser;
-}
-function parseNunjucksTemplate(template) {
-	const parser = getNunjucksParser();
-	if (!parser || typeof parser.parse !== "function") {
-		require_logger.logger.debug("[templates] nunjucks.parser.parse is not available; falling back to conservative detection");
-		return { ok: false };
-	}
-	try {
-		return {
-			ok: true,
-			ast: parser.parse(template)
-		};
-	} catch (err) {
-		require_logger.logger.debug("[templates] nunjucks parse failed; falling back to conservative detection", { error: err instanceof Error ? err.message : String(err) });
-		return { ok: false };
-	}
-}
-function isNonReferenceSymbol(parent) {
-	if (!parent) return false;
-	return parent.node.typename === "Pair" && parent.field === "key" || parent.node.typename === "Filter" && parent.field === "name" || parent.node.typename === "Is" && parent.field === "right" || parent.node.typename === "Set" && parent.field === "targets" || isForLikeNode(parent.node) && parent.field === "name" || parent.node.typename === "Macro" && parent.field === "name" || parent.node.typename === "Import" && parent.field === "target" || parent.node.typename === "FromImport" && parent.field === "names";
-}
-function getSymbolName(node) {
-	return isNunjucksAstNode(node) && node.typename === "Symbol" && typeof node.value === "string" ? node.value : void 0;
-}
-function collectBindingNames(node) {
-	if (Array.isArray(node)) return node.flatMap((item) => collectBindingNames(item));
-	const symbolName = getSymbolName(node);
-	if (symbolName) return [symbolName];
-	if (!isNunjucksAstNode(node) || !Array.isArray(node.children)) return [];
-	return node.children.flatMap((child) => collectBindingNames(child));
-}
-function addBindingNames(scope, node) {
-	const nextScope = new Set(scope);
-	for (const name of collectBindingNames(node)) nextScope.add(name);
-	return nextScope;
-}
-function addNamesToScope(scope, names) {
-	const nextScope = new Set(scope);
-	for (const name of names) nextScope.add(name);
-	return nextScope;
-}
-function collectFromImportBindingNames(namesNode) {
-	if (!isNunjucksAstNode(namesNode) || !Array.isArray(namesNode.children)) return [];
-	const names = [];
-	for (const child of namesNode.children) {
-		const plainImportName = getSymbolName(child);
-		if (plainImportName) {
-			names.push(plainImportName);
-			continue;
-		}
-		if (isNunjucksAstNode(child) && child.typename === "Pair") {
-			const aliasName = getSymbolName(child.value);
-			if (aliasName) names.push(aliasName);
-		}
-	}
-	return names;
-}
-function astNodeListReferencesVariable(nodes, variableName, parent, boundSymbols) {
-	let scope = new Set(boundSymbols);
-	for (const item of nodes) {
-		if (!isNunjucksAstNode(item)) continue;
-		if (astReferencesVariable(item, variableName, parent, scope)) return true;
-		if (item.typename === "Set") scope = addBindingNames(scope, item.targets);
-		else if (item.typename === "Macro") scope = addBindingNames(scope, item.name);
-		else if (item.typename === "Import") scope = addBindingNames(scope, item.target);
-		else if (item.typename === "FromImport") scope = addNamesToScope(scope, collectFromImportBindingNames(item.names));
-	}
-	return false;
-}
-function astChildReferencesVariable(node, field, variableName, boundSymbols) {
-	const child = node[field];
-	if (Array.isArray(child)) return child.some((item) => isNunjucksAstNode(item) && astReferencesVariable(item, variableName, {
-		field,
-		node
-	}, boundSymbols));
-	return isNunjucksAstNode(child) && astReferencesVariable(child, variableName, {
-		field,
-		node
-	}, boundSymbols);
-}
-function isForLikeNode(node) {
-	return node.typename === "For" || node.typename === "AsyncEach" || node.typename === "AsyncAll";
-}
-function forLikeNodeReferencesVariable(node, variableName, boundSymbols) {
-	const loopScope = addBindingNames(boundSymbols, node.name);
-	return astChildReferencesVariable(node, "arr", variableName, boundSymbols) || astChildReferencesVariable(node, "body", variableName, loopScope) || astChildReferencesVariable(node, "else_", variableName, boundSymbols);
-}
-/**
-* Walk a Nunjucks macro/caller signature left-to-right. Positional args and
-* earlier keyword defaults are visible to later defaults, but the current
-* keyword is not bound until its own default value has been evaluated.
-*/
-function analyzeSignatureArgs(argsNode, variableName, boundSymbols) {
-	const paramNames = [];
-	if (!isNunjucksAstNode(argsNode) || !Array.isArray(argsNode.children)) return {
-		paramNames,
-		referencesVariable: false
-	};
-	const defaultScope = new Set(boundSymbols);
-	for (const child of argsNode.children) {
-		if (!isNunjucksAstNode(child)) continue;
-		const positionalParamName = getSymbolName(child);
-		if (positionalParamName) {
-			paramNames.push(positionalParamName);
-			defaultScope.add(positionalParamName);
-			continue;
-		}
-		if (child.typename === "KeywordArgs" && Array.isArray(child.children)) {
-			for (const pair of child.children) {
-				const keywordParamName = isNunjucksAstNode(pair) && pair.typename === "Pair" ? getSymbolName(pair.key) : void 0;
-				if (isNunjucksAstNode(pair) && pair.typename === "Pair" && astChildReferencesVariable(pair, "value", variableName, defaultScope)) return {
-					paramNames,
-					referencesVariable: true
-				};
-				if (keywordParamName) {
-					paramNames.push(keywordParamName);
-					defaultScope.add(keywordParamName);
-				}
-			}
-			continue;
-		}
-		if (astReferencesVariable(child, variableName, {
-			field: "children",
-			node: argsNode
-		}, defaultScope)) return {
-			paramNames,
-			referencesVariable: true
-		};
-	}
-	return {
-		paramNames,
-		referencesVariable: false
-	};
-}
-function macroNodeReferencesVariable(node, variableName, boundSymbols) {
-	const { paramNames, referencesVariable } = analyzeSignatureArgs(node.args, variableName, boundSymbols);
-	if (referencesVariable) return true;
-	const macroScope = addNamesToScope(boundSymbols, paramNames);
-	const macroName = getSymbolName(node.name);
-	if (macroName) macroScope.add(macroName);
-	return astChildReferencesVariable(node, "body", variableName, macroScope);
-}
-function isNodeReferencesVariable(node, variableName, boundSymbols) {
-	if (astChildReferencesVariable(node, "left", variableName, boundSymbols)) return true;
-	const right = node.right;
-	if (!isNunjucksAstNode(right)) return false;
-	if (right.typename === "FunCall") return astChildReferencesVariable(right, "args", variableName, boundSymbols);
-	return right.typename !== "Symbol" && astReferencesVariable(right, variableName, {
-		field: "right",
-		node
-	}, boundSymbols);
-}
-function setNodeReferencesVariable(node, variableName, boundSymbols) {
-	return astChildReferencesVariable(node, "value", variableName, boundSymbols) || astChildReferencesVariable(node, "body", variableName, boundSymbols);
-}
-function fromImportNodeReferencesVariable(node, variableName, boundSymbols) {
-	return astChildReferencesVariable(node, "template", variableName, boundSymbols);
-}
-function blockNodeReferencesVariable(node, variableName, boundSymbols) {
-	return astChildReferencesVariable(node, "body", variableName, boundSymbols);
-}
-function callerNodeReferencesVariable(node, variableName, boundSymbols) {
-	const { paramNames, referencesVariable } = analyzeSignatureArgs(node.args, variableName, boundSymbols);
-	if (referencesVariable) return true;
-	return astChildReferencesVariable(node, "body", variableName, addNamesToScope(boundSymbols, paramNames));
-}
-function astFieldsReferenceVariable(node, variableName, boundSymbols) {
-	return (node.fields ?? []).some((field) => astChildReferencesVariable(node, field, variableName, boundSymbols));
-}
-function astReferencesVariable(node, variableName, parent, boundSymbols = /* @__PURE__ */ new Set()) {
-	if (node.typename === "Symbol" && node.value === variableName) return !boundSymbols.has(variableName) && !isNonReferenceSymbol(parent);
-	if ((node.typename === "Root" || node.typename === "NodeList") && Array.isArray(node.children)) return astNodeListReferencesVariable(node.children, variableName, {
-		field: "children",
-		node
-	}, boundSymbols);
-	if (isForLikeNode(node)) return forLikeNodeReferencesVariable(node, variableName, boundSymbols);
-	if (node.typename === "Macro") return macroNodeReferencesVariable(node, variableName, boundSymbols);
-	if (node.typename === "Is") return isNodeReferencesVariable(node, variableName, boundSymbols);
-	if (node.typename === "Set") return setNodeReferencesVariable(node, variableName, boundSymbols);
-	if (node.typename === "FromImport") return fromImportNodeReferencesVariable(node, variableName, boundSymbols);
-	if (node.typename === "Block") return blockNodeReferencesVariable(node, variableName, boundSymbols);
-	if (node.typename === "Caller") return callerNodeReferencesVariable(node, variableName, boundSymbols);
-	return astFieldsReferenceVariable(node, variableName, boundSymbols);
-}
-/**
-* Get a Nunjucks engine instance with optional filters and configuration.
-* @param filters - Optional map of custom Nunjucks filters.
-* @param throwOnUndefined - Whether to throw an error on undefined variables.
-* @param isGrader - Whether this engine is being used in a grader context.
-* Nunjucks is always enabled in grader mode.
-* @returns A configured Nunjucks environment.
-*/
-function getNunjucksEngine(filters, throwOnUndefined = false, isGrader = false) {
-	if (!isGrader && require_logger.getEnvBool("PROMPTFOO_DISABLE_TEMPLATING")) return { renderString: (template) => template };
-	const env = nunjucks.default.configure({
-		autoescape: false,
-		throwOnUndefined
-	});
-	const envGlobals = {
-		...require_logger.getEnvBool("PROMPTFOO_DISABLE_TEMPLATE_ENV_VARS", require_logger.getEnvBool("PROMPTFOO_SELF_HOSTED", false)) ? {} : process.env,
-		...require_logger.state.config?.env
-	};
-	env.addGlobal("env", envGlobals);
-	env.addFilter("load", function(str) {
-		return JSON.parse(str);
-	});
-	if (filters) for (const [name, filter] of Object.entries(filters)) env.addFilter(name, filter);
-	return env;
-}
-/**
-* Parse Nunjucks template to extract variables.
-* @param template - The Nunjucks template string.
-* @returns An array of variables used in the template.
-*/
-function extractVariablesFromTemplate(template) {
-	const variableSet = /* @__PURE__ */ new Set();
-	const regex = /\{\{[\s]*([^{}\s|]+)[\s]*(?:\|[^}]+)?\}\}|\{%[\s]*(?:if|for)[\s]+([^{}\s]+)[\s]*.*?%\}/g;
-	template = template.replace(/\{#[\s\S]*?#\}/g, "");
-	let match;
-	while ((match = regex.exec(template)) !== null) {
-		const variable = match[1] || match[2];
-		if (variable) variableSet.add(variable);
-	}
-	const forLoopRegex = /\{%[\s]*for[\s]+(\w+)[\s]+in[\s]+(\w+)[\s]*%\}/g;
-	while ((match = forLoopRegex.exec(template)) !== null) {
-		variableSet.delete(match[1]);
-		variableSet.add(match[2]);
-	}
-	return Array.from(variableSet);
-}
-/**
-* Extract variables from multiple Nunjucks templates.
-* @param templates - An array of Nunjucks template strings.
-* @returns An array of variables used in the templates.
-*/
-function extractVariablesFromTemplates(templates) {
-	const variableSet = /* @__PURE__ */ new Set();
-	for (const template of templates) extractVariablesFromTemplate(template).forEach((variable) => variableSet.add(variable));
-	return Array.from(variableSet);
-}
-/**
-* Classify whether a template references `variableName` as a real Nunjucks
-* expression symbol, and surface whether the template parsed successfully.
-*
-* Callers that cache results should avoid caching when `parsed` is `false`,
-* otherwise a transient parse failure would poison the cache for the lifetime
-* of the process.
-*/
-function analyzeTemplateReference(template, variableName) {
-	if (!variableName || !template.includes(variableName)) return {
-		referenced: false,
-		parsed: true
-	};
-	const parseResult = parseNunjucksTemplate(template);
-	if (!parseResult.ok) return {
-		referenced: true,
-		parsed: false
-	};
-	return {
-		referenced: astReferencesVariable(parseResult.ast, variableName),
-		parsed: true
-	};
-}
-//#endregion
-//#region src/util/render.ts
-/**
-* Renders ONLY environment variable templates in an object, leaving all other templates untouched.
-* This allows env vars to be resolved at provider load time while preserving runtime var templates.
-*
-* Supports full Nunjucks syntax for env vars including filters and expressions:
-* - {{ env.VAR_NAME }}
-* - {{ env['VAR-NAME'] }}
-* - {{ env["VAR-NAME"] }}
-* - {{ env.VAR | default('fallback') }}
-* - {{ env.VAR | upper }}
-*
-* Preserves non-env templates for runtime rendering:
-* - {{ vars.x }} - preserved as literal
-* - {{ prompt }} - preserved as literal
-*
-* Implementation: Uses regex to find env templates, delegates to Nunjucks for rendering.
-* This ensures full Nunjucks feature support while preserving non-env templates.
-*
-* @param obj - The object to process
-* @param envOverrides - Optional env vars to merge with (or replace) the base env
-* @param replaceBase - If true, envOverrides replaces the base env entirely instead of merging
-* @returns The object with only env templates rendered
-*/
-function renderEnvOnlyInObject(obj, envOverrides, replaceBase) {
-	if (require_logger.getEnvBool("PROMPTFOO_DISABLE_TEMPLATING")) return obj;
-	if (typeof obj === "string") {
-		const nunjucks = getNunjucksEngine();
-		const baseEnvGlobals = nunjucks.getGlobal("env");
-		const envGlobals = replaceBase ? envOverrides ?? {} : envOverrides ? {
-			...baseEnvGlobals,
-			...envOverrides
-		} : baseEnvGlobals;
-		return obj.replace(/\{\{(?:[^}]|\}(?!\}))*\}\}/g, (match) => {
-			if (!match.match(/\benv\.|env\[/)) return match;
-			const varMatch = match.match(/env\.(\w+)|env\[['"]([^'"]+)['"]\]/);
-			const varName = varMatch?.[1] || varMatch?.[2];
-			if (match.includes("|") || varName && varName in envGlobals && envGlobals[varName] !== void 0) try {
-				return nunjucks.renderString(match, { env: envGlobals });
-			} catch (error) {
-				require_logger.logger.debug(`Failed to render env template "${match}": ${error instanceof Error ? error.message : String(error)}`);
-				return match;
-			}
-			return match;
-		});
-	}
-	if (Array.isArray(obj)) return obj.map((item) => renderEnvOnlyInObject(item, envOverrides, replaceBase));
-	if (typeof obj === "object" && obj !== null) {
-		const result = {};
-		for (const key in obj) {
-			if (key === "_conversation") {
-				result[key] = obj[key];
-				continue;
-			}
-			result[key] = renderEnvOnlyInObject(obj[key], envOverrides, replaceBase);
-		}
-		return result;
-	}
-	return obj;
-}
-function renderVarsInObject(obj, vars) {
-	if (!vars || require_logger.getEnvBool("PROMPTFOO_DISABLE_TEMPLATING")) return obj;
-	if (typeof obj === "string") return getNunjucksEngine().renderString(obj, vars);
-	if (Array.isArray(obj)) return obj.map((item) => renderVarsInObject(item, vars));
-	if (typeof obj === "object" && obj !== null) {
-		const result = {};
-		for (const key in obj) result[key] = renderVarsInObject(obj[key], vars);
-		return result;
-	} else if (typeof obj === "function") return renderVarsInObject(obj({ vars }));
-	return obj;
-}
-//#endregion
 //#region src/util/functions/loadFunction.ts
 const functionCache = {};
 /**
@@ -837,7 +496,7 @@ async function readFilters(filters, basePath = "") {
 * @returns The processed configuration with variables rendered and files loaded
 */
 function maybeLoadFromExternalFileWithVars(config, vars) {
-	return maybeLoadFromExternalFile(renderVarsInObject(config, vars));
+	return maybeLoadFromExternalFile(require_render.renderVarsInObject(config, vars));
 }
 /**
 * Loads response_format configuration from an external file with variable rendering.
@@ -854,12 +513,12 @@ function maybeLoadFromExternalFileWithVars(config, vars) {
 */
 function maybeLoadResponseFormatFromExternalFile(responseFormat, vars) {
 	if (responseFormat === void 0 || responseFormat === null) return responseFormat;
-	const loaded = maybeLoadFromExternalFile(renderVarsInObject(responseFormat, vars));
+	const loaded = maybeLoadFromExternalFile(require_render.renderVarsInObject(responseFormat, vars));
 	if (!loaded || typeof loaded !== "object") return loaded;
 	if (loaded.type === "json_schema") {
 		const nestedSchema = loaded.schema || loaded.json_schema?.schema;
 		if (nestedSchema) {
-			const loadedSchema = maybeLoadFromExternalFile(renderVarsInObject(nestedSchema, vars));
+			const loadedSchema = maybeLoadFromExternalFile(require_render.renderVarsInObject(nestedSchema, vars));
 			if (loaded.schema !== void 0) return {
 				...loaded,
 				schema: loadedSchema
@@ -888,7 +547,7 @@ function maybeLoadResponseFormatFromExternalFile(responseFormat, vars) {
 * @throws {Error} If the loaded tools are in an invalid format
 */
 async function maybeLoadToolsFromExternalFile(tools, vars) {
-	const rendered = renderVarsInObject(tools, vars);
+	const rendered = require_render.renderVarsInObject(tools, vars);
 	if (typeof rendered === "string" && rendered.startsWith("file://")) {
 		const { filePath, functionName } = parseFileUrl(rendered);
 		if (functionName && (filePath.endsWith(".py") || require_fileExtensions.isJavascriptFile(filePath))) {
@@ -3477,7 +3136,7 @@ async function writeOutput(outputPath, evalRecord, shareableUrl, options = {}) {
 		const errors = reportOutputs.filter((output) => !output.pass && output.failureReason === require_tables.ResultFailureReason.ERROR).length;
 		const failures = totalResults - successes - errors;
 		const passRate = totalResults > 0 ? successes / totalResults * 100 : 0;
-		const htmlOutput = getNunjucksEngine().renderString(template, {
+		const htmlOutput = require_render.getNunjucksEngine().renderString(template, {
 			config: redactedConfig,
 			table: htmlTable,
 			results: summary,
@@ -3603,12 +3262,6 @@ Object.defineProperty(exports, "PROMPT_DELIMITER", {
 		return PROMPT_DELIMITER;
 	}
 });
-Object.defineProperty(exports, "analyzeTemplateReference", {
-	enumerable: true,
-	get: function() {
-		return analyzeTemplateReference;
-	}
-});
 Object.defineProperty(exports, "asEvaluateResult", {
 	enumerable: true,
 	get: function() {
@@ -3639,18 +3292,6 @@ Object.defineProperty(exports, "extractRuntimeVars", {
 		return extractRuntimeVars;
 	}
 });
-Object.defineProperty(exports, "extractVariablesFromTemplate", {
-	enumerable: true,
-	get: function() {
-		return extractVariablesFromTemplate;
-	}
-});
-Object.defineProperty(exports, "extractVariablesFromTemplates", {
-	enumerable: true,
-	get: function() {
-		return extractVariablesFromTemplates;
-	}
-});
 Object.defineProperty(exports, "fetchCsvFromGoogleSheet", {
 	enumerable: true,
 	get: function() {
@@ -3675,12 +3316,6 @@ Object.defineProperty(exports, "getCachedResultsCount", {
 		return getCachedResultsCount;
 	}
 });
-Object.defineProperty(exports, "getNunjucksEngine", {
-	enumerable: true,
-	get: function() {
-		return getNunjucksEngine;
-	}
-});
 Object.defineProperty(exports, "getNunjucksEngineForFilePath", {
 	enumerable: true,
 	get: function() {
@@ -3921,18 +3556,6 @@ Object.defineProperty(exports, "readProviderConfigFile", {
 		return readProviderConfigFile;
 	}
 });
-Object.defineProperty(exports, "renderEnvOnlyInObject", {
-	enumerable: true,
-	get: function() {
-		return renderEnvOnlyInObject;
-	}
-});
-Object.defineProperty(exports, "renderVarsInObject", {
-	enumerable: true,
-	get: function() {
-		return renderVarsInObject;
-	}
-});
 Object.defineProperty(exports, "resultIsForTestCase", {
 	enumerable: true,
 	get: function() {
@@ -3988,4 +3611,4 @@ Object.defineProperty(exports, "writeMultipleOutputs", {
 	}
 });
 
-//# sourceMappingURL=util-L5T81aJ0.cjs.map
+//# sourceMappingURL=util-BPkxeLuk.cjs.map

package/dist/src/{util-Ur5TGF_w.js → util-BPlSGvsQ.js}

@@ -1,6 +1,7 @@
 import { o as logger } from "./logger-D4ion89i.js";
-import { Et as renderVarsInObject } from "./util-uYYKYi7_.js";
-import { t as fetchWithProxy } from "./fetch-BdWQKDMj.js";
+import "./util-BPtpPW4O.js";
+import { t as fetchWithProxy } from "./fetch-DnDg-Ics.js";
+import { n as renderVarsInObject } from "./render-DIhuKgcQ.js";
 //#region src/util/oauth.ts
 /**
 * Buffer time before token expiry to trigger proactive refresh (60 seconds)
@@ -231,4 +232,4 @@ function requiresAsyncAuth(server) {
 //#endregion
 export { getOAuthToken as a, isMcpToolNameFilter as c, TOKEN_REFRESH_BUFFER_MS as d, getMcpErrorMessage as i, renderAuthVars as l, getAuthHeaders as n, getOAuthTokenWithExpiry as o, getAuthQueryParams as r, isMcpErrorResult as s, applyQueryParams as t, requiresAsyncAuth as u };
 
-//# sourceMappingURL=util-Ur5TGF_w.js.map
+//# sourceMappingURL=util-BPlSGvsQ.js.map