promptfoo 0.121.7 → 0.121.8

package/dist/src/index.cjs

@@ -6,47 +6,47 @@ const require_rolldown_runtime = require("./rolldown-runtime-D_mwlA32.cjs");
 const require_logger = require("./logger-cfNpzI4o.cjs");
 const require_invariant = require("./invariant-QtnLD03y.cjs");
 const require_types = require("./types-CxJvaY2S.cjs");
-const require_fetch = require("./fetch-Dw4XZHjj.cjs");
-const require_version = require("./version-BWCSaByA.cjs");
+const require_fetch = require("./fetch-Cfm4EuOB.cjs");
+const require_version = require("./version-VzUqOBZk.cjs");
 const require_fileExtensions = require("./fileExtensions-BhdwzYaD.cjs");
-const require_accounts = require("./accounts-CmWzeD2d.cjs");
+const require_accounts = require("./accounts-BVz5gHLK.cjs");
 const require_esm = require("./esm-BIKakvNa.cjs");
 const require_render = require("./render-BNTrbmBw.cjs");
 const require_providerRegistry = require("./providerRegistry-CZO_w7ue.cjs");
-const require_remoteGeneration = require("./remoteGeneration-DS9N3pgB.cjs");
-const require_server = require("./server-B8rqV126.cjs");
-const require_storage = require("./storage-CA-v9V2v.cjs");
+const require_remoteGeneration = require("./remoteGeneration-BpIYlb_O.cjs");
+const require_server = require("./server-BHOEL8p8.cjs");
+const require_storage = require("./storage-A4WnAeN3.cjs");
 const require_pythonUtils = require("./pythonUtils-Cokhluq3.cjs");
-const require_util = require("./util-SPsvFONY.cjs");
+const require_util = require("./util-COnzevXE.cjs");
 const require_createHash = require("./createHash-CSiqnK5P.cjs");
-const require_cache = require("./cache-CPGUA4Yl.cjs");
-const require_chat = require("./chat-DxTDQ83C.cjs");
-const require_transform = require("./transform-DhNkAUs8.cjs");
-const require_util$1 = require("./util-BSIuSLVK.cjs");
-const require_providers = require("./providers-BDVVIQM6.cjs");
+const require_cache = require("./cache-h5MWOBZI.cjs");
+const require_chat = require("./chat-DSyYuTYT.cjs");
+const require_transform = require("./transform-DJkt81VY.cjs");
+const require_util$1 = require("./util-NYQvo1C7.cjs");
+const require_providers = require("./providers-V6RBuieY.cjs");
 const require_tokenUsageUtils = require("./tokenUsageUtils-_B-P8IAi.cjs");
 const require_text = require("./text-nywWsRBM.cjs");
-const require_telemetry = require("./telemetry-CJ7FnCsc.cjs");
+const require_telemetry = require("./telemetry-C_ImlCOk.cjs");
 const require_tables = require("./tables-DBIJU0WE.cjs");
 const require_blobs = require("./blobs-B0977K1O.cjs");
 const require_processShim = require("./processShim-DSY9BV2T.cjs");
 const require_packageParser = require("./packageParser-QoCS1FMl.cjs");
 const require_rubyUtils = require("./rubyUtils-CqUWBZAt.cjs");
 const require_inputVariables = require("./inputVariables-Dq9W-Z3a.cjs");
-const require_util$2 = require("./util-CN8om2rz.cjs");
-const require_promptfoo = require("./promptfoo-b-baRMj-.cjs");
-const require_graders = require("./graders-ClrU2fnd.cjs");
-const require_shared = require("./shared-WkgnDkcg.cjs");
+const require_util$2 = require("./util-DNtUsus_.cjs");
+const require_promptfoo = require("./promptfoo-ClChwT74.cjs");
+const require_graders = require("./graders-CBQ2s6gz.cjs");
+const require_shared = require("./shared-aHWko3P1.cjs");
 const require_utils = require("./utils-Ve6kuJsa.cjs");
 const require_transform$1 = require("./transform-BnSXWmU_2.cjs");
 const require_store = require("./store-B2NDDooM.cjs");
-const require_extractor = require("./extractor-BdxEtt3J.cjs");
+const require_extractor = require("./extractor-bV_NOoaz.cjs");
 const require_promptLength = require("./promptLength-BbBbDHNj.cjs");
-const require_indirectWebPwn = require("./indirectWebPwn-BJ22AbQa.cjs");
+const require_indirectWebPwn = require("./indirectWebPwn-D7NA9Nsv.cjs");
 const require_toolAttributes = require("./toolAttributes-BAjwcBf0.cjs");
-const require_evaluatorHelpers = require("./evaluatorHelpers-CRqTvSux.cjs");
-const require_evalResult = require("./evalResult-CvtS8h8u.cjs");
-const require_strategies = require("./strategies-7CS3Alao.cjs");
+const require_evaluatorHelpers = require("./evaluatorHelpers-D06I9WFL.cjs");
+const require_evalResult = require("./evalResult-DHXs-9TL.cjs");
+const require_strategies = require("./strategies-CCcnEbFO.cjs");
 let fs = require("fs");
 fs = require_rolldown_runtime.__toESM(fs, 1);
 let path = require("path");
@@ -6334,7 +6334,7 @@ async function resolveDefaultTestProvider(defaultTest, testCase) {
 	const defaultProvider = defaultTest.provider;
 	if (require_types.isApiProvider(defaultProvider)) return defaultProvider;
 	if (typeof defaultProvider === "object" && defaultProvider.id) {
-		const { loadApiProvider } = await Promise.resolve().then(() => require("./providers-BDVVIQM6.cjs")).then((n) => n.providers_exports);
+		const { loadApiProvider } = await Promise.resolve().then(() => require("./providers-V6RBuieY.cjs")).then((n) => n.providers_exports);
 		return loadApiProvider(typeof defaultProvider.id === "function" ? defaultProvider.id() : defaultProvider.id, { options: defaultProvider });
 	}
 	return defaultProvider;
@@ -6494,7 +6494,7 @@ function buildRepeatCacheContextByTestIdx(runEvalOptions) {
 async function filterCompletedResumeSteps(runEvalOptions, evalRecord) {
 	if (!require_logger.state.resume || !evalRecord.persisted) return;
 	try {
-		const { default: EvalResult } = await Promise.resolve().then(() => require("./evalResult-CvtS8h8u.cjs")).then((n) => n.evalResult_exports);
+		const { default: EvalResult } = await Promise.resolve().then(() => require("./evalResult-DHXs-9TL.cjs")).then((n) => n.evalResult_exports);
 		const completedPairs = await EvalResult.getCompletedIndexPairs(evalRecord.id, { excludeErrors: require_logger.state.retryMode });
 		const originalCount = runEvalOptions.length;
 		for (let i = runEvalOptions.length - 1; i >= 0; i--) {
@@ -12135,6 +12135,7 @@ async function fetchRemoteTestCases(key, purpose, injectVar, n, config) {
 			require_logger.logger.error(`Error generating test cases for ${key}: ${statusText} ${JSON.stringify(data)}`);
 			return [];
 		}
+		if (require_promptfoo.requiresRemoteMaterialization(config?.inputs)) require_promptfoo.assertRemoteMaterializationHandled(data, `Remote plugin generation for ${key}`);
 		const ret = data.result;
 		require_logger.logger.debug(`Received remote generation for ${key}:\n${JSON.stringify(ret)}`);
 		return ret;
@@ -12143,31 +12144,6 @@ async function fetchRemoteTestCases(key, purpose, injectVar, n, config) {
 		return [];
 	}
 }
-async function materializeRemoteTestCaseInputs({ config, injectVar, pluginId, provider, purpose, testCases }) {
-	const inputs = config.inputs;
-	if (!inputs || Object.keys(inputs).length === 0) return testCases;
-	return Promise.all(testCases.map(async (testCase, materializationIndex) => {
-		const inputVars = require_util$2.extractInputVarsFromPrompt(String(testCase.vars?.[injectVar] ?? ""), inputs);
-		if (!inputVars) return testCase;
-		const materializedVars = await require_inputVariables.materializeInputVariablesWithMetadata(inputVars, inputs, {
-			materializationIndex,
-			pluginId,
-			provider,
-			purpose
-		});
-		return {
-			...testCase,
-			vars: {
-				...testCase.vars || {},
-				...materializedVars.vars
-			},
-			metadata: {
-				...testCase.metadata || {},
-				...materializedVars.metadata ? { inputMaterialization: materializedVars.metadata } : {}
-			}
-		};
-	}));
-}
 function createPluginFactory(PluginClass, key, validate) {
 	return {
 		key,
@@ -12179,14 +12155,7 @@ function createPluginFactory(PluginClass, key, validate) {
 				return new PluginClass(provider, purpose, injectVar, configWithDefaults).generateTests(n, delayMs);
 			}
 			const pluginId = require_util$2.getShortPluginId(key);
-			const testCases = await materializeRemoteTestCaseInputs({
-				config: configWithDefaults ?? {},
-				injectVar,
-				pluginId,
-				provider,
-				purpose,
-				testCases: await fetchRemoteTestCases(key, purpose, injectVar, n, configWithDefaults ?? {})
-			});
+			const testCases = await fetchRemoteTestCases(key, purpose, injectVar, n, configWithDefaults ?? {});
 			const computedModifiers = computeModifiersFromConfig(configWithDefaults);
 			return testCases.map((testCase) => ({
 				...testCase,
@@ -12273,14 +12242,7 @@ const piiPlugins = require_types.PII_PLUGINS.map((category) => ({
 	action: async (params) => {
 		if (require_remoteGeneration.shouldGenerateRemote()) {
 			const pluginId = require_util$2.getShortPluginId(category);
-			const testCases = await materializeRemoteTestCaseInputs({
-				config: params.config ?? {},
-				injectVar: params.injectVar,
-				pluginId,
-				provider: params.provider,
-				purpose: params.purpose,
-				testCases: await fetchRemoteTestCases(category, params.purpose, params.injectVar, params.n, params.config ?? {})
-			});
+			const testCases = await fetchRemoteTestCases(category, params.purpose, params.injectVar, params.n, params.config ?? {});
 			const computedModifiers = computeModifiersFromConfig(params.config);
 			return testCases.map((testCase) => ({
 				...testCase,
@@ -12312,14 +12274,7 @@ const biasPlugins = require_types.BIAS_PLUGINS.map((category) => ({
 			return [];
 		}
 		const pluginId = require_util$2.getShortPluginId(category);
-		const testCases = await materializeRemoteTestCaseInputs({
-			config: params.config ?? {},
-			injectVar: params.injectVar,
-			pluginId,
-			provider: params.provider,
-			purpose: params.purpose,
-			testCases: await fetchRemoteTestCases(category, params.purpose, params.injectVar, params.n, params.config ?? {})
-		});
+		const testCases = await fetchRemoteTestCases(category, params.purpose, params.injectVar, params.n, params.config ?? {});
 		const computedModifiers = computeModifiersFromConfig(params.config);
 		return testCases.map((testCase) => ({
 			...testCase,
@@ -12338,21 +12293,14 @@ function createRemotePlugin(key, validate) {
 	return {
 		key,
 		validate,
-		action: async ({ provider, purpose, injectVar, n, config }) => {
+		action: async ({ purpose, injectVar, n, config }) => {
 			const configWithDefaults = applyDefaultRemotePluginConfig(key, config);
 			if (require_remoteGeneration.neverGenerateRemote()) {
 				require_logger.logger.error(require_remoteGeneration.getRemoteGenerationExplicitlyDisabledError(`${key} plugin`));
 				return [];
 			}
 			const pluginId = require_util$2.getShortPluginId(key);
-			const testCases = await materializeRemoteTestCaseInputs({
-				config: configWithDefaults ?? {},
-				injectVar,
-				pluginId,
-				provider,
-				purpose,
-				testCases: await fetchRemoteTestCases(key, purpose, injectVar, n, configWithDefaults ?? {})
-			});
+			const testCases = await fetchRemoteTestCases(key, purpose, injectVar, n, configWithDefaults ?? {});
 			const computedModifiers = computeModifiersFromConfig(configWithDefaults);
 			const testsWithMetadata = testCases.map((testCase) => ({
 				...testCase,
@@ -12419,6 +12367,15 @@ async function validateSharpDependency(strategies, plugins, checkSharp = isSharp
 }
 //#endregion
 //#region src/redteam/index.ts
+const MATERIALIZED_MULTI_INPUT_PROMPT_METADATA_KEY = "__promptfooMaterializedMultiInputPrompt";
+function getMaterializedMultiInputPromptSnapshot(metadata) {
+	const snapshot = metadata?.[MATERIALIZED_MULTI_INPUT_PROMPT_METADATA_KEY];
+	return typeof snapshot === "string" ? snapshot : void 0;
+}
+function getMaterializedMultiInputPromptMetadata(vars) {
+	const prompt = vars?.[require_types.MULTI_INPUT_VAR];
+	return typeof prompt === "string" ? { [MATERIALIZED_MULTI_INPUT_PROMPT_METADATA_KEY]: prompt } : void 0;
+}
 function getPolicyText(metadata) {
 	if (!metadata || metadata.policy === void 0 || metadata.policy === null) return;
 	const policyValue = metadata.policy;
@@ -12431,12 +12388,19 @@ function getPolicyText(metadata) {
 async function rematerializeStrategyInputVars(testCase, injectVar, provider, purpose, materializationIndex) {
 	const inputs = testCase.metadata?.pluginConfig?.inputs;
 	const inputMaterialization = testCase.metadata?.inputMaterialization;
-	if (!inputs || Object.keys(inputs).length === 0 || !testCase.vars?.[injectVar]) return {
+	const materializedPromptSnapshot = getMaterializedMultiInputPromptSnapshot(testCase.metadata);
+	const currentInjectVar = testCase.vars?.[injectVar];
+	if (!inputs || Object.keys(inputs).length === 0 || !currentInjectVar) return {
+		inputMaterialization,
+		vars: testCase.vars
+	};
+	const promptChangedSinceMaterialization = typeof currentInjectVar === "string" && typeof materializedPromptSnapshot === "string" && currentInjectVar !== materializedPromptSnapshot;
+	if (Boolean(inputMaterialization) && Object.keys(inputs).every((key) => Object.prototype.hasOwnProperty.call(testCase.vars ?? {}, key)) && !promptChangedSinceMaterialization) return {
 		inputMaterialization,
 		vars: testCase.vars
 	};
 	try {
-		const materializedVars = await require_util$2.extractMaterializedVariablesFromJsonWithMetadata(JSON.parse(String(testCase.vars[injectVar])), inputs, {
+		const materializedVars = await require_util$2.extractMaterializedVariablesFromJsonWithMetadata(JSON.parse(String(currentInjectVar)), inputs, {
 			materializationIndex,
 			pluginId: String(testCase.metadata?.pluginId || "unknown-plugin"),
 			provider,
@@ -12668,7 +12632,8 @@ function addLanguageToPluginMetadata(test, lang, plugin, maxCharsPerMessage, tes
 				...test.metadata?.modifiers,
 				...languageToAdd
 			},
-			...languageToAdd
+			...languageToAdd,
+			...getMaterializedMultiInputPromptMetadata(test.vars)
 		}
 	};
 }
@@ -12767,7 +12732,8 @@ async function applyStrategies(testCases, strategies, injectVar, provider, purpo
 					...t?.metadata?.pluginId && { pluginId: t.metadata.pluginId },
 					...t?.metadata?.pluginConfig && { pluginConfig: t.metadata.pluginConfig },
 					...inputMaterialization && { inputMaterialization },
-					...Object.keys(strategyConfig).length > 0 && { strategyConfig }
+					...Object.keys(strategyConfig).length > 0 && { strategyConfig },
+					...getMaterializedMultiInputPromptMetadata(vars)
 				}
 			};
 		})));
@@ -13113,14 +13079,21 @@ async function synthesize({ abortSignal, delay, entities: entitiesOverride, inje
 				}
 			});
 			const languageResults = await Promise.allSettled(languagePromises);
-			for (const result of languageResults) if (result.status === "fulfilled") {
+			for (const [index, result] of languageResults.entries()) if (result.status === "fulfilled") {
 				const { lang, tests, requested, generated } = result.value;
 				allPluginTests.push(...tests);
 				resultsPerLanguage[lang || "default"] = {
 					requested,
 					generated
 				};
-			} else require_logger.logger.warn(`[Language Processing] Error generating tests for ${plugin.id}: ${result.reason}`);
+			} else {
+				const lang = languages[index];
+				require_logger.logger.warn(`[Language Processing] Error generating tests for ${plugin.id}: ${result.reason}`);
+				resultsPerLanguage[lang || "default"] = {
+					requested: plugin.numTests,
+					generated: 0
+				};
+			}
 			require_logger.logger.debug(`[Language Processing] Total tests generated for ${plugin.id}: ${allPluginTests.length} (across ${languages.length} language(s))`);
 			if (!Array.isArray(allPluginTests) || allPluginTests.length === 0) require_logger.logger.warn(`Failed to generate tests for ${plugin.id}`);
 			else {
@@ -13154,35 +13127,51 @@ async function synthesize({ abortSignal, delay, entities: entitiesOverride, inje
 				generated: allPluginTests.length
 			};
 		} else if (plugin.id.startsWith("file://")) try {
-			const customTests = await new CustomPlugin(redteamProvider, purpose, injectVar, plugin.id, resolvePluginConfigWithMaxChars(plugin.config, maxCharsPerMessage)).generateTests(plugin.numTests, delay);
-			const testCasesWithMetadata = filterOversizedTestCases(customTests.map((t) => {
-				const includePluginConfig = !(t.metadata && Object.hasOwn(t.metadata, "pluginConfig") && t.metadata.pluginConfig === void 0);
-				const pluginConfigWithMaxChars = {
+			const languageConfig = plugin.config?.language ?? language;
+			const languages = Array.isArray(languageConfig) ? languageConfig : languageConfig ? [languageConfig] : [void 0];
+			const allCustomTests = [];
+			const resultsPerLanguage = {};
+			const languagePromises = languages.map(async (lang) => {
+				const resolvedConfig = {
 					...resolvePluginConfigWithMaxChars(plugin.config, maxCharsPerMessage),
-					...t.metadata?.pluginConfig ?? {}
+					...lang ? { language: lang } : {},
+					...hasMultipleInputs ? { inputs } : {}
 				};
-				const modifiers = {
-					...buildRedteamModifiers({
+				const customPluginConfig = {
+					...resolvedConfig,
+					modifiers: buildRedteamModifiers({
 						maxCharsPerMessage,
-						pluginConfig: pluginConfigWithMaxChars,
+						pluginConfig: resolvedConfig,
 						testGenerationInstructions
-					}),
-					...t.metadata?.modifiers
+					})
 				};
+				const testCasesWithMetadata = filterOversizedTestCases((await new CustomPlugin(redteamProvider, purpose, injectVar, plugin.id, customPluginConfig).generateTests(plugin.numTests, delay)).map((t) => addLanguageToPluginMetadata(t, lang, plugin, maxCharsPerMessage, testGenerationInstructions)), injectVar, `Custom plugin ${plugin.id}`, maxCharsPerMessage);
 				return {
-					...t,
-					metadata: {
-						...t.metadata || {},
-						pluginId: plugin.id,
-						...includePluginConfig && { pluginConfig: pluginConfigWithMaxChars },
-						severity: plugin.severity || getPluginSeverity(plugin.id, resolvePluginConfig(plugin.config)),
-						modifiers
-					}
+					lang,
+					tests: testCasesWithMetadata,
+					requested: plugin.numTests,
+					generated: testCasesWithMetadata.length
+				};
+			});
+			const languageResults = await Promise.allSettled(languagePromises);
+			for (const [index, result] of languageResults.entries()) if (result.status === "fulfilled") {
+				const { lang, tests, requested, generated } = result.value;
+				allCustomTests.push(...tests);
+				resultsPerLanguage[lang || "default"] = {
+					requested,
+					generated
 				};
-			}), injectVar, `Custom plugin ${plugin.id}`, maxCharsPerMessage);
+			} else {
+				const lang = languages[index];
+				require_logger.logger.warn(`[Language Processing] Error generating tests for custom plugin ${plugin.id}: ${result.reason}`);
+				resultsPerLanguage[lang || "default"] = {
+					requested: plugin.numTests,
+					generated: 0
+				};
+			}
 			if (needsGoalExtraction) {
-				require_logger.logger.debug(`Extracting goal for ${testCasesWithMetadata.length} custom tests from ${plugin.id}...`);
-				for (const testCase of testCasesWithMetadata) {
+				require_logger.logger.debug(`Extracting goal for ${allCustomTests.length} custom tests from ${plugin.id}...`);
+				for (const testCase of allCustomTests) {
 					const promptVar = testCase.vars?.[injectVar];
 					const prompt = Array.isArray(promptVar) ? promptVar[0] : String(promptVar);
 					const policy = getPolicyText(testCase.metadata);
@@ -13190,13 +13179,21 @@ async function synthesize({ abortSignal, delay, entities: entitiesOverride, inje
 					testCase.metadata.goal = extractedGoal;
 				}
 			}
-			testCases.push(...testCasesWithMetadata);
-			require_logger.logger.debug(`Added ${customTests.length} custom test cases from ${plugin.id}`);
-			const displayId = getPluginDisplayId(plugin);
-			pluginResults[displayId] = {
-				requested: plugin.numTests,
-				generated: testCasesWithMetadata.length
+			testCases.push(...allCustomTests);
+			require_logger.logger.debug(`Added ${allCustomTests.length} custom test cases from ${plugin.id}`);
+			const baseDisplayId = getPluginDisplayId(plugin);
+			if (languages.filter((lang) => lang !== void 0).length > 1) for (const [langKey, result] of Object.entries(resultsPerLanguage)) {
+				const displayId = langKey === "en" ? baseDisplayId : `(${langKey}) ${baseDisplayId}`;
+				pluginResults[displayId] = {
+					requested: result.requested,
+					generated: result.generated
+				};
+			}
+			else pluginResults[baseDisplayId] = {
+				requested: plugin.numTests * languages.length,
+				generated: allCustomTests.length
 			};
+			progressBar?.increment(plugin.numTests * languages.length);
 		} catch (e) {
 			require_logger.logger.error(`Error generating tests for custom plugin ${plugin.id}: ${e}`);
 			const displayId = getPluginDisplayId(plugin);

package/dist/src/index.d.cts

@@ -480,6 +480,15 @@ interface ProviderResponse {
   cached?: boolean;
   cost?: number;
   error?: string;
+  /**
+   * Indicates that a remote Promptfoo server already materialized multi-input vars
+   * for this response. When true, callers must not re-materialize locally.
+   */
+  materializationHandled?: boolean;
+  /**
+   * Materialized per-input vars returned by a remote Promptfoo server.
+   */
+  materializedVars?: Record<string, string>;
   /**
    * Indicates that `output` contains base64-encoded binary data (often as JSON like OpenAI `b64_json`).
    * Used to enable blob externalization and avoid token bloat in downstream grading/agentic strategies.
@@ -513,6 +522,10 @@ interface ProviderResponse {
   prompt?: string | ChatMessage[];
   raw?: string | any;
   output?: string | any;
+  /**
+   * Input materialization metadata returned by a remote Promptfoo server.
+   */
+  inputMaterialization?: Record<string, unknown>;
   /**
    * Output after provider-level transform. Used by contextTransform to ensure
    * it operates on provider-normalized output, independent of test transforms.

package/dist/src/index.d.ts

@@ -481,6 +481,15 @@ interface ProviderResponse {
   cached?: boolean;
   cost?: number;
   error?: string;
+  /**
+   * Indicates that a remote Promptfoo server already materialized multi-input vars
+   * for this response. When true, callers must not re-materialize locally.
+   */
+  materializationHandled?: boolean;
+  /**
+   * Materialized per-input vars returned by a remote Promptfoo server.
+   */
+  materializedVars?: Record<string, string>;
   /**
    * Indicates that `output` contains base64-encoded binary data (often as JSON like OpenAI `b64_json`).
    * Used to enable blob externalization and avoid token bloat in downstream grading/agentic strategies.
@@ -514,6 +523,10 @@ interface ProviderResponse {
   prompt?: string | ChatMessage[];
   raw?: string | any;
   output?: string | any;
+  /**
+   * Input materialization metadata returned by a remote Promptfoo server.
+   */
+  inputMaterialization?: Record<string, unknown>;
   /**
    * Output after provider-level transform. Used by contextTransform to ensure
    * it operates on provider-normalized output, independent of test transforms.