promptfoo 0.121.7 → 0.121.9

package/dist/src/index.cjs

@@ -3,50 +3,50 @@ Object.defineProperties(exports, {
 	[Symbol.toStringTag]: { value: "Module" }
 });
 const require_rolldown_runtime = require("./rolldown-runtime-D_mwlA32.cjs");
-const require_logger = require("./logger-cfNpzI4o.cjs");
+const require_logger = require("./logger-Cm0Nb8YX.cjs");
 const require_invariant = require("./invariant-QtnLD03y.cjs");
 const require_types = require("./types-CxJvaY2S.cjs");
-const require_fetch = require("./fetch-Dw4XZHjj.cjs");
-const require_version = require("./version-BWCSaByA.cjs");
+const require_fetch = require("./fetch-B2HvaVUc.cjs");
+const require_version = require("./version-XjGPawtU.cjs");
 const require_fileExtensions = require("./fileExtensions-BhdwzYaD.cjs");
-const require_accounts = require("./accounts-CmWzeD2d.cjs");
-const require_esm = require("./esm-BIKakvNa.cjs");
-const require_render = require("./render-BNTrbmBw.cjs");
-const require_providerRegistry = require("./providerRegistry-CZO_w7ue.cjs");
-const require_remoteGeneration = require("./remoteGeneration-DS9N3pgB.cjs");
-const require_server = require("./server-B8rqV126.cjs");
-const require_storage = require("./storage-CA-v9V2v.cjs");
-const require_pythonUtils = require("./pythonUtils-Cokhluq3.cjs");
-const require_util = require("./util-SPsvFONY.cjs");
+const require_accounts = require("./accounts-DAcQXPwE.cjs");
+const require_esm = require("./esm-vv4rR-Dp.cjs");
+const require_render = require("./render-DJ4yipCD.cjs");
+const require_providerRegistry = require("./providerRegistry-DKbi2EbP.cjs");
+const require_remoteGeneration = require("./remoteGeneration-CD3c_rzc.cjs");
+const require_server = require("./server-yHhoBx98.cjs");
+const require_storage = require("./storage-Cv7QYUPO.cjs");
+const require_pythonUtils = require("./pythonUtils-BjuymNdM.cjs");
+const require_util = require("./util-BuYzkbEz.cjs");
 const require_createHash = require("./createHash-CSiqnK5P.cjs");
-const require_cache = require("./cache-CPGUA4Yl.cjs");
-const require_chat = require("./chat-DxTDQ83C.cjs");
-const require_transform = require("./transform-DhNkAUs8.cjs");
-const require_util$1 = require("./util-BSIuSLVK.cjs");
-const require_providers = require("./providers-BDVVIQM6.cjs");
+const require_cache = require("./cache-WuIsgzMA.cjs");
+const require_chat = require("./chat-DkitQHdt.cjs");
+const require_transform = require("./transform-DKw2FNwg.cjs");
+const require_util$1 = require("./util-Cs372i1_.cjs");
+const require_providers = require("./providers-DP1CDgd5.cjs");
 const require_tokenUsageUtils = require("./tokenUsageUtils-_B-P8IAi.cjs");
 const require_text = require("./text-nywWsRBM.cjs");
-const require_telemetry = require("./telemetry-CJ7FnCsc.cjs");
-const require_tables = require("./tables-DBIJU0WE.cjs");
-const require_blobs = require("./blobs-B0977K1O.cjs");
+const require_telemetry = require("./telemetry-DvzmDrSM.cjs");
+const require_tables = require("./tables-D_s2HMPP.cjs");
+const require_blobs = require("./blobs-5wbW7TbH.cjs");
 const require_processShim = require("./processShim-DSY9BV2T.cjs");
-const require_packageParser = require("./packageParser-QoCS1FMl.cjs");
-const require_rubyUtils = require("./rubyUtils-CqUWBZAt.cjs");
-const require_inputVariables = require("./inputVariables-Dq9W-Z3a.cjs");
-const require_util$2 = require("./util-CN8om2rz.cjs");
-const require_promptfoo = require("./promptfoo-b-baRMj-.cjs");
-const require_graders = require("./graders-ClrU2fnd.cjs");
-const require_shared = require("./shared-WkgnDkcg.cjs");
-const require_utils = require("./utils-Ve6kuJsa.cjs");
-const require_transform$1 = require("./transform-BnSXWmU_2.cjs");
-const require_store = require("./store-B2NDDooM.cjs");
-const require_extractor = require("./extractor-BdxEtt3J.cjs");
-const require_promptLength = require("./promptLength-BbBbDHNj.cjs");
-const require_indirectWebPwn = require("./indirectWebPwn-BJ22AbQa.cjs");
+const require_packageParser = require("./packageParser-DR054ceU.cjs");
+const require_rubyUtils = require("./rubyUtils-BEqeZynS.cjs");
+const require_inputVariables = require("./inputVariables-D5K2vTp4.cjs");
+const require_util$2 = require("./util-3BRss2KA.cjs");
+const require_promptfoo = require("./promptfoo-DAXBmrE6.cjs");
+const require_graders = require("./graders-CQv1GT5Q.cjs");
+const require_shared = require("./shared-DlmtkptR.cjs");
+const require_utils = require("./utils-B_HCobst.cjs");
+const require_transform$1 = require("./transform-0wnBYYaL2.cjs");
+const require_store = require("./store-BKN4VD-m.cjs");
+const require_extractor = require("./extractor-nyUwvU2U.cjs");
+const require_promptLength = require("./promptLength-CAcg7BmO.cjs");
+const require_indirectWebPwn = require("./indirectWebPwn-04ziKbly.cjs");
 const require_toolAttributes = require("./toolAttributes-BAjwcBf0.cjs");
-const require_evaluatorHelpers = require("./evaluatorHelpers-CRqTvSux.cjs");
-const require_evalResult = require("./evalResult-CvtS8h8u.cjs");
-const require_strategies = require("./strategies-7CS3Alao.cjs");
+const require_evaluatorHelpers = require("./evaluatorHelpers-2gpZ9Q_f.cjs");
+const require_evalResult = require("./evalResult-BZld6I3I.cjs");
+const require_strategies = require("./strategies-BD7US9WB.cjs");
 let fs = require("fs");
 fs = require_rolldown_runtime.__toESM(fs, 1);
 let path = require("path");
@@ -820,7 +820,7 @@ async function startOtlpReceiverIfNeeded(testSuite) {
 		require_telemetry.telemetry.record("feature_used", { feature: "tracing" });
 		try {
 			require_logger.logger.debug("[EvaluatorTracing] Tracing configuration detected, starting OTLP receiver");
-			const { startOTLPReceiver } = await Promise.resolve().then(() => require("./otlpReceiver-CvJdBGSc.cjs"));
+			const { startOTLPReceiver } = await Promise.resolve().then(() => require("./otlpReceiver-DOl5S07u.cjs"));
 			const port = testSuite.tracing.otlp.http.port || 4318;
 			const host = testSuite.tracing.otlp.http.host || "127.0.0.1";
 			const acceptFormats = normalizeOtlpAcceptFormats(testSuite.tracing.otlp.http.acceptFormats);
@@ -844,7 +844,7 @@ async function startOtlpReceiverIfNeeded(testSuite) {
 async function stopOtlpReceiverIfNeeded() {
 	if (otlpReceiverStarted) try {
 		require_logger.logger.debug("[EvaluatorTracing] Stopping OTLP receiver");
-		const { stopOTLPReceiver } = await Promise.resolve().then(() => require("./otlpReceiver-CvJdBGSc.cjs"));
+		const { stopOTLPReceiver } = await Promise.resolve().then(() => require("./otlpReceiver-DOl5S07u.cjs"));
 		await stopOTLPReceiver();
 		otlpReceiverStarted = false;
 		require_logger.logger.info("[EvaluatorTracing] OTLP receiver stopped successfully");
@@ -879,7 +879,7 @@ async function generateTraceContextIfNeeded(test, evaluateOptions, testIdx, prom
 	}
 	if (!tracingEnabled) return null;
 	require_logger.logger.debug("[EvaluatorTracing] Importing trace store");
-	const { getTraceStore } = await Promise.resolve().then(() => require("./store-B2NDDooM.cjs")).then((n) => n.store_exports);
+	const { getTraceStore } = await Promise.resolve().then(() => require("./store-BKN4VD-m.cjs")).then((n) => n.store_exports);
 	const traceStore = getTraceStore();
 	const traceId = generateTraceId();
 	const spanId = generateSpanId();
@@ -3133,7 +3133,7 @@ async function loadWebSearchProvider(preferAnthropic = false) {
 	};
 	const loadOpenAIWebSearch = async () => {
 		try {
-			return await require_providers.loadApiProvider("openai:responses:gpt-5.4-2026-03-05", { options: { config: { tools: [{ type: "web_search_preview" }] } } });
+			return await require_providers.loadApiProvider("openai:responses:gpt-5.5-2026-04-23", { options: { config: { tools: [{ type: "web_search_preview" }] } } });
 		} catch (err) {
 			require_logger.logger.debug(`Failed to load OpenAI web search provider: ${err}`);
 			return null;
@@ -4484,7 +4484,7 @@ async function runAssertion({ prompt, provider, assertion, test, vars, latencyMs
 			};
 		}
 		else if (filePath.endsWith(".rb")) try {
-			const { runRuby } = await Promise.resolve().then(() => require("./rubyUtils-CqUWBZAt.cjs")).then((n) => n.rubyUtils_exports);
+			const { runRuby } = await Promise.resolve().then(() => require("./rubyUtils-BEqeZynS.cjs")).then((n) => n.rubyUtils_exports);
 			valueFromScript = await runRuby(filePath, functionName || "get_assert", [output, context]);
 			require_logger.logger.debug(`Ruby script ${filePath} output: ${valueFromScript}`);
 		} catch (error) {
@@ -6334,7 +6334,7 @@ async function resolveDefaultTestProvider(defaultTest, testCase) {
 	const defaultProvider = defaultTest.provider;
 	if (require_types.isApiProvider(defaultProvider)) return defaultProvider;
 	if (typeof defaultProvider === "object" && defaultProvider.id) {
-		const { loadApiProvider } = await Promise.resolve().then(() => require("./providers-BDVVIQM6.cjs")).then((n) => n.providers_exports);
+		const { loadApiProvider } = await Promise.resolve().then(() => require("./providers-DP1CDgd5.cjs")).then((n) => n.providers_exports);
 		return loadApiProvider(typeof defaultProvider.id === "function" ? defaultProvider.id() : defaultProvider.id, { options: defaultProvider });
 	}
 	return defaultProvider;
@@ -6494,7 +6494,7 @@ function buildRepeatCacheContextByTestIdx(runEvalOptions) {
 async function filterCompletedResumeSteps(runEvalOptions, evalRecord) {
 	if (!require_logger.state.resume || !evalRecord.persisted) return;
 	try {
-		const { default: EvalResult } = await Promise.resolve().then(() => require("./evalResult-CvtS8h8u.cjs")).then((n) => n.evalResult_exports);
+		const { default: EvalResult } = await Promise.resolve().then(() => require("./evalResult-BZld6I3I.cjs")).then((n) => n.evalResult_exports);
 		const completedPairs = await EvalResult.getCompletedIndexPairs(evalRecord.id, { excludeErrors: require_logger.state.retryMode });
 		const originalCount = runEvalOptions.length;
 		for (let i = runEvalOptions.length - 1; i >= 0; i--) {
@@ -11076,7 +11076,7 @@ async function fetchRemoteGeneration(task, prompts) {
 			method: "POST",
 			headers: { "Content-Type": "application/json" },
 			body: JSON.stringify(body)
-		}, require_fetch.REQUEST_TIMEOUT_MS, "json");
+		}, require_fetch.getRequestTimeoutMs(), "json");
 		return RedTeamGenerationResponse.parse(response.data).result;
 	} catch (error) {
 		require_logger.logger.warn(`Error using remote generation for task '${task}': ${error}`);
@@ -11249,7 +11249,7 @@ const DATASET_URL$2 = `https://raw.githubusercontent.com/promptfoo/promptfoo/ref
 const DATASET_URL_MULTILINGUAL = `https://raw.githubusercontent.com/promptfoo/promptfoo/refs/tags/${DATASET_VERSION}/examples/cyberseceval/prompt_injection_multilingual.json`;
 async function fetchDataset$2(limit, isMultilingual) {
 	try {
-		const response = await require_fetch.fetchWithTimeout(isMultilingual ? DATASET_URL_MULTILINGUAL : DATASET_URL$2, {}, require_fetch.REQUEST_TIMEOUT_MS);
+		const response = await require_fetch.fetchWithTimeout(isMultilingual ? DATASET_URL_MULTILINGUAL : DATASET_URL$2, {}, require_fetch.getRequestTimeoutMs());
 		if (!response.ok) throw new Error(`[CyberSecEval] HTTP status: ${response.status} ${response.statusText}`);
 		const data = await response.json();
 		if (!data || !Array.isArray(data)) throw new Error(`[CyberSecEval] Invalid response from ${DATASET_URL$2}`);
@@ -11317,7 +11317,7 @@ async function fetchDataset$1(limit) {
 		require_logger.logger.debug(`[DoNotAnswer] Fetching dataset from ${DATASET_URL$1}`);
 		let csvData;
 		if (DATASET_URL$1.startsWith("http")) {
-			const response = await require_fetch.fetchWithTimeout(DATASET_URL$1, {}, require_fetch.REQUEST_TIMEOUT_MS);
+			const response = await require_fetch.fetchWithTimeout(DATASET_URL$1, {}, require_fetch.getRequestTimeoutMs());
 			if (!response.ok) throw new Error(`[DoNotAnswer] HTTP status: ${response.status} ${response.statusText}`);
 			csvData = await response.text();
 		} else try {
@@ -11852,7 +11852,7 @@ async function fetchDataset(limit) {
 		require_logger.logger.debug(`[XSTest] Fetching dataset from ${DATASET_URL}`);
 		let csvData;
 		if (DATASET_URL.startsWith("http")) {
-			const response = await require_fetch.fetchWithTimeout(DATASET_URL, {}, require_fetch.REQUEST_TIMEOUT_MS);
+			const response = await require_fetch.fetchWithTimeout(DATASET_URL, {}, require_fetch.getRequestTimeoutMs());
 			if (!response.ok) throw new Error(`[XSTest] HTTP status: ${response.status} ${response.statusText}`);
 			csvData = await response.text();
 		} else try {
@@ -12130,11 +12130,12 @@ async function fetchRemoteTestCases(key, purpose, injectVar, n, config) {
 			method: "POST",
 			headers: { "Content-Type": "application/json" },
 			body
-		}, require_fetch.REQUEST_TIMEOUT_MS);
+		}, require_fetch.getRequestTimeoutMs());
 		if (status !== 200 || !data || !data.result || !Array.isArray(data.result)) {
 			require_logger.logger.error(`Error generating test cases for ${key}: ${statusText} ${JSON.stringify(data)}`);
 			return [];
 		}
+		if (require_promptfoo.requiresRemoteMaterialization(config?.inputs)) require_promptfoo.assertRemoteMaterializationHandled(data, `Remote plugin generation for ${key}`);
 		const ret = data.result;
 		require_logger.logger.debug(`Received remote generation for ${key}:\n${JSON.stringify(ret)}`);
 		return ret;
@@ -12143,31 +12144,6 @@ async function fetchRemoteTestCases(key, purpose, injectVar, n, config) {
 		return [];
 	}
 }
-async function materializeRemoteTestCaseInputs({ config, injectVar, pluginId, provider, purpose, testCases }) {
-	const inputs = config.inputs;
-	if (!inputs || Object.keys(inputs).length === 0) return testCases;
-	return Promise.all(testCases.map(async (testCase, materializationIndex) => {
-		const inputVars = require_util$2.extractInputVarsFromPrompt(String(testCase.vars?.[injectVar] ?? ""), inputs);
-		if (!inputVars) return testCase;
-		const materializedVars = await require_inputVariables.materializeInputVariablesWithMetadata(inputVars, inputs, {
-			materializationIndex,
-			pluginId,
-			provider,
-			purpose
-		});
-		return {
-			...testCase,
-			vars: {
-				...testCase.vars || {},
-				...materializedVars.vars
-			},
-			metadata: {
-				...testCase.metadata || {},
-				...materializedVars.metadata ? { inputMaterialization: materializedVars.metadata } : {}
-			}
-		};
-	}));
-}
 function createPluginFactory(PluginClass, key, validate) {
 	return {
 		key,
@@ -12179,14 +12155,7 @@ function createPluginFactory(PluginClass, key, validate) {
 				return new PluginClass(provider, purpose, injectVar, configWithDefaults).generateTests(n, delayMs);
 			}
 			const pluginId = require_util$2.getShortPluginId(key);
-			const testCases = await materializeRemoteTestCaseInputs({
-				config: configWithDefaults ?? {},
-				injectVar,
-				pluginId,
-				provider,
-				purpose,
-				testCases: await fetchRemoteTestCases(key, purpose, injectVar, n, configWithDefaults ?? {})
-			});
+			const testCases = await fetchRemoteTestCases(key, purpose, injectVar, n, configWithDefaults ?? {});
 			const computedModifiers = computeModifiersFromConfig(configWithDefaults);
 			return testCases.map((testCase) => ({
 				...testCase,
@@ -12273,14 +12242,7 @@ const piiPlugins = require_types.PII_PLUGINS.map((category) => ({
 	action: async (params) => {
 		if (require_remoteGeneration.shouldGenerateRemote()) {
 			const pluginId = require_util$2.getShortPluginId(category);
-			const testCases = await materializeRemoteTestCaseInputs({
-				config: params.config ?? {},
-				injectVar: params.injectVar,
-				pluginId,
-				provider: params.provider,
-				purpose: params.purpose,
-				testCases: await fetchRemoteTestCases(category, params.purpose, params.injectVar, params.n, params.config ?? {})
-			});
+			const testCases = await fetchRemoteTestCases(category, params.purpose, params.injectVar, params.n, params.config ?? {});
 			const computedModifiers = computeModifiersFromConfig(params.config);
 			return testCases.map((testCase) => ({
 				...testCase,
@@ -12312,14 +12274,7 @@ const biasPlugins = require_types.BIAS_PLUGINS.map((category) => ({
 			return [];
 		}
 		const pluginId = require_util$2.getShortPluginId(category);
-		const testCases = await materializeRemoteTestCaseInputs({
-			config: params.config ?? {},
-			injectVar: params.injectVar,
-			pluginId,
-			provider: params.provider,
-			purpose: params.purpose,
-			testCases: await fetchRemoteTestCases(category, params.purpose, params.injectVar, params.n, params.config ?? {})
-		});
+		const testCases = await fetchRemoteTestCases(category, params.purpose, params.injectVar, params.n, params.config ?? {});
 		const computedModifiers = computeModifiersFromConfig(params.config);
 		return testCases.map((testCase) => ({
 			...testCase,
@@ -12338,21 +12293,14 @@ function createRemotePlugin(key, validate) {
 	return {
 		key,
 		validate,
-		action: async ({ provider, purpose, injectVar, n, config }) => {
+		action: async ({ purpose, injectVar, n, config }) => {
 			const configWithDefaults = applyDefaultRemotePluginConfig(key, config);
 			if (require_remoteGeneration.neverGenerateRemote()) {
 				require_logger.logger.error(require_remoteGeneration.getRemoteGenerationExplicitlyDisabledError(`${key} plugin`));
 				return [];
 			}
 			const pluginId = require_util$2.getShortPluginId(key);
-			const testCases = await materializeRemoteTestCaseInputs({
-				config: configWithDefaults ?? {},
-				injectVar,
-				pluginId,
-				provider,
-				purpose,
-				testCases: await fetchRemoteTestCases(key, purpose, injectVar, n, configWithDefaults ?? {})
-			});
+			const testCases = await fetchRemoteTestCases(key, purpose, injectVar, n, configWithDefaults ?? {});
 			const computedModifiers = computeModifiersFromConfig(configWithDefaults);
 			const testsWithMetadata = testCases.map((testCase) => ({
 				...testCase,
@@ -12419,6 +12367,15 @@ async function validateSharpDependency(strategies, plugins, checkSharp = isSharp
 }
 //#endregion
 //#region src/redteam/index.ts
+const MATERIALIZED_MULTI_INPUT_PROMPT_METADATA_KEY = "__promptfooMaterializedMultiInputPrompt";
+function getMaterializedMultiInputPromptSnapshot(metadata) {
+	const snapshot = metadata?.[MATERIALIZED_MULTI_INPUT_PROMPT_METADATA_KEY];
+	return typeof snapshot === "string" ? snapshot : void 0;
+}
+function getMaterializedMultiInputPromptMetadata(vars) {
+	const prompt = vars?.[require_types.MULTI_INPUT_VAR];
+	return typeof prompt === "string" ? { [MATERIALIZED_MULTI_INPUT_PROMPT_METADATA_KEY]: prompt } : void 0;
+}
 function getPolicyText(metadata) {
 	if (!metadata || metadata.policy === void 0 || metadata.policy === null) return;
 	const policyValue = metadata.policy;
@@ -12431,12 +12388,19 @@ function getPolicyText(metadata) {
 async function rematerializeStrategyInputVars(testCase, injectVar, provider, purpose, materializationIndex) {
 	const inputs = testCase.metadata?.pluginConfig?.inputs;
 	const inputMaterialization = testCase.metadata?.inputMaterialization;
-	if (!inputs || Object.keys(inputs).length === 0 || !testCase.vars?.[injectVar]) return {
+	const materializedPromptSnapshot = getMaterializedMultiInputPromptSnapshot(testCase.metadata);
+	const currentInjectVar = testCase.vars?.[injectVar];
+	if (!inputs || Object.keys(inputs).length === 0 || !currentInjectVar) return {
+		inputMaterialization,
+		vars: testCase.vars
+	};
+	const promptChangedSinceMaterialization = typeof currentInjectVar === "string" && typeof materializedPromptSnapshot === "string" && currentInjectVar !== materializedPromptSnapshot;
+	if (Boolean(inputMaterialization) && Object.keys(inputs).every((key) => Object.prototype.hasOwnProperty.call(testCase.vars ?? {}, key)) && !promptChangedSinceMaterialization) return {
 		inputMaterialization,
 		vars: testCase.vars
 	};
 	try {
-		const materializedVars = await require_util$2.extractMaterializedVariablesFromJsonWithMetadata(JSON.parse(String(testCase.vars[injectVar])), inputs, {
+		const materializedVars = await require_util$2.extractMaterializedVariablesFromJsonWithMetadata(JSON.parse(String(currentInjectVar)), inputs, {
 			materializationIndex,
 			pluginId: String(testCase.metadata?.pluginId || "unknown-plugin"),
 			provider,
@@ -12668,7 +12632,8 @@ function addLanguageToPluginMetadata(test, lang, plugin, maxCharsPerMessage, tes
 				...test.metadata?.modifiers,
 				...languageToAdd
 			},
-			...languageToAdd
+			...languageToAdd,
+			...getMaterializedMultiInputPromptMetadata(test.vars)
 		}
 	};
 }
@@ -12767,7 +12732,8 @@ async function applyStrategies(testCases, strategies, injectVar, provider, purpo
 					...t?.metadata?.pluginId && { pluginId: t.metadata.pluginId },
 					...t?.metadata?.pluginConfig && { pluginConfig: t.metadata.pluginConfig },
 					...inputMaterialization && { inputMaterialization },
-					...Object.keys(strategyConfig).length > 0 && { strategyConfig }
+					...Object.keys(strategyConfig).length > 0 && { strategyConfig },
+					...getMaterializedMultiInputPromptMetadata(vars)
 				}
 			};
 		})));
@@ -13113,14 +13079,21 @@ async function synthesize({ abortSignal, delay, entities: entitiesOverride, inje
 				}
 			});
 			const languageResults = await Promise.allSettled(languagePromises);
-			for (const result of languageResults) if (result.status === "fulfilled") {
+			for (const [index, result] of languageResults.entries()) if (result.status === "fulfilled") {
 				const { lang, tests, requested, generated } = result.value;
 				allPluginTests.push(...tests);
 				resultsPerLanguage[lang || "default"] = {
 					requested,
 					generated
 				};
-			} else require_logger.logger.warn(`[Language Processing] Error generating tests for ${plugin.id}: ${result.reason}`);
+			} else {
+				const lang = languages[index];
+				require_logger.logger.warn(`[Language Processing] Error generating tests for ${plugin.id}: ${result.reason}`);
+				resultsPerLanguage[lang || "default"] = {
+					requested: plugin.numTests,
+					generated: 0
+				};
+			}
 			require_logger.logger.debug(`[Language Processing] Total tests generated for ${plugin.id}: ${allPluginTests.length} (across ${languages.length} language(s))`);
 			if (!Array.isArray(allPluginTests) || allPluginTests.length === 0) require_logger.logger.warn(`Failed to generate tests for ${plugin.id}`);
 			else {
@@ -13154,35 +13127,51 @@ async function synthesize({ abortSignal, delay, entities: entitiesOverride, inje
 				generated: allPluginTests.length
 			};
 		} else if (plugin.id.startsWith("file://")) try {
-			const customTests = await new CustomPlugin(redteamProvider, purpose, injectVar, plugin.id, resolvePluginConfigWithMaxChars(plugin.config, maxCharsPerMessage)).generateTests(plugin.numTests, delay);
-			const testCasesWithMetadata = filterOversizedTestCases(customTests.map((t) => {
-				const includePluginConfig = !(t.metadata && Object.hasOwn(t.metadata, "pluginConfig") && t.metadata.pluginConfig === void 0);
-				const pluginConfigWithMaxChars = {
+			const languageConfig = plugin.config?.language ?? language;
+			const languages = Array.isArray(languageConfig) ? languageConfig : languageConfig ? [languageConfig] : [void 0];
+			const allCustomTests = [];
+			const resultsPerLanguage = {};
+			const languagePromises = languages.map(async (lang) => {
+				const resolvedConfig = {
 					...resolvePluginConfigWithMaxChars(plugin.config, maxCharsPerMessage),
-					...t.metadata?.pluginConfig ?? {}
+					...lang ? { language: lang } : {},
+					...hasMultipleInputs ? { inputs } : {}
 				};
-				const modifiers = {
-					...buildRedteamModifiers({
+				const customPluginConfig = {
+					...resolvedConfig,
+					modifiers: buildRedteamModifiers({
 						maxCharsPerMessage,
-						pluginConfig: pluginConfigWithMaxChars,
+						pluginConfig: resolvedConfig,
 						testGenerationInstructions
-					}),
-					...t.metadata?.modifiers
+					})
 				};
+				const testCasesWithMetadata = filterOversizedTestCases((await new CustomPlugin(redteamProvider, purpose, injectVar, plugin.id, customPluginConfig).generateTests(plugin.numTests, delay)).map((t) => addLanguageToPluginMetadata(t, lang, plugin, maxCharsPerMessage, testGenerationInstructions)), injectVar, `Custom plugin ${plugin.id}`, maxCharsPerMessage);
 				return {
-					...t,
-					metadata: {
-						...t.metadata || {},
-						pluginId: plugin.id,
-						...includePluginConfig && { pluginConfig: pluginConfigWithMaxChars },
-						severity: plugin.severity || getPluginSeverity(plugin.id, resolvePluginConfig(plugin.config)),
-						modifiers
-					}
+					lang,
+					tests: testCasesWithMetadata,
+					requested: plugin.numTests,
+					generated: testCasesWithMetadata.length
+				};
+			});
+			const languageResults = await Promise.allSettled(languagePromises);
+			for (const [index, result] of languageResults.entries()) if (result.status === "fulfilled") {
+				const { lang, tests, requested, generated } = result.value;
+				allCustomTests.push(...tests);
+				resultsPerLanguage[lang || "default"] = {
+					requested,
+					generated
 				};
-			}), injectVar, `Custom plugin ${plugin.id}`, maxCharsPerMessage);
+			} else {
+				const lang = languages[index];
+				require_logger.logger.warn(`[Language Processing] Error generating tests for custom plugin ${plugin.id}: ${result.reason}`);
+				resultsPerLanguage[lang || "default"] = {
+					requested: plugin.numTests,
+					generated: 0
+				};
+			}
 			if (needsGoalExtraction) {
-				require_logger.logger.debug(`Extracting goal for ${testCasesWithMetadata.length} custom tests from ${plugin.id}...`);
-				for (const testCase of testCasesWithMetadata) {
+				require_logger.logger.debug(`Extracting goal for ${allCustomTests.length} custom tests from ${plugin.id}...`);
+				for (const testCase of allCustomTests) {
 					const promptVar = testCase.vars?.[injectVar];
 					const prompt = Array.isArray(promptVar) ? promptVar[0] : String(promptVar);
 					const policy = getPolicyText(testCase.metadata);
@@ -13190,13 +13179,21 @@ async function synthesize({ abortSignal, delay, entities: entitiesOverride, inje
 					testCase.metadata.goal = extractedGoal;
 				}
 			}
-			testCases.push(...testCasesWithMetadata);
-			require_logger.logger.debug(`Added ${customTests.length} custom test cases from ${plugin.id}`);
-			const displayId = getPluginDisplayId(plugin);
-			pluginResults[displayId] = {
-				requested: plugin.numTests,
-				generated: testCasesWithMetadata.length
+			testCases.push(...allCustomTests);
+			require_logger.logger.debug(`Added ${allCustomTests.length} custom test cases from ${plugin.id}`);
+			const baseDisplayId = getPluginDisplayId(plugin);
+			if (languages.filter((lang) => lang !== void 0).length > 1) for (const [langKey, result] of Object.entries(resultsPerLanguage)) {
+				const displayId = langKey === "en" ? baseDisplayId : `(${langKey}) ${baseDisplayId}`;
+				pluginResults[displayId] = {
+					requested: result.requested,
+					generated: result.generated
+				};
+			}
+			else pluginResults[baseDisplayId] = {
+				requested: plugin.numTests * languages.length,
+				generated: allCustomTests.length
 			};
+			progressBar?.increment(plugin.numTests * languages.length);
 		} catch (e) {
 			require_logger.logger.error(`Error generating tests for custom plugin ${plugin.id}: ${e}`);
 			const displayId = getPluginDisplayId(plugin);
@@ -14939,7 +14936,7 @@ async function doEval(cmdObj, defaultConfig, defaultConfigPath, evaluateOptions)
 		const passRate = successes / totalTests * 100;
 		if (cmdObj.table && require_logger.getLogLevel() !== "debug" && totalTests < 500) {
 			const table = await evalRecord.getTable();
-			const outputTable = generateTable(table);
+			const outputTable = generateTable(table, cmdObj.tableCellMaxLength ?? commandLineOptions?.tableCellMaxLength);
 			require_logger.logger.info("\n" + outputTable.toString());
 			if (table.body.length > 25) {
 				const rowsLeft = table.body.length - 25;

package/dist/src/index.d.cts

@@ -480,6 +480,15 @@ interface ProviderResponse {
   cached?: boolean;
   cost?: number;
   error?: string;
+  /**
+   * Indicates that a remote Promptfoo server already materialized multi-input vars
+   * for this response. When true, callers must not re-materialize locally.
+   */
+  materializationHandled?: boolean;
+  /**
+   * Materialized per-input vars returned by a remote Promptfoo server.
+   */
+  materializedVars?: Record<string, string>;
   /**
    * Indicates that `output` contains base64-encoded binary data (often as JSON like OpenAI `b64_json`).
    * Used to enable blob externalization and avoid token bloat in downstream grading/agentic strategies.
@@ -513,6 +522,10 @@ interface ProviderResponse {
   prompt?: string | ChatMessage[];
   raw?: string | any;
   output?: string | any;
+  /**
+   * Input materialization metadata returned by a remote Promptfoo server.
+   */
+  inputMaterialization?: Record<string, unknown>;
   /**
    * Output after provider-level transform. Used by contextTransform to ensure
    * it operates on provider-normalized output, independent of test transforms.

package/dist/src/index.d.ts

@@ -481,6 +481,15 @@ interface ProviderResponse {
   cached?: boolean;
   cost?: number;
   error?: string;
+  /**
+   * Indicates that a remote Promptfoo server already materialized multi-input vars
+   * for this response. When true, callers must not re-materialize locally.
+   */
+  materializationHandled?: boolean;
+  /**
+   * Materialized per-input vars returned by a remote Promptfoo server.
+   */
+  materializedVars?: Record<string, string>;
   /**
    * Indicates that `output` contains base64-encoded binary data (often as JSON like OpenAI `b64_json`).
    * Used to enable blob externalization and avoid token bloat in downstream grading/agentic strategies.
@@ -514,6 +523,10 @@ interface ProviderResponse {
   prompt?: string | ChatMessage[];
   raw?: string | any;
   output?: string | any;
+  /**
+   * Input materialization metadata returned by a remote Promptfoo server.
+   */
+  inputMaterialization?: Record<string, unknown>;
   /**
    * Output after provider-level transform. Used by contextTransform to ensure
    * it operates on provider-normalized output, independent of test transforms.