promptfoo 0.121.7 → 0.121.8

package/dist/src/index.js

@@ -1,47 +1,47 @@
 import { C as getEnvFloat, D as getMaxEvalTimeMs, E as getEvalTimeoutMs, O as isCI, S as getEnvBool, T as getEnvString, a as logger, b as summarizeEvaluateResultForLogging, g as getAjv, h as extractJsonObjects, k as state, m as extractFirstJsonObject, n as globalLogCallback, o as setLogCallback, r as isDebugEnabled, s as setLogLevel, t as getLogLevel, v as orderKeys, w as getEnvInt, y as safeJsonStringify } from "./logger-Ct2S6Yx-.js";
 import { t as invariant } from "./invariant-Ddh24eXh.js";
 import { $ as riskCategorySeverityMap, A as RedteamConfigSchema, At as DocumentMediaInjectionPlacementValues, B as isUuid, Bt as getInputDescription, C as TestGeneratorConfigSchema, Ct as CODING_AGENT_CORE_PLUGINS, D as VarsSchema, Dt as BaseTokenUsageSchema, E as UnifiedConfigSchema, F as PartialGenerationError, Ft as InputDefinitionSchema, G as STRATEGY_COLLECTIONS, Gt as isProviderOptions, Ht as normalizeInputDefinition, I as PluginConfigSchema, It as InputTypeSchema, K as STRATEGY_COLLECTION_MAPPINGS, L as PolicyObjectSchema, Lt as InputTypeValues, Mt as DocxInjectionPlacementValues, N as ProvidersSchema, Nt as InputConfigSchema, O as isGradingResult, Ot as CompletionTokenDetailsSchema, P as ConversationMessageSchema, Pt as InputDefinitionObjectSchema, R as StrategyConfigSchema, Rt as InputsSchema, S as TestCasesWithMetadataSchema, St as UNALIGNED_PROVIDER_HARM_PLUGINS, T as TestSuiteSchema, U as DEFAULT_STRATEGIES, Ut as normalizeInputs, Vt as getInputType, Wt as isApiProvider, X as Severity, Y as isFanoutStrategy, Z as categoryAliases, _ as ScenarioSchema, _t as REDTEAM_PROVIDER_HARM_PLUGINS, a as AtomicTestCaseSchema, at as FINANCIAL_PLUGINS, b as TestCaseWithVarsFileSchema, bt as TEEN_SAFETY_PLUGINS, c as CompletedPromptSchema, ct as INSURANCE_PLUGINS, d as EvaluateOptionsSchema, dt as MEDICAL_PLUGINS, et as ALIASED_PLUGIN_MAPPINGS, f as GradingConfigSchema, ft as MULTI_INPUT_EXCLUDED_PLUGINS, g as ResultFailureReason, gt as PLUGIN_CATEGORIES, h as OutputFileExtension, ht as PII_PLUGINS, i as AssertionTypeSchema, it as DEFAULT_PLUGINS, j as PromptSchema, jt as DocxInjectionPlacementSchema, k as isResultFailureReason, kt as DocumentMediaInjectionPlacementSchema, l as DerivedMetricSchema, lt as LLAMA_GUARD_ENABLED_CATEGORIES, m as OutputConfigSchema, mt as PHARMACY_PLUGINS, n as AssertionSchema, nt as CANARY_BREAKING_STRATEGY_IDS, o as BaseAssertionTypesSchema, ot as FOUNDATION_PLUGINS, p as NotPrefixedAssertionTypesSchema, pt as MULTI_INPUT_VAR, q as getDefaultNFanout, r as AssertionSetSchema, rt as DATASET_EXEMPT_PLUGINS, s as CommandLineOptionsSchema, st as HARM_PLUGINS, t as AssertionOrSetSchema, tt as BIAS_PLUGINS, u as EvalResultsFilterMode, ut as LLAMA_GUARD_REPLICATE_PROVIDER, v as SpecialAssertionTypesSchema, vt as REMOTE_ONLY_PLUGIN_IDS, w as TestSuiteConfigSchema, wt as CODING_AGENT_PLUGINS, x as TestCasesWithMetadataPromptSchema, xt as TELECOM_PLUGINS, y as TestCaseSchema, zt as buildInputPromptDescription } from "./types-D6glLbdF.js";
-import { F as getShareApiBaseUrl, I as getShareViewBaseUrl, L as FILE_METADATA_KEY, N as TERMINAL_MAX_WIDTH, P as getDefaultShareViewBaseUrl, R as HUMAN_ASSERTION_TYPE, T as cloudConfig, _ as isPromptfooSampleTarget, b as parseChatPrompt, c as getCurrentTimestamp, l as sleep, n as fetchWithRetries, p as REQUEST_TIMEOUT_MS, r as fetchWithTimeout, t as fetchWithProxy, w as CloudConfig } from "./fetch-It34O8Ur.js";
-import { n as VERSION } from "./version-lpHV_53E.js";
+import { F as getShareApiBaseUrl, I as getShareViewBaseUrl, L as FILE_METADATA_KEY, N as TERMINAL_MAX_WIDTH, P as getDefaultShareViewBaseUrl, R as HUMAN_ASSERTION_TYPE, T as cloudConfig, _ as isPromptfooSampleTarget, b as parseChatPrompt, c as getCurrentTimestamp, l as sleep, n as fetchWithRetries, p as REQUEST_TIMEOUT_MS, r as fetchWithTimeout, t as fetchWithProxy, w as CloudConfig } from "./fetch-Doks14zQ.js";
+import { n as VERSION } from "./version-qVEN5qCm.js";
 import { i as isJavascriptFile } from "./fileExtensions-CXRfY3Ss.js";
-import { c as setUserEmail, i as getUserEmail, o as isLoggedIntoCloud, r as getAuthor, s as promptForEmailUnverified, t as checkEmailStatusAndMaybeExit } from "./accounts-Ca7WIoPY.js";
+import { c as setUserEmail, i as getUserEmail, o as isLoggedIntoCloud, r as getAuthor, s as promptForEmailUnverified, t as checkEmailStatusAndMaybeExit } from "./accounts-DAv_0iE7.js";
 import { r as importModule, t as getDirectory } from "./esm-BTK1W7lG.js";
 import { a as extractVariablesFromTemplates, i as extractVariablesFromTemplate, o as getNunjucksEngine, r as analyzeTemplateReference, t as renderEnvOnlyInObject } from "./render-DFfDeYUK.js";
 import { t as providerRegistry } from "./providerRegistry-DHcFiVWX.js";
-import { a as getRemoteHealthUrl, l as shouldGenerateRemote, n as getRemoteGenerationExplicitlyDisabledError, r as getRemoteGenerationUrl, s as neverGenerateRemote } from "./remoteGeneration-DsaSwmG2.js";
-import { r as promptYesNo } from "./server-CMJD10J4.js";
-import { a as getCloudDatabaseId, c as getPluginSeverityOverridesFromCloud, d as isCloudProvider, i as checkCloudPermissions, o as getEvalConfigFromCloud, p as resolveTeamId, s as getOrgContext } from "./storage-CD-GWAdx.js";
+import { a as getRemoteHealthUrl, l as shouldGenerateRemote, n as getRemoteGenerationExplicitlyDisabledError, r as getRemoteGenerationUrl, s as neverGenerateRemote } from "./remoteGeneration-CJC3E0aW.js";
+import { r as promptYesNo } from "./server-CbiJppij.js";
+import { a as getCloudDatabaseId, c as getPluginSeverityOverridesFromCloud, d as isCloudProvider, i as checkCloudPermissions, o as getEvalConfigFromCloud, p as resolveTeamId, s as getOrgContext } from "./storage-B2Ql_oq4.js";
 import { r as runPython } from "./pythonUtils-D0BYebvX.js";
-import { A as readFilters, M as loadFunction, N as parseFileUrl, O as maybeLoadToolsFromExternalFile, T as maybeLoadFromExternalFile, _ as isProviderAllowed, a as setupEnv, b as normalizeProviderRef, c as filterRuntimeVars, d as checkProviderApiKeys, f as doesProviderRefMatch, g as isOpenAiProvider, h as isGoogleProvider, i as fetchCsvFromGoogleSheet, j as readOutput, l as getTestCaseDeduplicationKey, m as isAnthropicProvider, n as writeMultipleOutputs, o as deduplicateTestCases, p as getProviderDescription, r as writeOutput, s as extractRuntimeVars, t as printBorder, u as resultIsForTestCase, w as maybeLoadConfigFromExternalFile } from "./util-Bx677_k2.js";
+import { A as readFilters, M as loadFunction, N as parseFileUrl, O as maybeLoadToolsFromExternalFile, T as maybeLoadFromExternalFile, _ as isProviderAllowed, a as setupEnv, b as normalizeProviderRef, c as filterRuntimeVars, d as checkProviderApiKeys, f as doesProviderRefMatch, g as isOpenAiProvider, h as isGoogleProvider, i as fetchCsvFromGoogleSheet, j as readOutput, l as getTestCaseDeduplicationKey, m as isAnthropicProvider, n as writeMultipleOutputs, o as deduplicateTestCases, p as getProviderDescription, r as writeOutput, s as extractRuntimeVars, t as printBorder, u as resultIsForTestCase, w as maybeLoadConfigFromExternalFile } from "./util-DRfqa4xz.js";
 import { n as sha256, t as randomSequence } from "./createHash-BtbSX3mj.js";
-import { c as NON_TRANSIENT_HTTP_STATUSES, i as getCache, l as isNonTransientHttpStatus, n as disableCache, r as fetchWithCache, s as withCacheNamespace, t as cache_exports } from "./cache-DIXbtkNO.js";
-import { t as OpenAiChatCompletionProvider } from "./chat-Dabu84Br.js";
-import { h as validateFunctionCall } from "./transform-DmvYBRll.js";
-import { l as validateFunctionCall$1 } from "./util-LKTmNsMQ.js";
-import { _ as AIStudioChatProvider, a as resolveProvider, f as MCPProvider, g as GoogleLiveProvider, h as VertexChatProvider, n as loadApiProvider, o as resolveProviderConfigs, r as loadApiProviders, t as getProviderIds } from "./providers-DVYRZP4E.js";
+import { c as NON_TRANSIENT_HTTP_STATUSES, i as getCache, l as isNonTransientHttpStatus, n as disableCache, r as fetchWithCache, s as withCacheNamespace, t as cache_exports } from "./cache-DdriHsNX.js";
+import { t as OpenAiChatCompletionProvider } from "./chat-BBEnnpQk.js";
+import { h as validateFunctionCall } from "./transform-Wp6s_5QE.js";
+import { l as validateFunctionCall$1 } from "./util-CYev3d-r.js";
+import { _ as AIStudioChatProvider, a as resolveProvider, f as MCPProvider, g as GoogleLiveProvider, h as VertexChatProvider, n as loadApiProvider, o as resolveProviderConfigs, r as loadApiProviders, t as getProviderIds } from "./providers-BGc7tDtQ.js";
 import { a as createEmptyTokenUsage, i as createEmptyAssertions, n as accumulateResponseTokenUsage, o as normalizeTokenUsage, r as accumulateTokenUsage, t as accumulateAssertionTokenUsage } from "./tokenUsageUtils-CmnQ0G2m.js";
 import { t as ellipsize } from "./text-DwYK5EBn.js";
-import { t as telemetry } from "./telemetry-DwX9XUN5.js";
+import { t as telemetry } from "./telemetry-C4bX-6Sr.js";
 import { a as evalsTable, c as evalsToTagsTable, d as tagsTable, i as evalResultsTable, l as promptsTable, m as getDbSignalPath, o as evalsToDatasetsTable, p as getDb, r as datasetsTable, s as evalsToPromptsTable } from "./tables-CRSXQ2Ke.js";
 import { t as getBlobByHash } from "./blobs-Dwef1Ao1.js";
 import { t as getProcessShim } from "./processShim-BcGzU8fY.js";
 import { n as loadFromPackage, t as isPackagePath } from "./packageParser-CgE-ziRo.js";
 import { n as runRuby } from "./rubyUtils-CnlW8AYb.js";
-import { n as materializeInputVariablesWithMetadata, t as buildPromptInputDescriptions } from "./inputVariables-DUGMb9Ka.js";
-import { a as extractPromptFromTags, c as isBasicRefusal, i as extractMaterializedVariablesFromJsonWithMetadata, n as extractGoalFromPrompt, o as getSessionId, r as extractInputVarsFromPrompt, s as getShortPluginId } from "./util-DNl96nNs.js";
-import { n as PromptfooHarmfulCompletionProvider } from "./promptfoo-Cm4hiy1Y.js";
-import { $ as readProviderPromptMap, A as ExcessiveAgencyPlugin, At as withProviderCallExecutionContext, B as retryWithDeduplication, C as PlinyPlugin, Ct as processFileReference, D as ImitationPlugin, Dt as getAndCheckProvider, E as IntentPlugin, Et as callProviderWithContext, F as BeavertailsPlugin, G as matchesFactuality, H as fetchHuggingFaceDataset, I as AegisPlugin, J as matchesPiScore, K as matchesGEval, L as RedteamGraderBase, M as DebugAccessPlugin, N as CrossSessionLeakPlugin, O as HarmbenchPlugin, Ot as getGradingProvider, P as ContractPlugin, Q as readPrompts, R as RedteamPluginBase, S as makeInlinePolicyIdSync, St as loadFromJavaScriptFile, T as OverreliancePlugin, Tt as DEFAULT_ANTHROPIC_MODEL, U as isGraderFailure, V as sampleArray, W as matchesClosedQa, X as doRemoteGrading, Y as matchesTrajectoryGoalSuccess, Z as processPrompts, _ as PromptExtractionPlugin, _t as normalizeMatcherTokenUsage, a as VLGuardPlugin, at as CONTEXT_FAITHFULNESS_NLI_STATEMENTS, b as determinePolicyTypeFromId, bt as coerceString, c as ToxicChatPlugin, ct as CONTEXT_RECALL_NOT_ATTRIBUTED_TOKEN, d as TeenSafetyDangerousRoleplayPlugin, dt as loadRubricPrompt, et as DEFAULT_WEB_SEARCH_PROMPT, f as TeenSafetyDangerousContentPlugin, ft as renderLlmRubricPrompt, g as RbacPlugin, gt as fail, h as ShellInjectionPlugin, ht as euclideanDistance, i as VLSUPlugin, it as CONTEXT_FAITHFULNESS_LONGFORM, j as DivergentRepetitionPlugin, k as HallucinationPlugin, kt as getProviderCallExecutionContext, l as ToolDiscoveryPlugin, lt as CONTEXT_RELEVANCE, m as SqlInjectionPlugin, mt as dotProduct, n as getGraderById, nt as SUGGEST_PROMPTS_SYSTEM_MESSAGE, o as UnverifiableClaimsPlugin, ot as CONTEXT_RECALL, p as TeenSafetyAgeRestrictedGoodsAndServicesPlugin, pt as cosineSimilarity, q as matchesLlmRubric, rt as ANSWER_RELEVANCY_GENERATE, s as UnsafeBenchPlugin, st as CONTEXT_RECALL_ATTRIBUTED_TOKEN, t as GRADERS, tt as SELECT_BEST_PROMPT, u as TeenSafetyHarmfulBodyIdealsPlugin, ut as CONTEXT_RELEVANCE_BAD, v as PoliticsPlugin, vt as splitIntoSentences, w as getPiiLeakTestsForCategory, wt as getDefaultProviders, x as isValidPolicyObject, xt as getFinalTest, y as PolicyPlugin, yt as tryParse, z as getCustomPolicies } from "./graders-BX0f2tvS.js";
-import { f as redteamProviderManager, g as createProviderRateLimitOptions, h as createRateLimitRegistry, m as TokenUsageTracker } from "./shared-D6IjElRI.js";
+import { t as buildPromptInputDescriptions } from "./inputVariables-DUGMb9Ka.js";
+import { a as extractPromptFromTags, c as isBasicRefusal, i as extractMaterializedVariablesFromJsonWithMetadata, n as extractGoalFromPrompt, o as getSessionId, s as getShortPluginId } from "./util-DpV6KT5i.js";
+import { a as assertRemoteMaterializationHandled, l as requiresRemoteMaterialization, n as PromptfooHarmfulCompletionProvider } from "./promptfoo-CqEpj6Sr.js";
+import { $ as readProviderPromptMap, A as ExcessiveAgencyPlugin, At as withProviderCallExecutionContext, B as retryWithDeduplication, C as PlinyPlugin, Ct as processFileReference, D as ImitationPlugin, Dt as getAndCheckProvider, E as IntentPlugin, Et as callProviderWithContext, F as BeavertailsPlugin, G as matchesFactuality, H as fetchHuggingFaceDataset, I as AegisPlugin, J as matchesPiScore, K as matchesGEval, L as RedteamGraderBase, M as DebugAccessPlugin, N as CrossSessionLeakPlugin, O as HarmbenchPlugin, Ot as getGradingProvider, P as ContractPlugin, Q as readPrompts, R as RedteamPluginBase, S as makeInlinePolicyIdSync, St as loadFromJavaScriptFile, T as OverreliancePlugin, Tt as DEFAULT_ANTHROPIC_MODEL, U as isGraderFailure, V as sampleArray, W as matchesClosedQa, X as doRemoteGrading, Y as matchesTrajectoryGoalSuccess, Z as processPrompts, _ as PromptExtractionPlugin, _t as normalizeMatcherTokenUsage, a as VLGuardPlugin, at as CONTEXT_FAITHFULNESS_NLI_STATEMENTS, b as determinePolicyTypeFromId, bt as coerceString, c as ToxicChatPlugin, ct as CONTEXT_RECALL_NOT_ATTRIBUTED_TOKEN, d as TeenSafetyDangerousRoleplayPlugin, dt as loadRubricPrompt, et as DEFAULT_WEB_SEARCH_PROMPT, f as TeenSafetyDangerousContentPlugin, ft as renderLlmRubricPrompt, g as RbacPlugin, gt as fail, h as ShellInjectionPlugin, ht as euclideanDistance, i as VLSUPlugin, it as CONTEXT_FAITHFULNESS_LONGFORM, j as DivergentRepetitionPlugin, k as HallucinationPlugin, kt as getProviderCallExecutionContext, l as ToolDiscoveryPlugin, lt as CONTEXT_RELEVANCE, m as SqlInjectionPlugin, mt as dotProduct, n as getGraderById, nt as SUGGEST_PROMPTS_SYSTEM_MESSAGE, o as UnverifiableClaimsPlugin, ot as CONTEXT_RECALL, p as TeenSafetyAgeRestrictedGoodsAndServicesPlugin, pt as cosineSimilarity, q as matchesLlmRubric, rt as ANSWER_RELEVANCY_GENERATE, s as UnsafeBenchPlugin, st as CONTEXT_RECALL_ATTRIBUTED_TOKEN, t as GRADERS, tt as SELECT_BEST_PROMPT, u as TeenSafetyHarmfulBodyIdealsPlugin, ut as CONTEXT_RELEVANCE_BAD, v as PoliticsPlugin, vt as splitIntoSentences, w as getPiiLeakTestsForCategory, wt as getDefaultProviders, x as isValidPolicyObject, xt as getFinalTest, y as PolicyPlugin, yt as tryParse, z as getCustomPolicies } from "./graders-KXzjnIim.js";
+import { f as redteamProviderManager, g as createProviderRateLimitOptions, h as createRateLimitRegistry, m as TokenUsageTracker } from "./shared-CSIGeGLl.js";
 import { i as generateIdFromPrompt, t as hashPrompt } from "./utils-BGY69tk_.js";
 import { a as getTransformLabel, i as getTransformErrorMessage, n as TRANSFORM_KEYS, o as transform, r as TransformInputType, t as INLINE_FUNCTION_LABEL } from "./transform-CGt7Kt3y2.js";
 import { t as getTraceStore } from "./store-HpopRVzl.js";
-import { n as isBlobStorageEnabled, t as extractAndStoreBinaryData } from "./extractor-CxRtnaHl.js";
+import { n as isBlobStorageEnabled, t as extractAndStoreBinaryData } from "./extractor-D3Fv_Tdh.js";
 import { i as throwIfTargetPromptExceedsMaxChars, n as getGeneratedPromptOverLimit, r as getMaxCharsPerMessageModifierValue, t as MAX_CHARS_PER_MESSAGE_MODIFIER_KEY } from "./promptLength-B9nZEfO6.js";
-import { n as checkExfilTracking } from "./indirectWebPwn-CbjUG0rh.js";
+import { n as checkExfilTracking } from "./indirectWebPwn-CiWB-vVH.js";
 import { n as getFirstStringAttribute, r as getToolNameFromAttributes, t as TOOL_ARGUMENT_ATTRIBUTE_KEYS } from "./toolAttributes-DJ9ZEKXD.js";
-import { i as filterFiniteScores, n as renderPrompt, r as runExtensionHook, t as collectFileMetadata } from "./evaluatorHelpers-DuqFFfq7.js";
-import { r as sanitizeProvider, t as EvalResult } from "./evalResult-2RRJvFyB.js";
-import { i as pluginMatchesStrategyTargets, n as loadStrategy, r as validateStrategies, t as Strategies } from "./strategies-CiSeroPH.js";
+import { i as filterFiniteScores, n as renderPrompt, r as runExtensionHook, t as collectFileMetadata } from "./evaluatorHelpers-CrRObe2z.js";
+import { r as sanitizeProvider, t as EvalResult } from "./evalResult-PGqEbasb.js";
+import { i as pluginMatchesStrategyTargets, n as loadStrategy, r as validateStrategies, t as Strategies } from "./strategies-CD1gHeeQ.js";
 import * as fs$2 from "fs";
 import fs, { createWriteStream } from "fs";
 import * as path$2 from "path";
@@ -6313,7 +6313,7 @@ async function resolveDefaultTestProvider(defaultTest, testCase) {
 	const defaultProvider = defaultTest.provider;
 	if (isApiProvider(defaultProvider)) return defaultProvider;
 	if (typeof defaultProvider === "object" && defaultProvider.id) {
-		const { loadApiProvider } = await import("./providers-DVYRZP4E.js").then((n) => n.i);
+		const { loadApiProvider } = await import("./providers-BGc7tDtQ.js").then((n) => n.i);
 		return loadApiProvider(typeof defaultProvider.id === "function" ? defaultProvider.id() : defaultProvider.id, { options: defaultProvider });
 	}
 	return defaultProvider;
@@ -6473,7 +6473,7 @@ function buildRepeatCacheContextByTestIdx(runEvalOptions) {
 async function filterCompletedResumeSteps(runEvalOptions, evalRecord) {
 	if (!state.resume || !evalRecord.persisted) return;
 	try {
-		const { default: EvalResult } = await import("./evalResult-2RRJvFyB.js").then((n) => n.n);
+		const { default: EvalResult } = await import("./evalResult-PGqEbasb.js").then((n) => n.n);
 		const completedPairs = await EvalResult.getCompletedIndexPairs(evalRecord.id, { excludeErrors: state.retryMode });
 		const originalCount = runEvalOptions.length;
 		for (let i = runEvalOptions.length - 1; i >= 0; i--) {
@@ -12118,6 +12118,7 @@ async function fetchRemoteTestCases(key, purpose, injectVar, n, config) {
 			logger.error(`Error generating test cases for ${key}: ${statusText} ${JSON.stringify(data)}`);
 			return [];
 		}
+		if (requiresRemoteMaterialization(config?.inputs)) assertRemoteMaterializationHandled(data, `Remote plugin generation for ${key}`);
 		const ret = data.result;
 		logger.debug(`Received remote generation for ${key}:\n${JSON.stringify(ret)}`);
 		return ret;
@@ -12126,31 +12127,6 @@ async function fetchRemoteTestCases(key, purpose, injectVar, n, config) {
 		return [];
 	}
 }
-async function materializeRemoteTestCaseInputs({ config, injectVar, pluginId, provider, purpose, testCases }) {
-	const inputs = config.inputs;
-	if (!inputs || Object.keys(inputs).length === 0) return testCases;
-	return Promise.all(testCases.map(async (testCase, materializationIndex) => {
-		const inputVars = extractInputVarsFromPrompt(String(testCase.vars?.[injectVar] ?? ""), inputs);
-		if (!inputVars) return testCase;
-		const materializedVars = await materializeInputVariablesWithMetadata(inputVars, inputs, {
-			materializationIndex,
-			pluginId,
-			provider,
-			purpose
-		});
-		return {
-			...testCase,
-			vars: {
-				...testCase.vars || {},
-				...materializedVars.vars
-			},
-			metadata: {
-				...testCase.metadata || {},
-				...materializedVars.metadata ? { inputMaterialization: materializedVars.metadata } : {}
-			}
-		};
-	}));
-}
 function createPluginFactory(PluginClass, key, validate) {
 	return {
 		key,
@@ -12162,14 +12138,7 @@ function createPluginFactory(PluginClass, key, validate) {
 				return new PluginClass(provider, purpose, injectVar, configWithDefaults).generateTests(n, delayMs);
 			}
 			const pluginId = getShortPluginId(key);
-			const testCases = await materializeRemoteTestCaseInputs({
-				config: configWithDefaults ?? {},
-				injectVar,
-				pluginId,
-				provider,
-				purpose,
-				testCases: await fetchRemoteTestCases(key, purpose, injectVar, n, configWithDefaults ?? {})
-			});
+			const testCases = await fetchRemoteTestCases(key, purpose, injectVar, n, configWithDefaults ?? {});
 			const computedModifiers = computeModifiersFromConfig(configWithDefaults);
 			return testCases.map((testCase) => ({
 				...testCase,
@@ -12256,14 +12225,7 @@ const piiPlugins = PII_PLUGINS.map((category) => ({
 	action: async (params) => {
 		if (shouldGenerateRemote()) {
 			const pluginId = getShortPluginId(category);
-			const testCases = await materializeRemoteTestCaseInputs({
-				config: params.config ?? {},
-				injectVar: params.injectVar,
-				pluginId,
-				provider: params.provider,
-				purpose: params.purpose,
-				testCases: await fetchRemoteTestCases(category, params.purpose, params.injectVar, params.n, params.config ?? {})
-			});
+			const testCases = await fetchRemoteTestCases(category, params.purpose, params.injectVar, params.n, params.config ?? {});
 			const computedModifiers = computeModifiersFromConfig(params.config);
 			return testCases.map((testCase) => ({
 				...testCase,
@@ -12295,14 +12257,7 @@ const biasPlugins = BIAS_PLUGINS.map((category) => ({
 			return [];
 		}
 		const pluginId = getShortPluginId(category);
-		const testCases = await materializeRemoteTestCaseInputs({
-			config: params.config ?? {},
-			injectVar: params.injectVar,
-			pluginId,
-			provider: params.provider,
-			purpose: params.purpose,
-			testCases: await fetchRemoteTestCases(category, params.purpose, params.injectVar, params.n, params.config ?? {})
-		});
+		const testCases = await fetchRemoteTestCases(category, params.purpose, params.injectVar, params.n, params.config ?? {});
 		const computedModifiers = computeModifiersFromConfig(params.config);
 		return testCases.map((testCase) => ({
 			...testCase,
@@ -12321,21 +12276,14 @@ function createRemotePlugin(key, validate) {
 	return {
 		key,
 		validate,
-		action: async ({ provider, purpose, injectVar, n, config }) => {
+		action: async ({ purpose, injectVar, n, config }) => {
 			const configWithDefaults = applyDefaultRemotePluginConfig(key, config);
 			if (neverGenerateRemote()) {
 				logger.error(getRemoteGenerationExplicitlyDisabledError(`${key} plugin`));
 				return [];
 			}
 			const pluginId = getShortPluginId(key);
-			const testCases = await materializeRemoteTestCaseInputs({
-				config: configWithDefaults ?? {},
-				injectVar,
-				pluginId,
-				provider,
-				purpose,
-				testCases: await fetchRemoteTestCases(key, purpose, injectVar, n, configWithDefaults ?? {})
-			});
+			const testCases = await fetchRemoteTestCases(key, purpose, injectVar, n, configWithDefaults ?? {});
 			const computedModifiers = computeModifiersFromConfig(configWithDefaults);
 			const testsWithMetadata = testCases.map((testCase) => ({
 				...testCase,
@@ -12402,6 +12350,15 @@ async function validateSharpDependency(strategies, plugins, checkSharp = isSharp
 }
 //#endregion
 //#region src/redteam/index.ts
+const MATERIALIZED_MULTI_INPUT_PROMPT_METADATA_KEY = "__promptfooMaterializedMultiInputPrompt";
+function getMaterializedMultiInputPromptSnapshot(metadata) {
+	const snapshot = metadata?.[MATERIALIZED_MULTI_INPUT_PROMPT_METADATA_KEY];
+	return typeof snapshot === "string" ? snapshot : void 0;
+}
+function getMaterializedMultiInputPromptMetadata(vars) {
+	const prompt = vars?.[MULTI_INPUT_VAR];
+	return typeof prompt === "string" ? { [MATERIALIZED_MULTI_INPUT_PROMPT_METADATA_KEY]: prompt } : void 0;
+}
 function getPolicyText(metadata) {
 	if (!metadata || metadata.policy === void 0 || metadata.policy === null) return;
 	const policyValue = metadata.policy;
@@ -12414,12 +12371,19 @@ function getPolicyText(metadata) {
 async function rematerializeStrategyInputVars(testCase, injectVar, provider, purpose, materializationIndex) {
 	const inputs = testCase.metadata?.pluginConfig?.inputs;
 	const inputMaterialization = testCase.metadata?.inputMaterialization;
-	if (!inputs || Object.keys(inputs).length === 0 || !testCase.vars?.[injectVar]) return {
+	const materializedPromptSnapshot = getMaterializedMultiInputPromptSnapshot(testCase.metadata);
+	const currentInjectVar = testCase.vars?.[injectVar];
+	if (!inputs || Object.keys(inputs).length === 0 || !currentInjectVar) return {
+		inputMaterialization,
+		vars: testCase.vars
+	};
+	const promptChangedSinceMaterialization = typeof currentInjectVar === "string" && typeof materializedPromptSnapshot === "string" && currentInjectVar !== materializedPromptSnapshot;
+	if (Boolean(inputMaterialization) && Object.keys(inputs).every((key) => Object.prototype.hasOwnProperty.call(testCase.vars ?? {}, key)) && !promptChangedSinceMaterialization) return {
 		inputMaterialization,
 		vars: testCase.vars
 	};
 	try {
-		const materializedVars = await extractMaterializedVariablesFromJsonWithMetadata(JSON.parse(String(testCase.vars[injectVar])), inputs, {
+		const materializedVars = await extractMaterializedVariablesFromJsonWithMetadata(JSON.parse(String(currentInjectVar)), inputs, {
 			materializationIndex,
 			pluginId: String(testCase.metadata?.pluginId || "unknown-plugin"),
 			provider,
@@ -12651,7 +12615,8 @@ function addLanguageToPluginMetadata(test, lang, plugin, maxCharsPerMessage, tes
 				...test.metadata?.modifiers,
 				...languageToAdd
 			},
-			...languageToAdd
+			...languageToAdd,
+			...getMaterializedMultiInputPromptMetadata(test.vars)
 		}
 	};
 }
@@ -12750,7 +12715,8 @@ async function applyStrategies(testCases, strategies, injectVar, provider, purpo
 					...t?.metadata?.pluginId && { pluginId: t.metadata.pluginId },
 					...t?.metadata?.pluginConfig && { pluginConfig: t.metadata.pluginConfig },
 					...inputMaterialization && { inputMaterialization },
-					...Object.keys(strategyConfig).length > 0 && { strategyConfig }
+					...Object.keys(strategyConfig).length > 0 && { strategyConfig },
+					...getMaterializedMultiInputPromptMetadata(vars)
 				}
 			};
 		})));
@@ -13096,14 +13062,21 @@ async function synthesize({ abortSignal, delay, entities: entitiesOverride, inje
 				}
 			});
 			const languageResults = await Promise.allSettled(languagePromises);
-			for (const result of languageResults) if (result.status === "fulfilled") {
+			for (const [index, result] of languageResults.entries()) if (result.status === "fulfilled") {
 				const { lang, tests, requested, generated } = result.value;
 				allPluginTests.push(...tests);
 				resultsPerLanguage[lang || "default"] = {
 					requested,
 					generated
 				};
-			} else logger.warn(`[Language Processing] Error generating tests for ${plugin.id}: ${result.reason}`);
+			} else {
+				const lang = languages[index];
+				logger.warn(`[Language Processing] Error generating tests for ${plugin.id}: ${result.reason}`);
+				resultsPerLanguage[lang || "default"] = {
+					requested: plugin.numTests,
+					generated: 0
+				};
+			}
 			logger.debug(`[Language Processing] Total tests generated for ${plugin.id}: ${allPluginTests.length} (across ${languages.length} language(s))`);
 			if (!Array.isArray(allPluginTests) || allPluginTests.length === 0) logger.warn(`Failed to generate tests for ${plugin.id}`);
 			else {
@@ -13137,35 +13110,51 @@ async function synthesize({ abortSignal, delay, entities: entitiesOverride, inje
 				generated: allPluginTests.length
 			};
 		} else if (plugin.id.startsWith("file://")) try {
-			const customTests = await new CustomPlugin(redteamProvider, purpose, injectVar, plugin.id, resolvePluginConfigWithMaxChars(plugin.config, maxCharsPerMessage)).generateTests(plugin.numTests, delay);
-			const testCasesWithMetadata = filterOversizedTestCases(customTests.map((t) => {
-				const includePluginConfig = !(t.metadata && Object.hasOwn(t.metadata, "pluginConfig") && t.metadata.pluginConfig === void 0);
-				const pluginConfigWithMaxChars = {
+			const languageConfig = plugin.config?.language ?? language;
+			const languages = Array.isArray(languageConfig) ? languageConfig : languageConfig ? [languageConfig] : [void 0];
+			const allCustomTests = [];
+			const resultsPerLanguage = {};
+			const languagePromises = languages.map(async (lang) => {
+				const resolvedConfig = {
 					...resolvePluginConfigWithMaxChars(plugin.config, maxCharsPerMessage),
-					...t.metadata?.pluginConfig ?? {}
+					...lang ? { language: lang } : {},
+					...hasMultipleInputs ? { inputs } : {}
 				};
-				const modifiers = {
-					...buildRedteamModifiers({
+				const customPluginConfig = {
+					...resolvedConfig,
+					modifiers: buildRedteamModifiers({
 						maxCharsPerMessage,
-						pluginConfig: pluginConfigWithMaxChars,
+						pluginConfig: resolvedConfig,
 						testGenerationInstructions
-					}),
-					...t.metadata?.modifiers
+					})
 				};
+				const testCasesWithMetadata = filterOversizedTestCases((await new CustomPlugin(redteamProvider, purpose, injectVar, plugin.id, customPluginConfig).generateTests(plugin.numTests, delay)).map((t) => addLanguageToPluginMetadata(t, lang, plugin, maxCharsPerMessage, testGenerationInstructions)), injectVar, `Custom plugin ${plugin.id}`, maxCharsPerMessage);
 				return {
-					...t,
-					metadata: {
-						...t.metadata || {},
-						pluginId: plugin.id,
-						...includePluginConfig && { pluginConfig: pluginConfigWithMaxChars },
-						severity: plugin.severity || getPluginSeverity(plugin.id, resolvePluginConfig(plugin.config)),
-						modifiers
-					}
+					lang,
+					tests: testCasesWithMetadata,
+					requested: plugin.numTests,
+					generated: testCasesWithMetadata.length
+				};
+			});
+			const languageResults = await Promise.allSettled(languagePromises);
+			for (const [index, result] of languageResults.entries()) if (result.status === "fulfilled") {
+				const { lang, tests, requested, generated } = result.value;
+				allCustomTests.push(...tests);
+				resultsPerLanguage[lang || "default"] = {
+					requested,
+					generated
 				};
-			}), injectVar, `Custom plugin ${plugin.id}`, maxCharsPerMessage);
+			} else {
+				const lang = languages[index];
+				logger.warn(`[Language Processing] Error generating tests for custom plugin ${plugin.id}: ${result.reason}`);
+				resultsPerLanguage[lang || "default"] = {
+					requested: plugin.numTests,
+					generated: 0
+				};
+			}
 			if (needsGoalExtraction) {
-				logger.debug(`Extracting goal for ${testCasesWithMetadata.length} custom tests from ${plugin.id}...`);
-				for (const testCase of testCasesWithMetadata) {
+				logger.debug(`Extracting goal for ${allCustomTests.length} custom tests from ${plugin.id}...`);
+				for (const testCase of allCustomTests) {
 					const promptVar = testCase.vars?.[injectVar];
 					const prompt = Array.isArray(promptVar) ? promptVar[0] : String(promptVar);
 					const policy = getPolicyText(testCase.metadata);
@@ -13173,13 +13162,21 @@ async function synthesize({ abortSignal, delay, entities: entitiesOverride, inje
 					testCase.metadata.goal = extractedGoal;
 				}
 			}
-			testCases.push(...testCasesWithMetadata);
-			logger.debug(`Added ${customTests.length} custom test cases from ${plugin.id}`);
-			const displayId = getPluginDisplayId(plugin);
-			pluginResults[displayId] = {
-				requested: plugin.numTests,
-				generated: testCasesWithMetadata.length
+			testCases.push(...allCustomTests);
+			logger.debug(`Added ${allCustomTests.length} custom test cases from ${plugin.id}`);
+			const baseDisplayId = getPluginDisplayId(plugin);
+			if (languages.filter((lang) => lang !== void 0).length > 1) for (const [langKey, result] of Object.entries(resultsPerLanguage)) {
+				const displayId = langKey === "en" ? baseDisplayId : `(${langKey}) ${baseDisplayId}`;
+				pluginResults[displayId] = {
+					requested: result.requested,
+					generated: result.generated
+				};
+			}
+			else pluginResults[baseDisplayId] = {
+				requested: plugin.numTests * languages.length,
+				generated: allCustomTests.length
 			};
+			progressBar?.increment(plugin.numTests * languages.length);
 		} catch (e) {
 			logger.error(`Error generating tests for custom plugin ${plugin.id}: ${e}`);
 			const displayId = getPluginDisplayId(plugin);

package/dist/src/{indirectWebPwn-DBQhOjoD.js → indirectWebPwn-B80dLlFC.js}

@@ -1,11 +1,11 @@
 #!/usr/bin/env node
 import { s as logger } from "./logger-BbY6ypFL.js";
-import { r as fetchWithRetries } from "./fetch-DXUnXkVU.js";
+import { r as fetchWithRetries } from "./fetch-Cpf1U1nO.js";
 import { t as invariant } from "./invariant-B2Rf6avk.js";
-import { o as getUserEmail } from "./accounts-CjFnOPmb.js";
-import { r as getRemoteGenerationUrl } from "./remoteGeneration-B1_XsKXU.js";
+import { o as getUserEmail } from "./accounts-D6IBfEE0.js";
+import { r as getRemoteGenerationUrl } from "./remoteGeneration-D6UjE2JT.js";
 import { a as createEmptyTokenUsage, n as accumulateResponseTokenUsage } from "./tokenUsageUtils-CDet74yk.js";
-import { c as getTargetResponse } from "./shared-9WHQ1oNE.js";
+import { c as getTargetResponse } from "./shared-DNvim54U.js";
 import { randomUUID } from "node:crypto";
 //#region src/redteam/providers/indirectWebPwn.ts
 /**
@@ -257,4 +257,4 @@ var IndirectWebPwnProvider = class {
 //#endregion
 export { IndirectWebPwnProvider as default };
 
-//# sourceMappingURL=indirectWebPwn-DBQhOjoD.js.map
+//# sourceMappingURL=indirectWebPwn-B80dLlFC.js.map

package/dist/src/{indirectWebPwn-uyWdHx04.js → indirectWebPwn-BMTXXznx.js}

@@ -1,8 +1,8 @@
 #!/usr/bin/env node
 import { s as logger } from "./logger-BbY6ypFL.js";
-import { r as fetchWithRetries } from "./fetch-DXUnXkVU.js";
-import { o as getUserEmail } from "./accounts-CjFnOPmb.js";
-import { r as getRemoteGenerationUrl } from "./remoteGeneration-B1_XsKXU.js";
+import { r as fetchWithRetries } from "./fetch-Cpf1U1nO.js";
+import { o as getUserEmail } from "./accounts-D6IBfEE0.js";
+import { r as getRemoteGenerationUrl } from "./remoteGeneration-D6UjE2JT.js";
 import { createHash, randomUUID } from "node:crypto";
 //#region src/redteam/strategies/indirectWebPwn.ts
 /**
@@ -383,4 +383,4 @@ async function transformForPerTurnLayer(testCases, injectVar, config) {
 //#endregion
 export { checkExfilTracking as n, addIndirectWebPwnTestCases as t };
 
-//# sourceMappingURL=indirectWebPwn-uyWdHx04.js.map
+//# sourceMappingURL=indirectWebPwn-BMTXXznx.js.map

package/dist/src/{indirectWebPwn-tNx9OZ35.js → indirectWebPwn-BZFPV7Q9.js}

@@ -1,7 +1,7 @@
 import { a as logger } from "./logger-KD8JjCRJ.js";
-import { n as fetchWithRetries } from "./fetch-BufrQtvR.js";
-import { o as getUserEmail } from "./accounts-DanM1wq_.js";
-import { r as getRemoteGenerationUrl } from "./remoteGeneration-COpWcmWd.js";
+import { n as fetchWithRetries } from "./fetch-BSSAcMxf.js";
+import { o as getUserEmail } from "./accounts-BWjqwsrf.js";
+import { r as getRemoteGenerationUrl } from "./remoteGeneration--D6WjzUm.js";
 import { createHash, randomUUID } from "node:crypto";
 //#region src/redteam/strategies/indirectWebPwn.ts
 /**
@@ -382,4 +382,4 @@ async function transformForPerTurnLayer(testCases, injectVar, config) {
 //#endregion
 export { checkExfilTracking as n, addIndirectWebPwnTestCases as t };
 
-//# sourceMappingURL=indirectWebPwn-tNx9OZ35.js.map
+//# sourceMappingURL=indirectWebPwn-BZFPV7Q9.js.map

package/dist/src/{indirectWebPwn-CfQJt3gk.cjs → indirectWebPwn-BaEQEOIO.cjs}

@@ -1,11 +1,11 @@
 require("./rolldown-runtime-D_mwlA32.cjs");
 const require_logger = require("./logger-cfNpzI4o.cjs");
 const require_invariant = require("./invariant-QtnLD03y.cjs");
-const require_fetch = require("./fetch-Dw4XZHjj.cjs");
-const require_accounts = require("./accounts-CmWzeD2d.cjs");
-const require_remoteGeneration = require("./remoteGeneration-DS9N3pgB.cjs");
+const require_fetch = require("./fetch-Cfm4EuOB.cjs");
+const require_accounts = require("./accounts-BVz5gHLK.cjs");
+const require_remoteGeneration = require("./remoteGeneration-BpIYlb_O.cjs");
 const require_tokenUsageUtils = require("./tokenUsageUtils-_B-P8IAi.cjs");
-const require_shared = require("./shared-WkgnDkcg.cjs");
+const require_shared = require("./shared-aHWko3P1.cjs");
 let node_crypto = require("node:crypto");
 //#region src/redteam/providers/indirectWebPwn.ts
 /**
@@ -257,4 +257,4 @@ var IndirectWebPwnProvider = class {
 //#endregion
 exports.default = IndirectWebPwnProvider;
 
-//# sourceMappingURL=indirectWebPwn-CfQJt3gk.cjs.map
+//# sourceMappingURL=indirectWebPwn-BaEQEOIO.cjs.map

package/dist/src/{indirectWebPwn-OsXnKejv.js → indirectWebPwn-BzpyMnFS.js}

@@ -1,10 +1,10 @@
 import { a as logger } from "./logger-Ct2S6Yx-.js";
 import { t as invariant } from "./invariant-Ddh24eXh.js";
-import { n as fetchWithRetries } from "./fetch-It34O8Ur.js";
-import { i as getUserEmail } from "./accounts-Ca7WIoPY.js";
-import { r as getRemoteGenerationUrl } from "./remoteGeneration-DsaSwmG2.js";
+import { n as fetchWithRetries } from "./fetch-Doks14zQ.js";
+import { i as getUserEmail } from "./accounts-DAv_0iE7.js";
+import { r as getRemoteGenerationUrl } from "./remoteGeneration-CJC3E0aW.js";
 import { a as createEmptyTokenUsage, n as accumulateResponseTokenUsage } from "./tokenUsageUtils-CmnQ0G2m.js";
-import { c as getTargetResponse } from "./shared-D6IjElRI.js";
+import { c as getTargetResponse } from "./shared-CSIGeGLl.js";
 import { randomUUID } from "node:crypto";
 //#region src/redteam/providers/indirectWebPwn.ts
 /**
@@ -256,4 +256,4 @@ var IndirectWebPwnProvider = class {
 //#endregion
 export { IndirectWebPwnProvider as default };
 
-//# sourceMappingURL=indirectWebPwn-OsXnKejv.js.map
+//# sourceMappingURL=indirectWebPwn-BzpyMnFS.js.map

package/dist/src/{indirectWebPwn-CbjUG0rh.js → indirectWebPwn-CiWB-vVH.js}

@@ -1,7 +1,7 @@
 import { a as logger } from "./logger-Ct2S6Yx-.js";
-import { n as fetchWithRetries } from "./fetch-It34O8Ur.js";
-import { i as getUserEmail } from "./accounts-Ca7WIoPY.js";
-import { r as getRemoteGenerationUrl } from "./remoteGeneration-DsaSwmG2.js";
+import { n as fetchWithRetries } from "./fetch-Doks14zQ.js";
+import { i as getUserEmail } from "./accounts-DAv_0iE7.js";
+import { r as getRemoteGenerationUrl } from "./remoteGeneration-CJC3E0aW.js";
 import { createHash, randomUUID } from "node:crypto";
 //#region src/redteam/strategies/indirectWebPwn.ts
 /**
@@ -382,4 +382,4 @@ async function transformForPerTurnLayer(testCases, injectVar, config) {
 //#endregion
 export { checkExfilTracking as n, addIndirectWebPwnTestCases as t };
 
-//# sourceMappingURL=indirectWebPwn-CbjUG0rh.js.map
+//# sourceMappingURL=indirectWebPwn-CiWB-vVH.js.map

package/dist/src/{indirectWebPwn-BJ22AbQa.cjs → indirectWebPwn-D7NA9Nsv.cjs}

@@ -1,8 +1,8 @@
 require("./rolldown-runtime-D_mwlA32.cjs");
 const require_logger = require("./logger-cfNpzI4o.cjs");
-const require_fetch = require("./fetch-Dw4XZHjj.cjs");
-const require_accounts = require("./accounts-CmWzeD2d.cjs");
-const require_remoteGeneration = require("./remoteGeneration-DS9N3pgB.cjs");
+const require_fetch = require("./fetch-Cfm4EuOB.cjs");
+const require_accounts = require("./accounts-BVz5gHLK.cjs");
+const require_remoteGeneration = require("./remoteGeneration-BpIYlb_O.cjs");
 let node_crypto = require("node:crypto");
 //#region src/redteam/strategies/indirectWebPwn.ts
 /**
@@ -394,4 +394,4 @@ Object.defineProperty(exports, "checkExfilTracking", {
 	}
 });
 
-//# sourceMappingURL=indirectWebPwn-BJ22AbQa.cjs.map
+//# sourceMappingURL=indirectWebPwn-D7NA9Nsv.cjs.map

package/dist/src/{indirectWebPwn-02ZIghCS.js → indirectWebPwn-o_bEFMjP.js}

@@ -1,10 +1,10 @@
 import { a as logger } from "./logger-KD8JjCRJ.js";
-import { n as fetchWithRetries } from "./fetch-BufrQtvR.js";
+import { n as fetchWithRetries } from "./fetch-BSSAcMxf.js";
 import { t as invariant } from "./invariant-DIYf9sP1.js";
-import { o as getUserEmail } from "./accounts-DanM1wq_.js";
-import { r as getRemoteGenerationUrl } from "./remoteGeneration-COpWcmWd.js";
+import { o as getUserEmail } from "./accounts-BWjqwsrf.js";
+import { r as getRemoteGenerationUrl } from "./remoteGeneration--D6WjzUm.js";
 import { a as createEmptyTokenUsage, n as accumulateResponseTokenUsage } from "./tokenUsageUtils-BjVkdk18.js";
-import { c as getTargetResponse } from "./shared-7pmVZLNO.js";
+import { c as getTargetResponse } from "./shared-CRq0gGVf.js";
 import { randomUUID } from "node:crypto";
 //#region src/redteam/providers/indirectWebPwn.ts
 /**
@@ -256,4 +256,4 @@ var IndirectWebPwnProvider = class {
 //#endregion
 export { IndirectWebPwnProvider as default };
 
-//# sourceMappingURL=indirectWebPwn-02ZIghCS.js.map
+//# sourceMappingURL=indirectWebPwn-o_bEFMjP.js.map