promptfoo 0.121.7 → 0.121.8

package/dist/src/{knowledgeBase-BadkINlJ.js → knowledgeBase-DxAq4n4z.js}

@@ -1,9 +1,9 @@
 import { a as logger, w as getEnvInt } from "./logger-Ct2S6Yx-.js";
 import { n as sha256 } from "./createHash-BtbSX3mj.js";
-import { i as getCache, o as isCacheEnabled } from "./cache-DIXbtkNO.js";
+import { i as getCache, o as isCacheEnabled } from "./cache-DdriHsNX.js";
 import { a as createEmptyTokenUsage } from "./tokenUsageUtils-CmnQ0G2m.js";
-import { t as telemetry } from "./telemetry-DwX9XUN5.js";
-import { n as createBedrockRequestHandler, r as hasProxyEnv, t as AwsBedrockGenericProvider } from "./base-CKLo890h.js";
+import { t as telemetry } from "./telemetry-C4bX-6Sr.js";
+import { n as createBedrockRequestHandler, r as hasProxyEnv, t as AwsBedrockGenericProvider } from "./base-BuNn-YBX.js";
 //#region src/providers/bedrock/knowledgeBase.ts
 /**
 * AWS Bedrock Knowledge Base provider for RAG (Retrieval Augmented Generation).
@@ -133,4 +133,4 @@ var AwsBedrockKnowledgeBaseProvider = class extends AwsBedrockGenericProvider {
 //#endregion
 export { AwsBedrockKnowledgeBaseProvider };
 
-//# sourceMappingURL=knowledgeBase-BadkINlJ.js.map
+//# sourceMappingURL=knowledgeBase-DxAq4n4z.js.map

package/dist/src/{knowledgeBase-CkMljjdg.js → knowledgeBase-o_wTLzrt.js}

@@ -1,10 +1,10 @@
 #!/usr/bin/env node
 import { D as getEnvInt, s as logger } from "./logger-BbY6ypFL.js";
-import { r as telemetry } from "./telemetry-00ezXr_t.js";
+import { r as telemetry } from "./telemetry-DOE567Wj.js";
 import { n as sha256 } from "./createHash-CgRvs4Fn.js";
-import { c as isCacheEnabled, o as getCache } from "./cache-DpPWrkTE.js";
+import { c as isCacheEnabled, o as getCache } from "./cache-BR77mdIR.js";
 import { a as createEmptyTokenUsage } from "./tokenUsageUtils-CDet74yk.js";
-import { n as createBedrockRequestHandler, r as hasProxyEnv, t as AwsBedrockGenericProvider } from "./base-Co80MMCi.js";
+import { n as createBedrockRequestHandler, r as hasProxyEnv, t as AwsBedrockGenericProvider } from "./base-Bp4c52YZ.js";
 //#region src/providers/bedrock/knowledgeBase.ts
 /**
 * AWS Bedrock Knowledge Base provider for RAG (Retrieval Augmented Generation).
@@ -134,4 +134,4 @@ var AwsBedrockKnowledgeBaseProvider = class extends AwsBedrockGenericProvider {
 //#endregion
 export { AwsBedrockKnowledgeBaseProvider };
 
-//# sourceMappingURL=knowledgeBase-CkMljjdg.js.map
+//# sourceMappingURL=knowledgeBase-o_wTLzrt.js.map

package/dist/src/{litellm-BKBo0jpC.js → litellm-B2gHwya_.js}

@@ -1,7 +1,7 @@
 import { T as getEnvString } from "./logger-KD8JjCRJ.js";
-import { t as OpenAiChatCompletionProvider } from "./chat-CUCorGiL.js";
-import { t as OpenAiEmbeddingProvider } from "./embedding-C251p1-8.js";
-import { t as OpenAiCompletionProvider } from "./completion-CO2e1_62.js";
+import { t as OpenAiChatCompletionProvider } from "./chat-BSos6PvZ.js";
+import { t as OpenAiEmbeddingProvider } from "./embedding-CV8lmCnU.js";
+import { t as OpenAiCompletionProvider } from "./completion-DT8cxo9T.js";
 //#region src/providers/litellm.ts
 /**
 * Base class for LiteLLM providers that maintains LiteLLM identity
@@ -133,4 +133,4 @@ function createLiteLLMProvider(providerPath, options = {}) {
 //#endregion
 export { createLiteLLMProvider };
 
-//# sourceMappingURL=litellm-BKBo0jpC.js.map
+//# sourceMappingURL=litellm-B2gHwya_.js.map

package/dist/src/{litellm-CtAr7bKG.cjs → litellm-CPpdlO7n.cjs}

@@ -1,7 +1,7 @@
 const require_logger = require("./logger-cfNpzI4o.cjs");
-const require_chat = require("./chat-DxTDQ83C.cjs");
-const require_embedding = require("./embedding-BbrwopfX.cjs");
-const require_completion = require("./completion-CSYfl2cd.cjs");
+const require_chat = require("./chat-DSyYuTYT.cjs");
+const require_embedding = require("./embedding-BQIApR18.cjs");
+const require_completion = require("./completion-CWtqdn3z.cjs");
 //#region src/providers/litellm.ts
 /**
 * Base class for LiteLLM providers that maintains LiteLLM identity
@@ -133,4 +133,4 @@ function createLiteLLMProvider(providerPath, options = {}) {
 //#endregion
 exports.createLiteLLMProvider = createLiteLLMProvider;
 
-//# sourceMappingURL=litellm-CtAr7bKG.cjs.map
+//# sourceMappingURL=litellm-CPpdlO7n.cjs.map

package/dist/src/{litellm-BXyn5kZK.js → litellm-CYfgxLrM.js}

@@ -1,8 +1,8 @@
 #!/usr/bin/env node
 import { O as getEnvString } from "./logger-BbY6ypFL.js";
-import { t as OpenAiChatCompletionProvider } from "./chat-DqUFcWI0.js";
-import { t as OpenAiEmbeddingProvider } from "./embedding--UZVe4_7.js";
-import { t as OpenAiCompletionProvider } from "./completion-DZNxcyfG.js";
+import { t as OpenAiChatCompletionProvider } from "./chat-DTdf-J5Q.js";
+import { t as OpenAiEmbeddingProvider } from "./embedding-CU78FMnw.js";
+import { t as OpenAiCompletionProvider } from "./completion-DUScduXp.js";
 //#region src/providers/litellm.ts
 /**
 * Base class for LiteLLM providers that maintains LiteLLM identity
@@ -134,4 +134,4 @@ function createLiteLLMProvider(providerPath, options = {}) {
 //#endregion
 export { createLiteLLMProvider };
 
-//# sourceMappingURL=litellm-BXyn5kZK.js.map
+//# sourceMappingURL=litellm-CYfgxLrM.js.map

package/dist/src/{litellm-CNcfbCfa.js → litellm-ojWBKU3C.js}

@@ -1,7 +1,7 @@
 import { T as getEnvString } from "./logger-Ct2S6Yx-.js";
-import { t as OpenAiChatCompletionProvider } from "./chat-Dabu84Br.js";
-import { t as OpenAiEmbeddingProvider } from "./embedding-Bi3rxrZF.js";
-import { t as OpenAiCompletionProvider } from "./completion-sNvCLTAP.js";
+import { t as OpenAiChatCompletionProvider } from "./chat-BBEnnpQk.js";
+import { t as OpenAiEmbeddingProvider } from "./embedding-CStK0TV6.js";
+import { t as OpenAiCompletionProvider } from "./completion-D_2IOAoS.js";
 //#region src/providers/litellm.ts
 /**
 * Base class for LiteLLM providers that maintains LiteLLM identity
@@ -133,4 +133,4 @@ function createLiteLLMProvider(providerPath, options = {}) {
 //#endregion
 export { createLiteLLMProvider };
 
-//# sourceMappingURL=litellm-CNcfbCfa.js.map
+//# sourceMappingURL=litellm-ojWBKU3C.js.map

package/dist/src/{luma-ray-D3FUc2K3.cjs → luma-ray-BCCO9XXG.cjs}

@@ -1,8 +1,8 @@
 const require_rolldown_runtime = require("./rolldown-runtime-D_mwlA32.cjs");
 const require_logger = require("./logger-cfNpzI4o.cjs");
-const require_fetch = require("./fetch-Dw4XZHjj.cjs");
+const require_fetch = require("./fetch-Cfm4EuOB.cjs");
 const require_text = require("./text-nywWsRBM.cjs");
-const require_base = require("./base-DGJW48uz.cjs");
+const require_base = require("./base-BzdS8tod.cjs");
 const require_blobs = require("./blobs-B0977K1O.cjs");
 let fs = require("fs");
 fs = require_rolldown_runtime.__toESM(fs, 1);
@@ -313,4 +313,4 @@ var LumaRayVideoProvider = class extends require_base.AwsBedrockGenericProvider
 //#endregion
 exports.LumaRayVideoProvider = LumaRayVideoProvider;
 
-//# sourceMappingURL=luma-ray-D3FUc2K3.cjs.map
+//# sourceMappingURL=luma-ray-BCCO9XXG.cjs.map

package/dist/src/{luma-ray-BMX1iEB6.js → luma-ray-BPrdihAb.js}

@@ -1,8 +1,8 @@
 import { a as logger } from "./logger-KD8JjCRJ.js";
-import { m as sleep } from "./fetch-BufrQtvR.js";
+import { m as sleep } from "./fetch-BSSAcMxf.js";
 import { i as storeBlob } from "./blobs-CeFdPn_T.js";
 import { t as ellipsize } from "./text-CZr46tp_.js";
-import { t as AwsBedrockGenericProvider } from "./base-E9I8zXjz.js";
+import { t as AwsBedrockGenericProvider } from "./base-0Gzzue9Z.js";
 import * as fs$1 from "fs";
 import * as path$1 from "path";
 //#region src/providers/bedrock/luma-ray.ts
@@ -310,4 +310,4 @@ var LumaRayVideoProvider = class extends AwsBedrockGenericProvider {
 //#endregion
 export { LumaRayVideoProvider };
 
-//# sourceMappingURL=luma-ray-BMX1iEB6.js.map
+//# sourceMappingURL=luma-ray-BPrdihAb.js.map

package/dist/src/{luma-ray-CR5TSpp4.js → luma-ray-DP5N79lB.js}

@@ -1,7 +1,7 @@
 import { a as logger } from "./logger-Ct2S6Yx-.js";
-import { l as sleep } from "./fetch-It34O8Ur.js";
+import { l as sleep } from "./fetch-Doks14zQ.js";
 import { t as ellipsize } from "./text-DwYK5EBn.js";
-import { t as AwsBedrockGenericProvider } from "./base-CKLo890h.js";
+import { t as AwsBedrockGenericProvider } from "./base-BuNn-YBX.js";
 import { r as storeBlob } from "./blobs-Dwef1Ao1.js";
 import * as fs$1 from "fs";
 import * as path$1 from "path";
@@ -310,4 +310,4 @@ var LumaRayVideoProvider = class extends AwsBedrockGenericProvider {
 //#endregion
 export { LumaRayVideoProvider };
 
-//# sourceMappingURL=luma-ray-CR5TSpp4.js.map
+//# sourceMappingURL=luma-ray-DP5N79lB.js.map

package/dist/src/{luma-ray-OEMmS1RB.js → luma-ray-DTDyoAlM.js}

@@ -1,8 +1,8 @@
 #!/usr/bin/env node
 import { s as logger } from "./logger-BbY6ypFL.js";
-import { f as sleep } from "./fetch-DXUnXkVU.js";
+import { f as sleep } from "./fetch-Cpf1U1nO.js";
 import { t as ellipsize } from "./text-Db-Wt2u2.js";
-import { t as AwsBedrockGenericProvider } from "./base-Co80MMCi.js";
+import { t as AwsBedrockGenericProvider } from "./base-Bp4c52YZ.js";
 import { i as storeBlob } from "./blobs-DODuTK-a.js";
 import * as fs$1 from "fs";
 import * as path$1 from "path";
@@ -311,4 +311,4 @@ var LumaRayVideoProvider = class extends AwsBedrockGenericProvider {
 //#endregion
 export { LumaRayVideoProvider };
 
-//# sourceMappingURL=luma-ray-OEMmS1RB.js.map
+//# sourceMappingURL=luma-ray-DTDyoAlM.js.map

package/dist/src/main.js

@@ -1,44 +1,44 @@
 #!/usr/bin/env node
 import { C as getConfigDirectoryPath, D as getEnvInt, E as getEnvFloat, M as isNonInteractive, N as state, O as getEnvString, T as getEnvBool, _ as extractJsonObjects, a as isDebugEnabled, b as orderKeys, c as setLogCallback, g as extractFirstJsonObject, i as initializeRunLogging, j as isCI, l as setLogLevel, m as sanitizeObject, n as getLogLevel, s as logger, t as closeLogger, w as setConfigDirectoryPath } from "./logger-BbY6ypFL.js";
-import { F as getShareApiBaseUrl, I as getShareViewBaseUrl, M as TERMINAL_MAX_WIDTH, N as getDefaultPort, O as CLOUD_PROVIDER_PREFIX, P as getDefaultShareViewBaseUrl, R as HUMAN_ASSERTION_TYPE, _ as REQUEST_TIMEOUT_MS, f as sleep, i as fetchWithTimeout, n as fetchWithProxy, r as fetchWithRetries, t as clearAgentCache, x as isPromptfooSampleTarget } from "./fetch-DXUnXkVU.js";
-import { n as VERSION } from "./version-eRkNuGv8.js";
+import { F as getShareApiBaseUrl, I as getShareViewBaseUrl, M as TERMINAL_MAX_WIDTH, N as getDefaultPort, O as CLOUD_PROVIDER_PREFIX, P as getDefaultShareViewBaseUrl, R as HUMAN_ASSERTION_TYPE, _ as REQUEST_TIMEOUT_MS, f as sleep, i as fetchWithTimeout, n as fetchWithProxy, r as fetchWithRetries, t as clearAgentCache, x as isPromptfooSampleTarget } from "./fetch-Cpf1U1nO.js";
+import { n as VERSION } from "./version-F0YDgb7J.js";
 import { t as invariant } from "./invariant-B2Rf6avk.js";
-import { a as getAuthor, c as isLoggedIntoCloud, l as promptForEmailUnverified, n as checkEmailStatusAndMaybeExit, o as getUserEmail, r as clearUserEmail, s as getUserId, t as checkEmailStatus, u as setUserEmail } from "./accounts-CjFnOPmb.js";
-import { a as cloudConfig, c as writeGlobalConfigPartial, o as readGlobalConfig, r as CloudConfig } from "./cloud-D3DiFqH6.js";
-import { n as TelemetryEventSchema, r as telemetry } from "./telemetry-00ezXr_t.js";
+import { a as getAuthor, c as isLoggedIntoCloud, l as promptForEmailUnverified, n as checkEmailStatusAndMaybeExit, o as getUserEmail, r as clearUserEmail, s as getUserId, t as checkEmailStatus, u as setUserEmail } from "./accounts-D6IBfEE0.js";
+import { a as cloudConfig, c as writeGlobalConfigPartial, o as readGlobalConfig, r as CloudConfig } from "./cloud-DqF5N1aJ.js";
+import { n as TelemetryEventSchema, r as telemetry } from "./telemetry-DOE567Wj.js";
 import { $ as HARM_PLUGINS, B as categoryAliases, D as ADDITIONAL_STRATEGIES, E as isUuid, F as getDefaultNFanout, G as ADDITIONAL_PLUGINS, H as riskCategorySeverityMap, J as CANARY_BREAKING_STRATEGY_IDS, K as ALL_PLUGINS, L as isFanoutStrategy, N as STRATEGY_COLLECTIONS, P as STRATEGY_COLLECTION_MAPPINGS, Q as FOUNDATION_PLUGINS, R as isMultiTurnStrategy, S as PluginConfigSchema, U as subCategoryDescriptions, W as ALIASED_PLUGIN_MAPPINGS, X as DEFAULT_PLUGINS, Y as DATASET_EXEMPT_PLUGINS, Z as FINANCIAL_PLUGINS, _ as PromptSchema, _t as CODING_AGENT_PLUGINS, a as EvaluateOptionsSchema, at as MULTI_INPUT_VAR, b as ConversationMessageSchema, c as TestSuiteConfigSchema, dt as REMOTE_ONLY_PLUGIN_IDS, et as INSURANCE_PLUGINS, g as RedteamGenerateOptionsSchema, gt as CODING_AGENT_CORE_PLUGINS, h as RedteamConfigSchema, ht as UNALIGNED_PROVIDER_HARM_PLUGINS, i as EvalResultsFilterMode, it as MULTI_INPUT_EXCLUDED_PLUGINS, j as DEFAULT_STRATEGIES, k as ALL_STRATEGIES, l as TestSuiteSchema, lt as REDTEAM_MODEL, mt as TELECOM_PLUGINS, n as BaseAssertionTypesSchema, nt as LLAMA_GUARD_REPLICATE_PROVIDER, o as OutputFileExtension, ot as PHARMACY_PLUGINS, p as isApiProvider, pt as TEEN_SAFETY_PLUGINS, q as BIAS_PLUGINS, r as CommandLineOptionsSchema, rt as MEDICAL_PLUGINS, s as ResultFailureReason, st as PII_PLUGINS, t as AssertionOrSetSchema, tt as LLAMA_GUARD_ENABLED_CATEGORIES, u as UnifiedConfigSchema, ut as REDTEAM_PROVIDER_HARM_PLUGINS, v as ProviderOptionsSchema, w as StrategyConfigSchema, x as PartialGenerationError, y as ProvidersSchema, z as Severity } from "./types-BFevViUY.js";
 import { i as isJavascriptFile } from "./fileExtensions-D4GCJ67J.js";
 import { r as importModule, t as getDirectory } from "./esm-Bexx2PFc.js";
 import { a as extractVariablesFromTemplates, i as extractVariablesFromTemplate, o as getNunjucksEngine, t as renderEnvOnlyInObject } from "./render-CSP99NLm.js";
-import { a as getRemoteHealthUrl, l as shouldGenerateRemote, n as getRemoteGenerationExplicitlyDisabledError, r as getRemoteGenerationUrl, s as neverGenerateRemote } from "./remoteGeneration-B1_XsKXU.js";
-import { a as openAuthBrowser, i as checkServerRunning, n as BrowserBehaviorNames, o as openBrowser, s as promptUser, t as BrowserBehavior } from "./server-DhMHosWj.js";
-import { _ as makeRequest$1, c as getCloudDatabaseId, d as getOrgContext, f as getPluginSeverityOverridesFromCloud, g as isCloudProvider, h as getUserTeams, i as retrieveMedia, l as getConfigFromCloud, m as getProviderFromCloud, o as canCreateTargets, r as mediaExists, s as checkCloudPermissions, t as getMediaStorage, u as getEvalConfigFromCloud, v as resolveTeamFromIdentifier, y as resolveTeamId } from "./storage-BU4qcnOb.js";
+import { a as getRemoteHealthUrl, l as shouldGenerateRemote, n as getRemoteGenerationExplicitlyDisabledError, r as getRemoteGenerationUrl, s as neverGenerateRemote } from "./remoteGeneration-D6UjE2JT.js";
+import { a as openAuthBrowser, i as checkServerRunning, n as BrowserBehaviorNames, o as openBrowser, s as promptUser, t as BrowserBehavior } from "./server-DLYjSFm2.js";
+import { _ as makeRequest$1, c as getCloudDatabaseId, d as getOrgContext, f as getPluginSeverityOverridesFromCloud, g as isCloudProvider, h as getUserTeams, i as retrieveMedia, l as getConfigFromCloud, m as getProviderFromCloud, o as canCreateTargets, r as mediaExists, s as checkCloudPermissions, t as getMediaStorage, u as getEvalConfigFromCloud, v as resolveTeamFromIdentifier, y as resolveTeamId } from "./storage-EKVWZBNY.js";
 import { n as sha256, t as randomSequence } from "./createHash-CgRvs4Fn.js";
-import { a as fetchWithCache, n as clearCache, r as disableCache, t as cache_exports } from "./cache-DpPWrkTE.js";
+import { a as fetchWithCache, n as clearCache, r as disableCache, t as cache_exports } from "./cache-BR77mdIR.js";
 import { r as runPython } from "./pythonUtils-CgYxeSmO.js";
-import { D as normalizeProviderRef, L as readFilters, M as maybeLoadFromExternalFile, R as readOutput, _ as resultIsForTestCase, a as ComparisonEvalNotFoundError, b as getProviderDescription, c as getEvalTableOutputPromptLocationsBySize, d as fetchCsvFromGoogleSheet, f as setupEnv, g as getTestCaseDeduplicationKey, h as filterRuntimeVars, i as writeOutput, j as maybeLoadConfigFromExternalFile, l as getEvalTablePromptStrippedPayload, m as extractRuntimeVars, n as createOutputMetadata, o as evalTableToJson, p as deduplicateTestCases, r as writeMultipleOutputs, s as generateEvalCsv, t as printBorder, u as mergeComparisonTables, v as checkProviderApiKeys, y as doesProviderRefMatch } from "./util-DURocbYR.js";
-import { a as resolveProviderConfigs, d as MCPProvider, f as HttpProvider, i as resolveProvider, m as createTransformResponse, n as loadApiProvider, p as createTransformRequest, r as loadApiProviders, t as getProviderIds } from "./providers-B9KzWxAX.js";
+import { D as normalizeProviderRef, L as readFilters, M as maybeLoadFromExternalFile, R as readOutput, _ as resultIsForTestCase, a as ComparisonEvalNotFoundError, b as getProviderDescription, c as getEvalTableOutputPromptLocationsBySize, d as fetchCsvFromGoogleSheet, f as setupEnv, g as getTestCaseDeduplicationKey, h as filterRuntimeVars, i as writeOutput, j as maybeLoadConfigFromExternalFile, l as getEvalTablePromptStrippedPayload, m as extractRuntimeVars, n as createOutputMetadata, o as evalTableToJson, p as deduplicateTestCases, r as writeMultipleOutputs, s as generateEvalCsv, t as printBorder, u as mergeComparisonTables, v as checkProviderApiKeys, y as doesProviderRefMatch } from "./util-jZRrXe1P.js";
+import { a as resolveProviderConfigs, d as MCPProvider, f as HttpProvider, i as resolveProvider, m as createTransformResponse, n as loadApiProvider, p as createTransformRequest, r as loadApiProviders, t as getProviderIds } from "./providers-B7TyByfj.js";
 import { a as createEmptyTokenUsage, i as createEmptyAssertions, n as accumulateResponseTokenUsage, r as accumulateTokenUsage, t as accumulateAssertionTokenUsage } from "./tokenUsageUtils-CDet74yk.js";
 import { n as escapeRegExp, t as ellipsize } from "./text-Db-Wt2u2.js";
 import { a as evalResultsTable, c as evalsToPromptsTable, d as promptsTable, g as getDb, h as closeDbIfOpen, i as datasetsTable, l as evalsToTagsTable, n as blobReferencesTable, o as evalsTable, p as tagsTable, r as configsTable, s as evalsToDatasetsTable, t as blobAssetsTable, u as modelAuditsTable } from "./tables-DafUHOeh.js";
 import { n as getBlobUrl, t as getBlobByHash } from "./blobs-DODuTK-a.js";
-import { n as createPlaceholderInputValue, r as materializeInputVariablesWithMetadata, t as buildPromptInputDescriptions } from "./inputVariables-DXFdi7AI.js";
-import { a as extractPromptFromTags, i as extractMaterializedVariablesFromJsonWithMetadata, n as extractGoalFromPrompt, r as extractInputVarsFromPrompt, s as getShortPluginId } from "./util-kDURhgJW.js";
-import { n as PromptfooHarmfulCompletionProvider } from "./promptfoo-Rjp-MeBb.js";
-import { A as DivergentRepetitionPlugin, B as sampleArray, C as getPiiLeakTestsForCategory, Ct as getDefaultProviders, D as HarmbenchPlugin, E as ImitationPlugin, F as AegisPlugin, I as RedteamGraderBase, L as RedteamPluginBase, M as CrossSessionLeakPlugin, N as ContractPlugin, O as HallucinationPlugin, P as BeavertailsPlugin, Q as readProviderPromptMap, R as getCustomPolicies, S as PlinyPlugin, T as IntentPlugin, V as fetchHuggingFaceDataset, X as processPrompts, Y as doRemoteGrading, Z as readPrompts, _ as PoliticsPlugin, a as UnverifiableClaimsPlugin, b as isValidPolicyObject, c as ToolDiscoveryPlugin, d as TeenSafetyDangerousContentPlugin, f as TeenSafetyAgeRestrictedGoodsAndServicesPlugin, g as PromptExtractionPlugin, h as RbacPlugin, i as VLGuardPlugin, j as DebugAccessPlugin, k as ExcessiveAgencyPlugin, l as TeenSafetyHarmfulBodyIdealsPlugin, m as ShellInjectionPlugin, o as UnsafeBenchPlugin, p as SqlInjectionPlugin, r as VLSUPlugin, s as ToxicChatPlugin, t as GRADERS, u as TeenSafetyDangerousRoleplayPlugin, v as PolicyPlugin, w as OverreliancePlugin, x as makeInlinePolicyIdSync, y as determinePolicyTypeFromId, z as retryWithDeduplication } from "./graders-BGP99PdK.js";
-import { f as redteamProviderManager, m as TokenUsageTracker } from "./shared-9WHQ1oNE.js";
-import { c as assertions_default, l as readAssertions, n as evaluate$1, o as accumulateNamedMetric, s as doesPromptRefMatch, u as runAssertions } from "./evaluator-DRoiYB2q.js";
+import { n as createPlaceholderInputValue, t as buildPromptInputDescriptions } from "./inputVariables-DXFdi7AI.js";
+import { a as extractPromptFromTags, i as extractMaterializedVariablesFromJsonWithMetadata, n as extractGoalFromPrompt, s as getShortPluginId } from "./util-ETfU_sS9.js";
+import { a as assertRemoteMaterializationHandled, l as requiresRemoteMaterialization, n as PromptfooHarmfulCompletionProvider } from "./promptfoo-BU4_0J85.js";
+import { A as DivergentRepetitionPlugin, B as sampleArray, C as getPiiLeakTestsForCategory, Ct as getDefaultProviders, D as HarmbenchPlugin, E as ImitationPlugin, F as AegisPlugin, I as RedteamGraderBase, L as RedteamPluginBase, M as CrossSessionLeakPlugin, N as ContractPlugin, O as HallucinationPlugin, P as BeavertailsPlugin, Q as readProviderPromptMap, R as getCustomPolicies, S as PlinyPlugin, T as IntentPlugin, V as fetchHuggingFaceDataset, X as processPrompts, Y as doRemoteGrading, Z as readPrompts, _ as PoliticsPlugin, a as UnverifiableClaimsPlugin, b as isValidPolicyObject, c as ToolDiscoveryPlugin, d as TeenSafetyDangerousContentPlugin, f as TeenSafetyAgeRestrictedGoodsAndServicesPlugin, g as PromptExtractionPlugin, h as RbacPlugin, i as VLGuardPlugin, j as DebugAccessPlugin, k as ExcessiveAgencyPlugin, l as TeenSafetyHarmfulBodyIdealsPlugin, m as ShellInjectionPlugin, o as UnsafeBenchPlugin, p as SqlInjectionPlugin, r as VLSUPlugin, s as ToxicChatPlugin, t as GRADERS, u as TeenSafetyDangerousRoleplayPlugin, v as PolicyPlugin, w as OverreliancePlugin, x as makeInlinePolicyIdSync, y as determinePolicyTypeFromId, z as retryWithDeduplication } from "./graders-BQt1BaQe.js";
+import { f as redteamProviderManager, m as TokenUsageTracker } from "./shared-DNvim54U.js";
+import { c as assertions_default, l as readAssertions, n as evaluate$1, o as accumulateNamedMetric, s as doesPromptRefMatch, u as runAssertions } from "./evaluator-n_dEb00o.js";
 import { i as generateIdFromPrompt } from "./utils-BFOh20Gb.js";
 import { n as TRANSFORM_KEYS, t as INLINE_FUNCTION_LABEL } from "./transform-BnSTnFlp.js";
 import { n as getTraceStore } from "./store-DKd5592Q.js";
 import { n as setupSignalWatcher, t as readSignalEvalId } from "./signal-CSurUUyV.js";
-import { n as isBlobStorageEnabled } from "./extractor-CIW3iN-b.js";
+import { n as isBlobStorageEnabled } from "./extractor-BVkZtk4R.js";
 import { n as getGeneratedPromptOverLimit, r as getMaxCharsPerMessageModifierValue, t as MAX_CHARS_PER_MESSAGE_MODIFIER_KEY } from "./promptLength-4X-Wd8PG.js";
-import { a as createEvalId, n as EvalQueries, s as getEvalSummaries, t as Eval } from "./eval-0VRANImH.js";
-import { n as sanitizeProvider, t as EvalResult } from "./evalResult-eUkJv9Ko.js";
-import { n as renderPrompt } from "./evaluatorHelpers-BsYP_muT.js";
+import { a as createEvalId, n as EvalQueries, s as getEvalSummaries, t as Eval } from "./eval-B3BaNBbO.js";
+import { n as sanitizeProvider, t as EvalResult } from "./evalResult-CcSqNl_Y.js";
+import { n as renderPrompt } from "./evaluatorHelpers-D1_kwvyp.js";
 import { t as formatDuration } from "./formatDuration-CMVNrYvE.js";
-import { i as pluginMatchesStrategyTargets, n as loadStrategy, r as validateStrategies, t as Strategies } from "./strategies-DRJjGTIY.js";
+import { i as pluginMatchesStrategyTargets, n as loadStrategy, r as validateStrategies, t as Strategies } from "./strategies-CBI1brSy.js";
 import { n as shouldUseInkUI } from "./interactiveCheck-C4QlIuoR.js";
 import { fileURLToPath } from "node:url";
 import { Command, InvalidArgumentError } from "commander";
@@ -111,7 +111,7 @@ function runCommand(program) {
 			hasGithubPr: !!cmdObj.githubPr,
 			hasGuidance: !!(cmdObj.guidance || cmdObj.guidanceFile)
 		});
-		const { executeScan } = await import("./scanner-DS0109SS.js");
+		const { executeScan } = await import("./scanner-BS-iFIp3.js");
 		await executeScan(repoPath, cmdObj);
 	});
 }
@@ -6649,6 +6649,7 @@ async function fetchRemoteTestCases(key, purpose, injectVar, n, config) {
 			logger.error(`Error generating test cases for ${key}: ${statusText} ${JSON.stringify(data)}`);
 			return [];
 		}
+		if (requiresRemoteMaterialization(config?.inputs)) assertRemoteMaterializationHandled(data, `Remote plugin generation for ${key}`);
 		const ret = data.result;
 		logger.debug(`Received remote generation for ${key}:\n${JSON.stringify(ret)}`);
 		return ret;
@@ -6657,31 +6658,6 @@ async function fetchRemoteTestCases(key, purpose, injectVar, n, config) {
 		return [];
 	}
 }
-async function materializeRemoteTestCaseInputs({ config, injectVar, pluginId, provider, purpose, testCases }) {
-	const inputs = config.inputs;
-	if (!inputs || Object.keys(inputs).length === 0) return testCases;
-	return Promise.all(testCases.map(async (testCase, materializationIndex) => {
-		const inputVars = extractInputVarsFromPrompt(String(testCase.vars?.[injectVar] ?? ""), inputs);
-		if (!inputVars) return testCase;
-		const materializedVars = await materializeInputVariablesWithMetadata(inputVars, inputs, {
-			materializationIndex,
-			pluginId,
-			provider,
-			purpose
-		});
-		return {
-			...testCase,
-			vars: {
-				...testCase.vars || {},
-				...materializedVars.vars
-			},
-			metadata: {
-				...testCase.metadata || {},
-				...materializedVars.metadata ? { inputMaterialization: materializedVars.metadata } : {}
-			}
-		};
-	}));
-}
 function createPluginFactory(PluginClass, key, validate) {
 	return {
 		key,
@@ -6693,14 +6669,7 @@ function createPluginFactory(PluginClass, key, validate) {
 				return new PluginClass(provider, purpose, injectVar, configWithDefaults).generateTests(n, delayMs);
 			}
 			const pluginId = getShortPluginId(key);
-			const testCases = await materializeRemoteTestCaseInputs({
-				config: configWithDefaults ?? {},
-				injectVar,
-				pluginId,
-				provider,
-				purpose,
-				testCases: await fetchRemoteTestCases(key, purpose, injectVar, n, configWithDefaults ?? {})
-			});
+			const testCases = await fetchRemoteTestCases(key, purpose, injectVar, n, configWithDefaults ?? {});
 			const computedModifiers = computeModifiersFromConfig(configWithDefaults);
 			return testCases.map((testCase) => ({
 				...testCase,
@@ -6787,14 +6756,7 @@ const piiPlugins = PII_PLUGINS.map((category) => ({
 	action: async (params) => {
 		if (shouldGenerateRemote()) {
 			const pluginId = getShortPluginId(category);
-			const testCases = await materializeRemoteTestCaseInputs({
-				config: params.config ?? {},
-				injectVar: params.injectVar,
-				pluginId,
-				provider: params.provider,
-				purpose: params.purpose,
-				testCases: await fetchRemoteTestCases(category, params.purpose, params.injectVar, params.n, params.config ?? {})
-			});
+			const testCases = await fetchRemoteTestCases(category, params.purpose, params.injectVar, params.n, params.config ?? {});
 			const computedModifiers = computeModifiersFromConfig(params.config);
 			return testCases.map((testCase) => ({
 				...testCase,
@@ -6826,14 +6788,7 @@ const biasPlugins = BIAS_PLUGINS.map((category) => ({
 			return [];
 		}
 		const pluginId = getShortPluginId(category);
-		const testCases = await materializeRemoteTestCaseInputs({
-			config: params.config ?? {},
-			injectVar: params.injectVar,
-			pluginId,
-			provider: params.provider,
-			purpose: params.purpose,
-			testCases: await fetchRemoteTestCases(category, params.purpose, params.injectVar, params.n, params.config ?? {})
-		});
+		const testCases = await fetchRemoteTestCases(category, params.purpose, params.injectVar, params.n, params.config ?? {});
 		const computedModifiers = computeModifiersFromConfig(params.config);
 		return testCases.map((testCase) => ({
 			...testCase,
@@ -6852,21 +6807,14 @@ function createRemotePlugin(key, validate) {
 	return {
 		key,
 		validate,
-		action: async ({ provider, purpose, injectVar, n, config }) => {
+		action: async ({ purpose, injectVar, n, config }) => {
 			const configWithDefaults = applyDefaultRemotePluginConfig(key, config);
 			if (neverGenerateRemote()) {
 				logger.error(getRemoteGenerationExplicitlyDisabledError(`${key} plugin`));
 				return [];
 			}
 			const pluginId = getShortPluginId(key);
-			const testCases = await materializeRemoteTestCaseInputs({
-				config: configWithDefaults ?? {},
-				injectVar,
-				pluginId,
-				provider,
-				purpose,
-				testCases: await fetchRemoteTestCases(key, purpose, injectVar, n, configWithDefaults ?? {})
-			});
+			const testCases = await fetchRemoteTestCases(key, purpose, injectVar, n, configWithDefaults ?? {});
 			const computedModifiers = computeModifiersFromConfig(configWithDefaults);
 			const testsWithMetadata = testCases.map((testCase) => ({
 				...testCase,
@@ -6933,6 +6881,15 @@ async function validateSharpDependency(strategies, plugins, checkSharp = isSharp
 }
 //#endregion
 //#region src/redteam/index.ts
+const MATERIALIZED_MULTI_INPUT_PROMPT_METADATA_KEY = "__promptfooMaterializedMultiInputPrompt";
+function getMaterializedMultiInputPromptSnapshot(metadata) {
+	const snapshot = metadata?.[MATERIALIZED_MULTI_INPUT_PROMPT_METADATA_KEY];
+	return typeof snapshot === "string" ? snapshot : void 0;
+}
+function getMaterializedMultiInputPromptMetadata(vars) {
+	const prompt = vars?.[MULTI_INPUT_VAR];
+	return typeof prompt === "string" ? { [MATERIALIZED_MULTI_INPUT_PROMPT_METADATA_KEY]: prompt } : void 0;
+}
 function getPolicyText(metadata) {
 	if (!metadata || metadata.policy === void 0 || metadata.policy === null) return;
 	const policyValue = metadata.policy;
@@ -6945,12 +6902,19 @@ function getPolicyText(metadata) {
 async function rematerializeStrategyInputVars(testCase, injectVar, provider, purpose, materializationIndex) {
 	const inputs = testCase.metadata?.pluginConfig?.inputs;
 	const inputMaterialization = testCase.metadata?.inputMaterialization;
-	if (!inputs || Object.keys(inputs).length === 0 || !testCase.vars?.[injectVar]) return {
+	const materializedPromptSnapshot = getMaterializedMultiInputPromptSnapshot(testCase.metadata);
+	const currentInjectVar = testCase.vars?.[injectVar];
+	if (!inputs || Object.keys(inputs).length === 0 || !currentInjectVar) return {
+		inputMaterialization,
+		vars: testCase.vars
+	};
+	const promptChangedSinceMaterialization = typeof currentInjectVar === "string" && typeof materializedPromptSnapshot === "string" && currentInjectVar !== materializedPromptSnapshot;
+	if (Boolean(inputMaterialization) && Object.keys(inputs).every((key) => Object.prototype.hasOwnProperty.call(testCase.vars ?? {}, key)) && !promptChangedSinceMaterialization) return {
 		inputMaterialization,
 		vars: testCase.vars
 	};
 	try {
-		const materializedVars = await extractMaterializedVariablesFromJsonWithMetadata(JSON.parse(String(testCase.vars[injectVar])), inputs, {
+		const materializedVars = await extractMaterializedVariablesFromJsonWithMetadata(JSON.parse(String(currentInjectVar)), inputs, {
 			materializationIndex,
 			pluginId: String(testCase.metadata?.pluginId || "unknown-plugin"),
 			provider,
@@ -7182,7 +7146,8 @@ function addLanguageToPluginMetadata(test, lang, plugin, maxCharsPerMessage, tes
 				...test.metadata?.modifiers,
 				...languageToAdd
 			},
-			...languageToAdd
+			...languageToAdd,
+			...getMaterializedMultiInputPromptMetadata(test.vars)
 		}
 	};
 }
@@ -7281,7 +7246,8 @@ async function applyStrategies(testCases, strategies, injectVar, provider, purpo
 					...t?.metadata?.pluginId && { pluginId: t.metadata.pluginId },
 					...t?.metadata?.pluginConfig && { pluginConfig: t.metadata.pluginConfig },
 					...inputMaterialization && { inputMaterialization },
-					...Object.keys(strategyConfig).length > 0 && { strategyConfig }
+					...Object.keys(strategyConfig).length > 0 && { strategyConfig },
+					...getMaterializedMultiInputPromptMetadata(vars)
 				}
 			};
 		})));
@@ -7627,14 +7593,21 @@ async function synthesize$1({ abortSignal, delay, entities: entitiesOverride, in
 				}
 			});
 			const languageResults = await Promise.allSettled(languagePromises);
-			for (const result of languageResults) if (result.status === "fulfilled") {
+			for (const [index, result] of languageResults.entries()) if (result.status === "fulfilled") {
 				const { lang, tests, requested, generated } = result.value;
 				allPluginTests.push(...tests);
 				resultsPerLanguage[lang || "default"] = {
 					requested,
 					generated
 				};
-			} else logger.warn(`[Language Processing] Error generating tests for ${plugin.id}: ${result.reason}`);
+			} else {
+				const lang = languages[index];
+				logger.warn(`[Language Processing] Error generating tests for ${plugin.id}: ${result.reason}`);
+				resultsPerLanguage[lang || "default"] = {
+					requested: plugin.numTests,
+					generated: 0
+				};
+			}
 			logger.debug(`[Language Processing] Total tests generated for ${plugin.id}: ${allPluginTests.length} (across ${languages.length} language(s))`);
 			if (!Array.isArray(allPluginTests) || allPluginTests.length === 0) logger.warn(`Failed to generate tests for ${plugin.id}`);
 			else {
@@ -7668,35 +7641,51 @@ async function synthesize$1({ abortSignal, delay, entities: entitiesOverride, in
 				generated: allPluginTests.length
 			};
 		} else if (plugin.id.startsWith("file://")) try {
-			const customTests = await new CustomPlugin(redteamProvider, purpose, injectVar, plugin.id, resolvePluginConfigWithMaxChars(plugin.config, maxCharsPerMessage)).generateTests(plugin.numTests, delay);
-			const testCasesWithMetadata = filterOversizedTestCases(customTests.map((t) => {
-				const includePluginConfig = !(t.metadata && Object.hasOwn(t.metadata, "pluginConfig") && t.metadata.pluginConfig === void 0);
-				const pluginConfigWithMaxChars = {
+			const languageConfig = plugin.config?.language ?? language;
+			const languages = Array.isArray(languageConfig) ? languageConfig : languageConfig ? [languageConfig] : [void 0];
+			const allCustomTests = [];
+			const resultsPerLanguage = {};
+			const languagePromises = languages.map(async (lang) => {
+				const resolvedConfig = {
 					...resolvePluginConfigWithMaxChars(plugin.config, maxCharsPerMessage),
-					...t.metadata?.pluginConfig ?? {}
+					...lang ? { language: lang } : {},
+					...hasMultipleInputs ? { inputs } : {}
 				};
-				const modifiers = {
-					...buildRedteamModifiers({
+				const customPluginConfig = {
+					...resolvedConfig,
+					modifiers: buildRedteamModifiers({
 						maxCharsPerMessage,
-						pluginConfig: pluginConfigWithMaxChars,
+						pluginConfig: resolvedConfig,
 						testGenerationInstructions
-					}),
-					...t.metadata?.modifiers
+					})
 				};
+				const testCasesWithMetadata = filterOversizedTestCases((await new CustomPlugin(redteamProvider, purpose, injectVar, plugin.id, customPluginConfig).generateTests(plugin.numTests, delay)).map((t) => addLanguageToPluginMetadata(t, lang, plugin, maxCharsPerMessage, testGenerationInstructions)), injectVar, `Custom plugin ${plugin.id}`, maxCharsPerMessage);
 				return {
-					...t,
-					metadata: {
-						...t.metadata || {},
-						pluginId: plugin.id,
-						...includePluginConfig && { pluginConfig: pluginConfigWithMaxChars },
-						severity: plugin.severity || getPluginSeverity(plugin.id, resolvePluginConfig(plugin.config)),
-						modifiers
-					}
+					lang,
+					tests: testCasesWithMetadata,
+					requested: plugin.numTests,
+					generated: testCasesWithMetadata.length
+				};
+			});
+			const languageResults = await Promise.allSettled(languagePromises);
+			for (const [index, result] of languageResults.entries()) if (result.status === "fulfilled") {
+				const { lang, tests, requested, generated } = result.value;
+				allCustomTests.push(...tests);
+				resultsPerLanguage[lang || "default"] = {
+					requested,
+					generated
+				};
+			} else {
+				const lang = languages[index];
+				logger.warn(`[Language Processing] Error generating tests for custom plugin ${plugin.id}: ${result.reason}`);
+				resultsPerLanguage[lang || "default"] = {
+					requested: plugin.numTests,
+					generated: 0
 				};
-			}), injectVar, `Custom plugin ${plugin.id}`, maxCharsPerMessage);
+			}
 			if (needsGoalExtraction) {
-				logger.debug(`Extracting goal for ${testCasesWithMetadata.length} custom tests from ${plugin.id}...`);
-				for (const testCase of testCasesWithMetadata) {
+				logger.debug(`Extracting goal for ${allCustomTests.length} custom tests from ${plugin.id}...`);
+				for (const testCase of allCustomTests) {
 					const promptVar = testCase.vars?.[injectVar];
 					const prompt = Array.isArray(promptVar) ? promptVar[0] : String(promptVar);
 					const policy = getPolicyText(testCase.metadata);
@@ -7704,13 +7693,21 @@ async function synthesize$1({ abortSignal, delay, entities: entitiesOverride, in
 					testCase.metadata.goal = extractedGoal;
 				}
 			}
-			testCases.push(...testCasesWithMetadata);
-			logger.debug(`Added ${customTests.length} custom test cases from ${plugin.id}`);
-			const displayId = getPluginDisplayId(plugin);
-			pluginResults[displayId] = {
-				requested: plugin.numTests,
-				generated: testCasesWithMetadata.length
+			testCases.push(...allCustomTests);
+			logger.debug(`Added ${allCustomTests.length} custom test cases from ${plugin.id}`);
+			const baseDisplayId = getPluginDisplayId(plugin);
+			if (languages.filter((lang) => lang !== void 0).length > 1) for (const [langKey, result] of Object.entries(resultsPerLanguage)) {
+				const displayId = langKey === "en" ? baseDisplayId : `(${langKey}) ${baseDisplayId}`;
+				pluginResults[displayId] = {
+					requested: result.requested,
+					generated: result.generated
+				};
+			}
+			else pluginResults[baseDisplayId] = {
+				requested: plugin.numTests * languages.length,
+				generated: allCustomTests.length
 			};
+			progressBar?.increment(plugin.numTests * languages.length);
 		} catch (e) {
 			logger.error(`Error generating tests for custom plugin ${plugin.id}: ${e}`);
 			const displayId = getPluginDisplayId(plugin);
@@ -9671,7 +9668,7 @@ async function checkForUpdates() {
 	} catch {
 		return false;
 	}
-	if (semverGt(latestVersion, "0.121.7")) {
+	if (semverGt(latestVersion, "0.121.8")) {
 		const border = "=".repeat(TERMINAL_MAX_WIDTH);
 		logger.info(`\n${border}
 ${chalk.yellow("⚠️")} The current version of promptfoo ${chalk.yellow(VERSION)} is lower than the latest available version ${chalk.green(latestVersion)}.
@@ -13274,7 +13271,7 @@ router.get("/", async (_req, res) => {
 				};
 			} catch (error) {
 				logger.debug(`Failed to fetch latest version: ${error}`);
-				latestVersion = versionCache.latestVersion ?? "0.121.7";
+				latestVersion = versionCache.latestVersion ?? "0.121.8";
 			}
 		}
 		const selfHosted = getEnvBool("PROMPTFOO_SELF_HOSTED");
@@ -13283,7 +13280,7 @@ router.get("/", async (_req, res) => {
 			selfHosted,
 			isNpx
 		});
-		const resolvedLatestVersion = latestVersion ?? "0.121.7";
+		const resolvedLatestVersion = latestVersion ?? "0.121.8";
 		const response = {
 			currentVersion: VERSION,
 			latestVersion: resolvedLatestVersion,
@@ -18443,8 +18440,8 @@ function registerRunEvaluationTool(server) {
 					}
 					filteredTestSuite.tests = filteredTests;
 				}
-				const { evaluate } = await import("./evaluator-DNdJF1Gv.js");
-				const evalRecord = await (await import("./eval-DscR5iOM.js")).default.create(config, filteredTestSuite.prompts, { id: `mcp-eval-${Date.now()}` });
+				const { evaluate } = await import("./evaluator-B9LGbKI8.js");
+				const evalRecord = await (await import("./eval-DfR9885C.js")).default.create(config, filteredTestSuite.prompts, { id: `mcp-eval-${Date.now()}` });
 				logger.debug(`Running filtered eval with ${filteredTestSuite.tests?.length || 0} test cases, ${filteredTestSuite.prompts.length} prompts, ${filteredTestSuite.providers.length} providers`);
 				const startTime = Date.now();
 				const result = await evaluate(filteredTestSuite, evalRecord, {

package/dist/src/{memoryPoisoning-Dp-btinn.cjs → memoryPoisoning-B6N3us35.cjs}

@@ -1,11 +1,11 @@
 const require_logger = require("./logger-cfNpzI4o.cjs");
 const require_invariant = require("./invariant-QtnLD03y.cjs");
-const require_fetch = require("./fetch-Dw4XZHjj.cjs");
-const require_version = require("./version-BWCSaByA.cjs");
-const require_accounts = require("./accounts-CmWzeD2d.cjs");
-const require_remoteGeneration = require("./remoteGeneration-DS9N3pgB.cjs");
+const require_fetch = require("./fetch-Cfm4EuOB.cjs");
+const require_version = require("./version-VzUqOBZk.cjs");
+const require_accounts = require("./accounts-BVz5gHLK.cjs");
+const require_remoteGeneration = require("./remoteGeneration-BpIYlb_O.cjs");
 const require_tokenUsageUtils = require("./tokenUsageUtils-_B-P8IAi.cjs");
-const require_shared = require("./shared-WkgnDkcg.cjs");
+const require_shared = require("./shared-aHWko3P1.cjs");
 const require_promptLength = require("./promptLength-BbBbDHNj.cjs");
 const require_constants = require("./constants-a2kYssQk.cjs");
 //#region src/redteam/providers/agentic/memoryPoisoning.ts
@@ -103,4 +103,4 @@ var MemoryPoisoningProvider = class {
 //#endregion
 exports.MemoryPoisoningProvider = MemoryPoisoningProvider;
 
-//# sourceMappingURL=memoryPoisoning-Dp-btinn.cjs.map
+//# sourceMappingURL=memoryPoisoning-B6N3us35.cjs.map