promptfoo 0.121.7 → 0.121.8

package/dist/src/{hydra-BSNZZm2M.js → hydra-CR0KyYDb.js}

@@ -1,19 +1,19 @@
 #!/usr/bin/env node
 import { s as logger, y as isValidJson } from "./logger-BbY6ypFL.js";
-import { f as sleep } from "./fetch-DXUnXkVU.js";
+import { f as sleep } from "./fetch-Cpf1U1nO.js";
 import { t as invariant } from "./invariant-B2Rf6avk.js";
-import { c as isLoggedIntoCloud } from "./accounts-CjFnOPmb.js";
-import { l as shouldGenerateRemote, n as getRemoteGenerationExplicitlyDisabledError, s as neverGenerateRemote, t as getRemoteGenerationDisabledError } from "./remoteGeneration-B1_XsKXU.js";
+import { c as isLoggedIntoCloud } from "./accounts-D6IBfEE0.js";
+import { l as shouldGenerateRemote, n as getRemoteGenerationExplicitlyDisabledError, s as neverGenerateRemote, t as getRemoteGenerationDisabledError } from "./remoteGeneration-D6UjE2JT.js";
 import { a as createEmptyTokenUsage, n as accumulateResponseTokenUsage } from "./tokenUsageUtils-CDet74yk.js";
 import { r as materializeInputVariablesWithMetadata } from "./inputVariables-DXFdi7AI.js";
-import { a as extractPromptFromTags, c as isBasicRefusal, o as getSessionId, r as extractInputVarsFromPrompt } from "./util-kDURhgJW.js";
-import { t as PromptfooChatCompletionProvider } from "./promptfoo-Rjp-MeBb.js";
-import { c as getTargetResponse, i as externalizeResponseForRedteamHistory, l as isConversationEndedResponse, o as getGraderAssertionValue, t as buildGraderResultAssertion, u as isValidChatMessageArray } from "./shared-9WHQ1oNE.js";
-import { n as isBlobStorageEnabled, r as shouldAttemptRemoteBlobUpload } from "./extractor-CIW3iN-b.js";
-import { n as checkExfilTracking } from "./indirectWebPwn-uyWdHx04.js";
-import { n as renderPrompt } from "./evaluatorHelpers-BsYP_muT.js";
-import { t as Strategies } from "./strategies-DRJjGTIY.js";
-import { a as fetchTraceContext, i as extractTraceIdFromTraceparent, n as formatTraceForMetadata, r as formatTraceSummary, t as resolveTracingOptions } from "./tracingOptions-BnwKCkSB.js";
+import { a as extractPromptFromTags, c as isBasicRefusal, o as getSessionId, r as extractInputVarsFromPrompt } from "./util-ETfU_sS9.js";
+import { a as assertRemoteMaterializationHandled, s as buildRemoteMaterializedInputVariables, t as PromptfooChatCompletionProvider } from "./promptfoo-BU4_0J85.js";
+import { c as getTargetResponse, i as externalizeResponseForRedteamHistory, l as isConversationEndedResponse, o as getGraderAssertionValue, t as buildGraderResultAssertion, u as isValidChatMessageArray } from "./shared-DNvim54U.js";
+import { n as isBlobStorageEnabled, r as shouldAttemptRemoteBlobUpload } from "./extractor-BVkZtk4R.js";
+import { n as checkExfilTracking } from "./indirectWebPwn-BMTXXznx.js";
+import { n as renderPrompt } from "./evaluatorHelpers-D1_kwvyp.js";
+import { t as Strategies } from "./strategies-CBI1brSy.js";
+import { a as fetchTraceContext, i as extractTraceIdFromTraceparent, n as formatTraceForMetadata, r as formatTraceSummary, t as resolveTracingOptions } from "./tracingOptions-xhjOg2b1.js";
 import { t as applyRuntimeTransforms } from "./runtimeTransform-Dgh_D7DU.js";
 //#region src/redteam/providers/hydra/index.ts
 const DEFAULT_MAX_TURNS = 10;
@@ -112,11 +112,12 @@ var HydraProvider = class {
 		let storedGraderResult = void 0;
 		let lastTargetResponse = void 0;
 		let backtrackCount = 0;
+		let agentFailureError;
 		const redteamHistory = [];
 		let lastTransformResult;
 		let lastTransformDisplayVars;
 		let lastFinalAttackPrompt;
-		const { getGraderById } = await import("./graders-BDT7dif6.js");
+		const { getGraderById } = await import("./graders-B3D7kCcD.js");
 		let assertToUse = test?.assert?.find((a) => a.type && a.type.includes(test.metadata?.pluginId));
 		if (!assertToUse) assertToUse = test?.assert?.find((a) => a.type);
 		let previousTraceSummary;
@@ -160,6 +161,7 @@ var HydraProvider = class {
 					testRunId,
 					error: agentResp.error
 				});
+				agentFailureError = agentResp.error;
 				continue;
 			}
 			let nextMessage;
@@ -170,18 +172,31 @@ var HydraProvider = class {
 			}
 			if (!nextMessage) {
 				logger.info("[Hydra] Missing message from agent", { turn });
+				agentFailureError = "Hydra agent did not return an attack message";
 				continue;
 			}
 			let processedMessage = nextMessage;
 			const extractedPrompt = extractPromptFromTags(nextMessage);
 			if (extractedPrompt) processedMessage = extractedPrompt;
+			if (this.config.inputs && shouldGenerateRemote()) assertRemoteMaterializationHandled(agentResp, "Hydra multi-input generation");
 			const currentInputVars = extractInputVarsFromPrompt(processedMessage, this.config.inputs);
-			const currentRenderInputVars = (currentInputVars && this.config.inputs ? await materializeInputVariablesWithMetadata(currentInputVars, this.config.inputs, {
+			let materializedInputVars;
+			if (this.config.inputs && shouldGenerateRemote() && !currentInputVars && !agentResp.materializedVars) {
+				logger.warn("[Hydra] Remote multi-input generation returned an invalid prompt format", {
+					turn,
+					messagePreview: processedMessage.slice(0, 200)
+				});
+				agentFailureError = "Hydra remote multi-input generation returned an invalid prompt format";
+				continue;
+			}
+			if ((currentInputVars || agentResp.materializedVars) && this.config.inputs) if (shouldGenerateRemote()) materializedInputVars = buildRemoteMaterializedInputVariables(agentResp, currentInputVars ?? {}, this.config.inputs);
+			else materializedInputVars = await materializeInputVariablesWithMetadata(currentInputVars, this.config.inputs, {
 				materializationIndex: turn,
 				pluginId: "hydra",
 				provider: this.agentProvider,
 				purpose: test?.metadata?.purpose
-			}) : void 0)?.vars ?? currentInputVars;
+			});
+			const currentRenderInputVars = materializedInputVars?.vars ?? currentInputVars;
 			this.conversationHistory.push({
 				role: "user",
 				content: processedMessage
@@ -513,13 +528,16 @@ var HydraProvider = class {
 			role: msg.role,
 			content: msg.content
 		}));
+		const targetProbeCount = totalTokenUsage.numRequests ?? 0;
+		const hydraRoundsCompleted = this.conversationHistory.filter((m) => m.role === "user").length;
+		const failClosedError = targetProbeCount === 0 ? agentFailureError || "Hydra did not execute any target probes" : void 0;
 		return {
 			output: lastTargetResponse?.output || "",
-			...lastTargetResponse?.error ? { error: lastTargetResponse.error } : {},
+			...failClosedError ? { error: failClosedError } : lastTargetResponse?.error ? { error: lastTargetResponse.error } : {},
 			metadata: {
 				sessionId: this.sessionId || getSessionId(lastTargetResponse, context),
 				messages,
-				hydraRoundsCompleted: this.conversationHistory.filter((m) => m.role === "user").length,
+				hydraRoundsCompleted,
 				hydraBacktrackCount: backtrackCount,
 				hydraResult: vulnerabilityAchieved,
 				stopReason,
@@ -540,4 +558,4 @@ var HydraProvider = class {
 //#endregion
 export { HydraProvider, HydraProvider as default };
 
-//# sourceMappingURL=hydra-BSNZZm2M.js.map
+//# sourceMappingURL=hydra-CR0KyYDb.js.map

package/dist/src/{hydra-DrJttnvw.cjs → hydra-RO9jBks7.cjs}

@@ -1,18 +1,18 @@
 const require_logger = require("./logger-cfNpzI4o.cjs");
 const require_invariant = require("./invariant-QtnLD03y.cjs");
-const require_fetch = require("./fetch-Dw4XZHjj.cjs");
-const require_accounts = require("./accounts-CmWzeD2d.cjs");
-const require_remoteGeneration = require("./remoteGeneration-DS9N3pgB.cjs");
+const require_fetch = require("./fetch-Cfm4EuOB.cjs");
+const require_accounts = require("./accounts-BVz5gHLK.cjs");
+const require_remoteGeneration = require("./remoteGeneration-BpIYlb_O.cjs");
 const require_tokenUsageUtils = require("./tokenUsageUtils-_B-P8IAi.cjs");
 const require_inputVariables = require("./inputVariables-Dq9W-Z3a.cjs");
-const require_util = require("./util-CN8om2rz.cjs");
-const require_promptfoo = require("./promptfoo-b-baRMj-.cjs");
-const require_shared = require("./shared-WkgnDkcg.cjs");
-const require_extractor = require("./extractor-BdxEtt3J.cjs");
-const require_indirectWebPwn = require("./indirectWebPwn-BJ22AbQa.cjs");
-const require_evaluatorHelpers = require("./evaluatorHelpers-CRqTvSux.cjs");
-const require_strategies = require("./strategies-7CS3Alao.cjs");
-const require_tracingOptions = require("./tracingOptions-DrbSFaKy.cjs");
+const require_util = require("./util-DNtUsus_.cjs");
+const require_promptfoo = require("./promptfoo-ClChwT74.cjs");
+const require_shared = require("./shared-aHWko3P1.cjs");
+const require_extractor = require("./extractor-bV_NOoaz.cjs");
+const require_indirectWebPwn = require("./indirectWebPwn-D7NA9Nsv.cjs");
+const require_evaluatorHelpers = require("./evaluatorHelpers-D06I9WFL.cjs");
+const require_strategies = require("./strategies-CCcnEbFO.cjs");
+const require_tracingOptions = require("./tracingOptions-DFBoE8O0.cjs");
 const require_runtimeTransform = require("./runtimeTransform-ON3YYILw.cjs");
 //#region src/redteam/providers/hydra/index.ts
 const DEFAULT_MAX_TURNS = 10;
@@ -111,11 +111,12 @@ var HydraProvider = class {
 		let storedGraderResult = void 0;
 		let lastTargetResponse = void 0;
 		let backtrackCount = 0;
+		let agentFailureError;
 		const redteamHistory = [];
 		let lastTransformResult;
 		let lastTransformDisplayVars;
 		let lastFinalAttackPrompt;
-		const { getGraderById } = await Promise.resolve().then(() => require("./graders-ClrU2fnd.cjs")).then((n) => n.graders_exports);
+		const { getGraderById } = await Promise.resolve().then(() => require("./graders-CBQ2s6gz.cjs")).then((n) => n.graders_exports);
 		let assertToUse = test?.assert?.find((a) => a.type && a.type.includes(test.metadata?.pluginId));
 		if (!assertToUse) assertToUse = test?.assert?.find((a) => a.type);
 		let previousTraceSummary;
@@ -159,6 +160,7 @@ var HydraProvider = class {
 					testRunId,
 					error: agentResp.error
 				});
+				agentFailureError = agentResp.error;
 				continue;
 			}
 			let nextMessage;
@@ -169,18 +171,31 @@ var HydraProvider = class {
 			}
 			if (!nextMessage) {
 				require_logger.logger.info("[Hydra] Missing message from agent", { turn });
+				agentFailureError = "Hydra agent did not return an attack message";
 				continue;
 			}
 			let processedMessage = nextMessage;
 			const extractedPrompt = require_util.extractPromptFromTags(nextMessage);
 			if (extractedPrompt) processedMessage = extractedPrompt;
+			if (this.config.inputs && require_remoteGeneration.shouldGenerateRemote()) require_promptfoo.assertRemoteMaterializationHandled(agentResp, "Hydra multi-input generation");
 			const currentInputVars = require_util.extractInputVarsFromPrompt(processedMessage, this.config.inputs);
-			const currentRenderInputVars = (currentInputVars && this.config.inputs ? await require_inputVariables.materializeInputVariablesWithMetadata(currentInputVars, this.config.inputs, {
+			let materializedInputVars;
+			if (this.config.inputs && require_remoteGeneration.shouldGenerateRemote() && !currentInputVars && !agentResp.materializedVars) {
+				require_logger.logger.warn("[Hydra] Remote multi-input generation returned an invalid prompt format", {
+					turn,
+					messagePreview: processedMessage.slice(0, 200)
+				});
+				agentFailureError = "Hydra remote multi-input generation returned an invalid prompt format";
+				continue;
+			}
+			if ((currentInputVars || agentResp.materializedVars) && this.config.inputs) if (require_remoteGeneration.shouldGenerateRemote()) materializedInputVars = require_promptfoo.buildRemoteMaterializedInputVariables(agentResp, currentInputVars ?? {}, this.config.inputs);
+			else materializedInputVars = await require_inputVariables.materializeInputVariablesWithMetadata(currentInputVars, this.config.inputs, {
 				materializationIndex: turn,
 				pluginId: "hydra",
 				provider: this.agentProvider,
 				purpose: test?.metadata?.purpose
-			}) : void 0)?.vars ?? currentInputVars;
+			});
+			const currentRenderInputVars = materializedInputVars?.vars ?? currentInputVars;
 			this.conversationHistory.push({
 				role: "user",
 				content: processedMessage
@@ -512,13 +527,16 @@ var HydraProvider = class {
 			role: msg.role,
 			content: msg.content
 		}));
+		const targetProbeCount = totalTokenUsage.numRequests ?? 0;
+		const hydraRoundsCompleted = this.conversationHistory.filter((m) => m.role === "user").length;
+		const failClosedError = targetProbeCount === 0 ? agentFailureError || "Hydra did not execute any target probes" : void 0;
 		return {
 			output: lastTargetResponse?.output || "",
-			...lastTargetResponse?.error ? { error: lastTargetResponse.error } : {},
+			...failClosedError ? { error: failClosedError } : lastTargetResponse?.error ? { error: lastTargetResponse.error } : {},
 			metadata: {
 				sessionId: this.sessionId || require_util.getSessionId(lastTargetResponse, context),
 				messages,
-				hydraRoundsCompleted: this.conversationHistory.filter((m) => m.role === "user").length,
+				hydraRoundsCompleted,
 				hydraBacktrackCount: backtrackCount,
 				hydraResult: vulnerabilityAchieved,
 				stopReason,
@@ -539,4 +557,4 @@ var HydraProvider = class {
 //#endregion
 exports.HydraProvider = HydraProvider;
 
-//# sourceMappingURL=hydra-DrJttnvw.cjs.map
+//# sourceMappingURL=hydra-RO9jBks7.cjs.map

package/dist/src/{image-B_fPIwdg.js → image-BBmZdKO3.js}

@@ -1,6 +1,6 @@
 import { T as getEnvString, a as logger } from "./logger-Ct2S6Yx-.js";
-import { p as REQUEST_TIMEOUT_MS } from "./fetch-It34O8Ur.js";
-import { r as fetchWithCache } from "./cache-DIXbtkNO.js";
+import { p as REQUEST_TIMEOUT_MS } from "./fetch-Doks14zQ.js";
+import { r as fetchWithCache } from "./cache-DdriHsNX.js";
 //#region src/providers/hyperbolic/image.ts
 const HYPERBOLIC_IMAGE_MODELS = [
 	{
@@ -160,4 +160,4 @@ function createHyperbolicImageProvider(providerPath, options = {}) {
 //#endregion
 export { createHyperbolicImageProvider };
 
-//# sourceMappingURL=image-B_fPIwdg.js.map
+//# sourceMappingURL=image-BBmZdKO3.js.map

package/dist/src/{image-QzmydkiG.js → image-B_PFG7IG.js}

@@ -1,6 +1,6 @@
 import { T as getEnvString, a as logger } from "./logger-KD8JjCRJ.js";
-import { v as REQUEST_TIMEOUT_MS } from "./fetch-BufrQtvR.js";
-import { r as fetchWithCache } from "./cache-roFAE0cI.js";
+import { v as REQUEST_TIMEOUT_MS } from "./fetch-BSSAcMxf.js";
+import { r as fetchWithCache } from "./cache-CrioYnaa.js";
 //#region src/providers/hyperbolic/image.ts
 const HYPERBOLIC_IMAGE_MODELS = [
 	{
@@ -160,4 +160,4 @@ function createHyperbolicImageProvider(providerPath, options = {}) {
 //#endregion
 export { createHyperbolicImageProvider };
 
-//# sourceMappingURL=image-QzmydkiG.js.map
+//# sourceMappingURL=image-B_PFG7IG.js.map

package/dist/src/{image-BvUAW344.js → image-BtODOZjh.js}

@@ -1,8 +1,8 @@
 import { a as logger } from "./logger-Ct2S6Yx-.js";
-import { p as REQUEST_TIMEOUT_MS } from "./fetch-It34O8Ur.js";
-import { r as fetchWithCache } from "./cache-DIXbtkNO.js";
+import { p as REQUEST_TIMEOUT_MS } from "./fetch-Doks14zQ.js";
+import { r as fetchWithCache } from "./cache-DdriHsNX.js";
 import { t as OpenAiGenericProvider } from "./openai-zgwBb4Ff.js";
-import { s as formatOpenAiError } from "./util-LKTmNsMQ.js";
+import { s as formatOpenAiError } from "./util-CYev3d-r.js";
 import { t as ellipsize } from "./text-DwYK5EBn.js";
 //#region src/providers/openai/image.ts
 const DALLE2_VALID_SIZES = [
@@ -439,4 +439,4 @@ var OpenAiImageProvider = class extends OpenAiGenericProvider {
 //#endregion
 export { formatOutput as i, buildStructuredImageOutputs as n, callOpenAiImageApi as r, OpenAiImageProvider as t };
 
-//# sourceMappingURL=image-BvUAW344.js.map
+//# sourceMappingURL=image-BtODOZjh.js.map

package/dist/src/{image-Cvjwx1uY.js → image-CYuNJIqd.js}

@@ -1,8 +1,8 @@
 import { a as logger } from "./logger-KD8JjCRJ.js";
-import { v as REQUEST_TIMEOUT_MS } from "./fetch-BufrQtvR.js";
-import { r as fetchWithCache } from "./cache-roFAE0cI.js";
+import { v as REQUEST_TIMEOUT_MS } from "./fetch-BSSAcMxf.js";
+import { r as fetchWithCache } from "./cache-CrioYnaa.js";
 import { t as OpenAiGenericProvider } from "./openai-BMHD2Huo.js";
-import { s as formatOpenAiError } from "./util-efByNxcr.js";
+import { s as formatOpenAiError } from "./util-o2Qg5rZv.js";
 import { t as ellipsize } from "./text-CZr46tp_.js";
 //#region src/providers/openai/image.ts
 const DALLE2_VALID_SIZES = [
@@ -439,4 +439,4 @@ var OpenAiImageProvider = class extends OpenAiGenericProvider {
 //#endregion
 export { formatOutput as i, buildStructuredImageOutputs as n, callOpenAiImageApi as r, OpenAiImageProvider as t };
 
-//# sourceMappingURL=image-Cvjwx1uY.js.map
+//# sourceMappingURL=image-CYuNJIqd.js.map

package/dist/src/{image-DfVCGPbI.cjs → image-E00WFZkq.cjs}

@@ -1,6 +1,6 @@
 const require_logger = require("./logger-cfNpzI4o.cjs");
-const require_fetch = require("./fetch-Dw4XZHjj.cjs");
-const require_cache = require("./cache-CPGUA4Yl.cjs");
+const require_fetch = require("./fetch-Cfm4EuOB.cjs");
+const require_cache = require("./cache-h5MWOBZI.cjs");
 //#region src/providers/hyperbolic/image.ts
 const HYPERBOLIC_IMAGE_MODELS = [
 	{
@@ -160,4 +160,4 @@ function createHyperbolicImageProvider(providerPath, options = {}) {
 //#endregion
 exports.createHyperbolicImageProvider = createHyperbolicImageProvider;
 
-//# sourceMappingURL=image-DfVCGPbI.cjs.map
+//# sourceMappingURL=image-E00WFZkq.cjs.map

package/dist/src/{image-X0oY4350.cjs → image-HK2Cfqb1.cjs}

@@ -1,8 +1,8 @@
 const require_logger = require("./logger-cfNpzI4o.cjs");
-const require_fetch = require("./fetch-Dw4XZHjj.cjs");
-const require_cache = require("./cache-CPGUA4Yl.cjs");
+const require_fetch = require("./fetch-Cfm4EuOB.cjs");
+const require_cache = require("./cache-h5MWOBZI.cjs");
 const require_openai = require("./openai-C3uXv8wS.cjs");
-const require_util = require("./util-BSIuSLVK.cjs");
+const require_util = require("./util-NYQvo1C7.cjs");
 const require_text = require("./text-nywWsRBM.cjs");
 //#region src/providers/openai/image.ts
 const DALLE2_VALID_SIZES = [
@@ -462,4 +462,4 @@ Object.defineProperty(exports, "formatOutput", {
 	}
 });
 
-//# sourceMappingURL=image-X0oY4350.cjs.map
+//# sourceMappingURL=image-HK2Cfqb1.cjs.map

package/dist/src/{image-BN-hjLL9.js → image-LGj8dTcr.js}

@@ -1,7 +1,7 @@
 #!/usr/bin/env node
 import { O as getEnvString, s as logger } from "./logger-BbY6ypFL.js";
-import { _ as REQUEST_TIMEOUT_MS } from "./fetch-DXUnXkVU.js";
-import { a as fetchWithCache } from "./cache-DpPWrkTE.js";
+import { _ as REQUEST_TIMEOUT_MS } from "./fetch-Cpf1U1nO.js";
+import { a as fetchWithCache } from "./cache-BR77mdIR.js";
 //#region src/providers/hyperbolic/image.ts
 const HYPERBOLIC_IMAGE_MODELS = [
 	{
@@ -161,4 +161,4 @@ function createHyperbolicImageProvider(providerPath, options = {}) {
 //#endregion
 export { createHyperbolicImageProvider };
 
-//# sourceMappingURL=image-BN-hjLL9.js.map
+//# sourceMappingURL=image-LGj8dTcr.js.map

package/dist/src/{image-B4oBtu6J.js → image-YMKejC0r.js}

@@ -1,9 +1,9 @@
 #!/usr/bin/env node
 import { s as logger } from "./logger-BbY6ypFL.js";
-import { _ as REQUEST_TIMEOUT_MS } from "./fetch-DXUnXkVU.js";
-import { a as fetchWithCache } from "./cache-DpPWrkTE.js";
+import { _ as REQUEST_TIMEOUT_MS } from "./fetch-Cpf1U1nO.js";
+import { a as fetchWithCache } from "./cache-BR77mdIR.js";
 import { t as OpenAiGenericProvider } from "./openai-CJrsh9n4.js";
-import { s as formatOpenAiError } from "./util-5WnCSb0h.js";
+import { s as formatOpenAiError } from "./util-DeY58psG.js";
 import { t as ellipsize } from "./text-Db-Wt2u2.js";
 //#region src/providers/openai/image.ts
 const DALLE2_VALID_SIZES = [
@@ -440,4 +440,4 @@ var OpenAiImageProvider = class extends OpenAiGenericProvider {
 //#endregion
 export { formatOutput as i, buildStructuredImageOutputs as n, callOpenAiImageApi as r, OpenAiImageProvider as t };
 
-//# sourceMappingURL=image-B4oBtu6J.js.map
+//# sourceMappingURL=image-YMKejC0r.js.map