promptfoo 0.121.7 → 0.121.9

package/dist/src/{hydra-DrJttnvw.cjs → hydra-Hdj65j0k.cjs}

@@ -1,19 +1,19 @@
-const require_logger = require("./logger-cfNpzI4o.cjs");
+const require_logger = require("./logger-Cm0Nb8YX.cjs");
 const require_invariant = require("./invariant-QtnLD03y.cjs");
-const require_fetch = require("./fetch-Dw4XZHjj.cjs");
-const require_accounts = require("./accounts-CmWzeD2d.cjs");
-const require_remoteGeneration = require("./remoteGeneration-DS9N3pgB.cjs");
+const require_fetch = require("./fetch-B2HvaVUc.cjs");
+const require_accounts = require("./accounts-DAcQXPwE.cjs");
+const require_remoteGeneration = require("./remoteGeneration-CD3c_rzc.cjs");
 const require_tokenUsageUtils = require("./tokenUsageUtils-_B-P8IAi.cjs");
-const require_inputVariables = require("./inputVariables-Dq9W-Z3a.cjs");
-const require_util = require("./util-CN8om2rz.cjs");
-const require_promptfoo = require("./promptfoo-b-baRMj-.cjs");
-const require_shared = require("./shared-WkgnDkcg.cjs");
-const require_extractor = require("./extractor-BdxEtt3J.cjs");
-const require_indirectWebPwn = require("./indirectWebPwn-BJ22AbQa.cjs");
-const require_evaluatorHelpers = require("./evaluatorHelpers-CRqTvSux.cjs");
-const require_strategies = require("./strategies-7CS3Alao.cjs");
-const require_tracingOptions = require("./tracingOptions-DrbSFaKy.cjs");
-const require_runtimeTransform = require("./runtimeTransform-ON3YYILw.cjs");
+const require_inputVariables = require("./inputVariables-D5K2vTp4.cjs");
+const require_util = require("./util-3BRss2KA.cjs");
+const require_promptfoo = require("./promptfoo-DAXBmrE6.cjs");
+const require_shared = require("./shared-DlmtkptR.cjs");
+const require_extractor = require("./extractor-nyUwvU2U.cjs");
+const require_indirectWebPwn = require("./indirectWebPwn-04ziKbly.cjs");
+const require_evaluatorHelpers = require("./evaluatorHelpers-2gpZ9Q_f.cjs");
+const require_strategies = require("./strategies-BD7US9WB.cjs");
+const require_tracingOptions = require("./tracingOptions-eqmq5sA2.cjs");
+const require_runtimeTransform = require("./runtimeTransform-DJEknTZQ.cjs");
 //#region src/redteam/providers/hydra/index.ts
 const DEFAULT_MAX_TURNS = 10;
 const DEFAULT_MAX_BACKTRACKS = 10;
@@ -111,11 +111,12 @@ var HydraProvider = class {
 		let storedGraderResult = void 0;
 		let lastTargetResponse = void 0;
 		let backtrackCount = 0;
+		let agentFailureError;
 		const redteamHistory = [];
 		let lastTransformResult;
 		let lastTransformDisplayVars;
 		let lastFinalAttackPrompt;
-		const { getGraderById } = await Promise.resolve().then(() => require("./graders-ClrU2fnd.cjs")).then((n) => n.graders_exports);
+		const { getGraderById } = await Promise.resolve().then(() => require("./graders-CQv1GT5Q.cjs")).then((n) => n.graders_exports);
 		let assertToUse = test?.assert?.find((a) => a.type && a.type.includes(test.metadata?.pluginId));
 		if (!assertToUse) assertToUse = test?.assert?.find((a) => a.type);
 		let previousTraceSummary;
@@ -159,6 +160,7 @@ var HydraProvider = class {
 					testRunId,
 					error: agentResp.error
 				});
+				agentFailureError = agentResp.error;
 				continue;
 			}
 			let nextMessage;
@@ -169,18 +171,31 @@ var HydraProvider = class {
 			}
 			if (!nextMessage) {
 				require_logger.logger.info("[Hydra] Missing message from agent", { turn });
+				agentFailureError = "Hydra agent did not return an attack message";
 				continue;
 			}
 			let processedMessage = nextMessage;
 			const extractedPrompt = require_util.extractPromptFromTags(nextMessage);
 			if (extractedPrompt) processedMessage = extractedPrompt;
+			if (this.config.inputs && require_remoteGeneration.shouldGenerateRemote()) require_promptfoo.assertRemoteMaterializationHandled(agentResp, "Hydra multi-input generation");
 			const currentInputVars = require_util.extractInputVarsFromPrompt(processedMessage, this.config.inputs);
-			const currentRenderInputVars = (currentInputVars && this.config.inputs ? await require_inputVariables.materializeInputVariablesWithMetadata(currentInputVars, this.config.inputs, {
+			let materializedInputVars;
+			if (this.config.inputs && require_remoteGeneration.shouldGenerateRemote() && !currentInputVars && !agentResp.materializedVars) {
+				require_logger.logger.warn("[Hydra] Remote multi-input generation returned an invalid prompt format", {
+					turn,
+					messagePreview: processedMessage.slice(0, 200)
+				});
+				agentFailureError = "Hydra remote multi-input generation returned an invalid prompt format";
+				continue;
+			}
+			if ((currentInputVars || agentResp.materializedVars) && this.config.inputs) if (require_remoteGeneration.shouldGenerateRemote()) materializedInputVars = require_promptfoo.buildRemoteMaterializedInputVariables(agentResp, currentInputVars ?? {}, this.config.inputs);
+			else materializedInputVars = await require_inputVariables.materializeInputVariablesWithMetadata(currentInputVars, this.config.inputs, {
 				materializationIndex: turn,
 				pluginId: "hydra",
 				provider: this.agentProvider,
 				purpose: test?.metadata?.purpose
-			}) : void 0)?.vars ?? currentInputVars;
+			});
+			const currentRenderInputVars = materializedInputVars?.vars ?? currentInputVars;
 			this.conversationHistory.push({
 				role: "user",
 				content: processedMessage
@@ -512,13 +527,16 @@ var HydraProvider = class {
 			role: msg.role,
 			content: msg.content
 		}));
+		const targetProbeCount = totalTokenUsage.numRequests ?? 0;
+		const hydraRoundsCompleted = this.conversationHistory.filter((m) => m.role === "user").length;
+		const failClosedError = targetProbeCount === 0 ? agentFailureError || "Hydra did not execute any target probes" : void 0;
 		return {
 			output: lastTargetResponse?.output || "",
-			...lastTargetResponse?.error ? { error: lastTargetResponse.error } : {},
+			...failClosedError ? { error: failClosedError } : lastTargetResponse?.error ? { error: lastTargetResponse.error } : {},
 			metadata: {
 				sessionId: this.sessionId || require_util.getSessionId(lastTargetResponse, context),
 				messages,
-				hydraRoundsCompleted: this.conversationHistory.filter((m) => m.role === "user").length,
+				hydraRoundsCompleted,
 				hydraBacktrackCount: backtrackCount,
 				hydraResult: vulnerabilityAchieved,
 				stopReason,
@@ -539,4 +557,4 @@ var HydraProvider = class {
 //#endregion
 exports.HydraProvider = HydraProvider;
 
-//# sourceMappingURL=hydra-DrJttnvw.cjs.map
+//# sourceMappingURL=hydra-Hdj65j0k.cjs.map

package/dist/src/{image-B_fPIwdg.js → image-6u3WZ_7H.js}

@@ -1,6 +1,6 @@
-import { T as getEnvString, a as logger } from "./logger-Ct2S6Yx-.js";
-import { p as REQUEST_TIMEOUT_MS } from "./fetch-It34O8Ur.js";
-import { r as fetchWithCache } from "./cache-DIXbtkNO.js";
+import { T as getEnvString, a as logger } from "./logger-Cs0I2v_y.js";
+import { m as getRequestTimeoutMs } from "./fetch-CHJu2KON.js";
+import { r as fetchWithCache } from "./cache-CT5D7d_A.js";
 //#region src/providers/hyperbolic/image.ts
 const HYPERBOLIC_IMAGE_MODELS = [
 	{
@@ -128,7 +128,7 @@ var HyperbolicImageProvider = class {
 				method: "POST",
 				headers,
 				body: JSON.stringify(body)
-			}, REQUEST_TIMEOUT_MS));
+			}, getRequestTimeoutMs()));
 			if (status < 200 || status >= 300) return { error: `API error: ${status} ${statusText}\n${typeof data === "string" ? data : JSON.stringify(data)}` };
 		} catch (err) {
 			logger.error(`API call error: ${String(err)}`);
@@ -160,4 +160,4 @@ function createHyperbolicImageProvider(providerPath, options = {}) {
 //#endregion
 export { createHyperbolicImageProvider };
 
-//# sourceMappingURL=image-B_fPIwdg.js.map
+//# sourceMappingURL=image-6u3WZ_7H.js.map

package/dist/src/{image-BN-hjLL9.js → image-B6hCz0z3.js}

@@ -1,7 +1,7 @@
 #!/usr/bin/env node
-import { O as getEnvString, s as logger } from "./logger-BbY6ypFL.js";
-import { _ as REQUEST_TIMEOUT_MS } from "./fetch-DXUnXkVU.js";
-import { a as fetchWithCache } from "./cache-DpPWrkTE.js";
+import { O as getEnvString, s as logger } from "./logger-h-eiS4iv.js";
+import { v as getRequestTimeoutMs } from "./fetch-C59QM4cf.js";
+import { a as fetchWithCache } from "./cache-BuudGL4I.js";
 //#region src/providers/hyperbolic/image.ts
 const HYPERBOLIC_IMAGE_MODELS = [
 	{
@@ -129,7 +129,7 @@ var HyperbolicImageProvider = class {
 				method: "POST",
 				headers,
 				body: JSON.stringify(body)
-			}, REQUEST_TIMEOUT_MS));
+			}, getRequestTimeoutMs()));
 			if (status < 200 || status >= 300) return { error: `API error: ${status} ${statusText}\n${typeof data === "string" ? data : JSON.stringify(data)}` };
 		} catch (err) {
 			logger.error(`API call error: ${String(err)}`);
@@ -161,4 +161,4 @@ function createHyperbolicImageProvider(providerPath, options = {}) {
 //#endregion
 export { createHyperbolicImageProvider };
 
-//# sourceMappingURL=image-BN-hjLL9.js.map
+//# sourceMappingURL=image-B6hCz0z3.js.map

package/dist/src/{image-BvUAW344.js → image-CgmaPlV_.js}

@@ -1,8 +1,8 @@
-import { a as logger } from "./logger-Ct2S6Yx-.js";
-import { p as REQUEST_TIMEOUT_MS } from "./fetch-It34O8Ur.js";
-import { r as fetchWithCache } from "./cache-DIXbtkNO.js";
-import { t as OpenAiGenericProvider } from "./openai-zgwBb4Ff.js";
-import { s as formatOpenAiError } from "./util-LKTmNsMQ.js";
+import { a as logger } from "./logger-Cs0I2v_y.js";
+import { m as getRequestTimeoutMs } from "./fetch-CHJu2KON.js";
+import { r as fetchWithCache } from "./cache-CT5D7d_A.js";
+import { t as OpenAiGenericProvider } from "./openai-Bfbg6L4c.js";
+import { s as formatOpenAiError } from "./util-B2Nt2Oy7.js";
 import { t as ellipsize } from "./text-DwYK5EBn.js";
 //#region src/providers/openai/image.ts
 const DALLE2_VALID_SIZES = [
@@ -426,7 +426,7 @@ var OpenAiImageProvider = class extends OpenAiGenericProvider {
 		let cached = false;
 		let latencyMs;
 		try {
-			({data, cached, status, statusText, latencyMs} = await callOpenAiImageApi(`${this.getApiUrl()}${endpoint}`, body, headers, REQUEST_TIMEOUT_MS));
+			({data, cached, status, statusText, latencyMs} = await callOpenAiImageApi(`${this.getApiUrl()}${endpoint}`, body, headers, getRequestTimeoutMs()));
 			if (status < 200 || status >= 300) return { error: `API error: ${status} ${statusText}\n${typeof data === "string" ? data : JSON.stringify(data)}` };
 		} catch (err) {
 			logger.error(`API call error: ${String(err)}`);
@@ -439,4 +439,4 @@ var OpenAiImageProvider = class extends OpenAiGenericProvider {
 //#endregion
 export { formatOutput as i, buildStructuredImageOutputs as n, callOpenAiImageApi as r, OpenAiImageProvider as t };
 
-//# sourceMappingURL=image-BvUAW344.js.map
+//# sourceMappingURL=image-CgmaPlV_.js.map

package/dist/src/{image-QzmydkiG.js → image-DXbwwL-r.js}

@@ -1,6 +1,6 @@
-import { T as getEnvString, a as logger } from "./logger-KD8JjCRJ.js";
-import { v as REQUEST_TIMEOUT_MS } from "./fetch-BufrQtvR.js";
-import { r as fetchWithCache } from "./cache-roFAE0cI.js";
+import { E as getEnvString, a as logger } from "./logger-BjYLHvGm.js";
+import { y as getRequestTimeoutMs } from "./fetch-CBe8pU8Q.js";
+import { r as fetchWithCache } from "./cache-x9llZxbx.js";
 //#region src/providers/hyperbolic/image.ts
 const HYPERBOLIC_IMAGE_MODELS = [
 	{
@@ -128,7 +128,7 @@ var HyperbolicImageProvider = class {
 				method: "POST",
 				headers,
 				body: JSON.stringify(body)
-			}, REQUEST_TIMEOUT_MS));
+			}, getRequestTimeoutMs()));
 			if (status < 200 || status >= 300) return { error: `API error: ${status} ${statusText}\n${typeof data === "string" ? data : JSON.stringify(data)}` };
 		} catch (err) {
 			logger.error(`API call error: ${String(err)}`);
@@ -160,4 +160,4 @@ function createHyperbolicImageProvider(providerPath, options = {}) {
 //#endregion
 export { createHyperbolicImageProvider };
 
-//# sourceMappingURL=image-QzmydkiG.js.map
+//# sourceMappingURL=image-DXbwwL-r.js.map

package/dist/src/{image-X0oY4350.cjs → image-Daa_X_YB.cjs}

@@ -1,8 +1,8 @@
-const require_logger = require("./logger-cfNpzI4o.cjs");
-const require_fetch = require("./fetch-Dw4XZHjj.cjs");
-const require_cache = require("./cache-CPGUA4Yl.cjs");
-const require_openai = require("./openai-C3uXv8wS.cjs");
-const require_util = require("./util-BSIuSLVK.cjs");
+const require_logger = require("./logger-Cm0Nb8YX.cjs");
+const require_fetch = require("./fetch-B2HvaVUc.cjs");
+const require_cache = require("./cache-WuIsgzMA.cjs");
+const require_openai = require("./openai-CCaz18I4.cjs");
+const require_util = require("./util-Cs372i1_.cjs");
 const require_text = require("./text-nywWsRBM.cjs");
 //#region src/providers/openai/image.ts
 const DALLE2_VALID_SIZES = [
@@ -426,7 +426,7 @@ var OpenAiImageProvider = class extends require_openai.OpenAiGenericProvider {
 		let cached = false;
 		let latencyMs;
 		try {
-			({data, cached, status, statusText, latencyMs} = await callOpenAiImageApi(`${this.getApiUrl()}${endpoint}`, body, headers, require_fetch.REQUEST_TIMEOUT_MS));
+			({data, cached, status, statusText, latencyMs} = await callOpenAiImageApi(`${this.getApiUrl()}${endpoint}`, body, headers, require_fetch.getRequestTimeoutMs()));
 			if (status < 200 || status >= 300) return { error: `API error: ${status} ${statusText}\n${typeof data === "string" ? data : JSON.stringify(data)}` };
 		} catch (err) {
 			require_logger.logger.error(`API call error: ${String(err)}`);
@@ -462,4 +462,4 @@ Object.defineProperty(exports, "formatOutput", {
 	}
 });
 
-//# sourceMappingURL=image-X0oY4350.cjs.map
+//# sourceMappingURL=image-Daa_X_YB.cjs.map

package/dist/src/{image-Cvjwx1uY.js → image-DmC5qwQx.js}

@@ -1,8 +1,8 @@
-import { a as logger } from "./logger-KD8JjCRJ.js";
-import { v as REQUEST_TIMEOUT_MS } from "./fetch-BufrQtvR.js";
-import { r as fetchWithCache } from "./cache-roFAE0cI.js";
-import { t as OpenAiGenericProvider } from "./openai-BMHD2Huo.js";
-import { s as formatOpenAiError } from "./util-efByNxcr.js";
+import { a as logger } from "./logger-BjYLHvGm.js";
+import { y as getRequestTimeoutMs } from "./fetch-CBe8pU8Q.js";
+import { r as fetchWithCache } from "./cache-x9llZxbx.js";
+import { t as OpenAiGenericProvider } from "./openai-uRacy72N.js";
+import { s as formatOpenAiError } from "./util-C6d3eV0A.js";
 import { t as ellipsize } from "./text-CZr46tp_.js";
 //#region src/providers/openai/image.ts
 const DALLE2_VALID_SIZES = [
@@ -426,7 +426,7 @@ var OpenAiImageProvider = class extends OpenAiGenericProvider {
 		let cached = false;
 		let latencyMs;
 		try {
-			({data, cached, status, statusText, latencyMs} = await callOpenAiImageApi(`${this.getApiUrl()}${endpoint}`, body, headers, REQUEST_TIMEOUT_MS));
+			({data, cached, status, statusText, latencyMs} = await callOpenAiImageApi(`${this.getApiUrl()}${endpoint}`, body, headers, getRequestTimeoutMs()));
 			if (status < 200 || status >= 300) return { error: `API error: ${status} ${statusText}\n${typeof data === "string" ? data : JSON.stringify(data)}` };
 		} catch (err) {
 			logger.error(`API call error: ${String(err)}`);
@@ -439,4 +439,4 @@ var OpenAiImageProvider = class extends OpenAiGenericProvider {
 //#endregion
 export { formatOutput as i, buildStructuredImageOutputs as n, callOpenAiImageApi as r, OpenAiImageProvider as t };
 
-//# sourceMappingURL=image-Cvjwx1uY.js.map
+//# sourceMappingURL=image-DmC5qwQx.js.map

package/dist/src/{image-DfVCGPbI.cjs → image-TY6FsHmn.cjs}

@@ -1,6 +1,6 @@
-const require_logger = require("./logger-cfNpzI4o.cjs");
-const require_fetch = require("./fetch-Dw4XZHjj.cjs");
-const require_cache = require("./cache-CPGUA4Yl.cjs");
+const require_logger = require("./logger-Cm0Nb8YX.cjs");
+const require_fetch = require("./fetch-B2HvaVUc.cjs");
+const require_cache = require("./cache-WuIsgzMA.cjs");
 //#region src/providers/hyperbolic/image.ts
 const HYPERBOLIC_IMAGE_MODELS = [
 	{
@@ -128,7 +128,7 @@ var HyperbolicImageProvider = class {
 				method: "POST",
 				headers,
 				body: JSON.stringify(body)
-			}, require_fetch.REQUEST_TIMEOUT_MS));
+			}, require_fetch.getRequestTimeoutMs()));
 			if (status < 200 || status >= 300) return { error: `API error: ${status} ${statusText}\n${typeof data === "string" ? data : JSON.stringify(data)}` };
 		} catch (err) {
 			require_logger.logger.error(`API call error: ${String(err)}`);
@@ -160,4 +160,4 @@ function createHyperbolicImageProvider(providerPath, options = {}) {
 //#endregion
 exports.createHyperbolicImageProvider = createHyperbolicImageProvider;
 
-//# sourceMappingURL=image-DfVCGPbI.cjs.map
+//# sourceMappingURL=image-TY6FsHmn.cjs.map

package/dist/src/{image-B4oBtu6J.js → image-V0-pYiBI.js}

@@ -1,9 +1,9 @@
 #!/usr/bin/env node
-import { s as logger } from "./logger-BbY6ypFL.js";
-import { _ as REQUEST_TIMEOUT_MS } from "./fetch-DXUnXkVU.js";
-import { a as fetchWithCache } from "./cache-DpPWrkTE.js";
-import { t as OpenAiGenericProvider } from "./openai-CJrsh9n4.js";
-import { s as formatOpenAiError } from "./util-5WnCSb0h.js";
+import { s as logger } from "./logger-h-eiS4iv.js";
+import { v as getRequestTimeoutMs } from "./fetch-C59QM4cf.js";
+import { a as fetchWithCache } from "./cache-BuudGL4I.js";
+import { t as OpenAiGenericProvider } from "./openai-CJOpg_dA.js";
+import { s as formatOpenAiError } from "./util-BLSQ_7Ky.js";
 import { t as ellipsize } from "./text-Db-Wt2u2.js";
 //#region src/providers/openai/image.ts
 const DALLE2_VALID_SIZES = [
@@ -427,7 +427,7 @@ var OpenAiImageProvider = class extends OpenAiGenericProvider {
 		let cached = false;
 		let latencyMs;
 		try {
-			({data, cached, status, statusText, latencyMs} = await callOpenAiImageApi(`${this.getApiUrl()}${endpoint}`, body, headers, REQUEST_TIMEOUT_MS));
+			({data, cached, status, statusText, latencyMs} = await callOpenAiImageApi(`${this.getApiUrl()}${endpoint}`, body, headers, getRequestTimeoutMs()));
 			if (status < 200 || status >= 300) return { error: `API error: ${status} ${statusText}\n${typeof data === "string" ? data : JSON.stringify(data)}` };
 		} catch (err) {
 			logger.error(`API call error: ${String(err)}`);
@@ -440,4 +440,4 @@ var OpenAiImageProvider = class extends OpenAiGenericProvider {
 //#endregion
 export { formatOutput as i, buildStructuredImageOutputs as n, callOpenAiImageApi as r, OpenAiImageProvider as t };
 
-//# sourceMappingURL=image-B4oBtu6J.js.map
+//# sourceMappingURL=image-V0-pYiBI.js.map