promptfoo 0.121.4 → 0.121.7

package/dist/src/goat-una6pZGP.cjs

@@ -0,0 +1,469 @@
+const require_rolldown_runtime = require("./rolldown-runtime-D_mwlA32.cjs");
+const require_logger = require("./logger-cfNpzI4o.cjs");
+const require_invariant = require("./invariant-QtnLD03y.cjs");
+const require_fetch = require("./fetch-Dw4XZHjj.cjs");
+const require_version = require("./version-BWCSaByA.cjs");
+const require_accounts = require("./accounts-CmWzeD2d.cjs");
+const require_render = require("./render-BNTrbmBw.cjs");
+const require_remoteGeneration = require("./remoteGeneration-DS9N3pgB.cjs");
+const require_tokenUsageUtils = require("./tokenUsageUtils-_B-P8IAi.cjs");
+const require_inputVariables = require("./inputVariables-Dq9W-Z3a.cjs");
+const require_util = require("./util-CN8om2rz.cjs");
+const require_shared = require("./shared-WkgnDkcg.cjs");
+const require_promptLength = require("./promptLength-BbBbDHNj.cjs");
+const require_indirectWebPwn = require("./indirectWebPwn-BJ22AbQa.cjs");
+const require_evaluatorHelpers = require("./evaluatorHelpers-CRqTvSux.cjs");
+const require_strategies = require("./strategies-7CS3Alao.cjs");
+const require_tracingOptions = require("./tracingOptions-DrbSFaKy.cjs");
+const require_runtimeTransform = require("./runtimeTransform-ON3YYILw.cjs");
+const require_prompts = require("./prompts-hNvWBD3z.cjs");
+let chalk = require("chalk");
+chalk = require_rolldown_runtime.__toESM(chalk, 1);
+let dedent = require("dedent");
+dedent = require_rolldown_runtime.__toESM(dedent, 1);
+//#region src/redteam/providers/goat.ts
+var GoatProvider = class {
+	config;
+	nunjucks;
+	perTurnLayers;
+	successfulAttacks = [];
+	id() {
+		return "promptfoo:redteam:goat";
+	}
+	constructor(options = {}) {
+		if (require_remoteGeneration.neverGenerateRemote()) throw new Error(`GOAT strategy requires remote grading to be enabled`);
+		require_invariant.invariant(typeof options.injectVar === "string", "Expected injectVar to be set");
+		let maxTurns = options.maxTurns ?? 5;
+		if (!require_accounts.isLoggedIntoCloud()) maxTurns = Math.min(maxTurns, 10);
+		this.config = {
+			maxTurns,
+			...options.maxCharsPerMessage ? { maxCharsPerMessage: options.maxCharsPerMessage } : {},
+			injectVar: options.injectVar,
+			stateful: options.stateful ?? false,
+			excludeTargetOutputFromAgenticAttackGeneration: options.excludeTargetOutputFromAgenticAttackGeneration ?? false,
+			continueAfterSuccess: options.continueAfterSuccess ?? false,
+			tracing: options.tracing,
+			_perTurnLayers: options._perTurnLayers,
+			inputs: options.inputs
+		};
+		this.perTurnLayers = options._perTurnLayers ?? [];
+		this.nunjucks = require_render.getNunjucksEngine();
+		require_logger.logger.debug("[GOAT] Constructor options", {
+			injectVar: options.injectVar,
+			maxTurns: options.maxTurns,
+			maxCharsPerMessage: options.maxCharsPerMessage,
+			stateful: options.stateful,
+			continueAfterSuccess: options.continueAfterSuccess,
+			perTurnLayers: this.perTurnLayers.map((l) => typeof l === "string" ? l : l.id),
+			inputs: options.inputs
+		});
+	}
+	async callApi(_prompt, context, options) {
+		this.successfulAttacks = [];
+		const tracingOptions = require_tracingOptions.resolveTracingOptions({
+			strategyId: "goat",
+			test: context?.test,
+			config: this.config
+		});
+		const shouldFetchTrace = tracingOptions.enabled && (tracingOptions.includeInAttack || tracingOptions.includeInGrading);
+		const traceSnapshots = [];
+		let response = void 0;
+		require_logger.logger.debug("[GOAT] callApi context", { context });
+		require_invariant.invariant(context?.originalProvider, "Expected originalProvider to be set");
+		require_invariant.invariant(context?.vars, "Expected vars to be set");
+		const targetProvider = context?.originalProvider;
+		require_invariant.invariant(targetProvider, "Expected originalProvider to be set");
+		const maxCharsPerMessage = this.config.maxCharsPerMessage ?? (context?.test?.metadata?.strategyConfig)?.maxCharsPerMessage ?? (context?.test?.metadata?.pluginConfig)?.maxCharsPerMessage;
+		const messages = [];
+		const totalTokenUsage = require_tokenUsageUtils.createEmptyTokenUsage();
+		const redteamHistory = [];
+		let lastTargetResponse = void 0;
+		let lastTransformDisplayVars;
+		let lastFinalAttackPrompt;
+		let assertToUse;
+		let graderPassed;
+		let storedGraderResult;
+		const { getGraderById } = await Promise.resolve().then(() => require("./graders-ClrU2fnd.cjs")).then((n) => n.graders_exports);
+		let test;
+		if (context?.test) {
+			test = context?.test;
+			assertToUse = test?.assert?.find((a) => a.type && a.type.includes(test?.metadata?.pluginId));
+			if (!assertToUse) assertToUse = test?.assert?.find((a) => a.type);
+		}
+		let previousAttackerMessage = "";
+		let previousTargetOutput = "";
+		let previousTraceSummary;
+		const additionalRubric = require_prompts.getGoalRubric(context?.test?.metadata?.goal || context?.vars[this.config.injectVar]);
+		let stopReason = "Max turns reached";
+		for (let turn = 0; turn < this.config.maxTurns; turn++) try {
+			if (turn > 0 && previousTargetOutput) {
+				const unblockingResult = await require_shared.tryUnblocking({
+					messages,
+					lastResponse: previousTargetOutput,
+					goal: context?.test?.metadata?.goal || context?.vars[this.config.injectVar],
+					purpose: context?.test?.metadata?.purpose
+				});
+				if (unblockingResult.success && unblockingResult.unblockingPrompt) {
+					require_logger.logger.debug(`[GOAT] Sending unblocking response: ${unblockingResult.unblockingPrompt}`);
+					messages.push({
+						role: "user",
+						content: unblockingResult.unblockingPrompt
+					});
+					let unblockingTargetPrompt = this.config.stateful ? unblockingResult.unblockingPrompt : JSON.stringify(messages);
+					if (this.perTurnLayers.length > 0) {
+						const transformResult = await require_runtimeTransform.applyRuntimeTransforms(unblockingResult.unblockingPrompt, this.config.injectVar, this.perTurnLayers, require_strategies.Strategies, {
+							evaluationId: context?.evaluationId,
+							testCaseId: context?.test?.metadata?.testCaseId,
+							purpose: context?.test?.metadata?.purpose,
+							goal: context?.test?.metadata?.goal
+						});
+						if (transformResult.error) {
+							require_logger.logger.warn("[GOAT] Transform failed for unblocking prompt", { error: transformResult.error });
+							continue;
+						}
+						unblockingTargetPrompt = transformResult.prompt;
+					}
+					require_promptLength.throwIfTargetPromptExceedsMaxChars(unblockingTargetPrompt, maxCharsPerMessage);
+					const unblockingResponse = await targetProvider.callApi(unblockingTargetPrompt, context, options);
+					if (!unblockingResponse.cached && targetProvider.delay && targetProvider.delay > 0) {
+						require_logger.logger.debug(`Sleeping for ${targetProvider.delay}ms`);
+						await require_fetch.sleep(targetProvider.delay);
+					}
+					require_tokenUsageUtils.accumulateResponseTokenUsage(totalTokenUsage, unblockingResponse);
+					const unblockingOutput = typeof unblockingResponse.output === "string" ? unblockingResponse.output : require_logger.safeJsonStringify(unblockingResponse.output);
+					if (unblockingOutput) messages.push({
+						role: "assistant",
+						content: unblockingOutput
+					});
+					if (unblockingResponse.error) require_logger.logger.error(`[GOAT] Target returned an error: ${unblockingResponse.error}`);
+				}
+			}
+			let body;
+			let failureReason;
+			if (this.config.excludeTargetOutputFromAgenticAttackGeneration && turn > 0) {
+				body = JSON.stringify({
+					goal: context?.test?.metadata?.goal || context?.vars[this.config.injectVar],
+					targetOutput: previousTargetOutput,
+					attackAttempt: previousAttackerMessage,
+					task: "extract-goat-failure",
+					modifiers: context?.test?.metadata?.modifiers,
+					traceSummary: previousTraceSummary
+				});
+				require_logger.logger.debug(`[GOAT] Sending request to ${require_remoteGeneration.getRemoteGenerationUrl()}: ${body}`);
+				response = await require_fetch.fetchWithProxy(require_remoteGeneration.getRemoteGenerationUrl(), {
+					body,
+					headers: { "Content-Type": "application/json" },
+					method: "POST"
+				}, options?.abortSignal);
+				const data = await response.json();
+				if (!data.message) {
+					require_logger.logger.info("[GOAT] Invalid message from GOAT, skipping turn", { data });
+					continue;
+				}
+				failureReason = data.message;
+				require_logger.logger.debug(`[GOAT] Previous attack attempt failure reason: ${failureReason}`);
+			}
+			body = JSON.stringify({
+				goal: context?.test?.metadata?.goal || context?.vars[this.config.injectVar],
+				i: turn,
+				messages: this.config.excludeTargetOutputFromAgenticAttackGeneration ? messages.filter((m) => m.role !== "assistant") : messages,
+				prompt: context?.prompt?.raw,
+				task: "goat",
+				version: require_version.VERSION,
+				email: require_accounts.getUserEmail(),
+				excludeTargetOutputFromAgenticAttackGeneration: this.config.excludeTargetOutputFromAgenticAttackGeneration,
+				failureReason,
+				purpose: context?.test?.metadata?.purpose,
+				modifiers: context?.test?.metadata?.modifiers,
+				traceSummary: previousTraceSummary,
+				inputs: this.config.inputs
+			});
+			require_logger.logger.debug(`[GOAT] Sending request to ${require_remoteGeneration.getRemoteGenerationUrl()}: ${body}`);
+			response = await require_fetch.fetchWithProxy(require_remoteGeneration.getRemoteGenerationUrl(), {
+				body,
+				headers: { "Content-Type": "application/json" },
+				method: "POST"
+			}, options?.abortSignal);
+			const data = await response.json();
+			if (typeof data?.message !== "object" || !data.message?.content || !data.message?.role) {
+				require_logger.logger.info("[GOAT] Invalid message from GOAT, skipping turn", { data });
+				continue;
+			}
+			const attackerMessage = data.message;
+			previousAttackerMessage = attackerMessage?.content;
+			let processedMessage = attackerMessage.content;
+			const extractedPrompt = require_util.extractPromptFromTags(attackerMessage.content);
+			if (extractedPrompt) processedMessage = extractedPrompt;
+			const currentInputVars = require_util.extractInputVarsFromPrompt(processedMessage, this.config.inputs);
+			const currentRenderInputVars = (currentInputVars && this.config.inputs ? await require_inputVariables.materializeInputVariablesWithMetadata(currentInputVars, this.config.inputs, {
+				materializationIndex: turn,
+				pluginId: "goat"
+			}) : void 0)?.vars ?? currentInputVars;
+			if (currentInputVars && this.config.inputs) try {
+				const parsed = JSON.parse(processedMessage);
+				if (typeof parsed.prompt === "string") processedMessage = parsed.prompt;
+			} catch {}
+			const attackerVars = {
+				[this.config.injectVar]: processedMessage,
+				...currentRenderInputVars || {}
+			};
+			const targetVars = {
+				...context.vars,
+				...attackerVars
+			};
+			const renderedAttackerPrompt = await require_evaluatorHelpers.renderPrompt(context.prompt, targetVars, context.filters, targetProvider, Object.keys(attackerVars));
+			messages.push({
+				role: attackerMessage.role,
+				content: renderedAttackerPrompt
+			});
+			require_logger.logger.debug(dedent.default`
+          ${chalk.default.bold.green(`GOAT turn ${turn} history:`)}
+          ${chalk.default.cyan(JSON.stringify(messages, null, 2))}
+        `);
+			const latestMessageContent = messages[messages.length - 1].content;
+			let targetPrompt = this.config.stateful ? latestMessageContent : JSON.stringify(messages);
+			require_logger.logger.debug(`GOAT turn ${turn} target prompt: ${renderedAttackerPrompt}`);
+			let lastTransformResult;
+			if (this.perTurnLayers.length > 0) {
+				require_logger.logger.debug("[GOAT] Applying per-turn transforms", {
+					turn,
+					layers: this.perTurnLayers.map((l) => typeof l === "string" ? l : l.id)
+				});
+				lastTransformResult = await require_runtimeTransform.applyRuntimeTransforms(latestMessageContent, this.config.injectVar, this.perTurnLayers, require_strategies.Strategies, {
+					evaluationId: context?.evaluationId,
+					testCaseId: context?.test?.metadata?.testCaseId,
+					purpose: context?.test?.metadata?.purpose,
+					goal: context?.test?.metadata?.goal
+				});
+				if (lastTransformResult.error) {
+					require_logger.logger.warn("[GOAT] Transform failed, skipping turn", {
+						turn,
+						error: lastTransformResult.error
+					});
+					continue;
+				}
+				if (lastTransformResult.audio || lastTransformResult.image) {
+					const historyWithoutCurrentTurn = messages.slice(0, -1);
+					const hybridPayload = {
+						_promptfoo_audio_hybrid: true,
+						history: historyWithoutCurrentTurn,
+						currentTurn: {
+							role: "user",
+							transcript: latestMessageContent,
+							...lastTransformResult.audio && { audio: lastTransformResult.audio },
+							...lastTransformResult.image && { image: lastTransformResult.image }
+						}
+					};
+					targetPrompt = JSON.stringify(hybridPayload);
+					require_logger.logger.debug("[GOAT] Using hybrid format (history + audio/image current turn)", {
+						turn,
+						historyLength: historyWithoutCurrentTurn.length,
+						hasAudio: !!lastTransformResult.audio,
+						hasImage: !!lastTransformResult.image
+					});
+				} else targetPrompt = lastTransformResult.prompt;
+				require_logger.logger.debug("[GOAT] Per-turn transforms applied", {
+					turn,
+					hasAudio: !!lastTransformResult.audio,
+					hasImage: !!lastTransformResult.image
+				});
+				if (lastTransformResult.displayVars) lastTransformDisplayVars = lastTransformResult.displayVars;
+				lastFinalAttackPrompt = lastTransformResult.prompt;
+			}
+			const iterationStart = Date.now();
+			require_promptLength.throwIfTargetPromptExceedsMaxChars(targetPrompt, maxCharsPerMessage);
+			const targetContext = context ? {
+				...context,
+				vars: {
+					...targetVars,
+					[this.config.injectVar]: targetPrompt
+				}
+			} : context;
+			const targetResponse = await targetProvider.callApi(targetPrompt, targetContext, options);
+			if (!targetResponse.cached && targetProvider.delay && targetProvider.delay > 0) {
+				require_logger.logger.debug(`Sleeping for ${targetProvider.delay}ms`);
+				await require_fetch.sleep(targetProvider.delay);
+			}
+			require_tokenUsageUtils.accumulateResponseTokenUsage(totalTokenUsage, targetResponse);
+			require_logger.logger.debug(`GOAT turn ${turn} target response`, { response: targetResponse });
+			let traceContext = null;
+			let computedTraceSummary;
+			if (shouldFetchTrace) {
+				const traceparent = context?.traceparent ?? void 0;
+				const traceId = traceparent ? require_tracingOptions.extractTraceIdFromTraceparent(traceparent) : null;
+				if (traceId) {
+					traceContext = await require_tracingOptions.fetchTraceContext(traceId, {
+						earliestStartTime: iterationStart,
+						includeInternalSpans: tracingOptions.includeInternalSpans,
+						maxSpans: tracingOptions.maxSpans,
+						maxDepth: tracingOptions.maxDepth,
+						maxRetries: tracingOptions.maxRetries,
+						retryDelayMs: tracingOptions.retryDelayMs,
+						spanFilter: tracingOptions.spanFilter,
+						sanitizeAttributes: tracingOptions.sanitizeAttributes
+					});
+					if (traceContext) {
+						targetResponse.traceContext = traceContext;
+						traceSnapshots.push(traceContext);
+						if (tracingOptions.includeInAttack || tracingOptions.includeInGrading) {
+							computedTraceSummary = require_tracingOptions.formatTraceSummary(traceContext);
+							targetResponse.traceSummary = computedTraceSummary;
+						}
+					}
+				}
+			}
+			if (targetResponse.sessionId) {
+				context = context ?? {
+					vars: {},
+					prompt: {
+						raw: "",
+						label: "target"
+					}
+				};
+				context.vars.sessionId = targetResponse.sessionId;
+			}
+			if (targetResponse.conversationEnded) {
+				require_logger.logger.info("[GOAT] Target ended conversation", {
+					turn,
+					reason: targetResponse.conversationEndReason
+				});
+				const endedOutput = typeof targetResponse.output === "string" ? targetResponse.output : require_logger.safeJsonStringify(targetResponse.output);
+				if (endedOutput) {
+					messages.push({
+						role: "assistant",
+						content: endedOutput
+					});
+					redteamHistory.push({
+						prompt: attackerMessage.content,
+						promptAudio: lastTransformResult?.audio,
+						promptImage: lastTransformResult?.image,
+						output: endedOutput,
+						outputAudio: targetResponse.audio?.data && targetResponse.audio?.format ? {
+							data: targetResponse.audio.data,
+							format: targetResponse.audio.format
+						} : void 0,
+						inputVars: currentRenderInputVars
+					});
+				}
+				lastTargetResponse = targetResponse;
+				stopReason = "Target ended conversation";
+				break;
+			}
+			if (targetResponse.error) throw new Error(`[GOAT] Target returned an error: ${targetResponse.error}`);
+			require_invariant.invariant(targetResponse.output, `[GOAT] Expected target response output to be set, but got: ${require_logger.safeJsonStringify(targetResponse)}`);
+			const stringifiedOutput = typeof targetResponse.output === "string" ? targetResponse.output : require_logger.safeJsonStringify(targetResponse.output);
+			const finalOutput = stringifiedOutput;
+			const finalResponse = targetResponse;
+			if (!stringifiedOutput) {
+				require_logger.logger.debug("[GOAT] Target response output is not a string or JSON", { response: targetResponse });
+				continue;
+			}
+			messages.push({
+				role: "assistant",
+				content: stringifiedOutput
+			});
+			redteamHistory.push({
+				prompt: attackerMessage.content,
+				promptAudio: lastTransformResult?.audio,
+				promptImage: lastTransformResult?.image,
+				output: stringifiedOutput,
+				outputAudio: targetResponse.audio?.data && targetResponse.audio?.format ? {
+					data: targetResponse.audio.data,
+					format: targetResponse.audio.format
+				} : void 0,
+				inputVars: currentRenderInputVars
+			});
+			const attackTraceSummary = tracingOptions.includeInAttack ? computedTraceSummary : void 0;
+			const gradingTraceSummary = tracingOptions.includeInGrading ? computedTraceSummary : void 0;
+			previousTraceSummary = attackTraceSummary;
+			previousTargetOutput = stringifiedOutput;
+			lastTargetResponse = finalResponse;
+			const grader = assertToUse ? getGraderById(assertToUse.type) : void 0;
+			if (test && assertToUse && grader && finalOutput) {
+				let gradingContext;
+				if (finalResponse.metadata?.wasExfiltrated === void 0) {
+					const webPageUuid = test.metadata?.webPageUuid;
+					if (webPageUuid) {
+						const evalId = context?.evaluationId ?? test.metadata?.evaluationId;
+						require_logger.logger.debug("[GOAT] Fetching exfil tracking from server API", {
+							webPageUuid,
+							evalId
+						});
+						const exfilData = await require_indirectWebPwn.checkExfilTracking(webPageUuid, evalId);
+						if (exfilData) gradingContext = {
+							...tracingOptions.includeInGrading ? {
+								traceContext: targetResponse.traceContext,
+								traceSummary: gradingTraceSummary
+							} : {},
+							wasExfiltrated: exfilData.wasExfiltrated,
+							exfilCount: exfilData.exfilCount,
+							exfilRecords: exfilData.exfilRecords
+						};
+					}
+				} else {
+					require_logger.logger.debug("[GOAT] Using exfil data from provider response metadata");
+					gradingContext = {
+						...tracingOptions.includeInGrading ? {
+							traceContext: targetResponse.traceContext,
+							traceSummary: gradingTraceSummary
+						} : {},
+						wasExfiltrated: Boolean(finalResponse.metadata.wasExfiltrated),
+						exfilCount: Number(finalResponse.metadata.exfilCount) || 0,
+						exfilRecords: []
+					};
+				}
+				if (!gradingContext && tracingOptions.includeInGrading) gradingContext = {
+					traceContext: targetResponse.traceContext,
+					traceSummary: gradingTraceSummary
+				};
+				const { grade, rubric } = await grader.getResult(attackerMessage.content, finalOutput, test, targetProvider, require_shared.getGraderAssertionValue(assertToUse), additionalRubric, void 0, gradingContext);
+				graderPassed = grade.pass;
+				storedGraderResult = {
+					...grade,
+					assertion: require_shared.buildGraderResultAssertion(grade.assertion, assertToUse, rubric)
+				};
+			}
+			if (graderPassed === false) {
+				this.successfulAttacks.push({
+					turn,
+					prompt: attackerMessage.content,
+					response: stringifiedOutput,
+					traceSummary: attackTraceSummary
+				});
+				if (this.config.continueAfterSuccess) {} else {
+					stopReason = "Grader failed";
+					break;
+				}
+			}
+		} catch (error) {
+			if (error instanceof Error && error.name === "AbortError") {
+				require_logger.logger.debug("[GOAT] Operation aborted");
+				throw error;
+			}
+			require_logger.logger.error(`[GOAT] An error occurred in GOAT turn ${turn}.  The test will continue to the next turn in the conversation.`, { error: error.message || error });
+		}
+		const finalPrompt = require_shared.getLastMessageContent(messages, "user") || "";
+		return {
+			output: require_shared.getLastMessageContent(messages, "assistant") || "",
+			prompt: finalPrompt,
+			metadata: {
+				redteamFinalPrompt: lastFinalAttackPrompt || finalPrompt,
+				messages,
+				stopReason,
+				redteamHistory,
+				successfulAttacks: this.successfulAttacks,
+				totalSuccessfulAttacks: this.successfulAttacks.length,
+				storedGraderResult,
+				traceSnapshots: traceSnapshots.length > 0 ? traceSnapshots.map((snapshot) => require_tracingOptions.formatTraceForMetadata(snapshot)) : void 0,
+				sessionId: require_util.getSessionId(lastTargetResponse, context),
+				...lastTransformDisplayVars && { transformDisplayVars: lastTransformDisplayVars }
+			},
+			tokenUsage: totalTokenUsage,
+			guardrails: lastTargetResponse?.guardrails
+		};
+	}
+};
+//#endregion
+exports.default = GoatProvider;
+
+//# sourceMappingURL=goat-una6pZGP.cjs.map

package/dist/src/graders-BDT7dif6.js

@@ -0,0 +1,3 @@
+#!/usr/bin/env node
+import { n as getGraderById } from "./graders-BGP99PdK.js";
+export { getGraderById };