promptfoo 0.121.5 → 0.121.7

package/dist/src/hydra-BSNZZm2M.js

@@ -0,0 +1,543 @@
+#!/usr/bin/env node
+import { s as logger, y as isValidJson } from "./logger-BbY6ypFL.js";
+import { f as sleep } from "./fetch-DXUnXkVU.js";
+import { t as invariant } from "./invariant-B2Rf6avk.js";
+import { c as isLoggedIntoCloud } from "./accounts-CjFnOPmb.js";
+import { l as shouldGenerateRemote, n as getRemoteGenerationExplicitlyDisabledError, s as neverGenerateRemote, t as getRemoteGenerationDisabledError } from "./remoteGeneration-B1_XsKXU.js";
+import { a as createEmptyTokenUsage, n as accumulateResponseTokenUsage } from "./tokenUsageUtils-CDet74yk.js";
+import { r as materializeInputVariablesWithMetadata } from "./inputVariables-DXFdi7AI.js";
+import { a as extractPromptFromTags, c as isBasicRefusal, o as getSessionId, r as extractInputVarsFromPrompt } from "./util-kDURhgJW.js";
+import { t as PromptfooChatCompletionProvider } from "./promptfoo-Rjp-MeBb.js";
+import { c as getTargetResponse, i as externalizeResponseForRedteamHistory, l as isConversationEndedResponse, o as getGraderAssertionValue, t as buildGraderResultAssertion, u as isValidChatMessageArray } from "./shared-9WHQ1oNE.js";
+import { n as isBlobStorageEnabled, r as shouldAttemptRemoteBlobUpload } from "./extractor-CIW3iN-b.js";
+import { n as checkExfilTracking } from "./indirectWebPwn-uyWdHx04.js";
+import { n as renderPrompt } from "./evaluatorHelpers-BsYP_muT.js";
+import { t as Strategies } from "./strategies-DRJjGTIY.js";
+import { a as fetchTraceContext, i as extractTraceIdFromTraceparent, n as formatTraceForMetadata, r as formatTraceSummary, t as resolveTracingOptions } from "./tracingOptions-BnwKCkSB.js";
+import { t as applyRuntimeTransforms } from "./runtimeTransform-Dgh_D7DU.js";
+//#region src/redteam/providers/hydra/index.ts
+const DEFAULT_MAX_TURNS = 10;
+const DEFAULT_MAX_BACKTRACKS = 10;
+function scrubOutputForHistory(output) {
+	if (typeof output !== "string") return output;
+	const b64Match = output.match(/"b64_json"\s*:\s*"([^"]{200,})"/);
+	if (b64Match) return `[binary output redacted; b64_json length=${b64Match[1].length}]`;
+	const compact = output.replace(/\s+/g, "");
+	if (compact.length > 2e3 && /^[A-Za-z0-9+/=]+$/.test(compact)) return `[binary output redacted; length≈${compact.length}]`;
+	return output;
+}
+var HydraProvider = class {
+	config;
+	scanId;
+	agentProvider;
+	injectVar;
+	maxTurns;
+	maxBacktracks;
+	stateful;
+	excludeTargetOutputFromAgenticAttackGeneration;
+	perTurnLayers;
+	conversationHistory = [];
+	sessionId;
+	constructor(config) {
+		this.config = config;
+		this.scanId = config.scanId;
+		this.injectVar = config.injectVar;
+		const configuredMaxTurns = config.maxTurns ?? DEFAULT_MAX_TURNS;
+		this.maxTurns = isLoggedIntoCloud() ? configuredMaxTurns : Math.min(configuredMaxTurns, 10);
+		this.maxBacktracks = config.maxBacktracks ?? DEFAULT_MAX_BACKTRACKS;
+		this.stateful = config.stateful ?? false;
+		this.excludeTargetOutputFromAgenticAttackGeneration = config.excludeTargetOutputFromAgenticAttackGeneration ?? false;
+		this.perTurnLayers = config._perTurnLayers ?? [];
+		if (this.stateful && this.maxBacktracks > 0) logger.debug("[Hydra] Backtracking disabled in stateful mode");
+		if (!shouldGenerateRemote()) throw new Error(neverGenerateRemote() ? getRemoteGenerationExplicitlyDisabledError("jailbreak:hydra strategy") : getRemoteGenerationDisabledError("jailbreak:hydra strategy"));
+		this.agentProvider = new PromptfooChatCompletionProvider({
+			task: "hydra-decision",
+			jsonOnly: true,
+			preferSmallModel: false,
+			inputs: this.config.inputs
+		});
+		logger.debug("[Hydra] Provider initialized", {
+			maxTurns: this.maxTurns,
+			maxBacktracks: this.maxBacktracks,
+			stateful: this.stateful,
+			injectVar: this.injectVar,
+			excludeTargetOutputFromAgenticAttackGeneration: this.excludeTargetOutputFromAgenticAttackGeneration,
+			perTurnLayers: this.perTurnLayers.map((l) => typeof l === "string" ? l : l.id)
+		});
+	}
+	id() {
+		return "promptfoo:redteam:hydra";
+	}
+	async callApi(_prompt, context, options) {
+		logger.debug("[Hydra] callApi invoked");
+		invariant(context?.originalProvider, "Expected originalProvider to be set");
+		invariant(context?.vars, "Expected vars to be set");
+		const goal = context.test?.metadata?.goal || String(context.vars[this.injectVar]);
+		return this.runAttack({
+			prompt: context.prompt,
+			filters: context.filters,
+			vars: context.vars,
+			goal,
+			targetProvider: context.originalProvider,
+			context,
+			options,
+			test: context.test
+		});
+	}
+	async runAttack({ prompt, filters, vars, goal, targetProvider, context, options, test }) {
+		if (!this.scanId) this.scanId = context?.evaluationId || crypto.randomUUID();
+		const scanId = context?.evaluationId || this.scanId;
+		const tracingOptions = resolveTracingOptions({
+			strategyId: "hydra",
+			test,
+			config: this.config
+		});
+		const shouldFetchTrace = tracingOptions.enabled && (tracingOptions.includeInAttack || tracingOptions.includeInGrading);
+		const traceSnapshots = [];
+		logger.debug("[Hydra] Starting attack", {
+			goal,
+			scanId,
+			maxTurns: this.maxTurns,
+			stateful: this.stateful,
+			tracingEnabled: tracingOptions.enabled
+		});
+		this.conversationHistory = [];
+		this.sessionId = void 0;
+		const sessionIds = [];
+		const successfulAttacks = [];
+		const totalTokenUsage = createEmptyTokenUsage();
+		const testRunId = `${context?.evaluationId || "local"}-tc${context?.testCaseId || crypto.randomUUID().slice(0, 8)}`;
+		let vulnerabilityAchieved = false;
+		let stopReason = "Max turns reached";
+		let storedGraderResult = void 0;
+		let lastTargetResponse = void 0;
+		let backtrackCount = 0;
+		const redteamHistory = [];
+		let lastTransformResult;
+		let lastTransformDisplayVars;
+		let lastFinalAttackPrompt;
+		const { getGraderById } = await import("./graders-BDT7dif6.js");
+		let assertToUse = test?.assert?.find((a) => a.type && a.type.includes(test.metadata?.pluginId));
+		if (!assertToUse) assertToUse = test?.assert?.find((a) => a.type);
+		let previousTraceSummary;
+		for (let turn = 1; turn <= this.maxTurns; turn++) {
+			logger.debug(`[Hydra] Turn ${turn}/${this.maxTurns}`);
+			const conversationHistoryForCloud = this.excludeTargetOutputFromAgenticAttackGeneration ? this.conversationHistory.map((msg) => msg.role === "assistant" ? {
+				...msg,
+				content: "[Response hidden for privacy - grader feedback provided]"
+			} : msg) : this.conversationHistory;
+			const cloudRequest = {
+				task: "hydra-decision",
+				testRunId,
+				scanId,
+				turn,
+				goal,
+				purpose: test?.metadata?.purpose,
+				modifiers: test?.metadata?.modifiers,
+				conversationHistory: conversationHistoryForCloud,
+				...this.config.inputs && { inputs: this.config.inputs },
+				lastGraderResult: turn > 1 && storedGraderResult ? {
+					pass: storedGraderResult.pass,
+					score: storedGraderResult.score
+				} : void 0,
+				stateful: this.stateful,
+				maxTurns: this.maxTurns,
+				excludeTargetOutputFromAgenticAttackGeneration: this.excludeTargetOutputFromAgenticAttackGeneration,
+				...tracingOptions.includeInAttack && previousTraceSummary ? { traceSummary: previousTraceSummary } : {}
+			};
+			const agentResp = await this.agentProvider.callApi(JSON.stringify(cloudRequest), {
+				prompt: {
+					raw: JSON.stringify(cloudRequest),
+					label: "hydra-agent"
+				},
+				vars: {}
+			}, options);
+			accumulateResponseTokenUsage(totalTokenUsage, agentResp, { countAsRequest: false });
+			if (this.agentProvider.delay) await sleep(this.agentProvider.delay);
+			if (agentResp.error) {
+				logger.debug("[Hydra] Agent provider error", {
+					turn,
+					testRunId,
+					error: agentResp.error
+				});
+				continue;
+			}
+			let nextMessage;
+			if (typeof agentResp.output === "string") nextMessage = agentResp.output;
+			else {
+				const cloudResponse = agentResp.output;
+				nextMessage = cloudResponse.result || cloudResponse.message;
+			}
+			if (!nextMessage) {
+				logger.info("[Hydra] Missing message from agent", { turn });
+				continue;
+			}
+			let processedMessage = nextMessage;
+			const extractedPrompt = extractPromptFromTags(nextMessage);
+			if (extractedPrompt) processedMessage = extractedPrompt;
+			const currentInputVars = extractInputVarsFromPrompt(processedMessage, this.config.inputs);
+			const currentRenderInputVars = (currentInputVars && this.config.inputs ? await materializeInputVariablesWithMetadata(currentInputVars, this.config.inputs, {
+				materializationIndex: turn,
+				pluginId: "hydra",
+				provider: this.agentProvider,
+				purpose: test?.metadata?.purpose
+			}) : void 0)?.vars ?? currentInputVars;
+			this.conversationHistory.push({
+				role: "user",
+				content: processedMessage
+			});
+			let targetPrompt;
+			if (this.stateful) {
+				const escapedMessage = processedMessage.replace(/\{\{/g, "{ {").replace(/\}\}/g, "} }").replace(/\{%/g, "{ %").replace(/%\}/g, "% }");
+				targetPrompt = await renderPrompt(prompt, {
+					...vars,
+					[this.injectVar]: escapedMessage,
+					...this.sessionId ? { sessionId: this.sessionId } : {},
+					...currentRenderInputVars || {}
+				}, filters, targetProvider, [this.injectVar]);
+			} else {
+				const samplePrompt = await renderPrompt(prompt, {
+					...vars,
+					[this.injectVar]: "test"
+				}, filters, targetProvider, [this.injectVar]);
+				if (isValidJson(samplePrompt)) if (isValidChatMessageArray(JSON.parse(samplePrompt))) targetPrompt = JSON.stringify(this.conversationHistory);
+				else targetPrompt = JSON.stringify(this.conversationHistory);
+				else targetPrompt = JSON.stringify(this.conversationHistory);
+			}
+			logger.debug("[Hydra] Sending to target", {
+				turn,
+				stateful: this.stateful,
+				messageLength: nextMessage.length
+			});
+			let finalTargetPrompt = targetPrompt;
+			lastTransformResult = void 0;
+			if (this.perTurnLayers.length > 0) {
+				logger.debug("[Hydra] Applying per-turn transforms", {
+					turn,
+					layers: this.perTurnLayers.map((l) => typeof l === "string" ? l : l.id)
+				});
+				lastTransformResult = await applyRuntimeTransforms(nextMessage, this.injectVar, this.perTurnLayers, Strategies, {
+					evaluationId: context?.evaluationId,
+					testCaseId: test?.metadata?.testCaseId,
+					purpose: test?.metadata?.purpose,
+					goal: test?.metadata?.goal
+				});
+				if (lastTransformResult.error) {
+					logger.warn("[Hydra] Transform failed, skipping turn", {
+						turn,
+						error: lastTransformResult.error
+					});
+					this.conversationHistory.pop();
+					continue;
+				}
+				if (lastTransformResult.audio || lastTransformResult.image) {
+					const historyWithoutCurrentTurn = this.conversationHistory.slice(0, -1);
+					const hybridPayload = {
+						_promptfoo_audio_hybrid: true,
+						history: historyWithoutCurrentTurn,
+						currentTurn: {
+							role: "user",
+							transcript: nextMessage,
+							...lastTransformResult.audio && { audio: lastTransformResult.audio },
+							...lastTransformResult.image && { image: lastTransformResult.image }
+						}
+					};
+					finalTargetPrompt = JSON.stringify(hybridPayload);
+					logger.debug("[Hydra] Using hybrid format (history + audio/image current turn)", {
+						turn,
+						historyLength: historyWithoutCurrentTurn.length,
+						hasAudio: !!lastTransformResult.audio,
+						hasImage: !!lastTransformResult.image
+					});
+				} else finalTargetPrompt = lastTransformResult.prompt;
+				logger.debug("[Hydra] Per-turn transforms applied", {
+					turn,
+					originalLength: nextMessage.length,
+					transformedLength: finalTargetPrompt.length,
+					hasAudio: !!lastTransformResult.audio,
+					hasImage: !!lastTransformResult.image
+				});
+				if (lastTransformResult.displayVars) lastTransformDisplayVars = lastTransformResult.displayVars;
+			}
+			lastFinalAttackPrompt = finalTargetPrompt;
+			const iterationStart = Date.now();
+			const targetContext = context ? {
+				...context,
+				vars: {
+					...vars,
+					...this.sessionId ? { sessionId: this.sessionId } : {},
+					...currentRenderInputVars || {},
+					[this.injectVar]: finalTargetPrompt
+				}
+			} : context;
+			let targetResponse = await getTargetResponse(targetProvider, finalTargetPrompt, targetContext, options);
+			lastTargetResponse = targetResponse;
+			accumulateResponseTokenUsage(totalTokenUsage, targetResponse);
+			let traceContext = null;
+			let computedTraceSummary;
+			if (shouldFetchTrace) {
+				const traceparent = context?.traceparent ?? void 0;
+				const traceId = traceparent ? extractTraceIdFromTraceparent(traceparent) : null;
+				if (traceId) {
+					traceContext = await fetchTraceContext(traceId, {
+						earliestStartTime: iterationStart,
+						includeInternalSpans: tracingOptions.includeInternalSpans,
+						maxSpans: tracingOptions.maxSpans,
+						maxDepth: tracingOptions.maxDepth,
+						maxRetries: tracingOptions.maxRetries,
+						retryDelayMs: tracingOptions.retryDelayMs,
+						spanFilter: tracingOptions.spanFilter,
+						sanitizeAttributes: tracingOptions.sanitizeAttributes
+					});
+					if (traceContext) {
+						traceSnapshots.push(traceContext);
+						if (tracingOptions.includeInAttack || tracingOptions.includeInGrading) computedTraceSummary = formatTraceSummary(traceContext);
+					}
+				}
+			}
+			logger.debug("[Hydra] Received target response", {
+				turn,
+				responseLength: targetResponse.output?.length,
+				hasTrace: !!traceContext
+			});
+			if (isConversationEndedResponse(targetResponse)) {
+				logger.info("[Hydra] Target ended conversation", {
+					turn,
+					reason: targetResponse.conversationEndReason
+				});
+				stopReason = "Target ended conversation";
+				break;
+			}
+			if (targetResponse.error) {
+				logger.info("[Hydra] Target error", {
+					turn,
+					error: targetResponse.error
+				});
+				continue;
+			}
+			if (!Object.prototype.hasOwnProperty.call(targetResponse, "output")) {
+				logger.info("[Hydra] Malformed target response - missing output", {
+					turn,
+					responseKeys: Object.keys(targetResponse)
+				});
+				continue;
+			}
+			if (!targetResponse.output || !targetResponse.output.trim()) {
+				logger.info("[Hydra] Empty target response", {
+					turn,
+					outputIsNull: targetResponse.output === null,
+					outputIsUndefined: targetResponse.output === void 0,
+					outputIsEmptyString: targetResponse.output === "",
+					outputValue: targetResponse.output,
+					outputTrimmed: targetResponse.output?.trim()
+				});
+				targetResponse.output = "[Target provided empty response - likely refused]";
+			}
+			if (this.stateful && targetResponse.sessionId) {
+				this.sessionId = targetResponse.sessionId;
+				sessionIds.push(targetResponse.sessionId);
+				vars["sessionId"] = targetResponse.sessionId;
+				if (!context) context = {
+					vars: {
+						...vars,
+						sessionId: targetResponse.sessionId
+					},
+					prompt
+				};
+				context.vars["sessionId"] = targetResponse.sessionId;
+			}
+			if (isBlobStorageEnabled() || shouldAttemptRemoteBlobUpload()) {
+				const beforeOutput = targetResponse.output;
+				targetResponse = await externalizeResponseForRedteamHistory(targetResponse, {
+					evalId: context?.evaluationId,
+					testIdx: context?.testIdx,
+					promptIdx: context?.promptIdx
+				});
+				if (targetResponse.output !== beforeOutput) logger.debug("[Hydra] Externalized binary output", {
+					turn,
+					beforeLength: beforeOutput?.length,
+					afterLength: targetResponse.output?.length,
+					blobUris: targetResponse.metadata && "blobUris" in targetResponse.metadata ? targetResponse.metadata.blobUris : void 0
+				});
+				else if (typeof targetResponse.output === "string") logger.debug("[Hydra] Binary output not externalized (using in-band)", {
+					turn,
+					responseLength: targetResponse.output.length
+				});
+			}
+			const historyOutput = isBlobStorageEnabled() || shouldAttemptRemoteBlobUpload() ? scrubOutputForHistory(targetResponse.output) : targetResponse.output;
+			this.conversationHistory.push({
+				role: "assistant",
+				content: historyOutput
+			});
+			const isRefusal = isBasicRefusal(targetResponse.output);
+			if (!this.stateful && this.maxBacktracks > 0 && isRefusal) {
+				logger.debug("[Hydra] Response rejected (basic refusal), backtracking...", {
+					turn,
+					backtrackCount: backtrackCount + 1,
+					maxBacktracks: this.maxBacktracks,
+					conversationLengthBefore: this.conversationHistory.length
+				});
+				backtrackCount++;
+				if (this.conversationHistory.length >= 2) {
+					this.conversationHistory.pop();
+					this.conversationHistory.pop();
+				}
+				logger.debug("[Hydra] After backtracking state", {
+					turn,
+					backtrackCount,
+					conversationLength: this.conversationHistory.length,
+					willDecrementTurn: turn > 1
+				});
+				if (backtrackCount >= this.maxBacktracks) {
+					logger.debug(`[Hydra] Max backtracks (${this.maxBacktracks}) reached. Exiting loop.`, {
+						backtrackCount,
+						maxBacktracks: this.maxBacktracks
+					});
+					stopReason = "Max backtracks reached";
+					break;
+				}
+				if (turn > 1) turn--;
+				continue;
+			}
+			let graderResult = void 0;
+			const attackTraceSummary = tracingOptions.includeInAttack ? computedTraceSummary : void 0;
+			const gradingTraceSummary = tracingOptions.includeInGrading ? computedTraceSummary : void 0;
+			previousTraceSummary = attackTraceSummary;
+			if (test && assertToUse) {
+				const grader = getGraderById(assertToUse.type);
+				if (grader) {
+					const gradingContext = {
+						providerResponse: targetResponse,
+						...tracingOptions.includeInGrading ? {
+							traceContext,
+							traceSummary: gradingTraceSummary
+						} : {}
+					};
+					const webPageUuid = lastTransformResult?.metadata?.webPageUuid;
+					if (webPageUuid) {
+						const webPageUrl = lastTransformResult?.metadata?.webPageUrl;
+						const evalId = context?.evaluationId ?? webPageUrl?.match(/\/dynamic-pages\/([^/]+)\//)?.[1];
+						logger.debug("[Hydra] Fetching exfil tracking from server API", {
+							webPageUuid,
+							evalId,
+							source: "lastTransformResult.metadata"
+						});
+						try {
+							const exfilData = await checkExfilTracking(webPageUuid, evalId);
+							if (exfilData) Object.assign(gradingContext, {
+								wasExfiltrated: exfilData.wasExfiltrated,
+								exfilCount: exfilData.exfilCount,
+								exfilRecords: exfilData.exfilRecords
+							});
+						} catch (error) {
+							logger.warn("[Hydra] Failed to fetch exfil tracking from server", {
+								error,
+								webPageUuid
+							});
+						}
+					}
+					if (gradingContext.wasExfiltrated === void 0 && targetResponse.metadata?.wasExfiltrated !== void 0) {
+						logger.debug("[Hydra] Using exfil data from provider response metadata (fallback)");
+						Object.assign(gradingContext, {
+							wasExfiltrated: Boolean(targetResponse.metadata.wasExfiltrated),
+							exfilCount: Number(targetResponse.metadata.exfilCount) || 0,
+							exfilRecords: []
+						});
+					}
+					const { grade, rubric } = await grader.getResult(nextMessage, targetResponse.output, test, targetProvider, getGraderAssertionValue(assertToUse), void 0, void 0, gradingContext);
+					graderResult = grade;
+					storedGraderResult = {
+						...grade,
+						assertion: buildGraderResultAssertion(grade.assertion, assertToUse, rubric)
+					};
+					logger.debug("[Hydra] Grader result", {
+						turn,
+						passed: grade.pass
+					});
+				}
+			}
+			redteamHistory.push({
+				prompt: nextMessage,
+				promptAudio: lastTransformResult?.audio,
+				promptImage: lastTransformResult?.image,
+				output: historyOutput,
+				outputAudio: targetResponse.audio ? {
+					data: targetResponse.audio.data || "",
+					format: targetResponse.audio.format || "wav"
+				} : void 0,
+				graderPassed: graderResult?.pass,
+				trace: traceContext ? formatTraceForMetadata(traceContext) : void 0,
+				traceSummary: computedTraceSummary,
+				inputVars: currentRenderInputVars
+			});
+			if (graderResult?.pass === false) {
+				vulnerabilityAchieved = true;
+				successfulAttacks.push({
+					turn,
+					message: nextMessage,
+					response: targetResponse.output,
+					traceSummary: computedTraceSummary
+				});
+				stopReason = "Grader failed";
+				logger.debug("[Hydra] Vulnerability achieved!", { turn });
+				break;
+			}
+		}
+		if (scanId) try {
+			const turnsCompleted = this.conversationHistory.filter((m) => m.role === "user").length;
+			const learningRequest = {
+				task: "hydra-decision",
+				testRunId,
+				scanId,
+				testComplete: true,
+				finalResult: {
+					success: vulnerabilityAchieved,
+					totalTurns: turnsCompleted
+				}
+			};
+			accumulateResponseTokenUsage(totalTokenUsage, await this.agentProvider.callApi(JSON.stringify(learningRequest), {
+				prompt: {
+					raw: JSON.stringify(learningRequest),
+					label: "hydra-learning-update"
+				},
+				vars: {}
+			}, options), { countAsRequest: false });
+			logger.debug("[Hydra] Scan learnings updated", {
+				scanId,
+				testRunId
+			});
+		} catch (error) {
+			logger.warn("[Hydra] Failed to update scan learnings", { error });
+		}
+		const messages = this.conversationHistory.map((msg) => ({
+			role: msg.role,
+			content: msg.content
+		}));
+		return {
+			output: lastTargetResponse?.output || "",
+			...lastTargetResponse?.error ? { error: lastTargetResponse.error } : {},
+			metadata: {
+				sessionId: this.sessionId || getSessionId(lastTargetResponse, context),
+				messages,
+				hydraRoundsCompleted: this.conversationHistory.filter((m) => m.role === "user").length,
+				hydraBacktrackCount: backtrackCount,
+				hydraResult: vulnerabilityAchieved,
+				stopReason,
+				successfulAttacks,
+				totalSuccessfulAttacks: successfulAttacks.length,
+				storedGraderResult,
+				redteamHistory,
+				sessionIds,
+				traceSnapshots: traceSnapshots.length > 0 ? traceSnapshots.map((t) => formatTraceForMetadata(t)) : void 0,
+				...lastTransformDisplayVars && { transformDisplayVars: lastTransformDisplayVars },
+				redteamFinalPrompt: lastFinalAttackPrompt || successfulAttacks[0]?.message
+			},
+			tokenUsage: totalTokenUsage,
+			guardrails: lastTargetResponse?.guardrails
+		};
+	}
+};
+//#endregion
+export { HydraProvider, HydraProvider as default };
+
+//# sourceMappingURL=hydra-BSNZZm2M.js.map