promptfoo 0.120.25 → 0.120.26

package/dist/src/{providers-BOvl9qv5.js → providers-BlEhY5mi.js}

@@ -1,37 +1,38 @@
 #!/usr/bin/env node
 import { C as getEnvString, O as cliState_default, S as getEnvInt, b as getEnvBool, d as extractFirstJsonObject, g as safeJsonStringify, l as sanitizeObject, m as isValidJson, o as logger_default, u as sanitizeUrl, v as getConfigDirectoryPath, x as getEnvFloat } from "./logger-BotXmWKW.js";
-import { P as VERSION, S as CLOUD_PROVIDER_PREFIX, _ as openaiToolsToBedrock, b as transformToolChoice, d as REQUEST_TIMEOUT_MS, f as calculateCost, g as openaiToolChoiceToBedrock, i as fetchWithTimeout, l as sleep, m as isOpenAIToolChoice, n as fetchWithProxy, p as isOpenAIToolArray, r as fetchWithRetries, u as LONG_RUNNING_MODEL_TIMEOUT_MS, v as parseChatPrompt, x as transformTools, y as toTitleCase } from "./fetch-BuP-gm0E.js";
+import { P as VERSION, S as CLOUD_PROVIDER_PREFIX, _ as openaiToolsToBedrock, b as transformToolChoice, d as REQUEST_TIMEOUT_MS, f as calculateCost, g as openaiToolChoiceToBedrock, i as fetchWithTimeout, l as sleep, m as isOpenAIToolChoice, n as fetchWithProxy, p as isOpenAIToolArray, r as fetchWithRetries, u as LONG_RUNNING_MODEL_TIMEOUT_MS, v as parseChatPrompt, x as transformTools, y as toTitleCase } from "./fetch-SRsE6Ctl.js";
 import { t as invariant } from "./invariant-DT20jrBd.js";
-import { o as getUserEmail } from "./accounts-Bzx8GnL6.js";
-import { i as cloudConfig } from "./cloud-lRosK6Nk.js";
-import { r as telemetry_default } from "./telemetry-wXyCQ1Nz.js";
-import { A as DATASET_PLUGINS, E as isUuid, H as pluginDescriptions, L as isCustomStrategy, N as MULTI_TURN_STRATEGIES, O as AGENTIC_STRATEGIES, ft as STRATEGY_EXEMPT_PLUGINS, m as isProviderOptions, p as isApiProvider, v as ProviderOptionsSchema } from "./types-CMDs7Kzu.js";
-import { f as neverGenerateRemote, l as getRemoteGenerationUrl, m as shouldGenerateRemote, p as neverGenerateRemoteForRegularEvals, r as checkServerFeatureSupport, u as getRemoteGenerationUrlForUnaligned } from "./server-HQtz_gUy.js";
-import { a as fetchWithCache, c as isTransientConnectionError, o as getCache, s as isCacheEnabled } from "./cache-Bg1Wpvzq.js";
-import { a as isVideoFile, i as isJavascriptFile, n as isAudioFile, r as isImageFile } from "./fileExtensions-CzcPkiKZ.js";
-import { C as extractVariablesFromTemplates, E as parseFileUrl, _ as parsePathOrGlob, b as renderEnvOnlyInObject, d as getResolvedRelativePath, f as maybeLoadConfigFromExternalFile, g as maybeLoadToolsFromExternalFile, h as maybeLoadResponseFormatFromExternalFile, m as maybeLoadFromExternalFileWithVars, p as maybeLoadFromExternalFile, w as getNunjucksEngine, x as renderVarsInObject } from "./util-BECSJ0T_.js";
-import { a as safeJoin, i as resolvePackageEntryPoint, n as getWrapperDir, o as safeResolve, r as importModule } from "./esm-79BpbQzg.js";
-import { i as validatePythonPath, n as getEnvInt$1, r as runPython, t as getConfiguredPythonPath } from "./pythonUtils-CUPlf1H0.js";
-import { n as transform, r as getProcessShim, t as TransformInputType } from "./transform-B86_0IIb.js";
-import { a as sha256, n as isBlobStorageEnabled, r as shouldAttemptRemoteBlobUpload, t as extractAndStoreBinaryData } from "./extractor-BFoThUOa.js";
-import { n as withGenAISpan } from "./genaiTracer-foKtbVa2.js";
-import { i as normalizeFinishReason, n as MCPClient, r as FINISH_REASON_MAP, t as OpenAiChatCompletionProvider } from "./chat-D_cro0wd.js";
-import { a as createEmptyTokenUsage, n as accumulateResponseTokenUsage, r as accumulateTokenUsage } from "./tokenUsageUtils-D2ko2eFQ.js";
-import { E as TOKEN_REFRESH_BUFFER_MS, a as calculateGoogleCost, c as geminiFormatAndSystemInstructions, d as mergeParts, f as normalizeTools, g as getGoogleClient, h as GoogleAuthManager, i as transformMCPToolsToOpenAi, l as getCandidate, m as CHAT_MODELS, o as createAuthCacheDiscriminator, r as transformMCPToolsToGoogle, s as formatCandidateContents, u as getGoogleAccessToken, v as loadCredentials, y as resolveProjectId } from "./transform-gpzlPrwG.js";
-import { n as AnthropicGenericProvider, t as AnthropicMessagesProvider } from "./messages-DpoCLD_8.js";
-import { a as parseMessages, i as outputFromMessage, t as ANTHROPIC_MODELS } from "./util-Ba18wQ8e.js";
-import { n as ResponsesProcessor, r as FunctionCallbackHandler, t as OpenAiResponsesProvider } from "./responses-B85halqh.js";
-import { t as OpenAiGenericProvider } from "./openai-jNIREoAW.js";
-import { a as calculateOpenAICost, c as getTokenUsage$2, o as failApiCall, r as OPENAI_REALTIME_MODELS, s as formatOpenAiError } from "./util-CttPoePn.js";
-import { n as OpenAiEmbeddingProvider, t as OpenAiCompletionProvider } from "./completion-BTbouQDz.js";
-import { i as storeBlob } from "./blobs-DlB5fwyF.js";
-import { a as evalResultsTable, g as getDb } from "./tables-ChjAuVAR.js";
-import { n as escapeRegExp, t as ellipsize } from "./text-C1cXk9ij.js";
-import { n as getTraceStore } from "./store-rueRdzcm.js";
-import { t as AwsBedrockGenericProvider } from "./base-DumUXbOH.js";
-import { n as callOpenAiImageApi, r as formatOutput, t as OpenAiImageProvider } from "./image-DFwp1aAV.js";
-import { t as providerRegistry } from "./providerRegistry-DAZoFmGv.js";
-import { n as runRuby } from "./rubyUtils-DYKoa7Ze.js";
+import { o as getUserEmail } from "./accounts-DHHiXsy6.js";
+import { i as cloudConfig } from "./cloud-BMbRVJFw.js";
+import { r as telemetry_default } from "./telemetry-BXyVqyAg.js";
+import { A as DATASET_PLUGINS, E as isUuid, H as pluginDescriptions, L as isCustomStrategy, N as MULTI_TURN_STRATEGIES, O as AGENTIC_STRATEGIES, ft as STRATEGY_EXEMPT_PLUGINS, m as isProviderOptions, p as isApiProvider, v as ProviderOptionsSchema } from "./types-t52w-XsS.js";
+import { f as neverGenerateRemote, l as getRemoteGenerationUrl, m as shouldGenerateRemote, p as neverGenerateRemoteForRegularEvals, r as checkServerFeatureSupport, u as getRemoteGenerationUrlForUnaligned } from "./server-DWmZLfCy.js";
+import { a as fetchWithCache, o as getCache, s as isCacheEnabled } from "./cache-mIszOnuz.js";
+import { r as isTransientConnectionError } from "./errors-DnGCbnx8.js";
+import { a as isVideoFile, i as isJavascriptFile, n as isAudioFile, r as isImageFile } from "./fileExtensions-BpuMmaFL.js";
+import { C as extractVariablesFromTemplates, E as parseFileUrl, _ as parsePathOrGlob, b as renderEnvOnlyInObject, d as getResolvedRelativePath, f as maybeLoadConfigFromExternalFile, g as maybeLoadToolsFromExternalFile, h as maybeLoadResponseFormatFromExternalFile, m as maybeLoadFromExternalFileWithVars, p as maybeLoadFromExternalFile, w as getNunjucksEngine, x as renderVarsInObject } from "./util-BSh4a_Q8.js";
+import { a as safeJoin, i as resolvePackageEntryPoint, n as getWrapperDir, o as safeResolve, r as importModule } from "./esm-CYhseqj4.js";
+import { i as validatePythonPath, n as getEnvInt$1, r as runPython, t as getConfiguredPythonPath } from "./pythonUtils-r1uBuA0n.js";
+import { n as transform, r as getProcessShim, t as TransformInputType } from "./transform-CqTFr7KR.js";
+import { a as sha256, n as isBlobStorageEnabled, r as shouldAttemptRemoteBlobUpload, t as extractAndStoreBinaryData } from "./extractor-LSYjrhK0.js";
+import { n as withGenAISpan } from "./genaiTracer-CqNnnXrE.js";
+import { i as normalizeFinishReason, n as MCPClient, r as FINISH_REASON_MAP, t as OpenAiChatCompletionProvider } from "./chat-DIywASPG.js";
+import { a as createEmptyTokenUsage, n as accumulateResponseTokenUsage, r as accumulateTokenUsage } from "./tokenUsageUtils-DmZSD2eU.js";
+import { O as TOKEN_REFRESH_BUFFER_MS, S as toDataUri, _ as getGoogleClient, a as calculateGoogleCost, b as resolveProjectId, c as geminiFormatAndSystemInstructions, d as mergeParts, f as normalizeSafetySettings, g as GoogleAuthManager, h as CHAT_MODELS, i as transformMCPToolsToOpenAi, l as getCandidate, o as createAuthCacheDiscriminator, p as normalizeTools, r as transformMCPToolsToGoogle, s as formatCandidateContents, u as getGoogleAccessToken, y as loadCredentials } from "./transform-CoP2bJ7P.js";
+import { n as AnthropicGenericProvider, t as AnthropicMessagesProvider } from "./messages-D61tPFQo.js";
+import { a as parseMessages, i as outputFromMessage, t as ANTHROPIC_MODELS } from "./util-vjscpUzy.js";
+import { n as ResponsesProcessor, r as FunctionCallbackHandler, t as OpenAiResponsesProvider } from "./responses-TsdODUpm.js";
+import { t as OpenAiGenericProvider } from "./openai-CT5fwbve.js";
+import { a as calculateOpenAICost, c as getTokenUsage$2, o as failApiCall, r as OPENAI_REALTIME_MODELS, s as formatOpenAiError } from "./util-CUEt0Vum.js";
+import { n as OpenAiEmbeddingProvider, t as OpenAiCompletionProvider } from "./completion-BBJ6zmG3.js";
+import { i as storeBlob } from "./blobs-kt8v3UyH.js";
+import { a as evalResultsTable, g as getDb } from "./tables-C7TT2XVn.js";
+import { n as escapeRegExp, t as ellipsize } from "./text-DgMr_tiM.js";
+import { n as getTraceStore } from "./store-D4gdn9ih.js";
+import { t as AwsBedrockGenericProvider } from "./base-BaXmtXYp.js";
+import { n as callOpenAiImageApi, r as formatOutput, t as OpenAiImageProvider } from "./image-mhAGP07h.js";
+import { t as providerRegistry } from "./providerRegistry-Xf0qdqGQ.js";
+import { n as runRuby } from "./rubyUtils-BtjlqyXt.js";
 import { Agent } from "undici";
 import { z } from "zod";
 import input from "@inquirer/input";
@@ -2845,7 +2846,7 @@ var AIStudioChatProvider = class extends GoogleGenericProvider {
 			temperature: config.temperature,
 			topP: config.topP,
 			topK: config.topK,
-			safetySettings: config.safetySettings,
+			safetySettings: normalizeSafetySettings(config.safetySettings),
 			stopSequences: config.stopSequences,
 			maxOutputTokens: config.maxOutputTokens
 		};
@@ -2919,7 +2920,7 @@ var AIStudioChatProvider = class extends GoogleGenericProvider {
 				...config.maxOutputTokens !== void 0 && { maxOutputTokens: config.maxOutputTokens },
 				...config.generationConfig
 			},
-			safetySettings: config.safetySettings,
+			safetySettings: normalizeSafetySettings(config.safetySettings),
 			...config.toolConfig ? { toolConfig: config.toolConfig } : {},
 			...allTools.length > 0 ? { tools: allTools } : {},
 			...systemInstruction ? { system_instruction: systemInstruction } : {}
@@ -3238,7 +3239,7 @@ var VertexChatProvider = class extends GoogleGenericProvider {
 				topK: config.topK,
 				...config.generationConfig
 			},
-			...config.safetySettings ? { safetySettings: config.safetySettings } : {},
+			...config.safetySettings ? { safetySettings: normalizeSafetySettings(config.safetySettings) } : {},
 			...config.toolConfig ? { toolConfig: config.toolConfig } : {},
 			...allTools.length > 0 ? { tools: allTools } : {},
 			...systemInstruction ? { systemInstruction } : {},
@@ -3452,7 +3453,7 @@ var VertexChatProvider = class extends GoogleGenericProvider {
 			parameters: {
 				context: this.config.context,
 				examples: this.config.examples,
-				safetySettings: this.config.safetySettings,
+				safetySettings: normalizeSafetySettings(this.config.safetySettings),
 				stopSequences: this.config.stopSequences,
 				temperature: this.config.temperature,
 				maxOutputTokens: this.config.maxOutputTokens,
@@ -5396,7 +5397,7 @@ async function loadRedteamProvider({ provider, jsonOnly = false, preferSmallMode
 		ret = redteamProvider;
 	} else if (typeof redteamProvider === "string" || isProviderOptions(redteamProvider)) {
 		logger_default.debug(`Loading ${purpose} provider`, { provider: redteamProvider });
-		ret = (await (await import("./providers-DVsoSup2.js")).loadApiProviders([redteamProvider]))[0];
+		ret = (await (await import("./providers-Ck8HyrC-.js")).loadApiProviders([redteamProvider]))[0];
 	} else {
 		const defaultModel = preferSmallModel ? ATTACKER_MODEL_SMALL : ATTACKER_MODEL;
 		logger_default.debug(`Using default ${purpose} provider: ${defaultModel}`);
@@ -5675,7 +5676,7 @@ async function externalizeResponseForRedteamHistory(response, context) {
 */
 async function tryUnblocking({ messages, lastResponse, goal, purpose }) {
 	try {
-		const { checkServerFeatureSupport } = await import("./server-Dhbq83-d.js");
+		const { checkServerFeatureSupport } = await import("./server-gfOx5Zrk.js");
 		const supportsUnblocking = await checkServerFeatureSupport("blocking-question-analysis", "2025-06-16T14:49:11-07:00");
 		if (!getEnvBool("PROMPTFOO_ENABLE_UNBLOCKING")) {
 			logger_default.debug("[Unblocking] Disabled by default (set PROMPTFOO_ENABLE_UNBLOCKING=true to enable)");
@@ -9745,6 +9746,7 @@ const Strategies = [
 	},
 	{
 		id: "crescendo",
+		requiresGoalExtraction: true,
 		action: async (testCases, injectVar, config) => {
 			logger_default.debug(`Adding Crescendo to ${testCases.length} test cases`);
 			const newTestCases = addCrescendo(testCases, injectVar, config);
@@ -9754,6 +9756,7 @@ const Strategies = [
 	},
 	{
 		id: "custom",
+		requiresGoalExtraction: true,
 		action: async (testCases, injectVar, config, strategyId = "custom") => {
 			logger_default.debug(`Adding Custom to ${testCases.length} test cases`);
 			const newTestCases = addCustom(testCases, injectVar, config, strategyId);
@@ -9772,6 +9775,7 @@ const Strategies = [
 	},
 	{
 		id: "goat",
+		requiresGoalExtraction: true,
 		action: async (testCases, injectVar, config) => {
 			logger_default.debug(`Adding GOAT to ${testCases.length} test cases`);
 			const newTestCases = await addGoatTestCases(testCases, injectVar, config);
@@ -9781,6 +9785,7 @@ const Strategies = [
 	},
 	{
 		id: "indirect-web-pwn",
+		requiresGoalExtraction: true,
 		action: async (testCases, injectVar, config) => {
 			logger_default.debug(`Adding Indirect Web Pwn to ${testCases.length} test cases`);
 			const newTestCases = await addIndirectWebPwnTestCases(testCases, injectVar, config);
@@ -9817,6 +9822,7 @@ const Strategies = [
 	},
 	{
 		id: "jailbreak",
+		requiresGoalExtraction: true,
 		action: async (testCases, injectVar, config) => {
 			logger_default.warn("Strategy \"jailbreak\" is deprecated. Use \"jailbreak:meta\" instead. The \"jailbreak\" strategy used outdated single-shot optimization techniques.");
 			logger_default.debug(`Adding meta-agent jailbreaks to ${testCases.length} test cases`);
@@ -9845,6 +9851,7 @@ const Strategies = [
 	},
 	{
 		id: "jailbreak:tree",
+		requiresGoalExtraction: true,
 		action: async (testCases, injectVar, config) => {
 			logger_default.debug(`Adding experimental tree jailbreaks to ${testCases.length} test cases`);
 			const newTestCases = addIterativeJailbreaks(testCases, injectVar, "iterative:tree", config);
@@ -9854,6 +9861,7 @@ const Strategies = [
 	},
 	{
 		id: "jailbreak:meta",
+		requiresGoalExtraction: true,
 		action: async (testCases, injectVar, config) => {
 			logger_default.debug(`Adding meta-agent jailbreaks to ${testCases.length} test cases`);
 			const newTestCases = addIterativeJailbreaks(testCases, injectVar, "iterative:meta", config);
@@ -9863,6 +9871,7 @@ const Strategies = [
 	},
 	{
 		id: "jailbreak:hydra",
+		requiresGoalExtraction: true,
 		action: async (testCases, injectVar, config) => {
 			logger_default.debug(`Adding hydra multi-turn jailbreaks to ${testCases.length} test cases`);
 			const newTestCases = addHydra(testCases, injectVar, config);
@@ -10766,7 +10775,7 @@ var CrescendoProvider = class {
 		});
 		let assertToUse = test?.assert?.find((a) => a.type && a.type.includes(test.metadata?.pluginId));
 		if (!assertToUse) assertToUse = test?.assert?.find((a) => a.type);
-		const { getGraderById } = await import("./graders-DgMr-E4I.js");
+		const { getGraderById } = await import("./graders-spkuVC-E.js");
 		let graderPassed;
 		const additionalRubric = getGoalRubric(this.userGoal);
 		while (roundNum < this.maxTurns) try {
@@ -11441,7 +11450,7 @@ var CustomProvider = class {
 		let lastTransformResult;
 		let assertToUse = test?.assert?.find((a) => a.type && a.type.includes(test.metadata?.pluginId));
 		if (!assertToUse) assertToUse = test?.assert?.find((a) => a.type);
-		const { getGraderById } = await import("./graders-DgMr-E4I.js");
+		const { getGraderById } = await import("./graders-spkuVC-E.js");
 		let graderPassed;
 		let storedGraderResult;
 		const additionalRubric = getGoalRubric(this.userGoal);
@@ -11923,7 +11932,7 @@ var GoatProvider = class {
 		let assertToUse;
 		let graderPassed;
 		let storedGraderResult;
-		const { getGraderById } = await import("./graders-DgMr-E4I.js");
+		const { getGraderById } = await import("./graders-spkuVC-E.js");
 		let test;
 		if (context?.test) {
 			test = context?.test;
@@ -12364,7 +12373,7 @@ var HydraProvider = class {
 		let lastTransformResult;
 		let lastTransformDisplayVars;
 		let lastFinalAttackPrompt;
-		const { getGraderById } = await import("./graders-DgMr-E4I.js");
+		const { getGraderById } = await import("./graders-spkuVC-E.js");
 		let assertToUse = test?.assert?.find((a) => a.type && a.type.includes(test.metadata?.pluginId));
 		if (!assertToUse) assertToUse = test?.assert?.find((a) => a.type);
 		let previousTraceSummary;
@@ -13203,7 +13212,7 @@ async function runRedteamConversation$2({ context, filters, injectVar, numIterat
 		if (sessionId) sessionIds.push(sessionId);
 		let assertToUse = test?.assert?.find((a) => a.type && a.type.includes(test.metadata?.pluginId));
 		if (!assertToUse) assertToUse = test?.assert?.find((a) => a.type);
-		const { getGraderById } = await import("./graders-DgMr-E4I.js");
+		const { getGraderById } = await import("./graders-spkuVC-E.js");
 		if (test && assertToUse) {
 			const grader = getGraderById(assertToUse.type);
 			if (grader) {
@@ -14033,7 +14042,7 @@ async function runMetaAgentRedteam({ context, filters, injectVar, numIterations,
 		previousTraceSummary = attackTraceSummary;
 		let assertToUse = test?.assert?.find((a) => a.type && a.type.includes(test.metadata?.pluginId));
 		if (!assertToUse) assertToUse = test?.assert?.find((a) => a.type);
-		const { getGraderById } = await import("./graders-DgMr-E4I.js");
+		const { getGraderById } = await import("./graders-spkuVC-E.js");
 		if (test && assertToUse) {
 			const grader = getGraderById(assertToUse.type);
 			if (grader) {
@@ -14610,7 +14619,7 @@ async function runRedteamConversation({ context, filters, injectVar, options, pr
 					noImprovementCount++;
 					if (noImprovementCount % 5 === 0) logger_default.debug(`[Depth ${depth}, Attempt ${attempts}] No improvement for ${noImprovementCount} consecutive iterations. Max score: ${maxScore}`);
 				}
-				const { getGraderById } = await import("./graders-DgMr-E4I.js");
+				const { getGraderById } = await import("./graders-spkuVC-E.js");
 				let graderPassed;
 				let assertToUse = test?.assert?.find((a) => a.type && a.type.includes(test.metadata?.pluginId));
 				if (!assertToUse) assertToUse = test?.assert?.find((a) => a.type);
@@ -22595,7 +22604,7 @@ var GeminiImageProvider = class {
 			...this.config.imageAspectRatio && { aspectRatio: this.config.imageAspectRatio },
 			...this.config.imageSize && isGemini3 && { imageSize: this.config.imageSize }
 		};
-		if (this.config.safetySettings) body.safetySettings = this.config.safetySettings;
+		if (this.config.safetySettings) body.safetySettings = normalizeSafetySettings(this.config.safetySettings);
 		return body;
 	}
 	processResponse(data, cached, latencyMs) {
@@ -22616,16 +22625,20 @@ var GeminiImageProvider = class {
 			"SPII"
 		].includes(candidate.finishReason)) return { error: `Response was blocked with finish reason: ${candidate.finishReason}` };
 		if (!candidate.content?.parts) return { error: "No content parts in response" };
-		const outputParts = [];
+		const textParts = [];
+		const imageParts = [];
 		let totalCost = 0;
-		for (const part of candidate.content.parts) if (part.text) outputParts.push(part.text);
+		for (const part of candidate.content.parts) if (part.text) textParts.push(part.text);
 		else if (part.inlineData) {
 			const mimeType = part.inlineData.mimeType || "image/png";
 			const base64Data = part.inlineData.data;
-			outputParts.push(`![Generated Image](data:${mimeType};base64,${base64Data})`);
+			imageParts.push({
+				mimeType,
+				base64Data
+			});
 			totalCost += this.getCostPerImage();
 		}
-		if (outputParts.length === 0) return { error: "No valid content generated" };
+		if (imageParts.length === 0 && textParts.length === 0) return { error: "No valid content generated" };
 		const tokenUsage = cached ? {
 			cached: data.usageMetadata?.totalTokenCount,
 			total: data.usageMetadata?.totalTokenCount,
@@ -22636,8 +22649,13 @@ var GeminiImageProvider = class {
 			total: data.usageMetadata?.totalTokenCount,
 			numRequests: 1
 		};
+		const images = imageParts.length > 0 ? imageParts.map((img) => ({
+			data: toDataUri(img.mimeType, img.base64Data),
+			mimeType: img.mimeType
+		})) : void 0;
 		return {
-			output: outputParts.join("\n\n"),
+			output: imageParts.length > 0 && textParts.length === 0 ? images[0].data : textParts.join("\n\n"),
+			images,
 			cached,
 			latencyMs,
 			cost: totalCost > 0 ? totalCost : void 0,
@@ -22775,13 +22793,17 @@ var GoogleImageProvider = class {
 			const base64Image = imageData.bytesBase64Encoded;
 			const mimeType = imageData.mimeType || "image/png";
 			if (base64Image) {
-				imageOutputs.push(`![Generated Image](data:${mimeType};base64,${base64Image})`);
+				imageOutputs.push({
+					data: toDataUri(mimeType, base64Image),
+					mimeType
+				});
 				totalCost += costPerImage;
 			}
 		}
 		if (imageOutputs.length === 0) return { error: "No valid images generated" };
 		return {
-			output: imageOutputs.join("\n\n"),
+			output: imageOutputs[0].data,
+			images: imageOutputs,
 			cached,
 			latencyMs,
 			cost: totalCost
@@ -24937,6 +24959,7 @@ var HttpProvider = class {
 		const vars = {
 			...context?.vars || {},
 			prompt,
+			...context?.evaluationId ? { evaluationId: context.evaluationId } : {},
 			...transformedTools !== void 0 ? { tools: serializeForTemplate(transformedTools) } : {},
 			...transformedToolChoice !== void 0 ? { tool_choice: serializeForTemplate(transformedToolChoice) } : {}
 		};
@@ -25160,9 +25183,17 @@ var HttpProvider = class {
 			logger_default.error(`Error parsing session ID: ${String(err)}. Got headers: ${safeJsonStringify(sanitizeObject(responseHeaders, { context: "response headers" }))} and parsed body: ${safeJsonStringify(sanitizeObject(parsedData, { context: "response body" }))}`);
 			throw err;
 		}
+		ret.metadata = {
+			...ret.metadata,
+			http: {
+				status,
+				statusText
+			}
+		};
 		if (context?.debug) {
 			ret.raw = data;
 			ret.metadata = {
+				...ret.metadata,
 				headers: sanitizeObject(responseHeaders, { context: "response headers" }),
 				transformedRequest: this.config.transformRequest ? transformedPrompt : parsedRequest.body?.text || renderedRequest.trim(),
 				finalRequestBody: parsedRequest.body?.text,
@@ -32197,7 +32228,7 @@ const providerMap = [
 	{
 		test: (providerPath) => providerPath.startsWith("opencode:") || providerPath === "opencode",
 		create: async (providerPath, providerOptions, context) => {
-			const { OpenCodeSDKProvider } = await import("./opencode-sdk-CqxTHrk0.js");
+			const { OpenCodeSDKProvider } = await import("./opencode-sdk-CImWVqy9.js");
 			return new OpenCodeSDKProvider({
 				...providerOptions,
 				id: providerPath,
@@ -32209,14 +32240,14 @@ const providerMap = [
 	{
 		test: (providerPath) => providerPath.startsWith("openclaw:") || providerPath === "openclaw",
 		create: async (providerPath, providerOptions, context) => {
-			const { createOpenClawProvider } = await import("./openclaw-CnAwBfPO.js");
+			const { createOpenClawProvider } = await import("./openclaw-B6XY2kUf.js");
 			return createOpenClawProvider(providerPath, providerOptions, context.env);
 		}
 	},
 	{
 		test: (providerPath) => providerPath.startsWith("anthropic:claude-agent-sdk") || providerPath.startsWith("anthropic:claude-code"),
 		create: async (_providerPath, providerOptions, context) => {
-			const { ClaudeCodeSDKProvider } = await import("./claude-agent-sdk-qxE5mGVi.js");
+			const { ClaudeCodeSDKProvider } = await import("./claude-agent-sdk-CmkTnKGH.js");
 			return new ClaudeCodeSDKProvider({
 				...providerOptions,
 				env: context.env
@@ -32273,25 +32304,25 @@ const providerMap = [
 			const modelName = splits.slice(2).join(":");
 			if (modelType === "converse") return new AwsBedrockConverseProvider(modelName, providerOptions);
 			if (modelType === "nova-sonic" || modelType.includes("amazon.nova-sonic")) {
-				const { NovaSonicProvider } = await import("./nova-sonic-CklvraBd.js");
+				const { NovaSonicProvider } = await import("./nova-sonic-BXRfQyF-.js");
 				return new NovaSonicProvider("amazon.nova-sonic-v1:0", providerOptions);
 			}
 			if (modelType.includes("luma.ray") || modelName.includes("luma.ray")) {
-				const { LumaRayVideoProvider } = await import("./luma-ray-vvE-O9EA.js");
+				const { LumaRayVideoProvider } = await import("./luma-ray-DgKLS0BF.js");
 				return new LumaRayVideoProvider(modelName.includes("luma.ray") ? modelName : splits.slice(1).join(":") || "luma.ray-v2:0", providerOptions);
 			}
 			if (modelType.includes("amazon.nova-reel") || modelType === "video" && (modelName.includes("amazon.nova-reel") || modelName === "")) {
-				const { NovaReelVideoProvider } = await import("./nova-reel-BM0snD2m.js");
+				const { NovaReelVideoProvider } = await import("./nova-reel-Bk5npr2q.js");
 				return new NovaReelVideoProvider(modelName || "amazon.nova-reel-v1:1", providerOptions);
 			}
 			if (modelType === "agents") {
-				const { AwsBedrockAgentsProvider } = await import("./agents-zHyWnnQ2.js");
+				const { AwsBedrockAgentsProvider } = await import("./agents-Cao4i7AX.js");
 				return new AwsBedrockAgentsProvider(modelName, providerOptions);
 			}
 			if (modelType === "completion") return new AwsBedrockCompletionProvider(modelName, providerOptions);
 			if (modelType === "embeddings" || modelType === "embedding") return new AwsBedrockEmbeddingProvider(modelName, providerOptions);
 			if (modelType === "kb" || modelType === "knowledge-base") {
-				const { AwsBedrockKnowledgeBaseProvider } = await import("./knowledgeBase-DZw2L9kf.js");
+				const { AwsBedrockKnowledgeBaseProvider } = await import("./knowledgeBase-CMvMlLZR.js");
 				return new AwsBedrockKnowledgeBaseProvider(modelName, providerOptions);
 			}
 			return new AwsBedrockCompletionProvider(splits.slice(1).join(":"), providerOptions);
@@ -32301,7 +32332,7 @@ const providerMap = [
 		test: (providerPath) => providerPath.startsWith("bedrock-agent:"),
 		create: async (providerPath, providerOptions, _context) => {
 			const agentId = providerPath.substring(14);
-			const { AwsBedrockAgentsProvider } = await import("./agents-zHyWnnQ2.js");
+			const { AwsBedrockAgentsProvider } = await import("./agents-Cao4i7AX.js");
 			return new AwsBedrockAgentsProvider(agentId, providerOptions);
 		}
 	},
@@ -32311,7 +32342,7 @@ const providerMap = [
 			const splits = providerPath.split(":");
 			const modelType = splits[1];
 			const endpointName = splits.slice(2).join(":");
-			const { SageMakerCompletionProvider, SageMakerEmbeddingProvider } = await import("./sagemaker-hRS_Jj3-.js");
+			const { SageMakerCompletionProvider, SageMakerEmbeddingProvider } = await import("./sagemaker-BcgLu0U4.js");
 			if (modelType === "embedding" || modelType === "embeddings") return new SageMakerEmbeddingProvider(endpointName || modelType, providerOptions);
 			if (splits.length === 2) return new SageMakerCompletionProvider(modelType, providerOptions);
 			if (endpointName.includes("jumpstart") || modelType === "jumpstart") return new SageMakerCompletionProvider(endpointName, {
@@ -32352,7 +32383,7 @@ const providerMap = [
 	{
 		test: (providerPath) => providerPath.startsWith("cloudflare-ai:"),
 		create: async (providerPath, providerOptions, context) => {
-			const { createCloudflareAiProvider } = await import("./cloudflare-ai-CB7OWRAP.js");
+			const { createCloudflareAiProvider } = await import("./cloudflare-ai-CobxMTR4.js");
 			return createCloudflareAiProvider(providerPath, {
 				...providerOptions,
 				env: context.env
@@ -32362,7 +32393,7 @@ const providerMap = [
 	{
 		test: (providerPath) => providerPath.startsWith("cloudflare-gateway:"),
 		create: async (providerPath, providerOptions, context) => {
-			const { createCloudflareGatewayProvider } = await import("./cloudflare-gateway-BaVwoBjE.js");
+			const { createCloudflareGatewayProvider } = await import("./cloudflare-gateway-D2_yi-Fh.js");
 			return createCloudflareGatewayProvider(providerPath, {
 				...providerOptions,
 				env: context.env
@@ -32514,27 +32545,27 @@ const providerMap = [
 		create: async (providerPath, providerOptions, context) => {
 			const modelType = providerPath.split(":")[1];
 			if (modelType === "image") {
-				const { createHyperbolicImageProvider } = await import("./image-DIn_AW1Z.js");
+				const { createHyperbolicImageProvider } = await import("./image-DB4sHxdJ.js");
 				return createHyperbolicImageProvider(providerPath, {
 					...providerOptions,
 					env: context.env
 				});
 			}
 			if (modelType === "audio") {
-				const { createHyperbolicAudioProvider } = await import("./audio-BDrAkpVc.js");
+				const { createHyperbolicAudioProvider } = await import("./audio-DUH4q0Xq.js");
 				return createHyperbolicAudioProvider(providerPath, {
 					...providerOptions,
 					env: context.env
 				});
 			}
-			const { createHyperbolicProvider } = await import("./chat-DQ6Z5Cbn.js");
+			const { createHyperbolicProvider } = await import("./chat-Cpz3O-Xl.js");
 			return createHyperbolicProvider(providerPath, providerOptions);
 		}
 	},
 	{
 		test: (providerPath) => providerPath.startsWith("litellm:"),
 		create: async (providerPath, providerOptions, context) => {
-			const { createLiteLLMProvider } = await import("./litellm-fU-xV6T4.js");
+			const { createLiteLLMProvider } = await import("./litellm-CHrRmPAe.js");
 			return createLiteLLMProvider(providerPath, {
 				config: providerOptions,
 				env: context.env
@@ -32590,7 +32621,7 @@ const providerMap = [
 			const modelType = splits[1];
 			const modelName = splits.slice(2).join(":");
 			if (modelType === "codex-sdk" || modelType === "codex") {
-				const { OpenAICodexSDKProvider } = await import("./codex-sdk-DNkCbb7c.js");
+				const { OpenAICodexSDKProvider } = await import("./codex-sdk-dSnGdgIp.js");
 				return new OpenAICodexSDKProvider({
 					...providerOptions,
 					env: context.env
@@ -32603,7 +32634,7 @@ const providerMap = [
 			if (modelType === "realtime") return new OpenAiRealtimeProvider(modelName || "gpt-4o-realtime-preview-2024-12-17", providerOptions);
 			if (modelType === "responses") return new OpenAiResponsesProvider(modelName || "gpt-4.1-2025-04-14", providerOptions);
 			if (modelType === "transcription") {
-				const { OpenAiTranscriptionProvider } = await import("./transcription-CcYTL4wi.js");
+				const { OpenAiTranscriptionProvider } = await import("./transcription-Ch7S-LWw.js");
 				return new OpenAiTranscriptionProvider(modelName || "gpt-4o-transcribe-diarize", providerOptions);
 			}
 			if (OpenAiChatCompletionProvider.OPENAI_CHAT_MODEL_NAMES.includes(modelType)) return new OpenAiChatCompletionProvider(modelType, providerOptions);
@@ -32611,11 +32642,11 @@ const providerMap = [
 			if (OpenAiRealtimeProvider.OPENAI_REALTIME_MODEL_NAMES.includes(modelType)) return new OpenAiRealtimeProvider(modelType, providerOptions);
 			if (OpenAiResponsesProvider.OPENAI_RESPONSES_MODEL_NAMES.includes(modelType)) return new OpenAiResponsesProvider(modelType, providerOptions);
 			if (modelType === "agents") {
-				const { OpenAiAgentsProvider } = await import("./agents-CkA9l77-.js");
+				const { OpenAiAgentsProvider } = await import("./agents-DYIT-hQy.js");
 				return new OpenAiAgentsProvider(modelName || "default-agent", providerOptions);
 			}
 			if (modelType === "chatkit") {
-				const { OpenAiChatKitProvider } = await import("./chatkit-TnC1YoVi.js");
+				const { OpenAiChatKitProvider } = await import("./chatkit-DEls11hE.js");
 				return new OpenAiChatKitProvider(modelName || "", providerOptions);
 			}
 			if (modelType === "assistant") return new OpenAiAssistantProvider(modelName, providerOptions);
@@ -32658,7 +32689,7 @@ const providerMap = [
 	{
 		test: (providerPath) => providerPath.startsWith("quiverai:"),
 		create: async (providerPath, providerOptions, context) => {
-			const { createQuiverAiProvider } = await import("./quiverai-Y7Is4wah.js");
+			const { createQuiverAiProvider } = await import("./quiverai-BQigKdIH.js");
 			return createQuiverAiProvider(providerPath, providerOptions, context.env);
 		}
 	},
@@ -32703,7 +32734,7 @@ const providerMap = [
 	{
 		test: (providerPath) => providerPath.startsWith("aimlapi:"),
 		create: async (providerPath, providerOptions, context) => {
-			const { createAimlApiProvider } = await import("./aimlapi-DILJovhN.js");
+			const { createAimlApiProvider } = await import("./aimlapi-DMF6v_vb.js");
 			return createAimlApiProvider(providerPath, {
 				...providerOptions,
 				env: context.env
@@ -32713,7 +32744,7 @@ const providerMap = [
 	{
 		test: (providerPath) => providerPath.startsWith("cometapi:"),
 		create: async (providerPath, providerOptions, context) => {
-			const { createCometApiProvider } = await import("./cometapi-BGxGZ7VA.js");
+			const { createCometApiProvider } = await import("./cometapi-B01btbfb.js");
 			return createCometApiProvider(providerPath, {
 				...providerOptions,
 				env: context.env
@@ -32723,7 +32754,7 @@ const providerMap = [
 	{
 		test: (providerPath) => providerPath.startsWith("docker:"),
 		create: async (providerPath, providerOptions, context) => {
-			const { createDockerProvider } = await import("./docker-Dx_FN1ag.js");
+			const { createDockerProvider } = await import("./docker-BuButc4D.js");
 			return createDockerProvider(providerPath, {
 				...providerOptions,
 				env: context.env
@@ -32989,7 +33020,7 @@ const providerMap = [
 	{
 		test: (providerPath) => providerPath.startsWith("transformers:") || providerPath.startsWith("transformers.js:"),
 		create: async (providerPath, providerOptions, _context) => {
-			const { validateTransformersDependency } = await import("./transformersAvailability-DM54tWzt.js");
+			const { validateTransformersDependency } = await import("./transformersAvailability-DwmezkVe.js");
 			await validateTransformersDependency();
 			const splits = providerPath.split(":");
 			if (splits.length < 3) throw new Error(`Invalid Transformers.js provider path: ${providerPath}. Format: transformers:<task>:<model>
@@ -33009,7 +33040,7 @@ Example: transformers:feature-extraction:Xenova/all-MiniLM-L6-v2`);
 		test: (providerPath) => providerPath === "slack" || providerPath.startsWith("slack:"),
 		create: async (providerPath, providerOptions, _context) => {
 			try {
-				const { SlackProvider } = await import("./slack-DTUSadx4.js");
+				const { SlackProvider } = await import("./slack-BfdBx2tO.js");
 				if (providerPath === "slack") return new SlackProvider(providerOptions);
 				const splits = providerPath.split(":");
 				if (splits.length < 2) throw new Error("Invalid Slack provider path. Use slack:<channel_id> or slack:channel:<channel_id>");
@@ -33306,4 +33337,4 @@ function getProviderIds(providerPaths) {
 
 //#endregion
 export { AzureEmbeddingProvider as $, renderPrompt as A, OpenAiModerationProvider as B, extractVariablesFromJson as C, isEmptyResponse as D, isBasicRefusal as E, TokenUsageTracker as F, VertexChatProvider as G, MistralEmbeddingProvider as H, createRateLimitRegistry as I, DefaultGradingProvider$1 as J, AIStudioChatProvider as K, createProviderRateLimitOptions as L, isPackagePath as M, loadFromPackage as N, removePrefix as O, redteamProviderManager as P, AzureModerationProvider as Q, PromptfooHarmfulCompletionProvider as R, extractPromptFromTags as S, getShortPluginId as T, DefaultEmbeddingProvider as U, MistralChatCompletionProvider as V, DefaultGradingProvider as W, DefaultSuggestionsProvider as X, DefaultLlmRubricProvider as Y, DefaultSynthesizeProvider as Z, pluginMatchesStrategyTargets as _, resolveTeamId as _t, resolveProviderConfigs as a, getCloudDatabaseId as at, extractGoalFromPrompt as b, createTransformRequest as c, getEvalConfigFromCloud as ct, Strategies as d, getPoliciesFromCloud as dt, AzureChatCompletionProvider as et, loadStrategy as f, getProviderFromCloud as ft, retrieveMedia as g, resolveTeamFromIdentifier as gt, mediaExists as h, makeRequest as ht, resolveProvider as i, checkCloudPermissions as it, runExtensionHook as j, collectFileMetadata as k, createTransformResponse$1 as l, getOrgContext as lt, getMediaStorage as m, isCloudProvider as mt, loadApiProvider as n, parseScriptParts as nt, MCPProvider as o, getConfigFromCloud as ot, validateStrategies as p, getUserTeams as pt, DefaultGradingJsonProvider as q, loadApiProviders as r, canCreateTargets as rt, HttpProvider as s, getDefaultTeam as st, getProviderIds as t, getFileHashes as tt, GoogleLiveProvider as u, getPluginSeverityOverridesFromCloud as ut, checkExfilTracking as v, getSessionId as w, extractInputVarsFromPrompt as x, extractAllPromptsFromTags as y, REDTEAM_MEMORY_POISONING_PLUGIN_ID as z };
-//# sourceMappingURL=providers-BOvl9qv5.js.map
+//# sourceMappingURL=providers-BlEhY5mi.js.map

package/dist/src/providers-CQQrNaJk.js

@@ -0,0 +1,32 @@
+import "./chunk-FhC4c-0y.js";
+import "./logger-w8Ozp0Td.js";
+import "./esm-rDtG_2rg.js";
+import "./pythonUtils-vMlk9Qp5.js";
+import "./transform-D8dILpfZ.js";
+import "./types-DMVjYLpx.js";
+import "./util-DiCePfDu.js";
+import "./fetch-18MuNu9i.js";
+import "./cache-BLLayYEN.js";
+import { a as resolveProviderConfigs, i as resolveProvider, n as loadApiProvider, r as loadApiProviders, t as getProviderIds } from "./providers-BiNq_Iyc.js";
+import "./extractor-Hs7la_19.js";
+import "./genaiTracer-Ce19n68P.js";
+import "./chat-C8Ei4f87.js";
+import "./transform-DMaxQwDx.js";
+import "./messages-CDZYGNlS.js";
+import "./util-mJ58qbbw.js";
+import "./responses-DLLjADw5.js";
+import "./openai-Dz3surb_.js";
+import "./util-Cl0zfT3V.js";
+import "./completion-D9_MDlnd.js";
+import "./accounts-R3gfCR_g.js";
+import "./server-CbMTRQkg.js";
+import "./blobs-C9J2mVgo.js";
+import "./tables-D-NSwNIb.js";
+import "./telemetry-DZ_7PaVq.js";
+import "./store-2ocbYY9D.js";
+import "./base-Dtp8b4_N.js";
+import "./image-BiEVdpdP.js";
+import "./providerRegistry-wCWd7sKQ.js";
+import "./rubyUtils-Cs35SDYa.js";
+
+export { loadApiProvider, loadApiProviders };

package/dist/src/providers-Ck8HyrC-.js

@@ -0,0 +1,34 @@
+#!/usr/bin/env node
+import "./chunk-DHDDz29n.js";
+import "./logger-BotXmWKW.js";
+import "./fetch-SRsE6Ctl.js";
+import "./accounts-DHHiXsy6.js";
+import "./cloud-BMbRVJFw.js";
+import "./telemetry-BXyVqyAg.js";
+import "./types-t52w-XsS.js";
+import "./server-DWmZLfCy.js";
+import { a as resolveProviderConfigs, i as resolveProvider, n as loadApiProvider, r as loadApiProviders, t as getProviderIds } from "./providers-BlEhY5mi.js";
+import "./cache-mIszOnuz.js";
+import "./util-BSh4a_Q8.js";
+import "./esm-CYhseqj4.js";
+import "./pythonUtils-r1uBuA0n.js";
+import "./transform-CqTFr7KR.js";
+import "./extractor-LSYjrhK0.js";
+import "./genaiTracer-CqNnnXrE.js";
+import "./chat-DIywASPG.js";
+import "./transform-CoP2bJ7P.js";
+import "./messages-D61tPFQo.js";
+import "./util-vjscpUzy.js";
+import "./responses-TsdODUpm.js";
+import "./openai-CT5fwbve.js";
+import "./util-CUEt0Vum.js";
+import "./completion-BBJ6zmG3.js";
+import "./blobs-kt8v3UyH.js";
+import "./tables-C7TT2XVn.js";
+import "./store-D4gdn9ih.js";
+import "./base-BaXmtXYp.js";
+import "./image-mhAGP07h.js";
+import "./providerRegistry-Xf0qdqGQ.js";
+import "./rubyUtils-BtjlqyXt.js";
+
+export { loadApiProvider, loadApiProviders };