promptfoo 0.121.10 → 0.121.11

package/dist/src/evaluator-CTl-lKuh.js

@@ -0,0 +1,3 @@
+#!/usr/bin/env node
+import { r as evaluate } from "./evaluator-DbOsHSRe.js";
+export { evaluate };

package/dist/src/{evaluator-C_gCkBe3.js → evaluator-DbOsHSRe.js}

@@ -1,44 +1,47 @@
 #!/usr/bin/env node
 import { t as getProcessShim } from "./processShim-PekFOUjm.js";
 import { A as getEvalTimeoutMs, C as summarizeEvaluateResultForLogging, E as getEnvBool, M as isCI, O as getEnvInt, P as state, S as safeJsonStringify, _ as extractFirstJsonObject, c as logger, i as globalLogCallback, j as getMaxEvalTimeMs, k as getEnvString, l as setLogCallback, v as extractJsonObjects, y as getAjv } from "./logger-DNHkR_3S.js";
-import { K as VERSION, M as parseChatPrompt, _ as isNonTransientHttpStatus, b as sleep, i as fetchWithRetries, k as isPromptfooSampleTarget, q as FILE_METADATA_KEY } from "./fetch-BBa1Lxgp.js";
+import { K as VERSION, M as parseChatPrompt, _ as isNonTransientHttpStatus, b as sleep, i as fetchWithRetries, k as isPromptfooSampleTarget, q as FILE_METADATA_KEY } from "./fetch-B1Rb0VdL.js";
 import { t as invariant } from "./invariant-BtWWVVhl.js";
-import { n as telemetry } from "./telemetry-Bh3HJU7r.js";
+import { n as telemetry } from "./telemetry-C7v0I4YX.js";
 import { at as LLAMA_GUARD_REPLICATE_PROVIDER, ct as MULTI_INPUT_VAR, d as isGradingResult, h as isCliEventSource, p as isApiProvider, s as ResultFailureReason, y as filterByRange } from "./types-C7iPXxc5.js";
 import { i as isJavascriptFile } from "./fileExtensions-PGPK8Wcw.js";
 import { t as providerRegistry } from "./providerRegistry-DaK_MD80.js";
-import { C as isAnthropicProvider, E as isProviderAllowed, H as loadFunction, K as analyzeTemplateReference, L as maybeLoadToolsFromExternalFile, T as isOpenAiProvider, U as parseFileUrl, Y as getNunjucksEngine, q as extractVariablesFromTemplate, w as isGoogleProvider } from "./util-DHG9r5sC.js";
-import { c as withCacheNamespace, i as getCache } from "./cache-CmidsPv2.js";
+import { C as isAnthropicProvider, E as isProviderAllowed, H as loadFunction, K as analyzeTemplateReference, L as maybeLoadToolsFromExternalFile, T as isOpenAiProvider, U as parseFileUrl, Y as getNunjucksEngine, q as extractVariablesFromTemplate, w as isGoogleProvider } from "./util-CcW4TZSn.js";
+import { c as withCacheNamespace, i as getCache } from "./cache-D2MfIrzr.js";
 import { r as runPython } from "./pythonUtils-yoUkCRXx.js";
-import { d as validateFunctionCall } from "./util-D9xyoiGG.js";
-import { l as shouldGenerateRemote } from "./remoteGeneration-BlwssRVS.js";
-import { c as promptYesNo } from "./server-CYxCnoVw.js";
-import { t as OpenAiChatCompletionProvider } from "./chat-CvN2w9fY.js";
-import { y as validateFunctionCall$1 } from "./transform-DmSzMsCP.js";
-import { _ as VertexChatProvider, b as GoogleLiveProvider, n as loadApiProvider, x as AIStudioChatProvider } from "./providers-DdNIebwz.js";
+import { d as validateFunctionCall } from "./util-CkLsVF3e.js";
+import { l as shouldGenerateRemote } from "./remoteGeneration-CW6zHjQx.js";
+import { c as promptYesNo } from "./server-DMCug-jv.js";
+import { t as OpenAiChatCompletionProvider } from "./chat-PfM8xgFA.js";
+import { y as validateFunctionCall$1 } from "./transform-CD9wrP31.js";
+import { _ as VertexChatProvider, b as GoogleLiveProvider, n as loadApiProvider, p as MCPProvider, x as AIStudioChatProvider } from "./providers-am7xTa5w.js";
 import { a as createEmptyTokenUsage, i as createEmptyAssertions, n as accumulateResponseTokenUsage, o as normalizeTokenUsage, r as accumulateTokenUsage, t as accumulateAssertionTokenUsage } from "./tokenUsageUtils-kXKUpxyr.js";
 import { n as loadFromPackage, t as isPackagePath } from "./packageParser-C_T8xu-D.js";
 import { n as runRuby } from "./rubyUtils-BjUJ_Cj-.js";
-import { c as isBasicRefusal, o as getSessionId } from "./util-16aq4oHW.js";
-import { $ as SELECT_BEST_PROMPT, Ct as DEFAULT_ANTHROPIC_MODEL, Dt as getProviderCallExecutionContext, Et as getGradingProvider, G as matchesLlmRubric, H as matchesClosedQa, J as doRemoteGrading, K as matchesPiScore, Ot as withProviderCallExecutionContext, Q as DEFAULT_WEB_SEARCH_PROMPT, St as getDefaultProviders, Tt as getAndCheckProvider, U as matchesFactuality, V as isGraderFailure, W as matchesGEval, _t as tryParse, at as CONTEXT_RECALL_ATTRIBUTED_TOKEN, bt as loadFromJavaScriptFile, ct as CONTEXT_RELEVANCE_BAD, dt as cosineSimilarity, et as SUGGEST_PROMPTS_SYSTEM_MESSAGE, ft as dotProduct, gt as splitIntoSentences, ht as normalizeMatcherTokenUsage, it as CONTEXT_RECALL, lt as loadRubricPrompt, mt as fail, n as getGraderById, nt as CONTEXT_FAITHFULNESS_LONGFORM, ot as CONTEXT_RECALL_NOT_ATTRIBUTED_TOKEN, pt as euclideanDistance, q as matchesTrajectoryGoalSuccess, rt as CONTEXT_FAITHFULNESS_NLI_STATEMENTS, st as CONTEXT_RELEVANCE, tt as ANSWER_RELEVANCY_GENERATE, ut as renderLlmRubricPrompt, vt as coerceString, wt as callProviderWithContext, xt as processFileReference, yt as getFinalTest } from "./graders-DEz7EeRy.js";
-import { f as redteamProviderManager, g as createProviderRateLimitOptions, h as createRateLimitRegistry, m as TokenUsageTracker } from "./shared-BavzGCp0.js";
+import { c as isBasicRefusal, o as getSessionId } from "./util-CD4LxH5h.js";
+import { $ as SELECT_BEST_PROMPT, Ct as DEFAULT_ANTHROPIC_MODEL, Dt as getProviderCallExecutionContext, Et as getGradingProvider, G as matchesLlmRubric, H as matchesClosedQa, J as doRemoteGrading, K as matchesPiScore, Ot as withProviderCallExecutionContext, Q as DEFAULT_WEB_SEARCH_PROMPT, St as getDefaultProviders, Tt as getAndCheckProvider, U as matchesFactuality, V as isGraderFailure, W as matchesGEval, _t as tryParse, at as CONTEXT_RECALL_ATTRIBUTED_TOKEN, bt as loadFromJavaScriptFile, ct as CONTEXT_RELEVANCE_BAD, dt as cosineSimilarity, et as SUGGEST_PROMPTS_SYSTEM_MESSAGE, ft as dotProduct, gt as splitIntoSentences, ht as normalizeMatcherTokenUsage, it as CONTEXT_RECALL, lt as loadRubricPrompt, mt as fail, n as getGraderById, nt as CONTEXT_FAITHFULNESS_LONGFORM, ot as CONTEXT_RECALL_NOT_ATTRIBUTED_TOKEN, pt as euclideanDistance, q as matchesTrajectoryGoalSuccess, rt as CONTEXT_FAITHFULNESS_NLI_STATEMENTS, st as CONTEXT_RELEVANCE, tt as ANSWER_RELEVANCY_GENERATE, ut as renderLlmRubricPrompt, vt as coerceString, wt as callProviderWithContext, xt as processFileReference, yt as getFinalTest } from "./graders-CpUlNVnh.js";
+import { f as redteamProviderManager, g as createProviderRateLimitOptions, h as createRateLimitRegistry, m as TokenUsageTracker } from "./shared-C-G3NQ1-.js";
 import { i as generateIdFromPrompt } from "./utils-CsmoLRsV.js";
 import { a as getTransformLabel, i as getTransformErrorMessage, o as transform, r as TransformInputType } from "./transform-Dou53MRF2.js";
 import { n as getTraceStore } from "./store-Bn1-Urtg.js";
 import { a as getActualPromptWithFallback, r as updateSignalFile } from "./signal-DdhkLwgQ.js";
-import { t as extractAndStoreBinaryData } from "./extractor-DV6WBfAi.js";
+import { t as extractAndStoreBinaryData } from "./extractor-DOhjGS0m.js";
 import { i as throwIfTargetPromptExceedsMaxChars } from "./promptLength-DYfhyE9j.js";
-import { n as checkExfilTracking } from "./indirectWebPwn-A16xYmav.js";
+import { n as checkExfilTracking } from "./indirectWebPwn-BbFMiv79.js";
 import { n as getFirstStringAttribute, r as getToolNameFromAttributes, t as TOOL_ARGUMENT_ATTRIBUTE_KEYS } from "./toolAttributes-DkzTVYQF.js";
-import { i as filterFiniteScores, n as renderPrompt, r as runExtensionHook, t as collectFileMetadata } from "./evaluatorHelpers-C6-aKnJe.js";
+import { i as filterFiniteScores, n as renderPrompt, r as runExtensionHook, t as collectFileMetadata } from "./evaluatorHelpers-CoLW5rKU.js";
 import { AsyncResource } from "node:async_hooks";
 import chalk from "chalk";
 import { createWriteStream } from "fs";
 import path from "path";
 import os from "os";
+import Ajv from "ajv";
+import addFormats from "ajv-formats";
 import yaml from "js-yaml";
 import fs$1 from "fs/promises";
 import util from "util";
+import dedent from "dedent";
 import { randomBytes } from "crypto";
 import { DiagConsoleLogger, DiagLogLevel, diag, propagation } from "@opentelemetry/api";
 import { globSync } from "glob";
@@ -5025,6 +5028,193 @@ function maybeEmitAzureOpenAiWarning(testSuite, tests) {
 	return false;
 }
 //#endregion
+//#region src/redteam/mcpMaterialization.ts
+const TOOL_NAME_FIELDS = [
+	"tool",
+	"toolName",
+	"function",
+	"functionName",
+	"name"
+];
+const TOOL_ARGS_FIELDS = [
+	"args",
+	"arguments",
+	"params",
+	"parameters"
+];
+const ajv = new Ajv({
+	allErrors: true,
+	strictSchema: false
+});
+addFormats(ajv);
+function parseMcpToolCall(value, allowedToolNames) {
+	let parsed = value;
+	if (typeof value === "string") try {
+		parsed = JSON.parse(value);
+	} catch {
+		return;
+	}
+	if (typeof parsed !== "object" || parsed === null || Array.isArray(parsed)) return;
+	const record = parsed;
+	const toolName = TOOL_NAME_FIELDS.map((field) => record[field]).find((fieldValue) => typeof fieldValue === "string" && allowedToolNames.has(fieldValue));
+	if (!toolName) return;
+	return {
+		tool: toolName,
+		args: TOOL_ARGS_FIELDS.map((field) => record[field]).find((fieldValue) => typeof fieldValue === "object" && fieldValue !== null && !Array.isArray(fieldValue)) ?? {}
+	};
+}
+function validateMcpToolCall(toolCall, toolByName) {
+	if (!toolCall) return false;
+	const tool = toolByName.get(toolCall.tool);
+	if (!tool) return false;
+	try {
+		return ajv.validate(tool.inputSchema ?? { type: "object" }, toolCall.args) === true;
+	} catch (error) {
+		logger.warn(`Failed to validate MCP tool call for ${tool.name}: ${error instanceof Error ? error.message : String(error)}`);
+		return false;
+	}
+}
+function stringifyToolCall(toolCall) {
+	return JSON.stringify({
+		tool: toolCall.tool,
+		args: toolCall.args
+	});
+}
+function getProviderOutputString(response) {
+	if (Array.isArray(response.output)) return response.output.join("\n");
+	return response.output === void 0 ? "" : String(response.output);
+}
+async function repairMcpToolCallWithProvider({ allowedToolNames, purpose, provider, toolByName, tools, value }) {
+	let response;
+	try {
+		response = await provider.callApi(dedent`
+        Convert this red team test intent into exactly one MCP tool call JSON object.
+
+        Return JSON only, with this exact shape:
+        {"tool":"tool_name","args":{}}
+
+        Choose only one of these MCP tools and use its input schema:
+        ${JSON.stringify(tools, null, 2)}
+
+        Application purpose:
+        ${purpose}
+
+        Red team test intent:
+        ${typeof value === "string" ? value : JSON.stringify(value)}
+      `.trim());
+	} catch (error) {
+		logger.debug(`Failed to repair MCP value with provider: ${error instanceof Error ? error.message : String(error)}`);
+		return;
+	}
+	if (response.error) {
+		logger.warn(`Failed to materialize MCP value: ${response.error}`);
+		return;
+	}
+	const parsedObjects = extractJsonObjects(getProviderOutputString(response));
+	for (const parsedObject of parsedObjects) {
+		const toolCall = parseMcpToolCall(parsedObject, allowedToolNames);
+		if (validateMcpToolCall(toolCall, toolByName)) return toolCall;
+	}
+}
+async function materializeMcpValue({ intentValue, purpose, provider, tools, value }) {
+	if (tools.length === 0) return typeof value === "string" ? value : value === void 0 ? "" : JSON.stringify(value);
+	const allowedToolNames = new Set(tools.map((tool) => tool.name));
+	const toolByName = new Map(tools.map((tool) => [tool.name, tool]));
+	const existingToolCall = parseMcpToolCall(value, allowedToolNames);
+	if (existingToolCall && validateMcpToolCall(existingToolCall, toolByName)) return stringifyToolCall(existingToolCall);
+	const materializationIntentValue = intentValue ?? value;
+	if (!provider) throw new Error(`Failed to materialize MCP value: inference provider is required for non-JSON MCP input ${JSON.stringify(value)}`);
+	const toolCall = await repairMcpToolCallWithProvider({
+		allowedToolNames,
+		provider,
+		purpose: purpose ?? "",
+		toolByName,
+		tools,
+		value: materializationIntentValue
+	});
+	if (!toolCall || !validateMcpToolCall(toolCall, toolByName)) throw new Error(`Failed to materialize MCP value: ${JSON.stringify(value)}`);
+	return stringifyToolCall(toolCall);
+}
+//#endregion
+//#region src/redteam/mcpTargetProvider.ts
+const WRAPPED_MCP_PROVIDER = Symbol("wrappedMcpProvider");
+function isRedteamTest(test) {
+	return Boolean(test?.metadata?.pluginId || test?.metadata?.strategyId);
+}
+function isMcpProviderWithTools(provider) {
+	return provider instanceof MCPProvider && typeof provider.getAvailableTools === "function";
+}
+var RedteamMcpTargetProvider = class {
+	[WRAPPED_MCP_PROVIDER] = true;
+	label;
+	config;
+	delay;
+	transform;
+	inputs;
+	toolsPromise;
+	constructor(target) {
+		this.target = target;
+		this.label = target.label;
+		this.config = target.config;
+		this.delay = target.delay;
+		this.transform = target.transform;
+		this.inputs = target.inputs;
+	}
+	id() {
+		return this.target.id();
+	}
+	toString() {
+		return this.target.toString?.() ?? this.id();
+	}
+	async callApi(prompt, context, options) {
+		const tools = await this.getTools();
+		if (tools.length === 0) return this.target.callApi(prompt, context, options);
+		try {
+			const intentValue = context?.test?.metadata?.goal ?? context?.test?.metadata?.originalPrompt ?? prompt;
+			const purpose = String(context?.test?.metadata?.purpose ?? "");
+			let materializedPrompt;
+			try {
+				materializedPrompt = await materializeMcpValue({
+					intentValue,
+					purpose,
+					tools,
+					value: prompt
+				});
+			} catch (error) {
+				logger.debug(`MCP target prompt requires inference materialization: ${error instanceof Error ? error.message : String(error)}`);
+				materializedPrompt = await materializeMcpValue({
+					intentValue,
+					provider: await redteamProviderManager.getProvider({ jsonOnly: true }),
+					purpose,
+					tools,
+					value: prompt
+				});
+			}
+			const materializedContext = context ? {
+				...context,
+				vars: {
+					...context.vars,
+					prompt: materializedPrompt
+				}
+			} : void 0;
+			return this.target.callApi(materializedPrompt, materializedContext, options);
+		} catch (error) {
+			return { error: `Failed to materialize MCP target prompt: ${error instanceof Error ? error.message : String(error)}` };
+		}
+	}
+	async cleanup() {
+		await this.target.cleanup?.();
+	}
+	getTools() {
+		this.toolsPromise ??= this.target.getAvailableTools();
+		return this.toolsPromise;
+	}
+};
+function maybeWrapMcpProviderForRedteam(provider, test) {
+	if (!isRedteamTest(test) || provider[WRAPPED_MCP_PROVIDER]) return provider;
+	return isMcpProviderWithTools(provider) ? new RedteamMcpTargetProvider(provider) : provider;
+}
+//#endregion
 //#region src/scheduler/providerCallQueue.ts
 var ProviderGroupedCallQueue = class {
 	jobs = [];
@@ -5834,12 +6024,13 @@ async function callProviderForRunEval({ abortSignal, evalId, evaluateOptions, fi
 	};
 }
 async function callActiveProvider({ abortSignal, evalId, filters, promptForRender, provider, rateLimitRegistry, renderedPrompt, repeatIndex, test, traceContext, vars }) {
-	const activeProvider = isApiProvider(test.provider) ? test.provider : provider;
+	const originalProvider = maybeWrapMcpProviderForRedteam(provider, test);
+	const activeProvider = maybeWrapMcpProviderForRedteam(isApiProvider(test.provider) ? test.provider : originalProvider, test);
 	logger.debug(`Provider type: ${activeProvider.id()}`);
 	const callApiContext = buildCallApiContext({
 		evalId,
 		filters,
-		originalProvider: provider,
+		originalProvider,
 		promptForRender,
 		repeatIndex,
 		test,
@@ -6565,7 +6756,7 @@ async function resolveDefaultTestProvider(defaultTest, testCase) {
 	const defaultProvider = defaultTest.provider;
 	if (isApiProvider(defaultProvider)) return defaultProvider;
 	if (typeof defaultProvider === "object" && defaultProvider.id) {
-		const { loadApiProvider } = await import("./providers-xrARCHcC.js");
+		const { loadApiProvider } = await import("./providers-BIgLuX4Z.js");
 		return loadApiProvider(typeof defaultProvider.id === "function" ? defaultProvider.id() : defaultProvider.id, { options: defaultProvider });
 	}
 	return defaultProvider;
@@ -6725,7 +6916,7 @@ function buildRepeatCacheContextByTestIdx(runEvalOptions) {
 async function filterCompletedResumeSteps(runEvalOptions, evalRecord) {
 	if (!state.resume || !evalRecord.persisted) return;
 	try {
-		const { default: EvalResult } = await import("./evalResult-uL1BStGk.js");
+		const { default: EvalResult } = await import("./evalResult-B6Jpy3kJ.js");
 		const completedPairs = await EvalResult.getCompletedIndexPairs(evalRecord.id, { excludeErrors: state.retryMode });
 		const originalCount = runEvalOptions.length;
 		for (let i = runEvalOptions.length - 1; i >= 0; i--) {
@@ -7835,4 +8026,4 @@ function evaluate(testSuite, evalRecord, options) {
 //#endregion
 export { generateVarCombinations as a, doesPromptRefMatch as c, runAssertions as d, formatVarsForDisplay as i, warnEmptyFilterRange as l, PromptSuggestionsRejectedError as n, isAllowedPrompt as o, evaluate as r, accumulateNamedMetric as s, ProgressBarManager as t, readAssertions as u };
 
-//# sourceMappingURL=evaluator-C_gCkBe3.js.map
+//# sourceMappingURL=evaluator-DbOsHSRe.js.map

package/dist/src/{evaluatorHelpers-BVvi8tWH.cjs → evaluatorHelpers-CaaC50RB.cjs}

@@ -1,12 +1,12 @@
 const require_rolldown_runtime = require("./rolldown-runtime-D_mwlA32.cjs");
 const require_logger = require("./logger-DPITYeiW.cjs");
 const require_invariant = require("./invariant-QtnLD03y.cjs");
-const require_fetch = require("./fetch-OePjzUmA.cjs");
+const require_fetch = require("./fetch-C7yU9CNr.cjs");
 const require_fileExtensions = require("./fileExtensions-DB9eTs4y.cjs");
 const require_esm = require("./esm-BXQgvlCe.cjs");
-const require_util = require("./util-DJFl6JZF.cjs");
+const require_util = require("./util-tFLdovkA.cjs");
 const require_pythonUtils = require("./pythonUtils-Dm5aZDp2.cjs");
-const require_telemetry = require("./telemetry-Be4rklbJ.cjs");
+const require_telemetry = require("./telemetry-hDovhwWp.cjs");
 const require_packageParser = require("./packageParser-ClroutLT.cjs");
 const require_transform = require("./transform-vjEQU-TM.cjs");
 let fs_promises = require("fs/promises");
@@ -533,4 +533,4 @@ Object.defineProperty(exports, "runExtensionHook", {
 	}
 });
 
-//# sourceMappingURL=evaluatorHelpers-BVvi8tWH.cjs.map
+//# sourceMappingURL=evaluatorHelpers-CaaC50RB.cjs.map

package/dist/src/{evaluatorHelpers-C6-aKnJe.js → evaluatorHelpers-CoLW5rKU.js}

@@ -1,11 +1,11 @@
 #!/usr/bin/env node
 import { E as getEnvBool, P as state, c as logger, k as getEnvString } from "./logger-DNHkR_3S.js";
-import { r as fetchWithProxy } from "./fetch-BBa1Lxgp.js";
+import { r as fetchWithProxy } from "./fetch-B1Rb0VdL.js";
 import { t as invariant } from "./invariant-BtWWVVhl.js";
-import { n as telemetry } from "./telemetry-Bh3HJU7r.js";
+import { n as telemetry } from "./telemetry-C7v0I4YX.js";
 import { a as isVideoFile, i as isJavascriptFile, n as isAudioFile, r as isImageFile } from "./fileExtensions-PGPK8Wcw.js";
 import { r as importModule } from "./esm-ClRKphVo.js";
-import { G as renderVarsInObject, Y as getNunjucksEngine, q as extractVariablesFromTemplate } from "./util-DHG9r5sC.js";
+import { G as renderVarsInObject, Y as getNunjucksEngine, q as extractVariablesFromTemplate } from "./util-CcW4TZSn.js";
 import { r as runPython } from "./pythonUtils-yoUkCRXx.js";
 import { n as loadFromPackage, t as isPackagePath } from "./packageParser-C_T8xu-D.js";
 import { o as transform } from "./transform-Dou53MRF2.js";
@@ -507,4 +507,4 @@ async function runExtensionHook(extensions, hookName, context) {
 //#endregion
 export { filterFiniteScores as i, renderPrompt as n, runExtensionHook as r, collectFileMetadata as t };
 
-//# sourceMappingURL=evaluatorHelpers-C6-aKnJe.js.map
+//# sourceMappingURL=evaluatorHelpers-CoLW5rKU.js.map

package/dist/src/{evaluatorHelpers-RQo-m90b.js → evaluatorHelpers-DTtYkrGO.js}

@@ -1,11 +1,11 @@
 import { A as state, C as getEnvBool, E as getEnvString, o as logger } from "./logger-2ZstxKQf.js";
 import { t as invariant } from "./invariant-Ddh24eXh.js";
-import { t as fetchWithProxy } from "./fetch-CFIU30wy.js";
+import { t as fetchWithProxy } from "./fetch-BNUDixbE.js";
 import { a as isVideoFile, i as isJavascriptFile, n as isAudioFile, r as isImageFile } from "./fileExtensions-CvD3YVfM.js";
 import { r as importModule } from "./esm-D78sHVp9.js";
-import { B as getNunjucksEngine, I as renderVarsInObject, R as extractVariablesFromTemplate } from "./util-CnqjxsAd.js";
+import { B as getNunjucksEngine, I as renderVarsInObject, R as extractVariablesFromTemplate } from "./util-BTAyQmdN.js";
 import { r as runPython } from "./pythonUtils-E3AlITsZ.js";
-import { t as telemetry } from "./telemetry-B-NTbvDj.js";
+import { t as telemetry } from "./telemetry-CTYOje0d.js";
 import { n as loadFromPackage, t as isPackagePath } from "./packageParser-D0SUUwqU.js";
 import { o as transform } from "./transform-5CliVIgB.js";
 import fs from "fs/promises";
@@ -506,4 +506,4 @@ async function runExtensionHook(extensions, hookName, context) {
 //#endregion
 export { filterFiniteScores as i, renderPrompt as n, runExtensionHook as r, collectFileMetadata as t };
 
-//# sourceMappingURL=evaluatorHelpers-RQo-m90b.js.map
+//# sourceMappingURL=evaluatorHelpers-DTtYkrGO.js.map

package/dist/src/{extractor-B1hELWwJ.js → extractor-BIqe0lMN.js}

@@ -1,7 +1,9 @@
 import { C as getEnvBool, o as logger } from "./logger-2ZstxKQf.js";
-import { M as cloudConfig } from "./fetch-CFIU30wy.js";
-import { s as isLoggedIntoCloud } from "./accounts-D4iTShVh.js";
-import { n as recordBlobReference, r as storeBlob } from "./blobs-7T2OXYkC.js";
+import { M as cloudConfig } from "./fetch-BNUDixbE.js";
+import { s as isLoggedIntoCloud } from "./accounts-BGs8AMhG.js";
+import { n as sha256 } from "./createHash-CM2Vkqac.js";
+import { a as BLOB_MIN_SIZE, n as recordBlobReference, o as BLOB_SCHEME, r as storeBlob } from "./blobs-B8d9jKvz.js";
+import { XMLParser, XMLValidator } from "fast-xml-parser";
 //#region src/blobs/remoteUpload.ts
 function buildRemoteUploadTarget() {
 	if (getEnvBool("PROMPTFOO_DISABLE_SHARING")) return null;
@@ -28,7 +30,7 @@ async function uploadBlobRemote(buffer, mimeType, context) {
 	const target = buildRemoteUploadTarget();
 	if (!target) return null;
 	try {
-		const { fetchWithProxy } = await import("./fetch-CFIU30wy.js").then((n) => n.i);
+		const { fetchWithProxy } = await import("./fetch-BNUDixbE.js").then((n) => n.i);
 		const response = await fetchWithProxy(target.url, {
 			method: "POST",
 			headers: {
@@ -86,9 +88,9 @@ function extractBase64(value) {
 		return null;
 	}
 }
-function shouldExternalize(buffer) {
+function shouldExternalize(buffer, minSizeBytes = BLOB_MIN_SIZE) {
 	const size = buffer.length;
-	return size >= 1024 && size <= 52428800;
+	return size >= minSizeBytes && size <= 52428800;
 }
 function getKindFromMimeType(mimeType) {
 	return mimeType.startsWith("audio/") ? "audio" : "image";
@@ -135,9 +137,9 @@ function parseBinary(base64OrDataUrl, defaultMimeType) {
 		return null;
 	}
 }
-async function maybeStore(base64OrDataUrl, defaultMimeType, context, location, kind) {
+async function maybeStore(base64OrDataUrl, defaultMimeType, context, location, kind, minSizeBytes = BLOB_MIN_SIZE) {
 	const parsed = parseBinary(base64OrDataUrl, defaultMimeType);
-	if (!parsed || !shouldExternalize(parsed.buffer)) return null;
+	if (!parsed || !shouldExternalize(parsed.buffer, minSizeBytes)) return null;
 	if (!isBlobStorageEnabled()) return null;
 	const mimeType = parsed.mimeType || "application/octet-stream";
 	const { ref } = await storeBlob(parsed.buffer, mimeType, {
@@ -161,13 +163,13 @@ async function maybeStore(base64OrDataUrl, defaultMimeType, context, location, k
 }
 function createStoreOnce(blobContext) {
 	const cache = /* @__PURE__ */ new Map();
-	return async (base64OrDataUrl, defaultMimeType, location, kind) => {
+	return async (base64OrDataUrl, defaultMimeType, location, kind, minSizeBytes) => {
 		const parsed = parseBinary(base64OrDataUrl, defaultMimeType);
-		if (!parsed || !shouldExternalize(parsed.buffer)) return null;
+		if (!parsed || !shouldExternalize(parsed.buffer, minSizeBytes)) return null;
 		const cacheKey = `${kind}:${parsed.buffer.toString("base64")}`;
 		const existing = cache.get(cacheKey);
 		if (existing) return existing;
-		const pendingStore = maybeStore(base64OrDataUrl, defaultMimeType, blobContext, location, kind);
+		const pendingStore = maybeStore(base64OrDataUrl, defaultMimeType, blobContext, location, kind, minSizeBytes);
 		cache.set(cacheKey, pendingStore);
 		try {
 			const stored = await pendingStore;
@@ -179,6 +181,57 @@ function createStoreOnce(blobContext) {
 		}
 	};
 }
+function getRawSvgOutputPreview(output) {
+	const trimmed = output.trim();
+	if (!trimmed.startsWith("<")) return null;
+	if (XMLValidator.validate(trimmed) !== true) return null;
+	try {
+		const parsed = new XMLParser({ ignoreAttributes: false }).parse(trimmed);
+		if (!parsed || typeof parsed !== "object" || Array.isArray(parsed) || !("svg" in parsed)) return null;
+	} catch {
+		return null;
+	}
+	const buffer = Buffer.from(trimmed, "utf8");
+	return {
+		dataUrl: `data:image/svg+xml;base64,${buffer.toString("base64")}`,
+		uri: `${BLOB_SCHEME}${sha256(buffer)}`
+	};
+}
+function appendMetadataBlobUri(metadata, uri) {
+	const existingBlobUris = Array.isArray(metadata?.blobUris) ? metadata.blobUris.filter((value) => typeof value === "string") : [];
+	return {
+		...metadata || {},
+		blobUris: [...new Set([...existingBlobUris, uri])]
+	};
+}
+async function storeRawSvgOutputPreview(output, metadata, storeOnce, context) {
+	if (typeof output !== "string") return {
+		metadata,
+		mutated: false
+	};
+	const preview = getRawSvgOutputPreview(output);
+	if (!preview) return {
+		metadata,
+		mutated: false
+	};
+	if ((Array.isArray(metadata?.blobUris) ? metadata.blobUris.filter((value) => typeof value === "string") : []).includes(preview.uri)) return {
+		metadata,
+		mutated: false
+	};
+	const stored = await storeOnce(preview.dataUrl, "image/svg+xml", "response.output", "image", 0);
+	if (!stored) return {
+		metadata,
+		mutated: false
+	};
+	logger.debug("[BlobExtractor] Stored raw SVG output blob", {
+		...context,
+		hash: stored.hash
+	});
+	return {
+		metadata: appendMetadataBlobUri(metadata, stored.uri),
+		mutated: true
+	};
+}
 async function externalizeDataUrls(value, storeOnce, location) {
 	if (typeof value === "string") {
 		if (!isDataUrl(value)) return {
@@ -312,6 +365,11 @@ async function extractAndStoreBinaryData(response, context) {
 		}
 		return img;
 	}));
+	const rawSvgPreview = await storeRawSvgOutputPreview(response.output, next.metadata || response.metadata, storeOnce, context);
+	if (rawSvgPreview.mutated) {
+		next.metadata = rawSvgPreview.metadata;
+		mutated = true;
+	}
 	const turns = response.turns;
 	if (Array.isArray(turns)) next.turns = await Promise.all(turns.map(async (turn, idx) => {
 		if (turn?.audio?.data && typeof turn.audio.data === "string") {
@@ -429,4 +487,4 @@ async function recordExistingBlobReferences(value, context, location) {
 //#endregion
 export { isBlobStorageEnabled as n, shouldAttemptRemoteBlobUpload as r, extractAndStoreBinaryData as t };
 
-//# sourceMappingURL=extractor-B1hELWwJ.js.map
+//# sourceMappingURL=extractor-BIqe0lMN.js.map

package/dist/src/{extractor-_E41W0Fl.cjs → extractor-CUVurbDW.cjs}

@@ -1,7 +1,10 @@
+require("./rolldown-runtime-D_mwlA32.cjs");
 const require_logger = require("./logger-DPITYeiW.cjs");
-const require_fetch = require("./fetch-OePjzUmA.cjs");
-const require_accounts = require("./accounts-5Sf1PCNo.cjs");
-const require_blobs = require("./blobs-DQ6pIlJQ.cjs");
+const require_fetch = require("./fetch-C7yU9CNr.cjs");
+const require_accounts = require("./accounts-C6NiTdum.cjs");
+const require_createHash = require("./createHash-CF-N1FBc.cjs");
+const require_blobs = require("./blobs-BcpoigIU.cjs");
+let fast_xml_parser = require("fast-xml-parser");
 //#region src/blobs/remoteUpload.ts
 function buildRemoteUploadTarget() {
 	if (require_logger.getEnvBool("PROMPTFOO_DISABLE_SHARING")) return null;
@@ -28,7 +31,7 @@ async function uploadBlobRemote(buffer, mimeType, context) {
 	const target = buildRemoteUploadTarget();
 	if (!target) return null;
 	try {
-		const { fetchWithProxy } = await Promise.resolve().then(() => require("./fetch-OePjzUmA.cjs")).then((n) => n.fetch_exports);
+		const { fetchWithProxy } = await Promise.resolve().then(() => require("./fetch-C7yU9CNr.cjs")).then((n) => n.fetch_exports);
 		const response = await fetchWithProxy(target.url, {
 			method: "POST",
 			headers: {
@@ -86,9 +89,9 @@ function extractBase64(value) {
 		return null;
 	}
 }
-function shouldExternalize(buffer) {
+function shouldExternalize(buffer, minSizeBytes = require_blobs.BLOB_MIN_SIZE) {
 	const size = buffer.length;
-	return size >= 1024 && size <= 52428800;
+	return size >= minSizeBytes && size <= 52428800;
 }
 function getKindFromMimeType(mimeType) {
 	return mimeType.startsWith("audio/") ? "audio" : "image";
@@ -135,9 +138,9 @@ function parseBinary(base64OrDataUrl, defaultMimeType) {
 		return null;
 	}
 }
-async function maybeStore(base64OrDataUrl, defaultMimeType, context, location, kind) {
+async function maybeStore(base64OrDataUrl, defaultMimeType, context, location, kind, minSizeBytes = require_blobs.BLOB_MIN_SIZE) {
 	const parsed = parseBinary(base64OrDataUrl, defaultMimeType);
-	if (!parsed || !shouldExternalize(parsed.buffer)) return null;
+	if (!parsed || !shouldExternalize(parsed.buffer, minSizeBytes)) return null;
 	if (!isBlobStorageEnabled()) return null;
 	const mimeType = parsed.mimeType || "application/octet-stream";
 	const { ref } = await require_blobs.storeBlob(parsed.buffer, mimeType, {
@@ -161,13 +164,13 @@ async function maybeStore(base64OrDataUrl, defaultMimeType, context, location, k
 }
 function createStoreOnce(blobContext) {
 	const cache = /* @__PURE__ */ new Map();
-	return async (base64OrDataUrl, defaultMimeType, location, kind) => {
+	return async (base64OrDataUrl, defaultMimeType, location, kind, minSizeBytes) => {
 		const parsed = parseBinary(base64OrDataUrl, defaultMimeType);
-		if (!parsed || !shouldExternalize(parsed.buffer)) return null;
+		if (!parsed || !shouldExternalize(parsed.buffer, minSizeBytes)) return null;
 		const cacheKey = `${kind}:${parsed.buffer.toString("base64")}`;
 		const existing = cache.get(cacheKey);
 		if (existing) return existing;
-		const pendingStore = maybeStore(base64OrDataUrl, defaultMimeType, blobContext, location, kind);
+		const pendingStore = maybeStore(base64OrDataUrl, defaultMimeType, blobContext, location, kind, minSizeBytes);
 		cache.set(cacheKey, pendingStore);
 		try {
 			const stored = await pendingStore;
@@ -179,6 +182,57 @@ function createStoreOnce(blobContext) {
 		}
 	};
 }
+function getRawSvgOutputPreview(output) {
+	const trimmed = output.trim();
+	if (!trimmed.startsWith("<")) return null;
+	if (fast_xml_parser.XMLValidator.validate(trimmed) !== true) return null;
+	try {
+		const parsed = new fast_xml_parser.XMLParser({ ignoreAttributes: false }).parse(trimmed);
+		if (!parsed || typeof parsed !== "object" || Array.isArray(parsed) || !("svg" in parsed)) return null;
+	} catch {
+		return null;
+	}
+	const buffer = Buffer.from(trimmed, "utf8");
+	return {
+		dataUrl: `data:image/svg+xml;base64,${buffer.toString("base64")}`,
+		uri: `${require_blobs.BLOB_SCHEME}${require_createHash.sha256(buffer)}`
+	};
+}
+function appendMetadataBlobUri(metadata, uri) {
+	const existingBlobUris = Array.isArray(metadata?.blobUris) ? metadata.blobUris.filter((value) => typeof value === "string") : [];
+	return {
+		...metadata || {},
+		blobUris: [...new Set([...existingBlobUris, uri])]
+	};
+}
+async function storeRawSvgOutputPreview(output, metadata, storeOnce, context) {
+	if (typeof output !== "string") return {
+		metadata,
+		mutated: false
+	};
+	const preview = getRawSvgOutputPreview(output);
+	if (!preview) return {
+		metadata,
+		mutated: false
+	};
+	if ((Array.isArray(metadata?.blobUris) ? metadata.blobUris.filter((value) => typeof value === "string") : []).includes(preview.uri)) return {
+		metadata,
+		mutated: false
+	};
+	const stored = await storeOnce(preview.dataUrl, "image/svg+xml", "response.output", "image", 0);
+	if (!stored) return {
+		metadata,
+		mutated: false
+	};
+	require_logger.logger.debug("[BlobExtractor] Stored raw SVG output blob", {
+		...context,
+		hash: stored.hash
+	});
+	return {
+		metadata: appendMetadataBlobUri(metadata, stored.uri),
+		mutated: true
+	};
+}
 async function externalizeDataUrls(value, storeOnce, location) {
 	if (typeof value === "string") {
 		if (!isDataUrl(value)) return {
@@ -312,6 +366,11 @@ async function extractAndStoreBinaryData(response, context) {
 		}
 		return img;
 	}));
+	const rawSvgPreview = await storeRawSvgOutputPreview(response.output, next.metadata || response.metadata, storeOnce, context);
+	if (rawSvgPreview.mutated) {
+		next.metadata = rawSvgPreview.metadata;
+		mutated = true;
+	}
 	const turns = response.turns;
 	if (Array.isArray(turns)) next.turns = await Promise.all(turns.map(async (turn, idx) => {
 		if (turn?.audio?.data && typeof turn.audio.data === "string") {
@@ -446,4 +505,4 @@ Object.defineProperty(exports, "shouldAttemptRemoteBlobUpload", {
 	}
 });
 
-//# sourceMappingURL=extractor-_E41W0Fl.cjs.map
+//# sourceMappingURL=extractor-CUVurbDW.cjs.map