promptfoo 0.121.10 → 0.121.11

package/dist/src/{extractor-DV6WBfAi.js → extractor-DOhjGS0m.js}

@@ -1,8 +1,10 @@
 #!/usr/bin/env node
 import { E as getEnvBool, c as logger } from "./logger-DNHkR_3S.js";
-import { l as isLoggedIntoCloud } from "./accounts-DbmMSaOW.js";
-import { a as cloudConfig } from "./cloud-DJB9yPQB.js";
-import { i as storeBlob, r as recordBlobReference } from "./blobs-CqhgM_fa.js";
+import { l as isLoggedIntoCloud } from "./accounts-C06-4dXo.js";
+import { a as cloudConfig } from "./cloud-DKKGeJBV.js";
+import { n as sha256 } from "./createHash-Ddi0OHLI.js";
+import { i as storeBlob, o as BLOB_MIN_SIZE, r as recordBlobReference, s as BLOB_SCHEME } from "./blobs-BTzPmpq-.js";
+import { XMLParser, XMLValidator } from "fast-xml-parser";
 //#region src/blobs/remoteUpload.ts
 function buildRemoteUploadTarget() {
 	if (getEnvBool("PROMPTFOO_DISABLE_SHARING")) return null;
@@ -29,7 +31,7 @@ async function uploadBlobRemote(buffer, mimeType, context) {
 	const target = buildRemoteUploadTarget();
 	if (!target) return null;
 	try {
-		const { fetchWithProxy } = await import("./fetch-DAe6a2ih.js");
+		const { fetchWithProxy } = await import("./fetch-DVj4dF4V.js");
 		const response = await fetchWithProxy(target.url, {
 			method: "POST",
 			headers: {
@@ -87,9 +89,9 @@ function extractBase64(value) {
 		return null;
 	}
 }
-function shouldExternalize(buffer) {
+function shouldExternalize(buffer, minSizeBytes = BLOB_MIN_SIZE) {
 	const size = buffer.length;
-	return size >= 1024 && size <= 52428800;
+	return size >= minSizeBytes && size <= 52428800;
 }
 function getKindFromMimeType(mimeType) {
 	return mimeType.startsWith("audio/") ? "audio" : "image";
@@ -136,9 +138,9 @@ function parseBinary(base64OrDataUrl, defaultMimeType) {
 		return null;
 	}
 }
-async function maybeStore(base64OrDataUrl, defaultMimeType, context, location, kind) {
+async function maybeStore(base64OrDataUrl, defaultMimeType, context, location, kind, minSizeBytes = BLOB_MIN_SIZE) {
 	const parsed = parseBinary(base64OrDataUrl, defaultMimeType);
-	if (!parsed || !shouldExternalize(parsed.buffer)) return null;
+	if (!parsed || !shouldExternalize(parsed.buffer, minSizeBytes)) return null;
 	if (!isBlobStorageEnabled()) return null;
 	const mimeType = parsed.mimeType || "application/octet-stream";
 	const { ref } = await storeBlob(parsed.buffer, mimeType, {
@@ -162,13 +164,13 @@ async function maybeStore(base64OrDataUrl, defaultMimeType, context, location, k
 }
 function createStoreOnce(blobContext) {
 	const cache = /* @__PURE__ */ new Map();
-	return async (base64OrDataUrl, defaultMimeType, location, kind) => {
+	return async (base64OrDataUrl, defaultMimeType, location, kind, minSizeBytes) => {
 		const parsed = parseBinary(base64OrDataUrl, defaultMimeType);
-		if (!parsed || !shouldExternalize(parsed.buffer)) return null;
+		if (!parsed || !shouldExternalize(parsed.buffer, minSizeBytes)) return null;
 		const cacheKey = `${kind}:${parsed.buffer.toString("base64")}`;
 		const existing = cache.get(cacheKey);
 		if (existing) return existing;
-		const pendingStore = maybeStore(base64OrDataUrl, defaultMimeType, blobContext, location, kind);
+		const pendingStore = maybeStore(base64OrDataUrl, defaultMimeType, blobContext, location, kind, minSizeBytes);
 		cache.set(cacheKey, pendingStore);
 		try {
 			const stored = await pendingStore;
@@ -180,6 +182,57 @@ function createStoreOnce(blobContext) {
 		}
 	};
 }
+function getRawSvgOutputPreview(output) {
+	const trimmed = output.trim();
+	if (!trimmed.startsWith("<")) return null;
+	if (XMLValidator.validate(trimmed) !== true) return null;
+	try {
+		const parsed = new XMLParser({ ignoreAttributes: false }).parse(trimmed);
+		if (!parsed || typeof parsed !== "object" || Array.isArray(parsed) || !("svg" in parsed)) return null;
+	} catch {
+		return null;
+	}
+	const buffer = Buffer.from(trimmed, "utf8");
+	return {
+		dataUrl: `data:image/svg+xml;base64,${buffer.toString("base64")}`,
+		uri: `${BLOB_SCHEME}${sha256(buffer)}`
+	};
+}
+function appendMetadataBlobUri(metadata, uri) {
+	const existingBlobUris = Array.isArray(metadata?.blobUris) ? metadata.blobUris.filter((value) => typeof value === "string") : [];
+	return {
+		...metadata || {},
+		blobUris: [...new Set([...existingBlobUris, uri])]
+	};
+}
+async function storeRawSvgOutputPreview(output, metadata, storeOnce, context) {
+	if (typeof output !== "string") return {
+		metadata,
+		mutated: false
+	};
+	const preview = getRawSvgOutputPreview(output);
+	if (!preview) return {
+		metadata,
+		mutated: false
+	};
+	if ((Array.isArray(metadata?.blobUris) ? metadata.blobUris.filter((value) => typeof value === "string") : []).includes(preview.uri)) return {
+		metadata,
+		mutated: false
+	};
+	const stored = await storeOnce(preview.dataUrl, "image/svg+xml", "response.output", "image", 0);
+	if (!stored) return {
+		metadata,
+		mutated: false
+	};
+	logger.debug("[BlobExtractor] Stored raw SVG output blob", {
+		...context,
+		hash: stored.hash
+	});
+	return {
+		metadata: appendMetadataBlobUri(metadata, stored.uri),
+		mutated: true
+	};
+}
 async function externalizeDataUrls(value, storeOnce, location) {
 	if (typeof value === "string") {
 		if (!isDataUrl(value)) return {
@@ -313,6 +366,11 @@ async function extractAndStoreBinaryData(response, context) {
 		}
 		return img;
 	}));
+	const rawSvgPreview = await storeRawSvgOutputPreview(response.output, next.metadata || response.metadata, storeOnce, context);
+	if (rawSvgPreview.mutated) {
+		next.metadata = rawSvgPreview.metadata;
+		mutated = true;
+	}
 	const turns = response.turns;
 	if (Array.isArray(turns)) next.turns = await Promise.all(turns.map(async (turn, idx) => {
 		if (turn?.audio?.data && typeof turn.audio.data === "string") {
@@ -430,4 +488,4 @@ async function recordExistingBlobReferences(value, context, location) {
 //#endregion
 export { isBlobStorageEnabled as n, shouldAttemptRemoteBlobUpload as r, extractAndStoreBinaryData as t };
 
-//# sourceMappingURL=extractor-DV6WBfAi.js.map
+//# sourceMappingURL=extractor-DOhjGS0m.js.map

package/dist/src/{extractor-ItsKh8lH.js → extractor-h1HS9f6y.js}

@@ -1,7 +1,9 @@
 import { o as logger, w as getEnvBool } from "./logger-Dz3siSzF.js";
-import { l as cloudConfig } from "./fetch-BtdsABrj.js";
-import { l as isLoggedIntoCloud } from "./accounts-ECb2xAYn.js";
-import { i as storeBlob, r as recordBlobReference } from "./blobs-Dxfq3woN.js";
+import { l as cloudConfig } from "./fetch-BErWGTUY.js";
+import { l as isLoggedIntoCloud } from "./accounts-BjCiVnil.js";
+import { n as sha256 } from "./createHash-BWWdtTvT.js";
+import { i as storeBlob, o as BLOB_MIN_SIZE, r as recordBlobReference, s as BLOB_SCHEME } from "./blobs-DQ4bXN5O.js";
+import { XMLParser, XMLValidator } from "fast-xml-parser";
 //#region src/blobs/remoteUpload.ts
 function buildRemoteUploadTarget() {
 	if (getEnvBool("PROMPTFOO_DISABLE_SHARING")) return null;
@@ -28,7 +30,7 @@ async function uploadBlobRemote(buffer, mimeType, context) {
 	const target = buildRemoteUploadTarget();
 	if (!target) return null;
 	try {
-		const { fetchWithProxy } = await import("./fetch-BtdsABrj.js").then((n) => n.i);
+		const { fetchWithProxy } = await import("./fetch-BErWGTUY.js").then((n) => n.i);
 		const response = await fetchWithProxy(target.url, {
 			method: "POST",
 			headers: {
@@ -86,9 +88,9 @@ function extractBase64(value) {
 		return null;
 	}
 }
-function shouldExternalize(buffer) {
+function shouldExternalize(buffer, minSizeBytes = BLOB_MIN_SIZE) {
 	const size = buffer.length;
-	return size >= 1024 && size <= 52428800;
+	return size >= minSizeBytes && size <= 52428800;
 }
 function getKindFromMimeType(mimeType) {
 	return mimeType.startsWith("audio/") ? "audio" : "image";
@@ -135,9 +137,9 @@ function parseBinary(base64OrDataUrl, defaultMimeType) {
 		return null;
 	}
 }
-async function maybeStore(base64OrDataUrl, defaultMimeType, context, location, kind) {
+async function maybeStore(base64OrDataUrl, defaultMimeType, context, location, kind, minSizeBytes = BLOB_MIN_SIZE) {
 	const parsed = parseBinary(base64OrDataUrl, defaultMimeType);
-	if (!parsed || !shouldExternalize(parsed.buffer)) return null;
+	if (!parsed || !shouldExternalize(parsed.buffer, minSizeBytes)) return null;
 	if (!isBlobStorageEnabled()) return null;
 	const mimeType = parsed.mimeType || "application/octet-stream";
 	const { ref } = await storeBlob(parsed.buffer, mimeType, {
@@ -161,13 +163,13 @@ async function maybeStore(base64OrDataUrl, defaultMimeType, context, location, k
 }
 function createStoreOnce(blobContext) {
 	const cache = /* @__PURE__ */ new Map();
-	return async (base64OrDataUrl, defaultMimeType, location, kind) => {
+	return async (base64OrDataUrl, defaultMimeType, location, kind, minSizeBytes) => {
 		const parsed = parseBinary(base64OrDataUrl, defaultMimeType);
-		if (!parsed || !shouldExternalize(parsed.buffer)) return null;
+		if (!parsed || !shouldExternalize(parsed.buffer, minSizeBytes)) return null;
 		const cacheKey = `${kind}:${parsed.buffer.toString("base64")}`;
 		const existing = cache.get(cacheKey);
 		if (existing) return existing;
-		const pendingStore = maybeStore(base64OrDataUrl, defaultMimeType, blobContext, location, kind);
+		const pendingStore = maybeStore(base64OrDataUrl, defaultMimeType, blobContext, location, kind, minSizeBytes);
 		cache.set(cacheKey, pendingStore);
 		try {
 			const stored = await pendingStore;
@@ -179,6 +181,57 @@ function createStoreOnce(blobContext) {
 		}
 	};
 }
+function getRawSvgOutputPreview(output) {
+	const trimmed = output.trim();
+	if (!trimmed.startsWith("<")) return null;
+	if (XMLValidator.validate(trimmed) !== true) return null;
+	try {
+		const parsed = new XMLParser({ ignoreAttributes: false }).parse(trimmed);
+		if (!parsed || typeof parsed !== "object" || Array.isArray(parsed) || !("svg" in parsed)) return null;
+	} catch {
+		return null;
+	}
+	const buffer = Buffer.from(trimmed, "utf8");
+	return {
+		dataUrl: `data:image/svg+xml;base64,${buffer.toString("base64")}`,
+		uri: `${BLOB_SCHEME}${sha256(buffer)}`
+	};
+}
+function appendMetadataBlobUri(metadata, uri) {
+	const existingBlobUris = Array.isArray(metadata?.blobUris) ? metadata.blobUris.filter((value) => typeof value === "string") : [];
+	return {
+		...metadata || {},
+		blobUris: [...new Set([...existingBlobUris, uri])]
+	};
+}
+async function storeRawSvgOutputPreview(output, metadata, storeOnce, context) {
+	if (typeof output !== "string") return {
+		metadata,
+		mutated: false
+	};
+	const preview = getRawSvgOutputPreview(output);
+	if (!preview) return {
+		metadata,
+		mutated: false
+	};
+	if ((Array.isArray(metadata?.blobUris) ? metadata.blobUris.filter((value) => typeof value === "string") : []).includes(preview.uri)) return {
+		metadata,
+		mutated: false
+	};
+	const stored = await storeOnce(preview.dataUrl, "image/svg+xml", "response.output", "image", 0);
+	if (!stored) return {
+		metadata,
+		mutated: false
+	};
+	logger.debug("[BlobExtractor] Stored raw SVG output blob", {
+		...context,
+		hash: stored.hash
+	});
+	return {
+		metadata: appendMetadataBlobUri(metadata, stored.uri),
+		mutated: true
+	};
+}
 async function externalizeDataUrls(value, storeOnce, location) {
 	if (typeof value === "string") {
 		if (!isDataUrl(value)) return {
@@ -312,6 +365,11 @@ async function extractAndStoreBinaryData(response, context) {
 		}
 		return img;
 	}));
+	const rawSvgPreview = await storeRawSvgOutputPreview(response.output, next.metadata || response.metadata, storeOnce, context);
+	if (rawSvgPreview.mutated) {
+		next.metadata = rawSvgPreview.metadata;
+		mutated = true;
+	}
 	const turns = response.turns;
 	if (Array.isArray(turns)) next.turns = await Promise.all(turns.map(async (turn, idx) => {
 		if (turn?.audio?.data && typeof turn.audio.data === "string") {
@@ -429,4 +487,4 @@ async function recordExistingBlobReferences(value, context, location) {
 //#endregion
 export { isBlobStorageEnabled as n, shouldAttemptRemoteBlobUpload as r, extractAndStoreBinaryData as t };
 
-//# sourceMappingURL=extractor-ItsKh8lH.js.map
+//# sourceMappingURL=extractor-h1HS9f6y.js.map

package/dist/src/{fetch-BBa1Lxgp.js → fetch-B1Rb0VdL.js}

@@ -1,7 +1,7 @@
 #!/usr/bin/env node
 import { E as getEnvBool, O as getEnvInt, P as state, c as logger, g as sanitizeUrl, k as getEnvString, s as logRequestResponse } from "./logger-DNHkR_3S.js";
 import { t as invariant } from "./invariant-BtWWVVhl.js";
-import { a as cloudConfig, n as CLOUD_API_HOST } from "./cloud-DJB9yPQB.js";
+import { a as cloudConfig, n as CLOUD_API_HOST } from "./cloud-DKKGeJBV.js";
 import { AsyncLocalStorage } from "node:async_hooks";
 import path from "path";
 import yaml from "js-yaml";
@@ -23,7 +23,7 @@ const HUMAN_ASSERTION_TYPE = "human";
 * Application version from package.json.
 * Injected at build time, or read from npm environment in development.
 */
-const VERSION = "0.121.10";
+const VERSION = "0.121.11";
 /**
 * PostHog analytics key.
 * Only populated during production builds via PROMPTFOO_POSTHOG_KEY env var.
@@ -1131,4 +1131,4 @@ async function fetchWithRetries(url, options = {}, timeout, maxRetries) {
 //#endregion
 export { openaiToolChoiceToBedrock as A, TERMINAL_MAX_WIDTH as B, parseRetryAfter as C, isOpenAIToolArray as D, getRequestTimeoutMs as E, transformTools as F, POSTHOG_KEY as G, getDefaultShareViewBaseUrl as H, CLOUD_PROVIDER_PREFIX as I, HUMAN_ASSERTION_TYPE as J, VERSION as K, CONSENT_ENDPOINT as L, parseChatPrompt as M, toTitleCase as N, isOpenAIToolChoice as O, transformToolChoice as P, EVENTS_ENDPOINT as R, parseRateLimitHeaders as S, calculateCost as T, getShareApiBaseUrl as U, getDefaultPort as V, getShareViewBaseUrl as W, isNonTransientHttpStatus as _, fetchWithTimeout as a, sleep as b, isRateLimited as c, isPromptfooCloudApiHost as d, HttpRateLimitError as f, isHttpRateLimitError as g, formatRateLimitErrorMessage as h, fetchWithRetries as i, openaiToolsToBedrock as j, isPromptfooSampleTarget as k, isTransientError as l, extractRateLimitErrorCode as m, computeRateLimitWaitMs as n, getFetchWithProxyHeaders as o, NON_TRANSIENT_HTTP_STATUSES as p, FILE_METADATA_KEY as q, fetchWithProxy as r, handleRateLimit as s, clearAgentCache as t, withFetchRetryContext as u, isTransientConnectionError as v, LONG_RUNNING_MODEL_TIMEOUT_MS as w, sleepWithAbort as x, getCurrentTimestamp as y, R_ENDPOINT as z };
 
-//# sourceMappingURL=fetch-BBa1Lxgp.js.map
+//# sourceMappingURL=fetch-B1Rb0VdL.js.map

package/dist/src/{fetch-BtdsABrj.js → fetch-BErWGTUY.js}

@@ -24,7 +24,7 @@ const HUMAN_ASSERTION_TYPE = "human";
 * Application version from package.json.
 * Injected at build time, or read from npm environment in development.
 */
-const VERSION = "0.121.10";
+const VERSION = "0.121.11";
 /**
 * PostHog analytics key.
 * Only populated during production builds via PROMPTFOO_POSTHOG_KEY env var.
@@ -1331,4 +1331,4 @@ async function fetchWithRetries(url, options = {}, timeout, maxRetries) {
 //#endregion
 export { openaiToolChoiceToBedrock as A, TERMINAL_MAX_WIDTH as B, parseRetryAfter as C, isOpenAIToolArray as D, getRequestTimeoutMs as E, transformTools as F, POSTHOG_KEY as G, getDefaultShareViewBaseUrl as H, CLOUD_PROVIDER_PREFIX as I, HUMAN_ASSERTION_TYPE as J, VERSION as K, CONSENT_ENDPOINT as L, parseChatPrompt as M, toTitleCase as N, isOpenAIToolChoice as O, transformToolChoice as P, EVENTS_ENDPOINT as R, parseRateLimitHeaders as S, calculateCost as T, getShareApiBaseUrl as U, getDefaultPort as V, getShareViewBaseUrl as W, isHttpRateLimitError as _, getFetchWithProxyHeaders as a, getCurrentTimestamp as b, CloudConfig as c, writeGlobalConfig as d, writeGlobalConfigPartial as f, formatRateLimitErrorMessage as g, extractRateLimitErrorCode as h, fetch_exports as i, openaiToolsToBedrock as j, isPromptfooSampleTarget as k, cloudConfig as l, NON_TRANSIENT_HTTP_STATUSES as m, fetchWithRetries as n, withFetchRetryContext as o, HttpRateLimitError as p, FILE_METADATA_KEY as q, fetchWithTimeout as r, isPromptfooCloudApiHost as s, fetchWithProxy as t, readGlobalConfig as u, isNonTransientHttpStatus as v, LONG_RUNNING_MODEL_TIMEOUT_MS as w, sleep as x, isTransientConnectionError as y, R_ENDPOINT as z };
 
-//# sourceMappingURL=fetch-BtdsABrj.js.map
+//# sourceMappingURL=fetch-BErWGTUY.js.map

package/dist/src/{fetch-CFIU30wy.js → fetch-BNUDixbE.js}

@@ -24,7 +24,7 @@ const HUMAN_ASSERTION_TYPE = "human";
 * Application version from package.json.
 * Injected at build time, or read from npm environment in development.
 */
-const VERSION = "0.121.10";
+const VERSION = "0.121.11";
 /**
 * PostHog analytics key.
 * Only populated during production builds via PROMPTFOO_POSTHOG_KEY env var.
@@ -1328,4 +1328,4 @@ async function fetchWithRetries(url, options = {}, timeout, maxRetries) {
 //#endregion
 export { transformTools as A, TERMINAL_MAX_WIDTH as B, isOpenAIToolChoice as C, parseChatPrompt as D, openaiToolsToBedrock as E, writeGlobalConfigPartial as F, VERSION as G, getShareApiBaseUrl as H, CLOUD_PROVIDER_PREFIX as I, FILE_METADATA_KEY as K, CONSENT_ENDPOINT as L, cloudConfig as M, readGlobalConfig as N, toTitleCase as O, writeGlobalConfig as P, EVENTS_ENDPOINT as R, isOpenAIToolArray as S, openaiToolChoiceToBedrock as T, getShareViewBaseUrl as U, getDefaultShareViewBaseUrl as V, POSTHOG_KEY as W, parseRateLimitHeaders as _, getFetchWithProxyHeaders as a, calculateCost as b, HttpRateLimitError as c, formatRateLimitErrorMessage as d, isHttpRateLimitError as f, sleep as g, getCurrentTimestamp as h, fetch_exports as i, CloudConfig as j, transformToolChoice as k, NON_TRANSIENT_HTTP_STATUSES as l, isTransientConnectionError as m, fetchWithRetries as n, withFetchRetryContext as o, isNonTransientHttpStatus as p, HUMAN_ASSERTION_TYPE as q, fetchWithTimeout as r, isPromptfooCloudApiHost as s, fetchWithProxy as t, extractRateLimitErrorCode as u, parseRetryAfter as v, isPromptfooSampleTarget as w, getRequestTimeoutMs as x, LONG_RUNNING_MODEL_TIMEOUT_MS as y, R_ENDPOINT as z };
 
-//# sourceMappingURL=fetch-CFIU30wy.js.map
+//# sourceMappingURL=fetch-BNUDixbE.js.map

package/dist/src/{fetch-OePjzUmA.cjs → fetch-C7yU9CNr.cjs}

@@ -27,7 +27,7 @@ const HUMAN_ASSERTION_TYPE = "human";
 * Application version from package.json.
 * Injected at build time, or read from npm environment in development.
 */
-const VERSION = "0.121.10";
+const VERSION = "0.121.11";
 /**
 * PostHog analytics key.
 * Only populated during production builds via PROMPTFOO_POSTHOG_KEY env var.
@@ -1612,4 +1612,4 @@ Object.defineProperty(exports, "writeGlobalConfigPartial", {
 	}
 });
 
-//# sourceMappingURL=fetch-OePjzUmA.cjs.map
+//# sourceMappingURL=fetch-C7yU9CNr.cjs.map

package/dist/src/fetch-DVj4dF4V.js

@@ -0,0 +1,3 @@
+#!/usr/bin/env node
+import { r as fetchWithProxy } from "./fetch-B1Rb0VdL.js";
+export { fetchWithProxy };

package/dist/src/{goat-kpeOOE-K.js → goat-BnB1zGir.js}

@@ -1,18 +1,18 @@
 import { b as safeJsonStringify, o as logger } from "./logger-2ZstxKQf.js";
 import { t as invariant } from "./invariant-Ddh24eXh.js";
-import { G as VERSION, g as sleep, t as fetchWithProxy } from "./fetch-CFIU30wy.js";
-import { a as getUserEmail, s as isLoggedIntoCloud } from "./accounts-D4iTShVh.js";
-import { B as getNunjucksEngine } from "./util-CnqjxsAd.js";
-import { r as getRemoteGenerationUrl, s as neverGenerateRemote } from "./remoteGeneration--4NPfyL-.js";
+import { G as VERSION, g as sleep, t as fetchWithProxy } from "./fetch-BNUDixbE.js";
+import { a as getUserEmail, s as isLoggedIntoCloud } from "./accounts-BGs8AMhG.js";
+import { B as getNunjucksEngine } from "./util-BTAyQmdN.js";
+import { r as getRemoteGenerationUrl, s as neverGenerateRemote } from "./remoteGeneration-DuxbIn94.js";
 import { a as createEmptyTokenUsage, n as accumulateResponseTokenUsage } from "./tokenUsageUtils-CmnQ0G2m.js";
-import { a as extractPromptFromTags, o as getSessionId, r as extractInputVarsFromPrompt } from "./util-CpS555tc2.js";
-import { a as assertRemoteMaterializationHandled, c as isRemoteMaterializationUpgradeError, s as buildRemoteMaterializedInputVariables } from "./promptfoo-x112UyE1.js";
-import { o as getGraderAssertionValue, p as tryUnblocking, s as getLastMessageContent, t as buildGraderResultAssertion } from "./shared-B3IkjkNV.js";
+import { a as extractPromptFromTags, o as getSessionId, r as extractInputVarsFromPrompt } from "./util-s0NU5XTE2.js";
+import { a as assertRemoteMaterializationHandled, c as isRemoteMaterializationUpgradeError, s as buildRemoteMaterializedInputVariables } from "./promptfoo-DhGR7OQn.js";
+import { o as getGraderAssertionValue, p as tryUnblocking, s as getLastMessageContent, t as buildGraderResultAssertion } from "./shared-DHDR-oS4.js";
 import { i as throwIfTargetPromptExceedsMaxChars } from "./promptLength-DwFSv-C5.js";
-import { n as checkExfilTracking } from "./indirectWebPwn-BRhR4Dk9.js";
-import { n as renderPrompt } from "./evaluatorHelpers-RQo-m90b.js";
-import { t as Strategies } from "./strategies-BuIPOuYL.js";
-import { a as fetchTraceContext, i as extractTraceIdFromTraceparent, n as formatTraceForMetadata, r as formatTraceSummary, t as resolveTracingOptions } from "./tracingOptions-Bi477wlC.js";
+import { n as checkExfilTracking } from "./indirectWebPwn-BnrxhAAL.js";
+import { n as renderPrompt } from "./evaluatorHelpers-DTtYkrGO.js";
+import { t as Strategies } from "./strategies-DE3UN_VH.js";
+import { a as fetchTraceContext, i as extractTraceIdFromTraceparent, n as formatTraceForMetadata, r as formatTraceSummary, t as resolveTracingOptions } from "./tracingOptions-xCBDF3nM.js";
 import { t as applyRuntimeTransforms } from "./runtimeTransform-D0y6T_QV.js";
 import { i as getGoalRubric } from "./prompts-CuZuRe7J.js";
 import chalk from "chalk";
@@ -79,7 +79,7 @@ var GoatProvider = class {
 		let assertToUse;
 		let graderPassed;
 		let storedGraderResult;
-		const { getGraderById } = await import("./graders-CY4Ft8A9.js").then((n) => n.r);
+		const { getGraderById } = await import("./graders-Z5qzvpuI.js").then((n) => n.r);
 		let test;
 		if (context?.test) {
 			test = context?.test;
@@ -463,4 +463,4 @@ var GoatProvider = class {
 //#endregion
 export { GoatProvider as default };
 
-//# sourceMappingURL=goat-kpeOOE-K.js.map
+//# sourceMappingURL=goat-BnB1zGir.js.map

package/dist/src/{goat-Coh2te0S.cjs → goat-C1RkjWv1.cjs}

@@ -1,19 +1,19 @@
 const require_rolldown_runtime = require("./rolldown-runtime-D_mwlA32.cjs");
 const require_logger = require("./logger-DPITYeiW.cjs");
 const require_invariant = require("./invariant-QtnLD03y.cjs");
-const require_fetch = require("./fetch-OePjzUmA.cjs");
-const require_accounts = require("./accounts-5Sf1PCNo.cjs");
-const require_util = require("./util-DJFl6JZF.cjs");
-const require_remoteGeneration = require("./remoteGeneration-B0K7iojS.cjs");
+const require_fetch = require("./fetch-C7yU9CNr.cjs");
+const require_accounts = require("./accounts-C6NiTdum.cjs");
+const require_util = require("./util-tFLdovkA.cjs");
+const require_remoteGeneration = require("./remoteGeneration-oSmtgmBY.cjs");
 const require_tokenUsageUtils = require("./tokenUsageUtils-_B-P8IAi.cjs");
-const require_util$1 = require("./util-DEMRW89M2.cjs");
-const require_promptfoo = require("./promptfoo-B9KuhWM0.cjs");
-const require_shared = require("./shared-DY3IzJlT.cjs");
+const require_util$1 = require("./util-jOGJmpWE2.cjs");
+const require_promptfoo = require("./promptfoo-Dtw3i6YK.cjs");
+const require_shared = require("./shared-D3PxrdDh.cjs");
 const require_promptLength = require("./promptLength-CIZ9JExv.cjs");
-const require_indirectWebPwn = require("./indirectWebPwn-DcKDbCn_.cjs");
-const require_evaluatorHelpers = require("./evaluatorHelpers-BVvi8tWH.cjs");
-const require_strategies = require("./strategies-2Q_uZxBs.cjs");
-const require_tracingOptions = require("./tracingOptions-CYCHrrQQ.cjs");
+const require_indirectWebPwn = require("./indirectWebPwn-CDo9rC06.cjs");
+const require_evaluatorHelpers = require("./evaluatorHelpers-CaaC50RB.cjs");
+const require_strategies = require("./strategies-B6Y6G9UV.cjs");
+const require_tracingOptions = require("./tracingOptions-DGai7e5l.cjs");
 const require_runtimeTransform = require("./runtimeTransform-BrI_mskc.cjs");
 const require_prompts = require("./prompts-CfS_cxI5.cjs");
 let chalk = require("chalk");
@@ -82,7 +82,7 @@ var GoatProvider = class {
 		let assertToUse;
 		let graderPassed;
 		let storedGraderResult;
-		const { getGraderById } = await Promise.resolve().then(() => require("./graders-CCrX0NG3.cjs")).then((n) => n.graders_exports);
+		const { getGraderById } = await Promise.resolve().then(() => require("./graders-t_K-nZ_X.cjs")).then((n) => n.graders_exports);
 		let test;
 		if (context?.test) {
 			test = context?.test;
@@ -466,4 +466,4 @@ var GoatProvider = class {
 //#endregion
 exports.default = GoatProvider;
 
-//# sourceMappingURL=goat-Coh2te0S.cjs.map
+//# sourceMappingURL=goat-C1RkjWv1.cjs.map

package/dist/src/{goat-BCFeW4zk.js → goat-CNpA_tT8.js}

@@ -1,17 +1,17 @@
 import { b as safeJsonStringify, o as logger } from "./logger-Dz3siSzF.js";
-import { K as VERSION, t as fetchWithProxy, x as sleep } from "./fetch-BtdsABrj.js";
+import { K as VERSION, t as fetchWithProxy, x as sleep } from "./fetch-BErWGTUY.js";
 import { t as invariant } from "./invariant-vgHWClmd.js";
-import { l as isLoggedIntoCloud, s as getUserEmail } from "./accounts-ECb2xAYn.js";
-import { K as getNunjucksEngine } from "./util-DJLHqGS5.js";
-import { r as getRemoteGenerationUrl, s as neverGenerateRemote } from "./remoteGeneration-DiAuGk9s.js";
+import { l as isLoggedIntoCloud, s as getUserEmail } from "./accounts-BjCiVnil.js";
+import { K as getNunjucksEngine } from "./util-Ci_vm5u1.js";
+import { r as getRemoteGenerationUrl, s as neverGenerateRemote } from "./remoteGeneration-wvnQpzN1.js";
 import { a as createEmptyTokenUsage, n as accumulateResponseTokenUsage } from "./tokenUsageUtils-DFcAf2Fd.js";
-import { a as extractPromptFromTags, o as getSessionId, r as extractInputVarsFromPrompt } from "./util-BIYgspDB.js";
-import { a as assertRemoteMaterializationHandled, c as isRemoteMaterializationUpgradeError, s as buildRemoteMaterializedInputVariables } from "./promptfoo-C4g2Lx9t.js";
-import { o as getGraderAssertionValue, p as tryUnblocking, s as getLastMessageContent, t as buildGraderResultAssertion } from "./shared-D_0FXjLS.js";
+import { a as extractPromptFromTags, o as getSessionId, r as extractInputVarsFromPrompt } from "./util-CKQ_tuWG.js";
+import { a as assertRemoteMaterializationHandled, c as isRemoteMaterializationUpgradeError, s as buildRemoteMaterializedInputVariables } from "./promptfoo-CkDcgxiW.js";
+import { o as getGraderAssertionValue, p as tryUnblocking, s as getLastMessageContent, t as buildGraderResultAssertion } from "./shared-CnrYC0Dk.js";
 import { i as throwIfTargetPromptExceedsMaxChars } from "./promptLength-Bx76P6F9.js";
-import { n as checkExfilTracking } from "./indirectWebPwn-BWGexuih.js";
+import { n as checkExfilTracking } from "./indirectWebPwn-C779yDi6.js";
 import { n as renderPrompt, t as Strategies } from "./server/index.js";
-import { a as fetchTraceContext, i as extractTraceIdFromTraceparent, n as formatTraceForMetadata, r as formatTraceSummary, t as resolveTracingOptions } from "./tracingOptions-CLKNUvF4.js";
+import { a as fetchTraceContext, i as extractTraceIdFromTraceparent, n as formatTraceForMetadata, r as formatTraceSummary, t as resolveTracingOptions } from "./tracingOptions-DL2vOmYS.js";
 import { t as applyRuntimeTransforms } from "./runtimeTransform-BbRV7Rrl.js";
 import { i as getGoalRubric } from "./prompts-k5KpuLAB.js";
 import chalk from "chalk";
@@ -78,7 +78,7 @@ var GoatProvider = class {
 		let assertToUse;
 		let graderPassed;
 		let storedGraderResult;
-		const { getGraderById } = await import("./graders-BrLOEmxX.js").then((n) => n.n);
+		const { getGraderById } = await import("./graders-CO5drf0x.js").then((n) => n.n);
 		let test;
 		if (context?.test) {
 			test = context?.test;
@@ -462,4 +462,4 @@ var GoatProvider = class {
 //#endregion
 export { GoatProvider as default };
 
-//# sourceMappingURL=goat-BCFeW4zk.js.map
+//# sourceMappingURL=goat-CNpA_tT8.js.map

package/dist/src/{goat-Dr2YGfiG.js → goat-ZRARvAD5.js}

@@ -1,19 +1,19 @@
 #!/usr/bin/env node
 import { S as safeJsonStringify, c as logger } from "./logger-DNHkR_3S.js";
-import { K as VERSION, b as sleep, r as fetchWithProxy } from "./fetch-BBa1Lxgp.js";
+import { K as VERSION, b as sleep, r as fetchWithProxy } from "./fetch-B1Rb0VdL.js";
 import { t as invariant } from "./invariant-BtWWVVhl.js";
-import { l as isLoggedIntoCloud, s as getUserEmail } from "./accounts-DbmMSaOW.js";
-import { Y as getNunjucksEngine } from "./util-DHG9r5sC.js";
-import { r as getRemoteGenerationUrl, s as neverGenerateRemote } from "./remoteGeneration-BlwssRVS.js";
+import { l as isLoggedIntoCloud, s as getUserEmail } from "./accounts-C06-4dXo.js";
+import { Y as getNunjucksEngine } from "./util-CcW4TZSn.js";
+import { r as getRemoteGenerationUrl, s as neverGenerateRemote } from "./remoteGeneration-CW6zHjQx.js";
 import { a as createEmptyTokenUsage, n as accumulateResponseTokenUsage } from "./tokenUsageUtils-kXKUpxyr.js";
-import { a as extractPromptFromTags, o as getSessionId, r as extractInputVarsFromPrompt } from "./util-16aq4oHW.js";
-import { a as assertRemoteMaterializationHandled, c as isRemoteMaterializationUpgradeError, s as buildRemoteMaterializedInputVariables } from "./promptfoo-DdsPYNm7.js";
-import { o as getGraderAssertionValue, p as tryUnblocking, s as getLastMessageContent, t as buildGraderResultAssertion } from "./shared-BavzGCp0.js";
+import { a as extractPromptFromTags, o as getSessionId, r as extractInputVarsFromPrompt } from "./util-CD4LxH5h.js";
+import { a as assertRemoteMaterializationHandled, c as isRemoteMaterializationUpgradeError, s as buildRemoteMaterializedInputVariables } from "./promptfoo-De6vQq9F.js";
+import { o as getGraderAssertionValue, p as tryUnblocking, s as getLastMessageContent, t as buildGraderResultAssertion } from "./shared-C-G3NQ1-.js";
 import { i as throwIfTargetPromptExceedsMaxChars } from "./promptLength-DYfhyE9j.js";
-import { n as checkExfilTracking } from "./indirectWebPwn-A16xYmav.js";
-import { n as renderPrompt } from "./evaluatorHelpers-C6-aKnJe.js";
-import { t as Strategies } from "./strategies-DVDC487a.js";
-import { a as fetchTraceContext, i as extractTraceIdFromTraceparent, n as formatTraceForMetadata, r as formatTraceSummary, t as resolveTracingOptions } from "./tracingOptions-CZpFCwyr.js";
+import { n as checkExfilTracking } from "./indirectWebPwn-BbFMiv79.js";
+import { n as renderPrompt } from "./evaluatorHelpers-CoLW5rKU.js";
+import { t as Strategies } from "./strategies-CaefFyHy.js";
+import { a as fetchTraceContext, i as extractTraceIdFromTraceparent, n as formatTraceForMetadata, r as formatTraceSummary, t as resolveTracingOptions } from "./tracingOptions-e_c5TlHv.js";
 import { t as applyRuntimeTransforms } from "./runtimeTransform-DjKoaR8A.js";
 import { i as getGoalRubric } from "./prompts-Co6vCI4z.js";
 import chalk from "chalk";
@@ -80,7 +80,7 @@ var GoatProvider = class {
 		let assertToUse;
 		let graderPassed;
 		let storedGraderResult;
-		const { getGraderById } = await import("./graders-BZe_w0E4.js");
+		const { getGraderById } = await import("./graders-CA3QVnK3.js");
 		let test;
 		if (context?.test) {
 			test = context?.test;
@@ -464,4 +464,4 @@ var GoatProvider = class {
 //#endregion
 export { GoatProvider as default };
 
-//# sourceMappingURL=goat-Dr2YGfiG.js.map
+//# sourceMappingURL=goat-ZRARvAD5.js.map

package/dist/src/graders-CA3QVnK3.js

@@ -0,0 +1,3 @@
+#!/usr/bin/env node
+import { n as getGraderById } from "./graders-CpUlNVnh.js";
+export { getGraderById };

package/dist/src/{graders-BrLOEmxX.js → graders-CO5drf0x.js}

@@ -1,31 +1,31 @@
 import { t as __exportAll } from "./rolldown-runtime-B_-DWIq7.js";
 import { A as isCI, C as state, D as getEnvString, b as safeJsonStringify, g as extractJsonObjects, h as extractFirstJsonObject, o as logger, w as getEnvBool } from "./logger-Dz3siSzF.js";
-import { E as getRequestTimeoutMs, r as fetchWithTimeout, t as fetchWithProxy, x as sleep } from "./fetch-BtdsABrj.js";
+import { E as getRequestTimeoutMs, r as fetchWithTimeout, t as fetchWithProxy, x as sleep } from "./fetch-BErWGTUY.js";
 import { t as invariant } from "./invariant-vgHWClmd.js";
-import { s as getUserEmail } from "./accounts-ECb2xAYn.js";
+import { s as getUserEmail } from "./accounts-BjCiVnil.js";
 import { r as importModule } from "./esm-DGM47-P2.js";
-import { A as maybeLoadConfigFromExternalFile, B as parseFileUrl, F as parsePathOrGlob, K as getNunjucksEngine, O as getNunjucksEngineForFilePath, P as maybeLoadToolsFromExternalFile, W as extractVariablesFromTemplate, j as maybeLoadFromExternalFile } from "./util-DJLHqGS5.js";
+import { A as maybeLoadConfigFromExternalFile, B as parseFileUrl, F as parsePathOrGlob, K as getNunjucksEngine, O as getNunjucksEngineForFilePath, P as maybeLoadToolsFromExternalFile, W as extractVariablesFromTemplate, j as maybeLoadFromExternalFile } from "./util-Ci_vm5u1.js";
 import { n as sha256 } from "./createHash-BWWdtTvT.js";
-import { i as getCache, o as isCacheEnabled, r as fetchWithCache } from "./cache-Z4cklSHj.js";
+import { i as getCache, o as isCacheEnabled, r as fetchWithCache } from "./cache-WT_2CYA-.js";
 import { S as PolicyObjectSchema, T as isValidReusablePolicyId, _t as PromptSchema, gt as CODING_AGENT_PLUGIN_DISPLAY_NAMES, ht as CODING_AGENT_PLUGIN_DESCRIPTIONS, j as MULTI_TURN_STRATEGIES, mt as CODING_AGENT_PLUGINS, p as isApiProvider, xt as buildInputPromptDescription } from "./types-13A1ySQW.js";
 import { o as isJavascriptFile } from "./shared-DdmcneC5.js";
 import { r as runPython } from "./pythonUtils-lzvSenw7.js";
-import { d as hasCodexDefaultCredentials, l as shouldGenerateRemote, r as getRemoteGenerationUrl, u as getCodexDefaultProviders } from "./remoteGeneration-DiAuGk9s.js";
+import { d as hasCodexDefaultCredentials, l as shouldGenerateRemote, r as getRemoteGenerationUrl, u as getCodexDefaultProviders } from "./remoteGeneration-wvnQpzN1.js";
 import { i as PROMPT_DELIMITER, n as maybeFilePath, r as normalizeInput } from "./utils-Dk9qWRdP.js";
 import { r as accumulateTokenUsage } from "./tokenUsageUtils-DFcAf2Fd.js";
-import { d as getPoliciesFromCloud } from "./storage-D_qaZpfe.js";
-import { t as OpenAiChatCompletionProvider } from "./chat-BE0BP7oZ.js";
-import { T as hasGoogleDefaultCredentials } from "./transform-CLzSt5fi.js";
-import { t as AnthropicMessagesProvider } from "./messages-DxzhBs74.js";
-import { t as OpenAiResponsesProvider } from "./responses-CTfu1ToN.js";
-import { C as getGoogleAiStudioProviders, E as AzureChatCompletionProvider, T as AzureEmbeddingProvider, _ as MistralChatCompletionProvider, b as getGoogleVertexEmbeddingProvider, c as parseScriptParts, g as OpenAiModerationProvider, h as createXAIProvider, m as XAIResponsesProvider, n as loadApiProvider, s as getFileHashes, v as MistralEmbeddingProvider, w as AzureModerationProvider, x as getGoogleVertexProviders } from "./providers-BnjFZtaq.js";
-import { t as OpenAiEmbeddingProvider } from "./embedding-LplgjBJ0.js";
+import { d as getPoliciesFromCloud } from "./storage-CK2YYGUg.js";
+import { t as OpenAiChatCompletionProvider } from "./chat-FNWhaysj.js";
+import { P as isMcpToolNameFilter, T as hasGoogleDefaultCredentials } from "./transform-D8kXC-nB.js";
+import { t as AnthropicMessagesProvider } from "./messages-aXox3d04.js";
+import { t as OpenAiResponsesProvider } from "./responses-JNrMb3zS.js";
+import { C as getGoogleAiStudioProviders, E as AzureChatCompletionProvider, T as AzureEmbeddingProvider, _ as MistralChatCompletionProvider, b as getGoogleVertexEmbeddingProvider, c as parseScriptParts, g as OpenAiModerationProvider, h as createXAIProvider, m as XAIResponsesProvider, n as loadApiProvider, s as getFileHashes, v as MistralEmbeddingProvider, w as AzureModerationProvider, x as getGoogleVertexProviders } from "./providers-gntyGa2T.js";
+import { t as OpenAiEmbeddingProvider } from "./embedding-Dy2o5RFQ.js";
 import { r as materializeInputVariablesWithMetadata } from "./inputVariables-BqNKeMvJ.js";
-import { a as extractPromptFromTags, c as isBasicRefusal, i as extractMaterializedVariablesFromJsonWithMetadata, l as isEmptyResponse, n as extractGoalFromPrompt, r as extractInputVarsFromPrompt, s as getShortPluginId, t as extractAllPromptsFromTags, u as removePrefix } from "./util-BIYgspDB.js";
-import { _ as isRateLimitWrapped, f as redteamProviderManager, g as createProviderRateLimitOptions } from "./shared-D_0FXjLS.js";
+import { a as extractPromptFromTags, c as isBasicRefusal, i as extractMaterializedVariablesFromJsonWithMetadata, l as isEmptyResponse, n as extractGoalFromPrompt, r as extractInputVarsFromPrompt, s as getShortPluginId, t as extractAllPromptsFromTags, u as removePrefix } from "./util-CKQ_tuWG.js";
+import { _ as isRateLimitWrapped, f as redteamProviderManager, g as createProviderRateLimitOptions } from "./shared-CnrYC0Dk.js";
 import { n as getGeneratedPromptOverLimit, r as getMaxCharsPerMessageModifierValue, t as MAX_CHARS_PER_MESSAGE_MODIFIER_KEY } from "./promptLength-Bx76P6F9.js";
 import { t as REDTEAM_MEMORY_POISONING_PLUGIN_ID } from "./constants-BjJV0cRr.js";
-import { n as checkExfilTracking } from "./indirectWebPwn-BWGexuih.js";
+import { n as checkExfilTracking } from "./indirectWebPwn-C779yDi6.js";
 import * as fs$2 from "fs";
 import fs from "fs";
 import path from "path";
@@ -2453,7 +2453,7 @@ var RedteamGraderBase = class {
 			goal: test.metadata?.goal || prompt,
 			prompt,
 			entities: test.metadata?.entities ?? [],
-			tools: provider?.config?.tools ? await maybeLoadToolsFromExternalFile(provider.config.tools) : void 0,
+			tools: provider?.config?.tools && !isMcpToolNameFilter(provider.config.tools) ? await maybeLoadToolsFromExternalFile(provider.config.tools) : void 0,
 			testVars: test.vars ?? {},
 			...gradingContext || {},
 			...typeof renderedValue === "object" && renderedValue !== null ? renderedValue : {},
@@ -16205,4 +16205,4 @@ function getGraderById(id) {
 //#endregion
 export { CONTEXT_FAITHFULNESS_LONGFORM as $, DivergentRepetitionPlugin as A, matchesFactuality as B, getPiiLeakTestsForCategory as C, withProviderCallExecutionContext as Ct, HarmbenchPlugin as D, getDefaultProviders as Dt, ImitationPlugin as E, sampleArray as Et, AegisPlugin as F, doRemoteGrading as G, matchesLlmRubric as H, RedteamPluginBase as I, readProviderPromptMap as J, processPrompts as K, fetchHuggingFaceDataset as L, CrossSessionLeakPlugin as M, ContractPlugin as N, HallucinationPlugin as O, DEFAULT_ANTHROPIC_MODEL as Ot, BeavertailsPlugin as P, ANSWER_RELEVANCY_GENERATE as Q, isGraderFailure as R, PlinyPlugin as S, getProviderCallExecutionContext as St, IntentPlugin as T, retryWithDeduplication as Tt, matchesPiScore as U, matchesGEval as V, matchesTrajectoryGoalSuccess as W, SELECT_BEST_PROMPT as X, DEFAULT_WEB_SEARCH_PROMPT as Y, SUGGEST_PROMPTS_SYSTEM_MESSAGE as Z, PoliticsPlugin as _, loadFromJavaScriptFile as _t, UnverifiableClaimsPlugin as a, CONTEXT_RELEVANCE_BAD as at, isValidPolicyObject as b, getAndCheckProvider as bt, ToolDiscoveryPlugin as c, cosineSimilarity as ct, TeenSafetyDangerousContentPlugin as d, fail as dt, CONTEXT_FAITHFULNESS_NLI_STATEMENTS as et, TeenSafetyAgeRestrictedGoodsAndServicesPlugin as f, normalizeMatcherTokenUsage as ft, PromptExtractionPlugin as g, getFinalTest as gt, RbacPlugin as h, coerceString as ht, VLGuardPlugin as i, CONTEXT_RELEVANCE as it, DebugAccessPlugin as j, ExcessiveAgencyPlugin as k, TeenSafetyHarmfulBodyIdealsPlugin as l, dotProduct as lt, ShellInjectionPlugin as m, tryParse as mt, graders_exports as n, CONTEXT_RECALL_ATTRIBUTED_TOKEN as nt, UnsafeBenchPlugin as o, loadRubricPrompt as ot, SqlInjectionPlugin as p, splitIntoSentences as pt, readPrompts as q, VLSUPlugin as r, CONTEXT_RECALL_NOT_ATTRIBUTED_TOKEN as rt, ToxicChatPlugin as s, renderLlmRubricPrompt as st, getGraderById as t, CONTEXT_RECALL as tt, TeenSafetyDangerousRoleplayPlugin as u, euclideanDistance as ut, PolicyPlugin as v, processFileReference as vt, OverreliancePlugin as w, getCustomPolicies as wt, makeInlinePolicyIdSync as x, getGradingProvider as xt, determinePolicyTypeFromId as y, callProviderWithContext as yt, matchesClosedQa as z };
 
-//# sourceMappingURL=graders-BrLOEmxX.js.map
+//# sourceMappingURL=graders-CO5drf0x.js.map