promptfoo 0.120.27 → 0.121.2

package/dist/src/evaluator-Jx6bRZV6.js

@@ -0,0 +1,36 @@
+#!/usr/bin/env node
+import "./logger-KkObSCzq.js";
+import "./fetch-Dxpd4_sr.js";
+import "./accounts-Bm2D8Db9.js";
+import "./cloud-C0dlstV_.js";
+import "./telemetry-CMrFgtPB.js";
+import "./types-Cbd8uOMq.js";
+import "./server-CqzrVGpF.js";
+import "./providers-BlqUifFg.js";
+import "./cache-BVeDlD87.js";
+import "./util-C1CeHl-P.js";
+import "./esm-C03C-mv3.js";
+import "./pythonUtils-C3py6GC1.js";
+import "./transform-DilY9wbS.js";
+import "./graders-Bj_Odv7c.js";
+import "./utils-f2-Moju7.js";
+import "./genaiTracer-C1rxGO8Q.js";
+import "./chat-BiKyneZl.js";
+import "./transform-D5HsjduX.js";
+import "./messages-D0lx5qK7.js";
+import "./util-B3xGByQh.js";
+import "./responses-WNGNYe3K.js";
+import "./openai-DhbB7eWK.js";
+import "./util-CMy69ZgQ.js";
+import "./completion-DZ083F31.js";
+import "./blobs-CMHN0Qcz.js";
+import "./tables-gftXzE9I.js";
+import "./extractor-DReVID0K.js";
+import "./store-Dim__MDd.js";
+import "./base-Dy1V8--Z.js";
+import "./image-DS-o-0ph.js";
+import "./providerRegistry-B0RUOLI_.js";
+import "./rubyUtils-Boc4HZzX.js";
+import { t as evaluate } from "./evaluator-BhoWwp5b.js";
+import "./signal-4U3mfRvL.js";
+export { evaluate };

package/dist/src/{extractor-V5x_m1i0.js → extractor-C0EVHewb.js}

@@ -1,8 +1,7 @@
-import { _ as getEnvBool, i as logger_default } from "./logger-CG1uZPbQ.js";
-import { c as cloudConfig } from "./fetch-D3OHf-lV.js";
-import { o as isLoggedIntoCloud } from "./accounts-CPDRAMND.js";
-import { a as BLOB_MIN_SIZE, i as BLOB_MAX_SIZE, n as recordBlobReference, r as storeBlob } from "./blobs-CR5C4Ihh.js";
-
+import { _ as getEnvBool, i as logger } from "./logger-CT3IKMKA.js";
+import { o as cloudConfig } from "./fetch-Cwxnd8zz.js";
+import { o as isLoggedIntoCloud } from "./accounts-CiBLOnA7.js";
+import { n as recordBlobReference, r as storeBlob } from "./blobs-CBO20krR.js";
 //#region src/blobs/remoteUpload.ts
 function buildRemoteUrl() {
 	const baseUrl = cloudConfig.getApiHost();
@@ -11,7 +10,7 @@ function buildRemoteUrl() {
 	try {
 		return new URL("/api/blobs", baseUrl).toString();
 	} catch (error) {
-		logger_default.debug("[RemoteBlob] Invalid remote blob URL", {
+		logger.debug("[RemoteBlob] Invalid remote blob URL", {
 			error: error instanceof Error ? error.message : String(error),
 			baseUrl
 		});
@@ -26,7 +25,7 @@ async function uploadBlobRemote(buffer, mimeType, context) {
 	const apiKey = cloudConfig.getApiKey();
 	if (!url || !apiKey) return null;
 	try {
-		const { fetchWithProxy } = await import("./fetch-ouKnrWK-.js");
+		const { fetchWithProxy } = await import("./fetch-Cwxnd8zz.js").then((n) => n.i);
 		const response = await fetchWithProxy(url, {
 			method: "POST",
 			headers: {
@@ -40,12 +39,12 @@ async function uploadBlobRemote(buffer, mimeType, context) {
 			})
 		});
 		if (response.status === 404 || response.status === 400) {
-			logger_default.debug("[RemoteBlob] Remote blob upload unavailable", { status: response.status });
+			logger.debug("[RemoteBlob] Remote blob upload unavailable", { status: response.status });
 			return null;
 		}
 		if (!response.ok) {
 			const text = await response.text();
-			logger_default.debug("[RemoteBlob] Failed to upload blob", {
+			logger.debug("[RemoteBlob] Failed to upload blob", {
 				status: response.status,
 				statusText: response.statusText,
 				body: text
@@ -54,16 +53,15 @@ async function uploadBlobRemote(buffer, mimeType, context) {
 		}
 		const data = await response.json();
 		if (!data?.ref?.hash) {
-			logger_default.debug("[RemoteBlob] Remote upload returned malformed response");
+			logger.debug("[RemoteBlob] Remote upload returned malformed response");
 			return null;
 		}
 		return data;
 	} catch (error) {
-		logger_default.debug("[RemoteBlob] Error uploading blob", { error: error instanceof Error ? error.message : String(error) });
+		logger.debug("[RemoteBlob] Error uploading blob", { error: error instanceof Error ? error.message : String(error) });
 		return null;
 	}
 }
-
 //#endregion
 //#region src/blobs/extractor.ts
 const BLOB_URI_REGEX = /^promptfoo:\/\/blob\/([a-f0-9]{64})$/i;
@@ -81,13 +79,13 @@ function extractBase64(value) {
 			mimeType
 		};
 	} catch (error) {
-		logger_default.warn("[BlobExtractor] Failed to parse base64 data URL", { error });
+		logger.warn("[BlobExtractor] Failed to parse base64 data URL", { error });
 		return null;
 	}
 }
 function shouldExternalize(buffer) {
 	const size = buffer.length;
-	return size >= BLOB_MIN_SIZE && size <= BLOB_MAX_SIZE;
+	return size >= 1024 && size <= 52428800;
 }
 function getKindFromMimeType(mimeType) {
 	return mimeType.startsWith("audio/") ? "audio" : "image";
@@ -113,7 +111,7 @@ function normalizeAudioMimeType(format) {
 	};
 	if (mimeMap[formatLower]) return mimeMap[formatLower];
 	if (!/^[a-z0-9_-]+$/i.test(formatLower)) {
-		logger_default.warn("[BlobExtractor] Invalid audio format, using default", { format });
+		logger.warn("[BlobExtractor] Invalid audio format, using default", { format });
 		return "audio/wav";
 	}
 	return `audio/${formatLower}`;
@@ -130,7 +128,7 @@ function parseBinary(base64OrDataUrl, defaultMimeType) {
 			mimeType: defaultMimeType
 		};
 	} catch (error) {
-		logger_default.warn("[BlobExtractor] Failed to parse base64 data", { error });
+		logger.warn("[BlobExtractor] Failed to parse base64 data", { error });
 		return null;
 	}
 }
@@ -151,7 +149,7 @@ async function maybeStore(base64OrDataUrl, defaultMimeType, context, location, k
 		location,
 		kind
 	}).catch((error) => {
-		logger_default.debug("[BlobExtractor] Cloud upload failed (non-fatal)", {
+		logger.debug("[BlobExtractor] Cloud upload failed (non-fatal)", {
 			error: error instanceof Error ? error.message : String(error),
 			hash: ref.hash
 		});
@@ -235,7 +233,7 @@ async function extractAndStoreBinaryData(response, context) {
 				blobRef: stored
 			};
 			mutated = true;
-			logger_default.debug("[BlobExtractor] Stored audio blob", {
+			logger.debug("[BlobExtractor] Stored audio blob", {
 				...context,
 				hash: stored.hash
 			});
@@ -246,7 +244,7 @@ async function extractAndStoreBinaryData(response, context) {
 		const stored = await maybeStore(img.data, img.mimeType || "image/png", blobContext, `response.images[${idx}].data`, "image");
 		if (stored) {
 			mutated = true;
-			logger_default.debug("[BlobExtractor] Stored image blob", {
+			logger.debug("[BlobExtractor] Stored image blob", {
 				...context,
 				hash: stored.hash
 			});
@@ -283,7 +281,7 @@ async function extractAndStoreBinaryData(response, context) {
 			if (stored) {
 				next.output = stored.uri;
 				mutated = true;
-				logger_default.debug("[BlobExtractor] Stored output blob", {
+				logger.debug("[BlobExtractor] Stored output blob", {
 					...context,
 					hash: stored.hash
 				});
@@ -302,7 +300,7 @@ async function extractAndStoreBinaryData(response, context) {
 					storedUris.push(stored.uri);
 					jsonMutated = true;
 					mutated = true;
-					logger_default.debug("[BlobExtractor] Stored image blob from b64_json", {
+					logger.debug("[BlobExtractor] Stored image blob from b64_json", {
 						...context,
 						hash: stored.hash
 					});
@@ -320,7 +318,7 @@ async function extractAndStoreBinaryData(response, context) {
 			}
 		}
 	} catch (err) {
-		logger_default.debug("[BlobExtractor] Failed to parse base64 JSON output", {
+		logger.debug("[BlobExtractor] Failed to parse base64 JSON output", {
 			error: err instanceof Error ? err.message : String(err),
 			location: "response.output"
 		});
@@ -370,7 +368,7 @@ async function recordExistingBlobReferences(value, context, location) {
 	}
 	if (value && typeof value === "object") for (const [key, child] of Object.entries(value)) await recordExistingBlobReferences(child, context, location ? `${location}.${key}` : key);
 }
-
 //#endregion
 export { isBlobStorageEnabled as n, shouldAttemptRemoteBlobUpload as r, extractAndStoreBinaryData as t };
-//# sourceMappingURL=extractor-V5x_m1i0.js.map
+
+//# sourceMappingURL=extractor-C0EVHewb.js.map

package/dist/src/extractor-D25qpmGX.js

@@ -0,0 +1,374 @@
+import { _ as getEnvBool, i as logger } from "./logger-DLcq4dWf.js";
+import { o as cloudConfig } from "./fetch-HaqdX7U1.js";
+import { c as isLoggedIntoCloud } from "./accounts-B0pgC1oV.js";
+import { i as storeBlob, r as recordBlobReference } from "./blobs-BDbfYdrJ.js";
+//#region src/blobs/remoteUpload.ts
+function buildRemoteUrl() {
+	const baseUrl = cloudConfig.getApiHost();
+	const apiKey = cloudConfig.getApiKey();
+	if (!baseUrl || !apiKey || !isLoggedIntoCloud()) return null;
+	try {
+		return new URL("/api/blobs", baseUrl).toString();
+	} catch (error) {
+		logger.debug("[RemoteBlob] Invalid remote blob URL", {
+			error: error instanceof Error ? error.message : String(error),
+			baseUrl
+		});
+		return null;
+	}
+}
+function shouldAttemptRemoteBlobUpload() {
+	return buildRemoteUrl() !== null;
+}
+async function uploadBlobRemote(buffer, mimeType, context) {
+	const url = buildRemoteUrl();
+	const apiKey = cloudConfig.getApiKey();
+	if (!url || !apiKey) return null;
+	try {
+		const { fetchWithProxy } = await import("./fetch-HaqdX7U1.js").then((n) => n.i);
+		const response = await fetchWithProxy(url, {
+			method: "POST",
+			headers: {
+				"Content-Type": "application/json",
+				Authorization: `Bearer ${apiKey}`
+			},
+			body: JSON.stringify({
+				data: buffer.toString("base64"),
+				mimeType,
+				context
+			})
+		});
+		if (response.status === 404 || response.status === 400) {
+			logger.debug("[RemoteBlob] Remote blob upload unavailable", { status: response.status });
+			return null;
+		}
+		if (!response.ok) {
+			const text = await response.text();
+			logger.debug("[RemoteBlob] Failed to upload blob", {
+				status: response.status,
+				statusText: response.statusText,
+				body: text
+			});
+			return null;
+		}
+		const data = await response.json();
+		if (!data?.ref?.hash) {
+			logger.debug("[RemoteBlob] Remote upload returned malformed response");
+			return null;
+		}
+		return data;
+	} catch (error) {
+		logger.debug("[RemoteBlob] Error uploading blob", { error: error instanceof Error ? error.message : String(error) });
+		return null;
+	}
+}
+//#endregion
+//#region src/blobs/extractor.ts
+const BLOB_URI_REGEX = /^promptfoo:\/\/blob\/([a-f0-9]{64})$/i;
+const BLOB_HASH_REGEX = /^[a-f0-9]{64}$/i;
+function isDataUrl(value) {
+	return /^data:(audio|image)\/[^;]+;base64,/.test(value);
+}
+function extractBase64(value) {
+	const match = value.match(/^data:([^;]+);base64,(.+)$/);
+	if (!match) return null;
+	const mimeType = match[1];
+	try {
+		return {
+			buffer: Buffer.from(match[2], "base64"),
+			mimeType
+		};
+	} catch (error) {
+		logger.warn("[BlobExtractor] Failed to parse base64 data URL", { error });
+		return null;
+	}
+}
+function shouldExternalize(buffer) {
+	const size = buffer.length;
+	return size >= 1024 && size <= 52428800;
+}
+function getKindFromMimeType(mimeType) {
+	return mimeType.startsWith("audio/") ? "audio" : "image";
+}
+/**
+* Normalize audio format to proper MIME type.
+* Some providers return just 'wav' instead of 'audio/wav'.
+* @internal Exported for testing
+*/
+function normalizeAudioMimeType(format) {
+	if (!format) return "audio/wav";
+	const trimmedFormat = format.trim();
+	if (/^audio\/[a-z0-9_+-]+$/i.test(trimmedFormat)) return trimmedFormat;
+	const formatLower = trimmedFormat.toLowerCase();
+	const mimeMap = {
+		wav: "audio/wav",
+		mp3: "audio/mpeg",
+		ogg: "audio/ogg",
+		flac: "audio/flac",
+		aac: "audio/aac",
+		m4a: "audio/mp4",
+		webm: "audio/webm"
+	};
+	if (mimeMap[formatLower]) return mimeMap[formatLower];
+	if (!/^[a-z0-9_-]+$/i.test(formatLower)) {
+		logger.warn("[BlobExtractor] Invalid audio format, using default", { format });
+		return "audio/wav";
+	}
+	return `audio/${formatLower}`;
+}
+function parseBinary(base64OrDataUrl, defaultMimeType) {
+	if (isDataUrl(base64OrDataUrl)) {
+		const parsed = extractBase64(base64OrDataUrl);
+		if (!parsed) return null;
+		return parsed;
+	}
+	try {
+		return {
+			buffer: Buffer.from(base64OrDataUrl, "base64"),
+			mimeType: defaultMimeType
+		};
+	} catch (error) {
+		logger.warn("[BlobExtractor] Failed to parse base64 data", { error });
+		return null;
+	}
+}
+async function maybeStore(base64OrDataUrl, defaultMimeType, context, location, kind) {
+	const parsed = parseBinary(base64OrDataUrl, defaultMimeType);
+	if (!parsed || !shouldExternalize(parsed.buffer)) return null;
+	if (!isBlobStorageEnabled()) return null;
+	const mimeType = parsed.mimeType || "application/octet-stream";
+	const { ref } = await storeBlob(parsed.buffer, mimeType, {
+		...context,
+		location,
+		kind
+	});
+	if (shouldAttemptRemoteBlobUpload()) uploadBlobRemote(parsed.buffer, mimeType, {
+		evalId: context.evalId,
+		testIdx: context.testIdx,
+		promptIdx: context.promptIdx,
+		location,
+		kind
+	}).catch((error) => {
+		logger.debug("[BlobExtractor] Cloud upload failed (non-fatal)", {
+			error: error instanceof Error ? error.message : String(error),
+			hash: ref.hash
+		});
+	});
+	return ref;
+}
+async function externalizeDataUrls(value, context, location) {
+	if (typeof value === "string") {
+		if (!isDataUrl(value)) return {
+			value,
+			mutated: false
+		};
+		const parsed = extractBase64(value);
+		if (!parsed || !shouldExternalize(parsed.buffer)) return {
+			value,
+			mutated: false
+		};
+		const storedRef = await maybeStore(parsed.buffer.toString("base64"), parsed.mimeType, context, location, getKindFromMimeType(parsed.mimeType)) || null;
+		if (!storedRef) return {
+			value,
+			mutated: false
+		};
+		return {
+			value: storedRef.uri,
+			mutated: true
+		};
+	}
+	if (Array.isArray(value)) {
+		let mutated = false;
+		const nextValues = await Promise.all(value.map(async (item, idx) => {
+			const { value: nextValue, mutated: childMutated } = await externalizeDataUrls(item, context, `${location}[${idx}]`);
+			mutated ||= childMutated;
+			return nextValue;
+		}));
+		return mutated ? {
+			value: nextValues,
+			mutated
+		} : {
+			value,
+			mutated: false
+		};
+	}
+	if (value && typeof value === "object") {
+		let mutated = false;
+		const nextObject = { ...value };
+		for (const [key, child] of Object.entries(value)) {
+			const { value: nextValue, mutated: childMutated } = await externalizeDataUrls(child, context, location ? `${location}.${key}` : key);
+			if (childMutated) {
+				nextObject[key] = nextValue;
+				mutated = true;
+			}
+		}
+		return mutated ? {
+			value: nextObject,
+			mutated: true
+		} : {
+			value,
+			mutated: false
+		};
+	}
+	return {
+		value,
+		mutated: false
+	};
+}
+/**
+* Best-effort extraction of binary data from provider responses.
+* Currently focuses on audio.data fields and data URL outputs.
+*/
+async function extractAndStoreBinaryData(response, context) {
+	if (!response) return response;
+	let mutated = false;
+	const next = { ...response };
+	const blobContext = context || {};
+	if (response.audio?.data && typeof response.audio.data === "string") {
+		const stored = await maybeStore(response.audio.data, normalizeAudioMimeType(response.audio.format), blobContext, "response.audio.data", "audio");
+		if (stored) {
+			next.audio = {
+				...response.audio,
+				data: void 0,
+				blobRef: stored
+			};
+			mutated = true;
+			logger.debug("[BlobExtractor] Stored audio blob", {
+				...context,
+				hash: stored.hash
+			});
+		}
+	}
+	if (response.images?.length) next.images = await Promise.all(response.images.map(async (img, idx) => {
+		if (!img.data || typeof img.data !== "string" || !isDataUrl(img.data)) return img;
+		const stored = await maybeStore(img.data, img.mimeType || "image/png", blobContext, `response.images[${idx}].data`, "image");
+		if (stored) {
+			mutated = true;
+			logger.debug("[BlobExtractor] Stored image blob", {
+				...context,
+				hash: stored.hash
+			});
+			return {
+				...img,
+				data: void 0,
+				blobRef: stored
+			};
+		}
+		return img;
+	}));
+	const turns = response.turns;
+	if (Array.isArray(turns)) next.turns = await Promise.all(turns.map(async (turn, idx) => {
+		if (turn?.audio?.data && typeof turn.audio.data === "string") {
+			const stored = await maybeStore(turn.audio.data, normalizeAudioMimeType(turn.audio.format), blobContext, `response.turns[${idx}].audio.data`, "audio");
+			if (stored) {
+				mutated = true;
+				return {
+					...turn,
+					audio: {
+						...turn.audio,
+						data: void 0,
+						blobRef: stored
+					}
+				};
+			}
+		}
+		return turn;
+	}));
+	if (typeof response.output === "string" && isDataUrl(response.output)) {
+		const parsed = extractBase64(response.output);
+		if (parsed && shouldExternalize(parsed.buffer)) {
+			const stored = await maybeStore(parsed.buffer.toString("base64"), parsed.mimeType, blobContext, "response.output", getKindFromMimeType(parsed.mimeType));
+			if (stored) {
+				next.output = stored.uri;
+				mutated = true;
+				logger.debug("[BlobExtractor] Stored output blob", {
+					...context,
+					hash: stored.hash
+				});
+			}
+		}
+	}
+	if (typeof response.output === "string" && response.output.trim().startsWith("{") && (response.isBase64 && response.format === "json" || response.output.includes("\"b64_json\"") || response.output.includes("b64_json"))) try {
+		const parsed = JSON.parse(response.output);
+		if (Array.isArray(parsed.data)) {
+			let jsonMutated = false;
+			const storedUris = [];
+			for (const item of parsed.data) if (item?.b64_json && typeof item.b64_json === "string") {
+				const stored = await maybeStore(item.b64_json, "image/png", blobContext, "response.output.data[].b64_json", "image");
+				if (stored) {
+					item.b64_json = stored.uri;
+					storedUris.push(stored.uri);
+					jsonMutated = true;
+					mutated = true;
+					logger.debug("[BlobExtractor] Stored image blob from b64_json", {
+						...context,
+						hash: stored.hash
+					});
+				}
+			}
+			if (jsonMutated) {
+				if (storedUris.length === 1) next.output = storedUris[0];
+				else if (storedUris.length > 1) next.output = JSON.stringify(storedUris);
+				else next.output = JSON.stringify(parsed);
+				next.metadata = {
+					...response.metadata || {},
+					blobUris: storedUris,
+					originalFormat: response.format
+				};
+			}
+		}
+	} catch (err) {
+		logger.debug("[BlobExtractor] Failed to parse base64 JSON output", {
+			error: err instanceof Error ? err.message : String(err),
+			location: "response.output"
+		});
+	}
+	if (response.metadata) {
+		const { value, mutated: metadataMutated } = await externalizeDataUrls(response.metadata, blobContext, "response.metadata");
+		if (metadataMutated) {
+			next.metadata = value;
+			mutated = true;
+		}
+	}
+	const finalResponse = mutated ? next : response;
+	if (blobContext.evalId) await recordExistingBlobReferences(finalResponse, blobContext, "response");
+	return finalResponse;
+}
+function isBlobStorageEnabled() {
+	return !getEnvBool("PROMPTFOO_INLINE_MEDIA", false);
+}
+function parseBlobHashFromValue(value) {
+	if (!value) return null;
+	if (typeof value === "string") {
+		const match = value.match(BLOB_URI_REGEX);
+		return match ? match[1] : null;
+	}
+	if (typeof value === "object") {
+		const candidate = value;
+		if (candidate.hash && BLOB_HASH_REGEX.test(candidate.hash)) return candidate.hash;
+		if (candidate.uri && typeof candidate.uri === "string") {
+			const match = candidate.uri.match(BLOB_URI_REGEX);
+			if (match) return match[1];
+		}
+	}
+	return null;
+}
+async function recordExistingBlobReferences(value, context, location) {
+	const hash = parseBlobHashFromValue(value);
+	if (hash) {
+		await recordBlobReference(hash, {
+			...context,
+			location
+		});
+		return;
+	}
+	if (Array.isArray(value)) {
+		await Promise.all(value.map((child, idx) => recordExistingBlobReferences(child, context, `${location}[${idx}]`)));
+		return;
+	}
+	if (value && typeof value === "object") for (const [key, child] of Object.entries(value)) await recordExistingBlobReferences(child, context, location ? `${location}.${key}` : key);
+}
+//#endregion
+export { isBlobStorageEnabled as n, shouldAttemptRemoteBlobUpload as r, extractAndStoreBinaryData as t };
+
+//# sourceMappingURL=extractor-D25qpmGX.js.map