promptfoo 0.120.21 → 0.120.22

package/dist/src/index.cjs

@@ -5,36 +5,36 @@ const require_esm = require('./esm-gWVPXn3r.cjs');
 const require_pythonUtils = require('./pythonUtils-rOCm9w_5.cjs');
 const require_fileExtensions = require('./fileExtensions-DkJYkWUy.cjs');
 const require_transform = require('./transform-DshYLyBq.cjs');
-const require_graders = require('./graders-C9Ni3pNQ.cjs');
-const require_types = require('./types-DAmPdu4_.cjs');
-const require_util = require('./util-B-Yo0qzc.cjs');
-const require_fetch = require('./fetch-MdGxb6Bv.cjs');
+const require_graders = require('./graders-B6ThhLLt.cjs');
+const require_types = require('./types-Cixl-wRO.cjs');
+const require_util = require('./util-CxWghxGa.cjs');
+const require_fetch = require('./fetch-E3xbPP9w.cjs');
 const require_time = require('./time-CbtsO5_a.cjs');
-const require_cache = require('./cache-DYJRe0Jt.cjs');
-const require_providers = require('./providers-Bz0U4NGd.cjs');
-const require_utils = require('./utils-BSpcJB4B.cjs');
-const require_extractor = require('./extractor-DjxI7POm.cjs');
+const require_cache = require('./cache-PWdK7nbt.cjs');
+const require_providers = require('./providers-lmV8J0jR.cjs');
+const require_utils = require('./utils-u1jU5_Iq.cjs');
+const require_extractor = require('./extractor-6TFoLBb3.cjs');
 require('./genaiTracer-zK0Mtd_D.cjs');
-const require_chat = require('./chat-Biz16BQv.cjs');
+const require_chat = require('./chat-DKNIBnnw.cjs');
 const require_tokenUsageUtils = require('./tokenUsageUtils-DFp2VFtG.cjs');
-const require_transform$1 = require('./transform-B5HUD86U.cjs');
-require('./messages-D4Cu-Nxe.cjs');
-require('./util-KE6uFWHG.cjs');
+const require_transform$1 = require('./transform-Bow1252l.cjs');
+require('./messages-DR9VV_jC.cjs');
+require('./util-9SFCdUzk.cjs');
 require('./openai-eZz8QsxY.cjs');
-const require_util$2 = require('./util-BT0p5c9X.cjs');
-require('./completion-Dw-9jqz1.cjs');
-const require_accounts = require('./accounts-uWRxEv-D.cjs');
-const require_server = require('./server-ByLwfSDf.cjs');
-const require_blobs = require('./blobs-D-Neij7m.cjs');
-const require_tables = require('./tables-D_VmN-ui.cjs');
-const require_telemetry = require('./telemetry-DrSTNTH-.cjs');
+const require_util$2 = require('./util-Dqg4mr2c.cjs');
+require('./completion-Xj-aJd1b.cjs');
+const require_accounts = require('./accounts-BGa3oLcq.cjs');
+const require_server = require('./server-PndEy2Nj.cjs');
+const require_blobs = require('./blobs-h2nW1G8J.cjs');
+const require_tables = require('./tables-B5ootZ1x.cjs');
+const require_telemetry = require('./telemetry-DMNm_KFn.cjs');
 const require_text = require('./text-DF2hMKdg.cjs');
-const require_store = require('./store-l9-mIRC-.cjs');
-require('./base-CRrQ15eu.cjs');
-require('./image-CpQ-fJt3.cjs');
+const require_store = require('./store-j9IVHCUH.cjs');
+require('./base-oCiJ630X.cjs');
+require('./image-CVIQmm2j.cjs');
 const require_providerRegistry = require('./providerRegistry-DFXamjVO.cjs');
 const require_rubyUtils = require('./rubyUtils-DCci_Y0f.cjs');
-const require_evalResult = require('./evalResult-CFSvp3Zh.cjs');
+const require_evalResult = require('./evalResult-CPVLU2Np.cjs');
 let fs = require("fs");
 fs = require_logger.__toESM(fs);
 let path = require("path");
@@ -307,7 +307,7 @@ async function startOtlpReceiverIfNeeded(testSuite) {
 		require_telemetry.telemetry_default.record("feature_used", { feature: "tracing" });
 		try {
 			require_logger.logger_default.debug("[EvaluatorTracing] Tracing configuration detected, starting OTLP receiver");
-			const { startOTLPReceiver } = await Promise.resolve().then(() => require("./otlpReceiver-COmdVimi.cjs"));
+			const { startOTLPReceiver } = await Promise.resolve().then(() => require("./otlpReceiver-BGH8dMy4.cjs"));
 			const port = testSuite.tracing.otlp.http.port || 4318;
 			const host = testSuite.tracing.otlp.http.host || "127.0.0.1";
 			require_logger.logger_default.debug(`[EvaluatorTracing] Starting OTLP receiver on ${host}:${port}`);
@@ -330,7 +330,7 @@ async function startOtlpReceiverIfNeeded(testSuite) {
 async function stopOtlpReceiverIfNeeded() {
 	if (otlpReceiverStarted) try {
 		require_logger.logger_default.debug("[EvaluatorTracing] Stopping OTLP receiver");
-		const { stopOTLPReceiver } = await Promise.resolve().then(() => require("./otlpReceiver-COmdVimi.cjs"));
+		const { stopOTLPReceiver } = await Promise.resolve().then(() => require("./otlpReceiver-BGH8dMy4.cjs"));
 		await stopOTLPReceiver();
 		otlpReceiverStarted = false;
 		require_logger.logger_default.info("[EvaluatorTracing] OTLP receiver stopped successfully");
@@ -365,7 +365,7 @@ async function generateTraceContextIfNeeded(test, evaluateOptions, testIdx, prom
 	}
 	if (!tracingEnabled) return null;
 	require_logger.logger_default.debug("[EvaluatorTracing] Importing trace store");
-	const { getTraceStore } = await Promise.resolve().then(() => require("./store-fsxOJl5O.cjs"));
+	const { getTraceStore } = await Promise.resolve().then(() => require("./store-BppHaTbi.cjs"));
 	const traceStore = getTraceStore();
 	const traceId = generateTraceId();
 	const spanId = generateSpanId();
@@ -4266,7 +4266,7 @@ var Evaluator = class {
 				const defaultProvider = testSuite.defaultTest.provider;
 				if (require_types.isApiProvider(defaultProvider)) testCase.provider = defaultProvider;
 				else if (typeof defaultProvider === "object" && defaultProvider.id) {
-					const { loadApiProvider } = await Promise.resolve().then(() => require("./providers-BR5P_qep.cjs"));
+					const { loadApiProvider } = await Promise.resolve().then(() => require("./providers-BGK6Bn3A.cjs"));
 					testCase.provider = await loadApiProvider(typeof defaultProvider.id === "function" ? defaultProvider.id() : defaultProvider.id, { options: defaultProvider });
 				} else testCase.provider = defaultProvider;
 			}
@@ -4341,7 +4341,7 @@ var Evaluator = class {
 			if (evalOption.test.assert?.some((a) => a.type === "max-score")) rowsWithMaxScoreAssertion.add(evalOption.testIdx);
 		}
 		if (require_logger.cliState_default.resume && this.evalRecord.persisted) try {
-			const { default: EvalResult } = await Promise.resolve().then(() => require("./evalResult-DqLWyktN.cjs"));
+			const { default: EvalResult } = await Promise.resolve().then(() => require("./evalResult-BeU2oOzP.cjs"));
 			const completedPairs = await EvalResult.getCompletedIndexPairs(this.evalRecord.id, { excludeErrors: require_logger.cliState_default.retryMode });
 			const originalCount = runEvalOptions.length;
 			for (let i = runEvalOptions.length - 1; i >= 0; i--) {
@@ -5273,7 +5273,7 @@ function convertResultsToTable(eval_) {
 			description: result.description || void 0,
 			outputs: [],
 			vars: result.vars ? Object.values(varsForHeader).map((varName) => {
-				const varValue = result.vars?.[varName] || "";
+				const varValue = result.vars?.[varName] ?? "";
 				if (typeof varValue === "string") return varValue;
 				return JSON.stringify(varValue, null, 2);
 			}).flat() : [],
@@ -5308,7 +5308,11 @@ function convertResultsToTable(eval_) {
 		varValuesForRow.set(result.testIdx, result.vars);
 		rowMap[result.testIdx] = row;
 		let resultText;
-		const outputTextDisplay = typeof result.response?.output === "object" ? JSON.stringify(result.response.output) : result.response?.output || result.error || "";
+		const rawOutput = result.response?.output;
+		let outputTextDisplay;
+		if (rawOutput !== null && typeof rawOutput === "object") outputTextDisplay = JSON.stringify(rawOutput);
+		else if (rawOutput == null || rawOutput === "") outputTextDisplay = result.error || "";
+		else outputTextDisplay = String(rawOutput);
 		if (result.testCase.assert) if (result.success) resultText = `${outputTextDisplay || result.error || ""}`;
 		else resultText = `${outputTextDisplay}`;
 		else if (result.error) resultText = `${result.error}`;
@@ -5355,7 +5359,7 @@ function convertResultsToTable(eval_) {
 	const rows = Object.values(rowMap);
 	const sortedVars = [...varsForHeader].sort();
 	for (const row of rows) row.vars = sortedVars.map((varName) => {
-		const varValue = varValuesForRow.get(row.testIdx)?.[varName] || "";
+		const varValue = varValuesForRow.get(row.testIdx)?.[varName] ?? "";
 		if (typeof varValue === "string") return varValue;
 		return JSON.stringify(varValue, null, 2);
 	});
@@ -5372,7 +5376,11 @@ function convertResultsToTable(eval_) {
 //#region src/util/exportToFile/index.ts
 function convertEvalResultToTableCell(result) {
 	let resultText;
-	const outputTextDisplay = typeof result.response?.output === "object" ? JSON.stringify(result.response.output) : result.response?.output || result.error || "";
+	const rawOutput = result.response?.output;
+	let outputTextDisplay;
+	if (rawOutput !== null && typeof rawOutput === "object") outputTextDisplay = JSON.stringify(rawOutput);
+	else if (rawOutput == null || rawOutput === "") outputTextDisplay = result.error || "";
+	else outputTextDisplay = String(rawOutput);
 	if (result.testCase.assert) if (result.success) resultText = `${outputTextDisplay || result.error || ""}`;
 	else resultText = `${outputTextDisplay}`;
 	else if (result.error) resultText = `${result.error}`;
@@ -5418,11 +5426,12 @@ function convertTestResultsToTableRow(results, varsForHeader) {
 		outputs: [],
 		vars: Object.values(varsForHeader).map((varName) => {
 			if (varName === "sessionId") {
-				const varValue = results[0].testCase.vars?.sessionId || results[0].metadata?.sessionId || "";
+				const sessionId = results[0].testCase.vars?.sessionId;
+				const varValue = sessionId == null || sessionId === "" ? results[0].metadata?.sessionId ?? "" : sessionId;
 				if (typeof varValue === "string") return varValue;
 				return JSON.stringify(varValue);
 			}
-			const varValue = results[0].testCase.vars?.[varName] || "";
+			const varValue = results[0].testCase.vars?.[varName] ?? "";
 			if (typeof varValue === "string") return varValue;
 			return JSON.stringify(varValue);
 		}).flat(),
@@ -9506,11 +9515,6 @@ async function synthesize({ abortSignal, delay, entities: entitiesOverride, inje
 	});
 	await require_providers.validateStrategies(strategies);
 	await validateSharpDependency(strategies, plugins);
-	if (strategies.some((s) => require_types.isLanguageDisallowedStrategy(s.id)) && language) {
-		const originalLanguage = Array.isArray(language) ? language.join(", ") : language;
-		language = "en";
-		require_logger.logger_default.info(`[Language Override] Detected language-disallowed strategy (audio/video/image/layer/math-prompt). Forcing language to 'en' (was: ${originalLanguage})`);
-	}
 	const redteamProvider = await require_providers.redteamProviderManager.getProvider({ provider });
 	const { effectiveStrategyCount, includeBasicTests, totalPluginTests, totalTests } = calculateTotalTests(plugins, strategies, language);
 	require_logger.logger_default.info(`Synthesizing test cases for ${prompts.length} ${prompts.length === 1 ? "prompt" : "prompts"}...\nUsing plugins:\n\n${chalk.default.yellow(plugins.map((p) => {
@@ -10995,60 +10999,101 @@ async function deleteErrorResults(resultIds) {
 	await require_tables.getDb().delete(require_tables.evalResultsTable).where((0, drizzle_orm.inArray)(require_tables.evalResultsTable.id, resultIds));
 	require_logger.logger_default.debug(`Deleted ${resultIds.length} error results from database`);
 }
+const RECALCULATE_BATCH_SIZE = 1e3;
 /**
-* Recalculates prompt metrics based on current results after ERROR results have been deleted
+* Recalculates prompt metrics based on current results after ERROR results have been deleted.
+* Uses streaming batched iteration to avoid OOM with large evaluations (40K+ results).
 */
 async function recalculatePromptMetrics(evalRecord) {
 	require_logger.logger_default.debug("Recalculating prompt metrics after deleting ERROR results");
-	await evalRecord.loadResults();
+	const startTime = Date.now();
+	let batchNumber = 0;
+	let totalProcessed = 0;
 	const promptMetricsMap = /* @__PURE__ */ new Map();
-	for (const prompt of evalRecord.prompts) {
-		const promptIdx = evalRecord.prompts.indexOf(prompt);
-		promptMetricsMap.set(promptIdx, {
-			score: 0,
-			testPassCount: 0,
-			testFailCount: 0,
-			testErrorCount: 0,
-			assertPassCount: 0,
-			assertFailCount: 0,
-			totalLatencyMs: 0,
-			tokenUsage: require_tokenUsageUtils.createEmptyTokenUsage(),
-			namedScores: {},
-			namedScoresCount: {},
-			cost: 0
-		});
-	}
-	for (const result of evalRecord.results) {
-		const metrics = promptMetricsMap.get(result.promptIdx);
-		if (!metrics) continue;
-		if (result.success) metrics.testPassCount++;
-		else if (result.failureReason === require_types.ResultFailureReason.ERROR) metrics.testErrorCount++;
-		else metrics.testFailCount++;
-		metrics.score += result.score || 0;
-		metrics.totalLatencyMs += result.latencyMs || 0;
-		metrics.cost += result.cost || 0;
-		for (const [key, value] of Object.entries(result.namedScores || {})) {
-			metrics.namedScores[key] = (metrics.namedScores[key] || 0) + value;
-			const testVars = result.testCase?.vars || {};
-			let contributingAssertions = 0;
-			result.gradingResult?.componentResults?.forEach((componentResult) => {
-				if (renderMetricName(componentResult.assertion?.metric, testVars) === key) contributingAssertions++;
-			});
-			metrics.namedScoresCount[key] = (metrics.namedScoresCount[key] || 0) + (contributingAssertions || 1);
-		}
-		if (result.gradingResult?.componentResults) {
-			metrics.assertPassCount += result.gradingResult.componentResults.filter((r) => r.pass).length;
-			metrics.assertFailCount += result.gradingResult.componentResults.filter((r) => !r.pass).length;
-		}
-		if (result.response?.tokenUsage) require_tokenUsageUtils.accumulateResponseTokenUsage(metrics.tokenUsage, { tokenUsage: result.response.tokenUsage });
-		if (result.gradingResult?.tokensUsed) {
-			if (!metrics.tokenUsage.assertions) metrics.tokenUsage.assertions = require_tokenUsageUtils.createEmptyAssertions();
-			require_tokenUsageUtils.accumulateAssertionTokenUsage(metrics.tokenUsage.assertions, result.gradingResult.tokensUsed);
+	for (const [promptIdx] of evalRecord.prompts.entries()) promptMetricsMap.set(promptIdx, {
+		score: 0,
+		testPassCount: 0,
+		testFailCount: 0,
+		testErrorCount: 0,
+		assertPassCount: 0,
+		assertFailCount: 0,
+		totalLatencyMs: 0,
+		tokenUsage: require_tokenUsageUtils.createEmptyTokenUsage(),
+		namedScores: {},
+		namedScoresCount: {},
+		cost: 0
+	});
+	let currentResultId;
+	try {
+		for await (const batch of evalRecord.fetchResultsBatched(RECALCULATE_BATCH_SIZE)) {
+			batchNumber++;
+			require_logger.logger_default.debug(`Processing batch ${batchNumber} with ${batch.length} results`);
+			for (const result of batch) {
+				currentResultId = result.id;
+				const metrics = promptMetricsMap.get(result.promptIdx);
+				if (!metrics) {
+					require_logger.logger_default.debug(`Skipping result with invalid promptIdx: ${result.promptIdx}`, {
+						resultId: result.id,
+						evalId: evalRecord.id
+					});
+					continue;
+				}
+				if (result.success) metrics.testPassCount++;
+				else if (result.failureReason === require_types.ResultFailureReason.ERROR) metrics.testErrorCount++;
+				else metrics.testFailCount++;
+				metrics.score += result.score ?? 0;
+				metrics.totalLatencyMs += result.latencyMs || 0;
+				metrics.cost += result.cost || 0;
+				for (const [key, value] of Object.entries(result.namedScores || {})) {
+					metrics.namedScores[key] = (metrics.namedScores[key] || 0) + value;
+					const testVars = result.testCase?.vars || {};
+					let contributingAssertions = 0;
+					result.gradingResult?.componentResults?.forEach((componentResult) => {
+						if (renderMetricName(componentResult.assertion?.metric, testVars) === key) contributingAssertions++;
+					});
+					metrics.namedScoresCount[key] = (metrics.namedScoresCount[key] || 0) + (contributingAssertions || 1);
+				}
+				if (result.gradingResult?.componentResults) {
+					metrics.assertPassCount += result.gradingResult.componentResults.filter((r) => r.pass).length;
+					metrics.assertFailCount += result.gradingResult.componentResults.filter((r) => !r.pass).length;
+				}
+				if (result.response?.tokenUsage) require_tokenUsageUtils.accumulateResponseTokenUsage(metrics.tokenUsage, { tokenUsage: result.response.tokenUsage });
+				if (result.gradingResult?.tokensUsed) {
+					if (!metrics.tokenUsage.assertions) metrics.tokenUsage.assertions = require_tokenUsageUtils.createEmptyAssertions();
+					require_tokenUsageUtils.accumulateAssertionTokenUsage(metrics.tokenUsage.assertions, result.gradingResult.tokensUsed);
+				}
+			}
+			totalProcessed += batch.length;
 		}
+	} catch (error) {
+		require_logger.logger_default.error("Error during batched metrics recalculation", {
+			phase: "calculation",
+			batchNumber,
+			totalProcessed,
+			currentResultId,
+			evalId: evalRecord.id,
+			error
+		});
+		throw error;
 	}
 	for (const [promptIdx, newMetrics] of promptMetricsMap.entries()) if (promptIdx < evalRecord.prompts.length) evalRecord.prompts[promptIdx].metrics = newMetrics;
-	if (evalRecord.persisted) await evalRecord.addPrompts(evalRecord.prompts);
-	require_logger.logger_default.debug("Prompt metrics recalculation completed");
+	if (evalRecord.persisted) try {
+		await evalRecord.addPrompts(evalRecord.prompts);
+	} catch (error) {
+		require_logger.logger_default.error("Error saving recalculated prompt metrics", {
+			phase: "save",
+			evalId: evalRecord.id,
+			promptCount: evalRecord.prompts.length,
+			error
+		});
+		throw error;
+	}
+	const durationMs = Date.now() - startTime;
+	require_logger.logger_default.debug("Prompt metrics recalculation completed", {
+		totalBatches: batchNumber,
+		totalResults: totalProcessed,
+		durationMs
+	});
 }
 
 //#endregion