promptfoo 0.120.21 → 0.120.23

package/dist/src/server/index.js

@@ -202,7 +202,7 @@ const HUMAN_ASSERTION_TYPE = "human";
 * Application version from package.json.
 * Injected at build time, or read from npm environment in development.
 */
-const VERSION = "0.120.21";
+const VERSION = "0.120.23";
 /**
 * PostHog analytics key.
 * Only populated during production builds via PROMPTFOO_POSTHOG_KEY env var.
@@ -4775,21 +4775,6 @@ const CONFIGURABLE_STRATEGIES = [
 ];
 const CONFIGURABLE_STRATEGIES_SET = new Set(CONFIGURABLE_STRATEGIES);
 /**
-* Strategies that should not have language configuration applied to them.
-*/
-const LANGUAGE_DISALLOWED_STRATEGIES = new Set([
-	"audio",
-	"video",
-	"image",
-	"math-prompt"
-]);
-/**
-* Determines if a strategy should not use language configuration
-*/
-function isLanguageDisallowedStrategy(strategyId) {
-	return strategyId ? LANGUAGE_DISALLOWED_STRATEGIES.has(strategyId) : false;
-}
-/**
 * Default 'n' fan out for strategies that can add additional test cases during generation
 */
 const DEFAULT_N_FAN_OUT_BY_STRATEGY = {
@@ -6101,7 +6086,11 @@ function calculateAttackSuccessRate(testCount, failCount) {
 //#region src/redteam/sharedFrontend.ts
 function getRiskCategorySeverityMap(plugins) {
 	const overrides = plugins?.reduce((acc, plugin) => {
-		if (plugin.severity) acc[plugin.id] = plugin.severity;
+		if (plugin.severity) {
+			acc[plugin.id] = plugin.severity;
+			const policyId = plugin.config?.policy?.id;
+			if (plugin.id === "policy" && policyId) acc[policyId] = plugin.severity;
+		}
 		return acc;
 	}, {}) || {};
 	return {
@@ -6655,7 +6644,7 @@ function convertResultsToTable(eval_) {
 			description: result.description || void 0,
 			outputs: [],
 			vars: result.vars ? Object.values(varsForHeader).map((varName) => {
-				const varValue = result.vars?.[varName] || "";
+				const varValue = result.vars?.[varName] ?? "";
 				if (typeof varValue === "string") return varValue;
 				return JSON.stringify(varValue, null, 2);
 			}).flat() : [],
@@ -6690,7 +6679,11 @@ function convertResultsToTable(eval_) {
 		varValuesForRow.set(result.testIdx, result.vars);
 		rowMap[result.testIdx] = row;
 		let resultText;
-		const outputTextDisplay = typeof result.response?.output === "object" ? JSON.stringify(result.response.output) : result.response?.output || result.error || "";
+		const rawOutput = result.response?.output;
+		let outputTextDisplay;
+		if (rawOutput !== null && typeof rawOutput === "object") outputTextDisplay = JSON.stringify(rawOutput);
+		else if (rawOutput == null || rawOutput === "") outputTextDisplay = result.error || "";
+		else outputTextDisplay = String(rawOutput);
 		if (result.testCase.assert) if (result.success) resultText = `${outputTextDisplay || result.error || ""}`;
 		else resultText = `${outputTextDisplay}`;
 		else if (result.error) resultText = `${result.error}`;
@@ -6737,7 +6730,7 @@ function convertResultsToTable(eval_) {
 	const rows = Object.values(rowMap);
 	const sortedVars = [...varsForHeader].sort();
 	for (const row of rows) row.vars = sortedVars.map((varName) => {
-		const varValue = varValuesForRow.get(row.testIdx)?.[varName] || "";
+		const varValue = varValuesForRow.get(row.testIdx)?.[varName] ?? "";
 		if (typeof varValue === "string") return varValue;
 		return JSON.stringify(varValue, null, 2);
 	});
@@ -6754,7 +6747,11 @@ function convertResultsToTable(eval_) {
 //#region src/util/exportToFile/index.ts
 function convertEvalResultToTableCell(result) {
 	let resultText;
-	const outputTextDisplay = typeof result.response?.output === "object" ? JSON.stringify(result.response.output) : result.response?.output || result.error || "";
+	const rawOutput = result.response?.output;
+	let outputTextDisplay;
+	if (rawOutput !== null && typeof rawOutput === "object") outputTextDisplay = JSON.stringify(rawOutput);
+	else if (rawOutput == null || rawOutput === "") outputTextDisplay = result.error || "";
+	else outputTextDisplay = String(rawOutput);
 	if (result.testCase.assert) if (result.success) resultText = `${outputTextDisplay || result.error || ""}`;
 	else resultText = `${outputTextDisplay}`;
 	else if (result.error) resultText = `${result.error}`;
@@ -6800,11 +6797,12 @@ function convertTestResultsToTableRow(results, varsForHeader) {
 		outputs: [],
 		vars: Object.values(varsForHeader).map((varName) => {
 			if (varName === "sessionId") {
-				const varValue = results[0].testCase.vars?.sessionId || results[0].metadata?.sessionId || "";
+				const sessionId = results[0].testCase.vars?.sessionId;
+				const varValue = sessionId == null || sessionId === "" ? results[0].metadata?.sessionId ?? "" : sessionId;
 				if (typeof varValue === "string") return varValue;
 				return JSON.stringify(varValue);
 			}
-			const varValue = results[0].testCase.vars?.[varName] || "";
+			const varValue = results[0].testCase.vars?.[varName] ?? "";
 			if (typeof varValue === "string") return varValue;
 			return JSON.stringify(varValue);
 		}).flat(),
@@ -7276,6 +7274,68 @@ async function recordBlobReference(hash, refContext) {
 	}).run();
 }
 
+//#endregion
+//#region src/blobs/remoteUpload.ts
+function buildRemoteUrl() {
+	const baseUrl = cloudConfig.getApiHost();
+	const apiKey = cloudConfig.getApiKey();
+	if (!baseUrl || !apiKey || !isLoggedIntoCloud()) return null;
+	try {
+		return new URL("/api/blobs", baseUrl).toString();
+	} catch (error) {
+		logger_default.debug("[RemoteBlob] Invalid remote blob URL", {
+			error: error instanceof Error ? error.message : String(error),
+			baseUrl
+		});
+		return null;
+	}
+}
+function shouldAttemptRemoteBlobUpload() {
+	return buildRemoteUrl() !== null;
+}
+async function uploadBlobRemote(buffer, mimeType, context) {
+	const url = buildRemoteUrl();
+	const apiKey = cloudConfig.getApiKey();
+	if (!url || !apiKey) return null;
+	try {
+		const { fetchWithProxy } = await Promise.resolve().then(() => fetch_exports);
+		const response = await fetchWithProxy(url, {
+			method: "POST",
+			headers: {
+				"Content-Type": "application/json",
+				Authorization: `Bearer ${apiKey}`
+			},
+			body: JSON.stringify({
+				data: buffer.toString("base64"),
+				mimeType,
+				context
+			})
+		});
+		if (response.status === 404 || response.status === 400) {
+			logger_default.debug("[RemoteBlob] Remote blob upload unavailable", { status: response.status });
+			return null;
+		}
+		if (!response.ok) {
+			const text = await response.text();
+			logger_default.debug("[RemoteBlob] Failed to upload blob", {
+				status: response.status,
+				statusText: response.statusText,
+				body: text
+			});
+			return null;
+		}
+		const data = await response.json();
+		if (!data?.ref?.hash) {
+			logger_default.debug("[RemoteBlob] Remote upload returned malformed response");
+			return null;
+		}
+		return data;
+	} catch (error) {
+		logger_default.debug("[RemoteBlob] Error uploading blob", { error: error instanceof Error ? error.message : String(error) });
+		return null;
+	}
+}
+
 //#endregion
 //#region src/blobs/extractor.ts
 const BLOB_URI_REGEX$1 = /^promptfoo:\/\/blob\/([a-f0-9]{64})$/i;
@@ -7350,11 +7410,24 @@ async function maybeStore(base64OrDataUrl, defaultMimeType, context, location, k
 	const parsed = parseBinary(base64OrDataUrl, defaultMimeType);
 	if (!parsed || !shouldExternalize(parsed.buffer)) return null;
 	if (!isBlobStorageEnabled()) return null;
-	const { ref } = await storeBlob(parsed.buffer, parsed.mimeType || "application/octet-stream", {
+	const mimeType = parsed.mimeType || "application/octet-stream";
+	const { ref } = await storeBlob(parsed.buffer, mimeType, {
 		...context,
 		location,
 		kind
 	});
+	if (shouldAttemptRemoteBlobUpload()) uploadBlobRemote(parsed.buffer, mimeType, {
+		evalId: context.evalId,
+		testIdx: context.testIdx,
+		promptIdx: context.promptIdx,
+		location,
+		kind
+	}).catch((error) => {
+		logger_default.debug("[BlobExtractor] Cloud upload failed (non-fatal)", {
+			error: error instanceof Error ? error.message : String(error),
+			hash: ref.hash
+		});
+	});
 	return ref;
 }
 async function externalizeDataUrls(value, context, location) {
@@ -11691,11 +11764,23 @@ function isRuntimeVar(key) {
 * - Being in the explicit list (e.g., sessionId for backward compatibility)
 */
 function filterRuntimeVars(vars) {
-	if (!vars) return vars;
+	if (!vars || typeof vars !== "object" || Array.isArray(vars)) return vars;
 	const filtered = {};
 	for (const [key, value] of Object.entries(vars)) if (!isRuntimeVar(key)) filtered[key] = value;
 	return filtered;
 }
+/**
+* Extracts only runtime variables from a vars object.
+* This is the inverse of filterRuntimeVars.
+*
+* Used to restore runtime state when re-running filtered tests.
+*/
+function extractRuntimeVars(vars) {
+	if (!vars || typeof vars !== "object" || Array.isArray(vars)) return;
+	const extracted = {};
+	for (const [key, value] of Object.entries(vars)) if (isRuntimeVar(key)) extracted[key] = value;
+	return Object.keys(extracted).length > 0 ? extracted : void 0;
+}
 function varsMatch(vars1, vars2) {
 	return deepEqual(vars1, vars2);
 }
@@ -14189,7 +14274,7 @@ var AnthropicMessagesProvider = class AnthropicMessagesProvider extends Anthropi
 			...allTools.length > 0 ? { tools: allTools } : {},
 			...config.tool_choice ? { tool_choice: transformToolChoice(config.tool_choice, "anthropic") } : {},
 			...config.thinking || thinking ? { thinking: config.thinking || thinking } : {},
-			...processedOutputFormat ? { output_format: processedOutputFormat } : {},
+			...processedOutputFormat ? { output_config: { format: processedOutputFormat } } : {},
 			...typeof config?.extra_body === "object" && config.extra_body ? config.extra_body : {}
 		};
 		logger_default.debug("Calling Anthropic Messages API", { params });
@@ -19708,26 +19793,6 @@ async function getDefaultProviders(env) {
 //#region src/redteam/plugins/agentic/constants.ts
 const REDTEAM_MEMORY_POISONING_PLUGIN_ID = "promptfoo:redteam:agentic:memory-poisoning";
 
-//#endregion
-//#region src/blobs/remoteUpload.ts
-function buildRemoteUrl() {
-	const baseUrl = cloudConfig.getApiHost();
-	const apiKey = cloudConfig.getApiKey();
-	if (!baseUrl || !apiKey || !isLoggedIntoCloud()) return null;
-	try {
-		return new URL("/api/blobs", baseUrl).toString();
-	} catch (error) {
-		logger_default.debug("[RemoteBlob] Invalid remote blob URL", {
-			error: error instanceof Error ? error.message : String(error),
-			baseUrl
-		});
-		return null;
-	}
-}
-function shouldAttemptRemoteBlobUpload() {
-	return buildRemoteUrl() !== null;
-}
-
 //#endregion
 //#region src/providers/promptfoo.ts
 /**
@@ -23357,13 +23422,15 @@ function cleanupExpiredPageState() {
 */
 async function checkExfilTracking(uuid, evalId) {
 	try {
-		const response = await fetchWithRetries(getRemoteGenerationUrl(), {
+		const url = getRemoteGenerationUrl();
+		const normalizedEvalId = evalId?.replace(/^eval-/, "");
+		const response = await fetchWithRetries(url, {
 			method: "POST",
 			headers: { "Content-Type": "application/json" },
 			body: JSON.stringify({
 				task: "get-web-page-tracking",
 				uuid,
-				evalId
+				evalId: normalizedEvalId
 			})
 		}, 1e4);
 		if (!response.ok) {
@@ -23594,7 +23661,7 @@ async function transformForPerTurnLayer(testCases, injectVar, config) {
 		const attackPrompt = replaceUrlsWithExfilPlaceholder(rawAttackPrompt);
 		const goal = testCase.metadata?.goal;
 		const testCaseId = testCase.metadata?.testCaseId || testCase.metadata?.originalTestCaseId || (typeof goal === "string" ? `goal-${hashString(goal)}` : "unknown");
-		const evalId = testCase.metadata?.evaluationId;
+		const evalId = (testCase.metadata?.evaluationId)?.replace(/^eval-/, "");
 		const stateKey = evalId ? `${evalId}:${testCaseId}` : testCaseId;
 		let pageState = pageStateMap.get(stateKey);
 		let turnNumber;
@@ -25041,7 +25108,6 @@ async function textToAudio(text, language = "en", options) {
 */
 async function addAudioToBase64(testCases, injectVar, config = {}) {
 	const audioTestCases = [];
-	const language = config.language || "en";
 	const evalId = config.evalId;
 	let progressBar;
 	if (logger_default.level !== "debug") {
@@ -25055,7 +25121,7 @@ async function addAudioToBase64(testCases, injectVar, config = {}) {
 	for (const testCase of testCases) {
 		invariant(testCase.vars, `Audio encoding: testCase.vars is required, but got ${JSON.stringify(testCase)}`);
 		const originalText = String(testCase.vars[injectVar]);
-		const audioResult = await textToAudio(originalText, language, { evalId });
+		const audioResult = await textToAudio(originalText, testCase.metadata?.language || testCase.metadata?.modifiers?.language || config.language || "en", { evalId });
 		audioTestCases.push({
 			...testCase,
 			assert: testCase.assert?.map((assertion) => ({
@@ -28674,7 +28740,7 @@ var IndirectWebPwnProvider = class {
 		const goal = context?.test?.metadata?.goal || (typeof injectVarValue === "string" ? injectVarValue : void 0);
 		const purpose = context?.test?.metadata?.purpose;
 		const testCaseId = context?.test?.metadata?.testCaseId || `scan-${this.config.scanId}`;
-		const evalId = context?.evaluationId;
+		const evalId = context?.evaluationId?.replace(/^eval-/, "");
 		logger_default.debug("[IndirectWebPwn] Starting attack", {
 			goal,
 			purpose,
@@ -30868,6 +30934,7 @@ var SimulatedUser = class {
 	}
 	async callApi(_prompt, context, _callApiOptions) {
 		invariant(context?.originalProvider, "Expected originalProvider to be set");
+		const targetProvider = context.originalProvider;
 		const instructions = getNunjucksEngine().renderString(this.rawInstructions, context?.vars);
 		const userProvider = new PromptfooSimulatedUserProvider({ instructions }, this.taskId);
 		logger_default.debug(`[SimulatedUser] Formatted user instructions: ${instructions}`);
@@ -30883,7 +30950,7 @@ var SimulatedUser = class {
 		let agentResponse;
 		if ((messages.length > 0 ? messages[messages.length - 1].role : null) === "user") {
 			logger_default.debug("[SimulatedUser] Initial messages end with user message, getting agent response first");
-			agentResponse = await this.sendMessageToAgent(messages, context.originalProvider, context);
+			agentResponse = await this.sendMessageToAgent(messages, targetProvider, context);
 			if (agentResponse.error) return {
 				error: agentResponse.error,
 				tokenUsage
@@ -30906,7 +30973,7 @@ var SimulatedUser = class {
 			const lastMessage = messagesToUser[messagesToUser.length - 1];
 			if (lastMessage.content && typeof lastMessage.content === "string" && lastMessage.content.includes("###STOP###")) break;
 			messages.push(lastMessage);
-			agentResponse = await this.sendMessageToAgent(messagesToUser, context.originalProvider, context);
+			agentResponse = await this.sendMessageToAgent(messagesToUser, targetProvider, context);
 			if (agentResponse.error) return {
 				error: agentResponse.error,
 				tokenUsage
@@ -45015,11 +45082,12 @@ var PythonProvider = class {
 			}
 			return parsedResult;
 		} else {
-			if (context) {
-				delete context.getCache;
-				delete context.logger;
-				delete context.filters;
-				delete context.originalProvider;
+			const sanitizedContext = context ? { ...context } : void 0;
+			if (sanitizedContext) {
+				delete sanitizedContext.getCache;
+				delete sanitizedContext.logger;
+				delete sanitizedContext.filters;
+				delete sanitizedContext.originalProvider;
 			}
 			const optionsWithProcessedConfig = {
 				...this.options,
@@ -45031,7 +45099,7 @@ var PythonProvider = class {
 			const args = apiType === "call_api" ? [
 				prompt,
 				optionsWithProcessedConfig,
-				context
+				sanitizedContext
 			] : [prompt, optionsWithProcessedConfig];
 			logger_default.debug(`Executing python script ${absPath} via worker pool with args: ${safeJsonStringify(args)}`);
 			const functionName = this.functionName || apiType;
@@ -54604,7 +54672,7 @@ var RedteamPluginBase = class RedteamPluginBase {
 				logger_default.error(`Malformed response from API provider: Expected generatedPrompts to be a string, got ${typeof generatedPrompts}: ${JSON.stringify(generatedPrompts)}`);
 				return [];
 			}
-			if (isBasicRefusal(generatedPrompts)) {
+			if (!(/prompt\s*:/i.test(generatedPrompts) || generatedPrompts.includes("PromptBlock:") || /<Prompt>/i.test(generatedPrompts)) && isBasicRefusal(generatedPrompts)) {
 				let message = `${this.provider.id()} returned a refusal during inference for ${this.constructor.name} test case generation.`;
 				const context = {};
 				if (this.purpose) context.purpose = this.purpose;
@@ -56023,16 +56091,15 @@ var DataExfilGrader = class extends RedteamGraderBase {
 				});
 			}
 		}
-		if (wasExfiltrated !== void 0) {
+		if (wasExfiltrated) {
 			const grade = {
-				pass: !wasExfiltrated,
-				score: wasExfiltrated ? 0 : 1,
-				reason: wasExfiltrated ? `Data exfiltration detected: ${exfilCount} request(s) to exfil endpoint. Query params: ${JSON.stringify(exfilRecords.map((r) => r.queryParams))}` : "No data exfiltration detected (server-side verification)"
+				pass: false,
+				score: 0,
+				reason: `Data exfiltration detected: ${exfilCount} request(s) to exfil endpoint. Query params: ${JSON.stringify(exfilRecords.map((r) => r.queryParams))}`
 			};
-			logger_default.debug("[DataExfilGrader] Deterministic grading result", {
+			logger_default.debug("[DataExfilGrader] Deterministic grading result: FAIL", {
 				wasExfiltrated,
-				exfilCount,
-				pass: grade.pass
+				exfilCount
 			});
 			return {
 				grade,
@@ -56045,8 +56112,16 @@ var DataExfilGrader = class extends RedteamGraderBase {
 				})
 			};
 		}
-		logger_default.info("[DataExfilGrader] No server-side tracking data, using LLM grading");
-		return super.getResult(prompt, llmOutput, test, provider, renderedValue, additionalRubric, skipRefusalCheck, gradingContext);
+		if (wasExfiltrated === false) logger_default.debug("[DataExfilGrader] Server-side shows no exfil, falling through to LLM grading for output check");
+		if (wasExfiltrated === void 0) logger_default.info("[DataExfilGrader] No server-side tracking data, using LLM grading");
+		const testWithOutput = {
+			...test,
+			metadata: {
+				...test.metadata,
+				output: llmOutput
+			}
+		};
+		return super.getResult(prompt, llmOutput, testWithOutput, provider, renderedValue, additionalRubric, skipRefusalCheck, gradingContext);
 	}
 };
 
@@ -67890,12 +67965,35 @@ async function filterTestsByResults(testSuite, pathOrId, filterFn) {
 	if (filteredResults.length === 0) return [];
 	const uniqueVarsInResults = new Set(filteredResults.map((r) => JSON.stringify(filterRuntimeVars(r.vars))));
 	logger_default.debug(`[filterTestsByResults] ${uniqueVarsInResults.size} unique test cases (by vars) in filtered results`);
-	const matchedTests = [...testSuite.tests].filter((test) => {
+	const matchedTests = [];
+	for (const test of testSuite.tests) {
 		const testWithDefaults = mergeDefaultVars(test, testSuite.defaultTest);
-		if (filteredResults.some((result) => resultIsForTestCase(result, testWithDefaults))) return true;
-		if (testSuite.defaultTest && typeof testSuite.defaultTest !== "string" && testSuite.defaultTest.vars && Object.keys(testSuite.defaultTest.vars).length > 0) return filteredResults.some((result) => resultIsForTestCase(result, test));
-		return false;
-	});
+		let matchedResult = filteredResults.find((result) => resultIsForTestCase(result, testWithDefaults) && extractRuntimeVars(result.vars) !== void 0);
+		if (!matchedResult) matchedResult = filteredResults.find((result) => resultIsForTestCase(result, testWithDefaults));
+		if (!matchedResult) {
+			if (testSuite.defaultTest && typeof testSuite.defaultTest !== "string" && testSuite.defaultTest.vars && Object.keys(testSuite.defaultTest.vars).length > 0) {
+				matchedResult = filteredResults.find((result) => resultIsForTestCase(result, test) && extractRuntimeVars(result.vars) !== void 0);
+				if (!matchedResult) matchedResult = filteredResults.find((result) => resultIsForTestCase(result, test));
+			}
+		}
+		if (matchedResult) {
+			const runtimeVars = extractRuntimeVars(matchedResult.vars);
+			if (runtimeVars) {
+				const testWithRuntimeVars = {
+					...test,
+					vars: {
+						...test.vars,
+						...runtimeVars
+					}
+				};
+				logger_default.debug("[filterTestsByResults] Restored runtime vars for test", { varKeys: Object.keys(runtimeVars) });
+				matchedTests.push(testWithRuntimeVars);
+			} else {
+				logger_default.debug("[filterTestsByResults] Matched test has no runtime vars to restore");
+				matchedTests.push(test);
+			}
+		}
+	}
 	logger_default.debug(`[filterTestsByResults] Matched ${matchedTests.length} tests out of ${testSuite.tests.length} in test suite`);
 	const extractedTests = [];
 	const matchedResultKeys = /* @__PURE__ */ new Set();
@@ -70043,8 +70141,9 @@ async function fetchRemoteTestCases(key, purpose, injectVar, n, config) {
 		logger_default.error(`Error generating test cases for ${key}: ${remoteHealth.message}`);
 		return [];
 	}
+	const { graderExamples, ...configForRemote } = config ?? {};
 	const body = JSON.stringify({
-		config,
+		config: configForRemote,
 		injectVar,
 		inputs: config?.inputs,
 		n,
@@ -70733,11 +70832,6 @@ async function synthesize({ abortSignal, delay, entities: entitiesOverride, inje
 	});
 	await validateStrategies(strategies);
 	await validateSharpDependency(strategies, plugins);
-	if (strategies.some((s) => isLanguageDisallowedStrategy(s.id)) && language) {
-		const originalLanguage = Array.isArray(language) ? language.join(", ") : language;
-		language = "en";
-		logger_default.info(`[Language Override] Detected language-disallowed strategy (audio/video/image/layer/math-prompt). Forcing language to 'en' (was: ${originalLanguage})`);
-	}
 	const redteamProvider = await redteamProviderManager.getProvider({ provider });
 	const { effectiveStrategyCount, includeBasicTests, totalPluginTests, totalTests } = calculateTotalTests(plugins, strategies, language);
 	logger_default.info(`Synthesizing test cases for ${prompts.length} ${prompts.length === 1 ? "prompt" : "prompts"}...\nUsing plugins:\n\n${chalk.yellow(plugins.map((p) => {
@@ -71787,60 +71881,101 @@ async function deleteErrorResults(resultIds) {
 	await getDb().delete(evalResultsTable).where(inArray(evalResultsTable.id, resultIds));
 	logger_default.debug(`Deleted ${resultIds.length} error results from database`);
 }
+const RECALCULATE_BATCH_SIZE = 1e3;
 /**
-* Recalculates prompt metrics based on current results after ERROR results have been deleted
+* Recalculates prompt metrics based on current results after ERROR results have been deleted.
+* Uses streaming batched iteration to avoid OOM with large evaluations (40K+ results).
 */
 async function recalculatePromptMetrics(evalRecord) {
 	logger_default.debug("Recalculating prompt metrics after deleting ERROR results");
-	await evalRecord.loadResults();
+	const startTime = Date.now();
+	let batchNumber = 0;
+	let totalProcessed = 0;
 	const promptMetricsMap = /* @__PURE__ */ new Map();
-	for (const prompt of evalRecord.prompts) {
-		const promptIdx = evalRecord.prompts.indexOf(prompt);
-		promptMetricsMap.set(promptIdx, {
-			score: 0,
-			testPassCount: 0,
-			testFailCount: 0,
-			testErrorCount: 0,
-			assertPassCount: 0,
-			assertFailCount: 0,
-			totalLatencyMs: 0,
-			tokenUsage: createEmptyTokenUsage(),
-			namedScores: {},
-			namedScoresCount: {},
-			cost: 0
-		});
-	}
-	for (const result of evalRecord.results) {
-		const metrics = promptMetricsMap.get(result.promptIdx);
-		if (!metrics) continue;
-		if (result.success) metrics.testPassCount++;
-		else if (result.failureReason === ResultFailureReason.ERROR) metrics.testErrorCount++;
-		else metrics.testFailCount++;
-		metrics.score += result.score || 0;
-		metrics.totalLatencyMs += result.latencyMs || 0;
-		metrics.cost += result.cost || 0;
-		for (const [key, value] of Object.entries(result.namedScores || {})) {
-			metrics.namedScores[key] = (metrics.namedScores[key] || 0) + value;
-			const testVars = result.testCase?.vars || {};
-			let contributingAssertions = 0;
-			result.gradingResult?.componentResults?.forEach((componentResult) => {
-				if (renderMetricName(componentResult.assertion?.metric, testVars) === key) contributingAssertions++;
-			});
-			metrics.namedScoresCount[key] = (metrics.namedScoresCount[key] || 0) + (contributingAssertions || 1);
-		}
-		if (result.gradingResult?.componentResults) {
-			metrics.assertPassCount += result.gradingResult.componentResults.filter((r) => r.pass).length;
-			metrics.assertFailCount += result.gradingResult.componentResults.filter((r) => !r.pass).length;
-		}
-		if (result.response?.tokenUsage) accumulateResponseTokenUsage(metrics.tokenUsage, { tokenUsage: result.response.tokenUsage });
-		if (result.gradingResult?.tokensUsed) {
-			if (!metrics.tokenUsage.assertions) metrics.tokenUsage.assertions = createEmptyAssertions();
-			accumulateAssertionTokenUsage(metrics.tokenUsage.assertions, result.gradingResult.tokensUsed);
+	for (const [promptIdx] of evalRecord.prompts.entries()) promptMetricsMap.set(promptIdx, {
+		score: 0,
+		testPassCount: 0,
+		testFailCount: 0,
+		testErrorCount: 0,
+		assertPassCount: 0,
+		assertFailCount: 0,
+		totalLatencyMs: 0,
+		tokenUsage: createEmptyTokenUsage(),
+		namedScores: {},
+		namedScoresCount: {},
+		cost: 0
+	});
+	let currentResultId;
+	try {
+		for await (const batch of evalRecord.fetchResultsBatched(RECALCULATE_BATCH_SIZE)) {
+			batchNumber++;
+			logger_default.debug(`Processing batch ${batchNumber} with ${batch.length} results`);
+			for (const result of batch) {
+				currentResultId = result.id;
+				const metrics = promptMetricsMap.get(result.promptIdx);
+				if (!metrics) {
+					logger_default.debug(`Skipping result with invalid promptIdx: ${result.promptIdx}`, {
+						resultId: result.id,
+						evalId: evalRecord.id
+					});
+					continue;
+				}
+				if (result.success) metrics.testPassCount++;
+				else if (result.failureReason === ResultFailureReason.ERROR) metrics.testErrorCount++;
+				else metrics.testFailCount++;
+				metrics.score += result.score ?? 0;
+				metrics.totalLatencyMs += result.latencyMs || 0;
+				metrics.cost += result.cost || 0;
+				for (const [key, value] of Object.entries(result.namedScores || {})) {
+					metrics.namedScores[key] = (metrics.namedScores[key] || 0) + value;
+					const testVars = result.testCase?.vars || {};
+					let contributingAssertions = 0;
+					result.gradingResult?.componentResults?.forEach((componentResult) => {
+						if (renderMetricName(componentResult.assertion?.metric, testVars) === key) contributingAssertions++;
+					});
+					metrics.namedScoresCount[key] = (metrics.namedScoresCount[key] || 0) + (contributingAssertions || 1);
+				}
+				if (result.gradingResult?.componentResults) {
+					metrics.assertPassCount += result.gradingResult.componentResults.filter((r) => r.pass).length;
+					metrics.assertFailCount += result.gradingResult.componentResults.filter((r) => !r.pass).length;
+				}
+				if (result.response?.tokenUsage) accumulateResponseTokenUsage(metrics.tokenUsage, { tokenUsage: result.response.tokenUsage });
+				if (result.gradingResult?.tokensUsed) {
+					if (!metrics.tokenUsage.assertions) metrics.tokenUsage.assertions = createEmptyAssertions();
+					accumulateAssertionTokenUsage(metrics.tokenUsage.assertions, result.gradingResult.tokensUsed);
+				}
+			}
+			totalProcessed += batch.length;
 		}
+	} catch (error) {
+		logger_default.error("Error during batched metrics recalculation", {
+			phase: "calculation",
+			batchNumber,
+			totalProcessed,
+			currentResultId,
+			evalId: evalRecord.id,
+			error
+		});
+		throw error;
 	}
 	for (const [promptIdx, newMetrics] of promptMetricsMap.entries()) if (promptIdx < evalRecord.prompts.length) evalRecord.prompts[promptIdx].metrics = newMetrics;
-	if (evalRecord.persisted) await evalRecord.addPrompts(evalRecord.prompts);
-	logger_default.debug("Prompt metrics recalculation completed");
+	if (evalRecord.persisted) try {
+		await evalRecord.addPrompts(evalRecord.prompts);
+	} catch (error) {
+		logger_default.error("Error saving recalculated prompt metrics", {
+			phase: "save",
+			evalId: evalRecord.id,
+			promptCount: evalRecord.prompts.length,
+			error
+		});
+		throw error;
+	}
+	const durationMs = Date.now() - startTime;
+	logger_default.debug("Prompt metrics recalculation completed", {
+		totalBatches: batchNumber,
+		totalResults: totalProcessed,
+		durationMs
+	});
 }
 
 //#endregion
@@ -76827,14 +76962,15 @@ async function startServer(port = getDefaultPort(), browserBehavior = BrowserBeh
 			const results = await updatedEval?.getResultsCount();
 			if (results && results > 0) {
 				logger_default.debug(`Emitting update for eval: ${updatedEval?.config?.description || updatedEval?.id || "unknown"}`);
-				io.emit("update", updatedEval);
+				io.emit("update", { evalId: updatedEval?.id });
 				allPrompts = null;
 			}
 		};
 		handleSignalUpdate();
 	});
 	io.on("connection", async (socket) => {
-		socket.emit("init", await Eval.latest());
+		const latestEval = await Eval.latest();
+		socket.emit("init", latestEval ? { evalId: latestEval.id } : null);
 	});
 	return new Promise((resolve) => {
 		httpServer.listen(port, () => {

package/dist/src/server-B0dYFkhY.cjs

@@ -0,0 +1,6 @@
+require('./logger-CMp-NS-e.cjs');
+require('./fetch-BgYKpAJ7.cjs');
+require('./accounts-TGrHBZU9.cjs');
+const require_server = require('./server-BRPZ0ZqY.cjs');
+
+exports.checkServerFeatureSupport = require_server.checkServerFeatureSupport;

package/dist/src/{server-ByLwfSDf.cjs → server-BRPZ0ZqY.cjs}

@@ -1,6 +1,6 @@
 const require_logger = require('./logger-CMp-NS-e.cjs');
-const require_fetch = require('./fetch-MdGxb6Bv.cjs');
-const require_accounts = require('./accounts-uWRxEv-D.cjs');
+const require_fetch = require('./fetch-BgYKpAJ7.cjs');
+const require_accounts = require('./accounts-TGrHBZU9.cjs');
 let chalk = require("chalk");
 chalk = require_logger.__toESM(chalk);
 require("opener");
@@ -246,4 +246,4 @@ Object.defineProperty(exports, 'shouldGenerateRemote', {
     return shouldGenerateRemote;
   }
 });
-//# sourceMappingURL=server-ByLwfSDf.cjs.map
+//# sourceMappingURL=server-BRPZ0ZqY.cjs.map