promptfoo 0.121.4 → 0.121.5

package/dist/src/index.cjs

@@ -4,34 +4,35 @@ Object.defineProperties(exports, {
 });
 const require_logger = require("./logger-COuQb2xB.cjs");
 const require_invariant = require("./invariant-kfQ8Bu82.cjs");
-const require_esm = require("./esm-CtEPLdAj.cjs");
-const require_pythonUtils = require("./pythonUtils-tAJvvpS-.cjs");
-const require_fileExtensions = require("./fileExtensions-bYh77CN8.cjs");
-const require_transform = require("./transform-CUnzlsbn.cjs");
-const require_graders = require("./graders--zknU_uk.cjs");
-const require_types = require("./types-CeaeaZdP.cjs");
-const require_util = require("./util-Bxn8emtE.cjs");
-const require_render = require("./render-DfQSFxGE.cjs");
-const require_fetch = require("./fetch-CJU5ELPa.cjs");
-const require_cache = require("./cache-BIyPcp5v.cjs");
-const require_providers = require("./providers-1eKkXBKp.cjs");
-const require_utils = require("./utils-B05gLxER.cjs");
-const require_createHash = require("./createHash-B7KvgoOD.cjs");
-const require_chat = require("./chat-CclRbxGf.cjs");
-const require_tokenUsageUtils = require("./tokenUsageUtils-Bb7DkZPz.cjs");
-const require_transform$1 = require("./transform-CTeuTR3S.cjs");
-const require_util$1 = require("./util-CN3SrLT4.cjs");
-const require_providerRegistry = require("./providerRegistry-CQMdTmHP.cjs");
-const require_server = require("./server-Dx2TyCH2.cjs");
-const require_accounts = require("./accounts-Dy17bs4D.cjs");
-const require_blobs = require("./blobs-D2FAd1Q5.cjs");
-const require_tables = require("./tables-DUfh1F7Z.cjs");
-const require_extractor = require("./extractor-WVPOrH43.cjs");
-const require_telemetry = require("./telemetry-Dsw_faFj.cjs");
-const require_text = require("./text-BVi-cLPJ.cjs");
-const require_store = require("./store-DAAyxcy6.cjs");
-const require_rubyUtils = require("./rubyUtils-B6eljPuh.cjs");
-const require_evalResult = require("./evalResult-KZqXl4XP.cjs");
+const require_fetch = require("./fetch-NuqXW1Xb.cjs");
+const require_version = require("./version-D9zu9FWB.cjs");
+const require_types = require("./types-CgG2rKiW.cjs");
+const require_accounts = require("./accounts-BIFntVWB.cjs");
+const require_esm = require("./esm-B_rGuPTo.cjs");
+const require_render = require("./render-tG6ir9_g.cjs");
+const require_providerRegistry = require("./providerRegistry-BESeALrr.cjs");
+const require_server = require("./server-BEECpeGG.cjs");
+const require_providers = require("./providers-eDShy16E.cjs");
+const require_pythonUtils = require("./pythonUtils-CoLaCwNY.cjs");
+const require_fileExtensions = require("./fileExtensions-D9h-8Wxg.cjs");
+const require_util = require("./util-DvpHnLt0.cjs");
+const require_tokenUsageUtils = require("./tokenUsageUtils-C9odhsbW.cjs");
+const require_blobs = require("./blobs-B95F_7vE.cjs");
+const require_tables = require("./tables-CXbaZ9y1.cjs");
+const require_extractor = require("./extractor-Dxr2J_wK.cjs");
+const require_cache = require("./cache-DGg-yTZG.cjs");
+const require_chat = require("./chat-vYqqv1gP.cjs");
+const require_transform = require("./transform-Dg4LcO1Y.cjs");
+const require_util$1 = require("./util-A5_ZsQUn.cjs");
+const require_transform$1 = require("./transform-C9izGX54.cjs");
+const require_telemetry = require("./telemetry-CEQxGnMZ.cjs");
+const require_text = require("./text-BiNME7QG.cjs");
+const require_store = require("./store-2K0kDi80.cjs");
+const require_createHash = require("./createHash-VvBIc-AW.cjs");
+const require_rubyUtils = require("./rubyUtils-DoifqkiA.cjs");
+const require_graders = require("./graders-Bw1wk_21.cjs");
+const require_utils = require("./utils-DkVeShIB.cjs");
+const require_evalResult = require("./evalResult-Cx-8OWkb.cjs");
 let fs = require("fs");
 fs = require_logger.__toESM(fs);
 let path = require("path");
@@ -41,34 +42,34 @@ async = require_logger.__toESM(async);
 let js_yaml = require("js-yaml");
 js_yaml = require_logger.__toESM(js_yaml);
 let node_async_hooks = require("node:async_hooks");
-require("node:path");
-require("node:url");
 let chalk = require("chalk");
 chalk = require_logger.__toESM(chalk);
 let os = require("os");
 os = require_logger.__toESM(os);
-let util = require("util");
-util = require_logger.__toESM(util);
 let dedent = require("dedent");
 dedent = require_logger.__toESM(dedent);
+let zod = require("zod");
 let fs_promises = require("fs/promises");
 fs_promises = require_logger.__toESM(fs_promises);
-let glob = require("glob");
-let zod = require("zod");
-let csv_parse_sync = require("csv-parse/sync");
-let fast_xml_parser = require("fast-xml-parser");
+let util = require("util");
+util = require_logger.__toESM(util);
+let _inquirer_input = require("@inquirer/input");
+_inquirer_input = require_logger.__toESM(_inquirer_input);
+require("node:path");
+require("node:url");
 let crypto$1 = require("crypto");
 crypto$1 = require_logger.__toESM(crypto$1);
 let _opentelemetry_api = require("@opentelemetry/api");
-let _inquirer_input = require("@inquirer/input");
-_inquirer_input = require_logger.__toESM(_inquirer_input);
 let readline = require("readline");
 readline = require_logger.__toESM(readline);
+let csv_parse_sync = require("csv-parse/sync");
+let glob = require("glob");
 let drizzle_orm = require("drizzle-orm");
+let fast_xml_parser = require("fast-xml-parser");
 let cli_progress = require("cli-progress");
 cli_progress = require_logger.__toESM(cli_progress);
 let url = require("url");
-let jsdom = require("jsdom");
+let parse5 = require("parse5");
 let fastest_levenshtein = require("fastest-levenshtein");
 let js_rouge = require("js-rouge");
 js_rouge = require_logger.__toESM(js_rouge);
@@ -262,6 +263,502 @@ const handleConversationRelevance = async ({ assertion, outputString, prompt, pr
 	};
 };
 //#endregion
+//#region src/matchers/classification.ts
+/**
+*
+* @param expected Expected classification. If undefined, matches any classification.
+* @param output Text to classify.
+* @param threshold Value between 0 and 1. If the expected classification is undefined, the threshold is the minimum score for any classification. If the expected classification is defined, the threshold is the minimum score for that classification.
+* @param grading
+* @returns Pass if the output matches the classification with a score greater than or equal to the threshold.
+*/
+async function matchesClassification(expected, output, threshold, grading) {
+	const resp = await (await require_graders.getAndCheckProvider("classification", grading?.provider, null, "classification check")).callClassificationApi(output);
+	if (!resp.classification) return require_graders.fail(resp.error || "Unknown error fetching classification");
+	let score;
+	if (expected === void 0) {
+		const scores = Object.values(resp.classification);
+		if (scores.length === 0) return {
+			pass: false,
+			score: 0,
+			reason: "No classification scores returned"
+		};
+		score = Math.max(...scores);
+	} else score = resp.classification[expected] || 0;
+	if (score >= threshold - Number.EPSILON) {
+		const reason = expected === void 0 ? `Maximum classification score ${score.toFixed(2)} >= ${threshold}` : `Classification ${expected} has score ${score.toFixed(2)} >= ${threshold}`;
+		return {
+			pass: true,
+			score,
+			reason
+		};
+	}
+	return {
+		pass: false,
+		score,
+		reason: expected === void 0 ? `Maximum classification score ${score.toFixed(2)} < ${threshold}` : `Classification ${expected} has score ${score.toFixed(2)} < ${threshold}`
+	};
+}
+//#endregion
+//#region src/matchers/comparison.ts
+async function matchesSelectBest(criteria, outputs, grading, vars, providerCallContext) {
+	require_invariant.invariant(outputs.length >= 2, "select-best assertion must have at least two outputs to compare between");
+	const resp = await require_graders.callProviderWithContext(await require_graders.getAndCheckProvider("text", grading?.provider, (await require_graders.getDefaultProviders()).gradingProvider, "select-best check"), await require_graders.renderLlmRubricPrompt(await require_graders.loadRubricPrompt(grading?.rubricPrompt, require_graders.SELECT_BEST_PROMPT), {
+		criteria,
+		outputs: outputs.map((o) => require_graders.tryParse(o)),
+		...vars || {}
+	}), "select-best", {
+		criteria,
+		outputs: outputs.map((o) => require_graders.tryParse(o)),
+		...vars || {}
+	}, providerCallContext);
+	if (resp.error || !resp.output) return Array.from({ length: outputs.length }, () => require_graders.fail(resp.error || "No output", resp.tokenUsage));
+	require_invariant.invariant(typeof resp.output === "string", "select-best produced malformed response");
+	const firstIntegerMatch = resp.output.trim().match(/\d+/);
+	const verdict = firstIntegerMatch ? Number.parseInt(firstIntegerMatch[0], 10) : NaN;
+	if (Number.isNaN(verdict) || verdict < 0 || verdict >= outputs.length) return Array.from({ length: outputs.length }, () => require_graders.fail(`Invalid select-best verdict: ${verdict}`, resp.tokenUsage));
+	const tokensUsed = require_graders.normalizeMatcherTokenUsage(resp.tokenUsage);
+	return outputs.map((_output, index) => {
+		if (index === verdict) return {
+			pass: true,
+			score: 1,
+			reason: `Output selected as the best: ${criteria}`,
+			tokensUsed
+		};
+		else return {
+			pass: false,
+			score: 0,
+			reason: `Output not selected: ${criteria}`,
+			tokensUsed
+		};
+	});
+}
+async function selectMaxScore(outputs, resultsWithGradingResults, assertion) {
+	require_invariant.invariant(outputs.length >= 2, "max-score assertion must have at least two outputs to compare between");
+	const value = assertion.value || {};
+	const options = {
+		method: typeof value === "object" && "method" in value ? value.method : "average",
+		weights: typeof value === "object" && "weights" in value ? value.weights : {},
+		threshold: typeof value === "object" && "threshold" in value ? value.threshold : void 0
+	};
+	const scores = resultsWithGradingResults.map((result, index) => {
+		const relevantResults = (result.gradingResult?.componentResults || []).filter((r) => r.assertion && r.assertion.type !== "max-score" && r.assertion.type !== "select-best");
+		if (relevantResults.length === 0) throw new Error("max-score requires at least one other assertion (besides max-score or select-best) to aggregate scores from");
+		let totalWeightedScore = 0;
+		let totalWeight = 0;
+		relevantResults.forEach((componentResult) => {
+			const assertionType = componentResult.assertion?.type || "unknown";
+			const weight = options.weights[assertionType] === void 0 ? 1 : options.weights[assertionType];
+			const score = componentResult.score || 0;
+			totalWeightedScore += score * weight;
+			totalWeight += weight;
+		});
+		let aggregateScore;
+		if (options.method === "sum") aggregateScore = totalWeightedScore;
+		else aggregateScore = totalWeight > 0 ? totalWeightedScore / totalWeight : 0;
+		return {
+			index,
+			score: aggregateScore,
+			componentCount: relevantResults.length,
+			totalWeight
+		};
+	});
+	let maxScore = -Infinity;
+	let winnerIndex = 0;
+	for (let i = 0; i < scores.length; i++) if (scores[i].score > maxScore) {
+		maxScore = scores[i].score;
+		winnerIndex = i;
+	}
+	const meetsThreshold = options.threshold === void 0 || maxScore >= options.threshold;
+	return scores.map(({ index, score, componentCount, totalWeight }) => {
+		const isWinner = index === winnerIndex && meetsThreshold;
+		return {
+			pass: isWinner,
+			score: isWinner ? 1 : 0,
+			reason: isWinner ? `Selected as highest scoring output (score: ${score.toFixed(3)})` : score === maxScore && !meetsThreshold ? `Not selected - score ${score.toFixed(3)} below threshold ${options.threshold}` : `Not selected (score: ${score.toFixed(3)}, max: ${maxScore.toFixed(3)})`,
+			namedScores: {
+				maxScore: score,
+				assertionCount: componentCount,
+				totalWeight
+			}
+		};
+	});
+}
+//#endregion
+//#region src/matchers/moderation.ts
+async function matchesModeration({ userPrompt, assistantResponse, categories = [] }, grading) {
+	if (!assistantResponse) return {
+		pass: true,
+		score: 1,
+		reason: "No output to moderate"
+	};
+	const defaultProviders = await require_graders.getDefaultProviders();
+	const defaultModerationProvider = !require_logger.getEnvString("OPENAI_API_KEY") && (require_logger.getEnvString("REPLICATE_API_KEY") || require_logger.getEnvString("REPLICATE_API_TOKEN")) ? await require_providers.loadApiProvider(require_types.LLAMA_GUARD_REPLICATE_PROVIDER) : defaultProviders.moderationProvider;
+	const moderationProvider = await require_graders.getAndCheckProvider("moderation", grading?.provider, defaultModerationProvider, "moderation check");
+	require_invariant.invariant(moderationProvider, "Moderation provider must be defined");
+	const resp = await moderationProvider.callModerationApi(userPrompt, assistantResponse);
+	if (resp.error) return {
+		pass: false,
+		score: 0,
+		reason: `Moderation API error: ${resp.error}`
+	};
+	const { flags } = resp;
+	if (!flags || flags.length === 0) return {
+		pass: true,
+		score: 1,
+		reason: "No moderation flags detected"
+	};
+	const filteredFlags = categories.length === 0 ? flags : flags.filter((flag) => categories.includes(flag.code));
+	if (filteredFlags.length > 0) return {
+		pass: false,
+		score: 0,
+		reason: `Moderation flags detected: ${filteredFlags.map((flag) => flag.description).join(", ")}`
+	};
+	return {
+		pass: true,
+		score: 1,
+		reason: "No relevant moderation flags detected"
+	};
+}
+//#endregion
+//#region src/assertions/contextUtils.ts
+/**
+* Resolves the context value for context-based assertions.
+* Supports extracting context from test variables or transforming from output.
+* Can return either a single context string or an array of context chunks.
+*
+* @param assertion - The assertion configuration
+* @param test - The test case
+* @param output - The provider output (after provider transform, before test transform)
+* @param prompt - The prompt text
+* @param fallbackContext - Optional fallback context (e.g., prompt for context-recall)
+* @param providerResponse - Optional full provider response for contextTransform
+* @returns The resolved context string or array of strings
+* @throws Error if context cannot be resolved or transform fails
+*/
+async function resolveContext(assertion, test, output, prompt, fallbackContext, providerResponse) {
+	let contextValue;
+	if (test.vars?.context) {
+		if (typeof test.vars.context === "string") contextValue = test.vars.context;
+		else if (Array.isArray(test.vars.context)) {
+			const invalidEntry = [...test.vars.context.entries()].find(([, v]) => typeof v !== "string");
+			if (invalidEntry) {
+				const [idx, val] = invalidEntry;
+				require_invariant.invariant(false, `Invalid context: expected an array of strings, but found ${typeof val} at index ${idx}`);
+			}
+			contextValue = test.vars.context;
+		}
+	} else if (fallbackContext) contextValue = fallbackContext;
+	if (assertion.contextTransform) try {
+		const outputForTransform = providerResponse?.providerTransformedOutput ?? output;
+		const transformed = await require_transform$1.transform(assertion.contextTransform, outputForTransform, {
+			vars: test.vars,
+			prompt: { label: prompt },
+			...providerResponse && providerResponse.metadata && { metadata: providerResponse.metadata }
+		});
+		require_invariant.invariant(typeof transformed === "string" || Array.isArray(transformed) && transformed.every((item) => typeof item === "string"), `contextTransform must return a string or array of strings. Got ${typeof transformed}. Check your transform expression: ${assertion.contextTransform}`);
+		contextValue = transformed;
+	} catch (error) {
+		throw new Error(`Failed to transform context using expression '${assertion.contextTransform}': ${error instanceof Error ? error.message : String(error)}`);
+	}
+	require_invariant.invariant(typeof contextValue === "string" && contextValue.length > 0 || Array.isArray(contextValue) && contextValue.length > 0 && contextValue.every((item) => typeof item === "string" && item.length > 0), "Context is required for context-based assertions. Provide either a \"context\" variable (string or array of strings) in your test case or use \"contextTransform\" to extract context from the provider response.");
+	return contextValue;
+}
+/**
+* Serializes context (string or string[]) to a single string for prompts.
+* Joins chunks with double newlines to preserve separation.
+*/
+function serializeContext(context) {
+	return Array.isArray(context) ? context.join("\n\n") : context;
+}
+//#endregion
+//#region src/matchers/rag.ts
+async function matchesAnswerRelevance(input, output, threshold, grading, providerCallContext) {
+	const defaults = await require_graders.getDefaultProviders();
+	const embeddingProvider = await require_graders.getAndCheckProvider("embedding", grading?.provider, defaults.embeddingProvider, "answer relevancy check");
+	const textProvider = await require_graders.getAndCheckProvider("text", grading?.provider, defaults.gradingProvider, "answer relevancy check");
+	const tokensUsed = require_graders.normalizeMatcherTokenUsage(void 0);
+	const rubricPrompt = await require_graders.loadRubricPrompt(grading?.rubricPrompt, require_graders.ANSWER_RELEVANCY_GENERATE);
+	const parsedOutput = require_graders.tryParse(output);
+	const promptText = await require_graders.renderLlmRubricPrompt(rubricPrompt, { answer: parsedOutput });
+	const candidateQuestions = [];
+	for (let i = 0; i < 3; i++) {
+		const resp = await require_graders.callProviderWithContext(textProvider, promptText, "answer-relevance", { answer: parsedOutput }, providerCallContext);
+		require_tokenUsageUtils.accumulateTokenUsage(tokensUsed, resp.tokenUsage);
+		if (resp.error || !resp.output) return require_graders.fail(resp.error || "No output", tokensUsed);
+		require_invariant.invariant(typeof resp.output === "string", "answer relevancy check produced malformed response");
+		candidateQuestions.push(resp.output);
+	}
+	require_invariant.invariant(typeof embeddingProvider.callEmbeddingApi === "function", `Provider ${embeddingProvider.id()} must implement callEmbeddingApi for similarity check`);
+	const inputEmbeddingResp = await embeddingProvider.callEmbeddingApi(input);
+	require_tokenUsageUtils.accumulateTokenUsage(tokensUsed, inputEmbeddingResp.tokenUsage);
+	if (inputEmbeddingResp.error || !inputEmbeddingResp.embedding) return require_graders.fail(inputEmbeddingResp.error || "No embedding", tokensUsed);
+	const inputEmbedding = inputEmbeddingResp.embedding;
+	const similarities = [];
+	const questionsWithScores = [];
+	for (const question of candidateQuestions) {
+		const resp = await embeddingProvider.callEmbeddingApi(question);
+		require_tokenUsageUtils.accumulateTokenUsage(tokensUsed, resp.tokenUsage);
+		if (resp.error || !resp.embedding) return require_graders.fail(resp.error || "No embedding", tokensUsed);
+		const questionSimilarity = require_graders.cosineSimilarity(inputEmbedding, resp.embedding);
+		similarities.push(questionSimilarity);
+		questionsWithScores.push({
+			question,
+			similarity: questionSimilarity
+		});
+	}
+	const similarity = similarities.reduce((a, b) => a + b, 0) / similarities.length;
+	const pass = similarity >= threshold - Number.EPSILON;
+	const greaterThanReason = `Relevance ${similarity.toFixed(2)} is greater than threshold ${threshold}`;
+	const lessThanReason = `Relevance ${similarity.toFixed(2)} is less than threshold ${threshold}`;
+	const metadata = {
+		generatedQuestions: questionsWithScores,
+		averageSimilarity: similarity,
+		threshold
+	};
+	if (pass) return {
+		pass: true,
+		score: similarity,
+		reason: greaterThanReason,
+		tokensUsed,
+		metadata
+	};
+	return {
+		pass: false,
+		score: similarity,
+		reason: lessThanReason,
+		tokensUsed,
+		metadata
+	};
+}
+async function matchesContextRecall(context, groundTruth, threshold, grading, vars, providerCallContext) {
+	const textProvider = await require_graders.getAndCheckProvider("text", grading?.provider, (await require_graders.getDefaultProviders()).gradingProvider, "context recall check");
+	const contextString = serializeContext(context);
+	const resp = await require_graders.callProviderWithContext(textProvider, await require_graders.renderLlmRubricPrompt(await require_graders.loadRubricPrompt(grading?.rubricPrompt, require_graders.CONTEXT_RECALL), {
+		context: contextString,
+		groundTruth,
+		...vars || {}
+	}), "context-recall", {
+		context: contextString,
+		groundTruth,
+		...vars || {}
+	}, providerCallContext);
+	if (resp.error || !resp.output) return require_graders.fail(resp.error || "No output", resp.tokenUsage);
+	require_invariant.invariant(typeof resp.output === "string", "context-recall produced malformed response");
+	const attributedTokenLower = require_graders.CONTEXT_RECALL_ATTRIBUTED_TOKEN.toLowerCase();
+	const notAttributedTokenLower = require_graders.CONTEXT_RECALL_NOT_ATTRIBUTED_TOKEN.toLowerCase();
+	const sentences = require_graders.splitIntoSentences(resp.output).filter((line) => {
+		const lowerLine = line.toLowerCase();
+		return lowerLine.includes(attributedTokenLower) || lowerLine.includes(notAttributedTokenLower);
+	});
+	const sentenceAttributions = [];
+	let numerator = 0;
+	for (const sentence of sentences) {
+		const lowerSentence = sentence.toLowerCase();
+		const isAttributed = !lowerSentence.includes(notAttributedTokenLower) && lowerSentence.includes(attributedTokenLower);
+		if (isAttributed) numerator++;
+		const sentenceMatch = sentence.match(/^\d+\.\s*([^\.]+\.)/);
+		const cleanSentence = sentenceMatch ? sentenceMatch[1].trim() : sentence.split(".")[0].trim();
+		sentenceAttributions.push({
+			sentence: cleanSentence,
+			attributed: isAttributed
+		});
+	}
+	const score = sentences.length > 0 ? numerator / sentences.length : 0;
+	const pass = score >= threshold - Number.EPSILON;
+	const metadata = {
+		sentenceAttributions,
+		totalSentences: sentences.length,
+		attributedSentences: numerator,
+		score
+	};
+	return {
+		pass,
+		score,
+		reason: pass ? `Recall ${score.toFixed(2)} is >= ${threshold}` : `Recall ${score.toFixed(2)} is < ${threshold}`,
+		tokensUsed: require_graders.normalizeMatcherTokenUsage(resp.tokenUsage),
+		metadata
+	};
+}
+async function matchesContextRelevance(question, context, threshold, grading, providerCallContext) {
+	const textProvider = await require_graders.getAndCheckProvider("text", grading?.provider, (await require_graders.getDefaultProviders()).gradingProvider, "context relevance check");
+	const contextString = serializeContext(context);
+	const resp = await require_graders.callProviderWithContext(textProvider, await require_graders.renderLlmRubricPrompt(await require_graders.loadRubricPrompt(grading?.rubricPrompt, require_graders.CONTEXT_RELEVANCE), {
+		context: contextString,
+		query: question
+	}), "context-relevance", {
+		context: contextString,
+		query: question
+	}, providerCallContext);
+	if (resp.error || !resp.output) return require_graders.fail(resp.error || "No output", resp.tokenUsage);
+	require_invariant.invariant(typeof resp.output === "string", "context-relevance produced malformed response");
+	const contextUnits = Array.isArray(context) ? context.filter((chunk) => chunk.trim().length > 0) : require_graders.splitIntoSentences(context);
+	const totalContextUnits = contextUnits.length;
+	const extractedSentences = require_graders.splitIntoSentences(resp.output);
+	const relevantSentences = [];
+	const insufficientInformation = resp.output.includes(require_graders.CONTEXT_RELEVANCE_BAD);
+	let numerator = 0;
+	if (insufficientInformation) numerator = 0;
+	else {
+		const uniqueRelevantSentences = [...new Set(extractedSentences)];
+		numerator = Math.min(uniqueRelevantSentences.length, totalContextUnits);
+		relevantSentences.push(...uniqueRelevantSentences);
+	}
+	const score = totalContextUnits > 0 ? numerator / totalContextUnits : 0;
+	const pass = score >= threshold - Number.EPSILON;
+	const metadata = {
+		extractedSentences: relevantSentences,
+		totalContextUnits,
+		totalContextSentences: totalContextUnits,
+		contextUnits,
+		relevantSentenceCount: numerator,
+		insufficientInformation,
+		score
+	};
+	return {
+		pass,
+		score,
+		reason: pass ? `Context relevance ${score.toFixed(2)} is >= ${threshold}` : `Context relevance ${score.toFixed(2)} is < ${threshold}`,
+		tokensUsed: require_graders.normalizeMatcherTokenUsage(resp.tokenUsage),
+		metadata
+	};
+}
+async function matchesContextFaithfulness(query, output, context, threshold, grading, vars, providerCallContext) {
+	const textProvider = await require_graders.getAndCheckProvider("text", grading?.provider, (await require_graders.getDefaultProviders()).gradingProvider, "faithfulness check");
+	const tokensUsed = require_graders.normalizeMatcherTokenUsage(void 0);
+	if (grading?.rubricPrompt) require_invariant.invariant(Array.isArray(grading.rubricPrompt), "rubricPrompt must be an array");
+	const rawLongformPrompt = typeof grading?.rubricPrompt?.[0] === "string" ? grading?.rubricPrompt?.[0] : grading?.rubricPrompt?.[0]?.content;
+	const rawNliPrompt = typeof grading?.rubricPrompt?.[1] === "string" ? grading?.rubricPrompt?.[1] : grading?.rubricPrompt?.[1]?.content;
+	const longformPrompt = await require_graders.loadRubricPrompt(rawLongformPrompt, require_graders.CONTEXT_FAITHFULNESS_LONGFORM);
+	const nliPrompt = await require_graders.loadRubricPrompt(rawNliPrompt, require_graders.CONTEXT_FAITHFULNESS_NLI_STATEMENTS);
+	let promptText = await require_graders.renderLlmRubricPrompt(longformPrompt, {
+		question: query,
+		answer: require_graders.tryParse(output),
+		...vars || {}
+	});
+	let resp = await require_graders.callProviderWithContext(textProvider, promptText, "context-faithfulness-longform", {
+		question: query,
+		answer: require_graders.tryParse(output),
+		...vars || {}
+	}, providerCallContext);
+	require_tokenUsageUtils.accumulateTokenUsage(tokensUsed, resp.tokenUsage);
+	if (resp.error || !resp.output) return require_graders.fail(resp.error || "No output", tokensUsed);
+	require_invariant.invariant(typeof resp.output === "string", "context-faithfulness produced malformed response");
+	const contextString = serializeContext(context);
+	const statements = require_graders.splitIntoSentences(resp.output);
+	promptText = await require_graders.renderLlmRubricPrompt(nliPrompt, {
+		context: contextString,
+		statements,
+		...vars || {}
+	});
+	resp = await require_graders.callProviderWithContext(textProvider, promptText, "context-faithfulness-nli", {
+		context: contextString,
+		statements,
+		...vars || {}
+	}, providerCallContext);
+	require_tokenUsageUtils.accumulateTokenUsage(tokensUsed, resp.tokenUsage);
+	if (resp.error || !resp.output) return require_graders.fail(resp.error || "No output", tokensUsed);
+	require_invariant.invariant(typeof resp.output === "string", "context-faithfulness produced malformed response");
+	let finalAnswer = "Final verdict for each statement in order:";
+	finalAnswer = finalAnswer.toLowerCase();
+	let verdicts = resp.output.toLowerCase().trim();
+	let score = 0;
+	if (statements.length > 0) if (verdicts.includes(finalAnswer)) {
+		verdicts = verdicts.slice(verdicts.indexOf(finalAnswer) + finalAnswer.length);
+		const parsedVerdicts = verdicts.split(".").filter((answer) => answer.trim() !== "");
+		if (parsedVerdicts.length > 0) score = 1 - parsedVerdicts.filter((answer) => !answer.includes("yes")).length / statements.length;
+	} else {
+		const noVerdictCount = verdicts.split("verdict: no").length - 1;
+		if (noVerdictCount + (verdicts.split("verdict: yes").length - 1) > 0) score = 1 - noVerdictCount / statements.length;
+	}
+	score = Math.min(1, Math.max(0, score));
+	const pass = score >= threshold - Number.EPSILON;
+	return {
+		pass,
+		score,
+		reason: pass ? `Faithfulness ${score.toFixed(2)} is >= ${threshold}` : `Faithfulness ${score.toFixed(2)} is < ${threshold}`,
+		tokensUsed
+	};
+}
+//#endregion
+//#region src/matchers/similarity.ts
+function calculateSimilarityScore(expectedEmbedding, outputEmbedding, metric, tokensUsed) {
+	switch (metric) {
+		case "cosine": return require_graders.cosineSimilarity(expectedEmbedding, outputEmbedding);
+		case "dot_product": return require_graders.dotProduct(expectedEmbedding, outputEmbedding);
+		case "euclidean": return require_graders.euclideanDistance(expectedEmbedding, outputEmbedding);
+		default: return require_graders.fail(`Unsupported metric: ${metric}`, tokensUsed);
+	}
+}
+function buildSimilarityResult(similarity, threshold, inverse, metric, tokensUsed) {
+	if (metric === "euclidean") {
+		const distance = similarity;
+		const pass = inverse ? distance >= threshold - Number.EPSILON : distance <= threshold + Number.EPSILON;
+		const normalizedScore = 1 / (1 + distance);
+		const score = inverse ? 1 - normalizedScore : normalizedScore;
+		const belowThresholdReason = `Distance ${distance.toFixed(2)} is less than or equal to threshold ${threshold}`;
+		const aboveThresholdReason = `Distance ${distance.toFixed(2)} is greater than threshold ${threshold}`;
+		return {
+			pass,
+			score,
+			reason: pass ? inverse ? aboveThresholdReason : belowThresholdReason : inverse ? belowThresholdReason : aboveThresholdReason,
+			tokensUsed
+		};
+	}
+	const pass = inverse ? similarity <= threshold + Number.EPSILON : similarity >= threshold - Number.EPSILON;
+	const score = inverse ? 1 - similarity : similarity;
+	const greaterThanReason = `Similarity ${similarity.toFixed(2)} is greater than or equal to threshold ${threshold}`;
+	const lessThanReason = `Similarity ${similarity.toFixed(2)} is less than threshold ${threshold}`;
+	return {
+		pass,
+		score,
+		reason: pass ? inverse ? lessThanReason : greaterThanReason : inverse ? greaterThanReason : lessThanReason,
+		tokensUsed
+	};
+}
+async function calculateProviderSimilarity(finalProvider, expected, output, metric, tokensUsed) {
+	if (metric === "cosine" && "callSimilarityApi" in finalProvider) {
+		const similarityResp = await finalProvider.callSimilarityApi(expected, output);
+		require_tokenUsageUtils.accumulateTokenUsage(tokensUsed, similarityResp.tokenUsage);
+		if (similarityResp.error) return require_graders.fail(similarityResp.error, tokensUsed);
+		if (similarityResp.similarity == null) return require_graders.fail("Unknown error fetching similarity", tokensUsed);
+		if (!Number.isFinite(similarityResp.similarity)) return require_graders.fail(`Invalid similarity score: ${similarityResp.similarity}`, tokensUsed);
+		return similarityResp.similarity;
+	}
+	const callEmbeddingApi = "callEmbeddingApi" in finalProvider ? finalProvider.callEmbeddingApi : void 0;
+	if (typeof callEmbeddingApi !== "function") {
+		if ("callSimilarityApi" in finalProvider) return require_graders.fail(`Provider ${finalProvider.id()} only supports cosine similarity via callSimilarityApi`, tokensUsed);
+		throw new Error("Provider must implement callSimilarityApi or callEmbeddingApi");
+	}
+	const [expectedEmbedding, outputEmbedding] = await Promise.all([callEmbeddingApi.call(finalProvider, expected), callEmbeddingApi.call(finalProvider, output)]);
+	const mergedUsage = require_graders.normalizeMatcherTokenUsage(void 0);
+	require_tokenUsageUtils.accumulateTokenUsage(mergedUsage, expectedEmbedding.tokenUsage);
+	require_tokenUsageUtils.accumulateTokenUsage(mergedUsage, outputEmbedding.tokenUsage);
+	require_tokenUsageUtils.accumulateTokenUsage(tokensUsed, mergedUsage);
+	if (expectedEmbedding.error || outputEmbedding.error) return require_graders.fail(expectedEmbedding.error || outputEmbedding.error || "Unknown error fetching embeddings", tokensUsed);
+	if (!expectedEmbedding.embedding || !outputEmbedding.embedding) return require_graders.fail("Embedding not found", tokensUsed);
+	return calculateSimilarityScore(expectedEmbedding.embedding, outputEmbedding.embedding, metric, tokensUsed);
+}
+async function matchesSimilarity(expected, output, threshold, inverse = false, grading, metric = "cosine") {
+	if (metric === "cosine" && require_logger.state.config?.redteam && require_server.shouldGenerateRemote({ requireEmbeddingProvider: true })) try {
+		return await require_graders.doRemoteGrading({
+			task: "similar",
+			expected,
+			output,
+			threshold,
+			inverse
+		});
+	} catch (error) {
+		return require_graders.fail(`Could not perform remote grading: ${error}`);
+	}
+	const defaults = await require_graders.getDefaultProviders();
+	const finalProvider = await require_graders.getAndCheckProvider("embedding", grading?.provider, defaults.embeddingProvider, "similarity check");
+	const tokensUsed = require_graders.normalizeMatcherTokenUsage(void 0);
+	const similarity = await calculateProviderSimilarity(finalProvider, expected, output, metric, tokensUsed);
+	if (typeof similarity !== "number") return similarity;
+	return buildSimilarityResult(similarity, threshold, inverse, metric, tokensUsed);
+}
+//#endregion
 //#region src/tracing/evaluatorTracing.ts
 let otlpReceiverStarted = false;
 const DEFAULT_OTLP_ACCEPT_FORMATS = ["json", "protobuf"];
@@ -305,7 +802,7 @@ async function startOtlpReceiverIfNeeded(testSuite) {
 		require_telemetry.telemetry.record("feature_used", { feature: "tracing" });
 		try {
 			require_logger.logger.debug("[EvaluatorTracing] Tracing configuration detected, starting OTLP receiver");
-			const { startOTLPReceiver } = await Promise.resolve().then(() => require("./otlpReceiver-BfcVq2Nq.cjs"));
+			const { startOTLPReceiver } = await Promise.resolve().then(() => require("./otlpReceiver-DNSQj6bf.cjs"));
 			const port = testSuite.tracing.otlp.http.port || 4318;
 			const host = testSuite.tracing.otlp.http.host || "127.0.0.1";
 			const acceptFormats = normalizeOtlpAcceptFormats(testSuite.tracing.otlp.http.acceptFormats);
@@ -329,7 +826,7 @@ async function startOtlpReceiverIfNeeded(testSuite) {
 async function stopOtlpReceiverIfNeeded() {
 	if (otlpReceiverStarted) try {
 		require_logger.logger.debug("[EvaluatorTracing] Stopping OTLP receiver");
-		const { stopOTLPReceiver } = await Promise.resolve().then(() => require("./otlpReceiver-BfcVq2Nq.cjs"));
+		const { stopOTLPReceiver } = await Promise.resolve().then(() => require("./otlpReceiver-DNSQj6bf.cjs"));
 		await stopOTLPReceiver();
 		otlpReceiverStarted = false;
 		require_logger.logger.info("[EvaluatorTracing] OTLP receiver stopped successfully");
@@ -364,7 +861,7 @@ async function generateTraceContextIfNeeded(test, evaluateOptions, testIdx, prom
 	}
 	if (!tracingEnabled) return null;
 	require_logger.logger.debug("[EvaluatorTracing] Importing trace store");
-	const { getTraceStore } = await Promise.resolve().then(() => require("./store-CWOSz6D_.cjs"));
+	const { getTraceStore } = await Promise.resolve().then(() => require("./store-uQZ4AjPe.cjs"));
 	const traceStore = getTraceStore();
 	const traceId = generateTraceId();
 	const spanId = generateSpanId();
@@ -406,7 +903,7 @@ const handleAnswerRelevance = async ({ assertion, output, prompt, test, provider
 	require_invariant.invariant(prompt, "answer-relevance assertion type must have a prompt");
 	return {
 		assertion,
-		...await require_graders.matchesAnswerRelevance(typeof test?.vars?.query === "string" ? test.vars.query : prompt, output, assertion.threshold ?? 0, test.options, providerCallContext)
+		...await matchesAnswerRelevance(typeof test?.vars?.query === "string" ? test.vars.query : prompt, output, assertion.threshold ?? 0, test.options, providerCallContext)
 	};
 };
 //#endregion
@@ -662,7 +1159,7 @@ function handleBleuScore({ assertion, inverse, outputString, renderedValue }) {
 //#region src/assertions/classifier.ts
 async function handleClassifier({ assertion, renderedValue, outputString, test, inverse }) {
 	require_invariant.invariant(typeof renderedValue === "string" || typeof renderedValue === "undefined", "\"classifier\" assertion type must have a string value or be undefined");
-	const classificationResult = await require_graders.matchesClassification(renderedValue, outputString, assertion.threshold ?? 1, test.options);
+	const classificationResult = await matchesClassification(renderedValue, outputString, assertion.threshold ?? 1, test.options);
 	if (inverse) {
 		classificationResult.pass = !classificationResult.pass;
 		classificationResult.score = 1 - classificationResult.score;
@@ -674,38 +1171,84 @@ async function handleClassifier({ assertion, renderedValue, outputString, test,
 }
 //#endregion
 //#region src/assertions/contains.ts
+/**
+* Advance over separators between parsed fields.
+*
+* Contains-any values allow whitespace around comma delimiters, and historical
+* parsing ignored repeated commas rather than producing empty fields.
+*/
+function skipWhitespaceAndCommas(value, startIndex) {
+	let i = startIndex;
+	while (i < value.length) {
+		i = skipWhitespace(value, i);
+		if (value[i] !== ",") break;
+		i++;
+	}
+	return i;
+}
+/**
+* Advance over whitespace while preserving comma delimiter handling for callers.
+*/
+function skipWhitespace(value, startIndex) {
+	let i = startIndex;
+	while (i < value.length && /\s/.test(value[i])) i++;
+	return i;
+}
+/**
+* Parse a quoted field using the assertion parser's CSV-like escape rules.
+*
+* Supports backslash-escaped quotes/backslashes and doubled quotes, and rejects
+* unterminated fields so malformed assertion values do not silently pass.
+*/
+function parseQuotedField(value, startIndex) {
+	let i = startIndex + 1;
+	let field = "";
+	let terminated = false;
+	while (i < value.length) if (value[i] === "\\" && i + 1 < value.length && ["\"", "\\"].includes(value[i + 1])) {
+		field += value[i + 1];
+		i += 2;
+	} else if (value[i] === "\"" && i + 1 < value.length && value[i + 1] === "\"") {
+		field += "\"";
+		i += 2;
+	} else if (value[i] === "\"") {
+		i++;
+		terminated = true;
+		break;
+	} else {
+		field += value[i];
+		i++;
+	}
+	require_invariant.invariant(terminated, "Unterminated quoted field in contains assertion value");
+	return {
+		field,
+		nextIndex: i
+	};
+}
+/**
+* Parse an unquoted field up to the next comma, trimming surrounding whitespace.
+*/
+function parseUnquotedField(value, startIndex) {
+	let i = startIndex;
+	while (i < value.length && value[i] !== ",") i++;
+	return {
+		field: value.substring(startIndex, i).trim(),
+		nextIndex: i
+	};
+}
+/**
+* Split a contains-any string into fields while preserving quoted commas.
+*/
 function parseCommaSeparatedValues(value) {
 	const results = [];
 	let i = 0;
 	while (i < value.length) {
-		while (i < value.length && /\s/.test(value[i])) i++;
+		i = skipWhitespaceAndCommas(value, i);
 		if (i >= value.length) break;
-		if (value[i] === ",") {
-			i++;
-			continue;
-		}
-		if (value[i] === "\"") {
-			i++;
-			let field = "";
-			while (i < value.length) if (value[i] === "\\" && i + 1 < value.length && (value[i + 1] === "\"" || value[i + 1] === "\\")) {
-				field += value[i + 1];
-				i += 2;
-			} else if (value[i] === "\"" && i + 1 < value.length && value[i + 1] === "\"") {
-				field += "\"";
-				i += 2;
-			} else if (value[i] === "\"") {
-				i++;
-				break;
-			} else {
-				field += value[i];
-				i++;
-			}
-			results.push(field);
-		} else {
-			const start = i;
-			while (i < value.length && value[i] !== ",") i++;
-			results.push(value.substring(start, i).trim());
-		}
+		const isQuotedField = value[i] === "\"";
+		const parsed = isQuotedField ? parseQuotedField(value, i) : parseUnquotedField(value, i);
+		results.push(parsed.field);
+		i = isQuotedField ? skipWhitespace(value, parsed.nextIndex) : parsed.nextIndex;
+		require_invariant.invariant(!isQuotedField || i >= value.length || value[i] === ",", "Expected comma after quoted field in contains assertion value");
 	}
 	return results;
 }
@@ -803,10 +1346,10 @@ async function handleContextFaithfulness({ assertion, test, output, prompt, prov
 	require_invariant.invariant(test.vars, "context-faithfulness assertion requires a test with variables");
 	require_invariant.invariant(typeof test.vars.query === "string", "context-faithfulness assertion requires a \"query\" variable with the user question");
 	require_invariant.invariant(typeof output === "string", "context-faithfulness assertion requires string output from the provider");
-	const context = await require_graders.resolveContext(assertion, test, output, prompt, void 0, providerResponse);
+	const context = await resolveContext(assertion, test, output, prompt, void 0, providerResponse);
 	return {
 		assertion,
-		...await require_graders.matchesContextFaithfulness(test.vars.query, output, context, assertion.threshold ?? 0, test.options, test.vars, providerCallContext),
+		...await matchesContextFaithfulness(test.vars.query, output, context, assertion.threshold ?? 0, test.options, test.vars, providerCallContext),
 		metadata: { context }
 	};
 }
@@ -825,8 +1368,8 @@ async function handleContextFaithfulness({ assertion, test, output, prompt, prov
 const handleContextRecall = async ({ assertion, renderedValue, prompt, test, output, providerResponse, providerCallContext }) => {
 	require_invariant.invariant(typeof renderedValue === "string", "context-recall assertion requires a string value (expected answer or fact to verify)");
 	require_invariant.invariant(prompt, "context-recall assertion requires a prompt");
-	const context = await require_graders.resolveContext(assertion, test, output, prompt, prompt, providerResponse);
-	const result = await require_graders.matchesContextRecall(context, renderedValue, assertion.threshold ?? 0, test.options, test.vars, providerCallContext);
+	const context = await resolveContext(assertion, test, output, prompt, prompt, providerResponse);
+	const result = await matchesContextRecall(context, renderedValue, assertion.threshold ?? 0, test.options, test.vars, providerCallContext);
 	return {
 		assertion,
 		...result,
@@ -851,8 +1394,8 @@ const handleContextRecall = async ({ assertion, renderedValue, prompt, test, out
 const handleContextRelevance = async ({ assertion, test, output, prompt, providerResponse, providerCallContext }) => {
 	require_invariant.invariant(test.vars, "context-relevance assertion requires a test with variables");
 	require_invariant.invariant(typeof test.vars.query === "string", "context-relevance assertion requires a \"query\" variable with the user question");
-	const context = await require_graders.resolveContext(assertion, test, output, prompt, void 0, providerResponse);
-	const result = await require_graders.matchesContextRelevance(test.vars.query, context, assertion.threshold ?? 0, test.options, providerCallContext);
+	const context = await resolveContext(assertion, test, output, prompt, void 0, providerResponse);
+	const result = await matchesContextRelevance(test.vars.query, context, assertion.threshold ?? 0, test.options, providerCallContext);
 	return {
 		assertion,
 		...result,
@@ -930,7 +1473,7 @@ function handleFinishReason({ assertion, inverse = false, renderedValue, provide
 //#region src/assertions/functionToolCall.ts
 const handleIsValidFunctionCall = ({ assertion, output, provider, test }) => {
 	try {
-		if (provider instanceof require_providers.AIStudioChatProvider || provider instanceof require_providers.GoogleLiveProvider || provider instanceof require_providers.VertexChatProvider) require_transform$1.validateFunctionCall(output, provider.config?.tools, test.vars);
+		if (provider instanceof require_providers.AIStudioChatProvider || provider instanceof require_providers.GoogleLiveProvider || provider instanceof require_providers.VertexChatProvider) require_transform.validateFunctionCall(output, provider.config?.tools, test.vars);
 		else if (provider instanceof require_chat.OpenAiChatCompletionProvider) require_util$1.validateFunctionCall(output, provider.config.functions, test.vars);
 		else throw new Error(`Provider does not have functionality for checking function call.`);
 		return {
@@ -1110,6 +1653,43 @@ const handleGuardrails = async ({ assertion, inverse, providerResponse }) => {
 };
 //#endregion
 //#region src/assertions/html.ts
+const LITERAL_WRAPPER_PATTERNS = {
+	html: /<html(?=[\s>/])/,
+	head: /<head(?=[\s>/])/,
+	body: /<body(?=[\s>/])/
+};
+function isWrapperTagName(tagName) {
+	return tagName === "html" || tagName === "head" || tagName === "body";
+}
+function isTextNode(node) {
+	return node.nodeName === "#text";
+}
+function isElementNode(node) {
+	return "tagName" in node;
+}
+function hasSourceCodeLocation(element) {
+	return "sourceCodeLocation" in element && element.sourceCodeLocation !== null && element.sourceCodeLocation !== void 0;
+}
+function getChildNodes(node) {
+	return "childNodes" in node ? node.childNodes : [];
+}
+function findFirstElement(root, predicate) {
+	const stack = [root];
+	while (stack.length > 0) {
+		const current = stack.pop();
+		if (isElementNode(current) && predicate(current)) return current;
+		const children = getChildNodes(current);
+		for (let i = children.length - 1; i >= 0; i--) stack.push(children[i]);
+	}
+}
+function hasTopLevelText(parentNode) {
+	return parentNode.childNodes.some((node) => isTextNode(node) && Boolean(node.value.trim()));
+}
+function isUserProvidedElement(element, inputLowercase) {
+	const tagName = element.tagName.toLowerCase();
+	if (isWrapperTagName(tagName)) return LITERAL_WRAPPER_PATTERNS[tagName].test(inputLowercase) && hasSourceCodeLocation(element);
+	return VALID_HTML_ELEMENTS.has(tagName) || tagName.includes("-");
+}
 const HTML_PATTERNS = {
 	openingTag: /<[a-zA-Z][a-zA-Z0-9-]*(?:\s[^>]*)?>/,
 	closingTag: /<\/[a-zA-Z][a-zA-Z0-9-]*\s*>/,
@@ -1265,37 +1845,21 @@ function validateHtml(htmlString) {
 		isValid: false,
 		reason: "Output appears to be XML, not HTML"
 	};
-	try {
-		const { document } = new jsdom.JSDOM(trimmed, { contentType: "text/html" }).window;
-		if (document.body && !trimmed.toLowerCase().includes("<body")) {
-			if (Array.from(document.body.childNodes).some((node) => node.nodeType === 3 && node.textContent?.trim())) return {
-				isValid: false,
-				reason: "Output must be wrapped in HTML tags"
-			};
-		}
-		const allElements = document.querySelectorAll("*");
-		if (!Array.from(allElements).find((element) => {
-			const tagName = element.tagName.toLowerCase();
-			if ([
-				"html",
-				"head",
-				"body"
-			].includes(tagName) && !trimmed.toLowerCase().includes(`<${tagName}`)) return false;
-			return VALID_HTML_ELEMENTS.has(tagName) || tagName.includes("-");
-		})) return {
-			isValid: false,
-			reason: "Output does not contain recognized HTML elements"
-		};
-		return {
-			isValid: true,
-			reason: "Output is valid HTML"
-		};
-	} catch (error) {
-		return {
-			isValid: false,
-			reason: `HTML parsing failed: ${error instanceof Error ? error.message : "Unknown error"}`
-		};
-	}
+	const document = (0, parse5.parse)(trimmed, { sourceCodeLocationInfo: true });
+	const inputLowercase = trimmed.toLowerCase();
+	const body = findFirstElement(document, (element) => element.tagName === "body");
+	if (!(body !== void 0 && LITERAL_WRAPPER_PATTERNS.body.test(inputLowercase) && hasSourceCodeLocation(body)) && body && hasTopLevelText(body)) return {
+		isValid: false,
+		reason: "Output must be wrapped in HTML tags"
+	};
+	if (!findFirstElement(document, (element) => isUserProvidedElement(element, inputLowercase))) return {
+		isValid: false,
+		reason: "Output does not contain recognized HTML elements"
+	};
+	return {
+		isValid: true,
+		reason: "Output is valid HTML"
+	};
 }
 const handleContainsHtml = ({ assertion, outputString, inverse }) => {
 	const pass = containsHtml(outputString) !== inverse;
@@ -1460,7 +2024,7 @@ const handleJavascript = async ({ assertion, renderedValue, valueFromScript, ass
 		let result;
 		if (typeof valueFromScript === "undefined") {
 			const functionBody = renderedValue.includes("\n") ? renderedValue : buildFunctionBody(renderedValue);
-			result = await validateResult(new Function("output", "context", "process", functionBody)(output, assertionValueContext, require_transform.getProcessShim()));
+			result = await validateResult(new Function("output", "context", "process", functionBody)(output, assertionValueContext, require_transform$1.getProcessShim()));
 		} else {
 			require_invariant.invariant(typeof valueFromScript === "boolean" || typeof valueFromScript === "number" || typeof valueFromScript === "object", `Javascript assertion script must return a boolean, number, or object (${assertion.value})`);
 			result = await validateResult(valueFromScript);
@@ -1667,7 +2231,7 @@ const handleModeration = async ({ assertion, test, outputString, providerRespons
 		const parsedPrompt = require_fetch.parseChatPrompt(promptToModerate, null);
 		if (parsedPrompt && parsedPrompt.length > 0) promptToModerate = getLastModerationPrompt(parsedPrompt) ?? promptToModerate;
 	} catch {}
-	const moderationResult = await require_graders.matchesModeration({
+	const moderationResult = await matchesModeration({
 		userPrompt: promptToModerate,
 		assistantResponse: outputString,
 		categories: Array.isArray(assertion.value) ? assertion.value : []
@@ -2400,11 +2964,10 @@ function handleRougeScore({ baseType, assertion, renderedValue, outputString, in
 	const rougeMethod = js_rouge[baseType[baseType.length - 1]];
 	const score = rougeMethod(outputString, renderedValue, {});
 	const threshold = assertion.threshold ?? .75;
-	const pass = score >= threshold != inverse;
 	return {
-		pass,
+		pass: score >= threshold !== inverse,
 		score: inverse ? 1 - score : score,
-		reason: pass ? `${baseType.toUpperCase()} score ${score.toFixed(2)} is greater than or equal to threshold ${threshold}` : `${baseType.toUpperCase()} score ${score.toFixed(2)} is less than threshold ${threshold}`,
+		reason: `${baseType.toUpperCase()} score ${score.toFixed(2)} is ${score >= threshold ? "greater than or equal to" : "less than"} threshold ${threshold}`,
 		assertion
 	};
 }
@@ -2466,10 +3029,196 @@ const handleRuby = async ({ assertion, renderedValue, valueFromScript, assertion
 	}
 };
 //#endregion
+//#region src/providers/webSearchUtils.ts
+function hasTool(provider, predicate) {
+	return Array.isArray(provider.config?.tools) && provider.config.tools.some(predicate);
+}
+function getProviderId(provider) {
+	if (typeof provider.id !== "function") return null;
+	try {
+		return provider.id();
+	} catch (err) {
+		require_logger.logger.debug(`Failed to read provider id: ${err}`);
+		return null;
+	}
+}
+function isOpenAiResponsesProvider(provider, id) {
+	return id.includes("openai:responses") || provider.constructor?.name === "OpenAiResponsesProvider";
+}
+/**
+* Check if a provider has web search capabilities
+* @param provider The provider to check
+* @returns true if the provider supports web search
+*/
+function hasWebSearchCapability(provider) {
+	if (!provider) return false;
+	const id = getProviderId(provider);
+	if (!id) return false;
+	if (id.includes("perplexity")) return true;
+	if ((id.includes("google") || id.includes("gemini") || id.includes("vertex")) && hasTool(provider, (t) => t.googleSearch !== void 0)) return true;
+	if (id.includes("xai") && provider.config?.search_parameters?.mode === "on") return true;
+	if (isOpenAiResponsesProvider(provider, id) && hasTool(provider, (t) => t.type === "web_search_preview")) return true;
+	if (id.startsWith("openai:codex") && (provider.config?.web_search_mode === "live" || provider.config?.web_search_mode === "cached" || provider.config?.web_search_enabled === true)) return true;
+	if (id.includes("anthropic") && hasTool(provider, (t) => t.type === "web_search_20250305")) return true;
+	return false;
+}
+/**
+* Load a provider with web search capabilities.
+* Tries multiple providers in order of preference until one succeeds.
+* Uses the latest and most capable models from each provider with specific checkpoint IDs.
+*
+* @param preferAnthropic Whether to try Anthropic first (true) or OpenAI first (false)
+* @returns A provider with web search capabilities or null
+*/
+async function loadWebSearchProvider(preferAnthropic = false) {
+	const loadAnthropicWebSearch = async () => {
+		try {
+			return await require_providers.loadApiProvider("anthropic:messages:claude-opus-4-6", { options: { config: { tools: [{
+				type: "web_search_20250305",
+				name: "web_search",
+				max_uses: 5
+			}] } } });
+		} catch (err) {
+			require_logger.logger.debug(`Failed to load Anthropic web search provider: ${err}`);
+			return null;
+		}
+	};
+	const loadOpenAIWebSearch = async () => {
+		try {
+			return await require_providers.loadApiProvider("openai:responses:gpt-5.4-2026-03-05", { options: { config: { tools: [{ type: "web_search_preview" }] } } });
+		} catch (err) {
+			require_logger.logger.debug(`Failed to load OpenAI web search provider: ${err}`);
+			return null;
+		}
+	};
+	const loadPerplexity = async () => {
+		try {
+			return await require_providers.loadApiProvider("perplexity:sonar-pro");
+		} catch (err) {
+			require_logger.logger.debug(`Failed to load Perplexity provider: ${err}`);
+			return null;
+		}
+	};
+	const loadGoogleWebSearch = async () => {
+		try {
+			return await require_providers.loadApiProvider("google:gemini-3-pro-preview", { options: { config: { tools: [{ googleSearch: {} }] } } });
+		} catch (err) {
+			require_logger.logger.debug(`Failed to load Google web search provider: ${err}`);
+			return null;
+		}
+	};
+	const loadVertexWebSearch = async () => {
+		try {
+			return await require_providers.loadApiProvider("vertex:gemini-3-pro-preview", { options: { config: { tools: [{ googleSearch: {} }] } } });
+		} catch (err) {
+			require_logger.logger.debug(`Failed to load Vertex web search provider: ${err}`);
+			return null;
+		}
+	};
+	const loadXaiWebSearch = async () => {
+		try {
+			return await require_providers.loadApiProvider("xai:grok-4-1-fast-reasoning", { options: { config: { search_parameters: { mode: "on" } } } });
+		} catch (err) {
+			require_logger.logger.debug(`Failed to load xAI web search provider: ${err}`);
+			return null;
+		}
+	};
+	const providers = preferAnthropic ? [
+		loadAnthropicWebSearch,
+		loadOpenAIWebSearch,
+		loadPerplexity,
+		loadGoogleWebSearch,
+		loadVertexWebSearch,
+		loadXaiWebSearch
+	] : [
+		loadOpenAIWebSearch,
+		loadAnthropicWebSearch,
+		loadPerplexity,
+		loadGoogleWebSearch,
+		loadVertexWebSearch,
+		loadXaiWebSearch
+	];
+	for (const getProvider of providers) {
+		const provider = await getProvider();
+		if (provider && hasWebSearchCapability(provider)) {
+			require_logger.logger.info(`Using ${getProviderId(provider) ?? "loaded provider"} as web search provider`);
+			return provider;
+		}
+		if (provider) require_logger.logger.debug(`Loaded provider ${getProviderId(provider) ?? "unknown"} does not support web search`);
+	}
+	return null;
+}
+//#endregion
+//#region src/matchers/search.ts
+async function matchesSearchRubric(rubric, llmOutput, grading, vars, assertion, _provider, providerCallContext) {
+	if (!grading) throw new Error("Cannot grade output without grading config. Specify --grader option or grading config.");
+	const defaultProviders = await require_graders.getDefaultProviders();
+	const defaultSearchProviders = [
+		defaultProviders.webSearchProvider,
+		defaultProviders.llmRubricProvider,
+		defaultProviders.gradingProvider
+	];
+	let searchProvider = (grading.provider ? await require_graders.getGradingProvider("text", grading.provider, null) : null) || defaultSearchProviders.find((provider) => Boolean(provider));
+	if (!hasWebSearchCapability(searchProvider)) {
+		const webSearchDefault = defaultSearchProviders.find((provider) => hasWebSearchCapability(provider));
+		if (webSearchDefault) searchProvider = webSearchDefault;
+	}
+	if (!hasWebSearchCapability(searchProvider)) {
+		const webSearchProvider = await loadWebSearchProvider(true);
+		if (webSearchProvider) searchProvider = webSearchProvider;
+	}
+	if (!searchProvider || !hasWebSearchCapability(searchProvider)) throw new Error(`search-rubric assertion requires a grading provider with web search capabilities. Use --grader with a web search provider (e.g., anthropic:messages:${require_graders.DEFAULT_ANTHROPIC_MODEL}, openai:responses:o4-mini with tools configured, perplexity:sonar) or configure one in defaultTest.options.provider`);
+	const prompt = await require_graders.renderLlmRubricPrompt(await require_graders.loadRubricPrompt(grading?.rubricPrompt, require_graders.DEFAULT_WEB_SEARCH_PROMPT), {
+		output: require_graders.tryParse(llmOutput),
+		rubric,
+		...vars || {}
+	});
+	const resp = await require_graders.callProviderWithContext(searchProvider, prompt, "search-rubric", {
+		output: require_graders.tryParse(llmOutput),
+		rubric,
+		...vars || {}
+	}, providerCallContext);
+	if (resp.error || !resp.output) return {
+		pass: false,
+		score: 0,
+		reason: `Search rubric evaluation failed: ${resp.error || "No output"}`,
+		tokensUsed: resp.tokenUsage,
+		assertion
+	};
+	try {
+		const result = require_logger.extractFirstJsonObject(String(resp.output));
+		let pass = result.pass ?? false;
+		const score = typeof result.score === "number" ? result.score : pass ? 1 : 0;
+		if (assertion?.threshold !== void 0) pass = pass && score >= assertion.threshold;
+		return {
+			pass,
+			score,
+			reason: result.reason || "No reason provided",
+			tokensUsed: resp.tokenUsage,
+			assertion,
+			metadata: {
+				searchResults: result.searchResults || [],
+				searchProvider: searchProvider.id()
+			}
+		};
+	} catch (err) {
+		require_logger.logger.warn(`[search-rubric] Could not parse structured JSON from provider response, falling back to substring matching: ${err.message}`);
+		const outputLower = String(resp.output).toLowerCase();
+		const pass = outputLower.includes("\"pass\":true") || outputLower.includes("\"pass\": true");
+		return {
+			pass,
+			score: pass ? 1 : 0,
+			reason: resp.output,
+			tokensUsed: resp.tokenUsage,
+			assertion
+		};
+	}
+}
+//#endregion
 //#region src/assertions/searchRubric.ts
 async function handleSearchRubric({ assertion, baseType: _baseType, inverse, provider, providerCallContext, renderedValue, test, providerResponse }) {
 	if (renderedValue == null) throw new Error("search-rubric assertion type must have a string value");
-	const result = await require_graders.matchesSearchRubric(String(renderedValue), providerResponse.output, test.options, test.vars, assertion, provider, providerCallContext);
+	const result = await matchesSearchRubric(String(renderedValue), providerResponse.output, test.options, test.vars, assertion, provider, providerCallContext);
 	if (inverse) {
 		result.pass = !result.pass;
 		result.reason = result.pass ? `Output does not require web search verification: ${result.reason}` : `Output requires web search verification: ${result.reason}`;
@@ -2500,7 +3249,7 @@ const handleSimilar = async ({ assertion, renderedValue, outputString, inverse,
 	if (Array.isArray(renderedValue)) {
 		let minScore = Number.POSITIVE_INFINITY;
 		for (const value of renderedValue) {
-			const result = await require_graders.matchesSimilarity(value, outputString, threshold, inverse, test.options, metric);
+			const result = await matchesSimilarity(value, outputString, threshold, inverse, test.options, metric);
 			if (result.pass) return {
 				assertion,
 				...result
@@ -2515,7 +3264,7 @@ const handleSimilar = async ({ assertion, renderedValue, outputString, inverse,
 		};
 	} else return {
 		assertion,
-		...await require_graders.matchesSimilarity(renderedValue, outputString, threshold, inverse, test.options, metric)
+		...await matchesSimilarity(renderedValue, outputString, threshold, inverse, test.options, metric)
 	};
 };
 //#endregion
@@ -3551,7 +4300,7 @@ const ASSERTION_HANDLERS = {
 	"llm-rubric": handleLlmRubric,
 	meteor: async (params) => {
 		try {
-			const { handleMeteorAssertion } = await Promise.resolve().then(() => require("./meteor-Co1VQ1u5.cjs"));
+			const { handleMeteorAssertion } = await Promise.resolve().then(() => require("./meteor-BBGcGeCa.cjs"));
 			return handleMeteorAssertion(params);
 		} catch (error) {
 			if (error instanceof Error && (error.message.includes("Cannot find module") || error.message.includes("natural\" package is required"))) return {
@@ -3633,7 +4382,7 @@ async function runAssertion({ prompt, provider, assertion, test, vars, latencyMs
 	const { cost, logProbs, output: originalOutput } = providerResponse;
 	let output = originalOutput;
 	require_invariant.invariant(assertion.type, `Assertion must have a type: ${JSON.stringify(assertion)}`);
-	if (assertion.transform) output = await require_transform.transform(assertion.transform, output, {
+	if (assertion.transform) output = await require_transform$1.transform(assertion.transform, output, {
 		vars: resolvedVars,
 		prompt: { label: prompt },
 		...providerResponse && providerResponse.metadata && { metadata: providerResponse.metadata }
@@ -3687,7 +4436,7 @@ async function runAssertion({ prompt, provider, assertion, test, vars, latencyMs
 			};
 		}
 		else if (filePath.endsWith(".rb")) try {
-			const { runRuby } = await Promise.resolve().then(() => require("./rubyUtils-DUbq4tff.cjs"));
+			const { runRuby } = await Promise.resolve().then(() => require("./rubyUtils-CO-tuszQ.cjs"));
 			valueFromScript = await runRuby(filePath, functionName || "get_assert", [output, context]);
 			require_logger.logger.debug(`Ruby script ${filePath} output: ${valueFromScript}`);
 		} catch (error) {
@@ -3840,7 +4589,7 @@ async function runAssertions({ assertScoringFunction, latencyMs, prompt, provide
 async function runCompareAssertion(test, assertion, outputs, context) {
 	require_invariant.invariant(typeof assertion.value === "string", "select-best must have a string value");
 	test = require_graders.getFinalTest(test, assertion);
-	return (await require_graders.matchesSelectBest(assertion.value, outputs, test.options, test.vars, context)).map((result) => ({
+	return (await matchesSelectBest(assertion.value, outputs, test.options, test.vars, context)).map((result) => ({
 		...result,
 		assertion
 	}));
@@ -3857,17 +4606,17 @@ async function readAssertions(filePath) {
 var assertions_default = {
 	runAssertion,
 	runAssertions,
-	matchesSimilarity: require_graders.matchesSimilarity,
-	matchesClassification: require_graders.matchesClassification,
+	matchesSimilarity,
+	matchesClassification,
 	matchesLlmRubric: require_graders.matchesLlmRubric,
 	matchesFactuality: require_graders.matchesFactuality,
 	matchesClosedQa: require_graders.matchesClosedQa,
-	matchesAnswerRelevance: require_graders.matchesAnswerRelevance,
-	matchesContextRecall: require_graders.matchesContextRecall,
-	matchesContextRelevance: require_graders.matchesContextRelevance,
-	matchesContextFaithfulness: require_graders.matchesContextFaithfulness,
-	matchesComparisonBoolean: require_graders.matchesSelectBest,
-	matchesModeration: require_graders.matchesModeration,
+	matchesAnswerRelevance,
+	matchesContextRecall,
+	matchesContextRelevance,
+	matchesContextFaithfulness,
+	matchesComparisonBoolean: matchesSelectBest,
+	matchesModeration,
 	matchesConversationRelevance
 };
 //#endregion
@@ -4242,7 +4991,7 @@ function initializeOtel(config) {
 	require_logger.logger.debug("[OtelSdk] Registered W3C Trace Context propagator");
 	const resource = (0, _opentelemetry_resources.resourceFromAttributes)({
 		[_opentelemetry_semantic_conventions.ATTR_SERVICE_NAME]: config.serviceName,
-		[_opentelemetry_semantic_conventions.ATTR_SERVICE_VERSION]: require_fetch.VERSION
+		[_opentelemetry_semantic_conventions.ATTR_SERVICE_VERSION]: require_version.VERSION
 	});
 	const spanProcessors = [];
 	if (config.localExport) {
@@ -4960,13 +5709,13 @@ async function gradeRunEvalResponse({ abortSignal, deferGrading, evalId, latency
 }
 async function transformRunEvalResponse({ evalId, prompt, promptIdx, provider, response, test, testIdx, vars }) {
 	const processedResponse = { ...response };
-	if (provider.transform) processedResponse.output = await require_transform.transform(provider.transform, processedResponse.output, {
+	if (provider.transform) processedResponse.output = await require_transform$1.transform(provider.transform, processedResponse.output, {
 		vars,
 		prompt
 	});
 	const providerTransformedOutput = processedResponse.output;
 	const testTransform = test.options?.transform || test.options?.postprocess;
-	if (testTransform) processedResponse.output = await require_transform.transform(testTransform, processedResponse.output, {
+	if (testTransform) processedResponse.output = await require_transform$1.transform(testTransform, processedResponse.output, {
 		vars,
 		prompt,
 		...response && response.metadata && { metadata: response.metadata }
@@ -5418,10 +6167,10 @@ async function prepareTestVariables(tests, testSuite) {
 async function applyInputTransform(testCase, inputTransformDefault) {
 	const inputTransform = testCase.options?.transformVars || inputTransformDefault;
 	if (!inputTransform) return;
-	const transformedVars = await require_transform.transform(inputTransform, testCase.vars, {
+	const transformedVars = await require_transform$1.transform(inputTransform, testCase.vars, {
 		prompt: {},
 		uuid: crypto.randomUUID()
-	}, true, require_transform.TransformInputType.VARS);
+	}, true, require_transform$1.TransformInputType.VARS);
 	require_invariant.invariant(typeof transformedVars === "object", "Transform function did not return a valid object");
 	testCase.vars = {
 		...testCase.vars,
@@ -5485,7 +6234,7 @@ async function resolveDefaultTestProvider(defaultTest, testCase) {
 	const defaultProvider = defaultTest.provider;
 	if (require_types.isApiProvider(defaultProvider)) return defaultProvider;
 	if (typeof defaultProvider === "object" && defaultProvider.id) {
-		const { loadApiProvider } = await Promise.resolve().then(() => require("./providers-DV3ax9e_.cjs"));
+		const { loadApiProvider } = await Promise.resolve().then(() => require("./providers-C7lNVBjX.cjs"));
 		return loadApiProvider(typeof defaultProvider.id === "function" ? defaultProvider.id() : defaultProvider.id, { options: defaultProvider });
 	}
 	return defaultProvider;
@@ -5645,7 +6394,7 @@ function buildRepeatCacheContextByTestIdx(runEvalOptions) {
 async function filterCompletedResumeSteps(runEvalOptions, evalRecord) {
 	if (!require_logger.state.resume || !evalRecord.persisted) return;
 	try {
-		const { default: EvalResult } = await Promise.resolve().then(() => require("./evalResult-Dvc-iucu.cjs"));
+		const { default: EvalResult } = await Promise.resolve().then(() => require("./evalResult-BBJAHAtw.cjs"));
 		const completedPairs = await EvalResult.getCompletedIndexPairs(evalRecord.id, { excludeErrors: require_logger.state.retryMode });
 		const originalCount = runEvalOptions.length;
 		for (let i = runEvalOptions.length - 1; i >= 0; i--) {
@@ -6104,9 +6853,8 @@ var Evaluator = class {
 		context.options.progressCallback?.(context.numComplete, context.runEvalOptionsLength, index, evalStep, metrics || createTimeoutMetrics(timeoutMs));
 	}
 	async executeEvalSteps({ checkAbort, ciProgressReporter, combinedAbortSignal, concurrentRunEvalOptions, evalStepIndexMap, globalTimeout, groupedRunEvalOptions, isEvalTimedOut, isWebUI, maxEvalTimeMs, processingContext, processedIndices, progressBarManager, prompts, serialRunEvalOptions, shouldGroupGradingByProvider }) {
-		let flushGroupedRows;
 		try {
-			if (shouldGroupGradingByProvider) flushGroupedRows = await this.runGroupedEvalSteps({
+			if (shouldGroupGradingByProvider) await this.runGroupedEvalSteps({
 				checkAbort,
 				evalStepIndexMap,
 				groupedRunEvalOptions,
@@ -6138,7 +6886,6 @@ var Evaluator = class {
 				cleanupProgressAfterError(progressBarManager, ciProgressReporter, err);
 				throw err;
 			}
-			await flushGroupedRows?.();
 			if (isEvalTimedOut()) require_logger.logger.warn(`Evaluation stopped after reaching max duration (${maxEvalTimeMs}ms)`);
 			else if (!processingContext.targetUnavailable) return this.saveInterruptedEval({
 				ciProgressReporter,
@@ -6363,7 +7110,7 @@ var Evaluator = class {
 		}
 		const maxScoreAssertion = resultsToCompare[0].testCase.assert?.find((a) => a.type === "max-score");
 		if (!maxScoreAssertion) return;
-		const maxScoreGradingResults = await require_graders.selectMaxScore(resultsToCompare.map((r) => r.response?.output || ""), resultsToCompare, maxScoreAssertion);
+		const maxScoreGradingResults = await selectMaxScore(resultsToCompare.map((r) => r.response?.output || ""), resultsToCompare, maxScoreAssertion);
 		updateComparisonReporterProgress({
 			ciProgressReporter,
 			compareCount,
@@ -8434,47 +9181,11 @@ function filterPrompts(prompts, filterPromptsOption) {
 //#endregion
 //#region src/commands/eval/filterProviders.ts
 /**
-* Checks if a value is a valid provider ID (non-empty string).
-*/
-function isValidProviderId(id) {
-	return id !== null && id !== void 0 && typeof id === "string" && id !== "";
-}
-/**
 * Extracts the id and label from a raw provider config without instantiating it.
 * Handles all provider config formats: string, function, ProviderOptions, ProviderOptionsMap.
 */
 function getProviderIdAndLabel(provider, index) {
-	if (typeof provider === "string") return { id: provider };
-	if (typeof provider === "function") {
-		const label = provider.label;
-		return {
-			id: label ?? `custom-function-${index}`,
-			label
-		};
-	}
-	const providerId = provider.id;
-	if ("id" in provider && isValidProviderId(providerId)) return {
-		id: providerId,
-		label: provider.label
-	};
-	const keys = Object.keys(provider);
-	if (keys.length > 0) {
-		const id = keys[0];
-		const value = provider[id];
-		if (typeof value === "object" && value !== null) return {
-			id: value.id || id,
-			label: value.label
-		};
-	}
-	const label = provider.label;
-	if (isValidProviderId(label)) return {
-		id: label,
-		label
-	};
-	return {
-		id: `unknown-${index}`,
-		label
-	};
+	return require_util.normalizeProviderRef(provider, { index });
 }
 /**
 * Filters raw provider configs BEFORE instantiation.
@@ -10227,7 +10938,7 @@ async function fetchRemoteGeneration(task, prompts) {
 		const body = {
 			task,
 			prompts,
-			version: require_fetch.VERSION,
+			version: require_version.VERSION,
 			email: require_accounts.getUserEmail()
 		};
 		const response = await require_cache.fetchWithCache(require_server.getRemoteGenerationUrl(), {
@@ -11179,9 +11890,10 @@ function dedupeTestCases(testCases) {
 	return deduped;
 }
 function buildMaxCharsRetryInstructions(rejectedPromptLengths, limit) {
+	const longestRejectedPromptText = rejectedPromptLengths.length > 0 ? `${Math.max(...rejectedPromptLengths)} characters` : "unknown length";
 	return dedent.default`
     Your previous response included ${rejectedPromptLengths.length} generated prompt${rejectedPromptLengths.length === 1 ? "" : "s"} that exceeded the ${limit ?? "configured"}-character limit.
-    The longest rejected prompt was ${Math.max(...rejectedPromptLengths)} characters.
+    The longest rejected prompt was ${longestRejectedPromptText}.
     Generate replacement prompts only, and keep every user message within the character limit.
   `.trim();
 }
@@ -11238,7 +11950,7 @@ async function fetchRemoteTestCases(key, purpose, injectVar, n, config) {
 		n,
 		purpose,
 		task: key,
-		version: require_fetch.VERSION,
+		version: require_version.VERSION,
 		email: require_accounts.getUserEmail()
 	});
 	try {
@@ -12317,7 +13029,7 @@ function handleFailedPlugins(failedPlugins, strict) {
 }
 function getConfigHash(configPath) {
 	const content = fs.readFileSync(configPath, "utf8");
-	return (0, crypto$1.createHash)("md5").update(`${require_fetch.VERSION}:${content}`).digest("hex");
+	return (0, crypto$1.createHash)("md5").update(`${require_version.VERSION}:${content}`).digest("hex");
 }
 function createHeaderComments({ title, timestampLabel, author, cloudHost, testCasesCount, plugins, strategies, isUpdate = false }) {
 	const sectionLabel = isUpdate ? "Changes:" : "Test Configuration:";
@@ -13230,7 +13942,7 @@ function generateTable(evaluateTable, tableCellMaxLength = 250, maxRows = 25) {
 	for (const row of evaluateTable.body.slice(0, maxRows)) table.push([...row.vars.map((v) => require_text.ellipsize(v, tableCellMaxLength)), ...row.outputs.map(({ pass, text, failureReason: failureType }) => {
 		text = require_text.ellipsize(text, tableCellMaxLength);
 		if (pass) return chalk.default.green("[PASS] ") + text;
-		else return chalk.default.red(failureType === require_types.ResultFailureReason.ASSERT ? "[FAIL] " : "[ERROR] ") + text.split("---").map((c, idx) => idx === 0 ? chalk.default.red.bold(c) : c).join("---");
+		return chalk.default.red(failureType === require_types.ResultFailureReason.ASSERT ? "[FAIL] " : "[ERROR] ") + text.split("---").map((c, idx) => idx === 0 ? chalk.default.red.bold(c) : c).join("---");
 	})]);
 	return table.toString();
 }
@@ -13321,6 +14033,115 @@ function formatDuration(seconds) {
 }
 //#endregion
 //#region src/commands/eval/summary.ts
+function getCompletionMessage({ completionType, evalId, shareableUrl, wasAborted, writeToDatabase, activelySharing }) {
+	if (wasAborted) {
+		const idSuffix = writeToDatabase ? ` (ID: ${chalk.default.cyan(evalId)})` : "";
+		return `${chalk.default.red("✗")} ${completionType} aborted${idSuffix}`;
+	}
+	if (writeToDatabase && shareableUrl) return `${chalk.default.green("✓")} ${completionType} complete: ${shareableUrl}`;
+	if (writeToDatabase && activelySharing) return `${chalk.default.green("✓")} ${completionType} complete`;
+	if (writeToDatabase) return `${chalk.default.green("✓")} ${completionType} complete (ID: ${chalk.default.cyan(evalId)})`;
+	return `${chalk.default.green("✓")} ${completionType} complete`;
+}
+function getAbortSummaryLines(targetErrorStatus) {
+	if (targetErrorStatus == null) return [];
+	return [
+		"",
+		chalk.default.red.bold("Scan stopped: Target is unavailable and will not recover on retry."),
+		chalk.default.red(`  Target returned HTTP ${targetErrorStatus}`),
+		"",
+		chalk.default.yellow("Possible causes:"),
+		chalk.default.yellow("  • Invalid API key or authentication (401/403)"),
+		chalk.default.yellow("  • Target endpoint does not exist (404)"),
+		chalk.default.yellow("  • Server does not support the request (501)"),
+		"",
+		chalk.default.cyan("To fix: Check your target configuration and credentials.")
+	];
+}
+function getGuidanceLines({ writeToDatabase, shareableUrl, wantsToShare, activelySharing, hasExplicitDisable, cloudEnabled }) {
+	if (!writeToDatabase || shareableUrl || wantsToShare || activelySharing) return [];
+	const lines = ["", `» View results: ${chalk.default.green.bold("promptfoo view")}`];
+	if (!hasExplicitDisable) lines.push(cloudEnabled ? `» Create shareable URL: ${chalk.default.green.bold("promptfoo share")}` : `» Share with your team: ${chalk.default.green.bold("https://promptfoo.app")}`);
+	lines.push(`» Feedback: ${chalk.default.green.bold("https://promptfoo.dev/feedback")}`);
+	return lines;
+}
+function buildUsageDetails(usage, total) {
+	const parts = [];
+	if (usage.prompt && usage.prompt > 0) parts.push(`${usage.prompt.toLocaleString()} prompt`);
+	if (usage.completion && usage.completion > 0) parts.push(`${usage.completion.toLocaleString()} completion`);
+	if (usage.cached && usage.cached > 0) parts.push(usage.cached === total && parts.length === 0 ? "cached" : `${usage.cached.toLocaleString()} cached`);
+	if (usage.completionDetails?.reasoning && usage.completionDetails.reasoning > 0) parts.push(`${usage.completionDetails.reasoning.toLocaleString()} reasoning`);
+	return parts;
+}
+function getTokenUsageLines(tokenUsage, isRedteam, tracker) {
+	const hasEvalTokens = (tokenUsage.total || 0) > 0 || (tokenUsage.prompt || 0) + (tokenUsage.completion || 0) > 0;
+	const hasGradingTokens = tokenUsage.assertions && (tokenUsage.assertions.total || 0) > 0;
+	if (!hasEvalTokens && !hasGradingTokens) return [];
+	const combinedTotal = (tokenUsage.prompt || 0) + (tokenUsage.completion || 0);
+	const evalTokens = {
+		prompt: tokenUsage.prompt || 0,
+		completion: tokenUsage.completion || 0,
+		total: tokenUsage.total || combinedTotal,
+		cached: tokenUsage.cached || 0,
+		numRequests: tokenUsage.numRequests || 0,
+		completionDetails: tokenUsage.completionDetails || {
+			reasoning: 0,
+			acceptedPrediction: 0,
+			rejectedPrediction: 0
+		}
+	};
+	const lines = [`${chalk.default.bold("Total Tokens:")} ${chalk.default.white.bold((evalTokens.total + (tokenUsage.assertions?.total || 0)).toLocaleString())}`];
+	if (isRedteam && tokenUsage.numRequests) lines.push(`  ${chalk.default.gray("Probes:")} ${chalk.default.white(tokenUsage.numRequests.toLocaleString())}`);
+	if (evalTokens.total > 0) {
+		const evalParts = buildUsageDetails(evalTokens, evalTokens.total);
+		lines.push(`  ${chalk.default.gray("Eval:")} ${chalk.default.white(evalTokens.total.toLocaleString())} (${evalParts.join(", ")})`);
+	}
+	if (tokenUsage.assertions?.total && tokenUsage.assertions.total > 0) {
+		const gradingParts = buildUsageDetails(tokenUsage.assertions, tokenUsage.assertions.total);
+		lines.push(`  ${chalk.default.gray("Grading:")} ${chalk.default.white(tokenUsage.assertions.total.toLocaleString())} (${gradingParts.join(", ")})`);
+	}
+	lines.push(...getProviderUsageLines(tracker));
+	return lines;
+}
+function getProviderUsageLines(tracker) {
+	const providerIds = tracker.getProviderIds();
+	if (providerIds.length <= 1) return [];
+	const sortedProviders = providerIds.map((id) => ({
+		id,
+		usage: tracker.getProviderUsage(id)
+	})).filter((p) => p.usage != null).sort((a, b) => (b.usage.total || 0) - (a.usage.total || 0));
+	const lines = ["", chalk.default.bold("Providers:")];
+	for (const { id, usage } of sortedProviders) {
+		if ((usage.total || 0) === 0 && (usage.prompt || 0) + (usage.completion || 0) === 0) continue;
+		const displayTotal = usage.total || (usage.prompt || 0) + (usage.completion || 0);
+		const displayId = id.includes(" (") ? id.substring(0, id.indexOf(" (")) : id;
+		const details = buildUsageDetails(usage, displayTotal);
+		const requestInfo = `${usage.numRequests || 0} requests`;
+		const separator = details.length > 0 ? "; " : "";
+		lines.push(`  ${chalk.default.gray(`${displayId}:`)} ${chalk.default.white(displayTotal.toLocaleString())} (${requestInfo}${separator}${details.join(", ")})`);
+	}
+	return lines;
+}
+function formatResultPercentage(count, totalTests) {
+	const percentage = totalTests === 0 ? 0 : count / totalTests * 100;
+	return percentage === 0 || percentage === 100 ? `${percentage.toFixed(0)}%` : `${percentage.toFixed(2)}%`;
+}
+function formatResultLine(count, label, icon, iconColor, totalTests) {
+	return `  ${icon ? `${iconColor(icon)} ` : ""}${chalk.default.white.bold(count.toLocaleString())} ${chalk.default.white(label)} ${chalk.default.gray(`(${formatResultPercentage(count, totalTests)})`)}`;
+}
+function getResultsLines({ successes, failures, errors, duration, maxConcurrency }) {
+	const totalTests = successes + failures + errors;
+	const errorLabel = errors === 1 ? "error" : "errors";
+	return [
+		"",
+		chalk.default.bold("Results:"),
+		formatResultLine(successes, "passed", successes > 0 ? "✓" : void 0, chalk.default.green, totalTests),
+		formatResultLine(failures, "failed", failures > 0 ? "✗" : void 0, chalk.default.red, totalTests),
+		formatResultLine(errors, errorLabel, errors > 0 ? "✗" : void 0, chalk.default.red, totalTests),
+		chalk.default.gray(`Duration: ${formatDuration(duration)} (concurrency: ${maxConcurrency})`),
+		""
+	];
+}
 /**
 * Generate formatted evaluation summary output for CLI display.
 *
@@ -13359,115 +14180,28 @@ function formatDuration(seconds) {
 * ```
 */
 function generateEvalSummary(params) {
-	const { evalId, isRedteam, writeToDatabase, shareableUrl, wantsToShare, hasExplicitDisable, cloudEnabled, activelySharing = false, tokenUsage, successes, failures, errors, duration, maxConcurrency, tracker, targetErrorStatus } = params;
-	const lines = [];
-	const completionType = isRedteam ? "Red team" : "Eval";
-	const wasAborted = targetErrorStatus != null;
-	let completionMessage;
-	if (wasAborted) {
-		completionMessage = `${chalk.default.red("✗")} ${completionType} aborted`;
-		if (writeToDatabase) completionMessage += ` (ID: ${chalk.default.cyan(evalId)})`;
-	} else if (writeToDatabase && shareableUrl) completionMessage = `${chalk.default.green("✓")} ${completionType} complete: ${shareableUrl}`;
-	else if (writeToDatabase && activelySharing) completionMessage = `${chalk.default.green("✓")} ${completionType} complete`;
-	else if (writeToDatabase) completionMessage = `${chalk.default.green("✓")} ${completionType} complete (ID: ${chalk.default.cyan(evalId)})`;
-	else completionMessage = `${chalk.default.green("✓")} ${completionType} complete`;
-	lines.push(completionMessage);
-	if (wasAborted && targetErrorStatus != null) {
-		lines.push("");
-		lines.push(chalk.default.red.bold("Scan stopped: Target is unavailable and will not recover on retry."));
-		lines.push(chalk.default.red(`  Target returned HTTP ${targetErrorStatus}`));
-		lines.push("");
-		lines.push(chalk.default.yellow("Possible causes:"));
-		lines.push(chalk.default.yellow("  • Invalid API key or authentication (401/403)"));
-		lines.push(chalk.default.yellow("  • Target endpoint does not exist (404)"));
-		lines.push(chalk.default.yellow("  • Server does not support the request (501)"));
-		lines.push("");
-		lines.push(chalk.default.cyan("To fix: Check your target configuration and credentials."));
-	}
-	if (writeToDatabase && !shareableUrl && !wantsToShare && !activelySharing) {
-		lines.push("");
-		lines.push(`» View results: ${chalk.default.green.bold("promptfoo view")}`);
-		if (!hasExplicitDisable) if (cloudEnabled) lines.push(`» Create shareable URL: ${chalk.default.green.bold("promptfoo share")}`);
-		else lines.push(`» Share with your team: ${chalk.default.green.bold("https://promptfoo.app")}`);
-		lines.push(`» Feedback: ${chalk.default.green.bold("https://promptfoo.dev/feedback")}`);
-	}
-	lines.push("");
-	const hasEvalTokens = (tokenUsage.total || 0) > 0 || (tokenUsage.prompt || 0) + (tokenUsage.completion || 0) > 0;
-	const hasGradingTokens = tokenUsage.assertions && (tokenUsage.assertions.total || 0) > 0;
-	if (hasEvalTokens || hasGradingTokens) {
-		const combinedTotal = (tokenUsage.prompt || 0) + (tokenUsage.completion || 0);
-		const evalTokens = {
-			prompt: tokenUsage.prompt || 0,
-			completion: tokenUsage.completion || 0,
-			total: tokenUsage.total || combinedTotal,
-			cached: tokenUsage.cached || 0,
-			completionDetails: tokenUsage.completionDetails || {
-				reasoning: 0,
-				acceptedPrediction: 0,
-				rejectedPrediction: 0
-			}
-		};
-		const grandTotal = evalTokens.total + (tokenUsage.assertions?.total || 0);
-		lines.push(`${chalk.default.bold("Total Tokens:")} ${chalk.default.white.bold(grandTotal.toLocaleString())}`);
-		if (isRedteam && tokenUsage.numRequests) lines.push(`  ${chalk.default.gray("Probes:")} ${chalk.default.white(tokenUsage.numRequests.toLocaleString())}`);
-		if (evalTokens.total > 0) {
-			const evalParts = [];
-			if (evalTokens.prompt > 0) evalParts.push(`${evalTokens.prompt.toLocaleString()} prompt`);
-			if (evalTokens.completion > 0) evalParts.push(`${evalTokens.completion.toLocaleString()} completion`);
-			if (evalTokens.cached > 0) if (evalTokens.cached === evalTokens.total && evalParts.length === 0) evalParts.push("cached");
-			else evalParts.push(`${evalTokens.cached.toLocaleString()} cached`);
-			if (evalTokens.completionDetails?.reasoning && evalTokens.completionDetails.reasoning > 0) evalParts.push(`${evalTokens.completionDetails.reasoning.toLocaleString()} reasoning`);
-			lines.push(`  ${chalk.default.gray("Eval:")} ${chalk.default.white(evalTokens.total.toLocaleString())} (${evalParts.join(", ")})`);
-		}
-		if (tokenUsage.assertions && tokenUsage.assertions.total && tokenUsage.assertions.total > 0) {
-			const gradingParts = [];
-			if (tokenUsage.assertions.prompt && tokenUsage.assertions.prompt > 0) gradingParts.push(`${tokenUsage.assertions.prompt.toLocaleString()} prompt`);
-			if (tokenUsage.assertions.completion && tokenUsage.assertions.completion > 0) gradingParts.push(`${tokenUsage.assertions.completion.toLocaleString()} completion`);
-			if (tokenUsage.assertions.cached && tokenUsage.assertions.cached > 0) if (tokenUsage.assertions.cached === tokenUsage.assertions.total && gradingParts.length === 0) gradingParts.push("cached");
-			else gradingParts.push(`${tokenUsage.assertions.cached.toLocaleString()} cached`);
-			if (tokenUsage.assertions.completionDetails?.reasoning && tokenUsage.assertions.completionDetails.reasoning > 0) gradingParts.push(`${tokenUsage.assertions.completionDetails.reasoning.toLocaleString()} reasoning`);
-			lines.push(`  ${chalk.default.gray("Grading:")} ${chalk.default.white(tokenUsage.assertions.total.toLocaleString())} (${gradingParts.join(", ")})`);
-		}
-		const providerIds = tracker.getProviderIds();
-		if (providerIds.length > 1) {
-			lines.push("");
-			lines.push(chalk.default.bold("Providers:"));
-			const sortedProviders = providerIds.map((id) => ({
-				id,
-				usage: tracker.getProviderUsage(id)
-			})).filter((p) => p.usage != null).sort((a, b) => (b.usage.total || 0) - (a.usage.total || 0));
-			for (const { id, usage } of sortedProviders) if ((usage.total || 0) > 0 || (usage.prompt || 0) + (usage.completion || 0) > 0) {
-				const displayTotal = usage.total || (usage.prompt || 0) + (usage.completion || 0);
-				const displayId = id.includes(" (") ? id.substring(0, id.indexOf(" (")) : id;
-				const details = [];
-				if (usage.prompt && usage.prompt > 0) details.push(`${usage.prompt.toLocaleString()} prompt`);
-				if (usage.completion && usage.completion > 0) details.push(`${usage.completion.toLocaleString()} completion`);
-				if (usage.cached && usage.cached > 0) if (usage.cached === displayTotal && details.length === 0) details.push("cached");
-				else details.push(`${usage.cached.toLocaleString()} cached`);
-				if (usage.completionDetails?.reasoning && usage.completionDetails.reasoning > 0) details.push(`${usage.completionDetails.reasoning.toLocaleString()} reasoning`);
-				const breakdown = ` (${`${usage.numRequests || 0} requests`}${details.length > 0 ? "; " : ""}${details.join(", ")})`;
-				lines.push(`  ${chalk.default.gray(displayId + ":")} ${chalk.default.white(displayTotal.toLocaleString())}${breakdown}`);
-			}
-		}
-	}
-	lines.push("");
-	const totalTests = successes + failures + errors;
-	const formatResultPercentage = (count) => {
-		const percentage = totalTests === 0 ? 0 : count / totalTests * 100;
-		return percentage === 0 || percentage === 100 ? `${percentage.toFixed(0)}%` : `${percentage.toFixed(2)}%`;
-	};
-	const formatResultLine = (count, label, icon, iconColor) => {
-		return `  ${icon ? `${iconColor(icon)} ` : ""}${chalk.default.white.bold(count.toLocaleString())} ${chalk.default.white(label)} ${chalk.default.gray(`(${formatResultPercentage(count)})`)}`;
-	};
-	const errorLabel = errors === 1 ? "error" : "errors";
-	lines.push(chalk.default.bold("Results:"));
-	lines.push(formatResultLine(successes, "passed", successes > 0 ? "✓" : void 0, chalk.default.green));
-	lines.push(formatResultLine(failures, "failed", failures > 0 ? "✗" : void 0, chalk.default.red));
-	lines.push(formatResultLine(errors, errorLabel, errors > 0 ? "✗" : void 0, chalk.default.red));
-	const durationDisplay = formatDuration(duration);
-	lines.push(chalk.default.gray(`Duration: ${durationDisplay} (concurrency: ${maxConcurrency})`));
-	lines.push("");
-	return lines;
+	return [
+		getCompletionMessage({
+			completionType: params.isRedteam ? "Red team" : "Eval",
+			evalId: params.evalId,
+			shareableUrl: params.shareableUrl,
+			wasAborted: params.targetErrorStatus != null,
+			writeToDatabase: params.writeToDatabase,
+			activelySharing: params.activelySharing ?? false
+		}),
+		...getAbortSummaryLines(params.targetErrorStatus),
+		...getGuidanceLines({
+			writeToDatabase: params.writeToDatabase,
+			shareableUrl: params.shareableUrl,
+			wantsToShare: params.wantsToShare,
+			activelySharing: params.activelySharing ?? false,
+			hasExplicitDisable: params.hasExplicitDisable,
+			cloudEnabled: params.cloudEnabled
+		}),
+		"",
+		...getTokenUsageLines(params.tokenUsage, params.isRedteam, params.tracker),
+		...getResultsLines(params)
+	];
 }
 //#endregion
 //#region src/commands/retry.ts
@@ -14237,6 +14971,26 @@ async function doRedteamRun(options) {
 }
 //#endregion
 //#region src/index.ts
+/**
+* Shallow-clone a test case so the caller can swap in resolved ApiProvider
+* instances on `options.provider` / `assert[].provider` without leaking those
+* mutations back to the input. The input may alias the unified config written
+* to the Eval record, and a live SDK client (e.g. Bedrock's BedrockRuntime,
+* Anthropic's client) holds circular references that break drizzle's JSON
+* serialization on `evalRecord.save()`. Fixes #8687.
+*
+* Detaches only `options` and `assert[]`. Other reference fields (`provider`,
+* `vars`, `metadata`, `providerOutput`) remain aliased — callers must reassign
+* those by reference rather than mutating in place. `assert-set` children are
+* not deep-cloned because the resolve loop skips `assert-set`; if that ever
+* changes, extend this helper.
+*/
+function cloneTestForResolve(test) {
+	const cloned = { ...test };
+	if (test.options) cloned.options = { ...test.options };
+	if (test.assert) cloned.assert = test.assert.map((assertion) => ({ ...assertion }));
+	return cloned;
+}
 async function evaluate(testSuite, options = {}) {
 	if (testSuite.writeLatestResults) await runDbMigrations();
 	const loadedProviders = await require_providers.loadApiProviders(testSuite.providers, { env: testSuite.env });
@@ -14256,22 +15010,24 @@ async function evaluate(testSuite, options = {}) {
 		nunjucksFilters: await require_util.readFilters(testSuite.nunjucksFilters || {}),
 		prompts: await require_graders.processPrompts(testSuite.prompts)
 	};
-	if (typeof constructedTestSuite.defaultTest === "object") {
-		if (constructedTestSuite.defaultTest?.provider && !require_types.isApiProvider(constructedTestSuite.defaultTest.provider)) constructedTestSuite.defaultTest.provider = await require_providers.resolveProvider(constructedTestSuite.defaultTest.provider, providerMap, {
+	if (typeof constructedTestSuite.defaultTest === "object" && constructedTestSuite.defaultTest) {
+		constructedTestSuite.defaultTest = cloneTestForResolve(constructedTestSuite.defaultTest);
+		if (constructedTestSuite.defaultTest.provider && !require_types.isApiProvider(constructedTestSuite.defaultTest.provider)) constructedTestSuite.defaultTest.provider = await require_providers.resolveProvider(constructedTestSuite.defaultTest.provider, providerMap, {
 			env: testSuite.env,
 			basePath: require_logger.state.basePath
 		});
-		if (constructedTestSuite.defaultTest?.options?.provider && !require_types.isApiProvider(constructedTestSuite.defaultTest.options.provider)) constructedTestSuite.defaultTest.options.provider = await require_providers.resolveProvider(constructedTestSuite.defaultTest.options.provider, providerMap, {
+		if (constructedTestSuite.defaultTest.options?.provider && !require_types.isApiProvider(constructedTestSuite.defaultTest.options.provider)) constructedTestSuite.defaultTest.options.provider = await require_providers.resolveProvider(constructedTestSuite.defaultTest.options.provider, providerMap, {
 			env: testSuite.env,
 			basePath: require_logger.state.basePath
 		});
 	}
-	for (const test of constructedTestSuite.tests || []) {
+	constructedTestSuite.tests = (constructedTestSuite.tests || []).map(cloneTestForResolve);
+	for (const test of constructedTestSuite.tests) {
 		if (test.options?.provider && !require_types.isApiProvider(test.options.provider)) test.options.provider = await require_providers.resolveProvider(test.options.provider, providerMap, {
 			env: testSuite.env,
 			basePath: require_logger.state.basePath
 		});
-		if (test.assert) for (const assertion of test.assert) {
+		for (const assertion of test.assert || []) {
 			if (assertion.type === "assert-set" || typeof assertion.provider === "function") continue;
 			if (assertion.provider && !require_types.isApiProvider(assertion.provider)) assertion.provider = await require_providers.resolveProvider(assertion.provider, providerMap, {
 				env: testSuite.env,