promptfoo 0.121.4 → 0.121.7

package/dist/src/{evaluator-IvuDYSvQ.js → evaluator-DRoiYB2q.js}

@@ -1,29 +1,39 @@
 #!/usr/bin/env node
-import { A as getMaxEvalTimeMs, D as getEnvInt, N as state, O as getEnvString, S as summarizeEvaluateResultForLogging, T as getEnvBool, _ as extractJsonObjects, c as setLogCallback, j as isCI, k as getEvalTimeoutMs, r as globalLogCallback, s as logger, v as getAjv, x as safeJsonStringify } from "./logger-DksKw1Qc.js";
-import { F as VERSION, I as FILE_METADATA_KEY, l as sleep, r as fetchWithRetries, v as isPromptfooSampleTarget, x as parseChatPrompt } from "./fetch-BEWnXrrG.js";
-import { t as invariant } from "./invariant-BtWWVVhl.js";
-import { r as telemetry } from "./telemetry-CQPez_Jp.js";
-import { at as MULTI_INPUT_VAR, d as isGradingResult, p as isApiProvider, s as ResultFailureReason } from "./types-Dm9JM6Vb.js";
-import { a as getNunjucksEngine, r as extractVariablesFromTemplate } from "./render-CMEpfLaO.js";
-import { t as providerRegistry } from "./providerRegistry-CWoPjKFZ.js";
-import { c as promptYesNo } from "./server-BNYztJkh.js";
-import { A as renderPrompt, E as isBasicRefusal, F as TokenUsageTracker, I as createRateLimitRegistry, L as createProviderRateLimitOptions, M as isPackagePath, N as loadFromPackage, P as redteamProviderManager, U as throwIfTargetPromptExceedsMaxChars, X as VertexChatProvider, Z as AIStudioChatProvider, j as runExtensionHook, k as collectFileMetadata, u as GoogleLiveProvider, v as checkExfilTracking, w as getSessionId } from "./providers-BV_KMZje.js";
-import { n as isNonTransientHttpStatus } from "./errors-Cw810C93.js";
-import { c as withCacheNamespace, o as getCache } from "./cache-mb7c8hbp.js";
-import { i as isJavascriptFile } from "./fileExtensions-AWa2ZML4.js";
-import { M as isGoogleProvider, N as isOpenAiProvider, P as isProviderAllowed, b as loadFunction, g as maybeLoadToolsFromExternalFile, j as isAnthropicProvider, x as parseFileUrl } from "./util-DxWpWjhc.js";
-import { r as runPython } from "./pythonUtils-CnndUbW-.js";
-import { n as transform, r as getProcessShim, t as TransformInputType } from "./transform-UN5UGu8U.js";
-import { $ as matchesGEval, G as loadRubricPrompt, H as callProviderWithContext, J as matchesClosedQa, K as matchesAnswerRelevance, Q as matchesFactuality, U as fail, W as getAndCheckProvider, X as matchesContextRecall, Y as matchesContextFaithfulness, Z as matchesContextRelevance, _t as loadFromJavaScriptFile, at as matchesSimilarity, ct as withProviderCallExecutionContext, et as matchesLlmRubric, gt as getFinalTest, ht as coerceString, it as matchesSelectBest, mt as SUGGEST_PROMPTS_SYSTEM_MESSAGE, n as getGraderById, nt as matchesPiScore, ot as matchesTrajectoryGoalSuccess, q as matchesClassification, rt as matchesSearchRubric, st as selectMaxScore, tt as matchesModeration, ut as getDefaultProviders, vt as processFileReference, yt as resolveContext } from "./graders-eIHhRqoC.js";
-import { i as generateIdFromPrompt } from "./utils-BLJKfv0y.js";
-import { t as OpenAiChatCompletionProvider } from "./chat-Dr3DUQ0D.js";
-import { a as createEmptyTokenUsage, i as createEmptyAssertions, n as accumulateResponseTokenUsage, o as normalizeTokenUsage, r as accumulateTokenUsage, t as accumulateAssertionTokenUsage } from "./tokenUsageUtils-CXrvO-wA.js";
-import { h as validateFunctionCall } from "./transform-zDhMmzwX.js";
-import { l as validateFunctionCall$1 } from "./util-C9J8ahRn.js";
-import { t as extractAndStoreBinaryData } from "./extractor-DNSeBVOJ.js";
-import { n as getTraceStore } from "./store-CYEy5J2D.js";
-import { n as runRuby } from "./rubyUtils-DsGrTx8R.js";
-import { a as getActualPromptWithFallback, r as updateSignalFile } from "./signal-CE5G3a7x.js";
+import { A as getMaxEvalTimeMs, D as getEnvInt, N as state, O as getEnvString, S as summarizeEvaluateResultForLogging, T as getEnvBool, _ as extractJsonObjects, c as setLogCallback, g as extractFirstJsonObject, j as isCI, k as getEvalTimeoutMs, r as globalLogCallback, s as logger, v as getAjv, x as safeJsonStringify } from "./logger-BbY6ypFL.js";
+import { L as FILE_METADATA_KEY, f as sleep, r as fetchWithRetries, w as parseChatPrompt, x as isPromptfooSampleTarget } from "./fetch-DXUnXkVU.js";
+import { n as VERSION } from "./version-eRkNuGv8.js";
+import { t as invariant } from "./invariant-B2Rf6avk.js";
+import { r as telemetry } from "./telemetry-00ezXr_t.js";
+import { at as MULTI_INPUT_VAR, d as isGradingResult, nt as LLAMA_GUARD_REPLICATE_PROVIDER, p as isApiProvider, s as ResultFailureReason } from "./types-BFevViUY.js";
+import { i as isJavascriptFile } from "./fileExtensions-D4GCJ67J.js";
+import { i as extractVariablesFromTemplate, o as getNunjucksEngine, r as analyzeTemplateReference } from "./render-CSP99NLm.js";
+import { t as providerRegistry } from "./providerRegistry-ReCd0sFa.js";
+import { l as shouldGenerateRemote } from "./remoteGeneration-B1_XsKXU.js";
+import { c as promptYesNo } from "./server-DhMHosWj.js";
+import { n as isNonTransientHttpStatus } from "./errors-9PcUL8BC.js";
+import { l as withCacheNamespace, o as getCache } from "./cache-DpPWrkTE.js";
+import { r as runPython } from "./pythonUtils-CgYxeSmO.js";
+import { B as parseFileUrl, C as isOpenAiProvider, F as maybeLoadToolsFromExternalFile, S as isGoogleProvider, w as isProviderAllowed, x as isAnthropicProvider, z as loadFunction } from "./util-DURocbYR.js";
+import { t as OpenAiChatCompletionProvider } from "./chat-DqUFcWI0.js";
+import { h as validateFunctionCall } from "./transform-EtD4jAWi.js";
+import { l as validateFunctionCall$1 } from "./util-5WnCSb0h.js";
+import { _ as VertexChatProvider, n as loadApiProvider, v as GoogleLiveProvider, y as AIStudioChatProvider } from "./providers-B9KzWxAX.js";
+import { a as createEmptyTokenUsage, i as createEmptyAssertions, n as accumulateResponseTokenUsage, o as normalizeTokenUsage, r as accumulateTokenUsage, t as accumulateAssertionTokenUsage } from "./tokenUsageUtils-CDet74yk.js";
+import { t as getProcessShim } from "./processShim-BBxt7LKO.js";
+import { n as loadFromPackage, t as isPackagePath } from "./packageParser--MWTSrPW.js";
+import { n as runRuby } from "./rubyUtils-CXlFM2rR.js";
+import { c as isBasicRefusal, o as getSessionId } from "./util-kDURhgJW.js";
+import { $ as DEFAULT_WEB_SEARCH_PROMPT, Ct as getDefaultProviders, Dt as getGradingProvider, Et as getAndCheckProvider, G as matchesGEval, H as isGraderFailure, J as matchesTrajectoryGoalSuccess, K as matchesLlmRubric, Ot as getProviderCallExecutionContext, St as processFileReference, Tt as callProviderWithContext, U as matchesClosedQa, W as matchesFactuality, Y as doRemoteGrading, _t as splitIntoSentences, at as CONTEXT_RECALL, bt as getFinalTest, ct as CONTEXT_RELEVANCE, dt as renderLlmRubricPrompt, et as SELECT_BEST_PROMPT, ft as cosineSimilarity, gt as normalizeMatcherTokenUsage, ht as fail, it as CONTEXT_FAITHFULNESS_NLI_STATEMENTS, kt as withProviderCallExecutionContext, lt as CONTEXT_RELEVANCE_BAD, mt as euclideanDistance, n as getGraderById, nt as ANSWER_RELEVANCY_GENERATE, ot as CONTEXT_RECALL_ATTRIBUTED_TOKEN, pt as dotProduct, q as matchesPiScore, rt as CONTEXT_FAITHFULNESS_LONGFORM, st as CONTEXT_RECALL_NOT_ATTRIBUTED_TOKEN, tt as SUGGEST_PROMPTS_SYSTEM_MESSAGE, ut as loadRubricPrompt, vt as tryParse, wt as DEFAULT_ANTHROPIC_MODEL, xt as loadFromJavaScriptFile, yt as coerceString } from "./graders-BGP99PdK.js";
+import { f as redteamProviderManager, g as createProviderRateLimitOptions, h as createRateLimitRegistry, m as TokenUsageTracker } from "./shared-9WHQ1oNE.js";
+import { i as generateIdFromPrompt } from "./utils-BFOh20Gb.js";
+import { a as getTransformLabel, i as getTransformErrorMessage, o as transform, r as TransformInputType } from "./transform-BnSTnFlp.js";
+import { n as getTraceStore } from "./store-DKd5592Q.js";
+import { a as getActualPromptWithFallback, r as updateSignalFile } from "./signal-CSurUUyV.js";
+import { t as extractAndStoreBinaryData } from "./extractor-CIW3iN-b.js";
+import { i as throwIfTargetPromptExceedsMaxChars } from "./promptLength-4X-Wd8PG.js";
+import { n as checkExfilTracking } from "./indirectWebPwn-uyWdHx04.js";
+import { n as getFirstStringAttribute, r as getToolNameFromAttributes, t as TOOL_ARGUMENT_ATTRIBUTE_KEYS } from "./toolAttributes-COVgDrBG.js";
+import { i as filterFiniteScores, n as renderPrompt, r as runExtensionHook, t as collectFileMetadata } from "./evaluatorHelpers-BsYP_muT.js";
 import { AsyncResource } from "node:async_hooks";
 import chalk from "chalk";
 import fs, { createWriteStream } from "fs";
@@ -37,11 +47,12 @@ import readline from "readline";
 import { globSync } from "glob";
 import { XMLParser } from "fast-xml-parser";
 import async from "async";
-import cliProgress from "cli-progress";
-import { JSDOM } from "jsdom";
+import { parse as parse$1 } from "parse5";
 import { distance } from "fastest-levenshtein";
+import cliProgress from "cli-progress";
 import * as rouge from "js-rouge";
 import { isDeepStrictEqual } from "node:util";
+import { LRUCache } from "lru-cache";
 import { ExportResultCode, W3CTraceContextPropagator } from "@opentelemetry/core";
 import { OTLPTraceExporter } from "@opentelemetry/exporter-trace-otlp-http";
 import { resourceFromAttributes } from "@opentelemetry/resources";
@@ -218,6 +229,505 @@ const handleConversationRelevance = async ({ assertion, outputString, prompt, pr
 	};
 };
 //#endregion
+//#region src/matchers/classification.ts
+/**
+*
+* @param expected Expected classification. If undefined, matches any classification.
+* @param output Text to classify.
+* @param threshold Value between 0 and 1. If the expected classification is undefined, the threshold is the minimum score for any classification. If the expected classification is defined, the threshold is the minimum score for that classification.
+* @param grading
+* @returns Pass if the output matches the classification with a score greater than or equal to the threshold.
+*/
+async function matchesClassification(expected, output, threshold, grading) {
+	const resp = await (await getAndCheckProvider("classification", grading?.provider, null, "classification check")).callClassificationApi(output);
+	if (!resp.classification) return fail(resp.error || "Unknown error fetching classification");
+	let score;
+	if (expected === void 0) {
+		const scores = Object.values(resp.classification);
+		if (scores.length === 0) return {
+			pass: false,
+			score: 0,
+			reason: "No classification scores returned"
+		};
+		score = Math.max(...scores);
+	} else score = resp.classification[expected] || 0;
+	if (score >= threshold - Number.EPSILON) {
+		const reason = expected === void 0 ? `Maximum classification score ${score.toFixed(2)} >= ${threshold}` : `Classification ${expected} has score ${score.toFixed(2)} >= ${threshold}`;
+		return {
+			pass: true,
+			score,
+			reason
+		};
+	}
+	return {
+		pass: false,
+		score,
+		reason: expected === void 0 ? `Maximum classification score ${score.toFixed(2)} < ${threshold}` : `Classification ${expected} has score ${score.toFixed(2)} < ${threshold}`
+	};
+}
+//#endregion
+//#region src/matchers/comparison.ts
+async function matchesSelectBest(criteria, outputs, grading, vars, providerCallContext) {
+	invariant(outputs.length >= 2, "select-best assertion must have at least two outputs to compare between");
+	const resp = await callProviderWithContext(await getAndCheckProvider("text", grading?.provider, (await getDefaultProviders()).gradingProvider, "select-best check"), await renderLlmRubricPrompt(await loadRubricPrompt(grading?.rubricPrompt, SELECT_BEST_PROMPT), {
+		criteria,
+		outputs: outputs.map((o) => tryParse(o)),
+		...vars || {}
+	}), "select-best", {
+		criteria,
+		outputs: outputs.map((o) => tryParse(o)),
+		...vars || {}
+	}, providerCallContext);
+	if (resp.error || !resp.output) return Array.from({ length: outputs.length }, () => fail(resp.error || "No output", resp.tokenUsage));
+	invariant(typeof resp.output === "string", "select-best produced malformed response");
+	const firstIntegerMatch = resp.output.trim().match(/\d+/);
+	const verdict = firstIntegerMatch ? Number.parseInt(firstIntegerMatch[0], 10) : NaN;
+	if (Number.isNaN(verdict) || verdict < 0 || verdict >= outputs.length) return Array.from({ length: outputs.length }, () => fail(`Invalid select-best verdict: ${verdict}`, resp.tokenUsage));
+	const tokensUsed = normalizeMatcherTokenUsage(resp.tokenUsage);
+	return outputs.map((_output, index) => {
+		if (index === verdict) return {
+			pass: true,
+			score: 1,
+			reason: `Output selected as the best: ${criteria}`,
+			tokensUsed
+		};
+		else return {
+			pass: false,
+			score: 0,
+			reason: `Output not selected: ${criteria}`,
+			tokensUsed
+		};
+	});
+}
+async function selectMaxScore(outputs, resultsWithGradingResults, assertion) {
+	invariant(outputs.length >= 2, "max-score assertion must have at least two outputs to compare between");
+	const value = assertion.value || {};
+	const options = {
+		method: typeof value === "object" && "method" in value ? value.method : "average",
+		weights: typeof value === "object" && "weights" in value ? value.weights : {},
+		threshold: typeof value === "object" && "threshold" in value ? value.threshold : void 0
+	};
+	const scores = resultsWithGradingResults.map((result, index) => {
+		const relevantResults = (result.gradingResult?.componentResults || []).filter((r) => r.assertion && r.assertion.type !== "max-score" && r.assertion.type !== "select-best");
+		if (relevantResults.length === 0) throw new Error("max-score requires at least one other assertion (besides max-score or select-best) to aggregate scores from");
+		let totalWeightedScore = 0;
+		let totalWeight = 0;
+		relevantResults.forEach((componentResult) => {
+			const assertionType = componentResult.assertion?.type || "unknown";
+			const weight = options.weights[assertionType] === void 0 ? 1 : options.weights[assertionType];
+			const score = componentResult.score || 0;
+			totalWeightedScore += score * weight;
+			totalWeight += weight;
+		});
+		let aggregateScore;
+		if (options.method === "sum") aggregateScore = totalWeightedScore;
+		else aggregateScore = totalWeight > 0 ? totalWeightedScore / totalWeight : 0;
+		return {
+			index,
+			score: aggregateScore,
+			componentCount: relevantResults.length,
+			totalWeight
+		};
+	});
+	let maxScore = -Infinity;
+	let winnerIndex = 0;
+	for (let i = 0; i < scores.length; i++) if (scores[i].score > maxScore) {
+		maxScore = scores[i].score;
+		winnerIndex = i;
+	}
+	const meetsThreshold = options.threshold === void 0 || maxScore >= options.threshold;
+	return scores.map(({ index, score, componentCount, totalWeight }) => {
+		const isWinner = index === winnerIndex && meetsThreshold;
+		return {
+			pass: isWinner,
+			score: isWinner ? 1 : 0,
+			reason: isWinner ? `Selected as highest scoring output (score: ${score.toFixed(3)})` : score === maxScore && !meetsThreshold ? `Not selected - score ${score.toFixed(3)} below threshold ${options.threshold}` : `Not selected (score: ${score.toFixed(3)}, max: ${maxScore.toFixed(3)})`,
+			namedScores: {
+				maxScore: score,
+				assertionCount: componentCount,
+				totalWeight
+			}
+		};
+	});
+}
+//#endregion
+//#region src/matchers/moderation.ts
+async function matchesModeration({ userPrompt, assistantResponse, categories = [] }, grading) {
+	if (!assistantResponse) return {
+		pass: true,
+		score: 1,
+		reason: "No output to moderate"
+	};
+	const defaultProviders = await getDefaultProviders();
+	const defaultModerationProvider = !getEnvString("OPENAI_API_KEY") && (getEnvString("REPLICATE_API_KEY") || getEnvString("REPLICATE_API_TOKEN")) ? await loadApiProvider(LLAMA_GUARD_REPLICATE_PROVIDER) : defaultProviders.moderationProvider;
+	const moderationProvider = await getAndCheckProvider("moderation", grading?.provider, defaultModerationProvider, "moderation check");
+	invariant(moderationProvider, "Moderation provider must be defined");
+	const resp = await moderationProvider.callModerationApi(userPrompt, assistantResponse);
+	if (resp.error) return {
+		pass: false,
+		score: 0,
+		reason: `Moderation API error: ${resp.error}`
+	};
+	const { flags } = resp;
+	if (!flags || flags.length === 0) return {
+		pass: true,
+		score: 1,
+		reason: "No moderation flags detected"
+	};
+	const filteredFlags = categories.length === 0 ? flags : flags.filter((flag) => categories.includes(flag.code));
+	if (filteredFlags.length > 0) return {
+		pass: false,
+		score: 0,
+		reason: `Moderation flags detected: ${filteredFlags.map((flag) => flag.description).join(", ")}`
+	};
+	return {
+		pass: true,
+		score: 1,
+		reason: "No relevant moderation flags detected"
+	};
+}
+//#endregion
+//#region src/assertions/contextUtils.ts
+/**
+* Resolves the context value for context-based assertions.
+* Supports extracting context from test variables or transforming from output.
+* Can return either a single context string or an array of context chunks.
+*
+* @param assertion - The assertion configuration
+* @param test - The test case
+* @param output - The provider output (after provider transform, before test transform)
+* @param prompt - The prompt text
+* @param fallbackContext - Optional fallback context (e.g., prompt for context-recall)
+* @param providerResponse - Optional full provider response for contextTransform
+* @returns The resolved context string or array of strings
+* @throws Error if context cannot be resolved or transform fails
+*/
+async function resolveContext(assertion, test, output, prompt, fallbackContext, providerResponse) {
+	let contextValue;
+	if (test.vars?.context) {
+		if (typeof test.vars.context === "string") contextValue = test.vars.context;
+		else if (Array.isArray(test.vars.context)) {
+			const invalidEntry = [...test.vars.context.entries()].find(([, v]) => typeof v !== "string");
+			if (invalidEntry) {
+				const [idx, val] = invalidEntry;
+				invariant(false, `Invalid context: expected an array of strings, but found ${typeof val} at index ${idx}`);
+			}
+			contextValue = test.vars.context;
+		}
+	} else if (fallbackContext) contextValue = fallbackContext;
+	if (assertion.contextTransform) {
+		const getLabel = () => getTransformLabel(assertion.contextTransform);
+		try {
+			const outputForTransform = providerResponse?.providerTransformedOutput ?? output;
+			const transformed = await transform(assertion.contextTransform, outputForTransform, {
+				vars: test.vars,
+				prompt: { label: prompt },
+				...providerResponse && providerResponse.metadata && { metadata: providerResponse.metadata }
+			});
+			invariant(typeof transformed === "string" || Array.isArray(transformed) && transformed.every((item) => typeof item === "string"), () => `contextTransform must return a string or array of strings. Got ${typeof transformed}. Check your transform expression: ${getLabel()}`);
+			contextValue = transformed;
+		} catch (error) {
+			throw new Error(`Failed to transform context using expression '${getLabel()}': ${getTransformErrorMessage(error)}`);
+		}
+	}
+	invariant(typeof contextValue === "string" && contextValue.length > 0 || Array.isArray(contextValue) && contextValue.length > 0 && contextValue.every((item) => typeof item === "string" && item.length > 0), "Context is required for context-based assertions. Provide either a \"context\" variable (string or array of strings) in your test case or use \"contextTransform\" to extract context from the provider response.");
+	return contextValue;
+}
+/**
+* Serializes context (string or string[]) to a single string for prompts.
+* Joins chunks with double newlines to preserve separation.
+*/
+function serializeContext(context) {
+	return Array.isArray(context) ? context.join("\n\n") : context;
+}
+//#endregion
+//#region src/matchers/rag.ts
+async function matchesAnswerRelevance(input, output, threshold, grading, providerCallContext) {
+	const defaults = await getDefaultProviders();
+	const embeddingProvider = await getAndCheckProvider("embedding", grading?.provider, defaults.embeddingProvider, "answer relevancy check");
+	const textProvider = await getAndCheckProvider("text", grading?.provider, defaults.gradingProvider, "answer relevancy check");
+	const tokensUsed = normalizeMatcherTokenUsage(void 0);
+	const rubricPrompt = await loadRubricPrompt(grading?.rubricPrompt, ANSWER_RELEVANCY_GENERATE);
+	const parsedOutput = tryParse(output);
+	const promptText = await renderLlmRubricPrompt(rubricPrompt, { answer: parsedOutput });
+	const candidateQuestions = [];
+	for (let i = 0; i < 3; i++) {
+		const resp = await callProviderWithContext(textProvider, promptText, "answer-relevance", { answer: parsedOutput }, providerCallContext);
+		accumulateTokenUsage(tokensUsed, resp.tokenUsage);
+		if (resp.error || !resp.output) return fail(resp.error || "No output", tokensUsed);
+		invariant(typeof resp.output === "string", "answer relevancy check produced malformed response");
+		candidateQuestions.push(resp.output);
+	}
+	invariant(typeof embeddingProvider.callEmbeddingApi === "function", `Provider ${embeddingProvider.id()} must implement callEmbeddingApi for similarity check`);
+	const inputEmbeddingResp = await embeddingProvider.callEmbeddingApi(input);
+	accumulateTokenUsage(tokensUsed, inputEmbeddingResp.tokenUsage);
+	if (inputEmbeddingResp.error || !inputEmbeddingResp.embedding) return fail(inputEmbeddingResp.error || "No embedding", tokensUsed);
+	const inputEmbedding = inputEmbeddingResp.embedding;
+	const similarities = [];
+	const questionsWithScores = [];
+	for (const question of candidateQuestions) {
+		const resp = await embeddingProvider.callEmbeddingApi(question);
+		accumulateTokenUsage(tokensUsed, resp.tokenUsage);
+		if (resp.error || !resp.embedding) return fail(resp.error || "No embedding", tokensUsed);
+		const questionSimilarity = cosineSimilarity(inputEmbedding, resp.embedding);
+		similarities.push(questionSimilarity);
+		questionsWithScores.push({
+			question,
+			similarity: questionSimilarity
+		});
+	}
+	const similarity = similarities.reduce((a, b) => a + b, 0) / similarities.length;
+	const pass = similarity >= threshold - Number.EPSILON;
+	const greaterThanReason = `Relevance ${similarity.toFixed(2)} is greater than threshold ${threshold}`;
+	const lessThanReason = `Relevance ${similarity.toFixed(2)} is less than threshold ${threshold}`;
+	const metadata = {
+		generatedQuestions: questionsWithScores,
+		averageSimilarity: similarity,
+		threshold
+	};
+	if (pass) return {
+		pass: true,
+		score: similarity,
+		reason: greaterThanReason,
+		tokensUsed,
+		metadata
+	};
+	return {
+		pass: false,
+		score: similarity,
+		reason: lessThanReason,
+		tokensUsed,
+		metadata
+	};
+}
+async function matchesContextRecall(context, groundTruth, threshold, grading, vars, providerCallContext) {
+	const textProvider = await getAndCheckProvider("text", grading?.provider, (await getDefaultProviders()).gradingProvider, "context recall check");
+	const contextString = serializeContext(context);
+	const resp = await callProviderWithContext(textProvider, await renderLlmRubricPrompt(await loadRubricPrompt(grading?.rubricPrompt, CONTEXT_RECALL), {
+		context: contextString,
+		groundTruth,
+		...vars || {}
+	}), "context-recall", {
+		context: contextString,
+		groundTruth,
+		...vars || {}
+	}, providerCallContext);
+	if (resp.error || !resp.output) return fail(resp.error || "No output", resp.tokenUsage);
+	invariant(typeof resp.output === "string", "context-recall produced malformed response");
+	const attributedTokenLower = CONTEXT_RECALL_ATTRIBUTED_TOKEN.toLowerCase();
+	const notAttributedTokenLower = CONTEXT_RECALL_NOT_ATTRIBUTED_TOKEN.toLowerCase();
+	const sentences = splitIntoSentences(resp.output).filter((line) => {
+		const lowerLine = line.toLowerCase();
+		return lowerLine.includes(attributedTokenLower) || lowerLine.includes(notAttributedTokenLower);
+	});
+	const sentenceAttributions = [];
+	let numerator = 0;
+	for (const sentence of sentences) {
+		const lowerSentence = sentence.toLowerCase();
+		const isAttributed = !lowerSentence.includes(notAttributedTokenLower) && lowerSentence.includes(attributedTokenLower);
+		if (isAttributed) numerator++;
+		const sentenceMatch = sentence.match(/^\d+\.\s*([^\.]+\.)/);
+		const cleanSentence = sentenceMatch ? sentenceMatch[1].trim() : sentence.split(".")[0].trim();
+		sentenceAttributions.push({
+			sentence: cleanSentence,
+			attributed: isAttributed
+		});
+	}
+	const score = sentences.length > 0 ? numerator / sentences.length : 0;
+	const pass = score >= threshold - Number.EPSILON;
+	const metadata = {
+		sentenceAttributions,
+		totalSentences: sentences.length,
+		attributedSentences: numerator,
+		score
+	};
+	return {
+		pass,
+		score,
+		reason: pass ? `Recall ${score.toFixed(2)} is >= ${threshold}` : `Recall ${score.toFixed(2)} is < ${threshold}`,
+		tokensUsed: normalizeMatcherTokenUsage(resp.tokenUsage),
+		metadata
+	};
+}
+async function matchesContextRelevance(question, context, threshold, grading, providerCallContext) {
+	const textProvider = await getAndCheckProvider("text", grading?.provider, (await getDefaultProviders()).gradingProvider, "context relevance check");
+	const contextString = serializeContext(context);
+	const resp = await callProviderWithContext(textProvider, await renderLlmRubricPrompt(await loadRubricPrompt(grading?.rubricPrompt, CONTEXT_RELEVANCE), {
+		context: contextString,
+		query: question
+	}), "context-relevance", {
+		context: contextString,
+		query: question
+	}, providerCallContext);
+	if (resp.error || !resp.output) return fail(resp.error || "No output", resp.tokenUsage);
+	invariant(typeof resp.output === "string", "context-relevance produced malformed response");
+	const contextUnits = Array.isArray(context) ? context.filter((chunk) => chunk.trim().length > 0) : splitIntoSentences(context);
+	const totalContextUnits = contextUnits.length;
+	const extractedSentences = splitIntoSentences(resp.output);
+	const relevantSentences = [];
+	const insufficientInformation = resp.output.includes(CONTEXT_RELEVANCE_BAD);
+	let numerator = 0;
+	if (insufficientInformation) numerator = 0;
+	else {
+		const uniqueRelevantSentences = [...new Set(extractedSentences)];
+		numerator = Math.min(uniqueRelevantSentences.length, totalContextUnits);
+		relevantSentences.push(...uniqueRelevantSentences);
+	}
+	const score = totalContextUnits > 0 ? numerator / totalContextUnits : 0;
+	const pass = score >= threshold - Number.EPSILON;
+	const metadata = {
+		extractedSentences: relevantSentences,
+		totalContextUnits,
+		totalContextSentences: totalContextUnits,
+		contextUnits,
+		relevantSentenceCount: numerator,
+		insufficientInformation,
+		score
+	};
+	return {
+		pass,
+		score,
+		reason: pass ? `Context relevance ${score.toFixed(2)} is >= ${threshold}` : `Context relevance ${score.toFixed(2)} is < ${threshold}`,
+		tokensUsed: normalizeMatcherTokenUsage(resp.tokenUsage),
+		metadata
+	};
+}
+async function matchesContextFaithfulness(query, output, context, threshold, grading, vars, providerCallContext) {
+	const textProvider = await getAndCheckProvider("text", grading?.provider, (await getDefaultProviders()).gradingProvider, "faithfulness check");
+	const tokensUsed = normalizeMatcherTokenUsage(void 0);
+	if (grading?.rubricPrompt) invariant(Array.isArray(grading.rubricPrompt), "rubricPrompt must be an array");
+	const rawLongformPrompt = typeof grading?.rubricPrompt?.[0] === "string" ? grading?.rubricPrompt?.[0] : grading?.rubricPrompt?.[0]?.content;
+	const rawNliPrompt = typeof grading?.rubricPrompt?.[1] === "string" ? grading?.rubricPrompt?.[1] : grading?.rubricPrompt?.[1]?.content;
+	const longformPrompt = await loadRubricPrompt(rawLongformPrompt, CONTEXT_FAITHFULNESS_LONGFORM);
+	const nliPrompt = await loadRubricPrompt(rawNliPrompt, CONTEXT_FAITHFULNESS_NLI_STATEMENTS);
+	let promptText = await renderLlmRubricPrompt(longformPrompt, {
+		question: query,
+		answer: tryParse(output),
+		...vars || {}
+	});
+	let resp = await callProviderWithContext(textProvider, promptText, "context-faithfulness-longform", {
+		question: query,
+		answer: tryParse(output),
+		...vars || {}
+	}, providerCallContext);
+	accumulateTokenUsage(tokensUsed, resp.tokenUsage);
+	if (resp.error || !resp.output) return fail(resp.error || "No output", tokensUsed);
+	invariant(typeof resp.output === "string", "context-faithfulness produced malformed response");
+	const contextString = serializeContext(context);
+	const statements = splitIntoSentences(resp.output);
+	promptText = await renderLlmRubricPrompt(nliPrompt, {
+		context: contextString,
+		statements,
+		...vars || {}
+	});
+	resp = await callProviderWithContext(textProvider, promptText, "context-faithfulness-nli", {
+		context: contextString,
+		statements,
+		...vars || {}
+	}, providerCallContext);
+	accumulateTokenUsage(tokensUsed, resp.tokenUsage);
+	if (resp.error || !resp.output) return fail(resp.error || "No output", tokensUsed);
+	invariant(typeof resp.output === "string", "context-faithfulness produced malformed response");
+	let finalAnswer = "Final verdict for each statement in order:";
+	finalAnswer = finalAnswer.toLowerCase();
+	let verdicts = resp.output.toLowerCase().trim();
+	let score = 0;
+	if (statements.length > 0) if (verdicts.includes(finalAnswer)) {
+		verdicts = verdicts.slice(verdicts.indexOf(finalAnswer) + finalAnswer.length);
+		const parsedVerdicts = verdicts.split(".").filter((answer) => answer.trim() !== "");
+		if (parsedVerdicts.length > 0) score = 1 - parsedVerdicts.filter((answer) => !answer.includes("yes")).length / statements.length;
+	} else {
+		const noVerdictCount = verdicts.split("verdict: no").length - 1;
+		if (noVerdictCount + (verdicts.split("verdict: yes").length - 1) > 0) score = 1 - noVerdictCount / statements.length;
+	}
+	score = Math.min(1, Math.max(0, score));
+	const pass = score >= threshold - Number.EPSILON;
+	return {
+		pass,
+		score,
+		reason: pass ? `Faithfulness ${score.toFixed(2)} is >= ${threshold}` : `Faithfulness ${score.toFixed(2)} is < ${threshold}`,
+		tokensUsed
+	};
+}
+//#endregion
+//#region src/matchers/similarity.ts
+function calculateSimilarityScore(expectedEmbedding, outputEmbedding, metric, tokensUsed) {
+	switch (metric) {
+		case "cosine": return cosineSimilarity(expectedEmbedding, outputEmbedding);
+		case "dot_product": return dotProduct(expectedEmbedding, outputEmbedding);
+		case "euclidean": return euclideanDistance(expectedEmbedding, outputEmbedding);
+		default: return fail(`Unsupported metric: ${metric}`, tokensUsed);
+	}
+}
+function buildSimilarityResult(similarity, threshold, inverse, metric, tokensUsed) {
+	if (metric === "euclidean") {
+		const distance = similarity;
+		const pass = inverse ? distance >= threshold - Number.EPSILON : distance <= threshold + Number.EPSILON;
+		const normalizedScore = 1 / (1 + distance);
+		const score = inverse ? 1 - normalizedScore : normalizedScore;
+		const belowThresholdReason = `Distance ${distance.toFixed(2)} is less than or equal to threshold ${threshold}`;
+		const aboveThresholdReason = `Distance ${distance.toFixed(2)} is greater than threshold ${threshold}`;
+		return {
+			pass,
+			score,
+			reason: pass ? inverse ? aboveThresholdReason : belowThresholdReason : inverse ? belowThresholdReason : aboveThresholdReason,
+			tokensUsed
+		};
+	}
+	const pass = inverse ? similarity <= threshold + Number.EPSILON : similarity >= threshold - Number.EPSILON;
+	const score = inverse ? 1 - similarity : similarity;
+	const greaterThanReason = `Similarity ${similarity.toFixed(2)} is greater than or equal to threshold ${threshold}`;
+	const lessThanReason = `Similarity ${similarity.toFixed(2)} is less than threshold ${threshold}`;
+	return {
+		pass,
+		score,
+		reason: pass ? inverse ? lessThanReason : greaterThanReason : inverse ? greaterThanReason : lessThanReason,
+		tokensUsed
+	};
+}
+async function calculateProviderSimilarity(finalProvider, expected, output, metric, tokensUsed) {
+	if (metric === "cosine" && "callSimilarityApi" in finalProvider) {
+		const similarityResp = await finalProvider.callSimilarityApi(expected, output);
+		accumulateTokenUsage(tokensUsed, similarityResp.tokenUsage);
+		if (similarityResp.error) return fail(similarityResp.error, tokensUsed);
+		if (similarityResp.similarity == null) return fail("Unknown error fetching similarity", tokensUsed);
+		if (!Number.isFinite(similarityResp.similarity)) return fail(`Invalid similarity score: ${similarityResp.similarity}`, tokensUsed);
+		return similarityResp.similarity;
+	}
+	const callEmbeddingApi = "callEmbeddingApi" in finalProvider ? finalProvider.callEmbeddingApi : void 0;
+	if (typeof callEmbeddingApi !== "function") {
+		if ("callSimilarityApi" in finalProvider) return fail(`Provider ${finalProvider.id()} only supports cosine similarity via callSimilarityApi`, tokensUsed);
+		throw new Error("Provider must implement callSimilarityApi or callEmbeddingApi");
+	}
+	const [expectedEmbedding, outputEmbedding] = await Promise.all([callEmbeddingApi.call(finalProvider, expected), callEmbeddingApi.call(finalProvider, output)]);
+	const mergedUsage = normalizeMatcherTokenUsage(void 0);
+	accumulateTokenUsage(mergedUsage, expectedEmbedding.tokenUsage);
+	accumulateTokenUsage(mergedUsage, outputEmbedding.tokenUsage);
+	accumulateTokenUsage(tokensUsed, mergedUsage);
+	if (expectedEmbedding.error || outputEmbedding.error) return fail(expectedEmbedding.error || outputEmbedding.error || "Unknown error fetching embeddings", tokensUsed);
+	if (!expectedEmbedding.embedding || !outputEmbedding.embedding) return fail("Embedding not found", tokensUsed);
+	return calculateSimilarityScore(expectedEmbedding.embedding, outputEmbedding.embedding, metric, tokensUsed);
+}
+async function matchesSimilarity(expected, output, threshold, inverse = false, grading, metric = "cosine") {
+	if (metric === "cosine" && state.config?.redteam && shouldGenerateRemote({ requireEmbeddingProvider: true })) try {
+		return await doRemoteGrading({
+			task: "similar",
+			expected,
+			output,
+			threshold,
+			inverse
+		});
+	} catch (error) {
+		return fail(`Could not perform remote grading: ${error}`);
+	}
+	const defaults = await getDefaultProviders();
+	const finalProvider = await getAndCheckProvider("embedding", grading?.provider, defaults.embeddingProvider, "similarity check");
+	const tokensUsed = normalizeMatcherTokenUsage(void 0);
+	const similarity = await calculateProviderSimilarity(finalProvider, expected, output, metric, tokensUsed);
+	if (typeof similarity !== "number") return similarity;
+	return buildSimilarityResult(similarity, threshold, inverse, metric, tokensUsed);
+}
+//#endregion
 //#region src/tracing/evaluatorTracing.ts
 let otlpReceiverStarted = false;
 const DEFAULT_OTLP_ACCEPT_FORMATS = ["json", "protobuf"];
@@ -261,7 +771,7 @@ async function startOtlpReceiverIfNeeded(testSuite) {
 		telemetry.record("feature_used", { feature: "tracing" });
 		try {
 			logger.debug("[EvaluatorTracing] Tracing configuration detected, starting OTLP receiver");
-			const { startOTLPReceiver } = await import("./otlpReceiver-BntK801g.js");
+			const { startOTLPReceiver } = await import("./otlpReceiver--gTpSagc.js");
 			const port = testSuite.tracing.otlp.http.port || 4318;
 			const host = testSuite.tracing.otlp.http.host || "127.0.0.1";
 			const acceptFormats = normalizeOtlpAcceptFormats(testSuite.tracing.otlp.http.acceptFormats);
@@ -285,7 +795,7 @@ async function startOtlpReceiverIfNeeded(testSuite) {
 async function stopOtlpReceiverIfNeeded() {
 	if (otlpReceiverStarted) try {
 		logger.debug("[EvaluatorTracing] Stopping OTLP receiver");
-		const { stopOTLPReceiver } = await import("./otlpReceiver-BntK801g.js");
+		const { stopOTLPReceiver } = await import("./otlpReceiver--gTpSagc.js");
 		await stopOTLPReceiver();
 		otlpReceiverStarted = false;
 		logger.info("[EvaluatorTracing] OTLP receiver stopped successfully");
@@ -320,7 +830,7 @@ async function generateTraceContextIfNeeded(test, evaluateOptions, testIdx, prom
 	}
 	if (!tracingEnabled) return null;
 	logger.debug("[EvaluatorTracing] Importing trace store");
-	const { getTraceStore } = await import("./store-DCDBhv7B.js");
+	const { getTraceStore } = await import("./store-IbiRIF3k.js");
 	const traceStore = getTraceStore();
 	const traceId = generateTraceId();
 	const spanId = generateSpanId();
@@ -630,38 +1140,84 @@ async function handleClassifier({ assertion, renderedValue, outputString, test,
 }
 //#endregion
 //#region src/assertions/contains.ts
+/**
+* Advance over separators between parsed fields.
+*
+* Contains-any values allow whitespace around comma delimiters, and historical
+* parsing ignored repeated commas rather than producing empty fields.
+*/
+function skipWhitespaceAndCommas(value, startIndex) {
+	let i = startIndex;
+	while (i < value.length) {
+		i = skipWhitespace(value, i);
+		if (value[i] !== ",") break;
+		i++;
+	}
+	return i;
+}
+/**
+* Advance over whitespace while preserving comma delimiter handling for callers.
+*/
+function skipWhitespace(value, startIndex) {
+	let i = startIndex;
+	while (i < value.length && /\s/.test(value[i])) i++;
+	return i;
+}
+/**
+* Parse a quoted field using the assertion parser's CSV-like escape rules.
+*
+* Supports backslash-escaped quotes/backslashes and doubled quotes, and rejects
+* unterminated fields so malformed assertion values do not silently pass.
+*/
+function parseQuotedField(value, startIndex) {
+	let i = startIndex + 1;
+	let field = "";
+	let terminated = false;
+	while (i < value.length) if (value[i] === "\\" && i + 1 < value.length && ["\"", "\\"].includes(value[i + 1])) {
+		field += value[i + 1];
+		i += 2;
+	} else if (value[i] === "\"" && i + 1 < value.length && value[i + 1] === "\"") {
+		field += "\"";
+		i += 2;
+	} else if (value[i] === "\"") {
+		i++;
+		terminated = true;
+		break;
+	} else {
+		field += value[i];
+		i++;
+	}
+	invariant(terminated, "Unterminated quoted field in contains assertion value");
+	return {
+		field,
+		nextIndex: i
+	};
+}
+/**
+* Parse an unquoted field up to the next comma, trimming surrounding whitespace.
+*/
+function parseUnquotedField(value, startIndex) {
+	let i = startIndex;
+	while (i < value.length && value[i] !== ",") i++;
+	return {
+		field: value.substring(startIndex, i).trim(),
+		nextIndex: i
+	};
+}
+/**
+* Split a contains-any string into fields while preserving quoted commas.
+*/
 function parseCommaSeparatedValues(value) {
 	const results = [];
 	let i = 0;
 	while (i < value.length) {
-		while (i < value.length && /\s/.test(value[i])) i++;
+		i = skipWhitespaceAndCommas(value, i);
 		if (i >= value.length) break;
-		if (value[i] === ",") {
-			i++;
-			continue;
-		}
-		if (value[i] === "\"") {
-			i++;
-			let field = "";
-			while (i < value.length) if (value[i] === "\\" && i + 1 < value.length && (value[i + 1] === "\"" || value[i + 1] === "\\")) {
-				field += value[i + 1];
-				i += 2;
-			} else if (value[i] === "\"" && i + 1 < value.length && value[i + 1] === "\"") {
-				field += "\"";
-				i += 2;
-			} else if (value[i] === "\"") {
-				i++;
-				break;
-			} else {
-				field += value[i];
-				i++;
-			}
-			results.push(field);
-		} else {
-			const start = i;
-			while (i < value.length && value[i] !== ",") i++;
-			results.push(value.substring(start, i).trim());
-		}
+		const isQuotedField = value[i] === "\"";
+		const parsed = isQuotedField ? parseQuotedField(value, i) : parseUnquotedField(value, i);
+		results.push(parsed.field);
+		i = isQuotedField ? skipWhitespace(value, parsed.nextIndex) : parsed.nextIndex;
+		invariant(!isQuotedField || i >= value.length || value[i] === ",", "Expected comma after quoted field in contains assertion value");
 	}
 	return results;
 }
@@ -906,27 +1462,67 @@ const handleIsValidFunctionCall = ({ assertion, output, provider, test }) => {
 };
 //#endregion
 //#region src/assertions/geval.ts
-const handleGEval = async ({ assertion, renderedValue, prompt, outputString, test, providerCallContext }) => {
-	invariant(typeof renderedValue === "string" || Array.isArray(renderedValue), "G-Eval assertion type must have a string or array of strings value");
+const handleGEval = async ({ assertion, inverse, renderedValue, prompt, outputString, test, providerCallContext }) => {
+	invariant(typeof renderedValue === "string" || Array.isArray(renderedValue) && renderedValue.every((value) => typeof value === "string"), "G-Eval assertion type must have a string or array of strings value");
 	const threshold = assertion.threshold ?? .7;
 	if (Array.isArray(renderedValue)) {
-		const scores = [];
-		const reasons = [];
-		for (const value of renderedValue) {
+		if (renderedValue.length === 0) return {
+			assertion,
+			pass: false,
+			score: 0,
+			reason: "G-Eval assertion requires at least one criterion string in the value array."
+		};
+		const responses = [];
+		let failure;
+		for (const [index, value] of renderedValue.entries()) {
 			const resp = await matchesGEval(value, prompt || "", outputString, threshold, test.options, providerCallContext);
-			scores.push(resp.score);
-			reasons.push(resp.reason);
+			responses.push(resp);
+			if (isGraderFailure(resp)) {
+				failure = {
+					index,
+					resp
+				};
+				break;
+			}
+		}
+		const tokensUsed = createEmptyTokenUsage();
+		for (const r of responses) accumulateTokenUsage(tokensUsed, r.tokensUsed);
+		if (failure) {
+			const criterion = renderedValue[failure.index];
+			return {
+				assertion,
+				pass: false,
+				score: 0,
+				reason: `G-Eval criterion ${failure.index + 1}/${renderedValue.length} (${JSON.stringify(criterion)}) failed: ${failure.resp.reason}`,
+				tokensUsed,
+				metadata: failure.resp.metadata
+			};
 		}
-		const scoresSum = scores.reduce((a, b) => a + b, 0);
+		const averageScore = responses.reduce((acc, r) => acc + r.score, 0) / responses.length;
+		const combinedReason = responses.map((r) => r.reason).join("\n\n");
 		return {
 			assertion,
-			pass: scoresSum / scores.length >= threshold,
-			score: scoresSum / scores.length,
-			reason: reasons.join("\n\n")
+			pass: averageScore >= threshold !== inverse,
+			score: inverse ? 1 - averageScore : averageScore,
+			reason: combinedReason,
+			tokensUsed
 		};
-	} else return {
+	}
+	const resp = await matchesGEval(renderedValue, prompt || "", outputString, threshold, test.options, providerCallContext);
+	if (isGraderFailure(resp)) return {
 		assertion,
-		...await matchesGEval(renderedValue, prompt || "", outputString, threshold, test.options, providerCallContext)
+		pass: false,
+		score: 0,
+		reason: resp.reason,
+		tokensUsed: resp.tokensUsed,
+		metadata: resp.metadata
+	};
+	const passed = resp.score >= threshold !== inverse;
+	return {
+		assertion,
+		...resp,
+		pass: passed,
+		score: inverse ? 1 - resp.score : resp.score
 	};
 };
 //#endregion
@@ -1066,6 +1662,43 @@ const handleGuardrails = async ({ assertion, inverse, providerResponse }) => {
 };
 //#endregion
 //#region src/assertions/html.ts
+const LITERAL_WRAPPER_PATTERNS = {
+	html: /<html(?=[\s>/])/,
+	head: /<head(?=[\s>/])/,
+	body: /<body(?=[\s>/])/
+};
+function isWrapperTagName(tagName) {
+	return tagName === "html" || tagName === "head" || tagName === "body";
+}
+function isTextNode(node) {
+	return node.nodeName === "#text";
+}
+function isElementNode(node) {
+	return "tagName" in node;
+}
+function hasSourceCodeLocation(element) {
+	return "sourceCodeLocation" in element && element.sourceCodeLocation !== null && element.sourceCodeLocation !== void 0;
+}
+function getChildNodes(node) {
+	return "childNodes" in node ? node.childNodes : [];
+}
+function findFirstElement(root, predicate) {
+	const stack = [root];
+	while (stack.length > 0) {
+		const current = stack.pop();
+		if (isElementNode(current) && predicate(current)) return current;
+		const children = getChildNodes(current);
+		for (let i = children.length - 1; i >= 0; i--) stack.push(children[i]);
+	}
+}
+function hasTopLevelText(parentNode) {
+	return parentNode.childNodes.some((node) => isTextNode(node) && Boolean(node.value.trim()));
+}
+function isUserProvidedElement(element, inputLowercase) {
+	const tagName = element.tagName.toLowerCase();
+	if (isWrapperTagName(tagName)) return LITERAL_WRAPPER_PATTERNS[tagName].test(inputLowercase) && hasSourceCodeLocation(element);
+	return VALID_HTML_ELEMENTS.has(tagName) || tagName.includes("-");
+}
 const HTML_PATTERNS = {
 	openingTag: /<[a-zA-Z][a-zA-Z0-9-]*(?:\s[^>]*)?>/,
 	closingTag: /<\/[a-zA-Z][a-zA-Z0-9-]*\s*>/,
@@ -1221,37 +1854,21 @@ function validateHtml(htmlString) {
 		isValid: false,
 		reason: "Output appears to be XML, not HTML"
 	};
-	try {
-		const { document } = new JSDOM(trimmed, { contentType: "text/html" }).window;
-		if (document.body && !trimmed.toLowerCase().includes("<body")) {
-			if (Array.from(document.body.childNodes).some((node) => node.nodeType === 3 && node.textContent?.trim())) return {
-				isValid: false,
-				reason: "Output must be wrapped in HTML tags"
-			};
-		}
-		const allElements = document.querySelectorAll("*");
-		if (!Array.from(allElements).find((element) => {
-			const tagName = element.tagName.toLowerCase();
-			if ([
-				"html",
-				"head",
-				"body"
-			].includes(tagName) && !trimmed.toLowerCase().includes(`<${tagName}`)) return false;
-			return VALID_HTML_ELEMENTS.has(tagName) || tagName.includes("-");
-		})) return {
-			isValid: false,
-			reason: "Output does not contain recognized HTML elements"
-		};
-		return {
-			isValid: true,
-			reason: "Output is valid HTML"
-		};
-	} catch (error) {
-		return {
-			isValid: false,
-			reason: `HTML parsing failed: ${error instanceof Error ? error.message : "Unknown error"}`
-		};
-	}
+	const document = parse$1(trimmed, { sourceCodeLocationInfo: true });
+	const inputLowercase = trimmed.toLowerCase();
+	const body = findFirstElement(document, (element) => element.tagName === "body");
+	if (!(body !== void 0 && LITERAL_WRAPPER_PATTERNS.body.test(inputLowercase) && hasSourceCodeLocation(body)) && body && hasTopLevelText(body)) return {
+		isValid: false,
+		reason: "Output must be wrapped in HTML tags"
+	};
+	if (!findFirstElement(document, (element) => isUserProvidedElement(element, inputLowercase))) return {
+		isValid: false,
+		reason: "Output does not contain recognized HTML elements"
+	};
+	return {
+		isValid: true,
+		reason: "Output is valid HTML"
+	};
 }
 const handleContainsHtml = ({ assertion, outputString, inverse }) => {
 	const pass = containsHtml(outputString) !== inverse;
@@ -1866,45 +2483,6 @@ function matchesPattern(spanName, pattern) {
 }
 //#endregion
 //#region src/assertions/trajectoryUtils.ts
-const TOOL_ATTRIBUTE_KEYS = [
-	"tool.name",
-	"tool_name",
-	"tool",
-	"function.name",
-	"function_name",
-	"gen_ai.tool.name",
-	"codex.mcp.tool",
-	"agent.tool",
-	"agent.tool_name",
-	"agent.toolName"
-];
-const TOOL_ARGUMENT_ATTRIBUTE_KEYS = [
-	"tool.arguments",
-	"tool.args",
-	"tool.input",
-	"tool_arguments",
-	"tool_args",
-	"tool_input",
-	"function.arguments",
-	"function.args",
-	"function.input",
-	"function_arguments",
-	"function_args",
-	"gen_ai.tool.arguments",
-	"gen_ai.tool.args",
-	"gen_ai.tool.input",
-	"gen_ai.tool.call.arguments",
-	"gen_ai.tool.call.args",
-	"agent.tool.arguments",
-	"agent.tool.args",
-	"agent.tool.input",
-	"codex.mcp.arguments",
-	"codex.mcp.args",
-	"codex.mcp.input",
-	"arguments",
-	"args",
-	"input"
-];
 const COMMAND_ATTRIBUTE_KEYS = [
 	"codex.command",
 	"command",
@@ -1917,16 +2495,15 @@ const SEARCH_ATTRIBUTE_KEYS = [
 	"search_query"
 ];
 const GENERIC_QUERY_ATTRIBUTE_KEYS = ["query"];
+const COMMAND_TOOL_NAMES = new Set([
+	"exec_command",
+	"local_shell",
+	"shell"
+]);
 const SEARCH_SPAN_NAME_PATTERN = /(^|[\s._:/-])(search|find|lookup|retriev(?:e|al))($|[\s._:/-])/i;
 const MAX_JUDGE_SUMMARY_STEPS = 24;
 const JUDGE_SUMMARY_HEAD_STEPS = 12;
 const JUDGE_SUMMARY_TAIL_STEPS = 12;
-function getStringAttribute(attributes, keys) {
-	for (const key of keys) {
-		const value = attributes[key];
-		if (typeof value === "string" && value.trim()) return value.trim();
-	}
-}
 function normalizeStructuredAttribute(value) {
 	if (value === void 0 || value === null) return;
 	if (typeof value === "string") {
@@ -1958,9 +2535,12 @@ function getTrajectoryStepStatus(step) {
 function getCommandExecutable(command) {
 	return command.trim().split(/\s+/)[0] || void 0;
 }
+function isCommandToolName(toolName) {
+	return !!toolName && COMMAND_TOOL_NAMES.has(toolName.trim().toLowerCase());
+}
 function extractToolName(span) {
 	const attributes = span.attributes || {};
-	const directMatch = getStringAttribute(attributes, TOOL_ATTRIBUTE_KEYS);
+	const directMatch = getToolNameFromAttributes(attributes);
 	if (directMatch) return directMatch;
 	for (const [key, value] of Object.entries(attributes)) {
 		if (typeof value !== "string" || !value.trim()) continue;
@@ -1985,21 +2565,31 @@ function extractToolArgs(span) {
 		if (value !== void 0) return value;
 	}
 }
-function extractCommand(span) {
+function extractCommand(span, toolName = extractToolName(span), getToolArgs = () => extractToolArgs(span)) {
 	const attributes = span.attributes || {};
-	const directMatch = getStringAttribute(attributes, COMMAND_ATTRIBUTE_KEYS);
+	const directMatch = getFirstStringAttribute(attributes, COMMAND_ATTRIBUTE_KEYS);
 	if (directMatch) return directMatch;
 	for (const [key, value] of Object.entries(attributes)) {
 		if (typeof value !== "string" || !value.trim()) continue;
 		if (/command/i.test(key) && !/output|result/i.test(key)) return value.trim();
 	}
+	const toolArgs = getToolArgs();
+	if (isCommandToolName(toolName) && toolArgs && typeof toolArgs === "object") {
+		const args = toolArgs;
+		const command = args.cmd ?? args.command;
+		if (typeof command === "string" && command.trim()) return command.trim();
+		if (Array.isArray(command)) {
+			const joined = command.map((part) => String(part).trim()).filter(Boolean).join(" ");
+			if (joined) return joined;
+		}
+	}
 	if (span.name.startsWith("exec ")) return span.name.slice(5).trim();
 }
 function extractSearchQuery(span) {
 	const attributes = span.attributes || {};
-	const directMatch = getStringAttribute(attributes, SEARCH_ATTRIBUTE_KEYS);
+	const directMatch = getFirstStringAttribute(attributes, SEARCH_ATTRIBUTE_KEYS);
 	if (directMatch) return directMatch;
-	const genericQuery = getStringAttribute(attributes, GENERIC_QUERY_ATTRIBUTE_KEYS);
+	const genericQuery = getFirstStringAttribute(attributes, GENERIC_QUERY_ATTRIBUTE_KEYS);
 	if (genericQuery && isSearchLikeSpan(span)) return genericQuery;
 	if (span.name.startsWith("search ")) return span.name.slice(7).replace(/^"|"$/g, "").trim();
 }
@@ -2023,17 +2613,34 @@ function extractTrajectorySteps(trace) {
 		return left.index - right.index;
 	}).map(({ span }) => {
 		const toolName = extractToolName(span);
-		const command = extractCommand(span);
+		let toolArgs;
+		let hasExtractedToolArgs = false;
+		const getToolArgs = () => {
+			if (!hasExtractedToolArgs) {
+				toolArgs = extractToolArgs(span);
+				hasExtractedToolArgs = true;
+			}
+			return toolArgs;
+		};
+		const command = extractCommand(span, toolName, getToolArgs);
 		const searchQuery = extractSearchQuery(span);
 		let type = "span";
 		let name = span.name;
 		const aliases = new Set([span.name]);
 		let args;
-		if (toolName) {
+		if (command && isCommandToolName(toolName)) {
+			type = "command";
+			name = command;
+			aliases.add(command);
+			args = getToolArgs();
+			if (toolName) aliases.add(toolName);
+			const executable = getCommandExecutable(command);
+			if (executable) aliases.add(executable);
+		} else if (toolName) {
 			type = "tool";
 			name = toolName;
 			aliases.add(toolName);
-			args = extractToolArgs(span);
+			args = getToolArgs();
 		} else if (command) {
 			type = "command";
 			name = command;
@@ -2314,11 +2921,10 @@ function handleRougeScore({ baseType, assertion, renderedValue, outputString, in
 	const rougeMethod = rouge[baseType[baseType.length - 1]];
 	const score = rougeMethod(outputString, renderedValue, {});
 	const threshold = assertion.threshold ?? .75;
-	const pass = score >= threshold != inverse;
 	return {
-		pass,
+		pass: score >= threshold !== inverse,
 		score: inverse ? 1 - score : score,
-		reason: pass ? `${baseType.toUpperCase()} score ${score.toFixed(2)} is greater than or equal to threshold ${threshold}` : `${baseType.toUpperCase()} score ${score.toFixed(2)} is less than threshold ${threshold}`,
+		reason: `${baseType.toUpperCase()} score ${score.toFixed(2)} is ${score >= threshold ? "greater than or equal to" : "less than"} threshold ${threshold}`,
 		assertion
 	};
 }
@@ -2380,6 +2986,192 @@ const handleRuby = async ({ assertion, renderedValue, valueFromScript, assertion
 	}
 };
 //#endregion
+//#region src/providers/webSearchUtils.ts
+function hasTool(provider, predicate) {
+	return Array.isArray(provider.config?.tools) && provider.config.tools.some(predicate);
+}
+function getProviderId(provider) {
+	if (typeof provider.id !== "function") return null;
+	try {
+		return provider.id();
+	} catch (err) {
+		logger.debug(`Failed to read provider id: ${err}`);
+		return null;
+	}
+}
+function isOpenAiResponsesProvider(provider, id) {
+	return id.includes("openai:responses") || provider.constructor?.name === "OpenAiResponsesProvider";
+}
+/**
+* Check if a provider has web search capabilities
+* @param provider The provider to check
+* @returns true if the provider supports web search
+*/
+function hasWebSearchCapability(provider) {
+	if (!provider) return false;
+	const id = getProviderId(provider);
+	if (!id) return false;
+	if (id.includes("perplexity")) return true;
+	if ((id.includes("google") || id.includes("gemini") || id.includes("vertex")) && hasTool(provider, (t) => t.googleSearch !== void 0)) return true;
+	if (id.includes("xai") && provider.config?.search_parameters?.mode === "on") return true;
+	if (isOpenAiResponsesProvider(provider, id) && hasTool(provider, (t) => t.type === "web_search_preview")) return true;
+	if (id.startsWith("openai:codex") && (provider.config?.web_search_mode === "live" || provider.config?.web_search_mode === "cached" || provider.config?.web_search_enabled === true)) return true;
+	if (id.includes("anthropic") && hasTool(provider, (t) => t.type === "web_search_20250305")) return true;
+	return false;
+}
+/**
+* Load a provider with web search capabilities.
+* Tries multiple providers in order of preference until one succeeds.
+* Uses the latest and most capable models from each provider with specific checkpoint IDs.
+*
+* @param preferAnthropic Whether to try Anthropic first (true) or OpenAI first (false)
+* @returns A provider with web search capabilities or null
+*/
+async function loadWebSearchProvider(preferAnthropic = false) {
+	const loadAnthropicWebSearch = async () => {
+		try {
+			return await loadApiProvider("anthropic:messages:claude-opus-4-6", { options: { config: { tools: [{
+				type: "web_search_20250305",
+				name: "web_search",
+				max_uses: 5
+			}] } } });
+		} catch (err) {
+			logger.debug(`Failed to load Anthropic web search provider: ${err}`);
+			return null;
+		}
+	};
+	const loadOpenAIWebSearch = async () => {
+		try {
+			return await loadApiProvider("openai:responses:gpt-5.4-2026-03-05", { options: { config: { tools: [{ type: "web_search_preview" }] } } });
+		} catch (err) {
+			logger.debug(`Failed to load OpenAI web search provider: ${err}`);
+			return null;
+		}
+	};
+	const loadPerplexity = async () => {
+		try {
+			return await loadApiProvider("perplexity:sonar-pro");
+		} catch (err) {
+			logger.debug(`Failed to load Perplexity provider: ${err}`);
+			return null;
+		}
+	};
+	const loadGoogleWebSearch = async () => {
+		try {
+			return await loadApiProvider("google:gemini-3-pro-preview", { options: { config: { tools: [{ googleSearch: {} }] } } });
+		} catch (err) {
+			logger.debug(`Failed to load Google web search provider: ${err}`);
+			return null;
+		}
+	};
+	const loadVertexWebSearch = async () => {
+		try {
+			return await loadApiProvider("vertex:gemini-3-pro-preview", { options: { config: { tools: [{ googleSearch: {} }] } } });
+		} catch (err) {
+			logger.debug(`Failed to load Vertex web search provider: ${err}`);
+			return null;
+		}
+	};
+	const loadXaiWebSearch = async () => {
+		try {
+			return await loadApiProvider("xai:grok-4-1-fast-reasoning", { options: { config: { search_parameters: { mode: "on" } } } });
+		} catch (err) {
+			logger.debug(`Failed to load xAI web search provider: ${err}`);
+			return null;
+		}
+	};
+	const providers = preferAnthropic ? [
+		loadAnthropicWebSearch,
+		loadOpenAIWebSearch,
+		loadPerplexity,
+		loadGoogleWebSearch,
+		loadVertexWebSearch,
+		loadXaiWebSearch
+	] : [
+		loadOpenAIWebSearch,
+		loadAnthropicWebSearch,
+		loadPerplexity,
+		loadGoogleWebSearch,
+		loadVertexWebSearch,
+		loadXaiWebSearch
+	];
+	for (const getProvider of providers) {
+		const provider = await getProvider();
+		if (provider && hasWebSearchCapability(provider)) {
+			logger.info(`Using ${getProviderId(provider) ?? "loaded provider"} as web search provider`);
+			return provider;
+		}
+		if (provider) logger.debug(`Loaded provider ${getProviderId(provider) ?? "unknown"} does not support web search`);
+	}
+	return null;
+}
+//#endregion
+//#region src/matchers/search.ts
+async function matchesSearchRubric(rubric, llmOutput, grading, vars, assertion, _provider, providerCallContext) {
+	if (!grading) throw new Error("Cannot grade output without grading config. Specify --grader option or grading config.");
+	const defaultProviders = await getDefaultProviders();
+	const defaultSearchProviders = [
+		defaultProviders.webSearchProvider,
+		defaultProviders.llmRubricProvider,
+		defaultProviders.gradingProvider
+	];
+	let searchProvider = (grading.provider ? await getGradingProvider("text", grading.provider, null) : null) || defaultSearchProviders.find((provider) => Boolean(provider));
+	if (!hasWebSearchCapability(searchProvider)) {
+		const webSearchDefault = defaultSearchProviders.find((provider) => hasWebSearchCapability(provider));
+		if (webSearchDefault) searchProvider = webSearchDefault;
+	}
+	if (!hasWebSearchCapability(searchProvider)) {
+		const webSearchProvider = await loadWebSearchProvider(true);
+		if (webSearchProvider) searchProvider = webSearchProvider;
+	}
+	if (!searchProvider || !hasWebSearchCapability(searchProvider)) throw new Error(`search-rubric assertion requires a grading provider with web search capabilities. Use --grader with a web search provider (e.g., anthropic:messages:${DEFAULT_ANTHROPIC_MODEL}, openai:responses:o4-mini with tools configured, perplexity:sonar) or configure one in defaultTest.options.provider`);
+	const prompt = await renderLlmRubricPrompt(await loadRubricPrompt(grading?.rubricPrompt, DEFAULT_WEB_SEARCH_PROMPT), {
+		output: tryParse(llmOutput),
+		rubric,
+		...vars || {}
+	});
+	const resp = await callProviderWithContext(searchProvider, prompt, "search-rubric", {
+		output: tryParse(llmOutput),
+		rubric,
+		...vars || {}
+	}, providerCallContext);
+	if (resp.error || !resp.output) return {
+		pass: false,
+		score: 0,
+		reason: `Search rubric evaluation failed: ${resp.error || "No output"}`,
+		tokensUsed: resp.tokenUsage,
+		assertion
+	};
+	try {
+		const result = extractFirstJsonObject(String(resp.output));
+		let pass = result.pass ?? false;
+		const score = typeof result.score === "number" ? result.score : pass ? 1 : 0;
+		if (assertion?.threshold !== void 0) pass = pass && score >= assertion.threshold;
+		return {
+			pass,
+			score,
+			reason: result.reason || "No reason provided",
+			tokensUsed: resp.tokenUsage,
+			assertion,
+			metadata: {
+				searchResults: result.searchResults || [],
+				searchProvider: searchProvider.id()
+			}
+		};
+	} catch (err) {
+		logger.warn(`[search-rubric] Could not parse structured JSON from provider response, falling back to substring matching: ${err.message}`);
+		const outputLower = String(resp.output).toLowerCase();
+		const pass = outputLower.includes("\"pass\":true") || outputLower.includes("\"pass\": true");
+		return {
+			pass,
+			score: pass ? 1 : 0,
+			reason: resp.output,
+			tokensUsed: resp.tokenUsage,
+			assertion
+		};
+	}
+}
+//#endregion
 //#region src/assertions/searchRubric.ts
 async function handleSearchRubric({ assertion, baseType: _baseType, inverse, provider, providerCallContext, renderedValue, test, providerResponse }) {
 	if (renderedValue == null) throw new Error("search-rubric assertion type must have a string value");
@@ -3412,7 +4204,7 @@ async function loadTraceData(traceId) {
 	let stableObservations = 0;
 	let latestTrace = null;
 	for (let attempt = 0; attempt < maxAttempts; attempt++) {
-		latestTrace = await traceStore.getTrace(traceId);
+		latestTrace = await traceStore.getTrace(traceId, { sanitizeAttributes: false });
 		const spanCount = latestTrace?.spans?.length ?? 0;
 		if (spanCount > 0) {
 			stableObservations = spanCount === lastSpanCount ? stableObservations + 1 : 1;
@@ -3465,7 +4257,7 @@ const ASSERTION_HANDLERS = {
 	"llm-rubric": handleLlmRubric,
 	meteor: async (params) => {
 		try {
-			const { handleMeteorAssertion } = await import("./meteor-DHdzY1Ss.js");
+			const { handleMeteorAssertion } = await import("./meteor--TZYICTI.js");
 			return handleMeteorAssertion(params);
 		} catch (error) {
 			if (error instanceof Error && (error.message.includes("Cannot find module") || error.message.includes("natural\" package is required"))) return {
@@ -3601,7 +4393,7 @@ async function runAssertion({ prompt, provider, assertion, test, vars, latencyMs
 			};
 		}
 		else if (filePath.endsWith(".rb")) try {
-			const { runRuby } = await import("./rubyUtils-D1L2d3jb.js");
+			const { runRuby } = await import("./rubyUtils-BYVlQ94c.js");
 			valueFromScript = await runRuby(filePath, functionName || "get_assert", [output, context]);
 			logger.debug(`Ruby script ${filePath} output: ${valueFromScript}`);
 		} catch (error) {
@@ -3718,7 +4510,8 @@ async function runAssertions({ assertScoringFunction, latencyMs, prompt, provide
 		logger.debug(`Failed to preload trace data for assertions: ${error}`);
 		preloadedTraceData = null;
 	}
-	await async.forEachOfLimit(asserts, ASSERTIONS_MAX_CONCURRENCY, async ({ assertion, assertResult, index }) => {
+	const concurrency = getProviderCallExecutionContext()?.providerCallQueue ? 1 : ASSERTIONS_MAX_CONCURRENCY;
+	await async.forEachOfLimit(asserts, concurrency, async ({ assertion, assertResult, index }) => {
 		if (assertion.type.startsWith("select-") || assertion.type === "max-score") return;
 		const result = await runAssertion({
 			prompt,
@@ -3865,7 +4658,8 @@ var CIProgressReporter = class {
 	}
 	updateTotalTests(newTotal) {
 		this.totalTests = Math.max(newTotal, 1);
-		this.highestPercentageSeen = Math.floor(this.completedTests / this.totalTests * 100);
+		const percentage = Math.floor(this.completedTests / this.totalTests * 100);
+		this.highestPercentageSeen = percentage;
 	}
 	finish() {
 		if (this.intervalId) {
@@ -4038,6 +4832,10 @@ function getDefaultOtelConfig() {
 }
 //#endregion
 //#region src/tracing/localSpanExporter.ts
+const MISSING_TRACE_RETRY_DELAY_MS = 50;
+function delay(ms) {
+	return new Promise((resolve) => setTimeout(resolve, ms));
+}
 /**
 * A span exporter that writes spans to the local TraceStore (SQLite).
 * This allows OTEL spans to be stored locally for analysis in the promptfoo UI.
@@ -4079,7 +4877,7 @@ var LocalSpanExporter = class {
 		}
 		let firstError;
 		for (const [traceId, spanDataList] of spansByTrace) try {
-			const result = await traceStore.addSpans(traceId, spanDataList, { skipTraceCheck: false });
+			const result = await this.addSpansWithTraceRetry(traceStore, traceId, spanDataList);
 			if (result.stored) logger.debug(`[LocalSpanExporter] Added ${spanDataList.length} spans to trace ${traceId}`);
 			else logger.debug(`[LocalSpanExporter] Skipping ${spanDataList.length} spans for orphan trace ${traceId}: ${result.reason}`);
 		} catch (error) {
@@ -4091,6 +4889,16 @@ var LocalSpanExporter = class {
 		}
 		return firstError;
 	}
+	async addSpansWithTraceRetry(traceStore, traceId, spans) {
+		const options = {
+			skipTraceCheck: false,
+			warnIfMissingTrace: false
+		};
+		const result = await traceStore.addSpans(traceId, spans, options);
+		if (result.stored) return result;
+		await delay(MISSING_TRACE_RETRY_DELAY_MS);
+		return traceStore.addSpans(traceId, spans, options);
+	}
 	/**
 	* Convert an OTEL ReadableSpan to our SpanData format.
 	*/
@@ -4336,6 +5144,15 @@ function backfillNamedScoreWeights(accumulator) {
 }
 //#endregion
 //#region src/evaluator.ts
+const CONVERSATION_VAR_NAME = "_conversation";
+const promptUsesConversationVariableCache = new LRUCache({ max: 1024 });
+function promptUsesConversationVariable(prompt) {
+	const cached = promptUsesConversationVariableCache.get(prompt.raw);
+	if (cached !== void 0) return cached;
+	const { referenced, parsed } = analyzeTemplateReference(prompt.raw, CONVERSATION_VAR_NAME);
+	if (parsed) promptUsesConversationVariableCache.set(prompt.raw, referenced);
+	return referenced;
+}
 /**
 * Manages a single progress bar for the evaluation
 */
@@ -4535,6 +5352,18 @@ function hasProviderGroupedAssertion(assertion) {
 function shouldDeferGradingForTest(test) {
 	return Boolean(test.assert?.some(hasProviderGroupedAssertion));
 }
+function logGroupedGradingStatus({ concurrency, hasEvalStepTimeout, runEvalOptions, shouldGroupGradingByProvider, usesConversationVar }) {
+	if (!runEvalOptions.some(({ test }) => shouldDeferGradingForTest(test))) return;
+	if (shouldGroupGradingByProvider) {
+		logger.info("Grouping model-graded assertions by provider to minimize local-model reload overhead.");
+		return;
+	}
+	if (concurrency !== 1) return;
+	const reasons = [];
+	if (hasEvalStepTimeout) reasons.push("per-eval-step timeout is configured");
+	if (usesConversationVar) reasons.push("conversation variables require per-row ordering");
+	if (reasons.length > 0) logger.info(`Serial grading grouping disabled because ${reasons.join(" and ")}; model-graded judges may reload between rows.`);
+}
 function applyGradingResult(row, checkResult) {
 	if (!checkResult.pass) {
 		row.error = checkResult.reason;
@@ -4549,14 +5378,29 @@ function applyGradingResult(row, checkResult) {
 	if (checkResult.tokensUsed) accumulateAssertionTokenUsage(row.tokenUsage.assertions, checkResult.tokensUsed);
 	row.gradingResult = checkResult;
 }
-function applyGradingError(row, error) {
-	const errorMessage = error instanceof Error ? error.stack ?? error.message : String(error);
-	logger.error("Assertion grading failed during eval", {
-		error: errorMessage,
-		promptIdx: row.promptIdx,
-		testIdx: row.testIdx
-	});
-	row.error = errorMessage;
+const ABORTED_GRADING_PREFIX = "Aborted: ";
+function isAbortShapedError(error) {
+	return error instanceof Error && (error.name === "AbortError" || error.name === "AbortException");
+}
+function applyGradingError(row, error, abortSignal) {
+	const errorAsError = error instanceof Error ? error : void 0;
+	if (Boolean(abortSignal?.aborted) && isAbortShapedError(error)) {
+		const shortMessage = errorAsError?.message ?? String(error);
+		logger.debug("Assertion grading aborted", {
+			error: shortMessage,
+			promptIdx: row.promptIdx,
+			testIdx: row.testIdx
+		});
+		row.error = `${ABORTED_GRADING_PREFIX}${shortMessage}`;
+	} else {
+		const fullMessage = errorAsError ? errorAsError.stack ?? errorAsError.message : String(error);
+		logger.error("Assertion grading failed during eval", {
+			error: fullMessage,
+			promptIdx: row.promptIdx,
+			testIdx: row.testIdx
+		});
+		row.error = fullMessage;
+	}
 	row.failureReason = ResultFailureReason.ERROR;
 	row.success = false;
 	row.score = 0;
@@ -4588,7 +5432,7 @@ function createRunEvalState({ provider, prompt, test }) {
 	};
 }
 function attachConversationVar({ conversations, conversationKey, prompt, test, vars }) {
-	const usesConversation = prompt.raw.includes("_conversation");
+	const usesConversation = promptUsesConversationVariable(prompt);
 	if (!getEnvBool("PROMPTFOO_DISABLE_CONVERSATION_VAR") && !test.options?.disableConversationVar && usesConversation) vars._conversation = conversations?.[conversationKey] || [];
 }
 function createRunEvalSetup({ provider, prompt, promptConfig, vars }) {
@@ -4835,7 +5679,7 @@ async function gradeRunEvalResponse({ abortSignal, deferGrading, evalId, latency
 			assertScoringFunction: test.assertScoringFunction,
 			traceId
 		}).then((checkResult) => applyGradingResult(ret, checkResult))).catch((error) => {
-			applyGradingError(ret, error);
+			applyGradingError(ret, error, abortSignal);
 		});
 		deferredGradingPromises.set(ret, gradingPromise);
 		return;
@@ -5382,7 +6226,7 @@ async function resolveDefaultTestProvider(defaultTest, testCase) {
 	const defaultProvider = defaultTest.provider;
 	if (isApiProvider(defaultProvider)) return defaultProvider;
 	if (typeof defaultProvider === "object" && defaultProvider.id) {
-		const { loadApiProvider } = await import("./providers-iUt5fbAN.js");
+		const { loadApiProvider } = await import("./providers-BYAn82cf.js");
 		return loadApiProvider(typeof defaultProvider.id === "function" ? defaultProvider.id() : defaultProvider.id, { options: defaultProvider });
 	}
 	return defaultProvider;
@@ -5542,7 +6386,7 @@ function buildRepeatCacheContextByTestIdx(runEvalOptions) {
 async function filterCompletedResumeSteps(runEvalOptions, evalRecord) {
 	if (!state.resume || !evalRecord.persisted) return;
 	try {
-		const { default: EvalResult } = await import("./evalResult-D8MT9p0s.js");
+		const { default: EvalResult } = await import("./evalResult-DqzsS6_W.js");
 		const completedPairs = await EvalResult.getCompletedIndexPairs(evalRecord.id, { excludeErrors: state.retryMode });
 		const originalCount = runEvalOptions.length;
 		for (let i = runEvalOptions.length - 1; i >= 0; i--) {
@@ -5556,14 +6400,14 @@ async function filterCompletedResumeSteps(runEvalOptions, evalRecord) {
 	}
 }
 function adjustConcurrencyForSerialFeatures({ concurrency, prompts, tests }) {
-	const usesConversationVar = prompts.some((p) => p.raw.includes("_conversation"));
+	const usesConversationVar = prompts.some(promptUsesConversationVariable);
 	if (concurrency <= 1) return {
 		concurrency,
 		usesConversationVar
 	};
 	const usesStoreOutputAs = tests.some((t) => t.options?.storeOutputAs);
 	if (usesConversationVar) {
-		logger.info(`Setting concurrency to 1 because the ${chalk.cyan("_conversation")} variable is used.`);
+		logger.info(`Setting concurrency to 1 because the ${chalk.cyan(CONVERSATION_VAR_NAME)} variable is used.`);
 		return {
 			concurrency: 1,
 			usesConversationVar
@@ -5793,7 +6637,8 @@ var Evaluator = class {
 		};
 		this.conversations = {};
 		this.registers = {};
-		this.fileWriters = (Array.isArray(evalRecord.config.outputPath) ? evalRecord.config.outputPath.filter((p) => p.endsWith(".jsonl")) : evalRecord.config.outputPath?.endsWith(".jsonl") ? [evalRecord.config.outputPath] : []).map((p) => new JsonlFileWriter(p));
+		const jsonlFiles = Array.isArray(evalRecord.config.outputPath) ? evalRecord.config.outputPath.filter((p) => p.endsWith(".jsonl")) : evalRecord.config.outputPath?.endsWith(".jsonl") ? [evalRecord.config.outputPath] : [];
+		this.fileWriters = jsonlFiles.map((p) => new JsonlFileWriter(p));
 		this.rateLimitRegistry = createRateLimitRegistry({ maxConcurrency: options.maxConcurrency || 4 });
 		this.rateLimitRegistry.on("ratelimit:hit", (data) => {
 			logger.debug(`[Scheduler] Rate limit hit for ${data.rateLimitKey}`, {
@@ -5913,6 +6758,25 @@ var Evaluator = class {
 			this.trackCompletedRow(evalStep, row, context);
 			context.numComplete++;
 			const promptEvalCount = reservePromptEvalCount(context, row.promptIdx);
+			if (context.testSuite.extensions?.length) try {
+				const afterEachOut = await runExtensionHook(context.testSuite.extensions, "afterEach", {
+					test: evalStep.test,
+					result: {
+						...row,
+						namedScores: { ...row.namedScores },
+						metadata: { ...row.metadata },
+						response: row.response ? {
+							...row.response,
+							metadata: { ...row.response.metadata }
+						} : row.response
+					}
+				});
+				row.namedScores = filterFiniteScores(afterEachOut.result.namedScores);
+				row.metadata = afterEachOut.result.metadata;
+				if (row.response && afterEachOut.result.response) row.response.metadata = afterEachOut.result.response.metadata;
+			} catch (error) {
+				logger.error(`afterEach extension hook failed, persisting row without hook modifications`, { error });
+			}
 			await this.persistEvalRow(row);
 			if (this.abortIfTargetUnavailable(row, context)) break;
 			const metrics = context.prompts[row.promptIdx].metrics;
@@ -5924,10 +6788,6 @@ var Evaluator = class {
 				promptEvalCount,
 				row
 			});
-			await runExtensionHook(context.testSuite.extensions, "afterEach", {
-				test: evalStep.test,
-				result: row
-			});
 			context.options.progressCallback?.(context.numComplete, context.runEvalOptionsLength, index, evalStep, metrics);
 		}
 	}
@@ -6001,9 +6861,8 @@ var Evaluator = class {
 		context.options.progressCallback?.(context.numComplete, context.runEvalOptionsLength, index, evalStep, metrics || createTimeoutMetrics(timeoutMs));
 	}
 	async executeEvalSteps({ checkAbort, ciProgressReporter, combinedAbortSignal, concurrentRunEvalOptions, evalStepIndexMap, globalTimeout, groupedRunEvalOptions, isEvalTimedOut, isWebUI, maxEvalTimeMs, processingContext, processedIndices, progressBarManager, prompts, serialRunEvalOptions, shouldGroupGradingByProvider }) {
-		let flushGroupedRows;
 		try {
-			if (shouldGroupGradingByProvider) flushGroupedRows = await this.runGroupedEvalSteps({
+			if (shouldGroupGradingByProvider) await this.runGroupedEvalSteps({
 				checkAbort,
 				evalStepIndexMap,
 				groupedRunEvalOptions,
@@ -6035,7 +6894,6 @@ var Evaluator = class {
 				cleanupProgressAfterError(progressBarManager, ciProgressReporter, err);
 				throw err;
 			}
-			await flushGroupedRows?.();
 			if (isEvalTimedOut()) logger.warn(`Evaluation stopped after reaching max duration (${maxEvalTimeMs}ms)`);
 			else if (!processingContext.targetUnavailable) return this.saveInterruptedEval({
 				ciProgressReporter,
@@ -6084,7 +6942,15 @@ var Evaluator = class {
 				})) break;
 			}
 		} catch (error) {
-			await flushGroupedRows();
+			const pendingRowCount = groupedRows.reduce((sum, entry) => sum + entry.rows.length, 0);
+			try {
+				await flushGroupedRows();
+			} catch (flushError) {
+				logger.warn("Failed to flush grouped rows after error; target outputs may be lost", {
+					error: flushError instanceof Error ? flushError.message : String(flushError),
+					pendingRowCount
+				});
+			}
 			throw error;
 		}
 		await flushGroupedRows();
@@ -6520,6 +7386,13 @@ var Evaluator = class {
 		if (!this.options.silent) {
 			if (serialRunEvalOptions.length > 0) logger.info(`Running ${serialRunEvalOptions.length} test cases serially...`);
 			if (concurrentRunEvalOptions.length > 0) logger.info(`Running ${concurrentRunEvalOptions.length} test cases (up to ${concurrency} at a time)...`);
+			logGroupedGradingStatus({
+				concurrency,
+				hasEvalStepTimeout,
+				runEvalOptions,
+				shouldGroupGradingByProvider,
+				usesConversationVar
+			});
 		}
 		if (this.options.showProgressBar && progressBarManager) {
 			await progressBarManager.initialize(runEvalOptions, concurrency, 0);
@@ -6622,4 +7495,4 @@ function evaluate(testSuite, evalRecord, options) {
 //#endregion
 export { isAllowedPrompt as a, assertions_default as c, generateVarCombinations as i, readAssertions as l, evaluate as n, accumulateNamedMetric as o, formatVarsForDisplay as r, doesPromptRefMatch as s, ProgressBarManager as t, runAssertions as u };
 
-//# sourceMappingURL=evaluator-IvuDYSvQ.js.map
+//# sourceMappingURL=evaluator-DRoiYB2q.js.map