promptfoo 0.121.5 → 0.121.8

package/dist/src/{evaluator-D-UIbbYq.js → evaluator-n_dEb00o.js}

@@ -1,30 +1,39 @@
 #!/usr/bin/env node
 import { A as getMaxEvalTimeMs, D as getEnvInt, N as state, O as getEnvString, S as summarizeEvaluateResultForLogging, T as getEnvBool, _ as extractJsonObjects, c as setLogCallback, g as extractFirstJsonObject, j as isCI, k as getEvalTimeoutMs, r as globalLogCallback, s as logger, v as getAjv, x as safeJsonStringify } from "./logger-BbY6ypFL.js";
-import { P as FILE_METADATA_KEY, l as sleep, r as fetchWithRetries, v as isPromptfooSampleTarget, x as parseChatPrompt } from "./fetch-B6ch2nU2.js";
-import { n as VERSION } from "./version-CbpiUINz.js";
+import { L as FILE_METADATA_KEY, f as sleep, r as fetchWithRetries, w as parseChatPrompt, x as isPromptfooSampleTarget } from "./fetch-Cpf1U1nO.js";
+import { n as VERSION } from "./version-F0YDgb7J.js";
 import { t as invariant } from "./invariant-B2Rf6avk.js";
-import { r as telemetry } from "./telemetry-DWdGHvEf.js";
-import { at as MULTI_INPUT_VAR, d as isGradingResult, nt as LLAMA_GUARD_REPLICATE_PROVIDER, p as isApiProvider, s as ResultFailureReason } from "./types-BVH9hjgW.js";
-import { a as getNunjucksEngine, r as extractVariablesFromTemplate } from "./render-eui5p5mL.js";
-import { t as providerRegistry } from "./providerRegistry-PMsleEzs.js";
-import { c as promptYesNo, m as shouldGenerateRemote } from "./server-ByiF3qlg.js";
-import { $ as createRateLimitRegistry, F as checkExfilTracking, J as runExtensionHook, K as collectFileMetadata, Q as TokenUsageTracker, U as isBasicRefusal, V as getSessionId, X as loadFromPackage, Y as isPackagePath, Z as redteamProviderManager, _ as VertexChatProvider, et as createProviderRateLimitOptions, n as loadApiProvider, ot as throwIfTargetPromptExceedsMaxChars, q as renderPrompt, v as GoogleLiveProvider, y as AIStudioChatProvider } from "./providers-DT-GtF2t.js";
-import { n as isNonTransientHttpStatus } from "./errors-DFHe4L-n.js";
-import { c as withCacheNamespace, o as getCache } from "./cache-BI5BY7ey.js";
-import { i as isJavascriptFile } from "./fileExtensions-DysCsxNG.js";
-import { r as runPython } from "./pythonUtils-DNqbnRdx.js";
-import { L as loadFunction, N as maybeLoadToolsFromExternalFile, R as parseFileUrl, S as isProviderAllowed, b as isGoogleProvider, x as isOpenAiProvider, y as isAnthropicProvider } from "./util-BQOCAHQC.js";
-import { a as createEmptyTokenUsage, i as createEmptyAssertions, n as accumulateResponseTokenUsage, o as normalizeTokenUsage, r as accumulateTokenUsage, t as accumulateAssertionTokenUsage } from "./tokenUsageUtils-2wIvAhB3.js";
-import { t as extractAndStoreBinaryData } from "./extractor-DxyiFhPk.js";
-import { t as OpenAiChatCompletionProvider } from "./chat-BLOdH60v.js";
-import { h as validateFunctionCall } from "./transform-B-b6Cq-q.js";
-import { l as validateFunctionCall$1 } from "./util-C-kmRosx.js";
-import { n as transform, r as getProcessShim, t as TransformInputType } from "./transform-DzCF-wqV.js";
-import { n as getTraceStore } from "./store-CPh25336.js";
-import { n as runRuby } from "./rubyUtils-CIQFnVz4.js";
-import { $ as SELECT_BEST_PROMPT, Ct as DEFAULT_ANTHROPIC_MODEL, Dt as withProviderCallExecutionContext, Et as getGradingProvider, G as matchesLlmRubric, H as matchesClosedQa, J as doRemoteGrading, K as matchesPiScore, Q as DEFAULT_WEB_SEARCH_PROMPT, St as getDefaultProviders, Tt as getAndCheckProvider, U as matchesFactuality, W as matchesGEval, _t as tryParse, at as CONTEXT_RECALL_ATTRIBUTED_TOKEN, bt as loadFromJavaScriptFile, ct as CONTEXT_RELEVANCE_BAD, dt as cosineSimilarity, et as SUGGEST_PROMPTS_SYSTEM_MESSAGE, ft as dotProduct, gt as splitIntoSentences, ht as normalizeMatcherTokenUsage, it as CONTEXT_RECALL, lt as loadRubricPrompt, mt as fail, n as getGraderById, nt as CONTEXT_FAITHFULNESS_LONGFORM, ot as CONTEXT_RECALL_NOT_ATTRIBUTED_TOKEN, pt as euclideanDistance, q as matchesTrajectoryGoalSuccess, rt as CONTEXT_FAITHFULNESS_NLI_STATEMENTS, st as CONTEXT_RELEVANCE, tt as ANSWER_RELEVANCY_GENERATE, ut as renderLlmRubricPrompt, vt as coerceString, wt as callProviderWithContext, xt as processFileReference, yt as getFinalTest } from "./graders-CgPn32yp.js";
-import { i as generateIdFromPrompt } from "./utils-kt7lv30R.js";
-import { a as getActualPromptWithFallback, r as updateSignalFile } from "./signal-DTtUuU3l.js";
+import { r as telemetry } from "./telemetry-DOE567Wj.js";
+import { at as MULTI_INPUT_VAR, d as isGradingResult, nt as LLAMA_GUARD_REPLICATE_PROVIDER, p as isApiProvider, s as ResultFailureReason } from "./types-BFevViUY.js";
+import { i as isJavascriptFile } from "./fileExtensions-D4GCJ67J.js";
+import { i as extractVariablesFromTemplate, o as getNunjucksEngine, r as analyzeTemplateReference } from "./render-CSP99NLm.js";
+import { t as providerRegistry } from "./providerRegistry-ReCd0sFa.js";
+import { l as shouldGenerateRemote } from "./remoteGeneration-D6UjE2JT.js";
+import { c as promptYesNo } from "./server-DLYjSFm2.js";
+import { n as isNonTransientHttpStatus } from "./errors-9PcUL8BC.js";
+import { l as withCacheNamespace, o as getCache } from "./cache-BR77mdIR.js";
+import { r as runPython } from "./pythonUtils-CgYxeSmO.js";
+import { B as parseFileUrl, C as isOpenAiProvider, F as maybeLoadToolsFromExternalFile, S as isGoogleProvider, w as isProviderAllowed, x as isAnthropicProvider, z as loadFunction } from "./util-jZRrXe1P.js";
+import { t as OpenAiChatCompletionProvider } from "./chat-DTdf-J5Q.js";
+import { h as validateFunctionCall } from "./transform-DtTfiGoh.js";
+import { l as validateFunctionCall$1 } from "./util-DeY58psG.js";
+import { _ as VertexChatProvider, n as loadApiProvider, v as GoogleLiveProvider, y as AIStudioChatProvider } from "./providers-B7TyByfj.js";
+import { a as createEmptyTokenUsage, i as createEmptyAssertions, n as accumulateResponseTokenUsage, o as normalizeTokenUsage, r as accumulateTokenUsage, t as accumulateAssertionTokenUsage } from "./tokenUsageUtils-CDet74yk.js";
+import { t as getProcessShim } from "./processShim-BBxt7LKO.js";
+import { n as loadFromPackage, t as isPackagePath } from "./packageParser--MWTSrPW.js";
+import { n as runRuby } from "./rubyUtils-CXlFM2rR.js";
+import { c as isBasicRefusal, o as getSessionId } from "./util-ETfU_sS9.js";
+import { $ as DEFAULT_WEB_SEARCH_PROMPT, Ct as getDefaultProviders, Dt as getGradingProvider, Et as getAndCheckProvider, G as matchesGEval, H as isGraderFailure, J as matchesTrajectoryGoalSuccess, K as matchesLlmRubric, Ot as getProviderCallExecutionContext, St as processFileReference, Tt as callProviderWithContext, U as matchesClosedQa, W as matchesFactuality, Y as doRemoteGrading, _t as splitIntoSentences, at as CONTEXT_RECALL, bt as getFinalTest, ct as CONTEXT_RELEVANCE, dt as renderLlmRubricPrompt, et as SELECT_BEST_PROMPT, ft as cosineSimilarity, gt as normalizeMatcherTokenUsage, ht as fail, it as CONTEXT_FAITHFULNESS_NLI_STATEMENTS, kt as withProviderCallExecutionContext, lt as CONTEXT_RELEVANCE_BAD, mt as euclideanDistance, n as getGraderById, nt as ANSWER_RELEVANCY_GENERATE, ot as CONTEXT_RECALL_ATTRIBUTED_TOKEN, pt as dotProduct, q as matchesPiScore, rt as CONTEXT_FAITHFULNESS_LONGFORM, st as CONTEXT_RECALL_NOT_ATTRIBUTED_TOKEN, tt as SUGGEST_PROMPTS_SYSTEM_MESSAGE, ut as loadRubricPrompt, vt as tryParse, wt as DEFAULT_ANTHROPIC_MODEL, xt as loadFromJavaScriptFile, yt as coerceString } from "./graders-BQt1BaQe.js";
+import { f as redteamProviderManager, g as createProviderRateLimitOptions, h as createRateLimitRegistry, m as TokenUsageTracker } from "./shared-DNvim54U.js";
+import { i as generateIdFromPrompt } from "./utils-BFOh20Gb.js";
+import { a as getTransformLabel, i as getTransformErrorMessage, o as transform, r as TransformInputType } from "./transform-BnSTnFlp.js";
+import { n as getTraceStore } from "./store-DKd5592Q.js";
+import { a as getActualPromptWithFallback, r as updateSignalFile } from "./signal-CSurUUyV.js";
+import { t as extractAndStoreBinaryData } from "./extractor-BVkZtk4R.js";
+import { i as throwIfTargetPromptExceedsMaxChars } from "./promptLength-4X-Wd8PG.js";
+import { n as checkExfilTracking } from "./indirectWebPwn-BMTXXznx.js";
+import { n as getFirstStringAttribute, r as getToolNameFromAttributes, t as TOOL_ARGUMENT_ATTRIBUTE_KEYS } from "./toolAttributes-COVgDrBG.js";
+import { i as filterFiniteScores, n as renderPrompt, r as runExtensionHook, t as collectFileMetadata } from "./evaluatorHelpers-D1_kwvyp.js";
 import { AsyncResource } from "node:async_hooks";
 import chalk from "chalk";
 import fs, { createWriteStream } from "fs";
@@ -38,11 +47,12 @@ import readline from "readline";
 import { globSync } from "glob";
 import { XMLParser } from "fast-xml-parser";
 import async from "async";
-import cliProgress from "cli-progress";
 import { parse as parse$1 } from "parse5";
 import { distance } from "fastest-levenshtein";
+import cliProgress from "cli-progress";
 import * as rouge from "js-rouge";
 import { isDeepStrictEqual } from "node:util";
+import { LRUCache } from "lru-cache";
 import { ExportResultCode, W3CTraceContextPropagator } from "@opentelemetry/core";
 import { OTLPTraceExporter } from "@opentelemetry/exporter-trace-otlp-http";
 import { resourceFromAttributes } from "@opentelemetry/resources";
@@ -405,17 +415,20 @@ async function resolveContext(assertion, test, output, prompt, fallbackContext,
 			contextValue = test.vars.context;
 		}
 	} else if (fallbackContext) contextValue = fallbackContext;
-	if (assertion.contextTransform) try {
-		const outputForTransform = providerResponse?.providerTransformedOutput ?? output;
-		const transformed = await transform(assertion.contextTransform, outputForTransform, {
-			vars: test.vars,
-			prompt: { label: prompt },
-			...providerResponse && providerResponse.metadata && { metadata: providerResponse.metadata }
-		});
-		invariant(typeof transformed === "string" || Array.isArray(transformed) && transformed.every((item) => typeof item === "string"), `contextTransform must return a string or array of strings. Got ${typeof transformed}. Check your transform expression: ${assertion.contextTransform}`);
-		contextValue = transformed;
-	} catch (error) {
-		throw new Error(`Failed to transform context using expression '${assertion.contextTransform}': ${error instanceof Error ? error.message : String(error)}`);
+	if (assertion.contextTransform) {
+		const getLabel = () => getTransformLabel(assertion.contextTransform);
+		try {
+			const outputForTransform = providerResponse?.providerTransformedOutput ?? output;
+			const transformed = await transform(assertion.contextTransform, outputForTransform, {
+				vars: test.vars,
+				prompt: { label: prompt },
+				...providerResponse && providerResponse.metadata && { metadata: providerResponse.metadata }
+			});
+			invariant(typeof transformed === "string" || Array.isArray(transformed) && transformed.every((item) => typeof item === "string"), () => `contextTransform must return a string or array of strings. Got ${typeof transformed}. Check your transform expression: ${getLabel()}`);
+			contextValue = transformed;
+		} catch (error) {
+			throw new Error(`Failed to transform context using expression '${getLabel()}': ${getTransformErrorMessage(error)}`);
+		}
 	}
 	invariant(typeof contextValue === "string" && contextValue.length > 0 || Array.isArray(contextValue) && contextValue.length > 0 && contextValue.every((item) => typeof item === "string" && item.length > 0), "Context is required for context-based assertions. Provide either a \"context\" variable (string or array of strings) in your test case or use \"contextTransform\" to extract context from the provider response.");
 	return contextValue;
@@ -758,7 +771,7 @@ async function startOtlpReceiverIfNeeded(testSuite) {
 		telemetry.record("feature_used", { feature: "tracing" });
 		try {
 			logger.debug("[EvaluatorTracing] Tracing configuration detected, starting OTLP receiver");
-			const { startOTLPReceiver } = await import("./otlpReceiver-UYMQx3sy.js");
+			const { startOTLPReceiver } = await import("./otlpReceiver--gTpSagc.js");
 			const port = testSuite.tracing.otlp.http.port || 4318;
 			const host = testSuite.tracing.otlp.http.host || "127.0.0.1";
 			const acceptFormats = normalizeOtlpAcceptFormats(testSuite.tracing.otlp.http.acceptFormats);
@@ -782,7 +795,7 @@ async function startOtlpReceiverIfNeeded(testSuite) {
 async function stopOtlpReceiverIfNeeded() {
 	if (otlpReceiverStarted) try {
 		logger.debug("[EvaluatorTracing] Stopping OTLP receiver");
-		const { stopOTLPReceiver } = await import("./otlpReceiver-UYMQx3sy.js");
+		const { stopOTLPReceiver } = await import("./otlpReceiver--gTpSagc.js");
 		await stopOTLPReceiver();
 		otlpReceiverStarted = false;
 		logger.info("[EvaluatorTracing] OTLP receiver stopped successfully");
@@ -817,7 +830,7 @@ async function generateTraceContextIfNeeded(test, evaluateOptions, testIdx, prom
 	}
 	if (!tracingEnabled) return null;
 	logger.debug("[EvaluatorTracing] Importing trace store");
-	const { getTraceStore } = await import("./store-BELqNwvz.js");
+	const { getTraceStore } = await import("./store-IbiRIF3k.js");
 	const traceStore = getTraceStore();
 	const traceId = generateTraceId();
 	const spanId = generateSpanId();
@@ -1449,27 +1462,67 @@ const handleIsValidFunctionCall = ({ assertion, output, provider, test }) => {
 };
 //#endregion
 //#region src/assertions/geval.ts
-const handleGEval = async ({ assertion, renderedValue, prompt, outputString, test, providerCallContext }) => {
-	invariant(typeof renderedValue === "string" || Array.isArray(renderedValue), "G-Eval assertion type must have a string or array of strings value");
+const handleGEval = async ({ assertion, inverse, renderedValue, prompt, outputString, test, providerCallContext }) => {
+	invariant(typeof renderedValue === "string" || Array.isArray(renderedValue) && renderedValue.every((value) => typeof value === "string"), "G-Eval assertion type must have a string or array of strings value");
 	const threshold = assertion.threshold ?? .7;
 	if (Array.isArray(renderedValue)) {
-		const scores = [];
-		const reasons = [];
-		for (const value of renderedValue) {
+		if (renderedValue.length === 0) return {
+			assertion,
+			pass: false,
+			score: 0,
+			reason: "G-Eval assertion requires at least one criterion string in the value array."
+		};
+		const responses = [];
+		let failure;
+		for (const [index, value] of renderedValue.entries()) {
 			const resp = await matchesGEval(value, prompt || "", outputString, threshold, test.options, providerCallContext);
-			scores.push(resp.score);
-			reasons.push(resp.reason);
+			responses.push(resp);
+			if (isGraderFailure(resp)) {
+				failure = {
+					index,
+					resp
+				};
+				break;
+			}
+		}
+		const tokensUsed = createEmptyTokenUsage();
+		for (const r of responses) accumulateTokenUsage(tokensUsed, r.tokensUsed);
+		if (failure) {
+			const criterion = renderedValue[failure.index];
+			return {
+				assertion,
+				pass: false,
+				score: 0,
+				reason: `G-Eval criterion ${failure.index + 1}/${renderedValue.length} (${JSON.stringify(criterion)}) failed: ${failure.resp.reason}`,
+				tokensUsed,
+				metadata: failure.resp.metadata
+			};
 		}
-		const scoresSum = scores.reduce((a, b) => a + b, 0);
+		const averageScore = responses.reduce((acc, r) => acc + r.score, 0) / responses.length;
+		const combinedReason = responses.map((r) => r.reason).join("\n\n");
 		return {
 			assertion,
-			pass: scoresSum / scores.length >= threshold,
-			score: scoresSum / scores.length,
-			reason: reasons.join("\n\n")
+			pass: averageScore >= threshold !== inverse,
+			score: inverse ? 1 - averageScore : averageScore,
+			reason: combinedReason,
+			tokensUsed
 		};
-	} else return {
+	}
+	const resp = await matchesGEval(renderedValue, prompt || "", outputString, threshold, test.options, providerCallContext);
+	if (isGraderFailure(resp)) return {
 		assertion,
-		...await matchesGEval(renderedValue, prompt || "", outputString, threshold, test.options, providerCallContext)
+		pass: false,
+		score: 0,
+		reason: resp.reason,
+		tokensUsed: resp.tokensUsed,
+		metadata: resp.metadata
+	};
+	const passed = resp.score >= threshold !== inverse;
+	return {
+		assertion,
+		...resp,
+		pass: passed,
+		score: inverse ? 1 - resp.score : resp.score
 	};
 };
 //#endregion
@@ -2430,45 +2483,6 @@ function matchesPattern(spanName, pattern) {
 }
 //#endregion
 //#region src/assertions/trajectoryUtils.ts
-const TOOL_ATTRIBUTE_KEYS = [
-	"tool.name",
-	"tool_name",
-	"tool",
-	"function.name",
-	"function_name",
-	"gen_ai.tool.name",
-	"codex.mcp.tool",
-	"agent.tool",
-	"agent.tool_name",
-	"agent.toolName"
-];
-const TOOL_ARGUMENT_ATTRIBUTE_KEYS = [
-	"tool.arguments",
-	"tool.args",
-	"tool.input",
-	"tool_arguments",
-	"tool_args",
-	"tool_input",
-	"function.arguments",
-	"function.args",
-	"function.input",
-	"function_arguments",
-	"function_args",
-	"gen_ai.tool.arguments",
-	"gen_ai.tool.args",
-	"gen_ai.tool.input",
-	"gen_ai.tool.call.arguments",
-	"gen_ai.tool.call.args",
-	"agent.tool.arguments",
-	"agent.tool.args",
-	"agent.tool.input",
-	"codex.mcp.arguments",
-	"codex.mcp.args",
-	"codex.mcp.input",
-	"arguments",
-	"args",
-	"input"
-];
 const COMMAND_ATTRIBUTE_KEYS = [
 	"codex.command",
 	"command",
@@ -2481,16 +2495,15 @@ const SEARCH_ATTRIBUTE_KEYS = [
 	"search_query"
 ];
 const GENERIC_QUERY_ATTRIBUTE_KEYS = ["query"];
+const COMMAND_TOOL_NAMES = new Set([
+	"exec_command",
+	"local_shell",
+	"shell"
+]);
 const SEARCH_SPAN_NAME_PATTERN = /(^|[\s._:/-])(search|find|lookup|retriev(?:e|al))($|[\s._:/-])/i;
 const MAX_JUDGE_SUMMARY_STEPS = 24;
 const JUDGE_SUMMARY_HEAD_STEPS = 12;
 const JUDGE_SUMMARY_TAIL_STEPS = 12;
-function getStringAttribute(attributes, keys) {
-	for (const key of keys) {
-		const value = attributes[key];
-		if (typeof value === "string" && value.trim()) return value.trim();
-	}
-}
 function normalizeStructuredAttribute(value) {
 	if (value === void 0 || value === null) return;
 	if (typeof value === "string") {
@@ -2522,9 +2535,12 @@ function getTrajectoryStepStatus(step) {
 function getCommandExecutable(command) {
 	return command.trim().split(/\s+/)[0] || void 0;
 }
+function isCommandToolName(toolName) {
+	return !!toolName && COMMAND_TOOL_NAMES.has(toolName.trim().toLowerCase());
+}
 function extractToolName(span) {
 	const attributes = span.attributes || {};
-	const directMatch = getStringAttribute(attributes, TOOL_ATTRIBUTE_KEYS);
+	const directMatch = getToolNameFromAttributes(attributes);
 	if (directMatch) return directMatch;
 	for (const [key, value] of Object.entries(attributes)) {
 		if (typeof value !== "string" || !value.trim()) continue;
@@ -2549,21 +2565,31 @@ function extractToolArgs(span) {
 		if (value !== void 0) return value;
 	}
 }
-function extractCommand(span) {
+function extractCommand(span, toolName = extractToolName(span), getToolArgs = () => extractToolArgs(span)) {
 	const attributes = span.attributes || {};
-	const directMatch = getStringAttribute(attributes, COMMAND_ATTRIBUTE_KEYS);
+	const directMatch = getFirstStringAttribute(attributes, COMMAND_ATTRIBUTE_KEYS);
 	if (directMatch) return directMatch;
 	for (const [key, value] of Object.entries(attributes)) {
 		if (typeof value !== "string" || !value.trim()) continue;
 		if (/command/i.test(key) && !/output|result/i.test(key)) return value.trim();
 	}
+	const toolArgs = getToolArgs();
+	if (isCommandToolName(toolName) && toolArgs && typeof toolArgs === "object") {
+		const args = toolArgs;
+		const command = args.cmd ?? args.command;
+		if (typeof command === "string" && command.trim()) return command.trim();
+		if (Array.isArray(command)) {
+			const joined = command.map((part) => String(part).trim()).filter(Boolean).join(" ");
+			if (joined) return joined;
+		}
+	}
 	if (span.name.startsWith("exec ")) return span.name.slice(5).trim();
 }
 function extractSearchQuery(span) {
 	const attributes = span.attributes || {};
-	const directMatch = getStringAttribute(attributes, SEARCH_ATTRIBUTE_KEYS);
+	const directMatch = getFirstStringAttribute(attributes, SEARCH_ATTRIBUTE_KEYS);
 	if (directMatch) return directMatch;
-	const genericQuery = getStringAttribute(attributes, GENERIC_QUERY_ATTRIBUTE_KEYS);
+	const genericQuery = getFirstStringAttribute(attributes, GENERIC_QUERY_ATTRIBUTE_KEYS);
 	if (genericQuery && isSearchLikeSpan(span)) return genericQuery;
 	if (span.name.startsWith("search ")) return span.name.slice(7).replace(/^"|"$/g, "").trim();
 }
@@ -2587,17 +2613,34 @@ function extractTrajectorySteps(trace) {
 		return left.index - right.index;
 	}).map(({ span }) => {
 		const toolName = extractToolName(span);
-		const command = extractCommand(span);
+		let toolArgs;
+		let hasExtractedToolArgs = false;
+		const getToolArgs = () => {
+			if (!hasExtractedToolArgs) {
+				toolArgs = extractToolArgs(span);
+				hasExtractedToolArgs = true;
+			}
+			return toolArgs;
+		};
+		const command = extractCommand(span, toolName, getToolArgs);
 		const searchQuery = extractSearchQuery(span);
 		let type = "span";
 		let name = span.name;
 		const aliases = new Set([span.name]);
 		let args;
-		if (toolName) {
+		if (command && isCommandToolName(toolName)) {
+			type = "command";
+			name = command;
+			aliases.add(command);
+			args = getToolArgs();
+			if (toolName) aliases.add(toolName);
+			const executable = getCommandExecutable(command);
+			if (executable) aliases.add(executable);
+		} else if (toolName) {
 			type = "tool";
 			name = toolName;
 			aliases.add(toolName);
-			args = extractToolArgs(span);
+			args = getToolArgs();
 		} else if (command) {
 			type = "command";
 			name = command;
@@ -4161,7 +4204,7 @@ async function loadTraceData(traceId) {
 	let stableObservations = 0;
 	let latestTrace = null;
 	for (let attempt = 0; attempt < maxAttempts; attempt++) {
-		latestTrace = await traceStore.getTrace(traceId);
+		latestTrace = await traceStore.getTrace(traceId, { sanitizeAttributes: false });
 		const spanCount = latestTrace?.spans?.length ?? 0;
 		if (spanCount > 0) {
 			stableObservations = spanCount === lastSpanCount ? stableObservations + 1 : 1;
@@ -4214,7 +4257,7 @@ const ASSERTION_HANDLERS = {
 	"llm-rubric": handleLlmRubric,
 	meteor: async (params) => {
 		try {
-			const { handleMeteorAssertion } = await import("./meteor-CeGo0Lu2.js");
+			const { handleMeteorAssertion } = await import("./meteor--TZYICTI.js");
 			return handleMeteorAssertion(params);
 		} catch (error) {
 			if (error instanceof Error && (error.message.includes("Cannot find module") || error.message.includes("natural\" package is required"))) return {
@@ -4350,7 +4393,7 @@ async function runAssertion({ prompt, provider, assertion, test, vars, latencyMs
 			};
 		}
 		else if (filePath.endsWith(".rb")) try {
-			const { runRuby } = await import("./rubyUtils-4hjGxvju.js");
+			const { runRuby } = await import("./rubyUtils-BYVlQ94c.js");
 			valueFromScript = await runRuby(filePath, functionName || "get_assert", [output, context]);
 			logger.debug(`Ruby script ${filePath} output: ${valueFromScript}`);
 		} catch (error) {
@@ -4467,7 +4510,8 @@ async function runAssertions({ assertScoringFunction, latencyMs, prompt, provide
 		logger.debug(`Failed to preload trace data for assertions: ${error}`);
 		preloadedTraceData = null;
 	}
-	await async.forEachOfLimit(asserts, ASSERTIONS_MAX_CONCURRENCY, async ({ assertion, assertResult, index }) => {
+	const concurrency = getProviderCallExecutionContext()?.providerCallQueue ? 1 : ASSERTIONS_MAX_CONCURRENCY;
+	await async.forEachOfLimit(asserts, concurrency, async ({ assertion, assertResult, index }) => {
 		if (assertion.type.startsWith("select-") || assertion.type === "max-score") return;
 		const result = await runAssertion({
 			prompt,
@@ -4614,7 +4658,8 @@ var CIProgressReporter = class {
 	}
 	updateTotalTests(newTotal) {
 		this.totalTests = Math.max(newTotal, 1);
-		this.highestPercentageSeen = Math.floor(this.completedTests / this.totalTests * 100);
+		const percentage = Math.floor(this.completedTests / this.totalTests * 100);
+		this.highestPercentageSeen = percentage;
 	}
 	finish() {
 		if (this.intervalId) {
@@ -4787,6 +4832,10 @@ function getDefaultOtelConfig() {
 }
 //#endregion
 //#region src/tracing/localSpanExporter.ts
+const MISSING_TRACE_RETRY_DELAY_MS = 50;
+function delay(ms) {
+	return new Promise((resolve) => setTimeout(resolve, ms));
+}
 /**
 * A span exporter that writes spans to the local TraceStore (SQLite).
 * This allows OTEL spans to be stored locally for analysis in the promptfoo UI.
@@ -4828,7 +4877,7 @@ var LocalSpanExporter = class {
 		}
 		let firstError;
 		for (const [traceId, spanDataList] of spansByTrace) try {
-			const result = await traceStore.addSpans(traceId, spanDataList, { skipTraceCheck: false });
+			const result = await this.addSpansWithTraceRetry(traceStore, traceId, spanDataList);
 			if (result.stored) logger.debug(`[LocalSpanExporter] Added ${spanDataList.length} spans to trace ${traceId}`);
 			else logger.debug(`[LocalSpanExporter] Skipping ${spanDataList.length} spans for orphan trace ${traceId}: ${result.reason}`);
 		} catch (error) {
@@ -4840,6 +4889,16 @@ var LocalSpanExporter = class {
 		}
 		return firstError;
 	}
+	async addSpansWithTraceRetry(traceStore, traceId, spans) {
+		const options = {
+			skipTraceCheck: false,
+			warnIfMissingTrace: false
+		};
+		const result = await traceStore.addSpans(traceId, spans, options);
+		if (result.stored) return result;
+		await delay(MISSING_TRACE_RETRY_DELAY_MS);
+		return traceStore.addSpans(traceId, spans, options);
+	}
 	/**
 	* Convert an OTEL ReadableSpan to our SpanData format.
 	*/
@@ -5085,6 +5144,15 @@ function backfillNamedScoreWeights(accumulator) {
 }
 //#endregion
 //#region src/evaluator.ts
+const CONVERSATION_VAR_NAME = "_conversation";
+const promptUsesConversationVariableCache = new LRUCache({ max: 1024 });
+function promptUsesConversationVariable(prompt) {
+	const cached = promptUsesConversationVariableCache.get(prompt.raw);
+	if (cached !== void 0) return cached;
+	const { referenced, parsed } = analyzeTemplateReference(prompt.raw, CONVERSATION_VAR_NAME);
+	if (parsed) promptUsesConversationVariableCache.set(prompt.raw, referenced);
+	return referenced;
+}
 /**
 * Manages a single progress bar for the evaluation
 */
@@ -5284,6 +5352,18 @@ function hasProviderGroupedAssertion(assertion) {
 function shouldDeferGradingForTest(test) {
 	return Boolean(test.assert?.some(hasProviderGroupedAssertion));
 }
+function logGroupedGradingStatus({ concurrency, hasEvalStepTimeout, runEvalOptions, shouldGroupGradingByProvider, usesConversationVar }) {
+	if (!runEvalOptions.some(({ test }) => shouldDeferGradingForTest(test))) return;
+	if (shouldGroupGradingByProvider) {
+		logger.info("Grouping model-graded assertions by provider to minimize local-model reload overhead.");
+		return;
+	}
+	if (concurrency !== 1) return;
+	const reasons = [];
+	if (hasEvalStepTimeout) reasons.push("per-eval-step timeout is configured");
+	if (usesConversationVar) reasons.push("conversation variables require per-row ordering");
+	if (reasons.length > 0) logger.info(`Serial grading grouping disabled because ${reasons.join(" and ")}; model-graded judges may reload between rows.`);
+}
 function applyGradingResult(row, checkResult) {
 	if (!checkResult.pass) {
 		row.error = checkResult.reason;
@@ -5298,14 +5378,29 @@ function applyGradingResult(row, checkResult) {
 	if (checkResult.tokensUsed) accumulateAssertionTokenUsage(row.tokenUsage.assertions, checkResult.tokensUsed);
 	row.gradingResult = checkResult;
 }
-function applyGradingError(row, error) {
-	const errorMessage = error instanceof Error ? error.stack ?? error.message : String(error);
-	logger.error("Assertion grading failed during eval", {
-		error: errorMessage,
-		promptIdx: row.promptIdx,
-		testIdx: row.testIdx
-	});
-	row.error = errorMessage;
+const ABORTED_GRADING_PREFIX = "Aborted: ";
+function isAbortShapedError(error) {
+	return error instanceof Error && (error.name === "AbortError" || error.name === "AbortException");
+}
+function applyGradingError(row, error, abortSignal) {
+	const errorAsError = error instanceof Error ? error : void 0;
+	if (Boolean(abortSignal?.aborted) && isAbortShapedError(error)) {
+		const shortMessage = errorAsError?.message ?? String(error);
+		logger.debug("Assertion grading aborted", {
+			error: shortMessage,
+			promptIdx: row.promptIdx,
+			testIdx: row.testIdx
+		});
+		row.error = `${ABORTED_GRADING_PREFIX}${shortMessage}`;
+	} else {
+		const fullMessage = errorAsError ? errorAsError.stack ?? errorAsError.message : String(error);
+		logger.error("Assertion grading failed during eval", {
+			error: fullMessage,
+			promptIdx: row.promptIdx,
+			testIdx: row.testIdx
+		});
+		row.error = fullMessage;
+	}
 	row.failureReason = ResultFailureReason.ERROR;
 	row.success = false;
 	row.score = 0;
@@ -5337,7 +5432,7 @@ function createRunEvalState({ provider, prompt, test }) {
 	};
 }
 function attachConversationVar({ conversations, conversationKey, prompt, test, vars }) {
-	const usesConversation = prompt.raw.includes("_conversation");
+	const usesConversation = promptUsesConversationVariable(prompt);
 	if (!getEnvBool("PROMPTFOO_DISABLE_CONVERSATION_VAR") && !test.options?.disableConversationVar && usesConversation) vars._conversation = conversations?.[conversationKey] || [];
 }
 function createRunEvalSetup({ provider, prompt, promptConfig, vars }) {
@@ -5584,7 +5679,7 @@ async function gradeRunEvalResponse({ abortSignal, deferGrading, evalId, latency
 			assertScoringFunction: test.assertScoringFunction,
 			traceId
 		}).then((checkResult) => applyGradingResult(ret, checkResult))).catch((error) => {
-			applyGradingError(ret, error);
+			applyGradingError(ret, error, abortSignal);
 		});
 		deferredGradingPromises.set(ret, gradingPromise);
 		return;
@@ -6131,7 +6226,7 @@ async function resolveDefaultTestProvider(defaultTest, testCase) {
 	const defaultProvider = defaultTest.provider;
 	if (isApiProvider(defaultProvider)) return defaultProvider;
 	if (typeof defaultProvider === "object" && defaultProvider.id) {
-		const { loadApiProvider } = await import("./providers-Ctcc592x.js");
+		const { loadApiProvider } = await import("./providers-CSOp-bCm.js");
 		return loadApiProvider(typeof defaultProvider.id === "function" ? defaultProvider.id() : defaultProvider.id, { options: defaultProvider });
 	}
 	return defaultProvider;
@@ -6291,7 +6386,7 @@ function buildRepeatCacheContextByTestIdx(runEvalOptions) {
 async function filterCompletedResumeSteps(runEvalOptions, evalRecord) {
 	if (!state.resume || !evalRecord.persisted) return;
 	try {
-		const { default: EvalResult } = await import("./evalResult-BBK58h2B.js");
+		const { default: EvalResult } = await import("./evalResult-BtZSUgQv.js");
 		const completedPairs = await EvalResult.getCompletedIndexPairs(evalRecord.id, { excludeErrors: state.retryMode });
 		const originalCount = runEvalOptions.length;
 		for (let i = runEvalOptions.length - 1; i >= 0; i--) {
@@ -6305,14 +6400,14 @@ async function filterCompletedResumeSteps(runEvalOptions, evalRecord) {
 	}
 }
 function adjustConcurrencyForSerialFeatures({ concurrency, prompts, tests }) {
-	const usesConversationVar = prompts.some((p) => p.raw.includes("_conversation"));
+	const usesConversationVar = prompts.some(promptUsesConversationVariable);
 	if (concurrency <= 1) return {
 		concurrency,
 		usesConversationVar
 	};
 	const usesStoreOutputAs = tests.some((t) => t.options?.storeOutputAs);
 	if (usesConversationVar) {
-		logger.info(`Setting concurrency to 1 because the ${chalk.cyan("_conversation")} variable is used.`);
+		logger.info(`Setting concurrency to 1 because the ${chalk.cyan(CONVERSATION_VAR_NAME)} variable is used.`);
 		return {
 			concurrency: 1,
 			usesConversationVar
@@ -6542,7 +6637,8 @@ var Evaluator = class {
 		};
 		this.conversations = {};
 		this.registers = {};
-		this.fileWriters = (Array.isArray(evalRecord.config.outputPath) ? evalRecord.config.outputPath.filter((p) => p.endsWith(".jsonl")) : evalRecord.config.outputPath?.endsWith(".jsonl") ? [evalRecord.config.outputPath] : []).map((p) => new JsonlFileWriter(p));
+		const jsonlFiles = Array.isArray(evalRecord.config.outputPath) ? evalRecord.config.outputPath.filter((p) => p.endsWith(".jsonl")) : evalRecord.config.outputPath?.endsWith(".jsonl") ? [evalRecord.config.outputPath] : [];
+		this.fileWriters = jsonlFiles.map((p) => new JsonlFileWriter(p));
 		this.rateLimitRegistry = createRateLimitRegistry({ maxConcurrency: options.maxConcurrency || 4 });
 		this.rateLimitRegistry.on("ratelimit:hit", (data) => {
 			logger.debug(`[Scheduler] Rate limit hit for ${data.rateLimitKey}`, {
@@ -6662,6 +6758,25 @@ var Evaluator = class {
 			this.trackCompletedRow(evalStep, row, context);
 			context.numComplete++;
 			const promptEvalCount = reservePromptEvalCount(context, row.promptIdx);
+			if (context.testSuite.extensions?.length) try {
+				const afterEachOut = await runExtensionHook(context.testSuite.extensions, "afterEach", {
+					test: evalStep.test,
+					result: {
+						...row,
+						namedScores: { ...row.namedScores },
+						metadata: { ...row.metadata },
+						response: row.response ? {
+							...row.response,
+							metadata: { ...row.response.metadata }
+						} : row.response
+					}
+				});
+				row.namedScores = filterFiniteScores(afterEachOut.result.namedScores);
+				row.metadata = afterEachOut.result.metadata;
+				if (row.response && afterEachOut.result.response) row.response.metadata = afterEachOut.result.response.metadata;
+			} catch (error) {
+				logger.error(`afterEach extension hook failed, persisting row without hook modifications`, { error });
+			}
 			await this.persistEvalRow(row);
 			if (this.abortIfTargetUnavailable(row, context)) break;
 			const metrics = context.prompts[row.promptIdx].metrics;
@@ -6673,10 +6788,6 @@ var Evaluator = class {
 				promptEvalCount,
 				row
 			});
-			await runExtensionHook(context.testSuite.extensions, "afterEach", {
-				test: evalStep.test,
-				result: row
-			});
 			context.options.progressCallback?.(context.numComplete, context.runEvalOptionsLength, index, evalStep, metrics);
 		}
 	}
@@ -6831,7 +6942,15 @@ var Evaluator = class {
 				})) break;
 			}
 		} catch (error) {
-			await flushGroupedRows();
+			const pendingRowCount = groupedRows.reduce((sum, entry) => sum + entry.rows.length, 0);
+			try {
+				await flushGroupedRows();
+			} catch (flushError) {
+				logger.warn("Failed to flush grouped rows after error; target outputs may be lost", {
+					error: flushError instanceof Error ? flushError.message : String(flushError),
+					pendingRowCount
+				});
+			}
 			throw error;
 		}
 		await flushGroupedRows();
@@ -7267,6 +7386,13 @@ var Evaluator = class {
 		if (!this.options.silent) {
 			if (serialRunEvalOptions.length > 0) logger.info(`Running ${serialRunEvalOptions.length} test cases serially...`);
 			if (concurrentRunEvalOptions.length > 0) logger.info(`Running ${concurrentRunEvalOptions.length} test cases (up to ${concurrency} at a time)...`);
+			logGroupedGradingStatus({
+				concurrency,
+				hasEvalStepTimeout,
+				runEvalOptions,
+				shouldGroupGradingByProvider,
+				usesConversationVar
+			});
 		}
 		if (this.options.showProgressBar && progressBarManager) {
 			await progressBarManager.initialize(runEvalOptions, concurrency, 0);
@@ -7369,4 +7495,4 @@ function evaluate(testSuite, evalRecord, options) {
 //#endregion
 export { isAllowedPrompt as a, assertions_default as c, generateVarCombinations as i, readAssertions as l, evaluate as n, accumulateNamedMetric as o, formatVarsForDisplay as r, doesPromptRefMatch as s, ProgressBarManager as t, runAssertions as u };
 
-//# sourceMappingURL=evaluator-D-UIbbYq.js.map
+//# sourceMappingURL=evaluator-n_dEb00o.js.map