promptfoo 0.121.1 → 0.121.3

package/dist/src/index.cjs

@@ -2,43 +2,43 @@ Object.defineProperties(exports, {
 	__esModule: { value: true },
 	[Symbol.toStringTag]: { value: "Module" }
 });
-const require_logger = require("./logger-Cp1GPUjj.cjs");
+const require_logger = require("./logger-D5iKBpu_.cjs");
 const require_invariant = require("./invariant-kfQ8Bu82.cjs");
-const require_esm = require("./esm-CnNt7sI4.cjs");
-const require_pythonUtils = require("./pythonUtils-CTU3Y3lw.cjs");
+const require_esm = require("./esm-CipptfDu.cjs");
+const require_pythonUtils = require("./pythonUtils-dAVigVK-.cjs");
 const require_fileExtensions = require("./fileExtensions-bYh77CN8.cjs");
-const require_transform = require("./transform-CzK1Q0zl.cjs");
-const require_graders = require("./graders-DOXycdlG.cjs");
-const require_types = require("./types-LJ0r3wbR.cjs");
-const require_util = require("./util-Yz-1aEhW.cjs");
-const require_fetch = require("./fetch-BxUk8odA.cjs");
-const require_cache = require("./cache-COish3-W.cjs");
-const require_providers = require("./providers-CFu-TZl-.cjs");
-const require_utils = require("./utils-Cz9qXqII.cjs");
-const require_createHash = require("./createHash-CfZSc0b4.cjs");
-require("./genaiTracer-DN4dQywX.cjs");
-const require_chat = require("./chat-DaqekjFr.cjs");
+const require_transform = require("./transform-ZrG2dvlo.cjs");
+const require_graders = require("./graders-BElhu9ZY.cjs");
+const require_types = require("./types-D8cGDZbL.cjs");
+const require_util = require("./util-CuLo2pMR.cjs");
+const require_fetch = require("./fetch-BnR9wSnm.cjs");
+const require_cache = require("./cache-C5yFZ4gC.cjs");
+const require_providers = require("./providers-CScd1wN6.cjs");
+const require_utils = require("./utils-DKw8mrgr.cjs");
+const require_createHash = require("./createHash-BYwImsYv.cjs");
+require("./genaiTracer-BfxrvSUb.cjs");
+const require_chat = require("./chat-CM8qWR3_.cjs");
 const require_tokenUsageUtils = require("./tokenUsageUtils-bVa1ga6f.cjs");
-const require_transform$1 = require("./transform-DOcQeLld.cjs");
-require("./messages-1JrJs91T.cjs");
-require("./util-CchiqXh_.cjs");
-require("./responses-tD4Bd4dc.cjs");
-require("./openai-Cuif0GEt.cjs");
-const require_util$2 = require("./util-Db0a0AFH.cjs");
-require("./completion-CDOouNzq.cjs");
-const require_accounts = require("./accounts-Bx-x3bmW.cjs");
-const require_server = require("./server-B0PPuDw-.cjs");
-const require_blobs = require("./blobs-DvS-O6be.cjs");
-const require_tables = require("./tables-C7K-XKWp.cjs");
-const require_extractor = require("./extractor-DX36oYEv.cjs");
-const require_telemetry = require("./telemetry-CbrnxHp_.cjs");
+const require_transform$1 = require("./transform-0BwoBsvO.cjs");
+require("./messages-HJsyEh4o.cjs");
+require("./util--9u9UVCt.cjs");
+require("./responses-mo0KQDbu.cjs");
+require("./openai-CoxGAQwn.cjs");
+const require_util$2 = require("./util-CFj4YKIn.cjs");
+require("./completion-DlXUhj5c.cjs");
+const require_accounts = require("./accounts-BPyfpSeU.cjs");
+const require_server = require("./server-BtoCXeXI.cjs");
+const require_blobs = require("./blobs-C6j0bvFz.cjs");
+const require_tables = require("./tables-BdZQEpRz.cjs");
+const require_extractor = require("./extractor-DG3sSfXE.cjs");
+const require_telemetry = require("./telemetry-re627Lre.cjs");
 const require_text = require("./text-CW1cyrwj.cjs");
-const require_store = require("./store-BSc-TF2w.cjs");
-require("./base-DBtwl2FR.cjs");
-require("./image-DTedmQPg.cjs");
-const require_providerRegistry = require("./providerRegistry-Civky8Ar.cjs");
-const require_rubyUtils = require("./rubyUtils-DhCAlxZr.cjs");
-const require_evalResult = require("./evalResult-Dap2CekP.cjs");
+const require_store = require("./store-CLyU7AtI.cjs");
+require("./base-BboXIF_0.cjs");
+require("./image--F58eEIn.cjs");
+const require_providerRegistry = require("./providerRegistry-BTDgfV5h.cjs");
+const require_rubyUtils = require("./rubyUtils-CGeUtCfW.cjs");
+const require_evalResult = require("./evalResult-DpARzUCb.cjs");
 let fs = require("fs");
 fs = require_logger.__toESM(fs);
 let path = require("path");
@@ -68,6 +68,8 @@ crypto$1 = require_logger.__toESM(crypto$1);
 let _opentelemetry_api = require("@opentelemetry/api");
 let _inquirer_input = require("@inquirer/input");
 _inquirer_input = require_logger.__toESM(_inquirer_input);
+let readline = require("readline");
+readline = require_logger.__toESM(readline);
 let drizzle_orm = require("drizzle-orm");
 let cli_progress = require("cli-progress");
 cli_progress = require_logger.__toESM(cli_progress);
@@ -75,6 +77,7 @@ let jsdom = require("jsdom");
 let fastest_levenshtein = require("fastest-levenshtein");
 let js_rouge = require("js-rouge");
 js_rouge = require_logger.__toESM(js_rouge);
+let node_util = require("node:util");
 require("debounce");
 let _opentelemetry_core = require("@opentelemetry/core");
 let _opentelemetry_exporter_trace_otlp_http = require("@opentelemetry/exporter-trace-otlp-http");
@@ -307,7 +310,7 @@ async function startOtlpReceiverIfNeeded(testSuite) {
 		require_telemetry.telemetry.record("feature_used", { feature: "tracing" });
 		try {
 			require_logger.logger.debug("[EvaluatorTracing] Tracing configuration detected, starting OTLP receiver");
-			const { startOTLPReceiver } = await Promise.resolve().then(() => require("./otlpReceiver-Diec4cln.cjs"));
+			const { startOTLPReceiver } = await Promise.resolve().then(() => require("./otlpReceiver-DHKqJlsz.cjs"));
 			const port = testSuite.tracing.otlp.http.port || 4318;
 			const host = testSuite.tracing.otlp.http.host || "127.0.0.1";
 			require_logger.logger.debug(`[EvaluatorTracing] Starting OTLP receiver on ${host}:${port}`);
@@ -330,7 +333,7 @@ async function startOtlpReceiverIfNeeded(testSuite) {
 async function stopOtlpReceiverIfNeeded() {
 	if (otlpReceiverStarted) try {
 		require_logger.logger.debug("[EvaluatorTracing] Stopping OTLP receiver");
-		const { stopOTLPReceiver } = await Promise.resolve().then(() => require("./otlpReceiver-Diec4cln.cjs"));
+		const { stopOTLPReceiver } = await Promise.resolve().then(() => require("./otlpReceiver-DHKqJlsz.cjs"));
 		await stopOTLPReceiver();
 		otlpReceiverStarted = false;
 		require_logger.logger.info("[EvaluatorTracing] OTLP receiver stopped successfully");
@@ -365,7 +368,7 @@ async function generateTraceContextIfNeeded(test, evaluateOptions, testIdx, prom
 	}
 	if (!tracingEnabled) return null;
 	require_logger.logger.debug("[EvaluatorTracing] Importing trace store");
-	const { getTraceStore } = await Promise.resolve().then(() => require("./store-BNmZ1KAz.cjs"));
+	const { getTraceStore } = await Promise.resolve().then(() => require("./store-CNHk-De4.cjs"));
 	const traceStore = getTraceStore();
 	const traceId = generateTraceId();
 	const spanId = generateSpanId();
@@ -1398,7 +1401,7 @@ const handleJavascript = async ({ assertion, renderedValue, valueFromScript, ass
 			pass = result !== inverse;
 			score = pass ? 1 : 0;
 		} else if (typeof result === "number") {
-			pass = assertion.threshold !== void 0 ? result >= assertion.threshold : result > 0;
+			pass = assertion.threshold === void 0 ? result > 0 : result >= assertion.threshold;
 			score = result;
 		} else if (typeof result === "object") return result;
 		else throw new Error("Custom function must return a boolean or number");
@@ -1431,7 +1434,7 @@ function handleIsJson({ outputString, renderedValue, inverse, valueFromScript, a
 	} catch {
 		pass = inverse;
 	}
-	if (pass && renderedValue) {
+	if (parsedJson !== void 0 && renderedValue) {
 		let validate;
 		if (typeof renderedValue === "string") if (renderedValue.startsWith("file://")) {
 			const schema = valueFromScript;
@@ -1443,11 +1446,12 @@ function handleIsJson({ outputString, renderedValue, inverse, valueFromScript, a
 		}
 		else if (typeof renderedValue === "object") validate = require_logger.getAjv().compile(renderedValue);
 		else throw new Error("is-json assertion must have a string or object value");
-		pass = validate(parsedJson);
+		const valid = validate(parsedJson);
+		pass = inverse ? !valid : valid;
 		if (!pass) return {
 			pass,
 			score: 0,
-			reason: `JSON does not conform to the provided schema. Errors: ${require_logger.getAjv().errorsText(validate.errors)}`,
+			reason: inverse ? "Output is JSON that conforms to the provided schema" : `JSON does not conform to the provided schema. Errors: ${require_logger.getAjv().errorsText(validate.errors)}`,
 			assertion
 		};
 	}
@@ -1474,9 +1478,12 @@ function handleContainsJson({ assertion, renderedValue, outputString, inverse, v
 		}
 		else if (typeof renderedValue === "object") validate = require_logger.getAjv().compile(renderedValue);
 		else throw new Error("contains-json assertion must have a string or object value");
-		pass = validate(jsonObject);
-		if (pass) break;
-		else errorMessage = `JSON does not conform to the provided schema. Errors: ${require_logger.getAjv().errorsText(validate.errors)}`;
+		const valid = validate(jsonObject);
+		pass = inverse ? !valid : valid;
+		if (valid) {
+			if (inverse) errorMessage = "Output contains JSON conforming to the provided schema";
+			break;
+		} else errorMessage = `JSON does not conform to the provided schema. Errors: ${require_logger.getAjv().errorsText(validate.errors)}`;
 	}
 	return {
 		pass,
@@ -1660,7 +1667,7 @@ function handlePerplexity({ logProbs, assertion }) {
 	if (!logProbs || logProbs.length === 0) throw new Error("Perplexity assertion does not support providers that do not return logProbs");
 	const avgLogProb = logProbs.reduce((acc, logProb) => acc + logProb, 0) / logProbs.length;
 	const perplexity = Math.exp(-avgLogProb);
-	const pass = assertion.threshold !== void 0 ? perplexity <= assertion.threshold : true;
+	const pass = assertion.threshold === void 0 ? true : perplexity <= assertion.threshold;
 	return {
 		pass,
 		score: pass ? 1 : 0,
@@ -1672,7 +1679,7 @@ function handlePerplexityScore({ logProbs, assertion }) {
 	if (!logProbs || logProbs.length === 0) throw new Error("perplexity-score assertion does not support providers that do not return logProbs");
 	const avgLogProb = logProbs.reduce((acc, logProb) => acc + logProb, 0) / logProbs.length;
 	const perplexityNorm = 1 / (1 + Math.exp(-avgLogProb));
-	const pass = assertion.threshold !== void 0 ? perplexityNorm >= assertion.threshold : true;
+	const pass = assertion.threshold === void 0 ? true : perplexityNorm >= assertion.threshold;
 	return {
 		pass,
 		score: perplexityNorm,
@@ -1787,7 +1794,7 @@ ${isMultiline ? renderedValue.split("\n").map((line) => `${indentStyle}${line}`)
 		} else {
 			score = Number.parseFloat(String(result));
 			if (Number.isNaN(score)) throw new Error(`Python assertion must return a boolean, number, or {pass, score, reason} object. Instead got:\n${result}`);
-			pass = assertion.threshold !== void 0 ? score >= assertion.threshold : score > 0;
+			pass = assertion.threshold === void 0 ? score > 0 : score >= assertion.threshold;
 		}
 	} catch (err) {
 		return {
@@ -2048,7 +2055,7 @@ end
 		} else {
 			score = Number.parseFloat(String(result));
 			if (Number.isNaN(score)) throw new Error(`Ruby assertion must return a boolean, number, or {pass, score, reason} object. Instead got:\n${result}`);
-			pass = assertion.threshold !== void 0 ? score >= assertion.threshold : score > 0;
+			pass = assertion.threshold === void 0 ? score > 0 : score >= assertion.threshold;
 		}
 	} catch (err) {
 		return {
@@ -2119,6 +2126,127 @@ const handleSimilar = async ({ assertion, renderedValue, outputString, inverse,
 	};
 };
 //#endregion
+//#region src/assertions/traceUtils.ts
+/**
+* Shared utilities for trace assertions
+*/
+/**
+* Match a span name against a glob-like pattern.
+* Supports * (any characters) and ? (single character) wildcards.
+*
+* @param spanName - The span name to match
+* @param pattern - The glob pattern to match against
+* @returns true if the span name matches the pattern
+*/
+function matchesPattern(spanName, pattern) {
+	const regexPattern = pattern.replace(/[.+^${}()|[\]\\]/g, "\\$&").replace(/\*/g, ".*").replace(/\?/g, ".");
+	return new RegExp(`^${regexPattern}$`, "i").test(spanName);
+}
+//#endregion
+//#region src/assertions/skill.ts
+function getSkillCalls(params) {
+	const rawSkillCalls = params.providerResponse?.metadata?.skillCalls;
+	if (!Array.isArray(rawSkillCalls)) return [];
+	return rawSkillCalls.filter((entry) => Boolean(entry) && typeof entry === "object" && typeof entry.name === "string");
+}
+function matchesSkill(skillCall, matcher) {
+	if (matcher.name && skillCall.name !== matcher.name) return false;
+	if (matcher.pattern && !matchesPattern(skillCall.name, matcher.pattern)) return false;
+	return true;
+}
+function formatSkillCall(skillCall) {
+	const details = [skillCall.source, skillCall.path].filter(Boolean).join(", ");
+	return details ? `${skillCall.name} (${details})` : skillCall.name;
+}
+function resolveSkillMatchers(value) {
+	const normalizeText = (text) => typeof text === "string" ? text.trim() : void 0;
+	const validateCount = (field, count) => {
+		if (!Number.isFinite(count) || !Number.isInteger(count) || count < 0) throw new Error(`skill-used assertion object ${field} must be a finite non-negative integer`);
+	};
+	if (typeof value === "string" && value.trim()) return {
+		kind: "list",
+		matchers: [{ name: normalizeText(value) }]
+	};
+	if (Array.isArray(value) && value.length > 0 && value.every((item) => typeof item === "string" && item.trim())) return {
+		kind: "list",
+		matchers: value.map((item) => ({ name: item.trim() }))
+	};
+	if (value && typeof value === "object" && !Array.isArray(value)) {
+		const rawMatcher = value;
+		const matcher = rawMatcher;
+		const name = normalizeText(matcher.name);
+		const pattern = normalizeText(matcher.pattern);
+		if (!name && !pattern) throw new Error("skill-used assertion object must include a name or pattern property");
+		if ("min" in rawMatcher) validateCount("min", matcher.min);
+		if ("max" in rawMatcher) validateCount("max", matcher.max);
+		if (typeof matcher.min === "number" && typeof matcher.max === "number" && matcher.max < matcher.min) throw new Error("skill-used assertion object max must be greater than or equal to min");
+		return {
+			kind: "count",
+			matcher: {
+				max: typeof matcher.max === "number" ? matcher.max : void 0,
+				min: typeof matcher.min === "number" ? matcher.min : void 0,
+				name,
+				pattern
+			}
+		};
+	}
+	throw new Error("skill-used assertion must have a string, string array, or object value");
+}
+function handleListSkillAssertion(params, skillCalls, actualSkills, expected) {
+	const missing = expected.matchers.filter((matcher) => !skillCalls.some((skillCall) => matchesSkill(skillCall, matcher)));
+	const matched = expected.matchers.filter((matcher) => skillCalls.some((skillCall) => matchesSkill(skillCall, matcher)));
+	const pass = params.inverse ? matched.length === 0 : missing.length === 0;
+	const expectedSkills = expected.matchers.map((matcher) => matcher.name);
+	const actualSummary = actualSkills.length > 0 ? actualSkills.join(", ") : "(none)";
+	let reason;
+	if (params.inverse) reason = pass ? `Forbidden skill(s) were not used: ${expectedSkills.join(", ")}` : `Forbidden skill(s) were used: ${matched.map((matcher) => matcher.name).join(", ")}. Actual skills: ${actualSummary}`;
+	else if (pass) reason = `Observed required skill(s): ${expectedSkills.join(", ")}. Actual skills: ${actualSummary}`;
+	else reason = `Missing required skill(s): ${missing.map((matcher) => matcher.name).join(", ")}. Actual skills: ${actualSummary}`;
+	return {
+		pass,
+		score: pass ? 1 : 0,
+		reason,
+		assertion: params.assertion
+	};
+}
+function handleCountSkillAssertion(params, skillCalls, actualSkills, matcher) {
+	const hasExplicitMin = matcher.min !== void 0;
+	const hasExplicitMax = matcher.max !== void 0;
+	const min = matcher.min ?? (hasExplicitMax ? 0 : 1);
+	const max = matcher.max;
+	const matchingSkillCalls = skillCalls.filter((skillCall) => matchesSkill(skillCall, matcher));
+	const count = matchingSkillCalls.length;
+	const matcherLabel = matcher.pattern || matcher.name || "*";
+	if (params.inverse) {
+		if (hasExplicitMin || hasExplicitMax && max !== 0) throw new Error("not-skill-used object assertions only support name/pattern with no count bounds, or max: 0");
+		const pass = count === 0;
+		const actualSummary = actualSkills.length > 0 ? actualSkills.join(", ") : "(none)";
+		return {
+			pass,
+			score: pass ? 1 : 0,
+			reason: pass ? `Forbidden skill "${matcherLabel}" was not used. Actual skills: ${actualSummary}` : `Forbidden skill "${matcherLabel}" was used ${count} time(s). Matches: ${matchingSkillCalls.map(formatSkillCall).join(", ")}`,
+			assertion: params.assertion
+		};
+	}
+	const pass = count >= min && (max === void 0 || count <= max);
+	let reason = `Matched skill "${matcherLabel}" ${count} time(s)`;
+	reason += max === void 0 ? ` (expected at least ${min})` : ` (expected ${min}-${max})`;
+	if (matchingSkillCalls.length > 0) reason += `. Matches: ${matchingSkillCalls.map(formatSkillCall).join(", ")}`;
+	return {
+		pass,
+		score: pass ? 1 : 0,
+		reason,
+		assertion: params.assertion
+	};
+}
+function handleSkillUsed(params) {
+	const skillCalls = getSkillCalls(params);
+	const actualSkills = skillCalls.map(formatSkillCall);
+	const expected = resolveSkillMatchers(params.renderedValue ?? params.assertion.value);
+	if (expected.kind === "list") return handleListSkillAssertion(params, skillCalls, actualSkills, expected);
+	return handleCountSkillAssertion(params, skillCalls, actualSkills, expected.matcher);
+}
+//#endregion
 //#region src/assertions/sql.ts
 const handleIsSql = async ({ assertion, renderedValue, outputString, inverse }) => {
 	let pass = false;
@@ -2351,23 +2479,6 @@ const handleToolCallF1 = ({ assertion, output, renderedValue, inverse }) => {
 	};
 };
 //#endregion
-//#region src/assertions/traceUtils.ts
-/**
-* Shared utilities for trace assertions
-*/
-/**
-* Match a span name against a glob-like pattern.
-* Supports * (any characters) and ? (single character) wildcards.
-*
-* @param spanName - The span name to match
-* @param pattern - The glob pattern to match against
-* @returns true if the span name matches the pattern
-*/
-function matchesPattern(spanName, pattern) {
-	const regexPattern = pattern.replace(/[.+^${}()|[\]\\]/g, "\\$&").replace(/\*/g, ".*").replace(/\?/g, ".");
-	return new RegExp(`^${regexPattern}$`, "i").test(spanName);
-}
-//#endregion
 //#region src/assertions/traceErrorSpans.ts
 function isErrorSpan(span) {
 	if (span.statusCode && span.statusCode >= 400) return true;
@@ -2536,6 +2647,524 @@ const handleTraceSpanDuration = ({ assertion, assertionValueContext }) => {
 	};
 };
 //#endregion
+//#region src/assertions/trajectoryUtils.ts
+const TOOL_ATTRIBUTE_KEYS = [
+	"tool.name",
+	"tool_name",
+	"tool",
+	"function.name",
+	"function_name",
+	"gen_ai.tool.name",
+	"codex.mcp.tool",
+	"agent.tool",
+	"agent.tool_name",
+	"agent.toolName"
+];
+const TOOL_ARGUMENT_ATTRIBUTE_KEYS = [
+	"tool.arguments",
+	"tool.args",
+	"tool.input",
+	"tool_arguments",
+	"tool_args",
+	"tool_input",
+	"function.arguments",
+	"function.args",
+	"function.input",
+	"function_arguments",
+	"function_args",
+	"gen_ai.tool.arguments",
+	"gen_ai.tool.args",
+	"gen_ai.tool.input",
+	"gen_ai.tool.call.arguments",
+	"gen_ai.tool.call.args",
+	"agent.tool.arguments",
+	"agent.tool.args",
+	"agent.tool.input",
+	"codex.mcp.arguments",
+	"codex.mcp.args",
+	"codex.mcp.input",
+	"arguments",
+	"args",
+	"input"
+];
+const COMMAND_ATTRIBUTE_KEYS = [
+	"codex.command",
+	"command",
+	"command.name",
+	"command_name"
+];
+const SEARCH_ATTRIBUTE_KEYS = [
+	"codex.search.query",
+	"search.query",
+	"search_query"
+];
+const GENERIC_QUERY_ATTRIBUTE_KEYS = ["query"];
+const SEARCH_SPAN_NAME_PATTERN = /(^|[\s._:/-])(search|find|lookup|retriev(?:e|al))($|[\s._:/-])/i;
+const MAX_JUDGE_SUMMARY_STEPS = 24;
+const JUDGE_SUMMARY_HEAD_STEPS = 12;
+const JUDGE_SUMMARY_TAIL_STEPS = 12;
+function getStringAttribute(attributes, keys) {
+	for (const key of keys) {
+		const value = attributes[key];
+		if (typeof value === "string" && value.trim()) return value.trim();
+	}
+}
+function normalizeStructuredAttribute(value) {
+	if (value === void 0 || value === null) return;
+	if (typeof value === "string") {
+		const trimmed = value.trim();
+		if (!trimmed) return;
+		try {
+			return JSON.parse(trimmed);
+		} catch {
+			return trimmed;
+		}
+	}
+	if (typeof value === "number" || typeof value === "boolean" || typeof value === "object") return value;
+}
+function hasSameStatus(left, right) {
+	return left?.code === right?.code && left?.message === right?.message;
+}
+function isSearchLikeSpan(span) {
+	const attributes = span.attributes || {};
+	if (SEARCH_SPAN_NAME_PATTERN.test(span.name) || span.name.startsWith("search ")) return true;
+	return Object.keys(attributes).some((key) => key !== "query" && /(^|[._])(search|lookup|retriev(?:e|al))($|[._])/i.test(key));
+}
+function getTrajectoryStepStatus(step) {
+	if (step.statusCode === void 0 || step.statusCode === 0) return;
+	return {
+		code: step.statusCode,
+		...step.statusMessage ? { message: step.statusMessage } : {}
+	};
+}
+function getCommandExecutable(command) {
+	return command.trim().split(/\s+/)[0] || void 0;
+}
+function extractToolName(span) {
+	const attributes = span.attributes || {};
+	const directMatch = getStringAttribute(attributes, TOOL_ATTRIBUTE_KEYS);
+	if (directMatch) return directMatch;
+	for (const [key, value] of Object.entries(attributes)) {
+		if (typeof value !== "string" || !value.trim()) continue;
+		if (/tool.?name|function.?name/i.test(key)) return value.trim();
+		if (/(^|[._])tool($|[._])/i.test(key) && !/result|output/i.test(key)) return value.trim();
+	}
+	if (span.name.startsWith("mcp ")) {
+		const slashIndex = span.name.lastIndexOf("/");
+		if (slashIndex !== -1 && slashIndex < span.name.length - 1) return span.name.slice(slashIndex + 1).trim();
+	}
+}
+function extractToolArgs(span) {
+	const attributes = span.attributes || {};
+	for (const key of TOOL_ARGUMENT_ATTRIBUTE_KEYS) {
+		const value = normalizeStructuredAttribute(attributes[key]);
+		if (value !== void 0) return value;
+	}
+	for (const [key, rawValue] of Object.entries(attributes)) {
+		if (/result|output|error|status/i.test(key)) continue;
+		if (!/(^|[._])(arguments|args|input)($|[._])/i.test(key)) continue;
+		const value = normalizeStructuredAttribute(rawValue);
+		if (value !== void 0) return value;
+	}
+}
+function extractCommand(span) {
+	const attributes = span.attributes || {};
+	const directMatch = getStringAttribute(attributes, COMMAND_ATTRIBUTE_KEYS);
+	if (directMatch) return directMatch;
+	for (const [key, value] of Object.entries(attributes)) {
+		if (typeof value !== "string" || !value.trim()) continue;
+		if (/command/i.test(key) && !/output|result/i.test(key)) return value.trim();
+	}
+	if (span.name.startsWith("exec ")) return span.name.slice(5).trim();
+}
+function extractSearchQuery(span) {
+	const attributes = span.attributes || {};
+	const directMatch = getStringAttribute(attributes, SEARCH_ATTRIBUTE_KEYS);
+	if (directMatch) return directMatch;
+	const genericQuery = getStringAttribute(attributes, GENERIC_QUERY_ATTRIBUTE_KEYS);
+	if (genericQuery && isSearchLikeSpan(span)) return genericQuery;
+	if (span.name.startsWith("search ")) return span.name.slice(7).replace(/^"|"$/g, "").trim();
+}
+function isReasoningSpan(span) {
+	if ((span.attributes || {})["codex.item.type"] === "reasoning") return true;
+	return /^reasoning([_\s]|$)/i.test(span.name) || span.name === "reasoning";
+}
+function isMessageSpan(span) {
+	if ((span.attributes || {})["codex.item.type"] === "agent_message") return true;
+	return span.name === "agent response" || span.name === "send input";
+}
+function extractTrajectorySteps(trace) {
+	return [...trace.spans || []].map((span, index) => ({
+		span,
+		index
+	})).sort((left, right) => {
+		const timeDiff = left.span.startTime - right.span.startTime;
+		if (timeDiff !== 0) return timeDiff;
+		const endDiff = (left.span.endTime ?? left.span.startTime) - (right.span.endTime ?? right.span.startTime);
+		if (endDiff !== 0) return endDiff;
+		return left.index - right.index;
+	}).map(({ span }) => {
+		const toolName = extractToolName(span);
+		const command = extractCommand(span);
+		const searchQuery = extractSearchQuery(span);
+		let type = "span";
+		let name = span.name;
+		const aliases = new Set([span.name]);
+		let args;
+		if (toolName) {
+			type = "tool";
+			name = toolName;
+			aliases.add(toolName);
+			args = extractToolArgs(span);
+		} else if (command) {
+			type = "command";
+			name = command;
+			aliases.add(command);
+			const executable = getCommandExecutable(command);
+			if (executable) aliases.add(executable);
+		} else if (searchQuery) {
+			type = "search";
+			name = searchQuery;
+			aliases.add(searchQuery);
+		} else if (isReasoningSpan(span)) {
+			type = "reasoning";
+			name = span.name;
+			aliases.add("reasoning");
+		} else if (isMessageSpan(span)) {
+			type = "message";
+			name = span.name;
+			aliases.add("message");
+		}
+		return {
+			aliases: [...aliases],
+			...args === void 0 ? {} : { args },
+			attributes: span.attributes || {},
+			endTime: span.endTime,
+			name,
+			spanId: span.spanId,
+			spanName: span.name,
+			startTime: span.startTime,
+			statusCode: span.statusCode,
+			statusMessage: span.statusMessage,
+			type
+		};
+	});
+}
+function normalizeTrajectoryMatcher(matcher, defaultType) {
+	if (typeof matcher === "string") return {
+		pattern: matcher,
+		...defaultType ? { type: defaultType } : {}
+	};
+	return {
+		...matcher,
+		...matcher.type ? {} : defaultType ? { type: defaultType } : {}
+	};
+}
+function matchesTrajectoryStep(step, matcher, defaultType) {
+	const { type, pattern, name } = normalizeTrajectoryMatcher(matcher, defaultType);
+	if (type) {
+		if (!(Array.isArray(type) ? type : [type]).includes(step.type)) return false;
+	}
+	const matchPattern = pattern || name;
+	if (!matchPattern) return true;
+	return step.aliases.some((alias) => matchesPattern(alias, matchPattern));
+}
+function formatTrajectoryStep(step) {
+	return `${step.type}:${step.name}`;
+}
+function formatTrajectoryArgs(args) {
+	if (args === void 0) return "(none)";
+	try {
+		const serialized = JSON.stringify(args);
+		if (serialized !== void 0) return serialized;
+	} catch {}
+	return String(args);
+}
+function compactJudgeTrajectorySteps(steps) {
+	const compacted = [];
+	for (const step of steps) {
+		const previousStep = compacted[compacted.length - 1];
+		if (previousStep && previousStep.type === step.type && previousStep.name === step.name && previousStep.spanName === step.spanName && hasSameStatus(previousStep.status, step.status)) {
+			previousStep.collapsedCount = (previousStep.collapsedCount ?? 1) + 1;
+			continue;
+		}
+		compacted.push(step);
+	}
+	return compacted;
+}
+function truncateJudgeTrajectorySteps(steps) {
+	if (steps.length <= MAX_JUDGE_SUMMARY_STEPS) return steps;
+	return [
+		...steps.slice(0, JUDGE_SUMMARY_HEAD_STEPS),
+		{ omittedCount: steps.length - MAX_JUDGE_SUMMARY_STEPS },
+		...steps.slice(-JUDGE_SUMMARY_TAIL_STEPS)
+	];
+}
+function summarizeTrajectoryForJudge(trace) {
+	const rawSteps = extractTrajectorySteps(trace).map((step, index) => ({
+		index: index + 1,
+		type: step.type,
+		name: step.name,
+		...step.spanName === step.name ? {} : { spanName: step.spanName },
+		...getTrajectoryStepStatus(step) ? { status: getTrajectoryStepStatus(step) } : {}
+	}));
+	const compactedSteps = compactJudgeTrajectorySteps(rawSteps);
+	const steps = truncateJudgeTrajectorySteps(compactedSteps);
+	return JSON.stringify({
+		traceId: trace.traceId,
+		stepCount: rawSteps.length,
+		compactedStepCount: compactedSteps.length,
+		steps
+	}, null, 2);
+}
+//#endregion
+//#region src/assertions/trajectory.ts
+function getTraceOrThrow(params) {
+	const trace = params.assertionValueContext.trace;
+	if (!trace || !trace.spans) throw new Error(`No trace data available for ${params.baseType} assertion`);
+	return trace;
+}
+function applyInverse(pass, inverse) {
+	return inverse ? !pass : pass;
+}
+function formatStepList(stepLabels) {
+	return stepLabels.length > 0 ? stepLabels.join(", ") : "(none)";
+}
+function requireNamedTrajectoryMatcher(matcher, assertionType, index) {
+	if (matcher.pattern || matcher.name) return;
+	const stepLabel = index === void 0 ? "object" : `step ${index + 1}`;
+	throw new Error(`${assertionType} assertion ${stepLabel} must include a name or pattern property`);
+}
+function resolveGoalSuccessValue(value) {
+	if (typeof value === "string" && value.trim()) return { goal: value.trim() };
+	if (value && typeof value === "object" && !Array.isArray(value) && typeof value.goal === "string" && value.goal.trim()) return { goal: value.goal.trim() };
+	throw new Error("trajectory:goal-success assertion must have a string value or an object with a goal property");
+}
+function resolveToolMatchers(value) {
+	if (typeof value === "string") return {
+		kind: "list",
+		matchers: [normalizeTrajectoryMatcher(value, "tool")]
+	};
+	if (Array.isArray(value) && value.every((item) => typeof item === "string")) return {
+		kind: "list",
+		matchers: value.map((item) => normalizeTrajectoryMatcher(item, "tool"))
+	};
+	if (value && typeof value === "object" && !Array.isArray(value)) return {
+		kind: "count",
+		matcher: {
+			...normalizeTrajectoryMatcher(value, "tool"),
+			max: typeof value.max === "number" ? value.max : void 0,
+			min: typeof value.min === "number" ? value.min : void 0
+		}
+	};
+	throw new Error("trajectory:tool-used assertion must have a string, string array, or object value");
+}
+const handleTrajectoryToolUsed = (params) => {
+	const steps = extractTrajectorySteps(getTraceOrThrow(params)).filter((step) => step.type === "tool");
+	const expected = resolveToolMatchers(params.renderedValue ?? params.assertion.value);
+	if (expected.kind === "list") {
+		if (expected.matchers.length === 0) throw new Error("trajectory:tool-used assertion requires at least one expected tool");
+		const missing = expected.matchers.filter((matcher) => !steps.some((step) => matchesTrajectoryStep(step, matcher)));
+		const matched = expected.matchers.filter((matcher) => steps.some((step) => matchesTrajectoryStep(step, matcher)));
+		const pass = params.inverse ? matched.length === 0 : missing.length === 0;
+		const actualTools = steps.map(formatTrajectoryStep);
+		const expectedTools = expected.matchers.map((matcher) => matcher.pattern || matcher.name || "*");
+		let reason;
+		if (params.inverse) reason = pass ? `Forbidden tool(s) were not used: ${expectedTools.join(", ")}` : `Forbidden tool(s) were used: ${matched.map((matcher) => matcher.pattern || matcher.name || "*").join(", ")}. Actual tools: ${formatStepList(actualTools)}`;
+		else if (pass) reason = `Observed required tool(s): ${expectedTools.join(", ")}. Actual tools: ${formatStepList(actualTools)}`;
+		else reason = `Missing required tool(s): ${missing.map((matcher) => matcher.pattern || matcher.name || "*").join(", ")}. Actual tools: ${formatStepList(actualTools)}`;
+		return {
+			pass,
+			score: pass ? 1 : 0,
+			reason,
+			assertion: params.assertion
+		};
+	}
+	const matcher = expected.matcher;
+	const min = matcher.min ?? 1;
+	const max = matcher.max;
+	if (!matcher.pattern && !matcher.name) throw new Error("trajectory:tool-used assertion object must include a name or pattern property");
+	const matchingSteps = steps.filter((step) => matchesTrajectoryStep(step, matcher));
+	const count = matchingSteps.length;
+	const basePass = count >= min && (max === void 0 || count <= max);
+	const pass = applyInverse(basePass, params.inverse);
+	const matcherLabel = matcher.pattern || matcher.name || "*";
+	let reason = `Matched tool "${matcherLabel}" ${count} time(s)`;
+	if (max === void 0) reason += ` (expected at least ${min})`;
+	else reason += ` (expected ${min}-${max})`;
+	if (matchingSteps.length > 0) reason += `. Matches: ${matchingSteps.map(formatTrajectoryStep).join(", ")}`;
+	if (params.inverse) reason = basePass ? `Tool "${matcherLabel}" matched ${count} time(s), which violates the inverse assertion` : `Tool "${matcherLabel}" did not satisfy the forbidden match condition`;
+	return {
+		pass,
+		score: pass ? 1 : 0,
+		reason,
+		assertion: params.assertion
+	};
+};
+function resolveSequenceValue(value) {
+	if (Array.isArray(value)) return {
+		mode: "in_order",
+		steps: value
+	};
+	if (value && typeof value === "object" && !Array.isArray(value)) {
+		const sequenceValue = value;
+		return {
+			mode: sequenceValue.mode || "in_order",
+			steps: sequenceValue.steps || []
+		};
+	}
+	throw new Error("trajectory:tool-sequence assertion must have an array or object value");
+}
+function isRecord(value) {
+	return typeof value === "object" && value !== null && !Array.isArray(value);
+}
+function matchesExpectedArgsPartial(actual, expected) {
+	if (Array.isArray(expected)) return Array.isArray(actual) && actual.length === expected.length && expected.every((item, index) => matchesExpectedArgsPartial(actual[index], item));
+	if (isRecord(expected)) {
+		if (!isRecord(actual)) return false;
+		return Object.entries(expected).every(([key, expectedValue]) => Object.prototype.hasOwnProperty.call(actual, key) && matchesExpectedArgsPartial(actual[key], expectedValue));
+	}
+	return (0, node_util.isDeepStrictEqual)(actual, expected);
+}
+function matchesToolArgs(actual, expected, mode) {
+	if (mode === "exact") return (0, node_util.isDeepStrictEqual)(actual, expected);
+	return matchesExpectedArgsPartial(actual, expected);
+}
+function resolveToolArgsMatchMode(mode) {
+	if (mode === void 0) return "partial";
+	if (mode === "partial" || mode === "exact") return mode;
+	throw new Error("trajectory:tool-args-match assertion mode must be \"partial\" or \"exact\"");
+}
+function resolveToolArgsMatchValue(value) {
+	if (!value || typeof value !== "object" || Array.isArray(value)) throw new Error("trajectory:tool-args-match assertion must have an object value");
+	const matcher = normalizeTrajectoryMatcher(value, "tool");
+	requireNamedTrajectoryMatcher(matcher, "trajectory:tool-args-match");
+	const expectedArgs = Object.prototype.hasOwnProperty.call(value, "args") ? value.args : value.arguments;
+	if (expectedArgs === void 0) throw new Error("trajectory:tool-args-match assertion must include an args or arguments property");
+	return {
+		matcher,
+		expectedArgs,
+		mode: resolveToolArgsMatchMode(value.mode)
+	};
+}
+const handleTrajectoryToolSequence = (params) => {
+	const toolSteps = extractTrajectorySteps(getTraceOrThrow(params)).filter((step) => step.type === "tool");
+	const value = resolveSequenceValue(params.renderedValue ?? params.assertion.value);
+	const expectedMatchers = value.steps.map((step, index) => {
+		const matcher = normalizeTrajectoryMatcher(step, "tool");
+		requireNamedTrajectoryMatcher(matcher, "trajectory:tool-sequence", index);
+		return matcher;
+	});
+	if (expectedMatchers.length === 0) throw new Error("trajectory:tool-sequence assertion requires at least one expected step");
+	const actualTools = toolSteps.map(formatTrajectoryStep);
+	let basePass = false;
+	let reason = "";
+	if (value.mode === "exact") {
+		basePass = toolSteps.length === expectedMatchers.length && expectedMatchers.every((matcher, index) => matchesTrajectoryStep(toolSteps[index], matcher));
+		if (basePass) reason = `Observed exact tool sequence: ${formatStepList(actualTools)}`;
+		else reason = `Expected exact tool sequence of ${expectedMatchers.map((matcher) => matcher.pattern || matcher.name || "*").join(", ")}, but actual tools were ${formatStepList(actualTools)}`;
+	} else {
+		let expectedIndex = 0;
+		const matchedSteps = [];
+		for (const step of toolSteps) {
+			if (expectedIndex >= expectedMatchers.length) break;
+			if (matchesTrajectoryStep(step, expectedMatchers[expectedIndex])) {
+				matchedSteps.push(formatTrajectoryStep(step));
+				expectedIndex += 1;
+			}
+		}
+		basePass = expectedIndex === expectedMatchers.length;
+		if (basePass) reason = `Observed tool sequence in order: ${matchedSteps.join(", ")}. Actual tools: ${formatStepList(actualTools)}`;
+		else reason = `Expected tool "${expectedMatchers[expectedIndex]?.pattern || expectedMatchers[expectedIndex]?.name || "*"}" was not observed in order. Actual tools: ${formatStepList(actualTools)}`;
+	}
+	const pass = applyInverse(basePass, params.inverse);
+	if (params.inverse) reason = basePass ? `Forbidden tool sequence was observed. Actual tools: ${formatStepList(actualTools)}` : `Forbidden tool sequence was not observed`;
+	return {
+		pass,
+		score: pass ? 1 : 0,
+		reason,
+		assertion: params.assertion
+	};
+};
+const handleTrajectoryToolArgsMatch = (params) => {
+	const toolSteps = extractTrajectorySteps(getTraceOrThrow(params)).filter((step) => step.type === "tool");
+	const { matcher, expectedArgs, mode } = resolveToolArgsMatchValue(params.renderedValue ?? params.assertion.value);
+	const matcherLabel = matcher.pattern || matcher.name || "*";
+	const actualTools = toolSteps.map(formatTrajectoryStep);
+	const matchingSteps = toolSteps.filter((step) => matchesTrajectoryStep(step, matcher));
+	const stepsWithArgs = matchingSteps.filter((step) => step.args !== void 0);
+	const matchedStep = stepsWithArgs.find((step) => matchesToolArgs(step.args, expectedArgs, mode));
+	const basePass = matchedStep !== void 0;
+	const pass = applyInverse(basePass, params.inverse);
+	const expectedArgsLabel = formatTrajectoryArgs(expectedArgs);
+	const observedArgsLabel = stepsWithArgs.length > 0 ? stepsWithArgs.map((step) => formatTrajectoryArgs(step.args)).join(", ") : "(none)";
+	let reason;
+	if (params.inverse) if (basePass) reason = `Forbidden argument match for tool "${matcherLabel}" was observed on ${formatTrajectoryStep(matchedStep)}. Args: ${formatTrajectoryArgs(matchedStep.args)}`;
+	else if (matchingSteps.length === 0) reason = `Forbidden argument match for tool "${matcherLabel}" was not observed because no tool call matched it`;
+	else reason = `Forbidden argument match for tool "${matcherLabel}" was not observed. Observed args: ${observedArgsLabel}`;
+	else if (basePass) reason = `Tool "${matcherLabel}" matched expected arguments (${mode}) on ${formatTrajectoryStep(matchedStep)}. Args: ${formatTrajectoryArgs(matchedStep.args)}`;
+	else if (matchingSteps.length === 0) reason = `No tool call matched "${matcherLabel}". Actual tools: ${formatStepList(actualTools)}`;
+	else if (stepsWithArgs.length === 0) reason = `Tool "${matcherLabel}" was observed but no arguments were captured. Actual tools: ${formatStepList(actualTools)}`;
+	else reason = `No call to tool "${matcherLabel}" matched expected arguments (${mode}): ${expectedArgsLabel}. Observed args: ${observedArgsLabel}`;
+	return {
+		pass,
+		score: pass ? 1 : 0,
+		reason,
+		assertion: params.assertion
+	};
+};
+function resolveStepCountValue(value) {
+	if (!value || typeof value !== "object" || Array.isArray(value)) throw new Error("trajectory:step-count assertion must have an object value");
+	return {
+		...normalizeTrajectoryMatcher(value),
+		max: typeof value.max === "number" ? value.max : void 0,
+		min: typeof value.min === "number" ? value.min : void 0
+	};
+}
+const handleTrajectoryStepCount = (params) => {
+	const steps = extractTrajectorySteps(getTraceOrThrow(params));
+	const matcher = resolveStepCountValue(params.renderedValue ?? params.assertion.value);
+	const { min, max } = matcher;
+	if (min === void 0 && max === void 0) throw new Error("trajectory:step-count assertion must include a min or max property");
+	const matchingSteps = steps.filter((step) => matchesTrajectoryStep(step, matcher));
+	const count = matchingSteps.length;
+	const basePass = (min === void 0 || count >= min) && (max === void 0 || count <= max);
+	const pass = applyInverse(basePass, params.inverse);
+	const filterParts = [];
+	if (matcher.type) {
+		const types = Array.isArray(matcher.type) ? matcher.type : [matcher.type];
+		filterParts.push(`type=${types.join("|")}`);
+	}
+	const pattern = matcher.pattern || matcher.name;
+	if (pattern) filterParts.push(`pattern=${pattern}`);
+	let reason = `Matched ${count} trajectory step(s)`;
+	if (filterParts.length > 0) reason += ` for ${filterParts.join(", ")}`;
+	if (min !== void 0 && max !== void 0) reason += ` (expected ${min}-${max})`;
+	else if (min !== void 0) reason += ` (expected at least ${min})`;
+	else if (max !== void 0) reason += ` (expected at most ${max})`;
+	if (matchingSteps.length > 0) reason += `. Matches: ${matchingSteps.map(formatTrajectoryStep).join(", ")}`;
+	if (params.inverse) reason = basePass ? `Trajectory step count satisfied the forbidden range` : `Trajectory step count did not satisfy the forbidden range`;
+	return {
+		pass,
+		score: pass ? 1 : 0,
+		reason,
+		assertion: params.assertion
+	};
+};
+const handleTrajectoryGoalSuccess = async (params) => {
+	const trace = getTraceOrThrow(params);
+	const { goal } = resolveGoalSuccessValue(params.renderedValue ?? params.assertion.value);
+	const result = await require_graders.matchesTrajectoryGoalSuccess(goal, summarizeTrajectoryForJudge(trace), params.outputString, params.test.options, params.assertionValueContext.vars, params.assertion, params.providerCallContext);
+	if (!params.inverse) return result;
+	return {
+		...result,
+		assertion: params.assertion,
+		pass: !result.pass,
+		score: result.pass ? 0 : 1,
+		reason: result.pass ? `Agent unexpectedly achieved the goal: ${goal}` : `Agent did not achieve the forbidden goal: ${goal}`
+	};
+};
+//#endregion
 //#region src/assertions/webhook.ts
 async function handleWebhook({ assertion, renderedValue, test, prompt, output, inverse }) {
 	require_invariant.invariant(renderedValue, "\"webhook\" assertion type must have a URL value");
@@ -2604,18 +3233,18 @@ const handleWordCount = ({ assertion, renderedValue, valueFromScript, outputStri
 			if (pass) reason = "Assertion passed";
 			else if (inverse) reason = `Expected word count to not be between ${min} and ${max}, but got ${wordCount}`;
 			else reason = `Word count ${wordCount} is not between ${min} and ${max}`;
-		} else if (min !== void 0) {
-			const basePass = wordCount >= min;
-			pass = inverse ? !basePass : basePass;
-			if (pass) reason = "Assertion passed";
-			else if (inverse) reason = `Expected word count to be less than ${min}, but got ${wordCount}`;
-			else reason = `Word count ${wordCount} is less than minimum ${min}`;
-		} else {
+		} else if (min === void 0) {
 			const basePass = wordCount <= max;
 			pass = inverse ? !basePass : basePass;
 			if (pass) reason = "Assertion passed";
 			else if (inverse) reason = `Expected word count to be greater than ${max}, but got ${wordCount}`;
 			else reason = `Word count ${wordCount} is greater than maximum ${max}`;
+		} else {
+			const basePass = wordCount >= min;
+			pass = inverse ? !basePass : basePass;
+			if (pass) reason = "Assertion passed";
+			else if (inverse) reason = `Expected word count to be less than ${min}, but got ${wordCount}`;
+			else reason = `Word count ${wordCount} is less than minimum ${min}`;
 		}
 	} else {
 		require_invariant.invariant(typeof value === "number" || typeof value === "string" && !Number.isNaN(Number(value)), "\"word-count\" assertion value must be a number or an object with min/max properties");
@@ -2710,6 +3339,12 @@ const handleIsXml = ({ assertion, renderedValue, outputString, inverse, baseType
 //#endregion
 //#region src/assertions/index.ts
 const ASSERTIONS_MAX_CONCURRENCY = require_logger.getEnvInt("PROMPTFOO_ASSERTIONS_MAX_CONCURRENCY", 3);
+const DEFAULT_TRACE_FETCH_MAX_ATTEMPTS = 6;
+const DEFAULT_TRACE_FETCH_RETRY_DELAY_MS = 250;
+const DEFAULT_TRACE_FETCH_STABLE_POLLS = 2;
+const MAX_TRACE_FETCH_MAX_ATTEMPTS = 30;
+const MAX_TRACE_FETCH_RETRY_DELAY_MS = 5e3;
+const MAX_TRACE_FETCH_STABLE_POLLS = 10;
 const MODEL_GRADED_ASSERTION_TYPES = new Set([
 	"answer-relevance",
 	"context-faithfulness",
@@ -2719,8 +3354,57 @@ const MODEL_GRADED_ASSERTION_TYPES = new Set([
 	"llm-rubric",
 	"model-graded-closedqa",
 	"model-graded-factuality",
-	"search-rubric"
+	"search-rubric",
+	"trajectory:goal-success"
+]);
+const TRACE_AWARE_ASSERTION_TYPES = new Set([
+	"javascript",
+	"python",
+	"ruby",
+	"trace-error-spans",
+	"trace-span-count",
+	"trace-span-duration",
+	"trajectory:goal-success",
+	"trajectory:step-count",
+	"trajectory:tool-args-match",
+	"trajectory:tool-sequence",
+	"trajectory:tool-used"
 ]);
+function assertionUsesTrace(assertion) {
+	if (assertion.type === "assert-set") return assertion.assert.some(assertionUsesTrace);
+	return TRACE_AWARE_ASSERTION_TYPES.has(getAssertionBaseType(assertion));
+}
+function assertionMayNeedTraceContext(assertion) {
+	if (assertionUsesTrace(assertion)) return true;
+	if (assertion.type === "assert-set") return assertion.assert.some(assertionMayNeedTraceContext);
+	return typeof assertion.value === "string" ? assertion.value.startsWith("file://") || require_providers.isPackagePath(assertion.value) : false;
+}
+function hasTraceAwareAssertions(assertions) {
+	return Boolean(assertions?.some(assertionMayNeedTraceContext));
+}
+async function loadTraceData(traceId) {
+	const traceStore = require_store.getTraceStore();
+	const maxAttempts = Math.min(MAX_TRACE_FETCH_MAX_ATTEMPTS, Math.max(1, require_logger.getEnvInt("PROMPTFOO_TRACE_FETCH_MAX_ATTEMPTS", DEFAULT_TRACE_FETCH_MAX_ATTEMPTS)));
+	const retryDelayMs = Math.min(MAX_TRACE_FETCH_RETRY_DELAY_MS, Math.max(0, require_logger.getEnvInt("PROMPTFOO_TRACE_FETCH_RETRY_DELAY_MS", DEFAULT_TRACE_FETCH_RETRY_DELAY_MS)));
+	const stablePolls = Math.min(MAX_TRACE_FETCH_STABLE_POLLS, Math.max(1, require_logger.getEnvInt("PROMPTFOO_TRACE_FETCH_STABLE_POLLS", DEFAULT_TRACE_FETCH_STABLE_POLLS)));
+	let lastSpanCount = -1;
+	let stableObservations = 0;
+	let latestTrace = null;
+	for (let attempt = 0; attempt < maxAttempts; attempt++) {
+		latestTrace = await traceStore.getTrace(traceId);
+		const spanCount = latestTrace?.spans?.length ?? 0;
+		if (spanCount > 0) {
+			stableObservations = spanCount === lastSpanCount ? stableObservations + 1 : 1;
+			lastSpanCount = spanCount;
+			if (stableObservations >= stablePolls || attempt === maxAttempts - 1) return latestTrace;
+		} else {
+			stableObservations = 0;
+			lastSpanCount = spanCount;
+		}
+		if (attempt < maxAttempts - 1) await require_fetch.sleep(retryDelayMs);
+	}
+	return latestTrace;
+}
 const ASSERTION_HANDLERS = {
 	"answer-relevance": handleAnswerRelevance,
 	bleu: handleBleuScore,
@@ -2783,12 +3467,18 @@ const ASSERTION_HANDLERS = {
 	ruby: handleRuby,
 	"rouge-n": handleRougeScore,
 	"search-rubric": handleSearchRubric,
+	"skill-used": handleSkillUsed,
 	similar: handleSimilar,
 	"similar:cosine": handleSimilar,
 	"similar:dot": handleSimilar,
 	"similar:euclidean": handleSimilar,
 	"starts-with": handleStartsWith,
 	"tool-call-f1": handleToolCallF1,
+	"trajectory:goal-success": handleTrajectoryGoalSuccess,
+	"trajectory:tool-args-match": handleTrajectoryToolArgsMatch,
+	"trajectory:step-count": handleTrajectoryStepCount,
+	"trajectory:tool-sequence": handleTrajectoryToolSequence,
+	"trajectory:tool-used": handleTrajectoryToolUsed,
 	"trace-error-spans": handleTraceErrorSpans,
 	"trace-span-count": handleTraceSpanCount,
 	"trace-span-duration": handleTraceSpanDuration,
@@ -2831,7 +3521,7 @@ function isAssertionInverse(assertion) {
 function getAssertionBaseType(assertion) {
 	return isAssertionInverse(assertion) ? assertion.type.slice(4) : assertion.type;
 }
-async function runAssertion({ prompt, provider, assertion, test, vars, latencyMs, providerResponse, traceId }) {
+async function runAssertion({ prompt, provider, assertion, test, vars, latencyMs, providerResponse, traceId, traceData }) {
 	const resolvedVars = vars || test.vars || {};
 	const { cost, logProbs, output: originalOutput } = providerResponse;
 	let output = originalOutput;
@@ -2850,14 +3540,14 @@ async function runAssertion({ prompt, provider, assertion, test, vars, latencyMs
 		providerResponse,
 		...assertion.config ? { config: structuredClone(assertion.config) } : {}
 	};
-	if (traceId) try {
-		const traceData = await require_store.getTraceStore().getTrace(traceId);
-		if (traceData) context.trace = {
-			traceId: traceData.traceId,
-			evaluationId: traceData.evaluationId,
-			testCaseId: traceData.testCaseId,
-			metadata: traceData.metadata,
-			spans: traceData.spans || []
+	if (traceId && assertionMayNeedTraceContext(assertion)) try {
+		const resolvedTraceData = traceData === void 0 ? await loadTraceData(traceId) : traceData;
+		if (resolvedTraceData) context.trace = {
+			traceId: resolvedTraceData.traceId,
+			evaluationId: resolvedTraceData.evaluationId,
+			testCaseId: resolvedTraceData.testCaseId,
+			metadata: resolvedTraceData.metadata,
+			spans: resolvedTraceData.spans || []
 		};
 	} catch (error) {
 		require_logger.logger.debug(`Failed to fetch trace data for assertion: ${error}`);
@@ -2890,7 +3580,7 @@ async function runAssertion({ prompt, provider, assertion, test, vars, latencyMs
 			};
 		}
 		else if (filePath.endsWith(".rb")) try {
-			const { runRuby } = await Promise.resolve().then(() => require("./rubyUtils-CP42kMvq.cjs"));
+			const { runRuby } = await Promise.resolve().then(() => require("./rubyUtils-B1HXG4ej.cjs"));
 			valueFromScript = await runRuby(filePath, functionName || "get_assert", [output, context]);
 			require_logger.logger.debug(`Ruby script ${filePath} output: ${valueFromScript}`);
 		} catch (error) {
@@ -2999,6 +3689,14 @@ async function runAssertions({ assertScoringFunction, latencyMs, prompt, provide
 			index: i
 		};
 	}).flat();
+	const shouldPreloadTrace = !!traceId && hasTraceAwareAssertions(asserts.map(({ assertion }) => assertion));
+	let preloadedTraceData;
+	if (shouldPreloadTrace && traceId) try {
+		preloadedTraceData = await loadTraceData(traceId);
+	} catch (error) {
+		require_logger.logger.debug(`Failed to preload trace data for assertions: ${error}`);
+		preloadedTraceData = null;
+	}
 	await async.default.forEachOfLimit(asserts, ASSERTIONS_MAX_CONCURRENCY, async ({ assertion, assertResult, index }) => {
 		if (assertion.type.startsWith("select-") || assertion.type === "max-score") return;
 		const result = await runAssertion({
@@ -3010,7 +3708,8 @@ async function runAssertions({ assertScoringFunction, latencyMs, prompt, provide
 			vars,
 			latencyMs,
 			assertIndex: index,
-			traceId
+			traceId,
+			traceData: preloadedTraceData
 		});
 		assertResult.addResult({
 			index,
@@ -3156,7 +3855,7 @@ var CIProgressReporter = class {
 		else {
 			const eta = remaining / rate;
 			if (eta > 1440) etaDisplay = ">24 hours";
-			else etaDisplay = `${Math.round(eta)} minute${Math.round(eta) !== 1 ? "s" : ""}`;
+			else etaDisplay = `${Math.round(eta)} minute${Math.round(eta) === 1 ? "" : "s"}`;
 		}
 		const percentage = Math.floor(this.completedTests / this.totalTests * 100);
 		require_logger.logger.info(`[CI Progress] Evaluation running for ${this.formatElapsedTime(elapsed)} - Completed ${this.completedTests}/${this.totalTests} tests (${percentage}%)`);
@@ -3557,12 +4256,55 @@ function isPromptAllowed(prompt, allowedPrompts) {
 var ProgressBarManager = class {
 	progressBar;
 	isWebUI;
+	originalLogCallback = null;
+	installedLogCallback = null;
+	pendingRender = null;
 	totalCount = 0;
 	completedCount = 0;
 	concurrency = 1;
 	constructor(isWebUI) {
 		this.isWebUI = isWebUI;
 	}
+	clearProgressBarLine() {
+		readline.default.cursorTo(process.stderr, 0);
+		readline.default.clearLine(process.stderr, 0);
+	}
+	scheduleRender() {
+		if (!this.progressBar || this.pendingRender) return;
+		this.pendingRender = setImmediate(() => {
+			this.pendingRender = null;
+			this.progressBar?.render();
+		});
+	}
+	handleLogMessage() {
+		if (!this.progressBar) return;
+		this.clearProgressBarLine();
+		this.scheduleRender();
+	}
+	/**
+	* Coordinate console logging with the progress bar to prevent visual corruption.
+	*/
+	installLogInterceptor() {
+		if (!this.progressBar || this.isWebUI || this.installedLogCallback) return;
+		this.originalLogCallback = require_logger.globalLogCallback;
+		this.installedLogCallback = (message) => {
+			this.originalLogCallback?.(message);
+			this.handleLogMessage();
+		};
+		require_logger.setLogCallback(this.installedLogCallback);
+	}
+	/**
+	* Remove the log interceptor and restore original logger callback behavior.
+	*/
+	removeLogInterceptor() {
+		if (this.pendingRender) {
+			clearImmediate(this.pendingRender);
+			this.pendingRender = null;
+		}
+		if (this.installedLogCallback && require_logger.globalLogCallback === this.installedLogCallback) require_logger.setLogCallback(this.originalLogCallback);
+		this.installedLogCallback = null;
+		this.originalLogCallback = null;
+	}
 	/**
 	* Initialize progress bar
 	*/
@@ -3582,7 +4324,8 @@ var ProgressBarManager = class {
 				return `Evaluating [${bar}${spaces}] ${percentage}% | ${params.value}/${params.total}${errorsText} | ${payload.provider} ${payload.prompt} ${payload.vars}`;
 			},
 			hideCursor: true,
-			gracefulExit: true
+			gracefulExit: true,
+			stream: process.stderr
 		}, cli_progress.default.Presets.shades_classic);
 		this.progressBar.start(this.totalCount, 0, {
 			provider: "",
@@ -3857,6 +4600,7 @@ async function runEval({ provider, prompt, test, testSuite, delay, nunjucksFilte
 				const parts = traceContext.traceparent.split("-");
 				if (parts.length >= 3) traceId = parts[1];
 			}
+			if (traceId && hasTraceAwareAssertions(test.assert)) await flushOtel();
 			const checkResult = await runAssertions({
 				prompt: renderedPrompt,
 				provider,
@@ -4254,7 +4998,7 @@ var Evaluator = class {
 				const defaultProvider = testSuite.defaultTest.provider;
 				if (require_types.isApiProvider(defaultProvider)) testCase.provider = defaultProvider;
 				else if (typeof defaultProvider === "object" && defaultProvider.id) {
-					const { loadApiProvider } = await Promise.resolve().then(() => require("./providers-CxmDwEFf.cjs"));
+					const { loadApiProvider } = await Promise.resolve().then(() => require("./providers-D-FnDg8k.cjs"));
 					testCase.provider = await loadApiProvider(typeof defaultProvider.id === "function" ? defaultProvider.id() : defaultProvider.id, { options: defaultProvider });
 				} else testCase.provider = defaultProvider;
 			}
@@ -4338,7 +5082,7 @@ var Evaluator = class {
 			if (evalOption.test.assert?.some((a) => a.type === "max-score")) rowsWithMaxScoreAssertion.add(evalOption.testIdx);
 		}
 		if (require_logger.state.resume && this.evalRecord.persisted) try {
-			const { default: EvalResult } = await Promise.resolve().then(() => require("./evalResult-DvcJAWJU.cjs"));
+			const { default: EvalResult } = await Promise.resolve().then(() => require("./evalResult-tGdilrWt.cjs"));
 			const completedPairs = await EvalResult.getCompletedIndexPairs(this.evalRecord.id, { excludeErrors: require_logger.state.retryMode });
 			const originalCount = runEvalOptions.length;
 			for (let i = runEvalOptions.length - 1; i >= 0; i--) {
@@ -4538,7 +5282,7 @@ var Evaluator = class {
 		if (require_logger.isCI() && !isWebUI) {
 			ciProgressReporter = new CIProgressReporter(runEvalOptions.length);
 			ciProgressReporter.start();
-		} else if (this.options.showProgressBar && process.stdout.isTTY) progressBarManager = new ProgressBarManager(isWebUI);
+		} else if (this.options.showProgressBar && process.stderr.isTTY) progressBarManager = new ProgressBarManager(isWebUI);
 		this.options.progressCallback = (completed, total, index, evalStep, metrics) => {
 			if (originalProgressCallback) originalProgressCallback(completed, total, index, evalStep, metrics);
 			if (isWebUI) {
@@ -4559,7 +5303,10 @@ var Evaluator = class {
 			if (serialRunEvalOptions.length > 0) require_logger.logger.info(`Running ${serialRunEvalOptions.length} test cases serially...`);
 			if (concurrentRunEvalOptions.length > 0) require_logger.logger.info(`Running ${concurrentRunEvalOptions.length} test cases (up to ${concurrency} at a time)...`);
 		}
-		if (this.options.showProgressBar && progressBarManager) await progressBarManager.initialize(runEvalOptions, concurrency, 0);
+		if (this.options.showProgressBar && progressBarManager) {
+			await progressBarManager.initialize(runEvalOptions, concurrency, 0);
+			progressBarManager.installLogInterceptor();
+		}
 		try {
 			if (serialRunEvalOptions.length > 0) for (const evalStep of serialRunEvalOptions) {
 				checkAbort();
@@ -4585,7 +5332,10 @@ var Evaluator = class {
 				else if (!targetUnavailable) {
 					require_logger.logger.info("Evaluation interrupted, saving progress...");
 					if (globalTimeout) clearTimeout(globalTimeout);
-					if (progressBarManager) progressBarManager.stop();
+					if (progressBarManager) {
+						progressBarManager.removeLogInterceptor();
+						progressBarManager.stop();
+					}
 					if (ciProgressReporter) ciProgressReporter.finish();
 					this.evalRecord.setVars(Array.from(vars));
 					await this.evalRecord.addPrompts(prompts);
@@ -4593,6 +5343,10 @@ var Evaluator = class {
 					return this.evalRecord;
 				}
 			} else {
+				if (progressBarManager) {
+					progressBarManager.removeLogInterceptor();
+					progressBarManager.stop();
+				}
 				if (ciProgressReporter) ciProgressReporter.error(`Evaluation failed: ${String(err)}`);
 				throw err;
 			}
@@ -4735,6 +5489,7 @@ var Evaluator = class {
 		await this.evalRecord.addPrompts(prompts);
 		try {
 			if (progressBarManager) {
+				progressBarManager.removeLogInterceptor();
 				progressBarManager.complete();
 				progressBarManager.stop();
 			} else if (ciProgressReporter) ciProgressReporter.finish();
@@ -7088,8 +7843,7 @@ function testCaseFromCsvRow(row) {
 			require_logger.logger.warn("The \"__metadata\" column requires a key, e.g. \"__metadata:category\". This column will be ignored.");
 		} else if (key.startsWith("__config:")) {
 			const configParts = key.slice(9).split(":");
-			if (configParts.length !== 2) require_logger.logger.warn(`Invalid __config column format: "${key}". Expected format: __config:__expected:threshold or __config:__expected<N>:threshold`);
-			else {
+			if (configParts.length === 2) {
 				const [expectedKey, configKey] = configParts;
 				let targetIndex;
 				if (expectedKey === "__expected") targetIndex = 0;
@@ -7115,7 +7869,7 @@ function testCaseFromCsvRow(row) {
 					}
 				}
 				assertionConfigs[targetIndex][configKey] = parsedValue;
-			}
+			} else require_logger.logger.warn(`Invalid __config column format: "${key}". Expected format: __config:__expected:threshold or __config:__expected<N>:threshold`);
 		} else vars[key] = value;
 	}
 	for (let i = 0; i < asserts.length; i++) {
@@ -7244,14 +7998,14 @@ async function parseXlsxFile(filePath) {
 		const sheetName = typeof sheetOption === "number" ? sheetNames[sheetOption - 1] : sheetOption;
 		const rows = await readXlsxFile(actualFilePath, { sheet: sheetOption });
 		if (rows.length === 0) throw new Error(`Sheet "${sheetName}" is empty or contains no valid data rows`);
-		const headers = rows[0].map((cell) => cell != null ? String(cell) : "");
+		const headers = rows[0].map((cell) => cell == null ? "" : String(cell));
 		if (headers.length === 0 || headers.every((h) => h === "")) throw new Error(`Sheet "${sheetName}" has no valid column headers`);
 		if (rows.length === 1) throw new Error(`Sheet "${sheetName}" is empty or contains no valid data rows`);
 		const data = rows.slice(1).map((row) => {
 			const obj = {};
 			headers.forEach((header, index) => {
 				const cellValue = row[index];
-				obj[header] = cellValue != null ? String(cellValue) : "";
+				obj[header] = cellValue == null ? "" : String(cellValue);
 			});
 			return obj;
 		});
@@ -11198,20 +11952,19 @@ function generateEvalSummary(params) {
 		}
 	}
 	lines.push("");
-	const passRate = successes / (successes + failures + errors) * 100;
-	let passRateDisplay;
-	if (!Number.isNaN(passRate)) {
-		const passRateFormatted = passRate === 0 || passRate === 100 ? `${passRate.toFixed(0)}%` : `${passRate.toFixed(2)}%`;
-		if (passRate >= 100) passRateDisplay = chalk.default.green.bold(passRateFormatted);
-		else if (passRate >= 80) passRateDisplay = chalk.default.yellow.bold(passRateFormatted);
-		else passRateDisplay = chalk.default.red.bold(passRateFormatted);
-	}
-	const passedPart = successes > 0 ? `${chalk.default.green("✓")} ${chalk.default.green.bold(successes.toLocaleString())} passed` : `${chalk.default.gray.bold(successes.toLocaleString())} passed`;
-	const failedPart = failures > 0 ? `${chalk.default.red("✗")} ${chalk.default.red.bold(failures.toLocaleString())} failed` : `${chalk.default.gray.bold(failures.toLocaleString())} failed`;
+	const totalTests = successes + failures + errors;
+	const formatResultPercentage = (count) => {
+		const percentage = totalTests === 0 ? 0 : count / totalTests * 100;
+		return percentage === 0 || percentage === 100 ? `${percentage.toFixed(0)}%` : `${percentage.toFixed(2)}%`;
+	};
+	const formatResultLine = (count, label, icon, iconColor) => {
+		return `  ${icon ? `${iconColor(icon)} ` : ""}${chalk.default.white.bold(count.toLocaleString())} ${chalk.default.white(label)} ${chalk.default.gray(`(${formatResultPercentage(count)})`)}`;
+	};
 	const errorLabel = errors === 1 ? "error" : "errors";
-	const resultsLine = `${passedPart}, ${failedPart}, ${errors > 0 ? `${chalk.default.red("✗")} ${chalk.default.red.bold(errors.toLocaleString())} ${errorLabel}` : `${chalk.default.gray.bold(errors.toLocaleString())} ${errorLabel}`}`;
-	if (Number.isNaN(passRate)) lines.push(`${chalk.default.bold("Results:")} ${resultsLine}`);
-	else lines.push(`${chalk.default.bold("Results:")} ${resultsLine} (${passRateDisplay})`);
+	lines.push(chalk.default.bold("Results:"));
+	lines.push(formatResultLine(successes, "passed", successes > 0 ? "✓" : void 0, chalk.default.green));
+	lines.push(formatResultLine(failures, "failed", failures > 0 ? "✗" : void 0, chalk.default.red));
+	lines.push(formatResultLine(errors, errorLabel, errors > 0 ? "✗" : void 0, chalk.default.red));
 	const durationDisplay = formatDuration(duration);
 	lines.push(chalk.default.gray(`Duration: ${durationDisplay} (concurrency: ${maxConcurrency})`));
 	lines.push("");
@@ -11545,7 +12298,7 @@ async function doEval(cmdObj, defaultConfig, defaultConfigPath, evaluateOptions)
 		await require_providers.checkCloudPermissions(config);
 		const options = {
 			...evaluateOptions,
-			showProgressBar: require_logger.getLogLevel() === "debug" ? false : cmdObj.progressBar !== void 0 ? cmdObj.progressBar !== false : evaluateOptions.showProgressBar !== void 0 ? evaluateOptions.showProgressBar : true,
+			showProgressBar: require_logger.getLogLevel() === "debug" ? false : cmdObj.progressBar === void 0 ? evaluateOptions.showProgressBar === void 0 ? true : evaluateOptions.showProgressBar : cmdObj.progressBar !== false,
 			repeat,
 			delay: !Number.isNaN(delay) && delay > 0 ? delay : void 0,
 			maxConcurrency,
@@ -11929,7 +12682,7 @@ async function doRedteamRun(options) {
 		redteamConfig = await doGenerateRedteam({
 			...passThroughOptions,
 			...options.liveRedteamConfig?.commandLineOptions || {},
-			...maxConcurrency !== void 0 ? { maxConcurrency } : {},
+			...maxConcurrency === void 0 ? {} : { maxConcurrency },
 			config: configPath,
 			output: redteamPath,
 			force: options.force,