promptfoo 0.121.1 → 0.121.3

package/dist/src/{evaluator-DPFRbFIL.js → evaluator-BcvOGaam.js}

@@ -1,34 +1,35 @@
 #!/usr/bin/env node
-import { C as getEnvString, E as isCI, O as state, S as getEnvInt, T as getMaxEvalTimeMs, _ as summarizeEvaluateResultForLogging, b as getEnvBool, f as extractJsonObjects, g as safeJsonStringify, o as logger, p as getAjv, w as getEvalTimeoutMs } from "./logger-KkObSCzq.js";
-import { N as VERSION, P as FILE_METADATA_KEY, g as isPromptfooSampleTarget, l as sleep, r as fetchWithRetries, y as parseChatPrompt } from "./fetch-BMv0O527.js";
+import { C as getEnvBool, D as getEvalTimeoutMs, E as getEnvString, O as getMaxEvalTimeMs, T as getEnvInt, b as summarizeEvaluateResultForLogging, c as setLogCallback, g as getAjv, h as extractJsonObjects, j as state, k as isCI, r as globalLogCallback, s as logger, y as safeJsonStringify } from "./logger-BcJBzSSA.js";
+import { N as VERSION, P as FILE_METADATA_KEY, g as isPromptfooSampleTarget, l as sleep, r as fetchWithRetries, y as parseChatPrompt } from "./fetch-DoVRJZhJ.js";
 import { t as invariant } from "./invariant-BtWWVVhl.js";
-import { r as telemetry } from "./telemetry-C2YDkUQH.js";
-import { d as isGradingResult, p as isApiProvider, s as ResultFailureReason } from "./types-CH3Ge2sE.js";
-import { c as promptYesNo } from "./server-DbFphssR.js";
-import { A as renderPrompt, E as isBasicRefusal, F as TokenUsageTracker, G as VertexChatProvider, I as createRateLimitRegistry, K as AIStudioChatProvider, L as createProviderRateLimitOptions, M as isPackagePath, N as loadFromPackage, P as redteamProviderManager, j as runExtensionHook, k as collectFileMetadata, u as GoogleLiveProvider, v as checkExfilTracking, w as getSessionId } from "./providers-CFLy1_ji.js";
-import { o as getCache } from "./cache-CG0SlR1d.js";
+import { r as telemetry } from "./telemetry-BnH5VJAU.js";
+import { d as isGradingResult, p as isApiProvider, s as ResultFailureReason } from "./types-CIhFeUC4.js";
+import { c as promptYesNo } from "./server-DZ9MtCn0.js";
+import { A as renderPrompt, E as isBasicRefusal, F as TokenUsageTracker, G as VertexChatProvider, I as createRateLimitRegistry, K as AIStudioChatProvider, L as createProviderRateLimitOptions, M as isPackagePath, N as loadFromPackage, P as redteamProviderManager, j as runExtensionHook, k as collectFileMetadata, u as GoogleLiveProvider, v as checkExfilTracking, w as getSessionId } from "./providers-Ch6Mr0gn.js";
+import { o as getCache } from "./cache-DSqR6ezl.js";
 import { n as isNonTransientHttpStatus } from "./errors-P6ll7XSJ.js";
 import { i as isJavascriptFile } from "./fileExtensions-Ds-foDzt.js";
-import { E as parseFileUrl, I as isAnthropicProvider, L as isGoogleProvider, R as isOpenAiProvider, T as loadFunction, g as maybeLoadToolsFromExternalFile, w as getNunjucksEngine, z as isProviderAllowed } from "./util-YT5HPZaS.js";
-import { r as runPython } from "./pythonUtils-C3py6GC1.js";
-import { n as transform, r as getProcessShim, t as TransformInputType } from "./transform-DilY9wbS.js";
-import { $ as matchesSearchRubric, B as getAndCheckProvider, G as matchesContextFaithfulness, H as matchesAnswerRelevance, J as matchesFactuality, K as matchesContextRecall, Q as matchesPiScore, R as callProviderWithContext, U as matchesClassification, V as loadRubricPrompt, W as matchesClosedQa, X as matchesLlmRubric, Y as matchesGEval, Z as matchesModeration, at as DefaultSuggestionsProvider, dt as getFinalTest, et as matchesSelectBest, ft as loadFromJavaScriptFile, it as getDefaultProviders, lt as SUGGEST_PROMPTS_SYSTEM_MESSAGE, mt as resolveContext, n as getGraderById, nt as selectMaxScore, pt as processFileReference, q as matchesContextRelevance, tt as matchesSimilarity, ut as coerceString, z as fail } from "./graders-CHO8EPM4.js";
-import { i as generateIdFromPrompt } from "./utils-f2-Moju7.js";
-import { t as OpenAiChatCompletionProvider } from "./chat-CRWNNq73.js";
+import { E as parseFileUrl, I as isAnthropicProvider, L as isGoogleProvider, R as isOpenAiProvider, T as loadFunction, g as maybeLoadToolsFromExternalFile, w as getNunjucksEngine, z as isProviderAllowed } from "./util-Bm3E9jpK.js";
+import { r as runPython } from "./pythonUtils-wIqk7zAf.js";
+import { n as transform, r as getProcessShim, t as TransformInputType } from "./transform-ljLYHEPh.js";
+import { $ as matchesSearchRubric, B as getAndCheckProvider, G as matchesContextFaithfulness, H as matchesAnswerRelevance, J as matchesFactuality, K as matchesContextRecall, Q as matchesPiScore, R as callProviderWithContext, U as matchesClassification, V as loadRubricPrompt, W as matchesClosedQa, X as matchesLlmRubric, Y as matchesGEval, Z as matchesModeration, at as getDefaultProviders, dt as coerceString, et as matchesSelectBest, ft as getFinalTest, ht as resolveContext, mt as processFileReference, n as getGraderById, nt as matchesTrajectoryGoalSuccess, ot as DefaultSuggestionsProvider, pt as loadFromJavaScriptFile, q as matchesContextRelevance, rt as selectMaxScore, tt as matchesSimilarity, ut as SUGGEST_PROMPTS_SYSTEM_MESSAGE, z as fail } from "./graders-RjHF8VfG.js";
+import { i as generateIdFromPrompt } from "./utils-DEuL4VNB.js";
+import { t as OpenAiChatCompletionProvider } from "./chat-BEwdgGEg.js";
 import { a as createEmptyTokenUsage, i as createEmptyAssertions, n as accumulateResponseTokenUsage, o as normalizeTokenUsage, r as accumulateTokenUsage, t as accumulateAssertionTokenUsage } from "./tokenUsageUtils-DflFMjS0.js";
-import { m as validateFunctionCall } from "./transform-Cgi24fJ7.js";
-import { l as validateFunctionCall$1 } from "./util-C-PPYSMq.js";
-import { t as extractAndStoreBinaryData } from "./extractor-CFG6bcWJ.js";
-import { n as getTraceStore } from "./store-D1tv90v3.js";
-import { t as providerRegistry } from "./providerRegistry-B0RUOLI_.js";
-import { n as runRuby } from "./rubyUtils-Boc4HZzX.js";
-import { a as getActualPromptWithFallback, r as updateSignalFile } from "./signal-BOTbd53Z.js";
+import { m as validateFunctionCall } from "./transform-DrleutM3.js";
+import { l as validateFunctionCall$1 } from "./util-DM2rTn_6.js";
+import { t as extractAndStoreBinaryData } from "./extractor-CAZ2G3Kh.js";
+import { n as getTraceStore } from "./store-P8OKm19S.js";
+import { t as providerRegistry } from "./providerRegistry-B9lh-_tx.js";
+import { n as runRuby } from "./rubyUtils-CiVfln3g.js";
+import { a as getActualPromptWithFallback, r as updateSignalFile } from "./signal-C3ZTsUgi.js";
 import chalk from "chalk";
 import fs, { createWriteStream } from "fs";
 import path from "path";
 import os from "os";
 import yaml from "js-yaml";
 import util from "util";
+import readline from "readline";
 import { randomBytes } from "crypto";
 import { globSync } from "glob";
 import { XMLParser } from "fast-xml-parser";
@@ -38,6 +39,7 @@ import cliProgress from "cli-progress";
 import { JSDOM } from "jsdom";
 import { distance } from "fastest-levenshtein";
 import * as rouge from "js-rouge";
+import { isDeepStrictEqual } from "node:util";
 import { ExportResultCode, W3CTraceContextPropagator } from "@opentelemetry/core";
 import { OTLPTraceExporter } from "@opentelemetry/exporter-trace-otlp-http";
 import { resourceFromAttributes } from "@opentelemetry/resources";
@@ -256,7 +258,7 @@ async function startOtlpReceiverIfNeeded(testSuite) {
 		telemetry.record("feature_used", { feature: "tracing" });
 		try {
 			logger.debug("[EvaluatorTracing] Tracing configuration detected, starting OTLP receiver");
-			const { startOTLPReceiver } = await import("./otlpReceiver-Bn5wGB1v.js");
+			const { startOTLPReceiver } = await import("./otlpReceiver-CavGAA6k.js");
 			const port = testSuite.tracing.otlp.http.port || 4318;
 			const host = testSuite.tracing.otlp.http.host || "127.0.0.1";
 			logger.debug(`[EvaluatorTracing] Starting OTLP receiver on ${host}:${port}`);
@@ -279,7 +281,7 @@ async function startOtlpReceiverIfNeeded(testSuite) {
 async function stopOtlpReceiverIfNeeded() {
 	if (otlpReceiverStarted) try {
 		logger.debug("[EvaluatorTracing] Stopping OTLP receiver");
-		const { stopOTLPReceiver } = await import("./otlpReceiver-Bn5wGB1v.js");
+		const { stopOTLPReceiver } = await import("./otlpReceiver-CavGAA6k.js");
 		await stopOTLPReceiver();
 		otlpReceiverStarted = false;
 		logger.info("[EvaluatorTracing] OTLP receiver stopped successfully");
@@ -314,7 +316,7 @@ async function generateTraceContextIfNeeded(test, evaluateOptions, testIdx, prom
 	}
 	if (!tracingEnabled) return null;
 	logger.debug("[EvaluatorTracing] Importing trace store");
-	const { getTraceStore } = await import("./store-BltJg2cd.js");
+	const { getTraceStore } = await import("./store-C5u6MgC8.js");
 	const traceStore = getTraceStore();
 	const traceId = generateTraceId();
 	const spanId = generateSpanId();
@@ -1347,7 +1349,7 @@ const handleJavascript = async ({ assertion, renderedValue, valueFromScript, ass
 			pass = result !== inverse;
 			score = pass ? 1 : 0;
 		} else if (typeof result === "number") {
-			pass = assertion.threshold !== void 0 ? result >= assertion.threshold : result > 0;
+			pass = assertion.threshold === void 0 ? result > 0 : result >= assertion.threshold;
 			score = result;
 		} else if (typeof result === "object") return result;
 		else throw new Error("Custom function must return a boolean or number");
@@ -1380,7 +1382,7 @@ function handleIsJson({ outputString, renderedValue, inverse, valueFromScript, a
 	} catch {
 		pass = inverse;
 	}
-	if (pass && renderedValue) {
+	if (parsedJson !== void 0 && renderedValue) {
 		let validate;
 		if (typeof renderedValue === "string") if (renderedValue.startsWith("file://")) {
 			const schema = valueFromScript;
@@ -1392,11 +1394,12 @@ function handleIsJson({ outputString, renderedValue, inverse, valueFromScript, a
 		}
 		else if (typeof renderedValue === "object") validate = getAjv().compile(renderedValue);
 		else throw new Error("is-json assertion must have a string or object value");
-		pass = validate(parsedJson);
+		const valid = validate(parsedJson);
+		pass = inverse ? !valid : valid;
 		if (!pass) return {
 			pass,
 			score: 0,
-			reason: `JSON does not conform to the provided schema. Errors: ${getAjv().errorsText(validate.errors)}`,
+			reason: inverse ? "Output is JSON that conforms to the provided schema" : `JSON does not conform to the provided schema. Errors: ${getAjv().errorsText(validate.errors)}`,
 			assertion
 		};
 	}
@@ -1423,9 +1426,12 @@ function handleContainsJson({ assertion, renderedValue, outputString, inverse, v
 		}
 		else if (typeof renderedValue === "object") validate = getAjv().compile(renderedValue);
 		else throw new Error("contains-json assertion must have a string or object value");
-		pass = validate(jsonObject);
-		if (pass) break;
-		else errorMessage = `JSON does not conform to the provided schema. Errors: ${getAjv().errorsText(validate.errors)}`;
+		const valid = validate(jsonObject);
+		pass = inverse ? !valid : valid;
+		if (valid) {
+			if (inverse) errorMessage = "Output contains JSON conforming to the provided schema";
+			break;
+		} else errorMessage = `JSON does not conform to the provided schema. Errors: ${getAjv().errorsText(validate.errors)}`;
 	}
 	return {
 		pass,
@@ -1567,7 +1573,7 @@ function handlePerplexity({ logProbs, assertion }) {
 	if (!logProbs || logProbs.length === 0) throw new Error("Perplexity assertion does not support providers that do not return logProbs");
 	const avgLogProb = logProbs.reduce((acc, logProb) => acc + logProb, 0) / logProbs.length;
 	const perplexity = Math.exp(-avgLogProb);
-	const pass = assertion.threshold !== void 0 ? perplexity <= assertion.threshold : true;
+	const pass = assertion.threshold === void 0 ? true : perplexity <= assertion.threshold;
 	return {
 		pass,
 		score: pass ? 1 : 0,
@@ -1579,7 +1585,7 @@ function handlePerplexityScore({ logProbs, assertion }) {
 	if (!logProbs || logProbs.length === 0) throw new Error("perplexity-score assertion does not support providers that do not return logProbs");
 	const avgLogProb = logProbs.reduce((acc, logProb) => acc + logProb, 0) / logProbs.length;
 	const perplexityNorm = 1 / (1 + Math.exp(-avgLogProb));
-	const pass = assertion.threshold !== void 0 ? perplexityNorm >= assertion.threshold : true;
+	const pass = assertion.threshold === void 0 ? true : perplexityNorm >= assertion.threshold;
 	return {
 		pass,
 		score: perplexityNorm,
@@ -1694,7 +1700,7 @@ ${isMultiline ? renderedValue.split("\n").map((line) => `${indentStyle}${line}`)
 		} else {
 			score = Number.parseFloat(String(result));
 			if (Number.isNaN(score)) throw new Error(`Python assertion must return a boolean, number, or {pass, score, reason} object. Instead got:\n${result}`);
-			pass = assertion.threshold !== void 0 ? score >= assertion.threshold : score > 0;
+			pass = assertion.threshold === void 0 ? score > 0 : score >= assertion.threshold;
 		}
 	} catch (err) {
 		return {
@@ -1955,7 +1961,7 @@ end
 		} else {
 			score = Number.parseFloat(String(result));
 			if (Number.isNaN(score)) throw new Error(`Ruby assertion must return a boolean, number, or {pass, score, reason} object. Instead got:\n${result}`);
-			pass = assertion.threshold !== void 0 ? score >= assertion.threshold : score > 0;
+			pass = assertion.threshold === void 0 ? score > 0 : score >= assertion.threshold;
 		}
 	} catch (err) {
 		return {
@@ -2026,6 +2032,127 @@ const handleSimilar = async ({ assertion, renderedValue, outputString, inverse,
 	};
 };
 //#endregion
+//#region src/assertions/traceUtils.ts
+/**
+* Shared utilities for trace assertions
+*/
+/**
+* Match a span name against a glob-like pattern.
+* Supports * (any characters) and ? (single character) wildcards.
+*
+* @param spanName - The span name to match
+* @param pattern - The glob pattern to match against
+* @returns true if the span name matches the pattern
+*/
+function matchesPattern(spanName, pattern) {
+	const regexPattern = pattern.replace(/[.+^${}()|[\]\\]/g, "\\$&").replace(/\*/g, ".*").replace(/\?/g, ".");
+	return new RegExp(`^${regexPattern}$`, "i").test(spanName);
+}
+//#endregion
+//#region src/assertions/skill.ts
+function getSkillCalls(params) {
+	const rawSkillCalls = params.providerResponse?.metadata?.skillCalls;
+	if (!Array.isArray(rawSkillCalls)) return [];
+	return rawSkillCalls.filter((entry) => Boolean(entry) && typeof entry === "object" && typeof entry.name === "string");
+}
+function matchesSkill(skillCall, matcher) {
+	if (matcher.name && skillCall.name !== matcher.name) return false;
+	if (matcher.pattern && !matchesPattern(skillCall.name, matcher.pattern)) return false;
+	return true;
+}
+function formatSkillCall(skillCall) {
+	const details = [skillCall.source, skillCall.path].filter(Boolean).join(", ");
+	return details ? `${skillCall.name} (${details})` : skillCall.name;
+}
+function resolveSkillMatchers(value) {
+	const normalizeText = (text) => typeof text === "string" ? text.trim() : void 0;
+	const validateCount = (field, count) => {
+		if (!Number.isFinite(count) || !Number.isInteger(count) || count < 0) throw new Error(`skill-used assertion object ${field} must be a finite non-negative integer`);
+	};
+	if (typeof value === "string" && value.trim()) return {
+		kind: "list",
+		matchers: [{ name: normalizeText(value) }]
+	};
+	if (Array.isArray(value) && value.length > 0 && value.every((item) => typeof item === "string" && item.trim())) return {
+		kind: "list",
+		matchers: value.map((item) => ({ name: item.trim() }))
+	};
+	if (value && typeof value === "object" && !Array.isArray(value)) {
+		const rawMatcher = value;
+		const matcher = rawMatcher;
+		const name = normalizeText(matcher.name);
+		const pattern = normalizeText(matcher.pattern);
+		if (!name && !pattern) throw new Error("skill-used assertion object must include a name or pattern property");
+		if ("min" in rawMatcher) validateCount("min", matcher.min);
+		if ("max" in rawMatcher) validateCount("max", matcher.max);
+		if (typeof matcher.min === "number" && typeof matcher.max === "number" && matcher.max < matcher.min) throw new Error("skill-used assertion object max must be greater than or equal to min");
+		return {
+			kind: "count",
+			matcher: {
+				max: typeof matcher.max === "number" ? matcher.max : void 0,
+				min: typeof matcher.min === "number" ? matcher.min : void 0,
+				name,
+				pattern
+			}
+		};
+	}
+	throw new Error("skill-used assertion must have a string, string array, or object value");
+}
+function handleListSkillAssertion(params, skillCalls, actualSkills, expected) {
+	const missing = expected.matchers.filter((matcher) => !skillCalls.some((skillCall) => matchesSkill(skillCall, matcher)));
+	const matched = expected.matchers.filter((matcher) => skillCalls.some((skillCall) => matchesSkill(skillCall, matcher)));
+	const pass = params.inverse ? matched.length === 0 : missing.length === 0;
+	const expectedSkills = expected.matchers.map((matcher) => matcher.name);
+	const actualSummary = actualSkills.length > 0 ? actualSkills.join(", ") : "(none)";
+	let reason;
+	if (params.inverse) reason = pass ? `Forbidden skill(s) were not used: ${expectedSkills.join(", ")}` : `Forbidden skill(s) were used: ${matched.map((matcher) => matcher.name).join(", ")}. Actual skills: ${actualSummary}`;
+	else if (pass) reason = `Observed required skill(s): ${expectedSkills.join(", ")}. Actual skills: ${actualSummary}`;
+	else reason = `Missing required skill(s): ${missing.map((matcher) => matcher.name).join(", ")}. Actual skills: ${actualSummary}`;
+	return {
+		pass,
+		score: pass ? 1 : 0,
+		reason,
+		assertion: params.assertion
+	};
+}
+function handleCountSkillAssertion(params, skillCalls, actualSkills, matcher) {
+	const hasExplicitMin = matcher.min !== void 0;
+	const hasExplicitMax = matcher.max !== void 0;
+	const min = matcher.min ?? (hasExplicitMax ? 0 : 1);
+	const max = matcher.max;
+	const matchingSkillCalls = skillCalls.filter((skillCall) => matchesSkill(skillCall, matcher));
+	const count = matchingSkillCalls.length;
+	const matcherLabel = matcher.pattern || matcher.name || "*";
+	if (params.inverse) {
+		if (hasExplicitMin || hasExplicitMax && max !== 0) throw new Error("not-skill-used object assertions only support name/pattern with no count bounds, or max: 0");
+		const pass = count === 0;
+		const actualSummary = actualSkills.length > 0 ? actualSkills.join(", ") : "(none)";
+		return {
+			pass,
+			score: pass ? 1 : 0,
+			reason: pass ? `Forbidden skill "${matcherLabel}" was not used. Actual skills: ${actualSummary}` : `Forbidden skill "${matcherLabel}" was used ${count} time(s). Matches: ${matchingSkillCalls.map(formatSkillCall).join(", ")}`,
+			assertion: params.assertion
+		};
+	}
+	const pass = count >= min && (max === void 0 || count <= max);
+	let reason = `Matched skill "${matcherLabel}" ${count} time(s)`;
+	reason += max === void 0 ? ` (expected at least ${min})` : ` (expected ${min}-${max})`;
+	if (matchingSkillCalls.length > 0) reason += `. Matches: ${matchingSkillCalls.map(formatSkillCall).join(", ")}`;
+	return {
+		pass,
+		score: pass ? 1 : 0,
+		reason,
+		assertion: params.assertion
+	};
+}
+function handleSkillUsed(params) {
+	const skillCalls = getSkillCalls(params);
+	const actualSkills = skillCalls.map(formatSkillCall);
+	const expected = resolveSkillMatchers(params.renderedValue ?? params.assertion.value);
+	if (expected.kind === "list") return handleListSkillAssertion(params, skillCalls, actualSkills, expected);
+	return handleCountSkillAssertion(params, skillCalls, actualSkills, expected.matcher);
+}
+//#endregion
 //#region src/assertions/sql.ts
 const handleIsSql = async ({ assertion, renderedValue, outputString, inverse }) => {
 	let pass = false;
@@ -2258,23 +2385,6 @@ const handleToolCallF1 = ({ assertion, output, renderedValue, inverse }) => {
 	};
 };
 //#endregion
-//#region src/assertions/traceUtils.ts
-/**
-* Shared utilities for trace assertions
-*/
-/**
-* Match a span name against a glob-like pattern.
-* Supports * (any characters) and ? (single character) wildcards.
-*
-* @param spanName - The span name to match
-* @param pattern - The glob pattern to match against
-* @returns true if the span name matches the pattern
-*/
-function matchesPattern(spanName, pattern) {
-	const regexPattern = pattern.replace(/[.+^${}()|[\]\\]/g, "\\$&").replace(/\*/g, ".*").replace(/\?/g, ".");
-	return new RegExp(`^${regexPattern}$`, "i").test(spanName);
-}
-//#endregion
 //#region src/assertions/traceErrorSpans.ts
 function isErrorSpan(span) {
 	if (span.statusCode && span.statusCode >= 400) return true;
@@ -2443,6 +2553,524 @@ const handleTraceSpanDuration = ({ assertion, assertionValueContext }) => {
 	};
 };
 //#endregion
+//#region src/assertions/trajectoryUtils.ts
+const TOOL_ATTRIBUTE_KEYS = [
+	"tool.name",
+	"tool_name",
+	"tool",
+	"function.name",
+	"function_name",
+	"gen_ai.tool.name",
+	"codex.mcp.tool",
+	"agent.tool",
+	"agent.tool_name",
+	"agent.toolName"
+];
+const TOOL_ARGUMENT_ATTRIBUTE_KEYS = [
+	"tool.arguments",
+	"tool.args",
+	"tool.input",
+	"tool_arguments",
+	"tool_args",
+	"tool_input",
+	"function.arguments",
+	"function.args",
+	"function.input",
+	"function_arguments",
+	"function_args",
+	"gen_ai.tool.arguments",
+	"gen_ai.tool.args",
+	"gen_ai.tool.input",
+	"gen_ai.tool.call.arguments",
+	"gen_ai.tool.call.args",
+	"agent.tool.arguments",
+	"agent.tool.args",
+	"agent.tool.input",
+	"codex.mcp.arguments",
+	"codex.mcp.args",
+	"codex.mcp.input",
+	"arguments",
+	"args",
+	"input"
+];
+const COMMAND_ATTRIBUTE_KEYS = [
+	"codex.command",
+	"command",
+	"command.name",
+	"command_name"
+];
+const SEARCH_ATTRIBUTE_KEYS = [
+	"codex.search.query",
+	"search.query",
+	"search_query"
+];
+const GENERIC_QUERY_ATTRIBUTE_KEYS = ["query"];
+const SEARCH_SPAN_NAME_PATTERN = /(^|[\s._:/-])(search|find|lookup|retriev(?:e|al))($|[\s._:/-])/i;
+const MAX_JUDGE_SUMMARY_STEPS = 24;
+const JUDGE_SUMMARY_HEAD_STEPS = 12;
+const JUDGE_SUMMARY_TAIL_STEPS = 12;
+function getStringAttribute(attributes, keys) {
+	for (const key of keys) {
+		const value = attributes[key];
+		if (typeof value === "string" && value.trim()) return value.trim();
+	}
+}
+function normalizeStructuredAttribute(value) {
+	if (value === void 0 || value === null) return;
+	if (typeof value === "string") {
+		const trimmed = value.trim();
+		if (!trimmed) return;
+		try {
+			return JSON.parse(trimmed);
+		} catch {
+			return trimmed;
+		}
+	}
+	if (typeof value === "number" || typeof value === "boolean" || typeof value === "object") return value;
+}
+function hasSameStatus(left, right) {
+	return left?.code === right?.code && left?.message === right?.message;
+}
+function isSearchLikeSpan(span) {
+	const attributes = span.attributes || {};
+	if (SEARCH_SPAN_NAME_PATTERN.test(span.name) || span.name.startsWith("search ")) return true;
+	return Object.keys(attributes).some((key) => key !== "query" && /(^|[._])(search|lookup|retriev(?:e|al))($|[._])/i.test(key));
+}
+function getTrajectoryStepStatus(step) {
+	if (step.statusCode === void 0 || step.statusCode === 0) return;
+	return {
+		code: step.statusCode,
+		...step.statusMessage ? { message: step.statusMessage } : {}
+	};
+}
+function getCommandExecutable(command) {
+	return command.trim().split(/\s+/)[0] || void 0;
+}
+function extractToolName(span) {
+	const attributes = span.attributes || {};
+	const directMatch = getStringAttribute(attributes, TOOL_ATTRIBUTE_KEYS);
+	if (directMatch) return directMatch;
+	for (const [key, value] of Object.entries(attributes)) {
+		if (typeof value !== "string" || !value.trim()) continue;
+		if (/tool.?name|function.?name/i.test(key)) return value.trim();
+		if (/(^|[._])tool($|[._])/i.test(key) && !/result|output/i.test(key)) return value.trim();
+	}
+	if (span.name.startsWith("mcp ")) {
+		const slashIndex = span.name.lastIndexOf("/");
+		if (slashIndex !== -1 && slashIndex < span.name.length - 1) return span.name.slice(slashIndex + 1).trim();
+	}
+}
+function extractToolArgs(span) {
+	const attributes = span.attributes || {};
+	for (const key of TOOL_ARGUMENT_ATTRIBUTE_KEYS) {
+		const value = normalizeStructuredAttribute(attributes[key]);
+		if (value !== void 0) return value;
+	}
+	for (const [key, rawValue] of Object.entries(attributes)) {
+		if (/result|output|error|status/i.test(key)) continue;
+		if (!/(^|[._])(arguments|args|input)($|[._])/i.test(key)) continue;
+		const value = normalizeStructuredAttribute(rawValue);
+		if (value !== void 0) return value;
+	}
+}
+function extractCommand(span) {
+	const attributes = span.attributes || {};
+	const directMatch = getStringAttribute(attributes, COMMAND_ATTRIBUTE_KEYS);
+	if (directMatch) return directMatch;
+	for (const [key, value] of Object.entries(attributes)) {
+		if (typeof value !== "string" || !value.trim()) continue;
+		if (/command/i.test(key) && !/output|result/i.test(key)) return value.trim();
+	}
+	if (span.name.startsWith("exec ")) return span.name.slice(5).trim();
+}
+function extractSearchQuery(span) {
+	const attributes = span.attributes || {};
+	const directMatch = getStringAttribute(attributes, SEARCH_ATTRIBUTE_KEYS);
+	if (directMatch) return directMatch;
+	const genericQuery = getStringAttribute(attributes, GENERIC_QUERY_ATTRIBUTE_KEYS);
+	if (genericQuery && isSearchLikeSpan(span)) return genericQuery;
+	if (span.name.startsWith("search ")) return span.name.slice(7).replace(/^"|"$/g, "").trim();
+}
+function isReasoningSpan(span) {
+	if ((span.attributes || {})["codex.item.type"] === "reasoning") return true;
+	return /^reasoning([_\s]|$)/i.test(span.name) || span.name === "reasoning";
+}
+function isMessageSpan(span) {
+	if ((span.attributes || {})["codex.item.type"] === "agent_message") return true;
+	return span.name === "agent response" || span.name === "send input";
+}
+function extractTrajectorySteps(trace) {
+	return [...trace.spans || []].map((span, index) => ({
+		span,
+		index
+	})).sort((left, right) => {
+		const timeDiff = left.span.startTime - right.span.startTime;
+		if (timeDiff !== 0) return timeDiff;
+		const endDiff = (left.span.endTime ?? left.span.startTime) - (right.span.endTime ?? right.span.startTime);
+		if (endDiff !== 0) return endDiff;
+		return left.index - right.index;
+	}).map(({ span }) => {
+		const toolName = extractToolName(span);
+		const command = extractCommand(span);
+		const searchQuery = extractSearchQuery(span);
+		let type = "span";
+		let name = span.name;
+		const aliases = new Set([span.name]);
+		let args;
+		if (toolName) {
+			type = "tool";
+			name = toolName;
+			aliases.add(toolName);
+			args = extractToolArgs(span);
+		} else if (command) {
+			type = "command";
+			name = command;
+			aliases.add(command);
+			const executable = getCommandExecutable(command);
+			if (executable) aliases.add(executable);
+		} else if (searchQuery) {
+			type = "search";
+			name = searchQuery;
+			aliases.add(searchQuery);
+		} else if (isReasoningSpan(span)) {
+			type = "reasoning";
+			name = span.name;
+			aliases.add("reasoning");
+		} else if (isMessageSpan(span)) {
+			type = "message";
+			name = span.name;
+			aliases.add("message");
+		}
+		return {
+			aliases: [...aliases],
+			...args === void 0 ? {} : { args },
+			attributes: span.attributes || {},
+			endTime: span.endTime,
+			name,
+			spanId: span.spanId,
+			spanName: span.name,
+			startTime: span.startTime,
+			statusCode: span.statusCode,
+			statusMessage: span.statusMessage,
+			type
+		};
+	});
+}
+function normalizeTrajectoryMatcher(matcher, defaultType) {
+	if (typeof matcher === "string") return {
+		pattern: matcher,
+		...defaultType ? { type: defaultType } : {}
+	};
+	return {
+		...matcher,
+		...matcher.type ? {} : defaultType ? { type: defaultType } : {}
+	};
+}
+function matchesTrajectoryStep(step, matcher, defaultType) {
+	const { type, pattern, name } = normalizeTrajectoryMatcher(matcher, defaultType);
+	if (type) {
+		if (!(Array.isArray(type) ? type : [type]).includes(step.type)) return false;
+	}
+	const matchPattern = pattern || name;
+	if (!matchPattern) return true;
+	return step.aliases.some((alias) => matchesPattern(alias, matchPattern));
+}
+function formatTrajectoryStep(step) {
+	return `${step.type}:${step.name}`;
+}
+function formatTrajectoryArgs(args) {
+	if (args === void 0) return "(none)";
+	try {
+		const serialized = JSON.stringify(args);
+		if (serialized !== void 0) return serialized;
+	} catch {}
+	return String(args);
+}
+function compactJudgeTrajectorySteps(steps) {
+	const compacted = [];
+	for (const step of steps) {
+		const previousStep = compacted[compacted.length - 1];
+		if (previousStep && previousStep.type === step.type && previousStep.name === step.name && previousStep.spanName === step.spanName && hasSameStatus(previousStep.status, step.status)) {
+			previousStep.collapsedCount = (previousStep.collapsedCount ?? 1) + 1;
+			continue;
+		}
+		compacted.push(step);
+	}
+	return compacted;
+}
+function truncateJudgeTrajectorySteps(steps) {
+	if (steps.length <= MAX_JUDGE_SUMMARY_STEPS) return steps;
+	return [
+		...steps.slice(0, JUDGE_SUMMARY_HEAD_STEPS),
+		{ omittedCount: steps.length - MAX_JUDGE_SUMMARY_STEPS },
+		...steps.slice(-JUDGE_SUMMARY_TAIL_STEPS)
+	];
+}
+function summarizeTrajectoryForJudge(trace) {
+	const rawSteps = extractTrajectorySteps(trace).map((step, index) => ({
+		index: index + 1,
+		type: step.type,
+		name: step.name,
+		...step.spanName === step.name ? {} : { spanName: step.spanName },
+		...getTrajectoryStepStatus(step) ? { status: getTrajectoryStepStatus(step) } : {}
+	}));
+	const compactedSteps = compactJudgeTrajectorySteps(rawSteps);
+	const steps = truncateJudgeTrajectorySteps(compactedSteps);
+	return JSON.stringify({
+		traceId: trace.traceId,
+		stepCount: rawSteps.length,
+		compactedStepCount: compactedSteps.length,
+		steps
+	}, null, 2);
+}
+//#endregion
+//#region src/assertions/trajectory.ts
+function getTraceOrThrow(params) {
+	const trace = params.assertionValueContext.trace;
+	if (!trace || !trace.spans) throw new Error(`No trace data available for ${params.baseType} assertion`);
+	return trace;
+}
+function applyInverse(pass, inverse) {
+	return inverse ? !pass : pass;
+}
+function formatStepList(stepLabels) {
+	return stepLabels.length > 0 ? stepLabels.join(", ") : "(none)";
+}
+function requireNamedTrajectoryMatcher(matcher, assertionType, index) {
+	if (matcher.pattern || matcher.name) return;
+	const stepLabel = index === void 0 ? "object" : `step ${index + 1}`;
+	throw new Error(`${assertionType} assertion ${stepLabel} must include a name or pattern property`);
+}
+function resolveGoalSuccessValue(value) {
+	if (typeof value === "string" && value.trim()) return { goal: value.trim() };
+	if (value && typeof value === "object" && !Array.isArray(value) && typeof value.goal === "string" && value.goal.trim()) return { goal: value.goal.trim() };
+	throw new Error("trajectory:goal-success assertion must have a string value or an object with a goal property");
+}
+function resolveToolMatchers(value) {
+	if (typeof value === "string") return {
+		kind: "list",
+		matchers: [normalizeTrajectoryMatcher(value, "tool")]
+	};
+	if (Array.isArray(value) && value.every((item) => typeof item === "string")) return {
+		kind: "list",
+		matchers: value.map((item) => normalizeTrajectoryMatcher(item, "tool"))
+	};
+	if (value && typeof value === "object" && !Array.isArray(value)) return {
+		kind: "count",
+		matcher: {
+			...normalizeTrajectoryMatcher(value, "tool"),
+			max: typeof value.max === "number" ? value.max : void 0,
+			min: typeof value.min === "number" ? value.min : void 0
+		}
+	};
+	throw new Error("trajectory:tool-used assertion must have a string, string array, or object value");
+}
+const handleTrajectoryToolUsed = (params) => {
+	const steps = extractTrajectorySteps(getTraceOrThrow(params)).filter((step) => step.type === "tool");
+	const expected = resolveToolMatchers(params.renderedValue ?? params.assertion.value);
+	if (expected.kind === "list") {
+		if (expected.matchers.length === 0) throw new Error("trajectory:tool-used assertion requires at least one expected tool");
+		const missing = expected.matchers.filter((matcher) => !steps.some((step) => matchesTrajectoryStep(step, matcher)));
+		const matched = expected.matchers.filter((matcher) => steps.some((step) => matchesTrajectoryStep(step, matcher)));
+		const pass = params.inverse ? matched.length === 0 : missing.length === 0;
+		const actualTools = steps.map(formatTrajectoryStep);
+		const expectedTools = expected.matchers.map((matcher) => matcher.pattern || matcher.name || "*");
+		let reason;
+		if (params.inverse) reason = pass ? `Forbidden tool(s) were not used: ${expectedTools.join(", ")}` : `Forbidden tool(s) were used: ${matched.map((matcher) => matcher.pattern || matcher.name || "*").join(", ")}. Actual tools: ${formatStepList(actualTools)}`;
+		else if (pass) reason = `Observed required tool(s): ${expectedTools.join(", ")}. Actual tools: ${formatStepList(actualTools)}`;
+		else reason = `Missing required tool(s): ${missing.map((matcher) => matcher.pattern || matcher.name || "*").join(", ")}. Actual tools: ${formatStepList(actualTools)}`;
+		return {
+			pass,
+			score: pass ? 1 : 0,
+			reason,
+			assertion: params.assertion
+		};
+	}
+	const matcher = expected.matcher;
+	const min = matcher.min ?? 1;
+	const max = matcher.max;
+	if (!matcher.pattern && !matcher.name) throw new Error("trajectory:tool-used assertion object must include a name or pattern property");
+	const matchingSteps = steps.filter((step) => matchesTrajectoryStep(step, matcher));
+	const count = matchingSteps.length;
+	const basePass = count >= min && (max === void 0 || count <= max);
+	const pass = applyInverse(basePass, params.inverse);
+	const matcherLabel = matcher.pattern || matcher.name || "*";
+	let reason = `Matched tool "${matcherLabel}" ${count} time(s)`;
+	if (max === void 0) reason += ` (expected at least ${min})`;
+	else reason += ` (expected ${min}-${max})`;
+	if (matchingSteps.length > 0) reason += `. Matches: ${matchingSteps.map(formatTrajectoryStep).join(", ")}`;
+	if (params.inverse) reason = basePass ? `Tool "${matcherLabel}" matched ${count} time(s), which violates the inverse assertion` : `Tool "${matcherLabel}" did not satisfy the forbidden match condition`;
+	return {
+		pass,
+		score: pass ? 1 : 0,
+		reason,
+		assertion: params.assertion
+	};
+};
+function resolveSequenceValue(value) {
+	if (Array.isArray(value)) return {
+		mode: "in_order",
+		steps: value
+	};
+	if (value && typeof value === "object" && !Array.isArray(value)) {
+		const sequenceValue = value;
+		return {
+			mode: sequenceValue.mode || "in_order",
+			steps: sequenceValue.steps || []
+		};
+	}
+	throw new Error("trajectory:tool-sequence assertion must have an array or object value");
+}
+function isRecord(value) {
+	return typeof value === "object" && value !== null && !Array.isArray(value);
+}
+function matchesExpectedArgsPartial(actual, expected) {
+	if (Array.isArray(expected)) return Array.isArray(actual) && actual.length === expected.length && expected.every((item, index) => matchesExpectedArgsPartial(actual[index], item));
+	if (isRecord(expected)) {
+		if (!isRecord(actual)) return false;
+		return Object.entries(expected).every(([key, expectedValue]) => Object.prototype.hasOwnProperty.call(actual, key) && matchesExpectedArgsPartial(actual[key], expectedValue));
+	}
+	return isDeepStrictEqual(actual, expected);
+}
+function matchesToolArgs(actual, expected, mode) {
+	if (mode === "exact") return isDeepStrictEqual(actual, expected);
+	return matchesExpectedArgsPartial(actual, expected);
+}
+function resolveToolArgsMatchMode(mode) {
+	if (mode === void 0) return "partial";
+	if (mode === "partial" || mode === "exact") return mode;
+	throw new Error("trajectory:tool-args-match assertion mode must be \"partial\" or \"exact\"");
+}
+function resolveToolArgsMatchValue(value) {
+	if (!value || typeof value !== "object" || Array.isArray(value)) throw new Error("trajectory:tool-args-match assertion must have an object value");
+	const matcher = normalizeTrajectoryMatcher(value, "tool");
+	requireNamedTrajectoryMatcher(matcher, "trajectory:tool-args-match");
+	const expectedArgs = Object.prototype.hasOwnProperty.call(value, "args") ? value.args : value.arguments;
+	if (expectedArgs === void 0) throw new Error("trajectory:tool-args-match assertion must include an args or arguments property");
+	return {
+		matcher,
+		expectedArgs,
+		mode: resolveToolArgsMatchMode(value.mode)
+	};
+}
+const handleTrajectoryToolSequence = (params) => {
+	const toolSteps = extractTrajectorySteps(getTraceOrThrow(params)).filter((step) => step.type === "tool");
+	const value = resolveSequenceValue(params.renderedValue ?? params.assertion.value);
+	const expectedMatchers = value.steps.map((step, index) => {
+		const matcher = normalizeTrajectoryMatcher(step, "tool");
+		requireNamedTrajectoryMatcher(matcher, "trajectory:tool-sequence", index);
+		return matcher;
+	});
+	if (expectedMatchers.length === 0) throw new Error("trajectory:tool-sequence assertion requires at least one expected step");
+	const actualTools = toolSteps.map(formatTrajectoryStep);
+	let basePass = false;
+	let reason = "";
+	if (value.mode === "exact") {
+		basePass = toolSteps.length === expectedMatchers.length && expectedMatchers.every((matcher, index) => matchesTrajectoryStep(toolSteps[index], matcher));
+		if (basePass) reason = `Observed exact tool sequence: ${formatStepList(actualTools)}`;
+		else reason = `Expected exact tool sequence of ${expectedMatchers.map((matcher) => matcher.pattern || matcher.name || "*").join(", ")}, but actual tools were ${formatStepList(actualTools)}`;
+	} else {
+		let expectedIndex = 0;
+		const matchedSteps = [];
+		for (const step of toolSteps) {
+			if (expectedIndex >= expectedMatchers.length) break;
+			if (matchesTrajectoryStep(step, expectedMatchers[expectedIndex])) {
+				matchedSteps.push(formatTrajectoryStep(step));
+				expectedIndex += 1;
+			}
+		}
+		basePass = expectedIndex === expectedMatchers.length;
+		if (basePass) reason = `Observed tool sequence in order: ${matchedSteps.join(", ")}. Actual tools: ${formatStepList(actualTools)}`;
+		else reason = `Expected tool "${expectedMatchers[expectedIndex]?.pattern || expectedMatchers[expectedIndex]?.name || "*"}" was not observed in order. Actual tools: ${formatStepList(actualTools)}`;
+	}
+	const pass = applyInverse(basePass, params.inverse);
+	if (params.inverse) reason = basePass ? `Forbidden tool sequence was observed. Actual tools: ${formatStepList(actualTools)}` : `Forbidden tool sequence was not observed`;
+	return {
+		pass,
+		score: pass ? 1 : 0,
+		reason,
+		assertion: params.assertion
+	};
+};
+const handleTrajectoryToolArgsMatch = (params) => {
+	const toolSteps = extractTrajectorySteps(getTraceOrThrow(params)).filter((step) => step.type === "tool");
+	const { matcher, expectedArgs, mode } = resolveToolArgsMatchValue(params.renderedValue ?? params.assertion.value);
+	const matcherLabel = matcher.pattern || matcher.name || "*";
+	const actualTools = toolSteps.map(formatTrajectoryStep);
+	const matchingSteps = toolSteps.filter((step) => matchesTrajectoryStep(step, matcher));
+	const stepsWithArgs = matchingSteps.filter((step) => step.args !== void 0);
+	const matchedStep = stepsWithArgs.find((step) => matchesToolArgs(step.args, expectedArgs, mode));
+	const basePass = matchedStep !== void 0;
+	const pass = applyInverse(basePass, params.inverse);
+	const expectedArgsLabel = formatTrajectoryArgs(expectedArgs);
+	const observedArgsLabel = stepsWithArgs.length > 0 ? stepsWithArgs.map((step) => formatTrajectoryArgs(step.args)).join(", ") : "(none)";
+	let reason;
+	if (params.inverse) if (basePass) reason = `Forbidden argument match for tool "${matcherLabel}" was observed on ${formatTrajectoryStep(matchedStep)}. Args: ${formatTrajectoryArgs(matchedStep.args)}`;
+	else if (matchingSteps.length === 0) reason = `Forbidden argument match for tool "${matcherLabel}" was not observed because no tool call matched it`;
+	else reason = `Forbidden argument match for tool "${matcherLabel}" was not observed. Observed args: ${observedArgsLabel}`;
+	else if (basePass) reason = `Tool "${matcherLabel}" matched expected arguments (${mode}) on ${formatTrajectoryStep(matchedStep)}. Args: ${formatTrajectoryArgs(matchedStep.args)}`;
+	else if (matchingSteps.length === 0) reason = `No tool call matched "${matcherLabel}". Actual tools: ${formatStepList(actualTools)}`;
+	else if (stepsWithArgs.length === 0) reason = `Tool "${matcherLabel}" was observed but no arguments were captured. Actual tools: ${formatStepList(actualTools)}`;
+	else reason = `No call to tool "${matcherLabel}" matched expected arguments (${mode}): ${expectedArgsLabel}. Observed args: ${observedArgsLabel}`;
+	return {
+		pass,
+		score: pass ? 1 : 0,
+		reason,
+		assertion: params.assertion
+	};
+};
+function resolveStepCountValue(value) {
+	if (!value || typeof value !== "object" || Array.isArray(value)) throw new Error("trajectory:step-count assertion must have an object value");
+	return {
+		...normalizeTrajectoryMatcher(value),
+		max: typeof value.max === "number" ? value.max : void 0,
+		min: typeof value.min === "number" ? value.min : void 0
+	};
+}
+const handleTrajectoryStepCount = (params) => {
+	const steps = extractTrajectorySteps(getTraceOrThrow(params));
+	const matcher = resolveStepCountValue(params.renderedValue ?? params.assertion.value);
+	const { min, max } = matcher;
+	if (min === void 0 && max === void 0) throw new Error("trajectory:step-count assertion must include a min or max property");
+	const matchingSteps = steps.filter((step) => matchesTrajectoryStep(step, matcher));
+	const count = matchingSteps.length;
+	const basePass = (min === void 0 || count >= min) && (max === void 0 || count <= max);
+	const pass = applyInverse(basePass, params.inverse);
+	const filterParts = [];
+	if (matcher.type) {
+		const types = Array.isArray(matcher.type) ? matcher.type : [matcher.type];
+		filterParts.push(`type=${types.join("|")}`);
+	}
+	const pattern = matcher.pattern || matcher.name;
+	if (pattern) filterParts.push(`pattern=${pattern}`);
+	let reason = `Matched ${count} trajectory step(s)`;
+	if (filterParts.length > 0) reason += ` for ${filterParts.join(", ")}`;
+	if (min !== void 0 && max !== void 0) reason += ` (expected ${min}-${max})`;
+	else if (min !== void 0) reason += ` (expected at least ${min})`;
+	else if (max !== void 0) reason += ` (expected at most ${max})`;
+	if (matchingSteps.length > 0) reason += `. Matches: ${matchingSteps.map(formatTrajectoryStep).join(", ")}`;
+	if (params.inverse) reason = basePass ? `Trajectory step count satisfied the forbidden range` : `Trajectory step count did not satisfy the forbidden range`;
+	return {
+		pass,
+		score: pass ? 1 : 0,
+		reason,
+		assertion: params.assertion
+	};
+};
+const handleTrajectoryGoalSuccess = async (params) => {
+	const trace = getTraceOrThrow(params);
+	const { goal } = resolveGoalSuccessValue(params.renderedValue ?? params.assertion.value);
+	const result = await matchesTrajectoryGoalSuccess(goal, summarizeTrajectoryForJudge(trace), params.outputString, params.test.options, params.assertionValueContext.vars, params.assertion, params.providerCallContext);
+	if (!params.inverse) return result;
+	return {
+		...result,
+		assertion: params.assertion,
+		pass: !result.pass,
+		score: result.pass ? 0 : 1,
+		reason: result.pass ? `Agent unexpectedly achieved the goal: ${goal}` : `Agent did not achieve the forbidden goal: ${goal}`
+	};
+};
+//#endregion
 //#region src/assertions/webhook.ts
 async function handleWebhook({ assertion, renderedValue, test, prompt, output, inverse }) {
 	invariant(renderedValue, "\"webhook\" assertion type must have a URL value");
@@ -2511,18 +3139,18 @@ const handleWordCount = ({ assertion, renderedValue, valueFromScript, outputStri
 			if (pass) reason = "Assertion passed";
 			else if (inverse) reason = `Expected word count to not be between ${min} and ${max}, but got ${wordCount}`;
 			else reason = `Word count ${wordCount} is not between ${min} and ${max}`;
-		} else if (min !== void 0) {
-			const basePass = wordCount >= min;
-			pass = inverse ? !basePass : basePass;
-			if (pass) reason = "Assertion passed";
-			else if (inverse) reason = `Expected word count to be less than ${min}, but got ${wordCount}`;
-			else reason = `Word count ${wordCount} is less than minimum ${min}`;
-		} else {
+		} else if (min === void 0) {
 			const basePass = wordCount <= max;
 			pass = inverse ? !basePass : basePass;
 			if (pass) reason = "Assertion passed";
 			else if (inverse) reason = `Expected word count to be greater than ${max}, but got ${wordCount}`;
 			else reason = `Word count ${wordCount} is greater than maximum ${max}`;
+		} else {
+			const basePass = wordCount >= min;
+			pass = inverse ? !basePass : basePass;
+			if (pass) reason = "Assertion passed";
+			else if (inverse) reason = `Expected word count to be less than ${min}, but got ${wordCount}`;
+			else reason = `Word count ${wordCount} is less than minimum ${min}`;
 		}
 	} else {
 		invariant(typeof value === "number" || typeof value === "string" && !Number.isNaN(Number(value)), "\"word-count\" assertion value must be a number or an object with min/max properties");
@@ -2617,6 +3245,12 @@ const handleIsXml = ({ assertion, renderedValue, outputString, inverse, baseType
 //#endregion
 //#region src/assertions/index.ts
 const ASSERTIONS_MAX_CONCURRENCY = getEnvInt("PROMPTFOO_ASSERTIONS_MAX_CONCURRENCY", 3);
+const DEFAULT_TRACE_FETCH_MAX_ATTEMPTS = 6;
+const DEFAULT_TRACE_FETCH_RETRY_DELAY_MS = 250;
+const DEFAULT_TRACE_FETCH_STABLE_POLLS = 2;
+const MAX_TRACE_FETCH_MAX_ATTEMPTS = 30;
+const MAX_TRACE_FETCH_RETRY_DELAY_MS = 5e3;
+const MAX_TRACE_FETCH_STABLE_POLLS = 10;
 const MODEL_GRADED_ASSERTION_TYPES = new Set([
 	"answer-relevance",
 	"context-faithfulness",
@@ -2626,8 +3260,57 @@ const MODEL_GRADED_ASSERTION_TYPES = new Set([
 	"llm-rubric",
 	"model-graded-closedqa",
 	"model-graded-factuality",
-	"search-rubric"
+	"search-rubric",
+	"trajectory:goal-success"
 ]);
+const TRACE_AWARE_ASSERTION_TYPES = new Set([
+	"javascript",
+	"python",
+	"ruby",
+	"trace-error-spans",
+	"trace-span-count",
+	"trace-span-duration",
+	"trajectory:goal-success",
+	"trajectory:step-count",
+	"trajectory:tool-args-match",
+	"trajectory:tool-sequence",
+	"trajectory:tool-used"
+]);
+function assertionUsesTrace(assertion) {
+	if (assertion.type === "assert-set") return assertion.assert.some(assertionUsesTrace);
+	return TRACE_AWARE_ASSERTION_TYPES.has(getAssertionBaseType(assertion));
+}
+function assertionMayNeedTraceContext(assertion) {
+	if (assertionUsesTrace(assertion)) return true;
+	if (assertion.type === "assert-set") return assertion.assert.some(assertionMayNeedTraceContext);
+	return typeof assertion.value === "string" ? assertion.value.startsWith("file://") || isPackagePath(assertion.value) : false;
+}
+function hasTraceAwareAssertions(assertions) {
+	return Boolean(assertions?.some(assertionMayNeedTraceContext));
+}
+async function loadTraceData(traceId) {
+	const traceStore = getTraceStore();
+	const maxAttempts = Math.min(MAX_TRACE_FETCH_MAX_ATTEMPTS, Math.max(1, getEnvInt("PROMPTFOO_TRACE_FETCH_MAX_ATTEMPTS", DEFAULT_TRACE_FETCH_MAX_ATTEMPTS)));
+	const retryDelayMs = Math.min(MAX_TRACE_FETCH_RETRY_DELAY_MS, Math.max(0, getEnvInt("PROMPTFOO_TRACE_FETCH_RETRY_DELAY_MS", DEFAULT_TRACE_FETCH_RETRY_DELAY_MS)));
+	const stablePolls = Math.min(MAX_TRACE_FETCH_STABLE_POLLS, Math.max(1, getEnvInt("PROMPTFOO_TRACE_FETCH_STABLE_POLLS", DEFAULT_TRACE_FETCH_STABLE_POLLS)));
+	let lastSpanCount = -1;
+	let stableObservations = 0;
+	let latestTrace = null;
+	for (let attempt = 0; attempt < maxAttempts; attempt++) {
+		latestTrace = await traceStore.getTrace(traceId);
+		const spanCount = latestTrace?.spans?.length ?? 0;
+		if (spanCount > 0) {
+			stableObservations = spanCount === lastSpanCount ? stableObservations + 1 : 1;
+			lastSpanCount = spanCount;
+			if (stableObservations >= stablePolls || attempt === maxAttempts - 1) return latestTrace;
+		} else {
+			stableObservations = 0;
+			lastSpanCount = spanCount;
+		}
+		if (attempt < maxAttempts - 1) await sleep(retryDelayMs);
+	}
+	return latestTrace;
+}
 const ASSERTION_HANDLERS = {
 	"answer-relevance": handleAnswerRelevance,
 	bleu: handleBleuScore,
@@ -2690,12 +3373,18 @@ const ASSERTION_HANDLERS = {
 	ruby: handleRuby,
 	"rouge-n": handleRougeScore,
 	"search-rubric": handleSearchRubric,
+	"skill-used": handleSkillUsed,
 	similar: handleSimilar,
 	"similar:cosine": handleSimilar,
 	"similar:dot": handleSimilar,
 	"similar:euclidean": handleSimilar,
 	"starts-with": handleStartsWith,
 	"tool-call-f1": handleToolCallF1,
+	"trajectory:goal-success": handleTrajectoryGoalSuccess,
+	"trajectory:tool-args-match": handleTrajectoryToolArgsMatch,
+	"trajectory:step-count": handleTrajectoryStepCount,
+	"trajectory:tool-sequence": handleTrajectoryToolSequence,
+	"trajectory:tool-used": handleTrajectoryToolUsed,
 	"trace-error-spans": handleTraceErrorSpans,
 	"trace-span-count": handleTraceSpanCount,
 	"trace-span-duration": handleTraceSpanDuration,
@@ -2738,7 +3427,7 @@ function isAssertionInverse(assertion) {
 function getAssertionBaseType(assertion) {
 	return isAssertionInverse(assertion) ? assertion.type.slice(4) : assertion.type;
 }
-async function runAssertion({ prompt, provider, assertion, test, vars, latencyMs, providerResponse, traceId }) {
+async function runAssertion({ prompt, provider, assertion, test, vars, latencyMs, providerResponse, traceId, traceData }) {
 	const resolvedVars = vars || test.vars || {};
 	const { cost, logProbs, output: originalOutput } = providerResponse;
 	let output = originalOutput;
@@ -2757,14 +3446,14 @@ async function runAssertion({ prompt, provider, assertion, test, vars, latencyMs
 		providerResponse,
 		...assertion.config ? { config: structuredClone(assertion.config) } : {}
 	};
-	if (traceId) try {
-		const traceData = await getTraceStore().getTrace(traceId);
-		if (traceData) context.trace = {
-			traceId: traceData.traceId,
-			evaluationId: traceData.evaluationId,
-			testCaseId: traceData.testCaseId,
-			metadata: traceData.metadata,
-			spans: traceData.spans || []
+	if (traceId && assertionMayNeedTraceContext(assertion)) try {
+		const resolvedTraceData = traceData === void 0 ? await loadTraceData(traceId) : traceData;
+		if (resolvedTraceData) context.trace = {
+			traceId: resolvedTraceData.traceId,
+			evaluationId: resolvedTraceData.evaluationId,
+			testCaseId: resolvedTraceData.testCaseId,
+			metadata: resolvedTraceData.metadata,
+			spans: resolvedTraceData.spans || []
 		};
 	} catch (error) {
 		logger.debug(`Failed to fetch trace data for assertion: ${error}`);
@@ -2797,7 +3486,7 @@ async function runAssertion({ prompt, provider, assertion, test, vars, latencyMs
 			};
 		}
 		else if (filePath.endsWith(".rb")) try {
-			const { runRuby } = await import("./rubyUtils-BUHu6PhO.js");
+			const { runRuby } = await import("./rubyUtils-Rt6pKA96.js");
 			valueFromScript = await runRuby(filePath, functionName || "get_assert", [output, context]);
 			logger.debug(`Ruby script ${filePath} output: ${valueFromScript}`);
 		} catch (error) {
@@ -2906,6 +3595,14 @@ async function runAssertions({ assertScoringFunction, latencyMs, prompt, provide
 			index: i
 		};
 	}).flat();
+	const shouldPreloadTrace = !!traceId && hasTraceAwareAssertions(asserts.map(({ assertion }) => assertion));
+	let preloadedTraceData;
+	if (shouldPreloadTrace && traceId) try {
+		preloadedTraceData = await loadTraceData(traceId);
+	} catch (error) {
+		logger.debug(`Failed to preload trace data for assertions: ${error}`);
+		preloadedTraceData = null;
+	}
 	await async.forEachOfLimit(asserts, ASSERTIONS_MAX_CONCURRENCY, async ({ assertion, assertResult, index }) => {
 		if (assertion.type.startsWith("select-") || assertion.type === "max-score") return;
 		const result = await runAssertion({
@@ -2917,7 +3614,8 @@ async function runAssertions({ assertScoringFunction, latencyMs, prompt, provide
 			vars,
 			latencyMs,
 			assertIndex: index,
-			traceId
+			traceId,
+			traceData: preloadedTraceData
 		});
 		assertResult.addResult({
 			index,
@@ -3083,7 +3781,7 @@ var CIProgressReporter = class {
 		else {
 			const eta = remaining / rate;
 			if (eta > 1440) etaDisplay = ">24 hours";
-			else etaDisplay = `${Math.round(eta)} minute${Math.round(eta) !== 1 ? "s" : ""}`;
+			else etaDisplay = `${Math.round(eta)} minute${Math.round(eta) === 1 ? "" : "s"}`;
 		}
 		const percentage = Math.floor(this.completedTests / this.totalTests * 100);
 		logger.info(`[CI Progress] Evaluation running for ${this.formatElapsedTime(elapsed)} - Completed ${this.completedTests}/${this.totalTests} tests (${percentage}%)`);
@@ -3447,12 +4145,55 @@ var JsonlFileWriter = class {
 var ProgressBarManager = class {
 	progressBar;
 	isWebUI;
+	originalLogCallback = null;
+	installedLogCallback = null;
+	pendingRender = null;
 	totalCount = 0;
 	completedCount = 0;
 	concurrency = 1;
 	constructor(isWebUI) {
 		this.isWebUI = isWebUI;
 	}
+	clearProgressBarLine() {
+		readline.cursorTo(process.stderr, 0);
+		readline.clearLine(process.stderr, 0);
+	}
+	scheduleRender() {
+		if (!this.progressBar || this.pendingRender) return;
+		this.pendingRender = setImmediate(() => {
+			this.pendingRender = null;
+			this.progressBar?.render();
+		});
+	}
+	handleLogMessage() {
+		if (!this.progressBar) return;
+		this.clearProgressBarLine();
+		this.scheduleRender();
+	}
+	/**
+	* Coordinate console logging with the progress bar to prevent visual corruption.
+	*/
+	installLogInterceptor() {
+		if (!this.progressBar || this.isWebUI || this.installedLogCallback) return;
+		this.originalLogCallback = globalLogCallback;
+		this.installedLogCallback = (message) => {
+			this.originalLogCallback?.(message);
+			this.handleLogMessage();
+		};
+		setLogCallback(this.installedLogCallback);
+	}
+	/**
+	* Remove the log interceptor and restore original logger callback behavior.
+	*/
+	removeLogInterceptor() {
+		if (this.pendingRender) {
+			clearImmediate(this.pendingRender);
+			this.pendingRender = null;
+		}
+		if (this.installedLogCallback && globalLogCallback === this.installedLogCallback) setLogCallback(this.originalLogCallback);
+		this.installedLogCallback = null;
+		this.originalLogCallback = null;
+	}
 	/**
 	* Initialize progress bar
 	*/
@@ -3472,7 +4213,8 @@ var ProgressBarManager = class {
 				return `Evaluating [${bar}${spaces}] ${percentage}% | ${params.value}/${params.total}${errorsText} | ${payload.provider} ${payload.prompt} ${payload.vars}`;
 			},
 			hideCursor: true,
-			gracefulExit: true
+			gracefulExit: true,
+			stream: process.stderr
 		}, cliProgress.Presets.shades_classic);
 		this.progressBar.start(this.totalCount, 0, {
 			provider: "",
@@ -3747,6 +4489,7 @@ async function runEval({ provider, prompt, test, testSuite, delay, nunjucksFilte
 				const parts = traceContext.traceparent.split("-");
 				if (parts.length >= 3) traceId = parts[1];
 			}
+			if (traceId && hasTraceAwareAssertions(test.assert)) await flushOtel();
 			const checkResult = await runAssertions({
 				prompt: renderedPrompt,
 				provider,
@@ -4144,7 +4887,7 @@ var Evaluator = class {
 				const defaultProvider = testSuite.defaultTest.provider;
 				if (isApiProvider(defaultProvider)) testCase.provider = defaultProvider;
 				else if (typeof defaultProvider === "object" && defaultProvider.id) {
-					const { loadApiProvider } = await import("./providers-C1rOSHiR.js");
+					const { loadApiProvider } = await import("./providers-BSLEaIQG.js");
 					testCase.provider = await loadApiProvider(typeof defaultProvider.id === "function" ? defaultProvider.id() : defaultProvider.id, { options: defaultProvider });
 				} else testCase.provider = defaultProvider;
 			}
@@ -4228,7 +4971,7 @@ var Evaluator = class {
 			if (evalOption.test.assert?.some((a) => a.type === "max-score")) rowsWithMaxScoreAssertion.add(evalOption.testIdx);
 		}
 		if (state.resume && this.evalRecord.persisted) try {
-			const { default: EvalResult } = await import("./evalResult-Cqj8pldJ.js");
+			const { default: EvalResult } = await import("./evalResult-CYNHkk5A.js");
 			const completedPairs = await EvalResult.getCompletedIndexPairs(this.evalRecord.id, { excludeErrors: state.retryMode });
 			const originalCount = runEvalOptions.length;
 			for (let i = runEvalOptions.length - 1; i >= 0; i--) {
@@ -4428,7 +5171,7 @@ var Evaluator = class {
 		if (isCI() && !isWebUI) {
 			ciProgressReporter = new CIProgressReporter(runEvalOptions.length);
 			ciProgressReporter.start();
-		} else if (this.options.showProgressBar && process.stdout.isTTY) progressBarManager = new ProgressBarManager(isWebUI);
+		} else if (this.options.showProgressBar && process.stderr.isTTY) progressBarManager = new ProgressBarManager(isWebUI);
 		this.options.progressCallback = (completed, total, index, evalStep, metrics) => {
 			if (originalProgressCallback) originalProgressCallback(completed, total, index, evalStep, metrics);
 			if (isWebUI) {
@@ -4449,7 +5192,10 @@ var Evaluator = class {
 			if (serialRunEvalOptions.length > 0) logger.info(`Running ${serialRunEvalOptions.length} test cases serially...`);
 			if (concurrentRunEvalOptions.length > 0) logger.info(`Running ${concurrentRunEvalOptions.length} test cases (up to ${concurrency} at a time)...`);
 		}
-		if (this.options.showProgressBar && progressBarManager) await progressBarManager.initialize(runEvalOptions, concurrency, 0);
+		if (this.options.showProgressBar && progressBarManager) {
+			await progressBarManager.initialize(runEvalOptions, concurrency, 0);
+			progressBarManager.installLogInterceptor();
+		}
 		try {
 			if (serialRunEvalOptions.length > 0) for (const evalStep of serialRunEvalOptions) {
 				checkAbort();
@@ -4475,7 +5221,10 @@ var Evaluator = class {
 				else if (!targetUnavailable) {
 					logger.info("Evaluation interrupted, saving progress...");
 					if (globalTimeout) clearTimeout(globalTimeout);
-					if (progressBarManager) progressBarManager.stop();
+					if (progressBarManager) {
+						progressBarManager.removeLogInterceptor();
+						progressBarManager.stop();
+					}
 					if (ciProgressReporter) ciProgressReporter.finish();
 					this.evalRecord.setVars(Array.from(vars));
 					await this.evalRecord.addPrompts(prompts);
@@ -4483,6 +5232,10 @@ var Evaluator = class {
 					return this.evalRecord;
 				}
 			} else {
+				if (progressBarManager) {
+					progressBarManager.removeLogInterceptor();
+					progressBarManager.stop();
+				}
 				if (ciProgressReporter) ciProgressReporter.error(`Evaluation failed: ${String(err)}`);
 				throw err;
 			}
@@ -4625,6 +5378,7 @@ var Evaluator = class {
 		await this.evalRecord.addPrompts(prompts);
 		try {
 			if (progressBarManager) {
+				progressBarManager.removeLogInterceptor();
 				progressBarManager.complete();
 				progressBarManager.stop();
 			} else if (ciProgressReporter) ciProgressReporter.finish();
@@ -4790,6 +5544,6 @@ function evaluate(testSuite, evalRecord, options) {
 	return new Evaluator(testSuite, evalRecord, options).evaluate();
 }
 //#endregion
-export { runEval as a, readAssertions as c, isAllowedPrompt as i, renderMetricName as l, formatVarsForDisplay as n, doesPromptRefMatch as o, generateVarCombinations as r, assertions_default as s, evaluate as t, runAssertions as u };
+export { isAllowedPrompt as a, assertions_default as c, runAssertions as d, generateVarCombinations as i, readAssertions as l, evaluate as n, runEval as o, formatVarsForDisplay as r, doesPromptRefMatch as s, ProgressBarManager as t, renderMetricName as u };
 
-//# sourceMappingURL=evaluator-DPFRbFIL.js.map
+//# sourceMappingURL=evaluator-BcvOGaam.js.map