promptfoo 0.121.1 → 0.121.3

package/dist/src/index.js

@@ -1,40 +1,40 @@
-import { C as isCI, S as getMaxEvalTimeMs, _ as getEnvBool, a as setLogCallback, b as getEnvString, d as getAjv, h as summarizeEvaluateResultForLogging, i as logger, m as safeJsonStringify, n as isDebugEnabled, o as setLogLevel, p as orderKeys, t as getLogLevel, u as extractJsonObjects, v as getEnvFloat, w as state, x as getEvalTimeoutMs, y as getEnvInt } from "./logger-CT3IKMKA.js";
+import { C as getEnvString, D as state, E as isCI, S as getEnvInt, T as getMaxEvalTimeMs, _ as safeJsonStringify, a as logger, b as getEnvBool, g as orderKeys, m as getAjv, n as globalLogCallback, o as setLogCallback, p as extractJsonObjects, r as isDebugEnabled, s as setLogLevel, t as getLogLevel, v as summarizeEvaluateResultForLogging, w as getEvalTimeoutMs, x as getEnvFloat } from "./logger-DO8_zM18.js";
 import { t as invariant } from "./invariant-Ddh24eXh.js";
-import { r as importModule, t as getDirectory } from "./esm-Cd1AjG1D.js";
-import { r as runPython } from "./pythonUtils-D5nxkQ0P.js";
+import { r as importModule, t as getDirectory } from "./esm-SUNIX1x3.js";
+import { r as runPython } from "./pythonUtils-Cpo0Ez1p.js";
 import { i as isJavascriptFile } from "./fileExtensions-DnqA1y9x.js";
-import { i as getProcessShim, n as transform, t as TransformInputType } from "./transform-DECvGmzp.js";
-import { $ as matchesSearchRubric, A as BeavertailsPlugin, B as getAndCheckProvider, C as HarmbenchPlugin, D as DebugAccessPlugin, E as DivergentRepetitionPlugin, F as retryWithDeduplication, G as matchesContextFaithfulness, H as matchesAnswerRelevance, I as sampleArray, J as matchesFactuality, K as matchesContextRecall, L as fetchHuggingFaceDataset, M as RedteamGraderBase, N as RedteamPluginBase, O as CrossSessionLeakPlugin, P as getCustomPolicies, Q as matchesPiScore, R as callProviderWithContext, S as ImitationPlugin, T as ExcessiveAgencyPlugin, U as matchesClassification, V as loadRubricPrompt, W as matchesClosedQa, X as matchesLlmRubric, Y as matchesGEval, Z as matchesModeration, _ as makeInlinePolicyIdSync, a as UnverifiableClaimsPlugin, at as processPrompts, b as OverreliancePlugin, c as ToolDiscoveryPlugin, ct as SUGGEST_PROMPTS_SYSTEM_MESSAGE, d as RbacPlugin, dt as loadFromJavaScriptFile, et as matchesSelectBest, f as PromptExtractionPlugin, ft as processFileReference, g as isValidPolicyObject, h as determinePolicyTypeFromId, i as VLGuardPlugin, it as DefaultSuggestionsProvider, j as AegisPlugin, k as ContractPlugin, l as SqlInjectionPlugin, lt as coerceString, m as PolicyPlugin, n as getGraderById, nt as selectMaxScore, o as UnsafeBenchPlugin, ot as readPrompts, p as PoliticsPlugin, pt as resolveContext, q as matchesContextRelevance, r as VLSUPlugin, rt as getDefaultProviders, s as ToxicChatPlugin, st as readProviderPromptMap, t as GRADERS, tt as matchesSimilarity, u as ShellInjectionPlugin, ut as getFinalTest, v as PlinyPlugin, w as HallucinationPlugin, x as IntentPlugin, y as getPiiLeakTestsForCategory, z as fail } from "./graders-CpdqD9PI.js";
-import { A as isApiProvider, C as TestGeneratorConfigSchema, Ct as BaseTokenUsageSchema, D as VarsSchema, E as UnifiedConfigSchema, F as ConversationMessageSchema, I as PartialGenerationError, J as getDefaultNFanout, K as STRATEGY_COLLECTIONS, L as PluginConfigSchema, M as RedteamConfigSchema, O as isGradingResult, P as ProvidersSchema, Q as categoryAliases, R as PolicyObjectSchema, S as TestCasesWithMetadataSchema, St as PromptSchema, T as TestSuiteSchema, Tt as InputsSchema, V as isUuid, W as DEFAULT_STRATEGIES, X as isFanoutStrategy, Z as Severity, _ as ScenarioSchema, _t as REDTEAM_PROVIDER_HARM_PLUGINS, a as AtomicTestCaseSchema, at as FINANCIAL_PLUGINS, b as TestCaseWithVarsFileSchema, bt as TELECOM_PLUGINS, c as CompletedPromptSchema, ct as INSURANCE_PLUGINS, d as EvaluateOptionsSchema, dt as MEDICAL_PLUGINS, et as riskCategorySeverityMap, f as GradingConfigSchema, ft as MULTI_INPUT_EXCLUDED_PLUGINS, g as ResultFailureReason, gt as PLUGIN_CATEGORIES, h as OutputFileExtension, ht as PII_PLUGINS, i as AssertionTypeSchema, it as DEFAULT_PLUGINS, j as isProviderOptions, k as isResultFailureReason, l as DerivedMetricSchema, lt as LLAMA_GUARD_ENABLED_CATEGORIES, m as OutputConfigSchema, mt as PHARMACY_PLUGINS, n as AssertionSchema, nt as BIAS_PLUGINS, o as BaseAssertionTypesSchema, ot as FOUNDATION_PLUGINS, p as NotPrefixedAssertionTypesSchema, pt as MULTI_INPUT_VAR, q as STRATEGY_COLLECTION_MAPPINGS, r as AssertionSetSchema, rt as DATASET_EXEMPT_PLUGINS, s as CommandLineOptionsSchema, st as HARM_PLUGINS, t as AssertionOrSetSchema, tt as ALIASED_PLUGIN_MAPPINGS, u as EvalResultsFilterMode, ut as LLAMA_GUARD_REPLICATE_PROVIDER, v as SpecialAssertionTypesSchema, vt as REMOTE_ONLY_PLUGIN_IDS, w as TestSuiteConfigSchema, wt as CompletionTokenDetailsSchema, x as TestCasesWithMetadataPromptSchema, xt as UNALIGNED_PROVIDER_HARM_PLUGINS, y as TestCaseSchema, z as StrategyConfigSchema } from "./types-CLKiCBW3.js";
-import { A as getProviderDescription, C as deduplicateTestCases, D as resultIsForTestCase, E as getTestCaseDeduplicationKey, M as isGoogleProvider, N as isOpenAiProvider, O as checkProviderApiKeys, P as isProviderAllowed, S as setupEnv, T as filterRuntimeVars, b as loadFunction, c as maybeLoadFromExternalFile, d as maybeLoadToolsFromExternalFile, h as renderEnvOnlyInObject, i as fetchCsvFromGoogleSheet, j as isAnthropicProvider, k as doesProviderRefMatch, m as readOutput, n as writeMultipleOutputs, p as readFilters, r as writeOutput, s as maybeLoadConfigFromExternalFile, t as printBorder, v as extractVariablesFromTemplates, w as extractRuntimeVars, x as parseFileUrl, y as getNunjucksEngine } from "./util-Dlz_Wvgm.js";
-import { A as getShareApiBaseUrl, F as HUMAN_ASSERTION_TYPE, N as VERSION, O as TERMINAL_MAX_WIDTH, P as FILE_METADATA_KEY, _ as isPromptfooSampleTarget, a as CloudConfig, b as parseChatPrompt, d as sleep, j as getShareViewBaseUrl, k as getDefaultShareViewBaseUrl, n as fetchWithRetries, o as cloudConfig, p as REQUEST_TIMEOUT_MS, r as fetchWithTimeout, t as fetchWithProxy, u as getCurrentTimestamp } from "./fetch-60Gzydls.js";
-import { i as getCache, n as disableCache, o as NON_TRANSIENT_HTTP_STATUSES, r as fetchWithCache, s as isNonTransientHttpStatus, t as cache_exports } from "./cache-8XhNqPKW.js";
-import { A as createRateLimitRegistry, B as isCloudProvider, C as collectFileMetadata, D as loadFromPackage, E as isPackagePath, F as getCloudDatabaseId, I as getEvalConfigFromCloud, J as AIStudioChatProvider, L as getOrgContext, M as PromptfooHarmfulCompletionProvider, O as redteamProviderManager, P as checkCloudPermissions, R as getPluginSeverityOverridesFromCloud, T as runExtensionHook, V as resolveTeamId, _ as extractVariablesFromJson, a as resolveProviderConfigs, b as isBasicRefusal, c as Strategies, d as pluginMatchesStrategyTargets, f as checkExfilTracking, g as extractPromptFromTags, i as resolveProvider, j as createProviderRateLimitOptions, k as TokenUsageTracker, l as loadStrategy, m as extractGoalFromPrompt, n as loadApiProvider, o as MCPProvider, q as VertexChatProvider, r as loadApiProviders, s as GoogleLiveProvider, t as getProviderIds, u as validateStrategies, v as getSessionId, w as renderPrompt, y as getShortPluginId } from "./providers-BKRJTjBz.js";
-import { i as generateIdFromPrompt, t as hashPrompt } from "./utils-XiOAgly5.js";
+import { i as getProcessShim, n as transform, t as TransformInputType } from "./transform-BqPkNPYm.js";
+import { $ as matchesSearchRubric, A as BeavertailsPlugin, B as getAndCheckProvider, C as HarmbenchPlugin, D as DebugAccessPlugin, E as DivergentRepetitionPlugin, F as retryWithDeduplication, G as matchesContextFaithfulness, H as matchesAnswerRelevance, I as sampleArray, J as matchesFactuality, K as matchesContextRecall, L as fetchHuggingFaceDataset, M as RedteamGraderBase, N as RedteamPluginBase, O as CrossSessionLeakPlugin, P as getCustomPolicies, Q as matchesPiScore, R as callProviderWithContext, S as ImitationPlugin, T as ExcessiveAgencyPlugin, U as matchesClassification, V as loadRubricPrompt, W as matchesClosedQa, X as matchesLlmRubric, Y as matchesGEval, Z as matchesModeration, _ as makeInlinePolicyIdSync, a as UnverifiableClaimsPlugin, at as DefaultSuggestionsProvider, b as OverreliancePlugin, c as ToolDiscoveryPlugin, ct as readProviderPromptMap, d as RbacPlugin, dt as getFinalTest, et as matchesSelectBest, f as PromptExtractionPlugin, ft as loadFromJavaScriptFile, g as isValidPolicyObject, h as determinePolicyTypeFromId, i as VLGuardPlugin, it as getDefaultProviders, j as AegisPlugin, k as ContractPlugin, l as SqlInjectionPlugin, lt as SUGGEST_PROMPTS_SYSTEM_MESSAGE, m as PolicyPlugin, mt as resolveContext, n as getGraderById, nt as matchesTrajectoryGoalSuccess, o as UnsafeBenchPlugin, ot as processPrompts, p as PoliticsPlugin, pt as processFileReference, q as matchesContextRelevance, r as VLSUPlugin, rt as selectMaxScore, s as ToxicChatPlugin, st as readPrompts, t as GRADERS, tt as matchesSimilarity, u as ShellInjectionPlugin, ut as coerceString, v as PlinyPlugin, w as HallucinationPlugin, x as IntentPlugin, y as getPiiLeakTestsForCategory, z as fail } from "./graders-DG7mhg-b.js";
+import { A as isApiProvider, C as TestGeneratorConfigSchema, Ct as BaseTokenUsageSchema, D as VarsSchema, E as UnifiedConfigSchema, F as ConversationMessageSchema, I as PartialGenerationError, J as getDefaultNFanout, K as STRATEGY_COLLECTIONS, L as PluginConfigSchema, M as RedteamConfigSchema, O as isGradingResult, P as ProvidersSchema, Q as categoryAliases, R as PolicyObjectSchema, S as TestCasesWithMetadataSchema, St as PromptSchema, T as TestSuiteSchema, Tt as InputsSchema, V as isUuid, W as DEFAULT_STRATEGIES, X as isFanoutStrategy, Z as Severity, _ as ScenarioSchema, _t as REDTEAM_PROVIDER_HARM_PLUGINS, a as AtomicTestCaseSchema, at as FINANCIAL_PLUGINS, b as TestCaseWithVarsFileSchema, bt as TELECOM_PLUGINS, c as CompletedPromptSchema, ct as INSURANCE_PLUGINS, d as EvaluateOptionsSchema, dt as MEDICAL_PLUGINS, et as riskCategorySeverityMap, f as GradingConfigSchema, ft as MULTI_INPUT_EXCLUDED_PLUGINS, g as ResultFailureReason, gt as PLUGIN_CATEGORIES, h as OutputFileExtension, ht as PII_PLUGINS, i as AssertionTypeSchema, it as DEFAULT_PLUGINS, j as isProviderOptions, k as isResultFailureReason, l as DerivedMetricSchema, lt as LLAMA_GUARD_ENABLED_CATEGORIES, m as OutputConfigSchema, mt as PHARMACY_PLUGINS, n as AssertionSchema, nt as BIAS_PLUGINS, o as BaseAssertionTypesSchema, ot as FOUNDATION_PLUGINS, p as NotPrefixedAssertionTypesSchema, pt as MULTI_INPUT_VAR, q as STRATEGY_COLLECTION_MAPPINGS, r as AssertionSetSchema, rt as DATASET_EXEMPT_PLUGINS, s as CommandLineOptionsSchema, st as HARM_PLUGINS, t as AssertionOrSetSchema, tt as ALIASED_PLUGIN_MAPPINGS, u as EvalResultsFilterMode, ut as LLAMA_GUARD_REPLICATE_PROVIDER, v as SpecialAssertionTypesSchema, vt as REMOTE_ONLY_PLUGIN_IDS, w as TestSuiteConfigSchema, wt as CompletionTokenDetailsSchema, x as TestCasesWithMetadataPromptSchema, xt as UNALIGNED_PROVIDER_HARM_PLUGINS, y as TestCaseSchema, z as StrategyConfigSchema } from "./types-q8GXGF65.js";
+import { A as getProviderDescription, C as deduplicateTestCases, D as resultIsForTestCase, E as getTestCaseDeduplicationKey, M as isGoogleProvider, N as isOpenAiProvider, O as checkProviderApiKeys, P as isProviderAllowed, S as setupEnv, T as filterRuntimeVars, b as loadFunction, c as maybeLoadFromExternalFile, d as maybeLoadToolsFromExternalFile, h as renderEnvOnlyInObject, i as fetchCsvFromGoogleSheet, j as isAnthropicProvider, k as doesProviderRefMatch, m as readOutput, n as writeMultipleOutputs, p as readFilters, r as writeOutput, s as maybeLoadConfigFromExternalFile, t as printBorder, v as extractVariablesFromTemplates, w as extractRuntimeVars, x as parseFileUrl, y as getNunjucksEngine } from "./util-CMMkIxfU.js";
+import { A as getShareApiBaseUrl, F as HUMAN_ASSERTION_TYPE, N as VERSION, O as TERMINAL_MAX_WIDTH, P as FILE_METADATA_KEY, _ as isPromptfooSampleTarget, a as CloudConfig, b as parseChatPrompt, d as sleep, j as getShareViewBaseUrl, k as getDefaultShareViewBaseUrl, n as fetchWithRetries, o as cloudConfig, p as REQUEST_TIMEOUT_MS, r as fetchWithTimeout, t as fetchWithProxy, u as getCurrentTimestamp } from "./fetch-CVAtKnI3.js";
+import { i as getCache, n as disableCache, o as NON_TRANSIENT_HTTP_STATUSES, r as fetchWithCache, s as isNonTransientHttpStatus, t as cache_exports } from "./cache-CaT5tPgo.js";
+import { A as createRateLimitRegistry, B as isCloudProvider, C as collectFileMetadata, D as loadFromPackage, E as isPackagePath, F as getCloudDatabaseId, I as getEvalConfigFromCloud, J as AIStudioChatProvider, L as getOrgContext, M as PromptfooHarmfulCompletionProvider, O as redteamProviderManager, P as checkCloudPermissions, R as getPluginSeverityOverridesFromCloud, T as runExtensionHook, V as resolveTeamId, _ as extractVariablesFromJson, a as resolveProviderConfigs, b as isBasicRefusal, c as Strategies, d as pluginMatchesStrategyTargets, f as checkExfilTracking, g as extractPromptFromTags, i as resolveProvider, j as createProviderRateLimitOptions, k as TokenUsageTracker, l as loadStrategy, m as extractGoalFromPrompt, n as loadApiProvider, o as MCPProvider, q as VertexChatProvider, r as loadApiProviders, s as GoogleLiveProvider, t as getProviderIds, u as validateStrategies, v as getSessionId, w as renderPrompt, y as getShortPluginId } from "./providers-Cn73d5sr.js";
+import { i as generateIdFromPrompt, t as hashPrompt } from "./utils-CFxO9KGo.js";
 import { n as sha256, t as randomSequence } from "./createHash-DmPQkvBh.js";
 import "./genaiTracer-D3fD9dNV.js";
-import { t as OpenAiChatCompletionProvider } from "./chat-CznLWr_D.js";
+import { t as OpenAiChatCompletionProvider } from "./chat-pxmiVpWe.js";
 import { a as createEmptyTokenUsage, i as createEmptyAssertions, n as accumulateResponseTokenUsage, o as normalizeTokenUsage, r as accumulateTokenUsage, t as accumulateAssertionTokenUsage } from "./tokenUsageUtils-NYT-WKS6.js";
-import { m as validateFunctionCall } from "./transform-DGLazrMm.js";
-import "./messages-BLbWdsyt.js";
-import "./util-DaWTWKBK.js";
-import "./responses-BKqJmhhc.js";
-import "./openai-DElQ-fPX.js";
-import { l as validateFunctionCall$1 } from "./util-Betm42rL.js";
-import "./completion-C_P3ypkJ.js";
-import { c as setUserEmail, i as getUserEmail, o as isLoggedIntoCloud, r as getAuthor, s as promptForEmailUnverified, t as checkEmailStatusAndMaybeExit } from "./accounts-xrUGFA6n.js";
-import { i as getRemoteGenerationUrl, l as shouldGenerateRemote, o as getRemoteHealthUrl, r as promptYesNo, s as neverGenerateRemote } from "./server-BC7XJFgr.js";
-import { t as getBlobByHash } from "./blobs-Bpg5rH6i.js";
-import { a as evalsTable, c as evalsToTagsTable, d as tagsTable, i as evalResultsTable, l as promptsTable, m as getDbSignalPath, o as evalsToDatasetsTable, p as getDb, r as datasetsTable, s as evalsToPromptsTable } from "./tables-5EvT_Bwn.js";
-import { n as isBlobStorageEnabled, t as extractAndStoreBinaryData } from "./extractor-M67RUtg6.js";
-import { t as telemetry } from "./telemetry-C15ziL8u.js";
+import { m as validateFunctionCall } from "./transform-DyDAwEpE.js";
+import "./messages-Dy9QecMs.js";
+import "./util-vNmDL5DT.js";
+import "./responses-CQb1Tj69.js";
+import "./openai-BcB5KlTk.js";
+import { l as validateFunctionCall$1 } from "./util-CgDCK4KI.js";
+import "./completion-DCjv7RZ3.js";
+import { c as setUserEmail, i as getUserEmail, o as isLoggedIntoCloud, r as getAuthor, s as promptForEmailUnverified, t as checkEmailStatusAndMaybeExit } from "./accounts-B2XmGjty.js";
+import { i as getRemoteGenerationUrl, l as shouldGenerateRemote, o as getRemoteHealthUrl, r as promptYesNo, s as neverGenerateRemote } from "./server-CP9qKM40.js";
+import { t as getBlobByHash } from "./blobs-DXTl6J3H.js";
+import { a as evalsTable, c as evalsToTagsTable, d as tagsTable, i as evalResultsTable, l as promptsTable, m as getDbSignalPath, o as evalsToDatasetsTable, p as getDb, r as datasetsTable, s as evalsToPromptsTable } from "./tables-kC7R5kiK.js";
+import { n as isBlobStorageEnabled, t as extractAndStoreBinaryData } from "./extractor-D_wd8jxt.js";
+import { t as telemetry } from "./telemetry-BugWqKiu.js";
 import { t as ellipsize } from "./text-B_UCRPp2.js";
-import { t as getTraceStore } from "./store-DQLEjuEO.js";
-import "./base-B0tcrnq_.js";
-import "./image-BmEZqVmk.js";
-import { t as providerRegistry } from "./providerRegistry-CD8MEar9.js";
-import { n as runRuby } from "./rubyUtils-BUVePouc.js";
-import { t as EvalResult } from "./evalResult-CDQiuUuf.js";
+import { t as getTraceStore } from "./store-Cj258DgL.js";
+import "./base-CqzQ4K8j.js";
+import "./image-CoxZp9PZ.js";
+import { t as providerRegistry } from "./providerRegistry-CUWki5mQ.js";
+import { n as runRuby } from "./rubyUtils-PgU-gHmx.js";
+import { t as EvalResult } from "./evalResult-BkIhRdTe.js";
 import * as fs$1 from "fs";
 import fs, { createWriteStream } from "fs";
 import * as path$2 from "path";
@@ -56,11 +56,13 @@ import { XMLParser } from "fast-xml-parser";
 import crypto$1, { createHash, randomBytes } from "crypto";
 import { DiagConsoleLogger, DiagLogLevel, diag, propagation } from "@opentelemetry/api";
 import input from "@inquirer/input";
+import readline from "readline";
 import { and, desc, eq, inArray, sql } from "drizzle-orm";
 import cliProgress from "cli-progress";
 import { JSDOM } from "jsdom";
 import { distance } from "fastest-levenshtein";
 import * as rouge from "js-rouge";
+import { isDeepStrictEqual } from "node:util";
 import "debounce";
 import { ExportResultCode, W3CTraceContextPropagator } from "@opentelemetry/core";
 import { OTLPTraceExporter } from "@opentelemetry/exporter-trace-otlp-http";
@@ -288,7 +290,7 @@ async function startOtlpReceiverIfNeeded(testSuite) {
 		telemetry.record("feature_used", { feature: "tracing" });
 		try {
 			logger.debug("[EvaluatorTracing] Tracing configuration detected, starting OTLP receiver");
-			const { startOTLPReceiver } = await import("./otlpReceiver--AIRW_S4.js");
+			const { startOTLPReceiver } = await import("./otlpReceiver-CZL48YfC.js");
 			const port = testSuite.tracing.otlp.http.port || 4318;
 			const host = testSuite.tracing.otlp.http.host || "127.0.0.1";
 			logger.debug(`[EvaluatorTracing] Starting OTLP receiver on ${host}:${port}`);
@@ -311,7 +313,7 @@ async function startOtlpReceiverIfNeeded(testSuite) {
 async function stopOtlpReceiverIfNeeded() {
 	if (otlpReceiverStarted) try {
 		logger.debug("[EvaluatorTracing] Stopping OTLP receiver");
-		const { stopOTLPReceiver } = await import("./otlpReceiver--AIRW_S4.js");
+		const { stopOTLPReceiver } = await import("./otlpReceiver-CZL48YfC.js");
 		await stopOTLPReceiver();
 		otlpReceiverStarted = false;
 		logger.info("[EvaluatorTracing] OTLP receiver stopped successfully");
@@ -346,7 +348,7 @@ async function generateTraceContextIfNeeded(test, evaluateOptions, testIdx, prom
 	}
 	if (!tracingEnabled) return null;
 	logger.debug("[EvaluatorTracing] Importing trace store");
-	const { getTraceStore } = await import("./store-DQLEjuEO.js").then((n) => n.n);
+	const { getTraceStore } = await import("./store-Cj258DgL.js").then((n) => n.n);
 	const traceStore = getTraceStore();
 	const traceId = generateTraceId();
 	const spanId = generateSpanId();
@@ -1379,7 +1381,7 @@ const handleJavascript = async ({ assertion, renderedValue, valueFromScript, ass
 			pass = result !== inverse;
 			score = pass ? 1 : 0;
 		} else if (typeof result === "number") {
-			pass = assertion.threshold !== void 0 ? result >= assertion.threshold : result > 0;
+			pass = assertion.threshold === void 0 ? result > 0 : result >= assertion.threshold;
 			score = result;
 		} else if (typeof result === "object") return result;
 		else throw new Error("Custom function must return a boolean or number");
@@ -1412,7 +1414,7 @@ function handleIsJson({ outputString, renderedValue, inverse, valueFromScript, a
 	} catch {
 		pass = inverse;
 	}
-	if (pass && renderedValue) {
+	if (parsedJson !== void 0 && renderedValue) {
 		let validate;
 		if (typeof renderedValue === "string") if (renderedValue.startsWith("file://")) {
 			const schema = valueFromScript;
@@ -1424,11 +1426,12 @@ function handleIsJson({ outputString, renderedValue, inverse, valueFromScript, a
 		}
 		else if (typeof renderedValue === "object") validate = getAjv().compile(renderedValue);
 		else throw new Error("is-json assertion must have a string or object value");
-		pass = validate(parsedJson);
+		const valid = validate(parsedJson);
+		pass = inverse ? !valid : valid;
 		if (!pass) return {
 			pass,
 			score: 0,
-			reason: `JSON does not conform to the provided schema. Errors: ${getAjv().errorsText(validate.errors)}`,
+			reason: inverse ? "Output is JSON that conforms to the provided schema" : `JSON does not conform to the provided schema. Errors: ${getAjv().errorsText(validate.errors)}`,
 			assertion
 		};
 	}
@@ -1455,9 +1458,12 @@ function handleContainsJson({ assertion, renderedValue, outputString, inverse, v
 		}
 		else if (typeof renderedValue === "object") validate = getAjv().compile(renderedValue);
 		else throw new Error("contains-json assertion must have a string or object value");
-		pass = validate(jsonObject);
-		if (pass) break;
-		else errorMessage = `JSON does not conform to the provided schema. Errors: ${getAjv().errorsText(validate.errors)}`;
+		const valid = validate(jsonObject);
+		pass = inverse ? !valid : valid;
+		if (valid) {
+			if (inverse) errorMessage = "Output contains JSON conforming to the provided schema";
+			break;
+		} else errorMessage = `JSON does not conform to the provided schema. Errors: ${getAjv().errorsText(validate.errors)}`;
 	}
 	return {
 		pass,
@@ -1641,7 +1647,7 @@ function handlePerplexity({ logProbs, assertion }) {
 	if (!logProbs || logProbs.length === 0) throw new Error("Perplexity assertion does not support providers that do not return logProbs");
 	const avgLogProb = logProbs.reduce((acc, logProb) => acc + logProb, 0) / logProbs.length;
 	const perplexity = Math.exp(-avgLogProb);
-	const pass = assertion.threshold !== void 0 ? perplexity <= assertion.threshold : true;
+	const pass = assertion.threshold === void 0 ? true : perplexity <= assertion.threshold;
 	return {
 		pass,
 		score: pass ? 1 : 0,
@@ -1653,7 +1659,7 @@ function handlePerplexityScore({ logProbs, assertion }) {
 	if (!logProbs || logProbs.length === 0) throw new Error("perplexity-score assertion does not support providers that do not return logProbs");
 	const avgLogProb = logProbs.reduce((acc, logProb) => acc + logProb, 0) / logProbs.length;
 	const perplexityNorm = 1 / (1 + Math.exp(-avgLogProb));
-	const pass = assertion.threshold !== void 0 ? perplexityNorm >= assertion.threshold : true;
+	const pass = assertion.threshold === void 0 ? true : perplexityNorm >= assertion.threshold;
 	return {
 		pass,
 		score: perplexityNorm,
@@ -1768,7 +1774,7 @@ ${isMultiline ? renderedValue.split("\n").map((line) => `${indentStyle}${line}`)
 		} else {
 			score = Number.parseFloat(String(result));
 			if (Number.isNaN(score)) throw new Error(`Python assertion must return a boolean, number, or {pass, score, reason} object. Instead got:\n${result}`);
-			pass = assertion.threshold !== void 0 ? score >= assertion.threshold : score > 0;
+			pass = assertion.threshold === void 0 ? score > 0 : score >= assertion.threshold;
 		}
 	} catch (err) {
 		return {
@@ -2029,7 +2035,7 @@ end
 		} else {
 			score = Number.parseFloat(String(result));
 			if (Number.isNaN(score)) throw new Error(`Ruby assertion must return a boolean, number, or {pass, score, reason} object. Instead got:\n${result}`);
-			pass = assertion.threshold !== void 0 ? score >= assertion.threshold : score > 0;
+			pass = assertion.threshold === void 0 ? score > 0 : score >= assertion.threshold;
 		}
 	} catch (err) {
 		return {
@@ -2100,6 +2106,127 @@ const handleSimilar = async ({ assertion, renderedValue, outputString, inverse,
 	};
 };
 //#endregion
+//#region src/assertions/traceUtils.ts
+/**
+* Shared utilities for trace assertions
+*/
+/**
+* Match a span name against a glob-like pattern.
+* Supports * (any characters) and ? (single character) wildcards.
+*
+* @param spanName - The span name to match
+* @param pattern - The glob pattern to match against
+* @returns true if the span name matches the pattern
+*/
+function matchesPattern(spanName, pattern) {
+	const regexPattern = pattern.replace(/[.+^${}()|[\]\\]/g, "\\$&").replace(/\*/g, ".*").replace(/\?/g, ".");
+	return new RegExp(`^${regexPattern}$`, "i").test(spanName);
+}
+//#endregion
+//#region src/assertions/skill.ts
+function getSkillCalls(params) {
+	const rawSkillCalls = params.providerResponse?.metadata?.skillCalls;
+	if (!Array.isArray(rawSkillCalls)) return [];
+	return rawSkillCalls.filter((entry) => Boolean(entry) && typeof entry === "object" && typeof entry.name === "string");
+}
+function matchesSkill(skillCall, matcher) {
+	if (matcher.name && skillCall.name !== matcher.name) return false;
+	if (matcher.pattern && !matchesPattern(skillCall.name, matcher.pattern)) return false;
+	return true;
+}
+function formatSkillCall(skillCall) {
+	const details = [skillCall.source, skillCall.path].filter(Boolean).join(", ");
+	return details ? `${skillCall.name} (${details})` : skillCall.name;
+}
+function resolveSkillMatchers(value) {
+	const normalizeText = (text) => typeof text === "string" ? text.trim() : void 0;
+	const validateCount = (field, count) => {
+		if (!Number.isFinite(count) || !Number.isInteger(count) || count < 0) throw new Error(`skill-used assertion object ${field} must be a finite non-negative integer`);
+	};
+	if (typeof value === "string" && value.trim()) return {
+		kind: "list",
+		matchers: [{ name: normalizeText(value) }]
+	};
+	if (Array.isArray(value) && value.length > 0 && value.every((item) => typeof item === "string" && item.trim())) return {
+		kind: "list",
+		matchers: value.map((item) => ({ name: item.trim() }))
+	};
+	if (value && typeof value === "object" && !Array.isArray(value)) {
+		const rawMatcher = value;
+		const matcher = rawMatcher;
+		const name = normalizeText(matcher.name);
+		const pattern = normalizeText(matcher.pattern);
+		if (!name && !pattern) throw new Error("skill-used assertion object must include a name or pattern property");
+		if ("min" in rawMatcher) validateCount("min", matcher.min);
+		if ("max" in rawMatcher) validateCount("max", matcher.max);
+		if (typeof matcher.min === "number" && typeof matcher.max === "number" && matcher.max < matcher.min) throw new Error("skill-used assertion object max must be greater than or equal to min");
+		return {
+			kind: "count",
+			matcher: {
+				max: typeof matcher.max === "number" ? matcher.max : void 0,
+				min: typeof matcher.min === "number" ? matcher.min : void 0,
+				name,
+				pattern
+			}
+		};
+	}
+	throw new Error("skill-used assertion must have a string, string array, or object value");
+}
+function handleListSkillAssertion(params, skillCalls, actualSkills, expected) {
+	const missing = expected.matchers.filter((matcher) => !skillCalls.some((skillCall) => matchesSkill(skillCall, matcher)));
+	const matched = expected.matchers.filter((matcher) => skillCalls.some((skillCall) => matchesSkill(skillCall, matcher)));
+	const pass = params.inverse ? matched.length === 0 : missing.length === 0;
+	const expectedSkills = expected.matchers.map((matcher) => matcher.name);
+	const actualSummary = actualSkills.length > 0 ? actualSkills.join(", ") : "(none)";
+	let reason;
+	if (params.inverse) reason = pass ? `Forbidden skill(s) were not used: ${expectedSkills.join(", ")}` : `Forbidden skill(s) were used: ${matched.map((matcher) => matcher.name).join(", ")}. Actual skills: ${actualSummary}`;
+	else if (pass) reason = `Observed required skill(s): ${expectedSkills.join(", ")}. Actual skills: ${actualSummary}`;
+	else reason = `Missing required skill(s): ${missing.map((matcher) => matcher.name).join(", ")}. Actual skills: ${actualSummary}`;
+	return {
+		pass,
+		score: pass ? 1 : 0,
+		reason,
+		assertion: params.assertion
+	};
+}
+function handleCountSkillAssertion(params, skillCalls, actualSkills, matcher) {
+	const hasExplicitMin = matcher.min !== void 0;
+	const hasExplicitMax = matcher.max !== void 0;
+	const min = matcher.min ?? (hasExplicitMax ? 0 : 1);
+	const max = matcher.max;
+	const matchingSkillCalls = skillCalls.filter((skillCall) => matchesSkill(skillCall, matcher));
+	const count = matchingSkillCalls.length;
+	const matcherLabel = matcher.pattern || matcher.name || "*";
+	if (params.inverse) {
+		if (hasExplicitMin || hasExplicitMax && max !== 0) throw new Error("not-skill-used object assertions only support name/pattern with no count bounds, or max: 0");
+		const pass = count === 0;
+		const actualSummary = actualSkills.length > 0 ? actualSkills.join(", ") : "(none)";
+		return {
+			pass,
+			score: pass ? 1 : 0,
+			reason: pass ? `Forbidden skill "${matcherLabel}" was not used. Actual skills: ${actualSummary}` : `Forbidden skill "${matcherLabel}" was used ${count} time(s). Matches: ${matchingSkillCalls.map(formatSkillCall).join(", ")}`,
+			assertion: params.assertion
+		};
+	}
+	const pass = count >= min && (max === void 0 || count <= max);
+	let reason = `Matched skill "${matcherLabel}" ${count} time(s)`;
+	reason += max === void 0 ? ` (expected at least ${min})` : ` (expected ${min}-${max})`;
+	if (matchingSkillCalls.length > 0) reason += `. Matches: ${matchingSkillCalls.map(formatSkillCall).join(", ")}`;
+	return {
+		pass,
+		score: pass ? 1 : 0,
+		reason,
+		assertion: params.assertion
+	};
+}
+function handleSkillUsed(params) {
+	const skillCalls = getSkillCalls(params);
+	const actualSkills = skillCalls.map(formatSkillCall);
+	const expected = resolveSkillMatchers(params.renderedValue ?? params.assertion.value);
+	if (expected.kind === "list") return handleListSkillAssertion(params, skillCalls, actualSkills, expected);
+	return handleCountSkillAssertion(params, skillCalls, actualSkills, expected.matcher);
+}
+//#endregion
 //#region src/assertions/sql.ts
 const handleIsSql = async ({ assertion, renderedValue, outputString, inverse }) => {
 	let pass = false;
@@ -2332,23 +2459,6 @@ const handleToolCallF1 = ({ assertion, output, renderedValue, inverse }) => {
 	};
 };
 //#endregion
-//#region src/assertions/traceUtils.ts
-/**
-* Shared utilities for trace assertions
-*/
-/**
-* Match a span name against a glob-like pattern.
-* Supports * (any characters) and ? (single character) wildcards.
-*
-* @param spanName - The span name to match
-* @param pattern - The glob pattern to match against
-* @returns true if the span name matches the pattern
-*/
-function matchesPattern(spanName, pattern) {
-	const regexPattern = pattern.replace(/[.+^${}()|[\]\\]/g, "\\$&").replace(/\*/g, ".*").replace(/\?/g, ".");
-	return new RegExp(`^${regexPattern}$`, "i").test(spanName);
-}
-//#endregion
 //#region src/assertions/traceErrorSpans.ts
 function isErrorSpan(span) {
 	if (span.statusCode && span.statusCode >= 400) return true;
@@ -2517,6 +2627,524 @@ const handleTraceSpanDuration = ({ assertion, assertionValueContext }) => {
 	};
 };
 //#endregion
+//#region src/assertions/trajectoryUtils.ts
+const TOOL_ATTRIBUTE_KEYS = [
+	"tool.name",
+	"tool_name",
+	"tool",
+	"function.name",
+	"function_name",
+	"gen_ai.tool.name",
+	"codex.mcp.tool",
+	"agent.tool",
+	"agent.tool_name",
+	"agent.toolName"
+];
+const TOOL_ARGUMENT_ATTRIBUTE_KEYS = [
+	"tool.arguments",
+	"tool.args",
+	"tool.input",
+	"tool_arguments",
+	"tool_args",
+	"tool_input",
+	"function.arguments",
+	"function.args",
+	"function.input",
+	"function_arguments",
+	"function_args",
+	"gen_ai.tool.arguments",
+	"gen_ai.tool.args",
+	"gen_ai.tool.input",
+	"gen_ai.tool.call.arguments",
+	"gen_ai.tool.call.args",
+	"agent.tool.arguments",
+	"agent.tool.args",
+	"agent.tool.input",
+	"codex.mcp.arguments",
+	"codex.mcp.args",
+	"codex.mcp.input",
+	"arguments",
+	"args",
+	"input"
+];
+const COMMAND_ATTRIBUTE_KEYS = [
+	"codex.command",
+	"command",
+	"command.name",
+	"command_name"
+];
+const SEARCH_ATTRIBUTE_KEYS = [
+	"codex.search.query",
+	"search.query",
+	"search_query"
+];
+const GENERIC_QUERY_ATTRIBUTE_KEYS = ["query"];
+const SEARCH_SPAN_NAME_PATTERN = /(^|[\s._:/-])(search|find|lookup|retriev(?:e|al))($|[\s._:/-])/i;
+const MAX_JUDGE_SUMMARY_STEPS = 24;
+const JUDGE_SUMMARY_HEAD_STEPS = 12;
+const JUDGE_SUMMARY_TAIL_STEPS = 12;
+function getStringAttribute(attributes, keys) {
+	for (const key of keys) {
+		const value = attributes[key];
+		if (typeof value === "string" && value.trim()) return value.trim();
+	}
+}
+function normalizeStructuredAttribute(value) {
+	if (value === void 0 || value === null) return;
+	if (typeof value === "string") {
+		const trimmed = value.trim();
+		if (!trimmed) return;
+		try {
+			return JSON.parse(trimmed);
+		} catch {
+			return trimmed;
+		}
+	}
+	if (typeof value === "number" || typeof value === "boolean" || typeof value === "object") return value;
+}
+function hasSameStatus(left, right) {
+	return left?.code === right?.code && left?.message === right?.message;
+}
+function isSearchLikeSpan(span) {
+	const attributes = span.attributes || {};
+	if (SEARCH_SPAN_NAME_PATTERN.test(span.name) || span.name.startsWith("search ")) return true;
+	return Object.keys(attributes).some((key) => key !== "query" && /(^|[._])(search|lookup|retriev(?:e|al))($|[._])/i.test(key));
+}
+function getTrajectoryStepStatus(step) {
+	if (step.statusCode === void 0 || step.statusCode === 0) return;
+	return {
+		code: step.statusCode,
+		...step.statusMessage ? { message: step.statusMessage } : {}
+	};
+}
+function getCommandExecutable(command) {
+	return command.trim().split(/\s+/)[0] || void 0;
+}
+function extractToolName(span) {
+	const attributes = span.attributes || {};
+	const directMatch = getStringAttribute(attributes, TOOL_ATTRIBUTE_KEYS);
+	if (directMatch) return directMatch;
+	for (const [key, value] of Object.entries(attributes)) {
+		if (typeof value !== "string" || !value.trim()) continue;
+		if (/tool.?name|function.?name/i.test(key)) return value.trim();
+		if (/(^|[._])tool($|[._])/i.test(key) && !/result|output/i.test(key)) return value.trim();
+	}
+	if (span.name.startsWith("mcp ")) {
+		const slashIndex = span.name.lastIndexOf("/");
+		if (slashIndex !== -1 && slashIndex < span.name.length - 1) return span.name.slice(slashIndex + 1).trim();
+	}
+}
+function extractToolArgs(span) {
+	const attributes = span.attributes || {};
+	for (const key of TOOL_ARGUMENT_ATTRIBUTE_KEYS) {
+		const value = normalizeStructuredAttribute(attributes[key]);
+		if (value !== void 0) return value;
+	}
+	for (const [key, rawValue] of Object.entries(attributes)) {
+		if (/result|output|error|status/i.test(key)) continue;
+		if (!/(^|[._])(arguments|args|input)($|[._])/i.test(key)) continue;
+		const value = normalizeStructuredAttribute(rawValue);
+		if (value !== void 0) return value;
+	}
+}
+function extractCommand(span) {
+	const attributes = span.attributes || {};
+	const directMatch = getStringAttribute(attributes, COMMAND_ATTRIBUTE_KEYS);
+	if (directMatch) return directMatch;
+	for (const [key, value] of Object.entries(attributes)) {
+		if (typeof value !== "string" || !value.trim()) continue;
+		if (/command/i.test(key) && !/output|result/i.test(key)) return value.trim();
+	}
+	if (span.name.startsWith("exec ")) return span.name.slice(5).trim();
+}
+function extractSearchQuery(span) {
+	const attributes = span.attributes || {};
+	const directMatch = getStringAttribute(attributes, SEARCH_ATTRIBUTE_KEYS);
+	if (directMatch) return directMatch;
+	const genericQuery = getStringAttribute(attributes, GENERIC_QUERY_ATTRIBUTE_KEYS);
+	if (genericQuery && isSearchLikeSpan(span)) return genericQuery;
+	if (span.name.startsWith("search ")) return span.name.slice(7).replace(/^"|"$/g, "").trim();
+}
+function isReasoningSpan(span) {
+	if ((span.attributes || {})["codex.item.type"] === "reasoning") return true;
+	return /^reasoning([_\s]|$)/i.test(span.name) || span.name === "reasoning";
+}
+function isMessageSpan(span) {
+	if ((span.attributes || {})["codex.item.type"] === "agent_message") return true;
+	return span.name === "agent response" || span.name === "send input";
+}
+function extractTrajectorySteps(trace) {
+	return [...trace.spans || []].map((span, index) => ({
+		span,
+		index
+	})).sort((left, right) => {
+		const timeDiff = left.span.startTime - right.span.startTime;
+		if (timeDiff !== 0) return timeDiff;
+		const endDiff = (left.span.endTime ?? left.span.startTime) - (right.span.endTime ?? right.span.startTime);
+		if (endDiff !== 0) return endDiff;
+		return left.index - right.index;
+	}).map(({ span }) => {
+		const toolName = extractToolName(span);
+		const command = extractCommand(span);
+		const searchQuery = extractSearchQuery(span);
+		let type = "span";
+		let name = span.name;
+		const aliases = new Set([span.name]);
+		let args;
+		if (toolName) {
+			type = "tool";
+			name = toolName;
+			aliases.add(toolName);
+			args = extractToolArgs(span);
+		} else if (command) {
+			type = "command";
+			name = command;
+			aliases.add(command);
+			const executable = getCommandExecutable(command);
+			if (executable) aliases.add(executable);
+		} else if (searchQuery) {
+			type = "search";
+			name = searchQuery;
+			aliases.add(searchQuery);
+		} else if (isReasoningSpan(span)) {
+			type = "reasoning";
+			name = span.name;
+			aliases.add("reasoning");
+		} else if (isMessageSpan(span)) {
+			type = "message";
+			name = span.name;
+			aliases.add("message");
+		}
+		return {
+			aliases: [...aliases],
+			...args === void 0 ? {} : { args },
+			attributes: span.attributes || {},
+			endTime: span.endTime,
+			name,
+			spanId: span.spanId,
+			spanName: span.name,
+			startTime: span.startTime,
+			statusCode: span.statusCode,
+			statusMessage: span.statusMessage,
+			type
+		};
+	});
+}
+function normalizeTrajectoryMatcher(matcher, defaultType) {
+	if (typeof matcher === "string") return {
+		pattern: matcher,
+		...defaultType ? { type: defaultType } : {}
+	};
+	return {
+		...matcher,
+		...matcher.type ? {} : defaultType ? { type: defaultType } : {}
+	};
+}
+function matchesTrajectoryStep(step, matcher, defaultType) {
+	const { type, pattern, name } = normalizeTrajectoryMatcher(matcher, defaultType);
+	if (type) {
+		if (!(Array.isArray(type) ? type : [type]).includes(step.type)) return false;
+	}
+	const matchPattern = pattern || name;
+	if (!matchPattern) return true;
+	return step.aliases.some((alias) => matchesPattern(alias, matchPattern));
+}
+function formatTrajectoryStep(step) {
+	return `${step.type}:${step.name}`;
+}
+function formatTrajectoryArgs(args) {
+	if (args === void 0) return "(none)";
+	try {
+		const serialized = JSON.stringify(args);
+		if (serialized !== void 0) return serialized;
+	} catch {}
+	return String(args);
+}
+function compactJudgeTrajectorySteps(steps) {
+	const compacted = [];
+	for (const step of steps) {
+		const previousStep = compacted[compacted.length - 1];
+		if (previousStep && previousStep.type === step.type && previousStep.name === step.name && previousStep.spanName === step.spanName && hasSameStatus(previousStep.status, step.status)) {
+			previousStep.collapsedCount = (previousStep.collapsedCount ?? 1) + 1;
+			continue;
+		}
+		compacted.push(step);
+	}
+	return compacted;
+}
+function truncateJudgeTrajectorySteps(steps) {
+	if (steps.length <= MAX_JUDGE_SUMMARY_STEPS) return steps;
+	return [
+		...steps.slice(0, JUDGE_SUMMARY_HEAD_STEPS),
+		{ omittedCount: steps.length - MAX_JUDGE_SUMMARY_STEPS },
+		...steps.slice(-JUDGE_SUMMARY_TAIL_STEPS)
+	];
+}
+function summarizeTrajectoryForJudge(trace) {
+	const rawSteps = extractTrajectorySteps(trace).map((step, index) => ({
+		index: index + 1,
+		type: step.type,
+		name: step.name,
+		...step.spanName === step.name ? {} : { spanName: step.spanName },
+		...getTrajectoryStepStatus(step) ? { status: getTrajectoryStepStatus(step) } : {}
+	}));
+	const compactedSteps = compactJudgeTrajectorySteps(rawSteps);
+	const steps = truncateJudgeTrajectorySteps(compactedSteps);
+	return JSON.stringify({
+		traceId: trace.traceId,
+		stepCount: rawSteps.length,
+		compactedStepCount: compactedSteps.length,
+		steps
+	}, null, 2);
+}
+//#endregion
+//#region src/assertions/trajectory.ts
+function getTraceOrThrow(params) {
+	const trace = params.assertionValueContext.trace;
+	if (!trace || !trace.spans) throw new Error(`No trace data available for ${params.baseType} assertion`);
+	return trace;
+}
+function applyInverse(pass, inverse) {
+	return inverse ? !pass : pass;
+}
+function formatStepList(stepLabels) {
+	return stepLabels.length > 0 ? stepLabels.join(", ") : "(none)";
+}
+function requireNamedTrajectoryMatcher(matcher, assertionType, index) {
+	if (matcher.pattern || matcher.name) return;
+	const stepLabel = index === void 0 ? "object" : `step ${index + 1}`;
+	throw new Error(`${assertionType} assertion ${stepLabel} must include a name or pattern property`);
+}
+function resolveGoalSuccessValue(value) {
+	if (typeof value === "string" && value.trim()) return { goal: value.trim() };
+	if (value && typeof value === "object" && !Array.isArray(value) && typeof value.goal === "string" && value.goal.trim()) return { goal: value.goal.trim() };
+	throw new Error("trajectory:goal-success assertion must have a string value or an object with a goal property");
+}
+function resolveToolMatchers(value) {
+	if (typeof value === "string") return {
+		kind: "list",
+		matchers: [normalizeTrajectoryMatcher(value, "tool")]
+	};
+	if (Array.isArray(value) && value.every((item) => typeof item === "string")) return {
+		kind: "list",
+		matchers: value.map((item) => normalizeTrajectoryMatcher(item, "tool"))
+	};
+	if (value && typeof value === "object" && !Array.isArray(value)) return {
+		kind: "count",
+		matcher: {
+			...normalizeTrajectoryMatcher(value, "tool"),
+			max: typeof value.max === "number" ? value.max : void 0,
+			min: typeof value.min === "number" ? value.min : void 0
+		}
+	};
+	throw new Error("trajectory:tool-used assertion must have a string, string array, or object value");
+}
+const handleTrajectoryToolUsed = (params) => {
+	const steps = extractTrajectorySteps(getTraceOrThrow(params)).filter((step) => step.type === "tool");
+	const expected = resolveToolMatchers(params.renderedValue ?? params.assertion.value);
+	if (expected.kind === "list") {
+		if (expected.matchers.length === 0) throw new Error("trajectory:tool-used assertion requires at least one expected tool");
+		const missing = expected.matchers.filter((matcher) => !steps.some((step) => matchesTrajectoryStep(step, matcher)));
+		const matched = expected.matchers.filter((matcher) => steps.some((step) => matchesTrajectoryStep(step, matcher)));
+		const pass = params.inverse ? matched.length === 0 : missing.length === 0;
+		const actualTools = steps.map(formatTrajectoryStep);
+		const expectedTools = expected.matchers.map((matcher) => matcher.pattern || matcher.name || "*");
+		let reason;
+		if (params.inverse) reason = pass ? `Forbidden tool(s) were not used: ${expectedTools.join(", ")}` : `Forbidden tool(s) were used: ${matched.map((matcher) => matcher.pattern || matcher.name || "*").join(", ")}. Actual tools: ${formatStepList(actualTools)}`;
+		else if (pass) reason = `Observed required tool(s): ${expectedTools.join(", ")}. Actual tools: ${formatStepList(actualTools)}`;
+		else reason = `Missing required tool(s): ${missing.map((matcher) => matcher.pattern || matcher.name || "*").join(", ")}. Actual tools: ${formatStepList(actualTools)}`;
+		return {
+			pass,
+			score: pass ? 1 : 0,
+			reason,
+			assertion: params.assertion
+		};
+	}
+	const matcher = expected.matcher;
+	const min = matcher.min ?? 1;
+	const max = matcher.max;
+	if (!matcher.pattern && !matcher.name) throw new Error("trajectory:tool-used assertion object must include a name or pattern property");
+	const matchingSteps = steps.filter((step) => matchesTrajectoryStep(step, matcher));
+	const count = matchingSteps.length;
+	const basePass = count >= min && (max === void 0 || count <= max);
+	const pass = applyInverse(basePass, params.inverse);
+	const matcherLabel = matcher.pattern || matcher.name || "*";
+	let reason = `Matched tool "${matcherLabel}" ${count} time(s)`;
+	if (max === void 0) reason += ` (expected at least ${min})`;
+	else reason += ` (expected ${min}-${max})`;
+	if (matchingSteps.length > 0) reason += `. Matches: ${matchingSteps.map(formatTrajectoryStep).join(", ")}`;
+	if (params.inverse) reason = basePass ? `Tool "${matcherLabel}" matched ${count} time(s), which violates the inverse assertion` : `Tool "${matcherLabel}" did not satisfy the forbidden match condition`;
+	return {
+		pass,
+		score: pass ? 1 : 0,
+		reason,
+		assertion: params.assertion
+	};
+};
+function resolveSequenceValue(value) {
+	if (Array.isArray(value)) return {
+		mode: "in_order",
+		steps: value
+	};
+	if (value && typeof value === "object" && !Array.isArray(value)) {
+		const sequenceValue = value;
+		return {
+			mode: sequenceValue.mode || "in_order",
+			steps: sequenceValue.steps || []
+		};
+	}
+	throw new Error("trajectory:tool-sequence assertion must have an array or object value");
+}
+function isRecord(value) {
+	return typeof value === "object" && value !== null && !Array.isArray(value);
+}
+function matchesExpectedArgsPartial(actual, expected) {
+	if (Array.isArray(expected)) return Array.isArray(actual) && actual.length === expected.length && expected.every((item, index) => matchesExpectedArgsPartial(actual[index], item));
+	if (isRecord(expected)) {
+		if (!isRecord(actual)) return false;
+		return Object.entries(expected).every(([key, expectedValue]) => Object.prototype.hasOwnProperty.call(actual, key) && matchesExpectedArgsPartial(actual[key], expectedValue));
+	}
+	return isDeepStrictEqual(actual, expected);
+}
+function matchesToolArgs(actual, expected, mode) {
+	if (mode === "exact") return isDeepStrictEqual(actual, expected);
+	return matchesExpectedArgsPartial(actual, expected);
+}
+function resolveToolArgsMatchMode(mode) {
+	if (mode === void 0) return "partial";
+	if (mode === "partial" || mode === "exact") return mode;
+	throw new Error("trajectory:tool-args-match assertion mode must be \"partial\" or \"exact\"");
+}
+function resolveToolArgsMatchValue(value) {
+	if (!value || typeof value !== "object" || Array.isArray(value)) throw new Error("trajectory:tool-args-match assertion must have an object value");
+	const matcher = normalizeTrajectoryMatcher(value, "tool");
+	requireNamedTrajectoryMatcher(matcher, "trajectory:tool-args-match");
+	const expectedArgs = Object.prototype.hasOwnProperty.call(value, "args") ? value.args : value.arguments;
+	if (expectedArgs === void 0) throw new Error("trajectory:tool-args-match assertion must include an args or arguments property");
+	return {
+		matcher,
+		expectedArgs,
+		mode: resolveToolArgsMatchMode(value.mode)
+	};
+}
+const handleTrajectoryToolSequence = (params) => {
+	const toolSteps = extractTrajectorySteps(getTraceOrThrow(params)).filter((step) => step.type === "tool");
+	const value = resolveSequenceValue(params.renderedValue ?? params.assertion.value);
+	const expectedMatchers = value.steps.map((step, index) => {
+		const matcher = normalizeTrajectoryMatcher(step, "tool");
+		requireNamedTrajectoryMatcher(matcher, "trajectory:tool-sequence", index);
+		return matcher;
+	});
+	if (expectedMatchers.length === 0) throw new Error("trajectory:tool-sequence assertion requires at least one expected step");
+	const actualTools = toolSteps.map(formatTrajectoryStep);
+	let basePass = false;
+	let reason = "";
+	if (value.mode === "exact") {
+		basePass = toolSteps.length === expectedMatchers.length && expectedMatchers.every((matcher, index) => matchesTrajectoryStep(toolSteps[index], matcher));
+		if (basePass) reason = `Observed exact tool sequence: ${formatStepList(actualTools)}`;
+		else reason = `Expected exact tool sequence of ${expectedMatchers.map((matcher) => matcher.pattern || matcher.name || "*").join(", ")}, but actual tools were ${formatStepList(actualTools)}`;
+	} else {
+		let expectedIndex = 0;
+		const matchedSteps = [];
+		for (const step of toolSteps) {
+			if (expectedIndex >= expectedMatchers.length) break;
+			if (matchesTrajectoryStep(step, expectedMatchers[expectedIndex])) {
+				matchedSteps.push(formatTrajectoryStep(step));
+				expectedIndex += 1;
+			}
+		}
+		basePass = expectedIndex === expectedMatchers.length;
+		if (basePass) reason = `Observed tool sequence in order: ${matchedSteps.join(", ")}. Actual tools: ${formatStepList(actualTools)}`;
+		else reason = `Expected tool "${expectedMatchers[expectedIndex]?.pattern || expectedMatchers[expectedIndex]?.name || "*"}" was not observed in order. Actual tools: ${formatStepList(actualTools)}`;
+	}
+	const pass = applyInverse(basePass, params.inverse);
+	if (params.inverse) reason = basePass ? `Forbidden tool sequence was observed. Actual tools: ${formatStepList(actualTools)}` : `Forbidden tool sequence was not observed`;
+	return {
+		pass,
+		score: pass ? 1 : 0,
+		reason,
+		assertion: params.assertion
+	};
+};
+const handleTrajectoryToolArgsMatch = (params) => {
+	const toolSteps = extractTrajectorySteps(getTraceOrThrow(params)).filter((step) => step.type === "tool");
+	const { matcher, expectedArgs, mode } = resolveToolArgsMatchValue(params.renderedValue ?? params.assertion.value);
+	const matcherLabel = matcher.pattern || matcher.name || "*";
+	const actualTools = toolSteps.map(formatTrajectoryStep);
+	const matchingSteps = toolSteps.filter((step) => matchesTrajectoryStep(step, matcher));
+	const stepsWithArgs = matchingSteps.filter((step) => step.args !== void 0);
+	const matchedStep = stepsWithArgs.find((step) => matchesToolArgs(step.args, expectedArgs, mode));
+	const basePass = matchedStep !== void 0;
+	const pass = applyInverse(basePass, params.inverse);
+	const expectedArgsLabel = formatTrajectoryArgs(expectedArgs);
+	const observedArgsLabel = stepsWithArgs.length > 0 ? stepsWithArgs.map((step) => formatTrajectoryArgs(step.args)).join(", ") : "(none)";
+	let reason;
+	if (params.inverse) if (basePass) reason = `Forbidden argument match for tool "${matcherLabel}" was observed on ${formatTrajectoryStep(matchedStep)}. Args: ${formatTrajectoryArgs(matchedStep.args)}`;
+	else if (matchingSteps.length === 0) reason = `Forbidden argument match for tool "${matcherLabel}" was not observed because no tool call matched it`;
+	else reason = `Forbidden argument match for tool "${matcherLabel}" was not observed. Observed args: ${observedArgsLabel}`;
+	else if (basePass) reason = `Tool "${matcherLabel}" matched expected arguments (${mode}) on ${formatTrajectoryStep(matchedStep)}. Args: ${formatTrajectoryArgs(matchedStep.args)}`;
+	else if (matchingSteps.length === 0) reason = `No tool call matched "${matcherLabel}". Actual tools: ${formatStepList(actualTools)}`;
+	else if (stepsWithArgs.length === 0) reason = `Tool "${matcherLabel}" was observed but no arguments were captured. Actual tools: ${formatStepList(actualTools)}`;
+	else reason = `No call to tool "${matcherLabel}" matched expected arguments (${mode}): ${expectedArgsLabel}. Observed args: ${observedArgsLabel}`;
+	return {
+		pass,
+		score: pass ? 1 : 0,
+		reason,
+		assertion: params.assertion
+	};
+};
+function resolveStepCountValue(value) {
+	if (!value || typeof value !== "object" || Array.isArray(value)) throw new Error("trajectory:step-count assertion must have an object value");
+	return {
+		...normalizeTrajectoryMatcher(value),
+		max: typeof value.max === "number" ? value.max : void 0,
+		min: typeof value.min === "number" ? value.min : void 0
+	};
+}
+const handleTrajectoryStepCount = (params) => {
+	const steps = extractTrajectorySteps(getTraceOrThrow(params));
+	const matcher = resolveStepCountValue(params.renderedValue ?? params.assertion.value);
+	const { min, max } = matcher;
+	if (min === void 0 && max === void 0) throw new Error("trajectory:step-count assertion must include a min or max property");
+	const matchingSteps = steps.filter((step) => matchesTrajectoryStep(step, matcher));
+	const count = matchingSteps.length;
+	const basePass = (min === void 0 || count >= min) && (max === void 0 || count <= max);
+	const pass = applyInverse(basePass, params.inverse);
+	const filterParts = [];
+	if (matcher.type) {
+		const types = Array.isArray(matcher.type) ? matcher.type : [matcher.type];
+		filterParts.push(`type=${types.join("|")}`);
+	}
+	const pattern = matcher.pattern || matcher.name;
+	if (pattern) filterParts.push(`pattern=${pattern}`);
+	let reason = `Matched ${count} trajectory step(s)`;
+	if (filterParts.length > 0) reason += ` for ${filterParts.join(", ")}`;
+	if (min !== void 0 && max !== void 0) reason += ` (expected ${min}-${max})`;
+	else if (min !== void 0) reason += ` (expected at least ${min})`;
+	else if (max !== void 0) reason += ` (expected at most ${max})`;
+	if (matchingSteps.length > 0) reason += `. Matches: ${matchingSteps.map(formatTrajectoryStep).join(", ")}`;
+	if (params.inverse) reason = basePass ? `Trajectory step count satisfied the forbidden range` : `Trajectory step count did not satisfy the forbidden range`;
+	return {
+		pass,
+		score: pass ? 1 : 0,
+		reason,
+		assertion: params.assertion
+	};
+};
+const handleTrajectoryGoalSuccess = async (params) => {
+	const trace = getTraceOrThrow(params);
+	const { goal } = resolveGoalSuccessValue(params.renderedValue ?? params.assertion.value);
+	const result = await matchesTrajectoryGoalSuccess(goal, summarizeTrajectoryForJudge(trace), params.outputString, params.test.options, params.assertionValueContext.vars, params.assertion, params.providerCallContext);
+	if (!params.inverse) return result;
+	return {
+		...result,
+		assertion: params.assertion,
+		pass: !result.pass,
+		score: result.pass ? 0 : 1,
+		reason: result.pass ? `Agent unexpectedly achieved the goal: ${goal}` : `Agent did not achieve the forbidden goal: ${goal}`
+	};
+};
+//#endregion
 //#region src/assertions/webhook.ts
 async function handleWebhook({ assertion, renderedValue, test, prompt, output, inverse }) {
 	invariant(renderedValue, "\"webhook\" assertion type must have a URL value");
@@ -2585,18 +3213,18 @@ const handleWordCount = ({ assertion, renderedValue, valueFromScript, outputStri
 			if (pass) reason = "Assertion passed";
 			else if (inverse) reason = `Expected word count to not be between ${min} and ${max}, but got ${wordCount}`;
 			else reason = `Word count ${wordCount} is not between ${min} and ${max}`;
-		} else if (min !== void 0) {
-			const basePass = wordCount >= min;
-			pass = inverse ? !basePass : basePass;
-			if (pass) reason = "Assertion passed";
-			else if (inverse) reason = `Expected word count to be less than ${min}, but got ${wordCount}`;
-			else reason = `Word count ${wordCount} is less than minimum ${min}`;
-		} else {
+		} else if (min === void 0) {
 			const basePass = wordCount <= max;
 			pass = inverse ? !basePass : basePass;
 			if (pass) reason = "Assertion passed";
 			else if (inverse) reason = `Expected word count to be greater than ${max}, but got ${wordCount}`;
 			else reason = `Word count ${wordCount} is greater than maximum ${max}`;
+		} else {
+			const basePass = wordCount >= min;
+			pass = inverse ? !basePass : basePass;
+			if (pass) reason = "Assertion passed";
+			else if (inverse) reason = `Expected word count to be less than ${min}, but got ${wordCount}`;
+			else reason = `Word count ${wordCount} is less than minimum ${min}`;
 		}
 	} else {
 		invariant(typeof value === "number" || typeof value === "string" && !Number.isNaN(Number(value)), "\"word-count\" assertion value must be a number or an object with min/max properties");
@@ -2691,6 +3319,12 @@ const handleIsXml = ({ assertion, renderedValue, outputString, inverse, baseType
 //#endregion
 //#region src/assertions/index.ts
 const ASSERTIONS_MAX_CONCURRENCY = getEnvInt("PROMPTFOO_ASSERTIONS_MAX_CONCURRENCY", 3);
+const DEFAULT_TRACE_FETCH_MAX_ATTEMPTS = 6;
+const DEFAULT_TRACE_FETCH_RETRY_DELAY_MS = 250;
+const DEFAULT_TRACE_FETCH_STABLE_POLLS = 2;
+const MAX_TRACE_FETCH_MAX_ATTEMPTS = 30;
+const MAX_TRACE_FETCH_RETRY_DELAY_MS = 5e3;
+const MAX_TRACE_FETCH_STABLE_POLLS = 10;
 const MODEL_GRADED_ASSERTION_TYPES = new Set([
 	"answer-relevance",
 	"context-faithfulness",
@@ -2700,8 +3334,57 @@ const MODEL_GRADED_ASSERTION_TYPES = new Set([
 	"llm-rubric",
 	"model-graded-closedqa",
 	"model-graded-factuality",
-	"search-rubric"
+	"search-rubric",
+	"trajectory:goal-success"
 ]);
+const TRACE_AWARE_ASSERTION_TYPES = new Set([
+	"javascript",
+	"python",
+	"ruby",
+	"trace-error-spans",
+	"trace-span-count",
+	"trace-span-duration",
+	"trajectory:goal-success",
+	"trajectory:step-count",
+	"trajectory:tool-args-match",
+	"trajectory:tool-sequence",
+	"trajectory:tool-used"
+]);
+function assertionUsesTrace(assertion) {
+	if (assertion.type === "assert-set") return assertion.assert.some(assertionUsesTrace);
+	return TRACE_AWARE_ASSERTION_TYPES.has(getAssertionBaseType(assertion));
+}
+function assertionMayNeedTraceContext(assertion) {
+	if (assertionUsesTrace(assertion)) return true;
+	if (assertion.type === "assert-set") return assertion.assert.some(assertionMayNeedTraceContext);
+	return typeof assertion.value === "string" ? assertion.value.startsWith("file://") || isPackagePath(assertion.value) : false;
+}
+function hasTraceAwareAssertions(assertions) {
+	return Boolean(assertions?.some(assertionMayNeedTraceContext));
+}
+async function loadTraceData(traceId) {
+	const traceStore = getTraceStore();
+	const maxAttempts = Math.min(MAX_TRACE_FETCH_MAX_ATTEMPTS, Math.max(1, getEnvInt("PROMPTFOO_TRACE_FETCH_MAX_ATTEMPTS", DEFAULT_TRACE_FETCH_MAX_ATTEMPTS)));
+	const retryDelayMs = Math.min(MAX_TRACE_FETCH_RETRY_DELAY_MS, Math.max(0, getEnvInt("PROMPTFOO_TRACE_FETCH_RETRY_DELAY_MS", DEFAULT_TRACE_FETCH_RETRY_DELAY_MS)));
+	const stablePolls = Math.min(MAX_TRACE_FETCH_STABLE_POLLS, Math.max(1, getEnvInt("PROMPTFOO_TRACE_FETCH_STABLE_POLLS", DEFAULT_TRACE_FETCH_STABLE_POLLS)));
+	let lastSpanCount = -1;
+	let stableObservations = 0;
+	let latestTrace = null;
+	for (let attempt = 0; attempt < maxAttempts; attempt++) {
+		latestTrace = await traceStore.getTrace(traceId);
+		const spanCount = latestTrace?.spans?.length ?? 0;
+		if (spanCount > 0) {
+			stableObservations = spanCount === lastSpanCount ? stableObservations + 1 : 1;
+			lastSpanCount = spanCount;
+			if (stableObservations >= stablePolls || attempt === maxAttempts - 1) return latestTrace;
+		} else {
+			stableObservations = 0;
+			lastSpanCount = spanCount;
+		}
+		if (attempt < maxAttempts - 1) await sleep(retryDelayMs);
+	}
+	return latestTrace;
+}
 const ASSERTION_HANDLERS = {
 	"answer-relevance": handleAnswerRelevance,
 	bleu: handleBleuScore,
@@ -2764,12 +3447,18 @@ const ASSERTION_HANDLERS = {
 	ruby: handleRuby,
 	"rouge-n": handleRougeScore,
 	"search-rubric": handleSearchRubric,
+	"skill-used": handleSkillUsed,
 	similar: handleSimilar,
 	"similar:cosine": handleSimilar,
 	"similar:dot": handleSimilar,
 	"similar:euclidean": handleSimilar,
 	"starts-with": handleStartsWith,
 	"tool-call-f1": handleToolCallF1,
+	"trajectory:goal-success": handleTrajectoryGoalSuccess,
+	"trajectory:tool-args-match": handleTrajectoryToolArgsMatch,
+	"trajectory:step-count": handleTrajectoryStepCount,
+	"trajectory:tool-sequence": handleTrajectoryToolSequence,
+	"trajectory:tool-used": handleTrajectoryToolUsed,
 	"trace-error-spans": handleTraceErrorSpans,
 	"trace-span-count": handleTraceSpanCount,
 	"trace-span-duration": handleTraceSpanDuration,
@@ -2812,7 +3501,7 @@ function isAssertionInverse(assertion) {
 function getAssertionBaseType(assertion) {
 	return isAssertionInverse(assertion) ? assertion.type.slice(4) : assertion.type;
 }
-async function runAssertion({ prompt, provider, assertion, test, vars, latencyMs, providerResponse, traceId }) {
+async function runAssertion({ prompt, provider, assertion, test, vars, latencyMs, providerResponse, traceId, traceData }) {
 	const resolvedVars = vars || test.vars || {};
 	const { cost, logProbs, output: originalOutput } = providerResponse;
 	let output = originalOutput;
@@ -2831,14 +3520,14 @@ async function runAssertion({ prompt, provider, assertion, test, vars, latencyMs
 		providerResponse,
 		...assertion.config ? { config: structuredClone(assertion.config) } : {}
 	};
-	if (traceId) try {
-		const traceData = await getTraceStore().getTrace(traceId);
-		if (traceData) context.trace = {
-			traceId: traceData.traceId,
-			evaluationId: traceData.evaluationId,
-			testCaseId: traceData.testCaseId,
-			metadata: traceData.metadata,
-			spans: traceData.spans || []
+	if (traceId && assertionMayNeedTraceContext(assertion)) try {
+		const resolvedTraceData = traceData === void 0 ? await loadTraceData(traceId) : traceData;
+		if (resolvedTraceData) context.trace = {
+			traceId: resolvedTraceData.traceId,
+			evaluationId: resolvedTraceData.evaluationId,
+			testCaseId: resolvedTraceData.testCaseId,
+			metadata: resolvedTraceData.metadata,
+			spans: resolvedTraceData.spans || []
 		};
 	} catch (error) {
 		logger.debug(`Failed to fetch trace data for assertion: ${error}`);
@@ -2871,7 +3560,7 @@ async function runAssertion({ prompt, provider, assertion, test, vars, latencyMs
 			};
 		}
 		else if (filePath.endsWith(".rb")) try {
-			const { runRuby } = await import("./rubyUtils-BUVePouc.js").then((n) => n.t);
+			const { runRuby } = await import("./rubyUtils-PgU-gHmx.js").then((n) => n.t);
 			valueFromScript = await runRuby(filePath, functionName || "get_assert", [output, context]);
 			logger.debug(`Ruby script ${filePath} output: ${valueFromScript}`);
 		} catch (error) {
@@ -2980,6 +3669,14 @@ async function runAssertions({ assertScoringFunction, latencyMs, prompt, provide
 			index: i
 		};
 	}).flat();
+	const shouldPreloadTrace = !!traceId && hasTraceAwareAssertions(asserts.map(({ assertion }) => assertion));
+	let preloadedTraceData;
+	if (shouldPreloadTrace && traceId) try {
+		preloadedTraceData = await loadTraceData(traceId);
+	} catch (error) {
+		logger.debug(`Failed to preload trace data for assertions: ${error}`);
+		preloadedTraceData = null;
+	}
 	await async.forEachOfLimit(asserts, ASSERTIONS_MAX_CONCURRENCY, async ({ assertion, assertResult, index }) => {
 		if (assertion.type.startsWith("select-") || assertion.type === "max-score") return;
 		const result = await runAssertion({
@@ -2991,7 +3688,8 @@ async function runAssertions({ assertScoringFunction, latencyMs, prompt, provide
 			vars,
 			latencyMs,
 			assertIndex: index,
-			traceId
+			traceId,
+			traceData: preloadedTraceData
 		});
 		assertResult.addResult({
 			index,
@@ -3137,7 +3835,7 @@ var CIProgressReporter = class {
 		else {
 			const eta = remaining / rate;
 			if (eta > 1440) etaDisplay = ">24 hours";
-			else etaDisplay = `${Math.round(eta)} minute${Math.round(eta) !== 1 ? "s" : ""}`;
+			else etaDisplay = `${Math.round(eta)} minute${Math.round(eta) === 1 ? "" : "s"}`;
 		}
 		const percentage = Math.floor(this.completedTests / this.totalTests * 100);
 		logger.info(`[CI Progress] Evaluation running for ${this.formatElapsedTime(elapsed)} - Completed ${this.completedTests}/${this.totalTests} tests (${percentage}%)`);
@@ -3538,12 +4236,55 @@ function isPromptAllowed(prompt, allowedPrompts) {
 var ProgressBarManager = class {
 	progressBar;
 	isWebUI;
+	originalLogCallback = null;
+	installedLogCallback = null;
+	pendingRender = null;
 	totalCount = 0;
 	completedCount = 0;
 	concurrency = 1;
 	constructor(isWebUI) {
 		this.isWebUI = isWebUI;
 	}
+	clearProgressBarLine() {
+		readline.cursorTo(process.stderr, 0);
+		readline.clearLine(process.stderr, 0);
+	}
+	scheduleRender() {
+		if (!this.progressBar || this.pendingRender) return;
+		this.pendingRender = setImmediate(() => {
+			this.pendingRender = null;
+			this.progressBar?.render();
+		});
+	}
+	handleLogMessage() {
+		if (!this.progressBar) return;
+		this.clearProgressBarLine();
+		this.scheduleRender();
+	}
+	/**
+	* Coordinate console logging with the progress bar to prevent visual corruption.
+	*/
+	installLogInterceptor() {
+		if (!this.progressBar || this.isWebUI || this.installedLogCallback) return;
+		this.originalLogCallback = globalLogCallback;
+		this.installedLogCallback = (message) => {
+			this.originalLogCallback?.(message);
+			this.handleLogMessage();
+		};
+		setLogCallback(this.installedLogCallback);
+	}
+	/**
+	* Remove the log interceptor and restore original logger callback behavior.
+	*/
+	removeLogInterceptor() {
+		if (this.pendingRender) {
+			clearImmediate(this.pendingRender);
+			this.pendingRender = null;
+		}
+		if (this.installedLogCallback && globalLogCallback === this.installedLogCallback) setLogCallback(this.originalLogCallback);
+		this.installedLogCallback = null;
+		this.originalLogCallback = null;
+	}
 	/**
 	* Initialize progress bar
 	*/
@@ -3563,7 +4304,8 @@ var ProgressBarManager = class {
 				return `Evaluating [${bar}${spaces}] ${percentage}% | ${params.value}/${params.total}${errorsText} | ${payload.provider} ${payload.prompt} ${payload.vars}`;
 			},
 			hideCursor: true,
-			gracefulExit: true
+			gracefulExit: true,
+			stream: process.stderr
 		}, cliProgress.Presets.shades_classic);
 		this.progressBar.start(this.totalCount, 0, {
 			provider: "",
@@ -3838,6 +4580,7 @@ async function runEval({ provider, prompt, test, testSuite, delay, nunjucksFilte
 				const parts = traceContext.traceparent.split("-");
 				if (parts.length >= 3) traceId = parts[1];
 			}
+			if (traceId && hasTraceAwareAssertions(test.assert)) await flushOtel();
 			const checkResult = await runAssertions({
 				prompt: renderedPrompt,
 				provider,
@@ -4235,7 +4978,7 @@ var Evaluator = class {
 				const defaultProvider = testSuite.defaultTest.provider;
 				if (isApiProvider(defaultProvider)) testCase.provider = defaultProvider;
 				else if (typeof defaultProvider === "object" && defaultProvider.id) {
-					const { loadApiProvider } = await import("./providers-GIQ2TcsA.js");
+					const { loadApiProvider } = await import("./providers-sS2WI8YD.js");
 					testCase.provider = await loadApiProvider(typeof defaultProvider.id === "function" ? defaultProvider.id() : defaultProvider.id, { options: defaultProvider });
 				} else testCase.provider = defaultProvider;
 			}
@@ -4319,7 +5062,7 @@ var Evaluator = class {
 			if (evalOption.test.assert?.some((a) => a.type === "max-score")) rowsWithMaxScoreAssertion.add(evalOption.testIdx);
 		}
 		if (state.resume && this.evalRecord.persisted) try {
-			const { default: EvalResult } = await import("./evalResult-CDQiuUuf.js").then((n) => n.n);
+			const { default: EvalResult } = await import("./evalResult-BkIhRdTe.js").then((n) => n.n);
 			const completedPairs = await EvalResult.getCompletedIndexPairs(this.evalRecord.id, { excludeErrors: state.retryMode });
 			const originalCount = runEvalOptions.length;
 			for (let i = runEvalOptions.length - 1; i >= 0; i--) {
@@ -4519,7 +5262,7 @@ var Evaluator = class {
 		if (isCI() && !isWebUI) {
 			ciProgressReporter = new CIProgressReporter(runEvalOptions.length);
 			ciProgressReporter.start();
-		} else if (this.options.showProgressBar && process.stdout.isTTY) progressBarManager = new ProgressBarManager(isWebUI);
+		} else if (this.options.showProgressBar && process.stderr.isTTY) progressBarManager = new ProgressBarManager(isWebUI);
 		this.options.progressCallback = (completed, total, index, evalStep, metrics) => {
 			if (originalProgressCallback) originalProgressCallback(completed, total, index, evalStep, metrics);
 			if (isWebUI) {
@@ -4540,7 +5283,10 @@ var Evaluator = class {
 			if (serialRunEvalOptions.length > 0) logger.info(`Running ${serialRunEvalOptions.length} test cases serially...`);
 			if (concurrentRunEvalOptions.length > 0) logger.info(`Running ${concurrentRunEvalOptions.length} test cases (up to ${concurrency} at a time)...`);
 		}
-		if (this.options.showProgressBar && progressBarManager) await progressBarManager.initialize(runEvalOptions, concurrency, 0);
+		if (this.options.showProgressBar && progressBarManager) {
+			await progressBarManager.initialize(runEvalOptions, concurrency, 0);
+			progressBarManager.installLogInterceptor();
+		}
 		try {
 			if (serialRunEvalOptions.length > 0) for (const evalStep of serialRunEvalOptions) {
 				checkAbort();
@@ -4566,7 +5312,10 @@ var Evaluator = class {
 				else if (!targetUnavailable) {
 					logger.info("Evaluation interrupted, saving progress...");
 					if (globalTimeout) clearTimeout(globalTimeout);
-					if (progressBarManager) progressBarManager.stop();
+					if (progressBarManager) {
+						progressBarManager.removeLogInterceptor();
+						progressBarManager.stop();
+					}
 					if (ciProgressReporter) ciProgressReporter.finish();
 					this.evalRecord.setVars(Array.from(vars));
 					await this.evalRecord.addPrompts(prompts);
@@ -4574,6 +5323,10 @@ var Evaluator = class {
 					return this.evalRecord;
 				}
 			} else {
+				if (progressBarManager) {
+					progressBarManager.removeLogInterceptor();
+					progressBarManager.stop();
+				}
 				if (ciProgressReporter) ciProgressReporter.error(`Evaluation failed: ${String(err)}`);
 				throw err;
 			}
@@ -4716,6 +5469,7 @@ var Evaluator = class {
 		await this.evalRecord.addPrompts(prompts);
 		try {
 			if (progressBarManager) {
+				progressBarManager.removeLogInterceptor();
 				progressBarManager.complete();
 				progressBarManager.stop();
 			} else if (ciProgressReporter) ciProgressReporter.finish();
@@ -7073,8 +7827,7 @@ function testCaseFromCsvRow(row) {
 			logger.warn("The \"__metadata\" column requires a key, e.g. \"__metadata:category\". This column will be ignored.");
 		} else if (key.startsWith("__config:")) {
 			const configParts = key.slice(9).split(":");
-			if (configParts.length !== 2) logger.warn(`Invalid __config column format: "${key}". Expected format: __config:__expected:threshold or __config:__expected<N>:threshold`);
-			else {
+			if (configParts.length === 2) {
 				const [expectedKey, configKey] = configParts;
 				let targetIndex;
 				if (expectedKey === "__expected") targetIndex = 0;
@@ -7100,7 +7853,7 @@ function testCaseFromCsvRow(row) {
 					}
 				}
 				assertionConfigs[targetIndex][configKey] = parsedValue;
-			}
+			} else logger.warn(`Invalid __config column format: "${key}". Expected format: __config:__expected:threshold or __config:__expected<N>:threshold`);
 		} else vars[key] = value;
 	}
 	for (let i = 0; i < asserts.length; i++) {
@@ -7229,14 +7982,14 @@ async function parseXlsxFile(filePath) {
 		const sheetName = typeof sheetOption === "number" ? sheetNames[sheetOption - 1] : sheetOption;
 		const rows = await readXlsxFile(actualFilePath, { sheet: sheetOption });
 		if (rows.length === 0) throw new Error(`Sheet "${sheetName}" is empty or contains no valid data rows`);
-		const headers = rows[0].map((cell) => cell != null ? String(cell) : "");
+		const headers = rows[0].map((cell) => cell == null ? "" : String(cell));
 		if (headers.length === 0 || headers.every((h) => h === "")) throw new Error(`Sheet "${sheetName}" has no valid column headers`);
 		if (rows.length === 1) throw new Error(`Sheet "${sheetName}" is empty or contains no valid data rows`);
 		const data = rows.slice(1).map((row) => {
 			const obj = {};
 			headers.forEach((header, index) => {
 				const cellValue = row[index];
-				obj[header] = cellValue != null ? String(cellValue) : "";
+				obj[header] = cellValue == null ? "" : String(cellValue);
 			});
 			return obj;
 		});
@@ -11183,20 +11936,19 @@ function generateEvalSummary(params) {
 		}
 	}
 	lines.push("");
-	const passRate = successes / (successes + failures + errors) * 100;
-	let passRateDisplay;
-	if (!Number.isNaN(passRate)) {
-		const passRateFormatted = passRate === 0 || passRate === 100 ? `${passRate.toFixed(0)}%` : `${passRate.toFixed(2)}%`;
-		if (passRate >= 100) passRateDisplay = chalk.green.bold(passRateFormatted);
-		else if (passRate >= 80) passRateDisplay = chalk.yellow.bold(passRateFormatted);
-		else passRateDisplay = chalk.red.bold(passRateFormatted);
-	}
-	const passedPart = successes > 0 ? `${chalk.green("✓")} ${chalk.green.bold(successes.toLocaleString())} passed` : `${chalk.gray.bold(successes.toLocaleString())} passed`;
-	const failedPart = failures > 0 ? `${chalk.red("✗")} ${chalk.red.bold(failures.toLocaleString())} failed` : `${chalk.gray.bold(failures.toLocaleString())} failed`;
+	const totalTests = successes + failures + errors;
+	const formatResultPercentage = (count) => {
+		const percentage = totalTests === 0 ? 0 : count / totalTests * 100;
+		return percentage === 0 || percentage === 100 ? `${percentage.toFixed(0)}%` : `${percentage.toFixed(2)}%`;
+	};
+	const formatResultLine = (count, label, icon, iconColor) => {
+		return `  ${icon ? `${iconColor(icon)} ` : ""}${chalk.white.bold(count.toLocaleString())} ${chalk.white(label)} ${chalk.gray(`(${formatResultPercentage(count)})`)}`;
+	};
 	const errorLabel = errors === 1 ? "error" : "errors";
-	const resultsLine = `${passedPart}, ${failedPart}, ${errors > 0 ? `${chalk.red("✗")} ${chalk.red.bold(errors.toLocaleString())} ${errorLabel}` : `${chalk.gray.bold(errors.toLocaleString())} ${errorLabel}`}`;
-	if (Number.isNaN(passRate)) lines.push(`${chalk.bold("Results:")} ${resultsLine}`);
-	else lines.push(`${chalk.bold("Results:")} ${resultsLine} (${passRateDisplay})`);
+	lines.push(chalk.bold("Results:"));
+	lines.push(formatResultLine(successes, "passed", successes > 0 ? "✓" : void 0, chalk.green));
+	lines.push(formatResultLine(failures, "failed", failures > 0 ? "✗" : void 0, chalk.red));
+	lines.push(formatResultLine(errors, errorLabel, errors > 0 ? "✗" : void 0, chalk.red));
 	const durationDisplay = formatDuration(duration);
 	lines.push(chalk.gray(`Duration: ${durationDisplay} (concurrency: ${maxConcurrency})`));
 	lines.push("");
@@ -11530,7 +12282,7 @@ async function doEval(cmdObj, defaultConfig, defaultConfigPath, evaluateOptions)
 		await checkCloudPermissions(config);
 		const options = {
 			...evaluateOptions,
-			showProgressBar: getLogLevel() === "debug" ? false : cmdObj.progressBar !== void 0 ? cmdObj.progressBar !== false : evaluateOptions.showProgressBar !== void 0 ? evaluateOptions.showProgressBar : true,
+			showProgressBar: getLogLevel() === "debug" ? false : cmdObj.progressBar === void 0 ? evaluateOptions.showProgressBar === void 0 ? true : evaluateOptions.showProgressBar : cmdObj.progressBar !== false,
 			repeat,
 			delay: !Number.isNaN(delay) && delay > 0 ? delay : void 0,
 			maxConcurrency,
@@ -11914,7 +12666,7 @@ async function doRedteamRun(options) {
 		redteamConfig = await doGenerateRedteam({
 			...passThroughOptions,
 			...options.liveRedteamConfig?.commandLineOptions || {},
-			...maxConcurrency !== void 0 ? { maxConcurrency } : {},
+			...maxConcurrency === void 0 ? {} : { maxConcurrency },
 			config: configPath,
 			output: redteamPath,
 			force: options.force,