promptfoo 0.121.4 → 0.121.5

package/dist/src/index.js

@@ -1,33 +1,34 @@
-import { C as getEnvFloat, D as getMaxEvalTimeMs, E as getEvalTimeoutMs, O as isCI, S as getEnvBool, T as getEnvString, a as logger, b as summarizeEvaluateResultForLogging, g as getAjv, h as extractJsonObjects, k as state, n as globalLogCallback, o as setLogCallback, r as isDebugEnabled, s as setLogLevel, t as getLogLevel, v as orderKeys, w as getEnvInt, y as safeJsonStringify } from "./logger-Ct2S6Yx-.js";
+import { C as getEnvFloat, D as getMaxEvalTimeMs, E as getEvalTimeoutMs, O as isCI, S as getEnvBool, T as getEnvString, a as logger, b as summarizeEvaluateResultForLogging, g as getAjv, h as extractJsonObjects, k as state, m as extractFirstJsonObject, n as globalLogCallback, o as setLogCallback, r as isDebugEnabled, s as setLogLevel, t as getLogLevel, v as orderKeys, w as getEnvInt, y as safeJsonStringify } from "./logger-Ct2S6Yx-.js";
 import { t as invariant } from "./invariant-Ddh24eXh.js";
-import { r as importModule, t as getDirectory } from "./esm-C7PnfdF8.js";
-import { r as runPython } from "./pythonUtils-C2UQ30Rz.js";
-import { i as isJavascriptFile } from "./fileExtensions-DnqA1y9x.js";
-import { i as getProcessShim, n as transform, t as TransformInputType } from "./transform-Bbg6A8Jk.js";
-import { $ as matchesGEval, A as DivergentRepetitionPlugin, B as sampleArray, C as getPiiLeakTestsForCategory, D as HarmbenchPlugin, E as ImitationPlugin, F as AegisPlugin, G as loadRubricPrompt, H as callProviderWithContext, I as RedteamGraderBase, J as matchesClosedQa, K as matchesAnswerRelevance, L as RedteamPluginBase, M as CrossSessionLeakPlugin, N as ContractPlugin, O as HallucinationPlugin, P as BeavertailsPlugin, Q as matchesFactuality, R as getCustomPolicies, S as PlinyPlugin, T as IntentPlugin, U as fail, V as fetchHuggingFaceDataset, W as getAndCheckProvider, X as matchesContextRecall, Y as matchesContextFaithfulness, Z as matchesContextRelevance, _ as PoliticsPlugin, _t as processFileReference, a as UnverifiableClaimsPlugin, at as matchesSimilarity, b as isValidPolicyObject, c as ToolDiscoveryPlugin, ct as withProviderCallExecutionContext, d as TeenSafetyDangerousContentPlugin, dt as readPrompts, et as matchesLlmRubric, f as TeenSafetyAgeRestrictedGoodsAndServicesPlugin, ft as readProviderPromptMap, g as PromptExtractionPlugin, gt as loadFromJavaScriptFile, h as RbacPlugin, ht as getFinalTest, i as VLGuardPlugin, it as matchesSelectBest, j as DebugAccessPlugin, k as ExcessiveAgencyPlugin, l as TeenSafetyHarmfulBodyIdealsPlugin, lt as getDefaultProviders, m as ShellInjectionPlugin, mt as coerceString, n as getGraderById, nt as matchesPiScore, o as UnsafeBenchPlugin, ot as matchesTrajectoryGoalSuccess, p as SqlInjectionPlugin, pt as SUGGEST_PROMPTS_SYSTEM_MESSAGE, q as matchesClassification, r as VLSUPlugin, rt as matchesSearchRubric, s as ToxicChatPlugin, st as selectMaxScore, t as GRADERS, tt as matchesModeration, u as TeenSafetyDangerousRoleplayPlugin, ut as processPrompts, v as PolicyPlugin, vt as resolveContext, w as OverreliancePlugin, x as makeInlinePolicyIdSync, y as determinePolicyTypeFromId, z as retryWithDeduplication } from "./graders-Zy3x0zqX.js";
-import { A as isApiProvider, At as CompletionTokenDetailsSchema, C as TestGeneratorConfigSchema, Ct as UNALIGNED_PROVIDER_HARM_PLUGINS, D as VarsSchema, E as UnifiedConfigSchema, F as ConversationMessageSchema, I as PartialGenerationError, J as getDefaultNFanout, K as STRATEGY_COLLECTIONS, L as PluginConfigSchema, M as RedteamConfigSchema, O as isGradingResult, Ot as PromptSchema, P as ProvidersSchema, Q as categoryAliases, R as PolicyObjectSchema, S as TestCasesWithMetadataSchema, St as TELECOM_PLUGINS, T as TestSuiteSchema, V as isUuid, W as DEFAULT_STRATEGIES, X as isFanoutStrategy, Z as Severity, _ as ScenarioSchema, _t as PLUGIN_CATEGORIES, a as AtomicTestCaseSchema, at as DEFAULT_PLUGINS, b as TestCaseWithVarsFileSchema, c as CompletedPromptSchema, ct as HARM_PLUGINS, d as EvaluateOptionsSchema, dt as LLAMA_GUARD_REPLICATE_PROVIDER, et as riskCategorySeverityMap, f as GradingConfigSchema, ft as MEDICAL_PLUGINS, g as ResultFailureReason, gt as PII_PLUGINS, h as OutputFileExtension, ht as PHARMACY_PLUGINS, i as AssertionTypeSchema, it as DATASET_EXEMPT_PLUGINS, j as isProviderOptions, jt as InputsSchema, k as isResultFailureReason, kt as BaseTokenUsageSchema, l as DerivedMetricSchema, lt as INSURANCE_PLUGINS, m as OutputConfigSchema, mt as MULTI_INPUT_VAR, n as AssertionSchema, nt as BIAS_PLUGINS, o as BaseAssertionTypesSchema, ot as FINANCIAL_PLUGINS, p as NotPrefixedAssertionTypesSchema, pt as MULTI_INPUT_EXCLUDED_PLUGINS, q as STRATEGY_COLLECTION_MAPPINGS, r as AssertionSetSchema, rt as CANARY_BREAKING_STRATEGY_IDS, s as CommandLineOptionsSchema, st as FOUNDATION_PLUGINS, t as AssertionOrSetSchema, tt as ALIASED_PLUGIN_MAPPINGS, u as EvalResultsFilterMode, ut as LLAMA_GUARD_ENABLED_CATEGORIES, v as SpecialAssertionTypesSchema, vt as REDTEAM_PROVIDER_HARM_PLUGINS, w as TestSuiteConfigSchema, wt as CODING_AGENT_CORE_PLUGINS, x as TestCasesWithMetadataPromptSchema, xt as TEEN_SAFETY_PLUGINS, y as TestCaseSchema, yt as REMOTE_ONLY_PLUGIN_IDS, z as StrategyConfigSchema } from "./types-BGQDAP8i.js";
-import { C as checkProviderApiKeys, D as isGoogleProvider, E as isAnthropicProvider, O as isOpenAiProvider, S as resultIsForTestCase, T as getProviderDescription, _ as setupEnv, b as filterRuntimeVars, c as maybeLoadFromExternalFile, d as maybeLoadToolsFromExternalFile, g as parseFileUrl, h as loadFunction, i as fetchCsvFromGoogleSheet, k as isProviderAllowed, m as readOutput, n as writeMultipleOutputs, p as readFilters, r as writeOutput, s as maybeLoadConfigFromExternalFile, t as printBorder, v as deduplicateTestCases, w as doesProviderRefMatch, x as getTestCaseDeduplicationKey, y as extractRuntimeVars } from "./util-oGMLA7vc.js";
-import { a as getNunjucksEngine, i as extractVariablesFromTemplates, r as extractVariablesFromTemplate, t as renderEnvOnlyInObject } from "./render-CH-62LbA.js";
-import { A as TERMINAL_MAX_WIDTH, F as VERSION, I as FILE_METADATA_KEY, L as HUMAN_ASSERTION_TYPE, M as getShareApiBaseUrl, N as getShareViewBaseUrl, S as parseChatPrompt, a as CloudConfig, d as sleep, h as REQUEST_TIMEOUT_MS, j as getDefaultShareViewBaseUrl, n as fetchWithRetries, o as cloudConfig, r as fetchWithTimeout, t as fetchWithProxy, u as getCurrentTimestamp, y as isPromptfooSampleTarget } from "./fetch-Di00EQrc.js";
-import { c as isNonTransientHttpStatus, i as getCache, n as disableCache, o as withCacheNamespace, r as fetchWithCache, s as NON_TRANSIENT_HTTP_STATUSES, t as cache_exports } from "./cache-D5NZmMiT.js";
-import { $ as AIStudioChatProvider, A as createRateLimitRegistry, B as getCloudDatabaseId, C as collectFileMetadata, D as loadFromPackage, E as isPackagePath, F as getGeneratedPromptOverLimit, G as isCloudProvider, H as getOrgContext, I as getMaxCharsPerMessageModifierValue, K as resolveTeamId, L as throwIfTargetPromptExceedsMaxChars, N as PromptfooHarmfulCompletionProvider, O as redteamProviderManager, P as MAX_CHARS_PER_MESSAGE_MODIFIER_KEY, Q as VertexChatProvider, T as runExtensionHook, U as getPluginSeverityOverridesFromCloud, V as getEvalConfigFromCloud, _ as extractVariablesFromJson, a as resolveProviderConfigs, b as isBasicRefusal, c as Strategies, d as pluginMatchesStrategyTargets, f as checkExfilTracking, g as extractPromptFromTags, i as resolveProvider, j as createProviderRateLimitOptions, k as TokenUsageTracker, l as loadStrategy, m as extractGoalFromPrompt, n as loadApiProvider, o as MCPProvider, r as loadApiProviders, s as GoogleLiveProvider, t as getProviderIds, u as validateStrategies, v as getSessionId, w as renderPrompt, y as getShortPluginId, z as checkCloudPermissions } from "./providers-DruaQfwu.js";
-import { i as generateIdFromPrompt, t as hashPrompt } from "./utils-hXtCYanr.js";
-import { n as sha256, t as randomSequence } from "./createHash-4gFQpDDv.js";
-import { t as OpenAiChatCompletionProvider } from "./chat-I9izLm49.js";
-import { a as createEmptyTokenUsage, i as createEmptyAssertions, n as accumulateResponseTokenUsage, o as normalizeTokenUsage, r as accumulateTokenUsage, t as accumulateAssertionTokenUsage } from "./tokenUsageUtils-C-bmyHoE.js";
-import { h as validateFunctionCall } from "./transform-CG0ehZNG.js";
-import { l as validateFunctionCall$1 } from "./util-D9TisOyk.js";
-import { t as providerRegistry } from "./providerRegistry-Bvh8mv85.js";
-import { i as getRemoteGenerationUrl, l as shouldGenerateRemote, o as getRemoteHealthUrl, r as promptYesNo, s as neverGenerateRemote } from "./server-D6Il2Sob.js";
-import { c as setUserEmail, i as getUserEmail, o as isLoggedIntoCloud, r as getAuthor, s as promptForEmailUnverified, t as checkEmailStatusAndMaybeExit } from "./accounts-DhMYUUbu.js";
-import { t as getBlobByHash } from "./blobs-C-F78Kfn.js";
-import { a as evalsTable, c as evalsToTagsTable, d as tagsTable, i as evalResultsTable, l as promptsTable, m as getDbSignalPath, o as evalsToDatasetsTable, p as getDb, r as datasetsTable, s as evalsToPromptsTable } from "./tables-DQ4WU5tX.js";
-import { n as isBlobStorageEnabled, t as extractAndStoreBinaryData } from "./extractor-Dk6bRWkv.js";
-import { t as telemetry } from "./telemetry-Dvqxv3YC.js";
-import { t as ellipsize } from "./text-DHxdyQqT.js";
-import { t as getTraceStore } from "./store-M0b1WfYb.js";
-import { n as runRuby } from "./rubyUtils-DVLeA2jg.js";
-import { t as EvalResult } from "./evalResult-Bgm9ZH31.js";
+import { A as TERMINAL_MAX_WIDTH, F as HUMAN_ASSERTION_TYPE, M as getShareApiBaseUrl, N as getShareViewBaseUrl, P as FILE_METADATA_KEY, S as cloudConfig, _ as parseChatPrompt, a as getCurrentTimestamp, j as getDefaultShareViewBaseUrl, m as isPromptfooSampleTarget, n as fetchWithRetries, o as sleep, r as fetchWithTimeout, t as fetchWithProxy, u as REQUEST_TIMEOUT_MS, x as CloudConfig } from "./fetch-D9xxyC1p.js";
+import { n as VERSION } from "./version-CbuBKu2U.js";
+import { $ as categoryAliases, A as isApiProvider, At as CompletionTokenDetailsSchema, B as StrategyConfigSchema, C as TestGeneratorConfigSchema, Ct as TELECOM_PLUGINS, D as VarsSchema, E as UnifiedConfigSchema, F as ProvidersSchema, G as DEFAULT_STRATEGIES, H as isUuid, I as ConversationMessageSchema, J as STRATEGY_COLLECTION_MAPPINGS, L as PartialGenerationError, M as RedteamConfigSchema, N as PromptSchema, O as isGradingResult, Q as Severity, R as PluginConfigSchema, S as TestCasesWithMetadataSchema, St as TEEN_SAFETY_PLUGINS, T as TestSuiteSchema, Tt as CODING_AGENT_CORE_PLUGINS, Y as getDefaultNFanout, Z as isFanoutStrategy, _ as ScenarioSchema, _t as PII_PLUGINS, a as AtomicTestCaseSchema, at as DATASET_EXEMPT_PLUGINS, b as TestCaseWithVarsFileSchema, bt as REMOTE_ONLY_PLUGIN_IDS, c as CompletedPromptSchema, ct as FOUNDATION_PLUGINS, d as EvaluateOptionsSchema, dt as LLAMA_GUARD_ENABLED_CATEGORIES, f as GradingConfigSchema, ft as LLAMA_GUARD_REPLICATE_PROVIDER, g as ResultFailureReason, gt as PHARMACY_PLUGINS, h as OutputFileExtension, ht as MULTI_INPUT_VAR, i as AssertionTypeSchema, it as CANARY_BREAKING_STRATEGY_IDS, j as isProviderOptions, jt as InputsSchema, k as isResultFailureReason, kt as BaseTokenUsageSchema, l as DerivedMetricSchema, lt as HARM_PLUGINS, m as OutputConfigSchema, mt as MULTI_INPUT_EXCLUDED_PLUGINS, n as AssertionSchema, nt as ALIASED_PLUGIN_MAPPINGS, o as BaseAssertionTypesSchema, ot as DEFAULT_PLUGINS, p as NotPrefixedAssertionTypesSchema, pt as MEDICAL_PLUGINS, q as STRATEGY_COLLECTIONS, r as AssertionSetSchema, rt as BIAS_PLUGINS, s as CommandLineOptionsSchema, st as FINANCIAL_PLUGINS, t as AssertionOrSetSchema, tt as riskCategorySeverityMap, u as EvalResultsFilterMode, ut as INSURANCE_PLUGINS, v as SpecialAssertionTypesSchema, vt as PLUGIN_CATEGORIES, w as TestSuiteConfigSchema, wt as UNALIGNED_PROVIDER_HARM_PLUGINS, x as TestCasesWithMetadataPromptSchema, y as TestCaseSchema, yt as REDTEAM_PROVIDER_HARM_PLUGINS, z as PolicyObjectSchema } from "./types-DNRZVOue.js";
+import { c as setUserEmail, i as getUserEmail, o as isLoggedIntoCloud, r as getAuthor, s as promptForEmailUnverified, t as checkEmailStatusAndMaybeExit } from "./accounts-CaLNYnf7.js";
+import { r as importModule, t as getDirectory } from "./esm-BRkfNsYs.js";
+import { a as getNunjucksEngine, i as extractVariablesFromTemplates, r as extractVariablesFromTemplate, t as renderEnvOnlyInObject } from "./render-DlscvAUJ.js";
+import { t as providerRegistry } from "./providerRegistry-DoACwqhD.js";
+import { i as getRemoteGenerationUrl, l as shouldGenerateRemote, o as getRemoteHealthUrl, r as promptYesNo, s as neverGenerateRemote } from "./server-gyd6d4Hc.js";
+import { $ as getMaxCharsPerMessageModifierValue, B as collectFileMetadata, D as validateStrategies, E as loadStrategy, F as getSessionId, G as redteamProviderManager, H as runExtensionHook, I as getShortPluginId, J as createProviderRateLimitOptions, K as TokenUsageTracker, L as isBasicRefusal, N as extractPromptFromTags, O as pluginMatchesStrategyTargets, P as extractVariablesFromJson, Q as getGeneratedPromptOverLimit, T as Strategies, U as isPackagePath, V as renderPrompt, W as loadFromPackage, X as PromptfooHarmfulCompletionProvider, Z as MAX_CHARS_PER_MESSAGE_MODIFIER_KEY, a as resolveProviderConfigs, at as getOrgContext, ct as isCloudProvider, d as MCPProvider, et as throwIfTargetPromptExceedsMaxChars, g as AIStudioChatProvider, h as GoogleLiveProvider, i as resolveProvider, it as getEvalConfigFromCloud, j as extractGoalFromPrompt, k as checkExfilTracking, lt as resolveTeamId, m as VertexChatProvider, n as loadApiProvider, nt as checkCloudPermissions, ot as getPluginSeverityOverridesFromCloud, q as createRateLimitRegistry, r as loadApiProviders, rt as getCloudDatabaseId, t as getProviderIds } from "./providers-CJh7iriU.js";
+import { r as runPython } from "./pythonUtils-DMO68Jg7.js";
+import { i as isJavascriptFile } from "./fileExtensions-8CjoL7vB.js";
+import { A as readFilters, M as loadFunction, N as parseFileUrl, O as maybeLoadToolsFromExternalFile, T as maybeLoadFromExternalFile, _ as isProviderAllowed, a as setupEnv, b as normalizeProviderRef, c as filterRuntimeVars, d as checkProviderApiKeys, f as doesProviderRefMatch, g as isOpenAiProvider, h as isGoogleProvider, i as fetchCsvFromGoogleSheet, j as readOutput, l as getTestCaseDeduplicationKey, m as isAnthropicProvider, n as writeMultipleOutputs, o as deduplicateTestCases, p as getProviderDescription, r as writeOutput, s as extractRuntimeVars, t as printBorder, u as resultIsForTestCase, w as maybeLoadConfigFromExternalFile } from "./util-Dub0f_ej.js";
+import { a as createEmptyTokenUsage, i as createEmptyAssertions, n as accumulateResponseTokenUsage, o as normalizeTokenUsage, r as accumulateTokenUsage, t as accumulateAssertionTokenUsage } from "./tokenUsageUtils-4c780gFd.js";
+import { t as getBlobByHash } from "./blobs-BW4U31ue.js";
+import { a as evalsTable, c as evalsToTagsTable, d as tagsTable, i as evalResultsTable, l as promptsTable, m as getDbSignalPath, o as evalsToDatasetsTable, p as getDb, r as datasetsTable, s as evalsToPromptsTable } from "./tables-WgdUZ8Ck.js";
+import { n as isBlobStorageEnabled, t as extractAndStoreBinaryData } from "./extractor-BM3jRERL.js";
+import { c as isNonTransientHttpStatus, i as getCache, n as disableCache, o as withCacheNamespace, r as fetchWithCache, s as NON_TRANSIENT_HTTP_STATUSES, t as cache_exports } from "./cache-Bzttsk0X.js";
+import { t as OpenAiChatCompletionProvider } from "./chat-Cx_LkwvZ.js";
+import { h as validateFunctionCall } from "./transform-_DpNB4qp.js";
+import { l as validateFunctionCall$1 } from "./util-BlFVL0UF.js";
+import { i as getProcessShim, n as transform, t as TransformInputType } from "./transform-eGiUAv86.js";
+import { t as telemetry } from "./telemetry-CgdVGV8N.js";
+import { t as ellipsize } from "./text-DDQP0tuQ.js";
+import { t as getTraceStore } from "./store-BPkzEyFM.js";
+import { n as sha256, t as randomSequence } from "./createHash-Un4Q_huE.js";
+import { n as runRuby } from "./rubyUtils-BI0p46eZ.js";
+import { $ as SELECT_BEST_PROMPT, A as DivergentRepetitionPlugin, B as sampleArray, C as getPiiLeakTestsForCategory, Ct as DEFAULT_ANTHROPIC_MODEL, D as HarmbenchPlugin, Dt as withProviderCallExecutionContext, E as ImitationPlugin, Et as getGradingProvider, F as AegisPlugin, G as matchesLlmRubric, H as matchesClosedQa, I as RedteamGraderBase, J as doRemoteGrading, K as matchesPiScore, L as RedteamPluginBase, M as CrossSessionLeakPlugin, N as ContractPlugin, O as HallucinationPlugin, P as BeavertailsPlugin, Q as DEFAULT_WEB_SEARCH_PROMPT, R as getCustomPolicies, S as PlinyPlugin, St as getDefaultProviders, T as IntentPlugin, Tt as getAndCheckProvider, U as matchesFactuality, V as fetchHuggingFaceDataset, W as matchesGEval, X as readPrompts, Y as processPrompts, Z as readProviderPromptMap, _ as PoliticsPlugin, _t as tryParse, a as UnverifiableClaimsPlugin, at as CONTEXT_RECALL_ATTRIBUTED_TOKEN, b as isValidPolicyObject, bt as loadFromJavaScriptFile, c as ToolDiscoveryPlugin, ct as CONTEXT_RELEVANCE_BAD, d as TeenSafetyDangerousContentPlugin, dt as cosineSimilarity, et as SUGGEST_PROMPTS_SYSTEM_MESSAGE, f as TeenSafetyAgeRestrictedGoodsAndServicesPlugin, ft as dotProduct, g as PromptExtractionPlugin, gt as splitIntoSentences, h as RbacPlugin, ht as normalizeMatcherTokenUsage, i as VLGuardPlugin, it as CONTEXT_RECALL, j as DebugAccessPlugin, k as ExcessiveAgencyPlugin, l as TeenSafetyHarmfulBodyIdealsPlugin, lt as loadRubricPrompt, m as ShellInjectionPlugin, mt as fail, n as getGraderById, nt as CONTEXT_FAITHFULNESS_LONGFORM, o as UnsafeBenchPlugin, ot as CONTEXT_RECALL_NOT_ATTRIBUTED_TOKEN, p as SqlInjectionPlugin, pt as euclideanDistance, q as matchesTrajectoryGoalSuccess, r as VLSUPlugin, rt as CONTEXT_FAITHFULNESS_NLI_STATEMENTS, s as ToxicChatPlugin, st as CONTEXT_RELEVANCE, t as GRADERS, tt as ANSWER_RELEVANCY_GENERATE, u as TeenSafetyDangerousRoleplayPlugin, ut as renderLlmRubricPrompt, v as PolicyPlugin, vt as coerceString, w as OverreliancePlugin, wt as callProviderWithContext, x as makeInlinePolicyIdSync, xt as processFileReference, y as determinePolicyTypeFromId, yt as getFinalTest, z as retryWithDeduplication } from "./graders-BoUqsCEm.js";
+import { i as generateIdFromPrompt, t as hashPrompt } from "./utils-o8S5huU2.js";
+import { t as EvalResult } from "./evalResult-pSvGWFMo.js";
 import * as fs$2 from "fs";
 import fs, { createWriteStream } from "fs";
 import * as path$2 from "path";
@@ -35,26 +36,26 @@ import path, { parse } from "path";
 import async from "async";
 import yaml from "js-yaml";
 import { AsyncResource } from "node:async_hooks";
-import { resolve } from "node:path";
-import { fileURLToPath } from "node:url";
 import chalk from "chalk";
 import * as os$1 from "os";
 import os from "os";
-import util from "util";
 import dedent from "dedent";
-import * as fsPromises from "fs/promises";
-import { globSync } from "glob";
 import { z } from "zod";
-import { parse as parse$1 } from "csv-parse/sync";
-import { XMLParser } from "fast-xml-parser";
+import * as fsPromises from "fs/promises";
+import util from "util";
+import input from "@inquirer/input";
+import { resolve } from "node:path";
+import { fileURLToPath } from "node:url";
 import crypto$1, { createHash, randomBytes } from "crypto";
 import { DiagConsoleLogger, DiagLogLevel, diag, propagation } from "@opentelemetry/api";
-import input from "@inquirer/input";
 import readline from "readline";
+import { parse as parse$1 } from "csv-parse/sync";
+import { globSync } from "glob";
 import { and, desc, eq, inArray, sql } from "drizzle-orm";
+import { XMLParser } from "fast-xml-parser";
 import cliProgress from "cli-progress";
 import { URL } from "url";
-import { JSDOM } from "jsdom";
+import { parse as parse$2 } from "parse5";
 import { distance } from "fastest-levenshtein";
 import * as rouge from "js-rouge";
 import { isDeepStrictEqual } from "node:util";
@@ -242,6 +243,502 @@ const handleConversationRelevance = async ({ assertion, outputString, prompt, pr
 	};
 };
 //#endregion
+//#region src/matchers/classification.ts
+/**
+*
+* @param expected Expected classification. If undefined, matches any classification.
+* @param output Text to classify.
+* @param threshold Value between 0 and 1. If the expected classification is undefined, the threshold is the minimum score for any classification. If the expected classification is defined, the threshold is the minimum score for that classification.
+* @param grading
+* @returns Pass if the output matches the classification with a score greater than or equal to the threshold.
+*/
+async function matchesClassification(expected, output, threshold, grading) {
+	const resp = await (await getAndCheckProvider("classification", grading?.provider, null, "classification check")).callClassificationApi(output);
+	if (!resp.classification) return fail(resp.error || "Unknown error fetching classification");
+	let score;
+	if (expected === void 0) {
+		const scores = Object.values(resp.classification);
+		if (scores.length === 0) return {
+			pass: false,
+			score: 0,
+			reason: "No classification scores returned"
+		};
+		score = Math.max(...scores);
+	} else score = resp.classification[expected] || 0;
+	if (score >= threshold - Number.EPSILON) {
+		const reason = expected === void 0 ? `Maximum classification score ${score.toFixed(2)} >= ${threshold}` : `Classification ${expected} has score ${score.toFixed(2)} >= ${threshold}`;
+		return {
+			pass: true,
+			score,
+			reason
+		};
+	}
+	return {
+		pass: false,
+		score,
+		reason: expected === void 0 ? `Maximum classification score ${score.toFixed(2)} < ${threshold}` : `Classification ${expected} has score ${score.toFixed(2)} < ${threshold}`
+	};
+}
+//#endregion
+//#region src/matchers/comparison.ts
+async function matchesSelectBest(criteria, outputs, grading, vars, providerCallContext) {
+	invariant(outputs.length >= 2, "select-best assertion must have at least two outputs to compare between");
+	const resp = await callProviderWithContext(await getAndCheckProvider("text", grading?.provider, (await getDefaultProviders()).gradingProvider, "select-best check"), await renderLlmRubricPrompt(await loadRubricPrompt(grading?.rubricPrompt, SELECT_BEST_PROMPT), {
+		criteria,
+		outputs: outputs.map((o) => tryParse(o)),
+		...vars || {}
+	}), "select-best", {
+		criteria,
+		outputs: outputs.map((o) => tryParse(o)),
+		...vars || {}
+	}, providerCallContext);
+	if (resp.error || !resp.output) return Array.from({ length: outputs.length }, () => fail(resp.error || "No output", resp.tokenUsage));
+	invariant(typeof resp.output === "string", "select-best produced malformed response");
+	const firstIntegerMatch = resp.output.trim().match(/\d+/);
+	const verdict = firstIntegerMatch ? Number.parseInt(firstIntegerMatch[0], 10) : NaN;
+	if (Number.isNaN(verdict) || verdict < 0 || verdict >= outputs.length) return Array.from({ length: outputs.length }, () => fail(`Invalid select-best verdict: ${verdict}`, resp.tokenUsage));
+	const tokensUsed = normalizeMatcherTokenUsage(resp.tokenUsage);
+	return outputs.map((_output, index) => {
+		if (index === verdict) return {
+			pass: true,
+			score: 1,
+			reason: `Output selected as the best: ${criteria}`,
+			tokensUsed
+		};
+		else return {
+			pass: false,
+			score: 0,
+			reason: `Output not selected: ${criteria}`,
+			tokensUsed
+		};
+	});
+}
+async function selectMaxScore(outputs, resultsWithGradingResults, assertion) {
+	invariant(outputs.length >= 2, "max-score assertion must have at least two outputs to compare between");
+	const value = assertion.value || {};
+	const options = {
+		method: typeof value === "object" && "method" in value ? value.method : "average",
+		weights: typeof value === "object" && "weights" in value ? value.weights : {},
+		threshold: typeof value === "object" && "threshold" in value ? value.threshold : void 0
+	};
+	const scores = resultsWithGradingResults.map((result, index) => {
+		const relevantResults = (result.gradingResult?.componentResults || []).filter((r) => r.assertion && r.assertion.type !== "max-score" && r.assertion.type !== "select-best");
+		if (relevantResults.length === 0) throw new Error("max-score requires at least one other assertion (besides max-score or select-best) to aggregate scores from");
+		let totalWeightedScore = 0;
+		let totalWeight = 0;
+		relevantResults.forEach((componentResult) => {
+			const assertionType = componentResult.assertion?.type || "unknown";
+			const weight = options.weights[assertionType] === void 0 ? 1 : options.weights[assertionType];
+			const score = componentResult.score || 0;
+			totalWeightedScore += score * weight;
+			totalWeight += weight;
+		});
+		let aggregateScore;
+		if (options.method === "sum") aggregateScore = totalWeightedScore;
+		else aggregateScore = totalWeight > 0 ? totalWeightedScore / totalWeight : 0;
+		return {
+			index,
+			score: aggregateScore,
+			componentCount: relevantResults.length,
+			totalWeight
+		};
+	});
+	let maxScore = -Infinity;
+	let winnerIndex = 0;
+	for (let i = 0; i < scores.length; i++) if (scores[i].score > maxScore) {
+		maxScore = scores[i].score;
+		winnerIndex = i;
+	}
+	const meetsThreshold = options.threshold === void 0 || maxScore >= options.threshold;
+	return scores.map(({ index, score, componentCount, totalWeight }) => {
+		const isWinner = index === winnerIndex && meetsThreshold;
+		return {
+			pass: isWinner,
+			score: isWinner ? 1 : 0,
+			reason: isWinner ? `Selected as highest scoring output (score: ${score.toFixed(3)})` : score === maxScore && !meetsThreshold ? `Not selected - score ${score.toFixed(3)} below threshold ${options.threshold}` : `Not selected (score: ${score.toFixed(3)}, max: ${maxScore.toFixed(3)})`,
+			namedScores: {
+				maxScore: score,
+				assertionCount: componentCount,
+				totalWeight
+			}
+		};
+	});
+}
+//#endregion
+//#region src/matchers/moderation.ts
+async function matchesModeration({ userPrompt, assistantResponse, categories = [] }, grading) {
+	if (!assistantResponse) return {
+		pass: true,
+		score: 1,
+		reason: "No output to moderate"
+	};
+	const defaultProviders = await getDefaultProviders();
+	const defaultModerationProvider = !getEnvString("OPENAI_API_KEY") && (getEnvString("REPLICATE_API_KEY") || getEnvString("REPLICATE_API_TOKEN")) ? await loadApiProvider(LLAMA_GUARD_REPLICATE_PROVIDER) : defaultProviders.moderationProvider;
+	const moderationProvider = await getAndCheckProvider("moderation", grading?.provider, defaultModerationProvider, "moderation check");
+	invariant(moderationProvider, "Moderation provider must be defined");
+	const resp = await moderationProvider.callModerationApi(userPrompt, assistantResponse);
+	if (resp.error) return {
+		pass: false,
+		score: 0,
+		reason: `Moderation API error: ${resp.error}`
+	};
+	const { flags } = resp;
+	if (!flags || flags.length === 0) return {
+		pass: true,
+		score: 1,
+		reason: "No moderation flags detected"
+	};
+	const filteredFlags = categories.length === 0 ? flags : flags.filter((flag) => categories.includes(flag.code));
+	if (filteredFlags.length > 0) return {
+		pass: false,
+		score: 0,
+		reason: `Moderation flags detected: ${filteredFlags.map((flag) => flag.description).join(", ")}`
+	};
+	return {
+		pass: true,
+		score: 1,
+		reason: "No relevant moderation flags detected"
+	};
+}
+//#endregion
+//#region src/assertions/contextUtils.ts
+/**
+* Resolves the context value for context-based assertions.
+* Supports extracting context from test variables or transforming from output.
+* Can return either a single context string or an array of context chunks.
+*
+* @param assertion - The assertion configuration
+* @param test - The test case
+* @param output - The provider output (after provider transform, before test transform)
+* @param prompt - The prompt text
+* @param fallbackContext - Optional fallback context (e.g., prompt for context-recall)
+* @param providerResponse - Optional full provider response for contextTransform
+* @returns The resolved context string or array of strings
+* @throws Error if context cannot be resolved or transform fails
+*/
+async function resolveContext(assertion, test, output, prompt, fallbackContext, providerResponse) {
+	let contextValue;
+	if (test.vars?.context) {
+		if (typeof test.vars.context === "string") contextValue = test.vars.context;
+		else if (Array.isArray(test.vars.context)) {
+			const invalidEntry = [...test.vars.context.entries()].find(([, v]) => typeof v !== "string");
+			if (invalidEntry) {
+				const [idx, val] = invalidEntry;
+				invariant(false, `Invalid context: expected an array of strings, but found ${typeof val} at index ${idx}`);
+			}
+			contextValue = test.vars.context;
+		}
+	} else if (fallbackContext) contextValue = fallbackContext;
+	if (assertion.contextTransform) try {
+		const outputForTransform = providerResponse?.providerTransformedOutput ?? output;
+		const transformed = await transform(assertion.contextTransform, outputForTransform, {
+			vars: test.vars,
+			prompt: { label: prompt },
+			...providerResponse && providerResponse.metadata && { metadata: providerResponse.metadata }
+		});
+		invariant(typeof transformed === "string" || Array.isArray(transformed) && transformed.every((item) => typeof item === "string"), `contextTransform must return a string or array of strings. Got ${typeof transformed}. Check your transform expression: ${assertion.contextTransform}`);
+		contextValue = transformed;
+	} catch (error) {
+		throw new Error(`Failed to transform context using expression '${assertion.contextTransform}': ${error instanceof Error ? error.message : String(error)}`);
+	}
+	invariant(typeof contextValue === "string" && contextValue.length > 0 || Array.isArray(contextValue) && contextValue.length > 0 && contextValue.every((item) => typeof item === "string" && item.length > 0), "Context is required for context-based assertions. Provide either a \"context\" variable (string or array of strings) in your test case or use \"contextTransform\" to extract context from the provider response.");
+	return contextValue;
+}
+/**
+* Serializes context (string or string[]) to a single string for prompts.
+* Joins chunks with double newlines to preserve separation.
+*/
+function serializeContext(context) {
+	return Array.isArray(context) ? context.join("\n\n") : context;
+}
+//#endregion
+//#region src/matchers/rag.ts
+async function matchesAnswerRelevance(input, output, threshold, grading, providerCallContext) {
+	const defaults = await getDefaultProviders();
+	const embeddingProvider = await getAndCheckProvider("embedding", grading?.provider, defaults.embeddingProvider, "answer relevancy check");
+	const textProvider = await getAndCheckProvider("text", grading?.provider, defaults.gradingProvider, "answer relevancy check");
+	const tokensUsed = normalizeMatcherTokenUsage(void 0);
+	const rubricPrompt = await loadRubricPrompt(grading?.rubricPrompt, ANSWER_RELEVANCY_GENERATE);
+	const parsedOutput = tryParse(output);
+	const promptText = await renderLlmRubricPrompt(rubricPrompt, { answer: parsedOutput });
+	const candidateQuestions = [];
+	for (let i = 0; i < 3; i++) {
+		const resp = await callProviderWithContext(textProvider, promptText, "answer-relevance", { answer: parsedOutput }, providerCallContext);
+		accumulateTokenUsage(tokensUsed, resp.tokenUsage);
+		if (resp.error || !resp.output) return fail(resp.error || "No output", tokensUsed);
+		invariant(typeof resp.output === "string", "answer relevancy check produced malformed response");
+		candidateQuestions.push(resp.output);
+	}
+	invariant(typeof embeddingProvider.callEmbeddingApi === "function", `Provider ${embeddingProvider.id()} must implement callEmbeddingApi for similarity check`);
+	const inputEmbeddingResp = await embeddingProvider.callEmbeddingApi(input);
+	accumulateTokenUsage(tokensUsed, inputEmbeddingResp.tokenUsage);
+	if (inputEmbeddingResp.error || !inputEmbeddingResp.embedding) return fail(inputEmbeddingResp.error || "No embedding", tokensUsed);
+	const inputEmbedding = inputEmbeddingResp.embedding;
+	const similarities = [];
+	const questionsWithScores = [];
+	for (const question of candidateQuestions) {
+		const resp = await embeddingProvider.callEmbeddingApi(question);
+		accumulateTokenUsage(tokensUsed, resp.tokenUsage);
+		if (resp.error || !resp.embedding) return fail(resp.error || "No embedding", tokensUsed);
+		const questionSimilarity = cosineSimilarity(inputEmbedding, resp.embedding);
+		similarities.push(questionSimilarity);
+		questionsWithScores.push({
+			question,
+			similarity: questionSimilarity
+		});
+	}
+	const similarity = similarities.reduce((a, b) => a + b, 0) / similarities.length;
+	const pass = similarity >= threshold - Number.EPSILON;
+	const greaterThanReason = `Relevance ${similarity.toFixed(2)} is greater than threshold ${threshold}`;
+	const lessThanReason = `Relevance ${similarity.toFixed(2)} is less than threshold ${threshold}`;
+	const metadata = {
+		generatedQuestions: questionsWithScores,
+		averageSimilarity: similarity,
+		threshold
+	};
+	if (pass) return {
+		pass: true,
+		score: similarity,
+		reason: greaterThanReason,
+		tokensUsed,
+		metadata
+	};
+	return {
+		pass: false,
+		score: similarity,
+		reason: lessThanReason,
+		tokensUsed,
+		metadata
+	};
+}
+async function matchesContextRecall(context, groundTruth, threshold, grading, vars, providerCallContext) {
+	const textProvider = await getAndCheckProvider("text", grading?.provider, (await getDefaultProviders()).gradingProvider, "context recall check");
+	const contextString = serializeContext(context);
+	const resp = await callProviderWithContext(textProvider, await renderLlmRubricPrompt(await loadRubricPrompt(grading?.rubricPrompt, CONTEXT_RECALL), {
+		context: contextString,
+		groundTruth,
+		...vars || {}
+	}), "context-recall", {
+		context: contextString,
+		groundTruth,
+		...vars || {}
+	}, providerCallContext);
+	if (resp.error || !resp.output) return fail(resp.error || "No output", resp.tokenUsage);
+	invariant(typeof resp.output === "string", "context-recall produced malformed response");
+	const attributedTokenLower = CONTEXT_RECALL_ATTRIBUTED_TOKEN.toLowerCase();
+	const notAttributedTokenLower = CONTEXT_RECALL_NOT_ATTRIBUTED_TOKEN.toLowerCase();
+	const sentences = splitIntoSentences(resp.output).filter((line) => {
+		const lowerLine = line.toLowerCase();
+		return lowerLine.includes(attributedTokenLower) || lowerLine.includes(notAttributedTokenLower);
+	});
+	const sentenceAttributions = [];
+	let numerator = 0;
+	for (const sentence of sentences) {
+		const lowerSentence = sentence.toLowerCase();
+		const isAttributed = !lowerSentence.includes(notAttributedTokenLower) && lowerSentence.includes(attributedTokenLower);
+		if (isAttributed) numerator++;
+		const sentenceMatch = sentence.match(/^\d+\.\s*([^\.]+\.)/);
+		const cleanSentence = sentenceMatch ? sentenceMatch[1].trim() : sentence.split(".")[0].trim();
+		sentenceAttributions.push({
+			sentence: cleanSentence,
+			attributed: isAttributed
+		});
+	}
+	const score = sentences.length > 0 ? numerator / sentences.length : 0;
+	const pass = score >= threshold - Number.EPSILON;
+	const metadata = {
+		sentenceAttributions,
+		totalSentences: sentences.length,
+		attributedSentences: numerator,
+		score
+	};
+	return {
+		pass,
+		score,
+		reason: pass ? `Recall ${score.toFixed(2)} is >= ${threshold}` : `Recall ${score.toFixed(2)} is < ${threshold}`,
+		tokensUsed: normalizeMatcherTokenUsage(resp.tokenUsage),
+		metadata
+	};
+}
+async function matchesContextRelevance(question, context, threshold, grading, providerCallContext) {
+	const textProvider = await getAndCheckProvider("text", grading?.provider, (await getDefaultProviders()).gradingProvider, "context relevance check");
+	const contextString = serializeContext(context);
+	const resp = await callProviderWithContext(textProvider, await renderLlmRubricPrompt(await loadRubricPrompt(grading?.rubricPrompt, CONTEXT_RELEVANCE), {
+		context: contextString,
+		query: question
+	}), "context-relevance", {
+		context: contextString,
+		query: question
+	}, providerCallContext);
+	if (resp.error || !resp.output) return fail(resp.error || "No output", resp.tokenUsage);
+	invariant(typeof resp.output === "string", "context-relevance produced malformed response");
+	const contextUnits = Array.isArray(context) ? context.filter((chunk) => chunk.trim().length > 0) : splitIntoSentences(context);
+	const totalContextUnits = contextUnits.length;
+	const extractedSentences = splitIntoSentences(resp.output);
+	const relevantSentences = [];
+	const insufficientInformation = resp.output.includes(CONTEXT_RELEVANCE_BAD);
+	let numerator = 0;
+	if (insufficientInformation) numerator = 0;
+	else {
+		const uniqueRelevantSentences = [...new Set(extractedSentences)];
+		numerator = Math.min(uniqueRelevantSentences.length, totalContextUnits);
+		relevantSentences.push(...uniqueRelevantSentences);
+	}
+	const score = totalContextUnits > 0 ? numerator / totalContextUnits : 0;
+	const pass = score >= threshold - Number.EPSILON;
+	const metadata = {
+		extractedSentences: relevantSentences,
+		totalContextUnits,
+		totalContextSentences: totalContextUnits,
+		contextUnits,
+		relevantSentenceCount: numerator,
+		insufficientInformation,
+		score
+	};
+	return {
+		pass,
+		score,
+		reason: pass ? `Context relevance ${score.toFixed(2)} is >= ${threshold}` : `Context relevance ${score.toFixed(2)} is < ${threshold}`,
+		tokensUsed: normalizeMatcherTokenUsage(resp.tokenUsage),
+		metadata
+	};
+}
+async function matchesContextFaithfulness(query, output, context, threshold, grading, vars, providerCallContext) {
+	const textProvider = await getAndCheckProvider("text", grading?.provider, (await getDefaultProviders()).gradingProvider, "faithfulness check");
+	const tokensUsed = normalizeMatcherTokenUsage(void 0);
+	if (grading?.rubricPrompt) invariant(Array.isArray(grading.rubricPrompt), "rubricPrompt must be an array");
+	const rawLongformPrompt = typeof grading?.rubricPrompt?.[0] === "string" ? grading?.rubricPrompt?.[0] : grading?.rubricPrompt?.[0]?.content;
+	const rawNliPrompt = typeof grading?.rubricPrompt?.[1] === "string" ? grading?.rubricPrompt?.[1] : grading?.rubricPrompt?.[1]?.content;
+	const longformPrompt = await loadRubricPrompt(rawLongformPrompt, CONTEXT_FAITHFULNESS_LONGFORM);
+	const nliPrompt = await loadRubricPrompt(rawNliPrompt, CONTEXT_FAITHFULNESS_NLI_STATEMENTS);
+	let promptText = await renderLlmRubricPrompt(longformPrompt, {
+		question: query,
+		answer: tryParse(output),
+		...vars || {}
+	});
+	let resp = await callProviderWithContext(textProvider, promptText, "context-faithfulness-longform", {
+		question: query,
+		answer: tryParse(output),
+		...vars || {}
+	}, providerCallContext);
+	accumulateTokenUsage(tokensUsed, resp.tokenUsage);
+	if (resp.error || !resp.output) return fail(resp.error || "No output", tokensUsed);
+	invariant(typeof resp.output === "string", "context-faithfulness produced malformed response");
+	const contextString = serializeContext(context);
+	const statements = splitIntoSentences(resp.output);
+	promptText = await renderLlmRubricPrompt(nliPrompt, {
+		context: contextString,
+		statements,
+		...vars || {}
+	});
+	resp = await callProviderWithContext(textProvider, promptText, "context-faithfulness-nli", {
+		context: contextString,
+		statements,
+		...vars || {}
+	}, providerCallContext);
+	accumulateTokenUsage(tokensUsed, resp.tokenUsage);
+	if (resp.error || !resp.output) return fail(resp.error || "No output", tokensUsed);
+	invariant(typeof resp.output === "string", "context-faithfulness produced malformed response");
+	let finalAnswer = "Final verdict for each statement in order:";
+	finalAnswer = finalAnswer.toLowerCase();
+	let verdicts = resp.output.toLowerCase().trim();
+	let score = 0;
+	if (statements.length > 0) if (verdicts.includes(finalAnswer)) {
+		verdicts = verdicts.slice(verdicts.indexOf(finalAnswer) + finalAnswer.length);
+		const parsedVerdicts = verdicts.split(".").filter((answer) => answer.trim() !== "");
+		if (parsedVerdicts.length > 0) score = 1 - parsedVerdicts.filter((answer) => !answer.includes("yes")).length / statements.length;
+	} else {
+		const noVerdictCount = verdicts.split("verdict: no").length - 1;
+		if (noVerdictCount + (verdicts.split("verdict: yes").length - 1) > 0) score = 1 - noVerdictCount / statements.length;
+	}
+	score = Math.min(1, Math.max(0, score));
+	const pass = score >= threshold - Number.EPSILON;
+	return {
+		pass,
+		score,
+		reason: pass ? `Faithfulness ${score.toFixed(2)} is >= ${threshold}` : `Faithfulness ${score.toFixed(2)} is < ${threshold}`,
+		tokensUsed
+	};
+}
+//#endregion
+//#region src/matchers/similarity.ts
+function calculateSimilarityScore(expectedEmbedding, outputEmbedding, metric, tokensUsed) {
+	switch (metric) {
+		case "cosine": return cosineSimilarity(expectedEmbedding, outputEmbedding);
+		case "dot_product": return dotProduct(expectedEmbedding, outputEmbedding);
+		case "euclidean": return euclideanDistance(expectedEmbedding, outputEmbedding);
+		default: return fail(`Unsupported metric: ${metric}`, tokensUsed);
+	}
+}
+function buildSimilarityResult(similarity, threshold, inverse, metric, tokensUsed) {
+	if (metric === "euclidean") {
+		const distance = similarity;
+		const pass = inverse ? distance >= threshold - Number.EPSILON : distance <= threshold + Number.EPSILON;
+		const normalizedScore = 1 / (1 + distance);
+		const score = inverse ? 1 - normalizedScore : normalizedScore;
+		const belowThresholdReason = `Distance ${distance.toFixed(2)} is less than or equal to threshold ${threshold}`;
+		const aboveThresholdReason = `Distance ${distance.toFixed(2)} is greater than threshold ${threshold}`;
+		return {
+			pass,
+			score,
+			reason: pass ? inverse ? aboveThresholdReason : belowThresholdReason : inverse ? belowThresholdReason : aboveThresholdReason,
+			tokensUsed
+		};
+	}
+	const pass = inverse ? similarity <= threshold + Number.EPSILON : similarity >= threshold - Number.EPSILON;
+	const score = inverse ? 1 - similarity : similarity;
+	const greaterThanReason = `Similarity ${similarity.toFixed(2)} is greater than or equal to threshold ${threshold}`;
+	const lessThanReason = `Similarity ${similarity.toFixed(2)} is less than threshold ${threshold}`;
+	return {
+		pass,
+		score,
+		reason: pass ? inverse ? lessThanReason : greaterThanReason : inverse ? greaterThanReason : lessThanReason,
+		tokensUsed
+	};
+}
+async function calculateProviderSimilarity(finalProvider, expected, output, metric, tokensUsed) {
+	if (metric === "cosine" && "callSimilarityApi" in finalProvider) {
+		const similarityResp = await finalProvider.callSimilarityApi(expected, output);
+		accumulateTokenUsage(tokensUsed, similarityResp.tokenUsage);
+		if (similarityResp.error) return fail(similarityResp.error, tokensUsed);
+		if (similarityResp.similarity == null) return fail("Unknown error fetching similarity", tokensUsed);
+		if (!Number.isFinite(similarityResp.similarity)) return fail(`Invalid similarity score: ${similarityResp.similarity}`, tokensUsed);
+		return similarityResp.similarity;
+	}
+	const callEmbeddingApi = "callEmbeddingApi" in finalProvider ? finalProvider.callEmbeddingApi : void 0;
+	if (typeof callEmbeddingApi !== "function") {
+		if ("callSimilarityApi" in finalProvider) return fail(`Provider ${finalProvider.id()} only supports cosine similarity via callSimilarityApi`, tokensUsed);
+		throw new Error("Provider must implement callSimilarityApi or callEmbeddingApi");
+	}
+	const [expectedEmbedding, outputEmbedding] = await Promise.all([callEmbeddingApi.call(finalProvider, expected), callEmbeddingApi.call(finalProvider, output)]);
+	const mergedUsage = normalizeMatcherTokenUsage(void 0);
+	accumulateTokenUsage(mergedUsage, expectedEmbedding.tokenUsage);
+	accumulateTokenUsage(mergedUsage, outputEmbedding.tokenUsage);
+	accumulateTokenUsage(tokensUsed, mergedUsage);
+	if (expectedEmbedding.error || outputEmbedding.error) return fail(expectedEmbedding.error || outputEmbedding.error || "Unknown error fetching embeddings", tokensUsed);
+	if (!expectedEmbedding.embedding || !outputEmbedding.embedding) return fail("Embedding not found", tokensUsed);
+	return calculateSimilarityScore(expectedEmbedding.embedding, outputEmbedding.embedding, metric, tokensUsed);
+}
+async function matchesSimilarity(expected, output, threshold, inverse = false, grading, metric = "cosine") {
+	if (metric === "cosine" && state.config?.redteam && shouldGenerateRemote({ requireEmbeddingProvider: true })) try {
+		return await doRemoteGrading({
+			task: "similar",
+			expected,
+			output,
+			threshold,
+			inverse
+		});
+	} catch (error) {
+		return fail(`Could not perform remote grading: ${error}`);
+	}
+	const defaults = await getDefaultProviders();
+	const finalProvider = await getAndCheckProvider("embedding", grading?.provider, defaults.embeddingProvider, "similarity check");
+	const tokensUsed = normalizeMatcherTokenUsage(void 0);
+	const similarity = await calculateProviderSimilarity(finalProvider, expected, output, metric, tokensUsed);
+	if (typeof similarity !== "number") return similarity;
+	return buildSimilarityResult(similarity, threshold, inverse, metric, tokensUsed);
+}
+//#endregion
 //#region src/tracing/evaluatorTracing.ts
 let otlpReceiverStarted = false;
 const DEFAULT_OTLP_ACCEPT_FORMATS = ["json", "protobuf"];
@@ -285,7 +782,7 @@ async function startOtlpReceiverIfNeeded(testSuite) {
 		telemetry.record("feature_used", { feature: "tracing" });
 		try {
 			logger.debug("[EvaluatorTracing] Tracing configuration detected, starting OTLP receiver");
-			const { startOTLPReceiver } = await import("./otlpReceiver-B2z58l4e.js");
+			const { startOTLPReceiver } = await import("./otlpReceiver-CcdIikOu.js");
 			const port = testSuite.tracing.otlp.http.port || 4318;
 			const host = testSuite.tracing.otlp.http.host || "127.0.0.1";
 			const acceptFormats = normalizeOtlpAcceptFormats(testSuite.tracing.otlp.http.acceptFormats);
@@ -309,7 +806,7 @@ async function startOtlpReceiverIfNeeded(testSuite) {
 async function stopOtlpReceiverIfNeeded() {
 	if (otlpReceiverStarted) try {
 		logger.debug("[EvaluatorTracing] Stopping OTLP receiver");
-		const { stopOTLPReceiver } = await import("./otlpReceiver-B2z58l4e.js");
+		const { stopOTLPReceiver } = await import("./otlpReceiver-CcdIikOu.js");
 		await stopOTLPReceiver();
 		otlpReceiverStarted = false;
 		logger.info("[EvaluatorTracing] OTLP receiver stopped successfully");
@@ -344,7 +841,7 @@ async function generateTraceContextIfNeeded(test, evaluateOptions, testIdx, prom
 	}
 	if (!tracingEnabled) return null;
 	logger.debug("[EvaluatorTracing] Importing trace store");
-	const { getTraceStore } = await import("./store-M0b1WfYb.js").then((n) => n.n);
+	const { getTraceStore } = await import("./store-BPkzEyFM.js").then((n) => n.n);
 	const traceStore = getTraceStore();
 	const traceId = generateTraceId();
 	const spanId = generateSpanId();
@@ -654,38 +1151,84 @@ async function handleClassifier({ assertion, renderedValue, outputString, test,
 }
 //#endregion
 //#region src/assertions/contains.ts
+/**
+* Advance over separators between parsed fields.
+*
+* Contains-any values allow whitespace around comma delimiters, and historical
+* parsing ignored repeated commas rather than producing empty fields.
+*/
+function skipWhitespaceAndCommas(value, startIndex) {
+	let i = startIndex;
+	while (i < value.length) {
+		i = skipWhitespace(value, i);
+		if (value[i] !== ",") break;
+		i++;
+	}
+	return i;
+}
+/**
+* Advance over whitespace while preserving comma delimiter handling for callers.
+*/
+function skipWhitespace(value, startIndex) {
+	let i = startIndex;
+	while (i < value.length && /\s/.test(value[i])) i++;
+	return i;
+}
+/**
+* Parse a quoted field using the assertion parser's CSV-like escape rules.
+*
+* Supports backslash-escaped quotes/backslashes and doubled quotes, and rejects
+* unterminated fields so malformed assertion values do not silently pass.
+*/
+function parseQuotedField(value, startIndex) {
+	let i = startIndex + 1;
+	let field = "";
+	let terminated = false;
+	while (i < value.length) if (value[i] === "\\" && i + 1 < value.length && ["\"", "\\"].includes(value[i + 1])) {
+		field += value[i + 1];
+		i += 2;
+	} else if (value[i] === "\"" && i + 1 < value.length && value[i + 1] === "\"") {
+		field += "\"";
+		i += 2;
+	} else if (value[i] === "\"") {
+		i++;
+		terminated = true;
+		break;
+	} else {
+		field += value[i];
+		i++;
+	}
+	invariant(terminated, "Unterminated quoted field in contains assertion value");
+	return {
+		field,
+		nextIndex: i
+	};
+}
+/**
+* Parse an unquoted field up to the next comma, trimming surrounding whitespace.
+*/
+function parseUnquotedField(value, startIndex) {
+	let i = startIndex;
+	while (i < value.length && value[i] !== ",") i++;
+	return {
+		field: value.substring(startIndex, i).trim(),
+		nextIndex: i
+	};
+}
+/**
+* Split a contains-any string into fields while preserving quoted commas.
+*/
 function parseCommaSeparatedValues(value) {
 	const results = [];
 	let i = 0;
 	while (i < value.length) {
-		while (i < value.length && /\s/.test(value[i])) i++;
+		i = skipWhitespaceAndCommas(value, i);
 		if (i >= value.length) break;
-		if (value[i] === ",") {
-			i++;
-			continue;
-		}
-		if (value[i] === "\"") {
-			i++;
-			let field = "";
-			while (i < value.length) if (value[i] === "\\" && i + 1 < value.length && (value[i + 1] === "\"" || value[i + 1] === "\\")) {
-				field += value[i + 1];
-				i += 2;
-			} else if (value[i] === "\"" && i + 1 < value.length && value[i + 1] === "\"") {
-				field += "\"";
-				i += 2;
-			} else if (value[i] === "\"") {
-				i++;
-				break;
-			} else {
-				field += value[i];
-				i++;
-			}
-			results.push(field);
-		} else {
-			const start = i;
-			while (i < value.length && value[i] !== ",") i++;
-			results.push(value.substring(start, i).trim());
-		}
+		const isQuotedField = value[i] === "\"";
+		const parsed = isQuotedField ? parseQuotedField(value, i) : parseUnquotedField(value, i);
+		results.push(parsed.field);
+		i = isQuotedField ? skipWhitespace(value, parsed.nextIndex) : parsed.nextIndex;
+		invariant(!isQuotedField || i >= value.length || value[i] === ",", "Expected comma after quoted field in contains assertion value");
 	}
 	return results;
 }
@@ -1090,6 +1633,43 @@ const handleGuardrails = async ({ assertion, inverse, providerResponse }) => {
 };
 //#endregion
 //#region src/assertions/html.ts
+const LITERAL_WRAPPER_PATTERNS = {
+	html: /<html(?=[\s>/])/,
+	head: /<head(?=[\s>/])/,
+	body: /<body(?=[\s>/])/
+};
+function isWrapperTagName(tagName) {
+	return tagName === "html" || tagName === "head" || tagName === "body";
+}
+function isTextNode(node) {
+	return node.nodeName === "#text";
+}
+function isElementNode(node) {
+	return "tagName" in node;
+}
+function hasSourceCodeLocation(element) {
+	return "sourceCodeLocation" in element && element.sourceCodeLocation !== null && element.sourceCodeLocation !== void 0;
+}
+function getChildNodes(node) {
+	return "childNodes" in node ? node.childNodes : [];
+}
+function findFirstElement(root, predicate) {
+	const stack = [root];
+	while (stack.length > 0) {
+		const current = stack.pop();
+		if (isElementNode(current) && predicate(current)) return current;
+		const children = getChildNodes(current);
+		for (let i = children.length - 1; i >= 0; i--) stack.push(children[i]);
+	}
+}
+function hasTopLevelText(parentNode) {
+	return parentNode.childNodes.some((node) => isTextNode(node) && Boolean(node.value.trim()));
+}
+function isUserProvidedElement(element, inputLowercase) {
+	const tagName = element.tagName.toLowerCase();
+	if (isWrapperTagName(tagName)) return LITERAL_WRAPPER_PATTERNS[tagName].test(inputLowercase) && hasSourceCodeLocation(element);
+	return VALID_HTML_ELEMENTS.has(tagName) || tagName.includes("-");
+}
 const HTML_PATTERNS = {
 	openingTag: /<[a-zA-Z][a-zA-Z0-9-]*(?:\s[^>]*)?>/,
 	closingTag: /<\/[a-zA-Z][a-zA-Z0-9-]*\s*>/,
@@ -1245,37 +1825,21 @@ function validateHtml(htmlString) {
 		isValid: false,
 		reason: "Output appears to be XML, not HTML"
 	};
-	try {
-		const { document } = new JSDOM(trimmed, { contentType: "text/html" }).window;
-		if (document.body && !trimmed.toLowerCase().includes("<body")) {
-			if (Array.from(document.body.childNodes).some((node) => node.nodeType === 3 && node.textContent?.trim())) return {
-				isValid: false,
-				reason: "Output must be wrapped in HTML tags"
-			};
-		}
-		const allElements = document.querySelectorAll("*");
-		if (!Array.from(allElements).find((element) => {
-			const tagName = element.tagName.toLowerCase();
-			if ([
-				"html",
-				"head",
-				"body"
-			].includes(tagName) && !trimmed.toLowerCase().includes(`<${tagName}`)) return false;
-			return VALID_HTML_ELEMENTS.has(tagName) || tagName.includes("-");
-		})) return {
-			isValid: false,
-			reason: "Output does not contain recognized HTML elements"
-		};
-		return {
-			isValid: true,
-			reason: "Output is valid HTML"
-		};
-	} catch (error) {
-		return {
-			isValid: false,
-			reason: `HTML parsing failed: ${error instanceof Error ? error.message : "Unknown error"}`
-		};
-	}
+	const document = parse$2(trimmed, { sourceCodeLocationInfo: true });
+	const inputLowercase = trimmed.toLowerCase();
+	const body = findFirstElement(document, (element) => element.tagName === "body");
+	if (!(body !== void 0 && LITERAL_WRAPPER_PATTERNS.body.test(inputLowercase) && hasSourceCodeLocation(body)) && body && hasTopLevelText(body)) return {
+		isValid: false,
+		reason: "Output must be wrapped in HTML tags"
+	};
+	if (!findFirstElement(document, (element) => isUserProvidedElement(element, inputLowercase))) return {
+		isValid: false,
+		reason: "Output does not contain recognized HTML elements"
+	};
+	return {
+		isValid: true,
+		reason: "Output is valid HTML"
+	};
 }
 const handleContainsHtml = ({ assertion, outputString, inverse }) => {
 	const pass = containsHtml(outputString) !== inverse;
@@ -2380,11 +2944,10 @@ function handleRougeScore({ baseType, assertion, renderedValue, outputString, in
 	const rougeMethod = rouge[baseType[baseType.length - 1]];
 	const score = rougeMethod(outputString, renderedValue, {});
 	const threshold = assertion.threshold ?? .75;
-	const pass = score >= threshold != inverse;
 	return {
-		pass,
+		pass: score >= threshold !== inverse,
 		score: inverse ? 1 - score : score,
-		reason: pass ? `${baseType.toUpperCase()} score ${score.toFixed(2)} is greater than or equal to threshold ${threshold}` : `${baseType.toUpperCase()} score ${score.toFixed(2)} is less than threshold ${threshold}`,
+		reason: `${baseType.toUpperCase()} score ${score.toFixed(2)} is ${score >= threshold ? "greater than or equal to" : "less than"} threshold ${threshold}`,
 		assertion
 	};
 }
@@ -2446,6 +3009,192 @@ const handleRuby = async ({ assertion, renderedValue, valueFromScript, assertion
 	}
 };
 //#endregion
+//#region src/providers/webSearchUtils.ts
+function hasTool(provider, predicate) {
+	return Array.isArray(provider.config?.tools) && provider.config.tools.some(predicate);
+}
+function getProviderId(provider) {
+	if (typeof provider.id !== "function") return null;
+	try {
+		return provider.id();
+	} catch (err) {
+		logger.debug(`Failed to read provider id: ${err}`);
+		return null;
+	}
+}
+function isOpenAiResponsesProvider(provider, id) {
+	return id.includes("openai:responses") || provider.constructor?.name === "OpenAiResponsesProvider";
+}
+/**
+* Check if a provider has web search capabilities
+* @param provider The provider to check
+* @returns true if the provider supports web search
+*/
+function hasWebSearchCapability(provider) {
+	if (!provider) return false;
+	const id = getProviderId(provider);
+	if (!id) return false;
+	if (id.includes("perplexity")) return true;
+	if ((id.includes("google") || id.includes("gemini") || id.includes("vertex")) && hasTool(provider, (t) => t.googleSearch !== void 0)) return true;
+	if (id.includes("xai") && provider.config?.search_parameters?.mode === "on") return true;
+	if (isOpenAiResponsesProvider(provider, id) && hasTool(provider, (t) => t.type === "web_search_preview")) return true;
+	if (id.startsWith("openai:codex") && (provider.config?.web_search_mode === "live" || provider.config?.web_search_mode === "cached" || provider.config?.web_search_enabled === true)) return true;
+	if (id.includes("anthropic") && hasTool(provider, (t) => t.type === "web_search_20250305")) return true;
+	return false;
+}
+/**
+* Load a provider with web search capabilities.
+* Tries multiple providers in order of preference until one succeeds.
+* Uses the latest and most capable models from each provider with specific checkpoint IDs.
+*
+* @param preferAnthropic Whether to try Anthropic first (true) or OpenAI first (false)
+* @returns A provider with web search capabilities or null
+*/
+async function loadWebSearchProvider(preferAnthropic = false) {
+	const loadAnthropicWebSearch = async () => {
+		try {
+			return await loadApiProvider("anthropic:messages:claude-opus-4-6", { options: { config: { tools: [{
+				type: "web_search_20250305",
+				name: "web_search",
+				max_uses: 5
+			}] } } });
+		} catch (err) {
+			logger.debug(`Failed to load Anthropic web search provider: ${err}`);
+			return null;
+		}
+	};
+	const loadOpenAIWebSearch = async () => {
+		try {
+			return await loadApiProvider("openai:responses:gpt-5.4-2026-03-05", { options: { config: { tools: [{ type: "web_search_preview" }] } } });
+		} catch (err) {
+			logger.debug(`Failed to load OpenAI web search provider: ${err}`);
+			return null;
+		}
+	};
+	const loadPerplexity = async () => {
+		try {
+			return await loadApiProvider("perplexity:sonar-pro");
+		} catch (err) {
+			logger.debug(`Failed to load Perplexity provider: ${err}`);
+			return null;
+		}
+	};
+	const loadGoogleWebSearch = async () => {
+		try {
+			return await loadApiProvider("google:gemini-3-pro-preview", { options: { config: { tools: [{ googleSearch: {} }] } } });
+		} catch (err) {
+			logger.debug(`Failed to load Google web search provider: ${err}`);
+			return null;
+		}
+	};
+	const loadVertexWebSearch = async () => {
+		try {
+			return await loadApiProvider("vertex:gemini-3-pro-preview", { options: { config: { tools: [{ googleSearch: {} }] } } });
+		} catch (err) {
+			logger.debug(`Failed to load Vertex web search provider: ${err}`);
+			return null;
+		}
+	};
+	const loadXaiWebSearch = async () => {
+		try {
+			return await loadApiProvider("xai:grok-4-1-fast-reasoning", { options: { config: { search_parameters: { mode: "on" } } } });
+		} catch (err) {
+			logger.debug(`Failed to load xAI web search provider: ${err}`);
+			return null;
+		}
+	};
+	const providers = preferAnthropic ? [
+		loadAnthropicWebSearch,
+		loadOpenAIWebSearch,
+		loadPerplexity,
+		loadGoogleWebSearch,
+		loadVertexWebSearch,
+		loadXaiWebSearch
+	] : [
+		loadOpenAIWebSearch,
+		loadAnthropicWebSearch,
+		loadPerplexity,
+		loadGoogleWebSearch,
+		loadVertexWebSearch,
+		loadXaiWebSearch
+	];
+	for (const getProvider of providers) {
+		const provider = await getProvider();
+		if (provider && hasWebSearchCapability(provider)) {
+			logger.info(`Using ${getProviderId(provider) ?? "loaded provider"} as web search provider`);
+			return provider;
+		}
+		if (provider) logger.debug(`Loaded provider ${getProviderId(provider) ?? "unknown"} does not support web search`);
+	}
+	return null;
+}
+//#endregion
+//#region src/matchers/search.ts
+async function matchesSearchRubric(rubric, llmOutput, grading, vars, assertion, _provider, providerCallContext) {
+	if (!grading) throw new Error("Cannot grade output without grading config. Specify --grader option or grading config.");
+	const defaultProviders = await getDefaultProviders();
+	const defaultSearchProviders = [
+		defaultProviders.webSearchProvider,
+		defaultProviders.llmRubricProvider,
+		defaultProviders.gradingProvider
+	];
+	let searchProvider = (grading.provider ? await getGradingProvider("text", grading.provider, null) : null) || defaultSearchProviders.find((provider) => Boolean(provider));
+	if (!hasWebSearchCapability(searchProvider)) {
+		const webSearchDefault = defaultSearchProviders.find((provider) => hasWebSearchCapability(provider));
+		if (webSearchDefault) searchProvider = webSearchDefault;
+	}
+	if (!hasWebSearchCapability(searchProvider)) {
+		const webSearchProvider = await loadWebSearchProvider(true);
+		if (webSearchProvider) searchProvider = webSearchProvider;
+	}
+	if (!searchProvider || !hasWebSearchCapability(searchProvider)) throw new Error(`search-rubric assertion requires a grading provider with web search capabilities. Use --grader with a web search provider (e.g., anthropic:messages:${DEFAULT_ANTHROPIC_MODEL}, openai:responses:o4-mini with tools configured, perplexity:sonar) or configure one in defaultTest.options.provider`);
+	const prompt = await renderLlmRubricPrompt(await loadRubricPrompt(grading?.rubricPrompt, DEFAULT_WEB_SEARCH_PROMPT), {
+		output: tryParse(llmOutput),
+		rubric,
+		...vars || {}
+	});
+	const resp = await callProviderWithContext(searchProvider, prompt, "search-rubric", {
+		output: tryParse(llmOutput),
+		rubric,
+		...vars || {}
+	}, providerCallContext);
+	if (resp.error || !resp.output) return {
+		pass: false,
+		score: 0,
+		reason: `Search rubric evaluation failed: ${resp.error || "No output"}`,
+		tokensUsed: resp.tokenUsage,
+		assertion
+	};
+	try {
+		const result = extractFirstJsonObject(String(resp.output));
+		let pass = result.pass ?? false;
+		const score = typeof result.score === "number" ? result.score : pass ? 1 : 0;
+		if (assertion?.threshold !== void 0) pass = pass && score >= assertion.threshold;
+		return {
+			pass,
+			score,
+			reason: result.reason || "No reason provided",
+			tokensUsed: resp.tokenUsage,
+			assertion,
+			metadata: {
+				searchResults: result.searchResults || [],
+				searchProvider: searchProvider.id()
+			}
+		};
+	} catch (err) {
+		logger.warn(`[search-rubric] Could not parse structured JSON from provider response, falling back to substring matching: ${err.message}`);
+		const outputLower = String(resp.output).toLowerCase();
+		const pass = outputLower.includes("\"pass\":true") || outputLower.includes("\"pass\": true");
+		return {
+			pass,
+			score: pass ? 1 : 0,
+			reason: resp.output,
+			tokensUsed: resp.tokenUsage,
+			assertion
+		};
+	}
+}
+//#endregion
 //#region src/assertions/searchRubric.ts
 async function handleSearchRubric({ assertion, baseType: _baseType, inverse, provider, providerCallContext, renderedValue, test, providerResponse }) {
 	if (renderedValue == null) throw new Error("search-rubric assertion type must have a string value");
@@ -3531,7 +4280,7 @@ const ASSERTION_HANDLERS = {
 	"llm-rubric": handleLlmRubric,
 	meteor: async (params) => {
 		try {
-			const { handleMeteorAssertion } = await import("./meteor-DuAFv6gF.js");
+			const { handleMeteorAssertion } = await import("./meteor-BKTM-7KS.js");
 			return handleMeteorAssertion(params);
 		} catch (error) {
 			if (error instanceof Error && (error.message.includes("Cannot find module") || error.message.includes("natural\" package is required"))) return {
@@ -3667,7 +4416,7 @@ async function runAssertion({ prompt, provider, assertion, test, vars, latencyMs
 			};
 		}
 		else if (filePath.endsWith(".rb")) try {
-			const { runRuby } = await import("./rubyUtils-DVLeA2jg.js").then((n) => n.t);
+			const { runRuby } = await import("./rubyUtils-BI0p46eZ.js").then((n) => n.t);
 			valueFromScript = await runRuby(filePath, functionName || "get_assert", [output, context]);
 			logger.debug(`Ruby script ${filePath} output: ${valueFromScript}`);
 		} catch (error) {
@@ -5465,7 +6214,7 @@ async function resolveDefaultTestProvider(defaultTest, testCase) {
 	const defaultProvider = defaultTest.provider;
 	if (isApiProvider(defaultProvider)) return defaultProvider;
 	if (typeof defaultProvider === "object" && defaultProvider.id) {
-		const { loadApiProvider } = await import("./providers-u9Enmfok.js");
+		const { loadApiProvider } = await import("./providers-CCE2COJi2.js");
 		return loadApiProvider(typeof defaultProvider.id === "function" ? defaultProvider.id() : defaultProvider.id, { options: defaultProvider });
 	}
 	return defaultProvider;
@@ -5625,7 +6374,7 @@ function buildRepeatCacheContextByTestIdx(runEvalOptions) {
 async function filterCompletedResumeSteps(runEvalOptions, evalRecord) {
 	if (!state.resume || !evalRecord.persisted) return;
 	try {
-		const { default: EvalResult } = await import("./evalResult-Bgm9ZH31.js").then((n) => n.n);
+		const { default: EvalResult } = await import("./evalResult-pSvGWFMo.js").then((n) => n.n);
 		const completedPairs = await EvalResult.getCompletedIndexPairs(evalRecord.id, { excludeErrors: state.retryMode });
 		const originalCount = runEvalOptions.length;
 		for (let i = runEvalOptions.length - 1; i >= 0; i--) {
@@ -6084,9 +6833,8 @@ var Evaluator = class {
 		context.options.progressCallback?.(context.numComplete, context.runEvalOptionsLength, index, evalStep, metrics || createTimeoutMetrics(timeoutMs));
 	}
 	async executeEvalSteps({ checkAbort, ciProgressReporter, combinedAbortSignal, concurrentRunEvalOptions, evalStepIndexMap, globalTimeout, groupedRunEvalOptions, isEvalTimedOut, isWebUI, maxEvalTimeMs, processingContext, processedIndices, progressBarManager, prompts, serialRunEvalOptions, shouldGroupGradingByProvider }) {
-		let flushGroupedRows;
 		try {
-			if (shouldGroupGradingByProvider) flushGroupedRows = await this.runGroupedEvalSteps({
+			if (shouldGroupGradingByProvider) await this.runGroupedEvalSteps({
 				checkAbort,
 				evalStepIndexMap,
 				groupedRunEvalOptions,
@@ -6118,7 +6866,6 @@ var Evaluator = class {
 				cleanupProgressAfterError(progressBarManager, ciProgressReporter, err);
 				throw err;
 			}
-			await flushGroupedRows?.();
 			if (isEvalTimedOut()) logger.warn(`Evaluation stopped after reaching max duration (${maxEvalTimeMs}ms)`);
 			else if (!processingContext.targetUnavailable) return this.saveInterruptedEval({
 				ciProgressReporter,
@@ -8418,47 +9165,11 @@ function filterPrompts(prompts, filterPromptsOption) {
 //#endregion
 //#region src/commands/eval/filterProviders.ts
 /**
-* Checks if a value is a valid provider ID (non-empty string).
-*/
-function isValidProviderId(id) {
-	return id !== null && id !== void 0 && typeof id === "string" && id !== "";
-}
-/**
 * Extracts the id and label from a raw provider config without instantiating it.
 * Handles all provider config formats: string, function, ProviderOptions, ProviderOptionsMap.
 */
 function getProviderIdAndLabel(provider, index) {
-	if (typeof provider === "string") return { id: provider };
-	if (typeof provider === "function") {
-		const label = provider.label;
-		return {
-			id: label ?? `custom-function-${index}`,
-			label
-		};
-	}
-	const providerId = provider.id;
-	if ("id" in provider && isValidProviderId(providerId)) return {
-		id: providerId,
-		label: provider.label
-	};
-	const keys = Object.keys(provider);
-	if (keys.length > 0) {
-		const id = keys[0];
-		const value = provider[id];
-		if (typeof value === "object" && value !== null) return {
-			id: value.id || id,
-			label: value.label
-		};
-	}
-	const label = provider.label;
-	if (isValidProviderId(label)) return {
-		id: label,
-		label
-	};
-	return {
-		id: `unknown-${index}`,
-		label
-	};
+	return normalizeProviderRef(provider, { index });
 }
 /**
 * Filters raw provider configs BEFORE instantiation.
@@ -11163,9 +11874,10 @@ function dedupeTestCases(testCases) {
 	return deduped;
 }
 function buildMaxCharsRetryInstructions(rejectedPromptLengths, limit) {
+	const longestRejectedPromptText = rejectedPromptLengths.length > 0 ? `${Math.max(...rejectedPromptLengths)} characters` : "unknown length";
 	return dedent`
     Your previous response included ${rejectedPromptLengths.length} generated prompt${rejectedPromptLengths.length === 1 ? "" : "s"} that exceeded the ${limit ?? "configured"}-character limit.
-    The longest rejected prompt was ${Math.max(...rejectedPromptLengths)} characters.
+    The longest rejected prompt was ${longestRejectedPromptText}.
     Generate replacement prompts only, and keep every user message within the character limit.
   `.trim();
 }
@@ -13214,7 +13926,7 @@ function generateTable(evaluateTable, tableCellMaxLength = 250, maxRows = 25) {
 	for (const row of evaluateTable.body.slice(0, maxRows)) table.push([...row.vars.map((v) => ellipsize(v, tableCellMaxLength)), ...row.outputs.map(({ pass, text, failureReason: failureType }) => {
 		text = ellipsize(text, tableCellMaxLength);
 		if (pass) return chalk.green("[PASS] ") + text;
-		else return chalk.red(failureType === ResultFailureReason.ASSERT ? "[FAIL] " : "[ERROR] ") + text.split("---").map((c, idx) => idx === 0 ? chalk.red.bold(c) : c).join("---");
+		return chalk.red(failureType === ResultFailureReason.ASSERT ? "[FAIL] " : "[ERROR] ") + text.split("---").map((c, idx) => idx === 0 ? chalk.red.bold(c) : c).join("---");
 	})]);
 	return table.toString();
 }
@@ -13305,6 +14017,115 @@ function formatDuration(seconds) {
 }
 //#endregion
 //#region src/commands/eval/summary.ts
+function getCompletionMessage({ completionType, evalId, shareableUrl, wasAborted, writeToDatabase, activelySharing }) {
+	if (wasAborted) {
+		const idSuffix = writeToDatabase ? ` (ID: ${chalk.cyan(evalId)})` : "";
+		return `${chalk.red("✗")} ${completionType} aborted${idSuffix}`;
+	}
+	if (writeToDatabase && shareableUrl) return `${chalk.green("✓")} ${completionType} complete: ${shareableUrl}`;
+	if (writeToDatabase && activelySharing) return `${chalk.green("✓")} ${completionType} complete`;
+	if (writeToDatabase) return `${chalk.green("✓")} ${completionType} complete (ID: ${chalk.cyan(evalId)})`;
+	return `${chalk.green("✓")} ${completionType} complete`;
+}
+function getAbortSummaryLines(targetErrorStatus) {
+	if (targetErrorStatus == null) return [];
+	return [
+		"",
+		chalk.red.bold("Scan stopped: Target is unavailable and will not recover on retry."),
+		chalk.red(`  Target returned HTTP ${targetErrorStatus}`),
+		"",
+		chalk.yellow("Possible causes:"),
+		chalk.yellow("  • Invalid API key or authentication (401/403)"),
+		chalk.yellow("  • Target endpoint does not exist (404)"),
+		chalk.yellow("  • Server does not support the request (501)"),
+		"",
+		chalk.cyan("To fix: Check your target configuration and credentials.")
+	];
+}
+function getGuidanceLines({ writeToDatabase, shareableUrl, wantsToShare, activelySharing, hasExplicitDisable, cloudEnabled }) {
+	if (!writeToDatabase || shareableUrl || wantsToShare || activelySharing) return [];
+	const lines = ["", `» View results: ${chalk.green.bold("promptfoo view")}`];
+	if (!hasExplicitDisable) lines.push(cloudEnabled ? `» Create shareable URL: ${chalk.green.bold("promptfoo share")}` : `» Share with your team: ${chalk.green.bold("https://promptfoo.app")}`);
+	lines.push(`» Feedback: ${chalk.green.bold("https://promptfoo.dev/feedback")}`);
+	return lines;
+}
+function buildUsageDetails(usage, total) {
+	const parts = [];
+	if (usage.prompt && usage.prompt > 0) parts.push(`${usage.prompt.toLocaleString()} prompt`);
+	if (usage.completion && usage.completion > 0) parts.push(`${usage.completion.toLocaleString()} completion`);
+	if (usage.cached && usage.cached > 0) parts.push(usage.cached === total && parts.length === 0 ? "cached" : `${usage.cached.toLocaleString()} cached`);
+	if (usage.completionDetails?.reasoning && usage.completionDetails.reasoning > 0) parts.push(`${usage.completionDetails.reasoning.toLocaleString()} reasoning`);
+	return parts;
+}
+function getTokenUsageLines(tokenUsage, isRedteam, tracker) {
+	const hasEvalTokens = (tokenUsage.total || 0) > 0 || (tokenUsage.prompt || 0) + (tokenUsage.completion || 0) > 0;
+	const hasGradingTokens = tokenUsage.assertions && (tokenUsage.assertions.total || 0) > 0;
+	if (!hasEvalTokens && !hasGradingTokens) return [];
+	const combinedTotal = (tokenUsage.prompt || 0) + (tokenUsage.completion || 0);
+	const evalTokens = {
+		prompt: tokenUsage.prompt || 0,
+		completion: tokenUsage.completion || 0,
+		total: tokenUsage.total || combinedTotal,
+		cached: tokenUsage.cached || 0,
+		numRequests: tokenUsage.numRequests || 0,
+		completionDetails: tokenUsage.completionDetails || {
+			reasoning: 0,
+			acceptedPrediction: 0,
+			rejectedPrediction: 0
+		}
+	};
+	const lines = [`${chalk.bold("Total Tokens:")} ${chalk.white.bold((evalTokens.total + (tokenUsage.assertions?.total || 0)).toLocaleString())}`];
+	if (isRedteam && tokenUsage.numRequests) lines.push(`  ${chalk.gray("Probes:")} ${chalk.white(tokenUsage.numRequests.toLocaleString())}`);
+	if (evalTokens.total > 0) {
+		const evalParts = buildUsageDetails(evalTokens, evalTokens.total);
+		lines.push(`  ${chalk.gray("Eval:")} ${chalk.white(evalTokens.total.toLocaleString())} (${evalParts.join(", ")})`);
+	}
+	if (tokenUsage.assertions?.total && tokenUsage.assertions.total > 0) {
+		const gradingParts = buildUsageDetails(tokenUsage.assertions, tokenUsage.assertions.total);
+		lines.push(`  ${chalk.gray("Grading:")} ${chalk.white(tokenUsage.assertions.total.toLocaleString())} (${gradingParts.join(", ")})`);
+	}
+	lines.push(...getProviderUsageLines(tracker));
+	return lines;
+}
+function getProviderUsageLines(tracker) {
+	const providerIds = tracker.getProviderIds();
+	if (providerIds.length <= 1) return [];
+	const sortedProviders = providerIds.map((id) => ({
+		id,
+		usage: tracker.getProviderUsage(id)
+	})).filter((p) => p.usage != null).sort((a, b) => (b.usage.total || 0) - (a.usage.total || 0));
+	const lines = ["", chalk.bold("Providers:")];
+	for (const { id, usage } of sortedProviders) {
+		if ((usage.total || 0) === 0 && (usage.prompt || 0) + (usage.completion || 0) === 0) continue;
+		const displayTotal = usage.total || (usage.prompt || 0) + (usage.completion || 0);
+		const displayId = id.includes(" (") ? id.substring(0, id.indexOf(" (")) : id;
+		const details = buildUsageDetails(usage, displayTotal);
+		const requestInfo = `${usage.numRequests || 0} requests`;
+		const separator = details.length > 0 ? "; " : "";
+		lines.push(`  ${chalk.gray(`${displayId}:`)} ${chalk.white(displayTotal.toLocaleString())} (${requestInfo}${separator}${details.join(", ")})`);
+	}
+	return lines;
+}
+function formatResultPercentage(count, totalTests) {
+	const percentage = totalTests === 0 ? 0 : count / totalTests * 100;
+	return percentage === 0 || percentage === 100 ? `${percentage.toFixed(0)}%` : `${percentage.toFixed(2)}%`;
+}
+function formatResultLine(count, label, icon, iconColor, totalTests) {
+	return `  ${icon ? `${iconColor(icon)} ` : ""}${chalk.white.bold(count.toLocaleString())} ${chalk.white(label)} ${chalk.gray(`(${formatResultPercentage(count, totalTests)})`)}`;
+}
+function getResultsLines({ successes, failures, errors, duration, maxConcurrency }) {
+	const totalTests = successes + failures + errors;
+	const errorLabel = errors === 1 ? "error" : "errors";
+	return [
+		"",
+		chalk.bold("Results:"),
+		formatResultLine(successes, "passed", successes > 0 ? "✓" : void 0, chalk.green, totalTests),
+		formatResultLine(failures, "failed", failures > 0 ? "✗" : void 0, chalk.red, totalTests),
+		formatResultLine(errors, errorLabel, errors > 0 ? "✗" : void 0, chalk.red, totalTests),
+		chalk.gray(`Duration: ${formatDuration(duration)} (concurrency: ${maxConcurrency})`),
+		""
+	];
+}
 /**
 * Generate formatted evaluation summary output for CLI display.
 *
@@ -13343,115 +14164,28 @@ function formatDuration(seconds) {
 * ```
 */
 function generateEvalSummary(params) {
-	const { evalId, isRedteam, writeToDatabase, shareableUrl, wantsToShare, hasExplicitDisable, cloudEnabled, activelySharing = false, tokenUsage, successes, failures, errors, duration, maxConcurrency, tracker, targetErrorStatus } = params;
-	const lines = [];
-	const completionType = isRedteam ? "Red team" : "Eval";
-	const wasAborted = targetErrorStatus != null;
-	let completionMessage;
-	if (wasAborted) {
-		completionMessage = `${chalk.red("✗")} ${completionType} aborted`;
-		if (writeToDatabase) completionMessage += ` (ID: ${chalk.cyan(evalId)})`;
-	} else if (writeToDatabase && shareableUrl) completionMessage = `${chalk.green("✓")} ${completionType} complete: ${shareableUrl}`;
-	else if (writeToDatabase && activelySharing) completionMessage = `${chalk.green("✓")} ${completionType} complete`;
-	else if (writeToDatabase) completionMessage = `${chalk.green("✓")} ${completionType} complete (ID: ${chalk.cyan(evalId)})`;
-	else completionMessage = `${chalk.green("✓")} ${completionType} complete`;
-	lines.push(completionMessage);
-	if (wasAborted && targetErrorStatus != null) {
-		lines.push("");
-		lines.push(chalk.red.bold("Scan stopped: Target is unavailable and will not recover on retry."));
-		lines.push(chalk.red(`  Target returned HTTP ${targetErrorStatus}`));
-		lines.push("");
-		lines.push(chalk.yellow("Possible causes:"));
-		lines.push(chalk.yellow("  • Invalid API key or authentication (401/403)"));
-		lines.push(chalk.yellow("  • Target endpoint does not exist (404)"));
-		lines.push(chalk.yellow("  • Server does not support the request (501)"));
-		lines.push("");
-		lines.push(chalk.cyan("To fix: Check your target configuration and credentials."));
-	}
-	if (writeToDatabase && !shareableUrl && !wantsToShare && !activelySharing) {
-		lines.push("");
-		lines.push(`» View results: ${chalk.green.bold("promptfoo view")}`);
-		if (!hasExplicitDisable) if (cloudEnabled) lines.push(`» Create shareable URL: ${chalk.green.bold("promptfoo share")}`);
-		else lines.push(`» Share with your team: ${chalk.green.bold("https://promptfoo.app")}`);
-		lines.push(`» Feedback: ${chalk.green.bold("https://promptfoo.dev/feedback")}`);
-	}
-	lines.push("");
-	const hasEvalTokens = (tokenUsage.total || 0) > 0 || (tokenUsage.prompt || 0) + (tokenUsage.completion || 0) > 0;
-	const hasGradingTokens = tokenUsage.assertions && (tokenUsage.assertions.total || 0) > 0;
-	if (hasEvalTokens || hasGradingTokens) {
-		const combinedTotal = (tokenUsage.prompt || 0) + (tokenUsage.completion || 0);
-		const evalTokens = {
-			prompt: tokenUsage.prompt || 0,
-			completion: tokenUsage.completion || 0,
-			total: tokenUsage.total || combinedTotal,
-			cached: tokenUsage.cached || 0,
-			completionDetails: tokenUsage.completionDetails || {
-				reasoning: 0,
-				acceptedPrediction: 0,
-				rejectedPrediction: 0
-			}
-		};
-		const grandTotal = evalTokens.total + (tokenUsage.assertions?.total || 0);
-		lines.push(`${chalk.bold("Total Tokens:")} ${chalk.white.bold(grandTotal.toLocaleString())}`);
-		if (isRedteam && tokenUsage.numRequests) lines.push(`  ${chalk.gray("Probes:")} ${chalk.white(tokenUsage.numRequests.toLocaleString())}`);
-		if (evalTokens.total > 0) {
-			const evalParts = [];
-			if (evalTokens.prompt > 0) evalParts.push(`${evalTokens.prompt.toLocaleString()} prompt`);
-			if (evalTokens.completion > 0) evalParts.push(`${evalTokens.completion.toLocaleString()} completion`);
-			if (evalTokens.cached > 0) if (evalTokens.cached === evalTokens.total && evalParts.length === 0) evalParts.push("cached");
-			else evalParts.push(`${evalTokens.cached.toLocaleString()} cached`);
-			if (evalTokens.completionDetails?.reasoning && evalTokens.completionDetails.reasoning > 0) evalParts.push(`${evalTokens.completionDetails.reasoning.toLocaleString()} reasoning`);
-			lines.push(`  ${chalk.gray("Eval:")} ${chalk.white(evalTokens.total.toLocaleString())} (${evalParts.join(", ")})`);
-		}
-		if (tokenUsage.assertions && tokenUsage.assertions.total && tokenUsage.assertions.total > 0) {
-			const gradingParts = [];
-			if (tokenUsage.assertions.prompt && tokenUsage.assertions.prompt > 0) gradingParts.push(`${tokenUsage.assertions.prompt.toLocaleString()} prompt`);
-			if (tokenUsage.assertions.completion && tokenUsage.assertions.completion > 0) gradingParts.push(`${tokenUsage.assertions.completion.toLocaleString()} completion`);
-			if (tokenUsage.assertions.cached && tokenUsage.assertions.cached > 0) if (tokenUsage.assertions.cached === tokenUsage.assertions.total && gradingParts.length === 0) gradingParts.push("cached");
-			else gradingParts.push(`${tokenUsage.assertions.cached.toLocaleString()} cached`);
-			if (tokenUsage.assertions.completionDetails?.reasoning && tokenUsage.assertions.completionDetails.reasoning > 0) gradingParts.push(`${tokenUsage.assertions.completionDetails.reasoning.toLocaleString()} reasoning`);
-			lines.push(`  ${chalk.gray("Grading:")} ${chalk.white(tokenUsage.assertions.total.toLocaleString())} (${gradingParts.join(", ")})`);
-		}
-		const providerIds = tracker.getProviderIds();
-		if (providerIds.length > 1) {
-			lines.push("");
-			lines.push(chalk.bold("Providers:"));
-			const sortedProviders = providerIds.map((id) => ({
-				id,
-				usage: tracker.getProviderUsage(id)
-			})).filter((p) => p.usage != null).sort((a, b) => (b.usage.total || 0) - (a.usage.total || 0));
-			for (const { id, usage } of sortedProviders) if ((usage.total || 0) > 0 || (usage.prompt || 0) + (usage.completion || 0) > 0) {
-				const displayTotal = usage.total || (usage.prompt || 0) + (usage.completion || 0);
-				const displayId = id.includes(" (") ? id.substring(0, id.indexOf(" (")) : id;
-				const details = [];
-				if (usage.prompt && usage.prompt > 0) details.push(`${usage.prompt.toLocaleString()} prompt`);
-				if (usage.completion && usage.completion > 0) details.push(`${usage.completion.toLocaleString()} completion`);
-				if (usage.cached && usage.cached > 0) if (usage.cached === displayTotal && details.length === 0) details.push("cached");
-				else details.push(`${usage.cached.toLocaleString()} cached`);
-				if (usage.completionDetails?.reasoning && usage.completionDetails.reasoning > 0) details.push(`${usage.completionDetails.reasoning.toLocaleString()} reasoning`);
-				const breakdown = ` (${`${usage.numRequests || 0} requests`}${details.length > 0 ? "; " : ""}${details.join(", ")})`;
-				lines.push(`  ${chalk.gray(displayId + ":")} ${chalk.white(displayTotal.toLocaleString())}${breakdown}`);
-			}
-		}
-	}
-	lines.push("");
-	const totalTests = successes + failures + errors;
-	const formatResultPercentage = (count) => {
-		const percentage = totalTests === 0 ? 0 : count / totalTests * 100;
-		return percentage === 0 || percentage === 100 ? `${percentage.toFixed(0)}%` : `${percentage.toFixed(2)}%`;
-	};
-	const formatResultLine = (count, label, icon, iconColor) => {
-		return `  ${icon ? `${iconColor(icon)} ` : ""}${chalk.white.bold(count.toLocaleString())} ${chalk.white(label)} ${chalk.gray(`(${formatResultPercentage(count)})`)}`;
-	};
-	const errorLabel = errors === 1 ? "error" : "errors";
-	lines.push(chalk.bold("Results:"));
-	lines.push(formatResultLine(successes, "passed", successes > 0 ? "✓" : void 0, chalk.green));
-	lines.push(formatResultLine(failures, "failed", failures > 0 ? "✗" : void 0, chalk.red));
-	lines.push(formatResultLine(errors, errorLabel, errors > 0 ? "✗" : void 0, chalk.red));
-	const durationDisplay = formatDuration(duration);
-	lines.push(chalk.gray(`Duration: ${durationDisplay} (concurrency: ${maxConcurrency})`));
-	lines.push("");
-	return lines;
+	return [
+		getCompletionMessage({
+			completionType: params.isRedteam ? "Red team" : "Eval",
+			evalId: params.evalId,
+			shareableUrl: params.shareableUrl,
+			wasAborted: params.targetErrorStatus != null,
+			writeToDatabase: params.writeToDatabase,
+			activelySharing: params.activelySharing ?? false
+		}),
+		...getAbortSummaryLines(params.targetErrorStatus),
+		...getGuidanceLines({
+			writeToDatabase: params.writeToDatabase,
+			shareableUrl: params.shareableUrl,
+			wantsToShare: params.wantsToShare,
+			activelySharing: params.activelySharing ?? false,
+			hasExplicitDisable: params.hasExplicitDisable,
+			cloudEnabled: params.cloudEnabled
+		}),
+		"",
+		...getTokenUsageLines(params.tokenUsage, params.isRedteam, params.tracker),
+		...getResultsLines(params)
+	];
 }
 //#endregion
 //#region src/commands/retry.ts
@@ -14221,6 +14955,26 @@ async function doRedteamRun(options) {
 }
 //#endregion
 //#region src/index.ts
+/**
+* Shallow-clone a test case so the caller can swap in resolved ApiProvider
+* instances on `options.provider` / `assert[].provider` without leaking those
+* mutations back to the input. The input may alias the unified config written
+* to the Eval record, and a live SDK client (e.g. Bedrock's BedrockRuntime,
+* Anthropic's client) holds circular references that break drizzle's JSON
+* serialization on `evalRecord.save()`. Fixes #8687.
+*
+* Detaches only `options` and `assert[]`. Other reference fields (`provider`,
+* `vars`, `metadata`, `providerOutput`) remain aliased — callers must reassign
+* those by reference rather than mutating in place. `assert-set` children are
+* not deep-cloned because the resolve loop skips `assert-set`; if that ever
+* changes, extend this helper.
+*/
+function cloneTestForResolve(test) {
+	const cloned = { ...test };
+	if (test.options) cloned.options = { ...test.options };
+	if (test.assert) cloned.assert = test.assert.map((assertion) => ({ ...assertion }));
+	return cloned;
+}
 async function evaluate(testSuite, options = {}) {
 	if (testSuite.writeLatestResults) await runDbMigrations();
 	const loadedProviders = await loadApiProviders(testSuite.providers, { env: testSuite.env });
@@ -14240,22 +14994,24 @@ async function evaluate(testSuite, options = {}) {
 		nunjucksFilters: await readFilters(testSuite.nunjucksFilters || {}),
 		prompts: await processPrompts(testSuite.prompts)
 	};
-	if (typeof constructedTestSuite.defaultTest === "object") {
-		if (constructedTestSuite.defaultTest?.provider && !isApiProvider(constructedTestSuite.defaultTest.provider)) constructedTestSuite.defaultTest.provider = await resolveProvider(constructedTestSuite.defaultTest.provider, providerMap, {
+	if (typeof constructedTestSuite.defaultTest === "object" && constructedTestSuite.defaultTest) {
+		constructedTestSuite.defaultTest = cloneTestForResolve(constructedTestSuite.defaultTest);
+		if (constructedTestSuite.defaultTest.provider && !isApiProvider(constructedTestSuite.defaultTest.provider)) constructedTestSuite.defaultTest.provider = await resolveProvider(constructedTestSuite.defaultTest.provider, providerMap, {
 			env: testSuite.env,
 			basePath: state.basePath
 		});
-		if (constructedTestSuite.defaultTest?.options?.provider && !isApiProvider(constructedTestSuite.defaultTest.options.provider)) constructedTestSuite.defaultTest.options.provider = await resolveProvider(constructedTestSuite.defaultTest.options.provider, providerMap, {
+		if (constructedTestSuite.defaultTest.options?.provider && !isApiProvider(constructedTestSuite.defaultTest.options.provider)) constructedTestSuite.defaultTest.options.provider = await resolveProvider(constructedTestSuite.defaultTest.options.provider, providerMap, {
 			env: testSuite.env,
 			basePath: state.basePath
 		});
 	}
-	for (const test of constructedTestSuite.tests || []) {
+	constructedTestSuite.tests = (constructedTestSuite.tests || []).map(cloneTestForResolve);
+	for (const test of constructedTestSuite.tests) {
 		if (test.options?.provider && !isApiProvider(test.options.provider)) test.options.provider = await resolveProvider(test.options.provider, providerMap, {
 			env: testSuite.env,
 			basePath: state.basePath
 		});
-		if (test.assert) for (const assertion of test.assert) {
+		for (const assertion of test.assert || []) {
 			if (assertion.type === "assert-set" || typeof assertion.provider === "function") continue;
 			if (assertion.provider && !isApiProvider(assertion.provider)) assertion.provider = await resolveProvider(assertion.provider, providerMap, {
 				env: testSuite.env,