promptfoo 0.121.4 → 0.121.7

package/dist/src/index.js

@@ -1,33 +1,47 @@
-import { C as getEnvFloat, D as getMaxEvalTimeMs, E as getEvalTimeoutMs, O as isCI, S as getEnvBool, T as getEnvString, a as logger, b as summarizeEvaluateResultForLogging, g as getAjv, h as extractJsonObjects, k as state, n as globalLogCallback, o as setLogCallback, r as isDebugEnabled, s as setLogLevel, t as getLogLevel, v as orderKeys, w as getEnvInt, y as safeJsonStringify } from "./logger-Ct2S6Yx-.js";
+import { C as getEnvFloat, D as getMaxEvalTimeMs, E as getEvalTimeoutMs, O as isCI, S as getEnvBool, T as getEnvString, a as logger, b as summarizeEvaluateResultForLogging, g as getAjv, h as extractJsonObjects, k as state, m as extractFirstJsonObject, n as globalLogCallback, o as setLogCallback, r as isDebugEnabled, s as setLogLevel, t as getLogLevel, v as orderKeys, w as getEnvInt, y as safeJsonStringify } from "./logger-Ct2S6Yx-.js";
 import { t as invariant } from "./invariant-Ddh24eXh.js";
-import { r as importModule, t as getDirectory } from "./esm-C7PnfdF8.js";
-import { r as runPython } from "./pythonUtils-C2UQ30Rz.js";
-import { i as isJavascriptFile } from "./fileExtensions-DnqA1y9x.js";
-import { i as getProcessShim, n as transform, t as TransformInputType } from "./transform-Bbg6A8Jk.js";
-import { $ as matchesGEval, A as DivergentRepetitionPlugin, B as sampleArray, C as getPiiLeakTestsForCategory, D as HarmbenchPlugin, E as ImitationPlugin, F as AegisPlugin, G as loadRubricPrompt, H as callProviderWithContext, I as RedteamGraderBase, J as matchesClosedQa, K as matchesAnswerRelevance, L as RedteamPluginBase, M as CrossSessionLeakPlugin, N as ContractPlugin, O as HallucinationPlugin, P as BeavertailsPlugin, Q as matchesFactuality, R as getCustomPolicies, S as PlinyPlugin, T as IntentPlugin, U as fail, V as fetchHuggingFaceDataset, W as getAndCheckProvider, X as matchesContextRecall, Y as matchesContextFaithfulness, Z as matchesContextRelevance, _ as PoliticsPlugin, _t as processFileReference, a as UnverifiableClaimsPlugin, at as matchesSimilarity, b as isValidPolicyObject, c as ToolDiscoveryPlugin, ct as withProviderCallExecutionContext, d as TeenSafetyDangerousContentPlugin, dt as readPrompts, et as matchesLlmRubric, f as TeenSafetyAgeRestrictedGoodsAndServicesPlugin, ft as readProviderPromptMap, g as PromptExtractionPlugin, gt as loadFromJavaScriptFile, h as RbacPlugin, ht as getFinalTest, i as VLGuardPlugin, it as matchesSelectBest, j as DebugAccessPlugin, k as ExcessiveAgencyPlugin, l as TeenSafetyHarmfulBodyIdealsPlugin, lt as getDefaultProviders, m as ShellInjectionPlugin, mt as coerceString, n as getGraderById, nt as matchesPiScore, o as UnsafeBenchPlugin, ot as matchesTrajectoryGoalSuccess, p as SqlInjectionPlugin, pt as SUGGEST_PROMPTS_SYSTEM_MESSAGE, q as matchesClassification, r as VLSUPlugin, rt as matchesSearchRubric, s as ToxicChatPlugin, st as selectMaxScore, t as GRADERS, tt as matchesModeration, u as TeenSafetyDangerousRoleplayPlugin, ut as processPrompts, v as PolicyPlugin, vt as resolveContext, w as OverreliancePlugin, x as makeInlinePolicyIdSync, y as determinePolicyTypeFromId, z as retryWithDeduplication } from "./graders-Zy3x0zqX.js";
-import { A as isApiProvider, At as CompletionTokenDetailsSchema, C as TestGeneratorConfigSchema, Ct as UNALIGNED_PROVIDER_HARM_PLUGINS, D as VarsSchema, E as UnifiedConfigSchema, F as ConversationMessageSchema, I as PartialGenerationError, J as getDefaultNFanout, K as STRATEGY_COLLECTIONS, L as PluginConfigSchema, M as RedteamConfigSchema, O as isGradingResult, Ot as PromptSchema, P as ProvidersSchema, Q as categoryAliases, R as PolicyObjectSchema, S as TestCasesWithMetadataSchema, St as TELECOM_PLUGINS, T as TestSuiteSchema, V as isUuid, W as DEFAULT_STRATEGIES, X as isFanoutStrategy, Z as Severity, _ as ScenarioSchema, _t as PLUGIN_CATEGORIES, a as AtomicTestCaseSchema, at as DEFAULT_PLUGINS, b as TestCaseWithVarsFileSchema, c as CompletedPromptSchema, ct as HARM_PLUGINS, d as EvaluateOptionsSchema, dt as LLAMA_GUARD_REPLICATE_PROVIDER, et as riskCategorySeverityMap, f as GradingConfigSchema, ft as MEDICAL_PLUGINS, g as ResultFailureReason, gt as PII_PLUGINS, h as OutputFileExtension, ht as PHARMACY_PLUGINS, i as AssertionTypeSchema, it as DATASET_EXEMPT_PLUGINS, j as isProviderOptions, jt as InputsSchema, k as isResultFailureReason, kt as BaseTokenUsageSchema, l as DerivedMetricSchema, lt as INSURANCE_PLUGINS, m as OutputConfigSchema, mt as MULTI_INPUT_VAR, n as AssertionSchema, nt as BIAS_PLUGINS, o as BaseAssertionTypesSchema, ot as FINANCIAL_PLUGINS, p as NotPrefixedAssertionTypesSchema, pt as MULTI_INPUT_EXCLUDED_PLUGINS, q as STRATEGY_COLLECTION_MAPPINGS, r as AssertionSetSchema, rt as CANARY_BREAKING_STRATEGY_IDS, s as CommandLineOptionsSchema, st as FOUNDATION_PLUGINS, t as AssertionOrSetSchema, tt as ALIASED_PLUGIN_MAPPINGS, u as EvalResultsFilterMode, ut as LLAMA_GUARD_ENABLED_CATEGORIES, v as SpecialAssertionTypesSchema, vt as REDTEAM_PROVIDER_HARM_PLUGINS, w as TestSuiteConfigSchema, wt as CODING_AGENT_CORE_PLUGINS, x as TestCasesWithMetadataPromptSchema, xt as TEEN_SAFETY_PLUGINS, y as TestCaseSchema, yt as REMOTE_ONLY_PLUGIN_IDS, z as StrategyConfigSchema } from "./types-BGQDAP8i.js";
-import { C as checkProviderApiKeys, D as isGoogleProvider, E as isAnthropicProvider, O as isOpenAiProvider, S as resultIsForTestCase, T as getProviderDescription, _ as setupEnv, b as filterRuntimeVars, c as maybeLoadFromExternalFile, d as maybeLoadToolsFromExternalFile, g as parseFileUrl, h as loadFunction, i as fetchCsvFromGoogleSheet, k as isProviderAllowed, m as readOutput, n as writeMultipleOutputs, p as readFilters, r as writeOutput, s as maybeLoadConfigFromExternalFile, t as printBorder, v as deduplicateTestCases, w as doesProviderRefMatch, x as getTestCaseDeduplicationKey, y as extractRuntimeVars } from "./util-oGMLA7vc.js";
-import { a as getNunjucksEngine, i as extractVariablesFromTemplates, r as extractVariablesFromTemplate, t as renderEnvOnlyInObject } from "./render-CH-62LbA.js";
-import { A as TERMINAL_MAX_WIDTH, F as VERSION, I as FILE_METADATA_KEY, L as HUMAN_ASSERTION_TYPE, M as getShareApiBaseUrl, N as getShareViewBaseUrl, S as parseChatPrompt, a as CloudConfig, d as sleep, h as REQUEST_TIMEOUT_MS, j as getDefaultShareViewBaseUrl, n as fetchWithRetries, o as cloudConfig, r as fetchWithTimeout, t as fetchWithProxy, u as getCurrentTimestamp, y as isPromptfooSampleTarget } from "./fetch-Di00EQrc.js";
-import { c as isNonTransientHttpStatus, i as getCache, n as disableCache, o as withCacheNamespace, r as fetchWithCache, s as NON_TRANSIENT_HTTP_STATUSES, t as cache_exports } from "./cache-D5NZmMiT.js";
-import { $ as AIStudioChatProvider, A as createRateLimitRegistry, B as getCloudDatabaseId, C as collectFileMetadata, D as loadFromPackage, E as isPackagePath, F as getGeneratedPromptOverLimit, G as isCloudProvider, H as getOrgContext, I as getMaxCharsPerMessageModifierValue, K as resolveTeamId, L as throwIfTargetPromptExceedsMaxChars, N as PromptfooHarmfulCompletionProvider, O as redteamProviderManager, P as MAX_CHARS_PER_MESSAGE_MODIFIER_KEY, Q as VertexChatProvider, T as runExtensionHook, U as getPluginSeverityOverridesFromCloud, V as getEvalConfigFromCloud, _ as extractVariablesFromJson, a as resolveProviderConfigs, b as isBasicRefusal, c as Strategies, d as pluginMatchesStrategyTargets, f as checkExfilTracking, g as extractPromptFromTags, i as resolveProvider, j as createProviderRateLimitOptions, k as TokenUsageTracker, l as loadStrategy, m as extractGoalFromPrompt, n as loadApiProvider, o as MCPProvider, r as loadApiProviders, s as GoogleLiveProvider, t as getProviderIds, u as validateStrategies, v as getSessionId, w as renderPrompt, y as getShortPluginId, z as checkCloudPermissions } from "./providers-DruaQfwu.js";
-import { i as generateIdFromPrompt, t as hashPrompt } from "./utils-hXtCYanr.js";
-import { n as sha256, t as randomSequence } from "./createHash-4gFQpDDv.js";
-import { t as OpenAiChatCompletionProvider } from "./chat-I9izLm49.js";
-import { a as createEmptyTokenUsage, i as createEmptyAssertions, n as accumulateResponseTokenUsage, o as normalizeTokenUsage, r as accumulateTokenUsage, t as accumulateAssertionTokenUsage } from "./tokenUsageUtils-C-bmyHoE.js";
-import { h as validateFunctionCall } from "./transform-CG0ehZNG.js";
-import { l as validateFunctionCall$1 } from "./util-D9TisOyk.js";
-import { t as providerRegistry } from "./providerRegistry-Bvh8mv85.js";
-import { i as getRemoteGenerationUrl, l as shouldGenerateRemote, o as getRemoteHealthUrl, r as promptYesNo, s as neverGenerateRemote } from "./server-D6Il2Sob.js";
-import { c as setUserEmail, i as getUserEmail, o as isLoggedIntoCloud, r as getAuthor, s as promptForEmailUnverified, t as checkEmailStatusAndMaybeExit } from "./accounts-DhMYUUbu.js";
-import { t as getBlobByHash } from "./blobs-C-F78Kfn.js";
-import { a as evalsTable, c as evalsToTagsTable, d as tagsTable, i as evalResultsTable, l as promptsTable, m as getDbSignalPath, o as evalsToDatasetsTable, p as getDb, r as datasetsTable, s as evalsToPromptsTable } from "./tables-DQ4WU5tX.js";
-import { n as isBlobStorageEnabled, t as extractAndStoreBinaryData } from "./extractor-Dk6bRWkv.js";
-import { t as telemetry } from "./telemetry-Dvqxv3YC.js";
-import { t as ellipsize } from "./text-DHxdyQqT.js";
-import { t as getTraceStore } from "./store-M0b1WfYb.js";
-import { n as runRuby } from "./rubyUtils-DVLeA2jg.js";
-import { t as EvalResult } from "./evalResult-Bgm9ZH31.js";
+import { $ as riskCategorySeverityMap, A as RedteamConfigSchema, At as DocumentMediaInjectionPlacementValues, B as isUuid, Bt as getInputDescription, C as TestGeneratorConfigSchema, Ct as CODING_AGENT_CORE_PLUGINS, D as VarsSchema, Dt as BaseTokenUsageSchema, E as UnifiedConfigSchema, F as PartialGenerationError, Ft as InputDefinitionSchema, G as STRATEGY_COLLECTIONS, Gt as isProviderOptions, Ht as normalizeInputDefinition, I as PluginConfigSchema, It as InputTypeSchema, K as STRATEGY_COLLECTION_MAPPINGS, L as PolicyObjectSchema, Lt as InputTypeValues, Mt as DocxInjectionPlacementValues, N as ProvidersSchema, Nt as InputConfigSchema, O as isGradingResult, Ot as CompletionTokenDetailsSchema, P as ConversationMessageSchema, Pt as InputDefinitionObjectSchema, R as StrategyConfigSchema, Rt as InputsSchema, S as TestCasesWithMetadataSchema, St as UNALIGNED_PROVIDER_HARM_PLUGINS, T as TestSuiteSchema, U as DEFAULT_STRATEGIES, Ut as normalizeInputs, Vt as getInputType, Wt as isApiProvider, X as Severity, Y as isFanoutStrategy, Z as categoryAliases, _ as ScenarioSchema, _t as REDTEAM_PROVIDER_HARM_PLUGINS, a as AtomicTestCaseSchema, at as FINANCIAL_PLUGINS, b as TestCaseWithVarsFileSchema, bt as TEEN_SAFETY_PLUGINS, c as CompletedPromptSchema, ct as INSURANCE_PLUGINS, d as EvaluateOptionsSchema, dt as MEDICAL_PLUGINS, et as ALIASED_PLUGIN_MAPPINGS, f as GradingConfigSchema, ft as MULTI_INPUT_EXCLUDED_PLUGINS, g as ResultFailureReason, gt as PLUGIN_CATEGORIES, h as OutputFileExtension, ht as PII_PLUGINS, i as AssertionTypeSchema, it as DEFAULT_PLUGINS, j as PromptSchema, jt as DocxInjectionPlacementSchema, k as isResultFailureReason, kt as DocumentMediaInjectionPlacementSchema, l as DerivedMetricSchema, lt as LLAMA_GUARD_ENABLED_CATEGORIES, m as OutputConfigSchema, mt as PHARMACY_PLUGINS, n as AssertionSchema, nt as CANARY_BREAKING_STRATEGY_IDS, o as BaseAssertionTypesSchema, ot as FOUNDATION_PLUGINS, p as NotPrefixedAssertionTypesSchema, pt as MULTI_INPUT_VAR, q as getDefaultNFanout, r as AssertionSetSchema, rt as DATASET_EXEMPT_PLUGINS, s as CommandLineOptionsSchema, st as HARM_PLUGINS, t as AssertionOrSetSchema, tt as BIAS_PLUGINS, u as EvalResultsFilterMode, ut as LLAMA_GUARD_REPLICATE_PROVIDER, v as SpecialAssertionTypesSchema, vt as REMOTE_ONLY_PLUGIN_IDS, w as TestSuiteConfigSchema, wt as CODING_AGENT_PLUGINS, x as TestCasesWithMetadataPromptSchema, xt as TELECOM_PLUGINS, y as TestCaseSchema, zt as buildInputPromptDescription } from "./types-D6glLbdF.js";
+import { F as getShareApiBaseUrl, I as getShareViewBaseUrl, L as FILE_METADATA_KEY, N as TERMINAL_MAX_WIDTH, P as getDefaultShareViewBaseUrl, R as HUMAN_ASSERTION_TYPE, T as cloudConfig, _ as isPromptfooSampleTarget, b as parseChatPrompt, c as getCurrentTimestamp, l as sleep, n as fetchWithRetries, p as REQUEST_TIMEOUT_MS, r as fetchWithTimeout, t as fetchWithProxy, w as CloudConfig } from "./fetch-It34O8Ur.js";
+import { n as VERSION } from "./version-lpHV_53E.js";
+import { i as isJavascriptFile } from "./fileExtensions-CXRfY3Ss.js";
+import { c as setUserEmail, i as getUserEmail, o as isLoggedIntoCloud, r as getAuthor, s as promptForEmailUnverified, t as checkEmailStatusAndMaybeExit } from "./accounts-Ca7WIoPY.js";
+import { r as importModule, t as getDirectory } from "./esm-BTK1W7lG.js";
+import { a as extractVariablesFromTemplates, i as extractVariablesFromTemplate, o as getNunjucksEngine, r as analyzeTemplateReference, t as renderEnvOnlyInObject } from "./render-DFfDeYUK.js";
+import { t as providerRegistry } from "./providerRegistry-DHcFiVWX.js";
+import { a as getRemoteHealthUrl, l as shouldGenerateRemote, n as getRemoteGenerationExplicitlyDisabledError, r as getRemoteGenerationUrl, s as neverGenerateRemote } from "./remoteGeneration-DsaSwmG2.js";
+import { r as promptYesNo } from "./server-CMJD10J4.js";
+import { a as getCloudDatabaseId, c as getPluginSeverityOverridesFromCloud, d as isCloudProvider, i as checkCloudPermissions, o as getEvalConfigFromCloud, p as resolveTeamId, s as getOrgContext } from "./storage-CD-GWAdx.js";
+import { r as runPython } from "./pythonUtils-D0BYebvX.js";
+import { A as readFilters, M as loadFunction, N as parseFileUrl, O as maybeLoadToolsFromExternalFile, T as maybeLoadFromExternalFile, _ as isProviderAllowed, a as setupEnv, b as normalizeProviderRef, c as filterRuntimeVars, d as checkProviderApiKeys, f as doesProviderRefMatch, g as isOpenAiProvider, h as isGoogleProvider, i as fetchCsvFromGoogleSheet, j as readOutput, l as getTestCaseDeduplicationKey, m as isAnthropicProvider, n as writeMultipleOutputs, o as deduplicateTestCases, p as getProviderDescription, r as writeOutput, s as extractRuntimeVars, t as printBorder, u as resultIsForTestCase, w as maybeLoadConfigFromExternalFile } from "./util-Bx677_k2.js";
+import { n as sha256, t as randomSequence } from "./createHash-BtbSX3mj.js";
+import { c as NON_TRANSIENT_HTTP_STATUSES, i as getCache, l as isNonTransientHttpStatus, n as disableCache, r as fetchWithCache, s as withCacheNamespace, t as cache_exports } from "./cache-DIXbtkNO.js";
+import { t as OpenAiChatCompletionProvider } from "./chat-Dabu84Br.js";
+import { h as validateFunctionCall } from "./transform-DmvYBRll.js";
+import { l as validateFunctionCall$1 } from "./util-LKTmNsMQ.js";
+import { _ as AIStudioChatProvider, a as resolveProvider, f as MCPProvider, g as GoogleLiveProvider, h as VertexChatProvider, n as loadApiProvider, o as resolveProviderConfigs, r as loadApiProviders, t as getProviderIds } from "./providers-DVYRZP4E.js";
+import { a as createEmptyTokenUsage, i as createEmptyAssertions, n as accumulateResponseTokenUsage, o as normalizeTokenUsage, r as accumulateTokenUsage, t as accumulateAssertionTokenUsage } from "./tokenUsageUtils-CmnQ0G2m.js";
+import { t as ellipsize } from "./text-DwYK5EBn.js";
+import { t as telemetry } from "./telemetry-DwX9XUN5.js";
+import { a as evalsTable, c as evalsToTagsTable, d as tagsTable, i as evalResultsTable, l as promptsTable, m as getDbSignalPath, o as evalsToDatasetsTable, p as getDb, r as datasetsTable, s as evalsToPromptsTable } from "./tables-CRSXQ2Ke.js";
+import { t as getBlobByHash } from "./blobs-Dwef1Ao1.js";
+import { t as getProcessShim } from "./processShim-BcGzU8fY.js";
+import { n as loadFromPackage, t as isPackagePath } from "./packageParser-CgE-ziRo.js";
+import { n as runRuby } from "./rubyUtils-CnlW8AYb.js";
+import { n as materializeInputVariablesWithMetadata, t as buildPromptInputDescriptions } from "./inputVariables-DUGMb9Ka.js";
+import { a as extractPromptFromTags, c as isBasicRefusal, i as extractMaterializedVariablesFromJsonWithMetadata, n as extractGoalFromPrompt, o as getSessionId, r as extractInputVarsFromPrompt, s as getShortPluginId } from "./util-DNl96nNs.js";
+import { n as PromptfooHarmfulCompletionProvider } from "./promptfoo-Cm4hiy1Y.js";
+import { $ as readProviderPromptMap, A as ExcessiveAgencyPlugin, At as withProviderCallExecutionContext, B as retryWithDeduplication, C as PlinyPlugin, Ct as processFileReference, D as ImitationPlugin, Dt as getAndCheckProvider, E as IntentPlugin, Et as callProviderWithContext, F as BeavertailsPlugin, G as matchesFactuality, H as fetchHuggingFaceDataset, I as AegisPlugin, J as matchesPiScore, K as matchesGEval, L as RedteamGraderBase, M as DebugAccessPlugin, N as CrossSessionLeakPlugin, O as HarmbenchPlugin, Ot as getGradingProvider, P as ContractPlugin, Q as readPrompts, R as RedteamPluginBase, S as makeInlinePolicyIdSync, St as loadFromJavaScriptFile, T as OverreliancePlugin, Tt as DEFAULT_ANTHROPIC_MODEL, U as isGraderFailure, V as sampleArray, W as matchesClosedQa, X as doRemoteGrading, Y as matchesTrajectoryGoalSuccess, Z as processPrompts, _ as PromptExtractionPlugin, _t as normalizeMatcherTokenUsage, a as VLGuardPlugin, at as CONTEXT_FAITHFULNESS_NLI_STATEMENTS, b as determinePolicyTypeFromId, bt as coerceString, c as ToxicChatPlugin, ct as CONTEXT_RECALL_NOT_ATTRIBUTED_TOKEN, d as TeenSafetyDangerousRoleplayPlugin, dt as loadRubricPrompt, et as DEFAULT_WEB_SEARCH_PROMPT, f as TeenSafetyDangerousContentPlugin, ft as renderLlmRubricPrompt, g as RbacPlugin, gt as fail, h as ShellInjectionPlugin, ht as euclideanDistance, i as VLSUPlugin, it as CONTEXT_FAITHFULNESS_LONGFORM, j as DivergentRepetitionPlugin, k as HallucinationPlugin, kt as getProviderCallExecutionContext, l as ToolDiscoveryPlugin, lt as CONTEXT_RELEVANCE, m as SqlInjectionPlugin, mt as dotProduct, n as getGraderById, nt as SUGGEST_PROMPTS_SYSTEM_MESSAGE, o as UnverifiableClaimsPlugin, ot as CONTEXT_RECALL, p as TeenSafetyAgeRestrictedGoodsAndServicesPlugin, pt as cosineSimilarity, q as matchesLlmRubric, rt as ANSWER_RELEVANCY_GENERATE, s as UnsafeBenchPlugin, st as CONTEXT_RECALL_ATTRIBUTED_TOKEN, t as GRADERS, tt as SELECT_BEST_PROMPT, u as TeenSafetyHarmfulBodyIdealsPlugin, ut as CONTEXT_RELEVANCE_BAD, v as PoliticsPlugin, vt as splitIntoSentences, w as getPiiLeakTestsForCategory, wt as getDefaultProviders, x as isValidPolicyObject, xt as getFinalTest, y as PolicyPlugin, yt as tryParse, z as getCustomPolicies } from "./graders-BX0f2tvS.js";
+import { f as redteamProviderManager, g as createProviderRateLimitOptions, h as createRateLimitRegistry, m as TokenUsageTracker } from "./shared-D6IjElRI.js";
+import { i as generateIdFromPrompt, t as hashPrompt } from "./utils-BGY69tk_.js";
+import { a as getTransformLabel, i as getTransformErrorMessage, n as TRANSFORM_KEYS, o as transform, r as TransformInputType, t as INLINE_FUNCTION_LABEL } from "./transform-CGt7Kt3y2.js";
+import { t as getTraceStore } from "./store-HpopRVzl.js";
+import { n as isBlobStorageEnabled, t as extractAndStoreBinaryData } from "./extractor-CxRtnaHl.js";
+import { i as throwIfTargetPromptExceedsMaxChars, n as getGeneratedPromptOverLimit, r as getMaxCharsPerMessageModifierValue, t as MAX_CHARS_PER_MESSAGE_MODIFIER_KEY } from "./promptLength-B9nZEfO6.js";
+import { n as checkExfilTracking } from "./indirectWebPwn-CbjUG0rh.js";
+import { n as getFirstStringAttribute, r as getToolNameFromAttributes, t as TOOL_ARGUMENT_ATTRIBUTE_KEYS } from "./toolAttributes-DJ9ZEKXD.js";
+import { i as filterFiniteScores, n as renderPrompt, r as runExtensionHook, t as collectFileMetadata } from "./evaluatorHelpers-DuqFFfq7.js";
+import { r as sanitizeProvider, t as EvalResult } from "./evalResult-2RRJvFyB.js";
+import { i as pluginMatchesStrategyTargets, n as loadStrategy, r as validateStrategies, t as Strategies } from "./strategies-CiSeroPH.js";
 import * as fs$2 from "fs";
 import fs, { createWriteStream } from "fs";
 import * as path$2 from "path";
@@ -35,29 +49,30 @@ import path, { parse } from "path";
 import async from "async";
 import yaml from "js-yaml";
 import { AsyncResource } from "node:async_hooks";
-import { resolve } from "node:path";
-import { fileURLToPath } from "node:url";
 import chalk from "chalk";
 import * as os$1 from "os";
 import os from "os";
-import util from "util";
 import dedent from "dedent";
-import * as fsPromises from "fs/promises";
-import { globSync } from "glob";
 import { z } from "zod";
-import { parse as parse$1 } from "csv-parse/sync";
-import { XMLParser } from "fast-xml-parser";
+import * as fsPromises from "fs/promises";
+import util from "util";
+import input from "@inquirer/input";
+import { resolve } from "node:path";
+import { fileURLToPath } from "node:url";
 import crypto$1, { createHash, randomBytes } from "crypto";
 import { DiagConsoleLogger, DiagLogLevel, diag, propagation } from "@opentelemetry/api";
-import input from "@inquirer/input";
 import readline from "readline";
+import { parse as parse$1 } from "csv-parse/sync";
+import { globSync } from "glob";
+import { XMLParser } from "fast-xml-parser";
 import { and, desc, eq, inArray, sql } from "drizzle-orm";
-import cliProgress from "cli-progress";
 import { URL } from "url";
-import { JSDOM } from "jsdom";
+import { parse as parse$2 } from "parse5";
 import { distance } from "fastest-levenshtein";
+import cliProgress from "cli-progress";
 import * as rouge from "js-rouge";
 import { isDeepStrictEqual } from "node:util";
+import { LRUCache } from "lru-cache";
 import "debounce";
 import { ExportResultCode, W3CTraceContextPropagator } from "@opentelemetry/core";
 import { OTLPTraceExporter } from "@opentelemetry/exporter-trace-otlp-http";
@@ -242,6 +257,505 @@ const handleConversationRelevance = async ({ assertion, outputString, prompt, pr
 	};
 };
 //#endregion
+//#region src/matchers/classification.ts
+/**
+*
+* @param expected Expected classification. If undefined, matches any classification.
+* @param output Text to classify.
+* @param threshold Value between 0 and 1. If the expected classification is undefined, the threshold is the minimum score for any classification. If the expected classification is defined, the threshold is the minimum score for that classification.
+* @param grading
+* @returns Pass if the output matches the classification with a score greater than or equal to the threshold.
+*/
+async function matchesClassification(expected, output, threshold, grading) {
+	const resp = await (await getAndCheckProvider("classification", grading?.provider, null, "classification check")).callClassificationApi(output);
+	if (!resp.classification) return fail(resp.error || "Unknown error fetching classification");
+	let score;
+	if (expected === void 0) {
+		const scores = Object.values(resp.classification);
+		if (scores.length === 0) return {
+			pass: false,
+			score: 0,
+			reason: "No classification scores returned"
+		};
+		score = Math.max(...scores);
+	} else score = resp.classification[expected] || 0;
+	if (score >= threshold - Number.EPSILON) {
+		const reason = expected === void 0 ? `Maximum classification score ${score.toFixed(2)} >= ${threshold}` : `Classification ${expected} has score ${score.toFixed(2)} >= ${threshold}`;
+		return {
+			pass: true,
+			score,
+			reason
+		};
+	}
+	return {
+		pass: false,
+		score,
+		reason: expected === void 0 ? `Maximum classification score ${score.toFixed(2)} < ${threshold}` : `Classification ${expected} has score ${score.toFixed(2)} < ${threshold}`
+	};
+}
+//#endregion
+//#region src/matchers/comparison.ts
+async function matchesSelectBest(criteria, outputs, grading, vars, providerCallContext) {
+	invariant(outputs.length >= 2, "select-best assertion must have at least two outputs to compare between");
+	const resp = await callProviderWithContext(await getAndCheckProvider("text", grading?.provider, (await getDefaultProviders()).gradingProvider, "select-best check"), await renderLlmRubricPrompt(await loadRubricPrompt(grading?.rubricPrompt, SELECT_BEST_PROMPT), {
+		criteria,
+		outputs: outputs.map((o) => tryParse(o)),
+		...vars || {}
+	}), "select-best", {
+		criteria,
+		outputs: outputs.map((o) => tryParse(o)),
+		...vars || {}
+	}, providerCallContext);
+	if (resp.error || !resp.output) return Array.from({ length: outputs.length }, () => fail(resp.error || "No output", resp.tokenUsage));
+	invariant(typeof resp.output === "string", "select-best produced malformed response");
+	const firstIntegerMatch = resp.output.trim().match(/\d+/);
+	const verdict = firstIntegerMatch ? Number.parseInt(firstIntegerMatch[0], 10) : NaN;
+	if (Number.isNaN(verdict) || verdict < 0 || verdict >= outputs.length) return Array.from({ length: outputs.length }, () => fail(`Invalid select-best verdict: ${verdict}`, resp.tokenUsage));
+	const tokensUsed = normalizeMatcherTokenUsage(resp.tokenUsage);
+	return outputs.map((_output, index) => {
+		if (index === verdict) return {
+			pass: true,
+			score: 1,
+			reason: `Output selected as the best: ${criteria}`,
+			tokensUsed
+		};
+		else return {
+			pass: false,
+			score: 0,
+			reason: `Output not selected: ${criteria}`,
+			tokensUsed
+		};
+	});
+}
+async function selectMaxScore(outputs, resultsWithGradingResults, assertion) {
+	invariant(outputs.length >= 2, "max-score assertion must have at least two outputs to compare between");
+	const value = assertion.value || {};
+	const options = {
+		method: typeof value === "object" && "method" in value ? value.method : "average",
+		weights: typeof value === "object" && "weights" in value ? value.weights : {},
+		threshold: typeof value === "object" && "threshold" in value ? value.threshold : void 0
+	};
+	const scores = resultsWithGradingResults.map((result, index) => {
+		const relevantResults = (result.gradingResult?.componentResults || []).filter((r) => r.assertion && r.assertion.type !== "max-score" && r.assertion.type !== "select-best");
+		if (relevantResults.length === 0) throw new Error("max-score requires at least one other assertion (besides max-score or select-best) to aggregate scores from");
+		let totalWeightedScore = 0;
+		let totalWeight = 0;
+		relevantResults.forEach((componentResult) => {
+			const assertionType = componentResult.assertion?.type || "unknown";
+			const weight = options.weights[assertionType] === void 0 ? 1 : options.weights[assertionType];
+			const score = componentResult.score || 0;
+			totalWeightedScore += score * weight;
+			totalWeight += weight;
+		});
+		let aggregateScore;
+		if (options.method === "sum") aggregateScore = totalWeightedScore;
+		else aggregateScore = totalWeight > 0 ? totalWeightedScore / totalWeight : 0;
+		return {
+			index,
+			score: aggregateScore,
+			componentCount: relevantResults.length,
+			totalWeight
+		};
+	});
+	let maxScore = -Infinity;
+	let winnerIndex = 0;
+	for (let i = 0; i < scores.length; i++) if (scores[i].score > maxScore) {
+		maxScore = scores[i].score;
+		winnerIndex = i;
+	}
+	const meetsThreshold = options.threshold === void 0 || maxScore >= options.threshold;
+	return scores.map(({ index, score, componentCount, totalWeight }) => {
+		const isWinner = index === winnerIndex && meetsThreshold;
+		return {
+			pass: isWinner,
+			score: isWinner ? 1 : 0,
+			reason: isWinner ? `Selected as highest scoring output (score: ${score.toFixed(3)})` : score === maxScore && !meetsThreshold ? `Not selected - score ${score.toFixed(3)} below threshold ${options.threshold}` : `Not selected (score: ${score.toFixed(3)}, max: ${maxScore.toFixed(3)})`,
+			namedScores: {
+				maxScore: score,
+				assertionCount: componentCount,
+				totalWeight
+			}
+		};
+	});
+}
+//#endregion
+//#region src/matchers/moderation.ts
+async function matchesModeration({ userPrompt, assistantResponse, categories = [] }, grading) {
+	if (!assistantResponse) return {
+		pass: true,
+		score: 1,
+		reason: "No output to moderate"
+	};
+	const defaultProviders = await getDefaultProviders();
+	const defaultModerationProvider = !getEnvString("OPENAI_API_KEY") && (getEnvString("REPLICATE_API_KEY") || getEnvString("REPLICATE_API_TOKEN")) ? await loadApiProvider(LLAMA_GUARD_REPLICATE_PROVIDER) : defaultProviders.moderationProvider;
+	const moderationProvider = await getAndCheckProvider("moderation", grading?.provider, defaultModerationProvider, "moderation check");
+	invariant(moderationProvider, "Moderation provider must be defined");
+	const resp = await moderationProvider.callModerationApi(userPrompt, assistantResponse);
+	if (resp.error) return {
+		pass: false,
+		score: 0,
+		reason: `Moderation API error: ${resp.error}`
+	};
+	const { flags } = resp;
+	if (!flags || flags.length === 0) return {
+		pass: true,
+		score: 1,
+		reason: "No moderation flags detected"
+	};
+	const filteredFlags = categories.length === 0 ? flags : flags.filter((flag) => categories.includes(flag.code));
+	if (filteredFlags.length > 0) return {
+		pass: false,
+		score: 0,
+		reason: `Moderation flags detected: ${filteredFlags.map((flag) => flag.description).join(", ")}`
+	};
+	return {
+		pass: true,
+		score: 1,
+		reason: "No relevant moderation flags detected"
+	};
+}
+//#endregion
+//#region src/assertions/contextUtils.ts
+/**
+* Resolves the context value for context-based assertions.
+* Supports extracting context from test variables or transforming from output.
+* Can return either a single context string or an array of context chunks.
+*
+* @param assertion - The assertion configuration
+* @param test - The test case
+* @param output - The provider output (after provider transform, before test transform)
+* @param prompt - The prompt text
+* @param fallbackContext - Optional fallback context (e.g., prompt for context-recall)
+* @param providerResponse - Optional full provider response for contextTransform
+* @returns The resolved context string or array of strings
+* @throws Error if context cannot be resolved or transform fails
+*/
+async function resolveContext(assertion, test, output, prompt, fallbackContext, providerResponse) {
+	let contextValue;
+	if (test.vars?.context) {
+		if (typeof test.vars.context === "string") contextValue = test.vars.context;
+		else if (Array.isArray(test.vars.context)) {
+			const invalidEntry = [...test.vars.context.entries()].find(([, v]) => typeof v !== "string");
+			if (invalidEntry) {
+				const [idx, val] = invalidEntry;
+				invariant(false, `Invalid context: expected an array of strings, but found ${typeof val} at index ${idx}`);
+			}
+			contextValue = test.vars.context;
+		}
+	} else if (fallbackContext) contextValue = fallbackContext;
+	if (assertion.contextTransform) {
+		const getLabel = () => getTransformLabel(assertion.contextTransform);
+		try {
+			const outputForTransform = providerResponse?.providerTransformedOutput ?? output;
+			const transformed = await transform(assertion.contextTransform, outputForTransform, {
+				vars: test.vars,
+				prompt: { label: prompt },
+				...providerResponse && providerResponse.metadata && { metadata: providerResponse.metadata }
+			});
+			invariant(typeof transformed === "string" || Array.isArray(transformed) && transformed.every((item) => typeof item === "string"), () => `contextTransform must return a string or array of strings. Got ${typeof transformed}. Check your transform expression: ${getLabel()}`);
+			contextValue = transformed;
+		} catch (error) {
+			throw new Error(`Failed to transform context using expression '${getLabel()}': ${getTransformErrorMessage(error)}`);
+		}
+	}
+	invariant(typeof contextValue === "string" && contextValue.length > 0 || Array.isArray(contextValue) && contextValue.length > 0 && contextValue.every((item) => typeof item === "string" && item.length > 0), "Context is required for context-based assertions. Provide either a \"context\" variable (string or array of strings) in your test case or use \"contextTransform\" to extract context from the provider response.");
+	return contextValue;
+}
+/**
+* Serializes context (string or string[]) to a single string for prompts.
+* Joins chunks with double newlines to preserve separation.
+*/
+function serializeContext(context) {
+	return Array.isArray(context) ? context.join("\n\n") : context;
+}
+//#endregion
+//#region src/matchers/rag.ts
+async function matchesAnswerRelevance(input, output, threshold, grading, providerCallContext) {
+	const defaults = await getDefaultProviders();
+	const embeddingProvider = await getAndCheckProvider("embedding", grading?.provider, defaults.embeddingProvider, "answer relevancy check");
+	const textProvider = await getAndCheckProvider("text", grading?.provider, defaults.gradingProvider, "answer relevancy check");
+	const tokensUsed = normalizeMatcherTokenUsage(void 0);
+	const rubricPrompt = await loadRubricPrompt(grading?.rubricPrompt, ANSWER_RELEVANCY_GENERATE);
+	const parsedOutput = tryParse(output);
+	const promptText = await renderLlmRubricPrompt(rubricPrompt, { answer: parsedOutput });
+	const candidateQuestions = [];
+	for (let i = 0; i < 3; i++) {
+		const resp = await callProviderWithContext(textProvider, promptText, "answer-relevance", { answer: parsedOutput }, providerCallContext);
+		accumulateTokenUsage(tokensUsed, resp.tokenUsage);
+		if (resp.error || !resp.output) return fail(resp.error || "No output", tokensUsed);
+		invariant(typeof resp.output === "string", "answer relevancy check produced malformed response");
+		candidateQuestions.push(resp.output);
+	}
+	invariant(typeof embeddingProvider.callEmbeddingApi === "function", `Provider ${embeddingProvider.id()} must implement callEmbeddingApi for similarity check`);
+	const inputEmbeddingResp = await embeddingProvider.callEmbeddingApi(input);
+	accumulateTokenUsage(tokensUsed, inputEmbeddingResp.tokenUsage);
+	if (inputEmbeddingResp.error || !inputEmbeddingResp.embedding) return fail(inputEmbeddingResp.error || "No embedding", tokensUsed);
+	const inputEmbedding = inputEmbeddingResp.embedding;
+	const similarities = [];
+	const questionsWithScores = [];
+	for (const question of candidateQuestions) {
+		const resp = await embeddingProvider.callEmbeddingApi(question);
+		accumulateTokenUsage(tokensUsed, resp.tokenUsage);
+		if (resp.error || !resp.embedding) return fail(resp.error || "No embedding", tokensUsed);
+		const questionSimilarity = cosineSimilarity(inputEmbedding, resp.embedding);
+		similarities.push(questionSimilarity);
+		questionsWithScores.push({
+			question,
+			similarity: questionSimilarity
+		});
+	}
+	const similarity = similarities.reduce((a, b) => a + b, 0) / similarities.length;
+	const pass = similarity >= threshold - Number.EPSILON;
+	const greaterThanReason = `Relevance ${similarity.toFixed(2)} is greater than threshold ${threshold}`;
+	const lessThanReason = `Relevance ${similarity.toFixed(2)} is less than threshold ${threshold}`;
+	const metadata = {
+		generatedQuestions: questionsWithScores,
+		averageSimilarity: similarity,
+		threshold
+	};
+	if (pass) return {
+		pass: true,
+		score: similarity,
+		reason: greaterThanReason,
+		tokensUsed,
+		metadata
+	};
+	return {
+		pass: false,
+		score: similarity,
+		reason: lessThanReason,
+		tokensUsed,
+		metadata
+	};
+}
+async function matchesContextRecall(context, groundTruth, threshold, grading, vars, providerCallContext) {
+	const textProvider = await getAndCheckProvider("text", grading?.provider, (await getDefaultProviders()).gradingProvider, "context recall check");
+	const contextString = serializeContext(context);
+	const resp = await callProviderWithContext(textProvider, await renderLlmRubricPrompt(await loadRubricPrompt(grading?.rubricPrompt, CONTEXT_RECALL), {
+		context: contextString,
+		groundTruth,
+		...vars || {}
+	}), "context-recall", {
+		context: contextString,
+		groundTruth,
+		...vars || {}
+	}, providerCallContext);
+	if (resp.error || !resp.output) return fail(resp.error || "No output", resp.tokenUsage);
+	invariant(typeof resp.output === "string", "context-recall produced malformed response");
+	const attributedTokenLower = CONTEXT_RECALL_ATTRIBUTED_TOKEN.toLowerCase();
+	const notAttributedTokenLower = CONTEXT_RECALL_NOT_ATTRIBUTED_TOKEN.toLowerCase();
+	const sentences = splitIntoSentences(resp.output).filter((line) => {
+		const lowerLine = line.toLowerCase();
+		return lowerLine.includes(attributedTokenLower) || lowerLine.includes(notAttributedTokenLower);
+	});
+	const sentenceAttributions = [];
+	let numerator = 0;
+	for (const sentence of sentences) {
+		const lowerSentence = sentence.toLowerCase();
+		const isAttributed = !lowerSentence.includes(notAttributedTokenLower) && lowerSentence.includes(attributedTokenLower);
+		if (isAttributed) numerator++;
+		const sentenceMatch = sentence.match(/^\d+\.\s*([^\.]+\.)/);
+		const cleanSentence = sentenceMatch ? sentenceMatch[1].trim() : sentence.split(".")[0].trim();
+		sentenceAttributions.push({
+			sentence: cleanSentence,
+			attributed: isAttributed
+		});
+	}
+	const score = sentences.length > 0 ? numerator / sentences.length : 0;
+	const pass = score >= threshold - Number.EPSILON;
+	const metadata = {
+		sentenceAttributions,
+		totalSentences: sentences.length,
+		attributedSentences: numerator,
+		score
+	};
+	return {
+		pass,
+		score,
+		reason: pass ? `Recall ${score.toFixed(2)} is >= ${threshold}` : `Recall ${score.toFixed(2)} is < ${threshold}`,
+		tokensUsed: normalizeMatcherTokenUsage(resp.tokenUsage),
+		metadata
+	};
+}
+async function matchesContextRelevance(question, context, threshold, grading, providerCallContext) {
+	const textProvider = await getAndCheckProvider("text", grading?.provider, (await getDefaultProviders()).gradingProvider, "context relevance check");
+	const contextString = serializeContext(context);
+	const resp = await callProviderWithContext(textProvider, await renderLlmRubricPrompt(await loadRubricPrompt(grading?.rubricPrompt, CONTEXT_RELEVANCE), {
+		context: contextString,
+		query: question
+	}), "context-relevance", {
+		context: contextString,
+		query: question
+	}, providerCallContext);
+	if (resp.error || !resp.output) return fail(resp.error || "No output", resp.tokenUsage);
+	invariant(typeof resp.output === "string", "context-relevance produced malformed response");
+	const contextUnits = Array.isArray(context) ? context.filter((chunk) => chunk.trim().length > 0) : splitIntoSentences(context);
+	const totalContextUnits = contextUnits.length;
+	const extractedSentences = splitIntoSentences(resp.output);
+	const relevantSentences = [];
+	const insufficientInformation = resp.output.includes(CONTEXT_RELEVANCE_BAD);
+	let numerator = 0;
+	if (insufficientInformation) numerator = 0;
+	else {
+		const uniqueRelevantSentences = [...new Set(extractedSentences)];
+		numerator = Math.min(uniqueRelevantSentences.length, totalContextUnits);
+		relevantSentences.push(...uniqueRelevantSentences);
+	}
+	const score = totalContextUnits > 0 ? numerator / totalContextUnits : 0;
+	const pass = score >= threshold - Number.EPSILON;
+	const metadata = {
+		extractedSentences: relevantSentences,
+		totalContextUnits,
+		totalContextSentences: totalContextUnits,
+		contextUnits,
+		relevantSentenceCount: numerator,
+		insufficientInformation,
+		score
+	};
+	return {
+		pass,
+		score,
+		reason: pass ? `Context relevance ${score.toFixed(2)} is >= ${threshold}` : `Context relevance ${score.toFixed(2)} is < ${threshold}`,
+		tokensUsed: normalizeMatcherTokenUsage(resp.tokenUsage),
+		metadata
+	};
+}
+async function matchesContextFaithfulness(query, output, context, threshold, grading, vars, providerCallContext) {
+	const textProvider = await getAndCheckProvider("text", grading?.provider, (await getDefaultProviders()).gradingProvider, "faithfulness check");
+	const tokensUsed = normalizeMatcherTokenUsage(void 0);
+	if (grading?.rubricPrompt) invariant(Array.isArray(grading.rubricPrompt), "rubricPrompt must be an array");
+	const rawLongformPrompt = typeof grading?.rubricPrompt?.[0] === "string" ? grading?.rubricPrompt?.[0] : grading?.rubricPrompt?.[0]?.content;
+	const rawNliPrompt = typeof grading?.rubricPrompt?.[1] === "string" ? grading?.rubricPrompt?.[1] : grading?.rubricPrompt?.[1]?.content;
+	const longformPrompt = await loadRubricPrompt(rawLongformPrompt, CONTEXT_FAITHFULNESS_LONGFORM);
+	const nliPrompt = await loadRubricPrompt(rawNliPrompt, CONTEXT_FAITHFULNESS_NLI_STATEMENTS);
+	let promptText = await renderLlmRubricPrompt(longformPrompt, {
+		question: query,
+		answer: tryParse(output),
+		...vars || {}
+	});
+	let resp = await callProviderWithContext(textProvider, promptText, "context-faithfulness-longform", {
+		question: query,
+		answer: tryParse(output),
+		...vars || {}
+	}, providerCallContext);
+	accumulateTokenUsage(tokensUsed, resp.tokenUsage);
+	if (resp.error || !resp.output) return fail(resp.error || "No output", tokensUsed);
+	invariant(typeof resp.output === "string", "context-faithfulness produced malformed response");
+	const contextString = serializeContext(context);
+	const statements = splitIntoSentences(resp.output);
+	promptText = await renderLlmRubricPrompt(nliPrompt, {
+		context: contextString,
+		statements,
+		...vars || {}
+	});
+	resp = await callProviderWithContext(textProvider, promptText, "context-faithfulness-nli", {
+		context: contextString,
+		statements,
+		...vars || {}
+	}, providerCallContext);
+	accumulateTokenUsage(tokensUsed, resp.tokenUsage);
+	if (resp.error || !resp.output) return fail(resp.error || "No output", tokensUsed);
+	invariant(typeof resp.output === "string", "context-faithfulness produced malformed response");
+	let finalAnswer = "Final verdict for each statement in order:";
+	finalAnswer = finalAnswer.toLowerCase();
+	let verdicts = resp.output.toLowerCase().trim();
+	let score = 0;
+	if (statements.length > 0) if (verdicts.includes(finalAnswer)) {
+		verdicts = verdicts.slice(verdicts.indexOf(finalAnswer) + finalAnswer.length);
+		const parsedVerdicts = verdicts.split(".").filter((answer) => answer.trim() !== "");
+		if (parsedVerdicts.length > 0) score = 1 - parsedVerdicts.filter((answer) => !answer.includes("yes")).length / statements.length;
+	} else {
+		const noVerdictCount = verdicts.split("verdict: no").length - 1;
+		if (noVerdictCount + (verdicts.split("verdict: yes").length - 1) > 0) score = 1 - noVerdictCount / statements.length;
+	}
+	score = Math.min(1, Math.max(0, score));
+	const pass = score >= threshold - Number.EPSILON;
+	return {
+		pass,
+		score,
+		reason: pass ? `Faithfulness ${score.toFixed(2)} is >= ${threshold}` : `Faithfulness ${score.toFixed(2)} is < ${threshold}`,
+		tokensUsed
+	};
+}
+//#endregion
+//#region src/matchers/similarity.ts
+function calculateSimilarityScore(expectedEmbedding, outputEmbedding, metric, tokensUsed) {
+	switch (metric) {
+		case "cosine": return cosineSimilarity(expectedEmbedding, outputEmbedding);
+		case "dot_product": return dotProduct(expectedEmbedding, outputEmbedding);
+		case "euclidean": return euclideanDistance(expectedEmbedding, outputEmbedding);
+		default: return fail(`Unsupported metric: ${metric}`, tokensUsed);
+	}
+}
+function buildSimilarityResult(similarity, threshold, inverse, metric, tokensUsed) {
+	if (metric === "euclidean") {
+		const distance = similarity;
+		const pass = inverse ? distance >= threshold - Number.EPSILON : distance <= threshold + Number.EPSILON;
+		const normalizedScore = 1 / (1 + distance);
+		const score = inverse ? 1 - normalizedScore : normalizedScore;
+		const belowThresholdReason = `Distance ${distance.toFixed(2)} is less than or equal to threshold ${threshold}`;
+		const aboveThresholdReason = `Distance ${distance.toFixed(2)} is greater than threshold ${threshold}`;
+		return {
+			pass,
+			score,
+			reason: pass ? inverse ? aboveThresholdReason : belowThresholdReason : inverse ? belowThresholdReason : aboveThresholdReason,
+			tokensUsed
+		};
+	}
+	const pass = inverse ? similarity <= threshold + Number.EPSILON : similarity >= threshold - Number.EPSILON;
+	const score = inverse ? 1 - similarity : similarity;
+	const greaterThanReason = `Similarity ${similarity.toFixed(2)} is greater than or equal to threshold ${threshold}`;
+	const lessThanReason = `Similarity ${similarity.toFixed(2)} is less than threshold ${threshold}`;
+	return {
+		pass,
+		score,
+		reason: pass ? inverse ? lessThanReason : greaterThanReason : inverse ? greaterThanReason : lessThanReason,
+		tokensUsed
+	};
+}
+async function calculateProviderSimilarity(finalProvider, expected, output, metric, tokensUsed) {
+	if (metric === "cosine" && "callSimilarityApi" in finalProvider) {
+		const similarityResp = await finalProvider.callSimilarityApi(expected, output);
+		accumulateTokenUsage(tokensUsed, similarityResp.tokenUsage);
+		if (similarityResp.error) return fail(similarityResp.error, tokensUsed);
+		if (similarityResp.similarity == null) return fail("Unknown error fetching similarity", tokensUsed);
+		if (!Number.isFinite(similarityResp.similarity)) return fail(`Invalid similarity score: ${similarityResp.similarity}`, tokensUsed);
+		return similarityResp.similarity;
+	}
+	const callEmbeddingApi = "callEmbeddingApi" in finalProvider ? finalProvider.callEmbeddingApi : void 0;
+	if (typeof callEmbeddingApi !== "function") {
+		if ("callSimilarityApi" in finalProvider) return fail(`Provider ${finalProvider.id()} only supports cosine similarity via callSimilarityApi`, tokensUsed);
+		throw new Error("Provider must implement callSimilarityApi or callEmbeddingApi");
+	}
+	const [expectedEmbedding, outputEmbedding] = await Promise.all([callEmbeddingApi.call(finalProvider, expected), callEmbeddingApi.call(finalProvider, output)]);
+	const mergedUsage = normalizeMatcherTokenUsage(void 0);
+	accumulateTokenUsage(mergedUsage, expectedEmbedding.tokenUsage);
+	accumulateTokenUsage(mergedUsage, outputEmbedding.tokenUsage);
+	accumulateTokenUsage(tokensUsed, mergedUsage);
+	if (expectedEmbedding.error || outputEmbedding.error) return fail(expectedEmbedding.error || outputEmbedding.error || "Unknown error fetching embeddings", tokensUsed);
+	if (!expectedEmbedding.embedding || !outputEmbedding.embedding) return fail("Embedding not found", tokensUsed);
+	return calculateSimilarityScore(expectedEmbedding.embedding, outputEmbedding.embedding, metric, tokensUsed);
+}
+async function matchesSimilarity(expected, output, threshold, inverse = false, grading, metric = "cosine") {
+	if (metric === "cosine" && state.config?.redteam && shouldGenerateRemote({ requireEmbeddingProvider: true })) try {
+		return await doRemoteGrading({
+			task: "similar",
+			expected,
+			output,
+			threshold,
+			inverse
+		});
+	} catch (error) {
+		return fail(`Could not perform remote grading: ${error}`);
+	}
+	const defaults = await getDefaultProviders();
+	const finalProvider = await getAndCheckProvider("embedding", grading?.provider, defaults.embeddingProvider, "similarity check");
+	const tokensUsed = normalizeMatcherTokenUsage(void 0);
+	const similarity = await calculateProviderSimilarity(finalProvider, expected, output, metric, tokensUsed);
+	if (typeof similarity !== "number") return similarity;
+	return buildSimilarityResult(similarity, threshold, inverse, metric, tokensUsed);
+}
+//#endregion
 //#region src/tracing/evaluatorTracing.ts
 let otlpReceiverStarted = false;
 const DEFAULT_OTLP_ACCEPT_FORMATS = ["json", "protobuf"];
@@ -285,7 +799,7 @@ async function startOtlpReceiverIfNeeded(testSuite) {
 		telemetry.record("feature_used", { feature: "tracing" });
 		try {
 			logger.debug("[EvaluatorTracing] Tracing configuration detected, starting OTLP receiver");
-			const { startOTLPReceiver } = await import("./otlpReceiver-B2z58l4e.js");
+			const { startOTLPReceiver } = await import("./otlpReceiver-BXjcRqAM.js");
 			const port = testSuite.tracing.otlp.http.port || 4318;
 			const host = testSuite.tracing.otlp.http.host || "127.0.0.1";
 			const acceptFormats = normalizeOtlpAcceptFormats(testSuite.tracing.otlp.http.acceptFormats);
@@ -309,7 +823,7 @@ async function startOtlpReceiverIfNeeded(testSuite) {
 async function stopOtlpReceiverIfNeeded() {
 	if (otlpReceiverStarted) try {
 		logger.debug("[EvaluatorTracing] Stopping OTLP receiver");
-		const { stopOTLPReceiver } = await import("./otlpReceiver-B2z58l4e.js");
+		const { stopOTLPReceiver } = await import("./otlpReceiver-BXjcRqAM.js");
 		await stopOTLPReceiver();
 		otlpReceiverStarted = false;
 		logger.info("[EvaluatorTracing] OTLP receiver stopped successfully");
@@ -344,7 +858,7 @@ async function generateTraceContextIfNeeded(test, evaluateOptions, testIdx, prom
 	}
 	if (!tracingEnabled) return null;
 	logger.debug("[EvaluatorTracing] Importing trace store");
-	const { getTraceStore } = await import("./store-M0b1WfYb.js").then((n) => n.n);
+	const { getTraceStore } = await import("./store-HpopRVzl.js").then((n) => n.n);
 	const traceStore = getTraceStore();
 	const traceId = generateTraceId();
 	const spanId = generateSpanId();
@@ -654,38 +1168,84 @@ async function handleClassifier({ assertion, renderedValue, outputString, test,
 }
 //#endregion
 //#region src/assertions/contains.ts
+/**
+* Advance over separators between parsed fields.
+*
+* Contains-any values allow whitespace around comma delimiters, and historical
+* parsing ignored repeated commas rather than producing empty fields.
+*/
+function skipWhitespaceAndCommas(value, startIndex) {
+	let i = startIndex;
+	while (i < value.length) {
+		i = skipWhitespace(value, i);
+		if (value[i] !== ",") break;
+		i++;
+	}
+	return i;
+}
+/**
+* Advance over whitespace while preserving comma delimiter handling for callers.
+*/
+function skipWhitespace(value, startIndex) {
+	let i = startIndex;
+	while (i < value.length && /\s/.test(value[i])) i++;
+	return i;
+}
+/**
+* Parse a quoted field using the assertion parser's CSV-like escape rules.
+*
+* Supports backslash-escaped quotes/backslashes and doubled quotes, and rejects
+* unterminated fields so malformed assertion values do not silently pass.
+*/
+function parseQuotedField(value, startIndex) {
+	let i = startIndex + 1;
+	let field = "";
+	let terminated = false;
+	while (i < value.length) if (value[i] === "\\" && i + 1 < value.length && ["\"", "\\"].includes(value[i + 1])) {
+		field += value[i + 1];
+		i += 2;
+	} else if (value[i] === "\"" && i + 1 < value.length && value[i + 1] === "\"") {
+		field += "\"";
+		i += 2;
+	} else if (value[i] === "\"") {
+		i++;
+		terminated = true;
+		break;
+	} else {
+		field += value[i];
+		i++;
+	}
+	invariant(terminated, "Unterminated quoted field in contains assertion value");
+	return {
+		field,
+		nextIndex: i
+	};
+}
+/**
+* Parse an unquoted field up to the next comma, trimming surrounding whitespace.
+*/
+function parseUnquotedField(value, startIndex) {
+	let i = startIndex;
+	while (i < value.length && value[i] !== ",") i++;
+	return {
+		field: value.substring(startIndex, i).trim(),
+		nextIndex: i
+	};
+}
+/**
+* Split a contains-any string into fields while preserving quoted commas.
+*/
 function parseCommaSeparatedValues(value) {
 	const results = [];
 	let i = 0;
 	while (i < value.length) {
-		while (i < value.length && /\s/.test(value[i])) i++;
+		i = skipWhitespaceAndCommas(value, i);
 		if (i >= value.length) break;
-		if (value[i] === ",") {
-			i++;
-			continue;
-		}
-		if (value[i] === "\"") {
-			i++;
-			let field = "";
-			while (i < value.length) if (value[i] === "\\" && i + 1 < value.length && (value[i + 1] === "\"" || value[i + 1] === "\\")) {
-				field += value[i + 1];
-				i += 2;
-			} else if (value[i] === "\"" && i + 1 < value.length && value[i + 1] === "\"") {
-				field += "\"";
-				i += 2;
-			} else if (value[i] === "\"") {
-				i++;
-				break;
-			} else {
-				field += value[i];
-				i++;
-			}
-			results.push(field);
-		} else {
-			const start = i;
-			while (i < value.length && value[i] !== ",") i++;
-			results.push(value.substring(start, i).trim());
-		}
+		const isQuotedField = value[i] === "\"";
+		const parsed = isQuotedField ? parseQuotedField(value, i) : parseUnquotedField(value, i);
+		results.push(parsed.field);
+		i = isQuotedField ? skipWhitespace(value, parsed.nextIndex) : parsed.nextIndex;
+		invariant(!isQuotedField || i >= value.length || value[i] === ",", "Expected comma after quoted field in contains assertion value");
 	}
 	return results;
 }
@@ -930,27 +1490,67 @@ const handleIsValidFunctionCall = ({ assertion, output, provider, test }) => {
 };
 //#endregion
 //#region src/assertions/geval.ts
-const handleGEval = async ({ assertion, renderedValue, prompt, outputString, test, providerCallContext }) => {
-	invariant(typeof renderedValue === "string" || Array.isArray(renderedValue), "G-Eval assertion type must have a string or array of strings value");
+const handleGEval = async ({ assertion, inverse, renderedValue, prompt, outputString, test, providerCallContext }) => {
+	invariant(typeof renderedValue === "string" || Array.isArray(renderedValue) && renderedValue.every((value) => typeof value === "string"), "G-Eval assertion type must have a string or array of strings value");
 	const threshold = assertion.threshold ?? .7;
 	if (Array.isArray(renderedValue)) {
-		const scores = [];
-		const reasons = [];
-		for (const value of renderedValue) {
+		if (renderedValue.length === 0) return {
+			assertion,
+			pass: false,
+			score: 0,
+			reason: "G-Eval assertion requires at least one criterion string in the value array."
+		};
+		const responses = [];
+		let failure;
+		for (const [index, value] of renderedValue.entries()) {
 			const resp = await matchesGEval(value, prompt || "", outputString, threshold, test.options, providerCallContext);
-			scores.push(resp.score);
-			reasons.push(resp.reason);
+			responses.push(resp);
+			if (isGraderFailure(resp)) {
+				failure = {
+					index,
+					resp
+				};
+				break;
+			}
+		}
+		const tokensUsed = createEmptyTokenUsage();
+		for (const r of responses) accumulateTokenUsage(tokensUsed, r.tokensUsed);
+		if (failure) {
+			const criterion = renderedValue[failure.index];
+			return {
+				assertion,
+				pass: false,
+				score: 0,
+				reason: `G-Eval criterion ${failure.index + 1}/${renderedValue.length} (${JSON.stringify(criterion)}) failed: ${failure.resp.reason}`,
+				tokensUsed,
+				metadata: failure.resp.metadata
+			};
 		}
-		const scoresSum = scores.reduce((a, b) => a + b, 0);
+		const averageScore = responses.reduce((acc, r) => acc + r.score, 0) / responses.length;
+		const combinedReason = responses.map((r) => r.reason).join("\n\n");
 		return {
 			assertion,
-			pass: scoresSum / scores.length >= threshold,
-			score: scoresSum / scores.length,
-			reason: reasons.join("\n\n")
+			pass: averageScore >= threshold !== inverse,
+			score: inverse ? 1 - averageScore : averageScore,
+			reason: combinedReason,
+			tokensUsed
 		};
-	} else return {
+	}
+	const resp = await matchesGEval(renderedValue, prompt || "", outputString, threshold, test.options, providerCallContext);
+	if (isGraderFailure(resp)) return {
+		assertion,
+		pass: false,
+		score: 0,
+		reason: resp.reason,
+		tokensUsed: resp.tokensUsed,
+		metadata: resp.metadata
+	};
+	const passed = resp.score >= threshold !== inverse;
+	return {
 		assertion,
-		...await matchesGEval(renderedValue, prompt || "", outputString, threshold, test.options, providerCallContext)
+		...resp,
+		pass: passed,
+		score: inverse ? 1 - resp.score : resp.score
 	};
 };
 //#endregion
@@ -1090,6 +1690,43 @@ const handleGuardrails = async ({ assertion, inverse, providerResponse }) => {
 };
 //#endregion
 //#region src/assertions/html.ts
+const LITERAL_WRAPPER_PATTERNS = {
+	html: /<html(?=[\s>/])/,
+	head: /<head(?=[\s>/])/,
+	body: /<body(?=[\s>/])/
+};
+function isWrapperTagName(tagName) {
+	return tagName === "html" || tagName === "head" || tagName === "body";
+}
+function isTextNode(node) {
+	return node.nodeName === "#text";
+}
+function isElementNode(node) {
+	return "tagName" in node;
+}
+function hasSourceCodeLocation(element) {
+	return "sourceCodeLocation" in element && element.sourceCodeLocation !== null && element.sourceCodeLocation !== void 0;
+}
+function getChildNodes(node) {
+	return "childNodes" in node ? node.childNodes : [];
+}
+function findFirstElement(root, predicate) {
+	const stack = [root];
+	while (stack.length > 0) {
+		const current = stack.pop();
+		if (isElementNode(current) && predicate(current)) return current;
+		const children = getChildNodes(current);
+		for (let i = children.length - 1; i >= 0; i--) stack.push(children[i]);
+	}
+}
+function hasTopLevelText(parentNode) {
+	return parentNode.childNodes.some((node) => isTextNode(node) && Boolean(node.value.trim()));
+}
+function isUserProvidedElement(element, inputLowercase) {
+	const tagName = element.tagName.toLowerCase();
+	if (isWrapperTagName(tagName)) return LITERAL_WRAPPER_PATTERNS[tagName].test(inputLowercase) && hasSourceCodeLocation(element);
+	return VALID_HTML_ELEMENTS.has(tagName) || tagName.includes("-");
+}
 const HTML_PATTERNS = {
 	openingTag: /<[a-zA-Z][a-zA-Z0-9-]*(?:\s[^>]*)?>/,
 	closingTag: /<\/[a-zA-Z][a-zA-Z0-9-]*\s*>/,
@@ -1245,37 +1882,21 @@ function validateHtml(htmlString) {
 		isValid: false,
 		reason: "Output appears to be XML, not HTML"
 	};
-	try {
-		const { document } = new JSDOM(trimmed, { contentType: "text/html" }).window;
-		if (document.body && !trimmed.toLowerCase().includes("<body")) {
-			if (Array.from(document.body.childNodes).some((node) => node.nodeType === 3 && node.textContent?.trim())) return {
-				isValid: false,
-				reason: "Output must be wrapped in HTML tags"
-			};
-		}
-		const allElements = document.querySelectorAll("*");
-		if (!Array.from(allElements).find((element) => {
-			const tagName = element.tagName.toLowerCase();
-			if ([
-				"html",
-				"head",
-				"body"
-			].includes(tagName) && !trimmed.toLowerCase().includes(`<${tagName}`)) return false;
-			return VALID_HTML_ELEMENTS.has(tagName) || tagName.includes("-");
-		})) return {
-			isValid: false,
-			reason: "Output does not contain recognized HTML elements"
-		};
-		return {
-			isValid: true,
-			reason: "Output is valid HTML"
-		};
-	} catch (error) {
-		return {
-			isValid: false,
-			reason: `HTML parsing failed: ${error instanceof Error ? error.message : "Unknown error"}`
-		};
-	}
+	const document = parse$2(trimmed, { sourceCodeLocationInfo: true });
+	const inputLowercase = trimmed.toLowerCase();
+	const body = findFirstElement(document, (element) => element.tagName === "body");
+	if (!(body !== void 0 && LITERAL_WRAPPER_PATTERNS.body.test(inputLowercase) && hasSourceCodeLocation(body)) && body && hasTopLevelText(body)) return {
+		isValid: false,
+		reason: "Output must be wrapped in HTML tags"
+	};
+	if (!findFirstElement(document, (element) => isUserProvidedElement(element, inputLowercase))) return {
+		isValid: false,
+		reason: "Output does not contain recognized HTML elements"
+	};
+	return {
+		isValid: true,
+		reason: "Output is valid HTML"
+	};
 }
 const handleContainsHtml = ({ assertion, outputString, inverse }) => {
 	const pass = containsHtml(outputString) !== inverse;
@@ -1932,45 +2553,6 @@ function matchesPattern(spanName, pattern) {
 }
 //#endregion
 //#region src/assertions/trajectoryUtils.ts
-const TOOL_ATTRIBUTE_KEYS = [
-	"tool.name",
-	"tool_name",
-	"tool",
-	"function.name",
-	"function_name",
-	"gen_ai.tool.name",
-	"codex.mcp.tool",
-	"agent.tool",
-	"agent.tool_name",
-	"agent.toolName"
-];
-const TOOL_ARGUMENT_ATTRIBUTE_KEYS = [
-	"tool.arguments",
-	"tool.args",
-	"tool.input",
-	"tool_arguments",
-	"tool_args",
-	"tool_input",
-	"function.arguments",
-	"function.args",
-	"function.input",
-	"function_arguments",
-	"function_args",
-	"gen_ai.tool.arguments",
-	"gen_ai.tool.args",
-	"gen_ai.tool.input",
-	"gen_ai.tool.call.arguments",
-	"gen_ai.tool.call.args",
-	"agent.tool.arguments",
-	"agent.tool.args",
-	"agent.tool.input",
-	"codex.mcp.arguments",
-	"codex.mcp.args",
-	"codex.mcp.input",
-	"arguments",
-	"args",
-	"input"
-];
 const COMMAND_ATTRIBUTE_KEYS = [
 	"codex.command",
 	"command",
@@ -1983,16 +2565,15 @@ const SEARCH_ATTRIBUTE_KEYS = [
 	"search_query"
 ];
 const GENERIC_QUERY_ATTRIBUTE_KEYS = ["query"];
+const COMMAND_TOOL_NAMES = new Set([
+	"exec_command",
+	"local_shell",
+	"shell"
+]);
 const SEARCH_SPAN_NAME_PATTERN = /(^|[\s._:/-])(search|find|lookup|retriev(?:e|al))($|[\s._:/-])/i;
 const MAX_JUDGE_SUMMARY_STEPS = 24;
 const JUDGE_SUMMARY_HEAD_STEPS = 12;
 const JUDGE_SUMMARY_TAIL_STEPS = 12;
-function getStringAttribute(attributes, keys) {
-	for (const key of keys) {
-		const value = attributes[key];
-		if (typeof value === "string" && value.trim()) return value.trim();
-	}
-}
 function normalizeStructuredAttribute(value) {
 	if (value === void 0 || value === null) return;
 	if (typeof value === "string") {
@@ -2024,9 +2605,12 @@ function getTrajectoryStepStatus(step) {
 function getCommandExecutable(command) {
 	return command.trim().split(/\s+/)[0] || void 0;
 }
+function isCommandToolName(toolName) {
+	return !!toolName && COMMAND_TOOL_NAMES.has(toolName.trim().toLowerCase());
+}
 function extractToolName(span) {
 	const attributes = span.attributes || {};
-	const directMatch = getStringAttribute(attributes, TOOL_ATTRIBUTE_KEYS);
+	const directMatch = getToolNameFromAttributes(attributes);
 	if (directMatch) return directMatch;
 	for (const [key, value] of Object.entries(attributes)) {
 		if (typeof value !== "string" || !value.trim()) continue;
@@ -2051,21 +2635,31 @@ function extractToolArgs(span) {
 		if (value !== void 0) return value;
 	}
 }
-function extractCommand(span) {
+function extractCommand(span, toolName = extractToolName(span), getToolArgs = () => extractToolArgs(span)) {
 	const attributes = span.attributes || {};
-	const directMatch = getStringAttribute(attributes, COMMAND_ATTRIBUTE_KEYS);
+	const directMatch = getFirstStringAttribute(attributes, COMMAND_ATTRIBUTE_KEYS);
 	if (directMatch) return directMatch;
 	for (const [key, value] of Object.entries(attributes)) {
 		if (typeof value !== "string" || !value.trim()) continue;
 		if (/command/i.test(key) && !/output|result/i.test(key)) return value.trim();
 	}
+	const toolArgs = getToolArgs();
+	if (isCommandToolName(toolName) && toolArgs && typeof toolArgs === "object") {
+		const args = toolArgs;
+		const command = args.cmd ?? args.command;
+		if (typeof command === "string" && command.trim()) return command.trim();
+		if (Array.isArray(command)) {
+			const joined = command.map((part) => String(part).trim()).filter(Boolean).join(" ");
+			if (joined) return joined;
+		}
+	}
 	if (span.name.startsWith("exec ")) return span.name.slice(5).trim();
 }
 function extractSearchQuery(span) {
 	const attributes = span.attributes || {};
-	const directMatch = getStringAttribute(attributes, SEARCH_ATTRIBUTE_KEYS);
+	const directMatch = getFirstStringAttribute(attributes, SEARCH_ATTRIBUTE_KEYS);
 	if (directMatch) return directMatch;
-	const genericQuery = getStringAttribute(attributes, GENERIC_QUERY_ATTRIBUTE_KEYS);
+	const genericQuery = getFirstStringAttribute(attributes, GENERIC_QUERY_ATTRIBUTE_KEYS);
 	if (genericQuery && isSearchLikeSpan(span)) return genericQuery;
 	if (span.name.startsWith("search ")) return span.name.slice(7).replace(/^"|"$/g, "").trim();
 }
@@ -2089,17 +2683,34 @@ function extractTrajectorySteps(trace) {
 		return left.index - right.index;
 	}).map(({ span }) => {
 		const toolName = extractToolName(span);
-		const command = extractCommand(span);
+		let toolArgs;
+		let hasExtractedToolArgs = false;
+		const getToolArgs = () => {
+			if (!hasExtractedToolArgs) {
+				toolArgs = extractToolArgs(span);
+				hasExtractedToolArgs = true;
+			}
+			return toolArgs;
+		};
+		const command = extractCommand(span, toolName, getToolArgs);
 		const searchQuery = extractSearchQuery(span);
 		let type = "span";
 		let name = span.name;
 		const aliases = new Set([span.name]);
 		let args;
-		if (toolName) {
+		if (command && isCommandToolName(toolName)) {
+			type = "command";
+			name = command;
+			aliases.add(command);
+			args = getToolArgs();
+			if (toolName) aliases.add(toolName);
+			const executable = getCommandExecutable(command);
+			if (executable) aliases.add(executable);
+		} else if (toolName) {
 			type = "tool";
 			name = toolName;
 			aliases.add(toolName);
-			args = extractToolArgs(span);
+			args = getToolArgs();
 		} else if (command) {
 			type = "command";
 			name = command;
@@ -2380,11 +2991,10 @@ function handleRougeScore({ baseType, assertion, renderedValue, outputString, in
 	const rougeMethod = rouge[baseType[baseType.length - 1]];
 	const score = rougeMethod(outputString, renderedValue, {});
 	const threshold = assertion.threshold ?? .75;
-	const pass = score >= threshold != inverse;
 	return {
-		pass,
+		pass: score >= threshold !== inverse,
 		score: inverse ? 1 - score : score,
-		reason: pass ? `${baseType.toUpperCase()} score ${score.toFixed(2)} is greater than or equal to threshold ${threshold}` : `${baseType.toUpperCase()} score ${score.toFixed(2)} is less than threshold ${threshold}`,
+		reason: `${baseType.toUpperCase()} score ${score.toFixed(2)} is ${score >= threshold ? "greater than or equal to" : "less than"} threshold ${threshold}`,
 		assertion
 	};
 }
@@ -2446,6 +3056,192 @@ const handleRuby = async ({ assertion, renderedValue, valueFromScript, assertion
 	}
 };
 //#endregion
+//#region src/providers/webSearchUtils.ts
+function hasTool(provider, predicate) {
+	return Array.isArray(provider.config?.tools) && provider.config.tools.some(predicate);
+}
+function getProviderId(provider) {
+	if (typeof provider.id !== "function") return null;
+	try {
+		return provider.id();
+	} catch (err) {
+		logger.debug(`Failed to read provider id: ${err}`);
+		return null;
+	}
+}
+function isOpenAiResponsesProvider(provider, id) {
+	return id.includes("openai:responses") || provider.constructor?.name === "OpenAiResponsesProvider";
+}
+/**
+* Check if a provider has web search capabilities
+* @param provider The provider to check
+* @returns true if the provider supports web search
+*/
+function hasWebSearchCapability(provider) {
+	if (!provider) return false;
+	const id = getProviderId(provider);
+	if (!id) return false;
+	if (id.includes("perplexity")) return true;
+	if ((id.includes("google") || id.includes("gemini") || id.includes("vertex")) && hasTool(provider, (t) => t.googleSearch !== void 0)) return true;
+	if (id.includes("xai") && provider.config?.search_parameters?.mode === "on") return true;
+	if (isOpenAiResponsesProvider(provider, id) && hasTool(provider, (t) => t.type === "web_search_preview")) return true;
+	if (id.startsWith("openai:codex") && (provider.config?.web_search_mode === "live" || provider.config?.web_search_mode === "cached" || provider.config?.web_search_enabled === true)) return true;
+	if (id.includes("anthropic") && hasTool(provider, (t) => t.type === "web_search_20250305")) return true;
+	return false;
+}
+/**
+* Load a provider with web search capabilities.
+* Tries multiple providers in order of preference until one succeeds.
+* Uses the latest and most capable models from each provider with specific checkpoint IDs.
+*
+* @param preferAnthropic Whether to try Anthropic first (true) or OpenAI first (false)
+* @returns A provider with web search capabilities or null
+*/
+async function loadWebSearchProvider(preferAnthropic = false) {
+	const loadAnthropicWebSearch = async () => {
+		try {
+			return await loadApiProvider("anthropic:messages:claude-opus-4-6", { options: { config: { tools: [{
+				type: "web_search_20250305",
+				name: "web_search",
+				max_uses: 5
+			}] } } });
+		} catch (err) {
+			logger.debug(`Failed to load Anthropic web search provider: ${err}`);
+			return null;
+		}
+	};
+	const loadOpenAIWebSearch = async () => {
+		try {
+			return await loadApiProvider("openai:responses:gpt-5.4-2026-03-05", { options: { config: { tools: [{ type: "web_search_preview" }] } } });
+		} catch (err) {
+			logger.debug(`Failed to load OpenAI web search provider: ${err}`);
+			return null;
+		}
+	};
+	const loadPerplexity = async () => {
+		try {
+			return await loadApiProvider("perplexity:sonar-pro");
+		} catch (err) {
+			logger.debug(`Failed to load Perplexity provider: ${err}`);
+			return null;
+		}
+	};
+	const loadGoogleWebSearch = async () => {
+		try {
+			return await loadApiProvider("google:gemini-3-pro-preview", { options: { config: { tools: [{ googleSearch: {} }] } } });
+		} catch (err) {
+			logger.debug(`Failed to load Google web search provider: ${err}`);
+			return null;
+		}
+	};
+	const loadVertexWebSearch = async () => {
+		try {
+			return await loadApiProvider("vertex:gemini-3-pro-preview", { options: { config: { tools: [{ googleSearch: {} }] } } });
+		} catch (err) {
+			logger.debug(`Failed to load Vertex web search provider: ${err}`);
+			return null;
+		}
+	};
+	const loadXaiWebSearch = async () => {
+		try {
+			return await loadApiProvider("xai:grok-4-1-fast-reasoning", { options: { config: { search_parameters: { mode: "on" } } } });
+		} catch (err) {
+			logger.debug(`Failed to load xAI web search provider: ${err}`);
+			return null;
+		}
+	};
+	const providers = preferAnthropic ? [
+		loadAnthropicWebSearch,
+		loadOpenAIWebSearch,
+		loadPerplexity,
+		loadGoogleWebSearch,
+		loadVertexWebSearch,
+		loadXaiWebSearch
+	] : [
+		loadOpenAIWebSearch,
+		loadAnthropicWebSearch,
+		loadPerplexity,
+		loadGoogleWebSearch,
+		loadVertexWebSearch,
+		loadXaiWebSearch
+	];
+	for (const getProvider of providers) {
+		const provider = await getProvider();
+		if (provider && hasWebSearchCapability(provider)) {
+			logger.info(`Using ${getProviderId(provider) ?? "loaded provider"} as web search provider`);
+			return provider;
+		}
+		if (provider) logger.debug(`Loaded provider ${getProviderId(provider) ?? "unknown"} does not support web search`);
+	}
+	return null;
+}
+//#endregion
+//#region src/matchers/search.ts
+async function matchesSearchRubric(rubric, llmOutput, grading, vars, assertion, _provider, providerCallContext) {
+	if (!grading) throw new Error("Cannot grade output without grading config. Specify --grader option or grading config.");
+	const defaultProviders = await getDefaultProviders();
+	const defaultSearchProviders = [
+		defaultProviders.webSearchProvider,
+		defaultProviders.llmRubricProvider,
+		defaultProviders.gradingProvider
+	];
+	let searchProvider = (grading.provider ? await getGradingProvider("text", grading.provider, null) : null) || defaultSearchProviders.find((provider) => Boolean(provider));
+	if (!hasWebSearchCapability(searchProvider)) {
+		const webSearchDefault = defaultSearchProviders.find((provider) => hasWebSearchCapability(provider));
+		if (webSearchDefault) searchProvider = webSearchDefault;
+	}
+	if (!hasWebSearchCapability(searchProvider)) {
+		const webSearchProvider = await loadWebSearchProvider(true);
+		if (webSearchProvider) searchProvider = webSearchProvider;
+	}
+	if (!searchProvider || !hasWebSearchCapability(searchProvider)) throw new Error(`search-rubric assertion requires a grading provider with web search capabilities. Use --grader with a web search provider (e.g., anthropic:messages:${DEFAULT_ANTHROPIC_MODEL}, openai:responses:o4-mini with tools configured, perplexity:sonar) or configure one in defaultTest.options.provider`);
+	const prompt = await renderLlmRubricPrompt(await loadRubricPrompt(grading?.rubricPrompt, DEFAULT_WEB_SEARCH_PROMPT), {
+		output: tryParse(llmOutput),
+		rubric,
+		...vars || {}
+	});
+	const resp = await callProviderWithContext(searchProvider, prompt, "search-rubric", {
+		output: tryParse(llmOutput),
+		rubric,
+		...vars || {}
+	}, providerCallContext);
+	if (resp.error || !resp.output) return {
+		pass: false,
+		score: 0,
+		reason: `Search rubric evaluation failed: ${resp.error || "No output"}`,
+		tokensUsed: resp.tokenUsage,
+		assertion
+	};
+	try {
+		const result = extractFirstJsonObject(String(resp.output));
+		let pass = result.pass ?? false;
+		const score = typeof result.score === "number" ? result.score : pass ? 1 : 0;
+		if (assertion?.threshold !== void 0) pass = pass && score >= assertion.threshold;
+		return {
+			pass,
+			score,
+			reason: result.reason || "No reason provided",
+			tokensUsed: resp.tokenUsage,
+			assertion,
+			metadata: {
+				searchResults: result.searchResults || [],
+				searchProvider: searchProvider.id()
+			}
+		};
+	} catch (err) {
+		logger.warn(`[search-rubric] Could not parse structured JSON from provider response, falling back to substring matching: ${err.message}`);
+		const outputLower = String(resp.output).toLowerCase();
+		const pass = outputLower.includes("\"pass\":true") || outputLower.includes("\"pass\": true");
+		return {
+			pass,
+			score: pass ? 1 : 0,
+			reason: resp.output,
+			tokensUsed: resp.tokenUsage,
+			assertion
+		};
+	}
+}
+//#endregion
 //#region src/assertions/searchRubric.ts
 async function handleSearchRubric({ assertion, baseType: _baseType, inverse, provider, providerCallContext, renderedValue, test, providerResponse }) {
 	if (renderedValue == null) throw new Error("search-rubric assertion type must have a string value");
@@ -3101,13 +3897,13 @@ function resolveSequenceValue(value) {
 	}
 	throw new Error("trajectory:tool-sequence assertion must have an array or object value");
 }
-function isRecord(value) {
+function isRecord$1(value) {
 	return typeof value === "object" && value !== null && !Array.isArray(value);
 }
 function matchesExpectedArgsPartial(actual, expected) {
 	if (Array.isArray(expected)) return Array.isArray(actual) && actual.length === expected.length && expected.every((item, index) => matchesExpectedArgsPartial(actual[index], item));
-	if (isRecord(expected)) {
-		if (!isRecord(actual)) return false;
+	if (isRecord$1(expected)) {
+		if (!isRecord$1(actual)) return false;
 		return Object.entries(expected).every(([key, expectedValue]) => Object.prototype.hasOwnProperty.call(actual, key) && matchesExpectedArgsPartial(actual[key], expectedValue));
 	}
 	return isDeepStrictEqual(actual, expected);
@@ -3478,7 +4274,7 @@ async function loadTraceData(traceId) {
 	let stableObservations = 0;
 	let latestTrace = null;
 	for (let attempt = 0; attempt < maxAttempts; attempt++) {
-		latestTrace = await traceStore.getTrace(traceId);
+		latestTrace = await traceStore.getTrace(traceId, { sanitizeAttributes: false });
 		const spanCount = latestTrace?.spans?.length ?? 0;
 		if (spanCount > 0) {
 			stableObservations = spanCount === lastSpanCount ? stableObservations + 1 : 1;
@@ -3531,7 +4327,7 @@ const ASSERTION_HANDLERS = {
 	"llm-rubric": handleLlmRubric,
 	meteor: async (params) => {
 		try {
-			const { handleMeteorAssertion } = await import("./meteor-DuAFv6gF.js");
+			const { handleMeteorAssertion } = await import("./meteor-Dce-_zGQ.js");
 			return handleMeteorAssertion(params);
 		} catch (error) {
 			if (error instanceof Error && (error.message.includes("Cannot find module") || error.message.includes("natural\" package is required"))) return {
@@ -3667,7 +4463,7 @@ async function runAssertion({ prompt, provider, assertion, test, vars, latencyMs
 			};
 		}
 		else if (filePath.endsWith(".rb")) try {
-			const { runRuby } = await import("./rubyUtils-DVLeA2jg.js").then((n) => n.t);
+			const { runRuby } = await import("./rubyUtils-CnlW8AYb.js").then((n) => n.t);
 			valueFromScript = await runRuby(filePath, functionName || "get_assert", [output, context]);
 			logger.debug(`Ruby script ${filePath} output: ${valueFromScript}`);
 		} catch (error) {
@@ -3784,7 +4580,8 @@ async function runAssertions({ assertScoringFunction, latencyMs, prompt, provide
 		logger.debug(`Failed to preload trace data for assertions: ${error}`);
 		preloadedTraceData = null;
 	}
-	await async.forEachOfLimit(asserts, ASSERTIONS_MAX_CONCURRENCY, async ({ assertion, assertResult, index }) => {
+	const concurrency = getProviderCallExecutionContext()?.providerCallQueue ? 1 : ASSERTIONS_MAX_CONCURRENCY;
+	await async.forEachOfLimit(asserts, concurrency, async ({ assertion, assertResult, index }) => {
 		if (assertion.type.startsWith("select-") || assertion.type === "max-score") return;
 		const result = await runAssertion({
 			prompt,
@@ -3911,7 +4708,8 @@ var CIProgressReporter = class {
 	}
 	updateTotalTests(newTotal) {
 		this.totalTests = Math.max(newTotal, 1);
-		this.highestPercentageSeen = Math.floor(this.completedTests / this.totalTests * 100);
+		const percentage = Math.floor(this.completedTests / this.totalTests * 100);
+		this.highestPercentageSeen = percentage;
 	}
 	finish() {
 		if (this.intervalId) {
@@ -4084,6 +4882,10 @@ function getDefaultOtelConfig() {
 }
 //#endregion
 //#region src/tracing/localSpanExporter.ts
+const MISSING_TRACE_RETRY_DELAY_MS = 50;
+function delay(ms) {
+	return new Promise((resolve) => setTimeout(resolve, ms));
+}
 /**
 * A span exporter that writes spans to the local TraceStore (SQLite).
 * This allows OTEL spans to be stored locally for analysis in the promptfoo UI.
@@ -4125,7 +4927,7 @@ var LocalSpanExporter = class {
 		}
 		let firstError;
 		for (const [traceId, spanDataList] of spansByTrace) try {
-			const result = await traceStore.addSpans(traceId, spanDataList, { skipTraceCheck: false });
+			const result = await this.addSpansWithTraceRetry(traceStore, traceId, spanDataList);
 			if (result.stored) logger.debug(`[LocalSpanExporter] Added ${spanDataList.length} spans to trace ${traceId}`);
 			else logger.debug(`[LocalSpanExporter] Skipping ${spanDataList.length} spans for orphan trace ${traceId}: ${result.reason}`);
 		} catch (error) {
@@ -4137,6 +4939,16 @@ var LocalSpanExporter = class {
 		}
 		return firstError;
 	}
+	async addSpansWithTraceRetry(traceStore, traceId, spans) {
+		const options = {
+			skipTraceCheck: false,
+			warnIfMissingTrace: false
+		};
+		const result = await traceStore.addSpans(traceId, spans, options);
+		if (result.stored) return result;
+		await delay(MISSING_TRACE_RETRY_DELAY_MS);
+		return traceStore.addSpans(traceId, spans, options);
+	}
 	/**
 	* Convert an OTEL ReadableSpan to our SpanData format.
 	*/
@@ -4419,6 +5231,15 @@ function isPromptAllowed(prompt, allowedPrompts) {
 }
 //#endregion
 //#region src/evaluator.ts
+const CONVERSATION_VAR_NAME = "_conversation";
+const promptUsesConversationVariableCache = new LRUCache({ max: 1024 });
+function promptUsesConversationVariable(prompt) {
+	const cached = promptUsesConversationVariableCache.get(prompt.raw);
+	if (cached !== void 0) return cached;
+	const { referenced, parsed } = analyzeTemplateReference(prompt.raw, CONVERSATION_VAR_NAME);
+	if (parsed) promptUsesConversationVariableCache.set(prompt.raw, referenced);
+	return referenced;
+}
 /**
 * Manages a single progress bar for the evaluation
 */
@@ -4618,6 +5439,18 @@ function hasProviderGroupedAssertion(assertion) {
 function shouldDeferGradingForTest(test) {
 	return Boolean(test.assert?.some(hasProviderGroupedAssertion));
 }
+function logGroupedGradingStatus({ concurrency, hasEvalStepTimeout, runEvalOptions, shouldGroupGradingByProvider, usesConversationVar }) {
+	if (!runEvalOptions.some(({ test }) => shouldDeferGradingForTest(test))) return;
+	if (shouldGroupGradingByProvider) {
+		logger.info("Grouping model-graded assertions by provider to minimize local-model reload overhead.");
+		return;
+	}
+	if (concurrency !== 1) return;
+	const reasons = [];
+	if (hasEvalStepTimeout) reasons.push("per-eval-step timeout is configured");
+	if (usesConversationVar) reasons.push("conversation variables require per-row ordering");
+	if (reasons.length > 0) logger.info(`Serial grading grouping disabled because ${reasons.join(" and ")}; model-graded judges may reload between rows.`);
+}
 function applyGradingResult(row, checkResult) {
 	if (!checkResult.pass) {
 		row.error = checkResult.reason;
@@ -4632,14 +5465,29 @@ function applyGradingResult(row, checkResult) {
 	if (checkResult.tokensUsed) accumulateAssertionTokenUsage(row.tokenUsage.assertions, checkResult.tokensUsed);
 	row.gradingResult = checkResult;
 }
-function applyGradingError(row, error) {
-	const errorMessage = error instanceof Error ? error.stack ?? error.message : String(error);
-	logger.error("Assertion grading failed during eval", {
-		error: errorMessage,
-		promptIdx: row.promptIdx,
-		testIdx: row.testIdx
-	});
-	row.error = errorMessage;
+const ABORTED_GRADING_PREFIX = "Aborted: ";
+function isAbortShapedError(error) {
+	return error instanceof Error && (error.name === "AbortError" || error.name === "AbortException");
+}
+function applyGradingError(row, error, abortSignal) {
+	const errorAsError = error instanceof Error ? error : void 0;
+	if (Boolean(abortSignal?.aborted) && isAbortShapedError(error)) {
+		const shortMessage = errorAsError?.message ?? String(error);
+		logger.debug("Assertion grading aborted", {
+			error: shortMessage,
+			promptIdx: row.promptIdx,
+			testIdx: row.testIdx
+		});
+		row.error = `${ABORTED_GRADING_PREFIX}${shortMessage}`;
+	} else {
+		const fullMessage = errorAsError ? errorAsError.stack ?? errorAsError.message : String(error);
+		logger.error("Assertion grading failed during eval", {
+			error: fullMessage,
+			promptIdx: row.promptIdx,
+			testIdx: row.testIdx
+		});
+		row.error = fullMessage;
+	}
 	row.failureReason = ResultFailureReason.ERROR;
 	row.success = false;
 	row.score = 0;
@@ -4671,7 +5519,7 @@ function createRunEvalState({ provider, prompt, test }) {
 	};
 }
 function attachConversationVar({ conversations, conversationKey, prompt, test, vars }) {
-	const usesConversation = prompt.raw.includes("_conversation");
+	const usesConversation = promptUsesConversationVariable(prompt);
 	if (!getEnvBool("PROMPTFOO_DISABLE_CONVERSATION_VAR") && !test.options?.disableConversationVar && usesConversation) vars._conversation = conversations?.[conversationKey] || [];
 }
 function createRunEvalSetup({ provider, prompt, promptConfig, vars }) {
@@ -4918,7 +5766,7 @@ async function gradeRunEvalResponse({ abortSignal, deferGrading, evalId, latency
 			assertScoringFunction: test.assertScoringFunction,
 			traceId
 		}).then((checkResult) => applyGradingResult(ret, checkResult))).catch((error) => {
-			applyGradingError(ret, error);
+			applyGradingError(ret, error, abortSignal);
 		});
 		deferredGradingPromises.set(ret, gradingPromise);
 		return;
@@ -5465,7 +6313,7 @@ async function resolveDefaultTestProvider(defaultTest, testCase) {
 	const defaultProvider = defaultTest.provider;
 	if (isApiProvider(defaultProvider)) return defaultProvider;
 	if (typeof defaultProvider === "object" && defaultProvider.id) {
-		const { loadApiProvider } = await import("./providers-u9Enmfok.js");
+		const { loadApiProvider } = await import("./providers-DVYRZP4E.js").then((n) => n.i);
 		return loadApiProvider(typeof defaultProvider.id === "function" ? defaultProvider.id() : defaultProvider.id, { options: defaultProvider });
 	}
 	return defaultProvider;
@@ -5625,7 +6473,7 @@ function buildRepeatCacheContextByTestIdx(runEvalOptions) {
 async function filterCompletedResumeSteps(runEvalOptions, evalRecord) {
 	if (!state.resume || !evalRecord.persisted) return;
 	try {
-		const { default: EvalResult } = await import("./evalResult-Bgm9ZH31.js").then((n) => n.n);
+		const { default: EvalResult } = await import("./evalResult-2RRJvFyB.js").then((n) => n.n);
 		const completedPairs = await EvalResult.getCompletedIndexPairs(evalRecord.id, { excludeErrors: state.retryMode });
 		const originalCount = runEvalOptions.length;
 		for (let i = runEvalOptions.length - 1; i >= 0; i--) {
@@ -5639,14 +6487,14 @@ async function filterCompletedResumeSteps(runEvalOptions, evalRecord) {
 	}
 }
 function adjustConcurrencyForSerialFeatures({ concurrency, prompts, tests }) {
-	const usesConversationVar = prompts.some((p) => p.raw.includes("_conversation"));
+	const usesConversationVar = prompts.some(promptUsesConversationVariable);
 	if (concurrency <= 1) return {
 		concurrency,
 		usesConversationVar
 	};
 	const usesStoreOutputAs = tests.some((t) => t.options?.storeOutputAs);
 	if (usesConversationVar) {
-		logger.info(`Setting concurrency to 1 because the ${chalk.cyan("_conversation")} variable is used.`);
+		logger.info(`Setting concurrency to 1 because the ${chalk.cyan(CONVERSATION_VAR_NAME)} variable is used.`);
 		return {
 			concurrency: 1,
 			usesConversationVar
@@ -5876,7 +6724,8 @@ var Evaluator = class {
 		};
 		this.conversations = {};
 		this.registers = {};
-		this.fileWriters = (Array.isArray(evalRecord.config.outputPath) ? evalRecord.config.outputPath.filter((p) => p.endsWith(".jsonl")) : evalRecord.config.outputPath?.endsWith(".jsonl") ? [evalRecord.config.outputPath] : []).map((p) => new JsonlFileWriter(p));
+		const jsonlFiles = Array.isArray(evalRecord.config.outputPath) ? evalRecord.config.outputPath.filter((p) => p.endsWith(".jsonl")) : evalRecord.config.outputPath?.endsWith(".jsonl") ? [evalRecord.config.outputPath] : [];
+		this.fileWriters = jsonlFiles.map((p) => new JsonlFileWriter(p));
 		this.rateLimitRegistry = createRateLimitRegistry({ maxConcurrency: options.maxConcurrency || 4 });
 		this.rateLimitRegistry.on("ratelimit:hit", (data) => {
 			logger.debug(`[Scheduler] Rate limit hit for ${data.rateLimitKey}`, {
@@ -5996,6 +6845,25 @@ var Evaluator = class {
 			this.trackCompletedRow(evalStep, row, context);
 			context.numComplete++;
 			const promptEvalCount = reservePromptEvalCount(context, row.promptIdx);
+			if (context.testSuite.extensions?.length) try {
+				const afterEachOut = await runExtensionHook(context.testSuite.extensions, "afterEach", {
+					test: evalStep.test,
+					result: {
+						...row,
+						namedScores: { ...row.namedScores },
+						metadata: { ...row.metadata },
+						response: row.response ? {
+							...row.response,
+							metadata: { ...row.response.metadata }
+						} : row.response
+					}
+				});
+				row.namedScores = filterFiniteScores(afterEachOut.result.namedScores);
+				row.metadata = afterEachOut.result.metadata;
+				if (row.response && afterEachOut.result.response) row.response.metadata = afterEachOut.result.response.metadata;
+			} catch (error) {
+				logger.error(`afterEach extension hook failed, persisting row without hook modifications`, { error });
+			}
 			await this.persistEvalRow(row);
 			if (this.abortIfTargetUnavailable(row, context)) break;
 			const metrics = context.prompts[row.promptIdx].metrics;
@@ -6007,10 +6875,6 @@ var Evaluator = class {
 				promptEvalCount,
 				row
 			});
-			await runExtensionHook(context.testSuite.extensions, "afterEach", {
-				test: evalStep.test,
-				result: row
-			});
 			context.options.progressCallback?.(context.numComplete, context.runEvalOptionsLength, index, evalStep, metrics);
 		}
 	}
@@ -6084,9 +6948,8 @@ var Evaluator = class {
 		context.options.progressCallback?.(context.numComplete, context.runEvalOptionsLength, index, evalStep, metrics || createTimeoutMetrics(timeoutMs));
 	}
 	async executeEvalSteps({ checkAbort, ciProgressReporter, combinedAbortSignal, concurrentRunEvalOptions, evalStepIndexMap, globalTimeout, groupedRunEvalOptions, isEvalTimedOut, isWebUI, maxEvalTimeMs, processingContext, processedIndices, progressBarManager, prompts, serialRunEvalOptions, shouldGroupGradingByProvider }) {
-		let flushGroupedRows;
 		try {
-			if (shouldGroupGradingByProvider) flushGroupedRows = await this.runGroupedEvalSteps({
+			if (shouldGroupGradingByProvider) await this.runGroupedEvalSteps({
 				checkAbort,
 				evalStepIndexMap,
 				groupedRunEvalOptions,
@@ -6118,7 +6981,6 @@ var Evaluator = class {
 				cleanupProgressAfterError(progressBarManager, ciProgressReporter, err);
 				throw err;
 			}
-			await flushGroupedRows?.();
 			if (isEvalTimedOut()) logger.warn(`Evaluation stopped after reaching max duration (${maxEvalTimeMs}ms)`);
 			else if (!processingContext.targetUnavailable) return this.saveInterruptedEval({
 				ciProgressReporter,
@@ -6167,7 +7029,15 @@ var Evaluator = class {
 				})) break;
 			}
 		} catch (error) {
-			await flushGroupedRows();
+			const pendingRowCount = groupedRows.reduce((sum, entry) => sum + entry.rows.length, 0);
+			try {
+				await flushGroupedRows();
+			} catch (flushError) {
+				logger.warn("Failed to flush grouped rows after error; target outputs may be lost", {
+					error: flushError instanceof Error ? flushError.message : String(flushError),
+					pendingRowCount
+				});
+			}
 			throw error;
 		}
 		await flushGroupedRows();
@@ -6603,6 +7473,13 @@ var Evaluator = class {
 		if (!this.options.silent) {
 			if (serialRunEvalOptions.length > 0) logger.info(`Running ${serialRunEvalOptions.length} test cases serially...`);
 			if (concurrentRunEvalOptions.length > 0) logger.info(`Running ${concurrentRunEvalOptions.length} test cases (up to ${concurrency} at a time)...`);
+			logGroupedGradingStatus({
+				concurrency,
+				hasEvalStepTimeout,
+				runEvalOptions,
+				shouldGroupGradingByProvider,
+				usesConversationVar
+			});
 		}
 		if (this.options.showProgressBar && progressBarManager) {
 			await progressBarManager.initialize(runEvalOptions, concurrency, 0);
@@ -7606,7 +8483,7 @@ var Eval = class Eval {
 		const evalInstance = new Eval(eval_.config, {
 			id: eval_.id,
 			createdAt: new Date(eval_.createdAt),
-			author: eval_.author || void 0,
+			author: eval_.author,
 			description: eval_.description || void 0,
 			prompts: eval_.prompts || [],
 			datasetId,
@@ -7629,7 +8506,7 @@ var Eval = class Eval {
 		return (await getDb().select().from(evalsTable).limit(limit).orderBy(desc(evalsTable.createdAt)).all()).map((e) => new Eval(e.config, {
 			id: e.id,
 			createdAt: new Date(e.createdAt),
-			author: e.author || void 0,
+			author: e.author,
 			description: e.description || void 0,
 			prompts: e.prompts || [],
 			persisted: true
@@ -7644,7 +8521,7 @@ var Eval = class Eval {
 		return (await getDb().select().from(evalsTable).orderBy(desc(evalsTable.createdAt)).limit(limit).offset(offset).all()).map((e) => new Eval(e.config, {
 			id: e.id,
 			createdAt: new Date(e.createdAt),
-			author: e.author || void 0,
+			author: e.author,
 			description: e.description || void 0,
 			prompts: e.prompts || [],
 			persisted: true
@@ -7659,7 +8536,7 @@ var Eval = class Eval {
 	static async create(config, renderedPrompts, opts) {
 		const createdAt = opts?.createdAt || /* @__PURE__ */ new Date();
 		const evalId = opts?.id || createEvalId(createdAt);
-		const author = opts?.author || getUserEmail();
+		const author = opts && "author" in opts ? opts.author ?? null : getAuthor();
 		const db = getDb();
 		const datasetId = sha256(JSON.stringify(config.tests || []));
 		db.transaction(() => {
@@ -7721,7 +8598,7 @@ var Eval = class Eval {
 		});
 		return new Eval(config, {
 			id: evalId,
-			author: opts?.author,
+			author,
 			createdAt,
 			persisted: true,
 			runtimeOptions: sanitizeRuntimeOptions(opts?.runtimeOptions)
@@ -7731,7 +8608,7 @@ var Eval = class Eval {
 		const createdAt = opts?.createdAt || /* @__PURE__ */ new Date();
 		this.createdAt = createdAt.getTime();
 		this.id = opts?.id || createEvalId(createdAt);
-		this.author = opts?.author;
+		this.author = opts?.author ?? null;
 		this.config = config;
 		this.results = [];
 		this.prompts = opts?.prompts || [];
@@ -8261,7 +9138,7 @@ var Eval = class Eval {
 		newConfig.description = copyDescription;
 		const newPrompts = structuredClone(this.prompts);
 		const newVars = this.vars ? structuredClone(this.vars) : [];
-		const author = getUserEmail();
+		const author = getAuthor();
 		const db = getDb();
 		let copiedCount = 0;
 		db.transaction(() => {
@@ -8418,47 +9295,11 @@ function filterPrompts(prompts, filterPromptsOption) {
 //#endregion
 //#region src/commands/eval/filterProviders.ts
 /**
-* Checks if a value is a valid provider ID (non-empty string).
-*/
-function isValidProviderId(id) {
-	return id !== null && id !== void 0 && typeof id === "string" && id !== "";
-}
-/**
 * Extracts the id and label from a raw provider config without instantiating it.
 * Handles all provider config formats: string, function, ProviderOptions, ProviderOptionsMap.
 */
 function getProviderIdAndLabel(provider, index) {
-	if (typeof provider === "string") return { id: provider };
-	if (typeof provider === "function") {
-		const label = provider.label;
-		return {
-			id: label ?? `custom-function-${index}`,
-			label
-		};
-	}
-	const providerId = provider.id;
-	if ("id" in provider && isValidProviderId(providerId)) return {
-		id: providerId,
-		label: provider.label
-	};
-	const keys = Object.keys(provider);
-	if (keys.length > 0) {
-		const id = keys[0];
-		const value = provider[id];
-		if (typeof value === "object" && value !== null) return {
-			id: value.id || id,
-			label: value.label
-		};
-	}
-	const label = provider.label;
-	if (isValidProviderId(label)) return {
-		id: label,
-		label
-	};
-	return {
-		id: `unknown-${index}`,
-		label
-	};
+	return normalizeProviderRef(provider, { index });
 }
 /**
 * Filters raw provider configs BEFORE instantiation.
@@ -10691,25 +11532,45 @@ var AlignedHarmfulPlugin = class extends RedteamPluginBase {
 	getAssertions(_prompt) {
 		return getHarmfulAssertions(this.harmCategory);
 	}
-	promptsToTestCases(prompts) {
+	async promptsToTestCases(prompts) {
 		const hasMultipleInputs = this.config.inputs && Object.keys(this.config.inputs).length > 0;
 		const harmCategoryLabel = HARM_PLUGINS[this.harmCategory] || this.harmCategory;
-		return prompts.map(({ __prompt }) => {
+		const pluginId = getShortPluginId(this.harmCategory);
+		return Promise.all([...prompts].sort((a, b) => a.__prompt.localeCompare(b.__prompt)).map(async ({ __prompt }, materializationIndex) => {
 			const vars = { [this.injectVar]: __prompt };
-			if (hasMultipleInputs) try {
-				const parsed = JSON.parse(__prompt);
-				Object.assign(vars, extractVariablesFromJson(parsed, this.config.inputs));
-			} catch {}
+			let inputMaterialization;
+			if (hasMultipleInputs) {
+				let parsed;
+				try {
+					parsed = JSON.parse(__prompt);
+				} catch (error) {
+					logger.debug("[AlignedHarmful] Could not parse prompt as JSON for multi-input mode", { error });
+				}
+				if (parsed) try {
+					const materializedVars = await extractMaterializedVariablesFromJsonWithMetadata(parsed, this.config.inputs, {
+						materializationIndex,
+						pluginId,
+						provider: this.provider,
+						purpose: this.purpose
+					});
+					Object.assign(vars, materializedVars.vars);
+					inputMaterialization = materializedVars.metadata;
+				} catch (error) {
+					logger.debug("[AlignedHarmful] Failed to materialize prompt inputs", { error });
+					throw error;
+				}
+			}
 			return {
 				vars,
 				metadata: {
 					harmCategory: harmCategoryLabel,
-					pluginId: getShortPluginId(this.harmCategory),
-					pluginConfig: this.config
+					pluginId,
+					pluginConfig: this.config,
+					...inputMaterialization ? { inputMaterialization } : {}
 				},
 				assert: getHarmfulAssertions(this.harmCategory)
 			};
-		});
+		}));
 	}
 };
 //#endregion
@@ -10718,20 +11579,37 @@ var AlignedHarmfulPlugin = class extends RedteamPluginBase {
 * Extract content from <Prompt> tags and parse JSON if inputs are defined.
 * Returns the processed prompt and any additional vars extracted from JSON.
 */
-function processPromptForInputs(prompt, _injectVar, inputs) {
+async function processPromptForInputs(prompt, inputs, plugin, provider, purpose, materializationIndex) {
 	let processedPrompt = prompt.trim();
 	const additionalVars = {};
+	let additionalMetadata;
 	const extractedPrompt = extractPromptFromTags(processedPrompt);
 	if (extractedPrompt) processedPrompt = extractedPrompt;
-	if (inputs && Object.keys(inputs).length > 0) try {
-		const parsed = JSON.parse(processedPrompt);
-		Object.assign(additionalVars, extractVariablesFromJson(parsed, inputs));
-	} catch {
-		logger.debug("[Harmful] Could not parse prompt as JSON for multi-input mode");
+	if (inputs && Object.keys(inputs).length > 0) {
+		let parsed;
+		try {
+			parsed = JSON.parse(processedPrompt);
+		} catch (error) {
+			logger.debug("[Harmful] Could not parse prompt as JSON for multi-input mode", { error });
+		}
+		if (parsed) try {
+			const materializedVars = await extractMaterializedVariablesFromJsonWithMetadata(parsed, inputs, {
+				materializationIndex,
+				pluginId: plugin,
+				provider,
+				purpose
+			});
+			Object.assign(additionalVars, materializedVars.vars);
+			additionalMetadata = materializedVars.metadata;
+		} catch (error) {
+			logger.debug("[Harmful] Failed to materialize prompt inputs", { error });
+			throw error;
+		}
 	}
 	return {
 		processedPrompt,
-		additionalVars
+		additionalVars,
+		additionalMetadata
 	};
 }
 async function getHarmfulTests({ purpose, injectVar, n, delayMs = 0, config }, plugin) {
@@ -10752,15 +11630,19 @@ async function getHarmfulTests({ purpose, injectVar, n, delayMs = 0, config }, p
 	};
 	const allPrompts = await retryWithDeduplication(generatePrompts, n);
 	const inputs = config?.inputs;
-	return sampleArray(allPrompts, n).map((prompt) => {
-		const { processedPrompt, additionalVars } = processPromptForInputs(prompt, injectVar, inputs);
+	return Promise.all(sampleArray(allPrompts, n).map(async (prompt, materializationIndex) => {
+		const { processedPrompt, additionalVars, additionalMetadata } = await processPromptForInputs(prompt, inputs, plugin, unalignedProvider, purpose, materializationIndex);
 		const testCase = createTestCase(injectVar, processedPrompt, plugin);
 		if (Object.keys(additionalVars).length > 0) testCase.vars = {
 			...testCase.vars,
 			...additionalVars
 		};
+		if (additionalMetadata) testCase.metadata = {
+			...testCase.metadata,
+			inputMaterialization: additionalMetadata
+		};
 		return testCase;
-	});
+	}));
 }
 //#endregion
 //#region src/redteam/plugins/teenSafety/graderExamples.ts
@@ -11078,7 +11960,7 @@ const MAX_CHARS_RETRY_MODIFIER_KEY = "__maxCharsPerMessageRetry";
 function computeModifiersFromConfig(config) {
 	const modifiers = { ...config?.modifiers };
 	if (config?.language && typeof config.language === "string") modifiers.language = config.language;
-	if (config?.inputs && Object.keys(config.inputs).length > 0) modifiers.__outputFormat = `Output each test case as JSON wrapped in <Prompt> tags: <Prompt>{${Object.entries(config.inputs).map(([k, description]) => `"${k}": "${description}"`).join(", ")}}</Prompt>`;
+	if (config?.inputs && Object.keys(config.inputs).length > 0) modifiers.__outputFormat = `Output each test case as JSON wrapped in <Prompt> tags: <Prompt>{${Object.entries(buildPromptInputDescriptions(config.inputs) ?? {}).map(([k, description]) => `"${k}": "${description}"`).join(", ")}}</Prompt>`;
 	const maxCharsModifier = getMaxCharsPerMessageModifierValue(config?.maxCharsPerMessage);
 	if (maxCharsModifier) modifiers[MAX_CHARS_PER_MESSAGE_MODIFIER_KEY] = maxCharsModifier;
 	return modifiers;
@@ -11163,9 +12045,10 @@ function dedupeTestCases(testCases) {
 	return deduped;
 }
 function buildMaxCharsRetryInstructions(rejectedPromptLengths, limit) {
+	const longestRejectedPromptText = rejectedPromptLengths.length > 0 ? `${Math.max(...rejectedPromptLengths)} characters` : "unknown length";
 	return dedent`
     Your previous response included ${rejectedPromptLengths.length} generated prompt${rejectedPromptLengths.length === 1 ? "" : "s"} that exceeded the ${limit ?? "configured"}-character limit.
-    The longest rejected prompt was ${Math.max(...rejectedPromptLengths)} characters.
+    The longest rejected prompt was ${longestRejectedPromptText}.
     Generate replacement prompts only, and keep every user message within the character limit.
   `.trim();
 }
@@ -11243,6 +12126,31 @@ async function fetchRemoteTestCases(key, purpose, injectVar, n, config) {
 		return [];
 	}
 }
+async function materializeRemoteTestCaseInputs({ config, injectVar, pluginId, provider, purpose, testCases }) {
+	const inputs = config.inputs;
+	if (!inputs || Object.keys(inputs).length === 0) return testCases;
+	return Promise.all(testCases.map(async (testCase, materializationIndex) => {
+		const inputVars = extractInputVarsFromPrompt(String(testCase.vars?.[injectVar] ?? ""), inputs);
+		if (!inputVars) return testCase;
+		const materializedVars = await materializeInputVariablesWithMetadata(inputVars, inputs, {
+			materializationIndex,
+			pluginId,
+			provider,
+			purpose
+		});
+		return {
+			...testCase,
+			vars: {
+				...testCase.vars || {},
+				...materializedVars.vars
+			},
+			metadata: {
+				...testCase.metadata || {},
+				...materializedVars.metadata ? { inputMaterialization: materializedVars.metadata } : {}
+			}
+		};
+	}));
+}
 function createPluginFactory(PluginClass, key, validate) {
 	return {
 		key,
@@ -11253,13 +12161,21 @@ function createPluginFactory(PluginClass, key, validate) {
 				logger.debug(`Using local redteam generation for ${key}`);
 				return new PluginClass(provider, purpose, injectVar, configWithDefaults).generateTests(n, delayMs);
 			}
-			const testCases = await fetchRemoteTestCases(key, purpose, injectVar, n, configWithDefaults ?? {});
+			const pluginId = getShortPluginId(key);
+			const testCases = await materializeRemoteTestCaseInputs({
+				config: configWithDefaults ?? {},
+				injectVar,
+				pluginId,
+				provider,
+				purpose,
+				testCases: await fetchRemoteTestCases(key, purpose, injectVar, n, configWithDefaults ?? {})
+			});
 			const computedModifiers = computeModifiersFromConfig(configWithDefaults);
 			return testCases.map((testCase) => ({
 				...testCase,
 				metadata: {
 					...testCase.metadata,
-					pluginId: getShortPluginId(key),
+					pluginId,
 					pluginConfig: {
 						...configWithDefaults,
 						modifiers: computedModifiers
@@ -11316,7 +12232,7 @@ const pluginFactories = [
 		key: category,
 		action: async (params) => {
 			if (neverGenerateRemote()) {
-				logger.error(`${category} plugin requires remote generation to be enabled`);
+				logger.error(getRemoteGenerationExplicitlyDisabledError(`${category} plugin`));
 				return [];
 			}
 			const testCases = await getHarmfulTests(params, category);
@@ -11339,13 +12255,21 @@ const piiPlugins = PII_PLUGINS.map((category) => ({
 	key: category,
 	action: async (params) => {
 		if (shouldGenerateRemote()) {
-			const testCases = await fetchRemoteTestCases(category, params.purpose, params.injectVar, params.n, params.config ?? {});
+			const pluginId = getShortPluginId(category);
+			const testCases = await materializeRemoteTestCaseInputs({
+				config: params.config ?? {},
+				injectVar: params.injectVar,
+				pluginId,
+				provider: params.provider,
+				purpose: params.purpose,
+				testCases: await fetchRemoteTestCases(category, params.purpose, params.injectVar, params.n, params.config ?? {})
+			});
 			const computedModifiers = computeModifiersFromConfig(params.config);
 			return testCases.map((testCase) => ({
 				...testCase,
 				metadata: {
 					...testCase.metadata,
-					pluginId: getShortPluginId(category),
+					pluginId,
 					pluginConfig: {
 						...params.config,
 						modifiers: computedModifiers
@@ -11367,16 +12291,24 @@ const biasPlugins = BIAS_PLUGINS.map((category) => ({
 	key: category,
 	action: async (params) => {
 		if (neverGenerateRemote()) {
-			logger.error(`${category} plugin requires remote generation to be enabled`);
+			logger.error(getRemoteGenerationExplicitlyDisabledError(`${category} plugin`));
 			return [];
 		}
-		const testCases = await fetchRemoteTestCases(category, params.purpose, params.injectVar, params.n, params.config ?? {});
+		const pluginId = getShortPluginId(category);
+		const testCases = await materializeRemoteTestCaseInputs({
+			config: params.config ?? {},
+			injectVar: params.injectVar,
+			pluginId,
+			provider: params.provider,
+			purpose: params.purpose,
+			testCases: await fetchRemoteTestCases(category, params.purpose, params.injectVar, params.n, params.config ?? {})
+		});
 		const computedModifiers = computeModifiersFromConfig(params.config);
 		return testCases.map((testCase) => ({
 			...testCase,
 			metadata: {
 				...testCase.metadata,
-				pluginId: getShortPluginId(category),
+				pluginId,
 				pluginConfig: {
 					...params.config,
 					modifiers: computedModifiers
@@ -11389,19 +12321,27 @@ function createRemotePlugin(key, validate) {
 	return {
 		key,
 		validate,
-		action: async ({ purpose, injectVar, n, config }) => {
+		action: async ({ provider, purpose, injectVar, n, config }) => {
 			const configWithDefaults = applyDefaultRemotePluginConfig(key, config);
 			if (neverGenerateRemote()) {
-				logger.error(`${key} plugin requires remote generation to be enabled`);
+				logger.error(getRemoteGenerationExplicitlyDisabledError(`${key} plugin`));
 				return [];
 			}
-			const testCases = await fetchRemoteTestCases(key, purpose, injectVar, n, configWithDefaults ?? {});
+			const pluginId = getShortPluginId(key);
+			const testCases = await materializeRemoteTestCaseInputs({
+				config: configWithDefaults ?? {},
+				injectVar,
+				pluginId,
+				provider,
+				purpose,
+				testCases: await fetchRemoteTestCases(key, purpose, injectVar, n, configWithDefaults ?? {})
+			});
 			const computedModifiers = computeModifiersFromConfig(configWithDefaults);
 			const testsWithMetadata = testCases.map((testCase) => ({
 				...testCase,
 				metadata: {
 					...testCase.metadata,
-					pluginId: getShortPluginId(key),
+					pluginId,
 					pluginConfig: {
 						...configWithDefaults,
 						modifiers: computedModifiers
@@ -11471,6 +12411,37 @@ function getPolicyText(metadata) {
 		return typeof policyObject.text === "string" && policyObject.text.length > 0 ? policyObject.text : void 0;
 	}
 }
+async function rematerializeStrategyInputVars(testCase, injectVar, provider, purpose, materializationIndex) {
+	const inputs = testCase.metadata?.pluginConfig?.inputs;
+	const inputMaterialization = testCase.metadata?.inputMaterialization;
+	if (!inputs || Object.keys(inputs).length === 0 || !testCase.vars?.[injectVar]) return {
+		inputMaterialization,
+		vars: testCase.vars
+	};
+	try {
+		const materializedVars = await extractMaterializedVariablesFromJsonWithMetadata(JSON.parse(String(testCase.vars[injectVar])), inputs, {
+			materializationIndex,
+			pluginId: String(testCase.metadata?.pluginId || "unknown-plugin"),
+			provider,
+			purpose
+		});
+		return {
+			inputMaterialization: materializedVars.metadata ? {
+				...inputMaterialization,
+				...materializedVars.metadata
+			} : inputMaterialization,
+			vars: {
+				...testCase.vars,
+				...materializedVars.vars
+			}
+		};
+	} catch {
+		return {
+			inputMaterialization,
+			vars: testCase.vars
+		};
+	}
+}
 /**
 * Gets the severity level for a plugin based on its ID and configuration.
 * @param pluginId - The ID of the plugin.
@@ -11612,6 +12583,7 @@ const categories = {
 	foundation: FOUNDATION_PLUGINS,
 	harmful: Object.keys(HARM_PLUGINS),
 	"coding-agent:core": CODING_AGENT_CORE_PLUGINS,
+	"coding-agent:all": CODING_AGENT_PLUGINS,
 	bias: BIAS_PLUGINS,
 	pii: PII_PLUGINS,
 	medical: MEDICAL_PLUGINS,
@@ -11706,7 +12678,7 @@ function addLanguageToPluginMetadata(test, lang, plugin, maxCharsPerMessage, tes
 * @param injectVar - The variable to inject.
 * @returns An array of new test cases generated by strategies.
 */
-async function applyStrategies(testCases, strategies, injectVar, excludeTargetOutputFromAgenticAttackGeneration, maxCharsPerMessage) {
+async function applyStrategies(testCases, strategies, injectVar, provider, purpose, excludeTargetOutputFromAgenticAttackGeneration, maxCharsPerMessage) {
 	const newTestCases = [];
 	const strategyResults = {};
 	for (const strategy of strategies) {
@@ -11762,14 +12734,8 @@ async function applyStrategies(testCases, strategies, injectVar, excludeTargetOu
 			}
 		}
 		resultTestCases = filterOversizedTestCases(resultTestCases, injectVar, `Strategy ${strategy.id}`, maxCharsPerMessage);
-		newTestCases.push(...resultTestCases.map((t) => {
-			const inputs = t?.metadata?.pluginConfig?.inputs;
-			let updatedVars = t.vars;
-			if (inputs && Object.keys(inputs).length > 0 && t.vars?.[injectVar]) try {
-				const parsed = JSON.parse(String(t.vars[injectVar]));
-				updatedVars = { ...t.vars };
-				Object.assign(updatedVars, extractVariablesFromJson(parsed, inputs));
-			} catch {}
+		newTestCases.push(...await Promise.all(resultTestCases.map(async (t, materializationIndex) => {
+			const { inputMaterialization, vars } = await rematerializeStrategyInputVars(t, injectVar, provider, purpose, materializationIndex);
 			const strategyConfig = {
 				...strategy.config || {},
 				...maxCharsPerMessage ? { maxCharsPerMessage } : {},
@@ -11777,16 +12743,17 @@ async function applyStrategies(testCases, strategies, injectVar, excludeTargetOu
 			};
 			return {
 				...t,
-				vars: updatedVars,
+				vars,
 				metadata: {
 					...t?.metadata || {},
 					...strategy.id !== "retry" && { strategyId: t?.metadata?.strategyId || strategy.id },
 					...t?.metadata?.pluginId && { pluginId: t.metadata.pluginId },
 					...t?.metadata?.pluginConfig && { pluginConfig: t.metadata.pluginConfig },
+					...inputMaterialization && { inputMaterialization },
 					...Object.keys(strategyConfig).length > 0 && { strategyConfig }
 				}
 			};
-		}));
+		})));
 		const displayId = strategy.id === "layer" && Array.isArray(strategy.config?.steps) ? `layer(${strategy.config.steps.map((st) => typeof st === "string" ? st : st.id).join("→")})` : strategy.id;
 		const languagesInResults = new Set(strategyTestCases.map((t) => getLanguageForTestCase(t)).filter((lang) => lang !== void 0));
 		const applyNumTestsCap = (calculatedRequested) => {
@@ -12241,7 +13208,7 @@ async function synthesize({ abortSignal, delay, entities: entitiesOverride, inje
 			targetIds,
 			...retryStrategy.config
 		};
-		const { testCases: retryTestCases, strategyResults: retryResults } = await applyStrategies(pluginTestCases, [retryStrategy], injectVar, void 0, maxCharsPerMessage);
+		const { testCases: retryTestCases, strategyResults: retryResults } = await applyStrategies(pluginTestCases, [retryStrategy], injectVar, redteamProvider, purpose, void 0, maxCharsPerMessage);
 		pluginTestCases.push(...retryTestCases);
 		Object.assign(strategyResults, retryResults);
 		if (showProgressBar) progressBar?.increment(retryTestCases.length);
@@ -12249,7 +13216,7 @@ async function synthesize({ abortSignal, delay, entities: entitiesOverride, inje
 	checkAbort();
 	const nonBasicStrategies = strategies.filter((s) => !["basic", "retry"].includes(s.id));
 	if (showProgressBar && nonBasicStrategies.length > 0) progressBar?.update({ task: "Applying strategies" });
-	const { testCases: strategyTestCases, strategyResults: otherStrategyResults } = await applyStrategies(pluginTestCases, nonBasicStrategies, injectVar, excludeTargetOutputFromAgenticAttackGeneration, maxCharsPerMessage);
+	const { testCases: strategyTestCases, strategyResults: otherStrategyResults } = await applyStrategies(pluginTestCases, nonBasicStrategies, injectVar, redteamProvider, purpose, excludeTargetOutputFromAgenticAttackGeneration, maxCharsPerMessage);
 	Object.assign(strategyResults, otherStrategyResults);
 	if (showProgressBar && strategyTestCases.length > 0) progressBar?.increment(strategyTestCases.length);
 	const finalTestCases = [...includeBasicTests ? pluginTestCases : [], ...strategyTestCases];
@@ -13142,6 +14109,10 @@ function stripAuthFromUrl(urlString) {
 	}
 }
 async function handleEmailCollection(evalRecord) {
+	if (evalRecord.author) {
+		logger.debug(`[Share] Skipping email collection because author is already set`, { evalId: evalRecord.id });
+		return;
+	}
 	if (!process.stdout.isTTY || isCI() || getEnvBool("PROMPTFOO_DISABLE_SHARE_EMAIL_REQUEST")) return;
 	let email = getUserEmail();
 	if (!email) {
@@ -13214,7 +14185,7 @@ function generateTable(evaluateTable, tableCellMaxLength = 250, maxRows = 25) {
 	for (const row of evaluateTable.body.slice(0, maxRows)) table.push([...row.vars.map((v) => ellipsize(v, tableCellMaxLength)), ...row.outputs.map(({ pass, text, failureReason: failureType }) => {
 		text = ellipsize(text, tableCellMaxLength);
 		if (pass) return chalk.green("[PASS] ") + text;
-		else return chalk.red(failureType === ResultFailureReason.ASSERT ? "[FAIL] " : "[ERROR] ") + text.split("---").map((c, idx) => idx === 0 ? chalk.red.bold(c) : c).join("---");
+		return chalk.red(failureType === ResultFailureReason.ASSERT ? "[FAIL] " : "[ERROR] ") + text.split("---").map((c, idx) => idx === 0 ? chalk.red.bold(c) : c).join("---");
 	})]);
 	return table.toString();
 }
@@ -13285,6 +14256,14 @@ function shouldShareResults(opts) {
 	return cloudConfig.isEnabled() && sharing !== false;
 }
 //#endregion
+//#region src/commands/eval/redteamWarning.ts
+function warnIfRedteamConfigHasNoTests(config, testSuite) {
+	if (config.redteam && (!testSuite.tests || testSuite.tests.length === 0) && (!testSuite.scenarios || testSuite.scenarios.length === 0)) logger.warn(chalk.yellow(dedent`
+        Warning: Config file has a redteam section but no test cases.
+        Did you mean to run ${chalk.bold("promptfoo redteam generate")} instead?
+        `));
+}
+//#endregion
 //#region src/util/formatDuration.ts
 /**
 * Formats a duration in seconds into a human-readable string
@@ -13305,6 +14284,115 @@ function formatDuration(seconds) {
 }
 //#endregion
 //#region src/commands/eval/summary.ts
+function getCompletionMessage({ completionType, evalId, shareableUrl, wasAborted, writeToDatabase, activelySharing }) {
+	if (wasAborted) {
+		const idSuffix = writeToDatabase ? ` (ID: ${chalk.cyan(evalId)})` : "";
+		return `${chalk.red("✗")} ${completionType} aborted${idSuffix}`;
+	}
+	if (writeToDatabase && shareableUrl) return `${chalk.green("✓")} ${completionType} complete: ${shareableUrl}`;
+	if (writeToDatabase && activelySharing) return `${chalk.green("✓")} ${completionType} complete`;
+	if (writeToDatabase) return `${chalk.green("✓")} ${completionType} complete (ID: ${chalk.cyan(evalId)})`;
+	return `${chalk.green("✓")} ${completionType} complete`;
+}
+function getAbortSummaryLines(targetErrorStatus) {
+	if (targetErrorStatus == null) return [];
+	return [
+		"",
+		chalk.red.bold("Scan stopped: Target is unavailable and will not recover on retry."),
+		chalk.red(`  Target returned HTTP ${targetErrorStatus}`),
+		"",
+		chalk.yellow("Possible causes:"),
+		chalk.yellow("  • Invalid API key or authentication (401/403)"),
+		chalk.yellow("  • Target endpoint does not exist (404)"),
+		chalk.yellow("  • Server does not support the request (501)"),
+		"",
+		chalk.cyan("To fix: Check your target configuration and credentials.")
+	];
+}
+function getGuidanceLines({ writeToDatabase, shareableUrl, wantsToShare, activelySharing, hasExplicitDisable, cloudEnabled }) {
+	if (!writeToDatabase || shareableUrl || wantsToShare || activelySharing) return [];
+	const lines = ["", `» View results: ${chalk.green.bold("promptfoo view")}`];
+	if (!hasExplicitDisable) lines.push(cloudEnabled ? `» Create shareable URL: ${chalk.green.bold("promptfoo share")}` : `» Share with your team: ${chalk.green.bold("https://promptfoo.app")}`);
+	lines.push(`» Feedback: ${chalk.green.bold("https://promptfoo.dev/feedback")}`);
+	return lines;
+}
+function buildUsageDetails(usage, total) {
+	const parts = [];
+	if (usage.prompt && usage.prompt > 0) parts.push(`${usage.prompt.toLocaleString()} prompt`);
+	if (usage.completion && usage.completion > 0) parts.push(`${usage.completion.toLocaleString()} completion`);
+	if (usage.cached && usage.cached > 0) parts.push(usage.cached === total && parts.length === 0 ? "cached" : `${usage.cached.toLocaleString()} cached`);
+	if (usage.completionDetails?.reasoning && usage.completionDetails.reasoning > 0) parts.push(`${usage.completionDetails.reasoning.toLocaleString()} reasoning`);
+	return parts;
+}
+function getTokenUsageLines(tokenUsage, isRedteam, tracker) {
+	const hasEvalTokens = (tokenUsage.total || 0) > 0 || (tokenUsage.prompt || 0) + (tokenUsage.completion || 0) > 0;
+	const hasGradingTokens = tokenUsage.assertions && (tokenUsage.assertions.total || 0) > 0;
+	if (!hasEvalTokens && !hasGradingTokens) return [];
+	const combinedTotal = (tokenUsage.prompt || 0) + (tokenUsage.completion || 0);
+	const evalTokens = {
+		prompt: tokenUsage.prompt || 0,
+		completion: tokenUsage.completion || 0,
+		total: tokenUsage.total || combinedTotal,
+		cached: tokenUsage.cached || 0,
+		numRequests: tokenUsage.numRequests || 0,
+		completionDetails: tokenUsage.completionDetails || {
+			reasoning: 0,
+			acceptedPrediction: 0,
+			rejectedPrediction: 0
+		}
+	};
+	const lines = [`${chalk.bold("Total Tokens:")} ${chalk.white.bold((evalTokens.total + (tokenUsage.assertions?.total || 0)).toLocaleString())}`];
+	if (isRedteam && tokenUsage.numRequests) lines.push(`  ${chalk.gray("Probes:")} ${chalk.white(tokenUsage.numRequests.toLocaleString())}`);
+	if (evalTokens.total > 0) {
+		const evalParts = buildUsageDetails(evalTokens, evalTokens.total);
+		lines.push(`  ${chalk.gray("Eval:")} ${chalk.white(evalTokens.total.toLocaleString())} (${evalParts.join(", ")})`);
+	}
+	if (tokenUsage.assertions?.total && tokenUsage.assertions.total > 0) {
+		const gradingParts = buildUsageDetails(tokenUsage.assertions, tokenUsage.assertions.total);
+		lines.push(`  ${chalk.gray("Grading:")} ${chalk.white(tokenUsage.assertions.total.toLocaleString())} (${gradingParts.join(", ")})`);
+	}
+	lines.push(...getProviderUsageLines(tracker));
+	return lines;
+}
+function getProviderUsageLines(tracker) {
+	const providerIds = tracker.getProviderIds();
+	if (providerIds.length <= 1) return [];
+	const sortedProviders = providerIds.map((id) => ({
+		id,
+		usage: tracker.getProviderUsage(id)
+	})).filter((p) => p.usage != null).sort((a, b) => (b.usage.total || 0) - (a.usage.total || 0));
+	const lines = ["", chalk.bold("Providers:")];
+	for (const { id, usage } of sortedProviders) {
+		if ((usage.total || 0) === 0 && (usage.prompt || 0) + (usage.completion || 0) === 0) continue;
+		const displayTotal = usage.total || (usage.prompt || 0) + (usage.completion || 0);
+		const displayId = id.includes(" (") ? id.substring(0, id.indexOf(" (")) : id;
+		const details = buildUsageDetails(usage, displayTotal);
+		const requestInfo = `${usage.numRequests || 0} requests`;
+		const separator = details.length > 0 ? "; " : "";
+		lines.push(`  ${chalk.gray(`${displayId}:`)} ${chalk.white(displayTotal.toLocaleString())} (${requestInfo}${separator}${details.join(", ")})`);
+	}
+	return lines;
+}
+function formatResultPercentage(count, totalTests) {
+	const percentage = totalTests === 0 ? 0 : count / totalTests * 100;
+	return percentage === 0 || percentage === 100 ? `${percentage.toFixed(0)}%` : `${percentage.toFixed(2)}%`;
+}
+function formatResultLine(count, label, icon, iconColor, totalTests) {
+	return `  ${icon ? `${iconColor(icon)} ` : ""}${chalk.white.bold(count.toLocaleString())} ${chalk.white(label)} ${chalk.gray(`(${formatResultPercentage(count, totalTests)})`)}`;
+}
+function getResultsLines({ successes, failures, errors, duration, maxConcurrency }) {
+	const totalTests = successes + failures + errors;
+	const errorLabel = errors === 1 ? "error" : "errors";
+	return [
+		"",
+		chalk.bold("Results:"),
+		formatResultLine(successes, "passed", successes > 0 ? "✓" : void 0, chalk.green, totalTests),
+		formatResultLine(failures, "failed", failures > 0 ? "✗" : void 0, chalk.red, totalTests),
+		formatResultLine(errors, errorLabel, errors > 0 ? "✗" : void 0, chalk.red, totalTests),
+		chalk.gray(`Duration: ${formatDuration(duration)} (concurrency: ${maxConcurrency})`),
+		""
+	];
+}
 /**
 * Generate formatted evaluation summary output for CLI display.
 *
@@ -13343,115 +14431,28 @@ function formatDuration(seconds) {
 * ```
 */
 function generateEvalSummary(params) {
-	const { evalId, isRedteam, writeToDatabase, shareableUrl, wantsToShare, hasExplicitDisable, cloudEnabled, activelySharing = false, tokenUsage, successes, failures, errors, duration, maxConcurrency, tracker, targetErrorStatus } = params;
-	const lines = [];
-	const completionType = isRedteam ? "Red team" : "Eval";
-	const wasAborted = targetErrorStatus != null;
-	let completionMessage;
-	if (wasAborted) {
-		completionMessage = `${chalk.red("✗")} ${completionType} aborted`;
-		if (writeToDatabase) completionMessage += ` (ID: ${chalk.cyan(evalId)})`;
-	} else if (writeToDatabase && shareableUrl) completionMessage = `${chalk.green("✓")} ${completionType} complete: ${shareableUrl}`;
-	else if (writeToDatabase && activelySharing) completionMessage = `${chalk.green("✓")} ${completionType} complete`;
-	else if (writeToDatabase) completionMessage = `${chalk.green("✓")} ${completionType} complete (ID: ${chalk.cyan(evalId)})`;
-	else completionMessage = `${chalk.green("✓")} ${completionType} complete`;
-	lines.push(completionMessage);
-	if (wasAborted && targetErrorStatus != null) {
-		lines.push("");
-		lines.push(chalk.red.bold("Scan stopped: Target is unavailable and will not recover on retry."));
-		lines.push(chalk.red(`  Target returned HTTP ${targetErrorStatus}`));
-		lines.push("");
-		lines.push(chalk.yellow("Possible causes:"));
-		lines.push(chalk.yellow("  • Invalid API key or authentication (401/403)"));
-		lines.push(chalk.yellow("  • Target endpoint does not exist (404)"));
-		lines.push(chalk.yellow("  • Server does not support the request (501)"));
-		lines.push("");
-		lines.push(chalk.cyan("To fix: Check your target configuration and credentials."));
-	}
-	if (writeToDatabase && !shareableUrl && !wantsToShare && !activelySharing) {
-		lines.push("");
-		lines.push(`» View results: ${chalk.green.bold("promptfoo view")}`);
-		if (!hasExplicitDisable) if (cloudEnabled) lines.push(`» Create shareable URL: ${chalk.green.bold("promptfoo share")}`);
-		else lines.push(`» Share with your team: ${chalk.green.bold("https://promptfoo.app")}`);
-		lines.push(`» Feedback: ${chalk.green.bold("https://promptfoo.dev/feedback")}`);
-	}
-	lines.push("");
-	const hasEvalTokens = (tokenUsage.total || 0) > 0 || (tokenUsage.prompt || 0) + (tokenUsage.completion || 0) > 0;
-	const hasGradingTokens = tokenUsage.assertions && (tokenUsage.assertions.total || 0) > 0;
-	if (hasEvalTokens || hasGradingTokens) {
-		const combinedTotal = (tokenUsage.prompt || 0) + (tokenUsage.completion || 0);
-		const evalTokens = {
-			prompt: tokenUsage.prompt || 0,
-			completion: tokenUsage.completion || 0,
-			total: tokenUsage.total || combinedTotal,
-			cached: tokenUsage.cached || 0,
-			completionDetails: tokenUsage.completionDetails || {
-				reasoning: 0,
-				acceptedPrediction: 0,
-				rejectedPrediction: 0
-			}
-		};
-		const grandTotal = evalTokens.total + (tokenUsage.assertions?.total || 0);
-		lines.push(`${chalk.bold("Total Tokens:")} ${chalk.white.bold(grandTotal.toLocaleString())}`);
-		if (isRedteam && tokenUsage.numRequests) lines.push(`  ${chalk.gray("Probes:")} ${chalk.white(tokenUsage.numRequests.toLocaleString())}`);
-		if (evalTokens.total > 0) {
-			const evalParts = [];
-			if (evalTokens.prompt > 0) evalParts.push(`${evalTokens.prompt.toLocaleString()} prompt`);
-			if (evalTokens.completion > 0) evalParts.push(`${evalTokens.completion.toLocaleString()} completion`);
-			if (evalTokens.cached > 0) if (evalTokens.cached === evalTokens.total && evalParts.length === 0) evalParts.push("cached");
-			else evalParts.push(`${evalTokens.cached.toLocaleString()} cached`);
-			if (evalTokens.completionDetails?.reasoning && evalTokens.completionDetails.reasoning > 0) evalParts.push(`${evalTokens.completionDetails.reasoning.toLocaleString()} reasoning`);
-			lines.push(`  ${chalk.gray("Eval:")} ${chalk.white(evalTokens.total.toLocaleString())} (${evalParts.join(", ")})`);
-		}
-		if (tokenUsage.assertions && tokenUsage.assertions.total && tokenUsage.assertions.total > 0) {
-			const gradingParts = [];
-			if (tokenUsage.assertions.prompt && tokenUsage.assertions.prompt > 0) gradingParts.push(`${tokenUsage.assertions.prompt.toLocaleString()} prompt`);
-			if (tokenUsage.assertions.completion && tokenUsage.assertions.completion > 0) gradingParts.push(`${tokenUsage.assertions.completion.toLocaleString()} completion`);
-			if (tokenUsage.assertions.cached && tokenUsage.assertions.cached > 0) if (tokenUsage.assertions.cached === tokenUsage.assertions.total && gradingParts.length === 0) gradingParts.push("cached");
-			else gradingParts.push(`${tokenUsage.assertions.cached.toLocaleString()} cached`);
-			if (tokenUsage.assertions.completionDetails?.reasoning && tokenUsage.assertions.completionDetails.reasoning > 0) gradingParts.push(`${tokenUsage.assertions.completionDetails.reasoning.toLocaleString()} reasoning`);
-			lines.push(`  ${chalk.gray("Grading:")} ${chalk.white(tokenUsage.assertions.total.toLocaleString())} (${gradingParts.join(", ")})`);
-		}
-		const providerIds = tracker.getProviderIds();
-		if (providerIds.length > 1) {
-			lines.push("");
-			lines.push(chalk.bold("Providers:"));
-			const sortedProviders = providerIds.map((id) => ({
-				id,
-				usage: tracker.getProviderUsage(id)
-			})).filter((p) => p.usage != null).sort((a, b) => (b.usage.total || 0) - (a.usage.total || 0));
-			for (const { id, usage } of sortedProviders) if ((usage.total || 0) > 0 || (usage.prompt || 0) + (usage.completion || 0) > 0) {
-				const displayTotal = usage.total || (usage.prompt || 0) + (usage.completion || 0);
-				const displayId = id.includes(" (") ? id.substring(0, id.indexOf(" (")) : id;
-				const details = [];
-				if (usage.prompt && usage.prompt > 0) details.push(`${usage.prompt.toLocaleString()} prompt`);
-				if (usage.completion && usage.completion > 0) details.push(`${usage.completion.toLocaleString()} completion`);
-				if (usage.cached && usage.cached > 0) if (usage.cached === displayTotal && details.length === 0) details.push("cached");
-				else details.push(`${usage.cached.toLocaleString()} cached`);
-				if (usage.completionDetails?.reasoning && usage.completionDetails.reasoning > 0) details.push(`${usage.completionDetails.reasoning.toLocaleString()} reasoning`);
-				const breakdown = ` (${`${usage.numRequests || 0} requests`}${details.length > 0 ? "; " : ""}${details.join(", ")})`;
-				lines.push(`  ${chalk.gray(displayId + ":")} ${chalk.white(displayTotal.toLocaleString())}${breakdown}`);
-			}
-		}
-	}
-	lines.push("");
-	const totalTests = successes + failures + errors;
-	const formatResultPercentage = (count) => {
-		const percentage = totalTests === 0 ? 0 : count / totalTests * 100;
-		return percentage === 0 || percentage === 100 ? `${percentage.toFixed(0)}%` : `${percentage.toFixed(2)}%`;
-	};
-	const formatResultLine = (count, label, icon, iconColor) => {
-		return `  ${icon ? `${iconColor(icon)} ` : ""}${chalk.white.bold(count.toLocaleString())} ${chalk.white(label)} ${chalk.gray(`(${formatResultPercentage(count)})`)}`;
-	};
-	const errorLabel = errors === 1 ? "error" : "errors";
-	lines.push(chalk.bold("Results:"));
-	lines.push(formatResultLine(successes, "passed", successes > 0 ? "✓" : void 0, chalk.green));
-	lines.push(formatResultLine(failures, "failed", failures > 0 ? "✗" : void 0, chalk.red));
-	lines.push(formatResultLine(errors, errorLabel, errors > 0 ? "✗" : void 0, chalk.red));
-	const durationDisplay = formatDuration(duration);
-	lines.push(chalk.gray(`Duration: ${durationDisplay} (concurrency: ${maxConcurrency})`));
-	lines.push("");
-	return lines;
+	return [
+		getCompletionMessage({
+			completionType: params.isRedteam ? "Red team" : "Eval",
+			evalId: params.evalId,
+			shareableUrl: params.shareableUrl,
+			wasAborted: params.targetErrorStatus != null,
+			writeToDatabase: params.writeToDatabase,
+			activelySharing: params.activelySharing ?? false
+		}),
+		...getAbortSummaryLines(params.targetErrorStatus),
+		...getGuidanceLines({
+			writeToDatabase: params.writeToDatabase,
+			shareableUrl: params.shareableUrl,
+			wantsToShare: params.wantsToShare,
+			activelySharing: params.activelySharing ?? false,
+			hasExplicitDisable: params.hasExplicitDisable,
+			cloudEnabled: params.cloudEnabled
+		}),
+		"",
+		...getTokenUsageLines(params.tokenUsage, params.isRedteam, params.tracker),
+		...getResultsLines(params)
+	];
 }
 //#endregion
 //#region src/commands/retry.ts
@@ -13703,14 +14704,11 @@ async function doEval(cmdObj, defaultConfig, defaultConfigPath, evaluateOptions)
 			state.resume = true;
 			state.retryMode = true;
 		} else ({config, testSuite, basePath: _basePath, commandLineOptions} = await resolveConfigs(cmdObj, defaultConfig));
-		if (!cmdObj.envPath && commandLineOptions?.envPath) {
+		if ((!cmdObj.envPath || cmdObj.envPath.length === 0) && commandLineOptions?.envPath) {
 			logger.debug(`Loading additional environment from config: ${commandLineOptions.envPath}`);
 			setupEnv(commandLineOptions.envPath);
 		}
-		if (config.redteam && (!testSuite.tests || testSuite.tests.length === 0) && (!testSuite.scenarios || testSuite.scenarios.length === 0)) logger.warn(chalk.yellow(dedent`
-        Warning: Config file has a redteam section but no test cases.
-        Did you mean to run ${chalk.bold("promptfoo redteam generate")} instead?
-        `));
+		warnIfRedteamConfigHasNoTests(config, testSuite);
 		if (config.redteam && Array.isArray(config.providers) && config.providers.length > 0 && typeof config.providers[0] === "object" && config.providers[0].id === "http") {
 			const maybeUrl = config.providers[0]?.config?.url;
 			if (typeof maybeUrl === "string" && maybeUrl.includes("promptfoo.app")) telemetry.record("feature_used", { feature: "redteam_run_with_example" });
@@ -13818,7 +14816,14 @@ async function doEval(cmdObj, defaultConfig, defaultConfigPath, evaluateOptions)
         ${z.prettifyError(testSuiteSchema.error)}
 
       Please review your promptfooconfig.yaml configuration.`));
-		const evalRecord = resumeEval ? resumeEval : cmdObj.write ? await Eval.create(config, testSuite.prompts, { runtimeOptions: options }) : new Eval(config, { runtimeOptions: options });
+		const author = getAuthor();
+		const evalRecord = resumeEval ? resumeEval : cmdObj.write ? await Eval.create(config, testSuite.prompts, {
+			author,
+			runtimeOptions: options
+		}) : new Eval(config, {
+			author,
+			runtimeOptions: options
+		});
 		const abortController = new AbortController();
 		const previousAbortSignal = evaluateOptions.abortSignal;
 		evaluateOptions.abortSignal = previousAbortSignal ? AbortSignal.any([previousAbortSignal, abortController.signal]) : abortController.signal;
@@ -14220,65 +15225,175 @@ async function doRedteamRun(options) {
 	return evalResult;
 }
 //#endregion
+//#region src/types/transform.ts
+/** Runtime type guard for `TransformFunction` values. */
+function isTransformFunction(value) {
+	return typeof value === "function";
+}
+//#endregion
 //#region src/index.ts
+/**
+* Shallow-clone a test case so the caller can swap in resolved ApiProvider
+* instances on `options.provider` / `assert[].provider` without leaking those
+* mutations back to the input. The input may alias the unified config written
+* to the Eval record, and a live SDK client (e.g. Bedrock's BedrockRuntime,
+* Anthropic's client) holds circular references that break drizzle's JSON
+* serialization on `evalRecord.save()`. Fixes #8687.
+*
+* Detaches only `options` and `assert[]`. Other reference fields (`provider`,
+* `vars`, `metadata`, `providerOutput`) remain aliased — callers must reassign
+* those by reference rather than mutating in place. `assert-set` children are
+* not deep-cloned because the resolve loop skips `assert-set`; if that ever
+* changes, extend this helper.
+*/
+function cloneTestForResolve(test) {
+	const cloned = { ...test };
+	if (test.options) cloned.options = { ...test.options };
+	if (test.assert) cloned.assert = test.assert.map((assertion) => ({ ...assertion }));
+	return cloned;
+}
+function toSerializableProviderRef(provider) {
+	if (isApiProvider(provider)) return sanitizeProvider(provider);
+	if (Array.isArray(provider)) return provider.map(toSerializableProviderRef);
+	return provider;
+}
+function isRecord(value) {
+	return Boolean(value && typeof value === "object" && !Array.isArray(value));
+}
+function withSerializableProvider(record) {
+	if (!isApiProvider(record.provider)) return record;
+	return {
+		...record,
+		provider: sanitizeProvider(record.provider)
+	};
+}
+/**
+* Function-valued transforms are first-class at runtime but are silently dropped
+* by `JSON.stringify`. Persisted eval configs (drizzle-stored) must never retain
+* a function reference, so replace every `transform`-like field with a
+* `[inline function]: name` marker. Non-function values pass through unchanged.
+*
+* `droppedRef.value` is flipped to `true` the first time a function is replaced
+* so the caller can emit a single warning instead of logging per field.
+*/
+function replaceFunctionTransforms(record, droppedRef) {
+	let result;
+	for (const key of TRANSFORM_KEYS) {
+		const value = record[key];
+		if (!isTransformFunction(value)) continue;
+		if (!result) result = { ...record };
+		result[key] = value.name ? `${INLINE_FUNCTION_LABEL}: ${value.name}` : INLINE_FUNCTION_LABEL;
+		droppedRef.value = true;
+	}
+	return result ?? record;
+}
+function toSerializableAssertion(assertion, droppedRef) {
+	if (!isRecord(assertion)) return assertion;
+	let sanitizedAssertion = withSerializableProvider(assertion);
+	sanitizedAssertion = replaceFunctionTransforms(sanitizedAssertion, droppedRef);
+	if (Array.isArray(assertion.assert)) sanitizedAssertion = {
+		...sanitizedAssertion,
+		assert: assertion.assert.map((a) => toSerializableAssertion(a, droppedRef))
+	};
+	return sanitizedAssertion;
+}
+function toSerializableTestCase(test, droppedRef) {
+	if (!isRecord(test)) return test;
+	let sanitizedTest = withSerializableProvider(test);
+	if (isRecord(test.options)) {
+		let options = withSerializableProvider(test.options);
+		options = replaceFunctionTransforms(options, droppedRef);
+		if (options !== test.options) sanitizedTest = {
+			...sanitizedTest,
+			options
+		};
+	}
+	if (Array.isArray(test.assert)) sanitizedTest = {
+		...sanitizedTest,
+		assert: test.assert.map((a) => toSerializableAssertion(a, droppedRef))
+	};
+	return sanitizedTest;
+}
+function toSerializableScenario(scenario, droppedRef) {
+	if (!isRecord(scenario)) return scenario;
+	if (!Array.isArray(scenario.tests)) return scenario;
+	return {
+		...scenario,
+		tests: scenario.tests.map((t) => toSerializableTestCase(t, droppedRef))
+	};
+}
+function createSerializableUnifiedConfig(testSuite, prompts) {
+	const droppedRef = { value: false };
+	const config = {
+		...testSuite,
+		providers: toSerializableProviderRef(testSuite.providers),
+		defaultTest: toSerializableTestCase(testSuite.defaultTest, droppedRef),
+		tests: Array.isArray(testSuite.tests) ? testSuite.tests.map((t) => toSerializableTestCase(t, droppedRef)) : testSuite.tests,
+		scenarios: Array.isArray(testSuite.scenarios) ? testSuite.scenarios.map((s) => toSerializableScenario(s, droppedRef)) : testSuite.scenarios,
+		prompts
+	};
+	if (droppedRef.value && testSuite.writeLatestResults) logger.warn("Function-valued transform(s) in testSuite were replaced with \"[inline function]\" markers in the persisted config. Re-running the saved eval will not invoke them; use string expressions or file:// references if you need the config to round-trip.");
+	return config;
+}
 async function evaluate(testSuite, options = {}) {
-	if (testSuite.writeLatestResults) await runDbMigrations();
-	const loadedProviders = await loadApiProviders(testSuite.providers, { env: testSuite.env });
+	const { author: suiteAuthor, ...testSuiteConfig } = testSuite;
+	if (testSuiteConfig.writeLatestResults) await runDbMigrations();
+	const loadedProviders = await loadApiProviders(testSuiteConfig.providers, { env: testSuiteConfig.env });
 	const providerMap = {};
 	for (const p of loadedProviders) {
 		providerMap[p.id()] = p;
 		if (p.label) providerMap[p.label] = p;
 	}
-	let resolvedDefaultTest = testSuite.defaultTest;
-	if (typeof testSuite.defaultTest === "string" && testSuite.defaultTest.startsWith("file://")) resolvedDefaultTest = await maybeLoadFromExternalFile(testSuite.defaultTest);
+	let resolvedDefaultTest = testSuiteConfig.defaultTest;
+	if (typeof testSuiteConfig.defaultTest === "string" && testSuiteConfig.defaultTest.startsWith("file://")) resolvedDefaultTest = await maybeLoadFromExternalFile(testSuiteConfig.defaultTest);
 	const constructedTestSuite = {
-		...testSuite,
+		...testSuiteConfig,
 		defaultTest: resolvedDefaultTest,
-		scenarios: testSuite.scenarios,
+		scenarios: testSuiteConfig.scenarios,
 		providers: loadedProviders,
-		tests: await readTests(testSuite.tests),
-		nunjucksFilters: await readFilters(testSuite.nunjucksFilters || {}),
-		prompts: await processPrompts(testSuite.prompts)
-	};
-	if (typeof constructedTestSuite.defaultTest === "object") {
-		if (constructedTestSuite.defaultTest?.provider && !isApiProvider(constructedTestSuite.defaultTest.provider)) constructedTestSuite.defaultTest.provider = await resolveProvider(constructedTestSuite.defaultTest.provider, providerMap, {
-			env: testSuite.env,
+		tests: await readTests(testSuiteConfig.tests),
+		nunjucksFilters: await readFilters(testSuiteConfig.nunjucksFilters || {}),
+		prompts: await processPrompts(testSuiteConfig.prompts)
+	};
+	if (typeof constructedTestSuite.defaultTest === "object" && constructedTestSuite.defaultTest) {
+		constructedTestSuite.defaultTest = cloneTestForResolve(constructedTestSuite.defaultTest);
+		if (constructedTestSuite.defaultTest.provider && !isApiProvider(constructedTestSuite.defaultTest.provider)) constructedTestSuite.defaultTest.provider = await resolveProvider(constructedTestSuite.defaultTest.provider, providerMap, {
+			env: testSuiteConfig.env,
 			basePath: state.basePath
 		});
-		if (constructedTestSuite.defaultTest?.options?.provider && !isApiProvider(constructedTestSuite.defaultTest.options.provider)) constructedTestSuite.defaultTest.options.provider = await resolveProvider(constructedTestSuite.defaultTest.options.provider, providerMap, {
-			env: testSuite.env,
+		if (constructedTestSuite.defaultTest.options?.provider && !isApiProvider(constructedTestSuite.defaultTest.options.provider)) constructedTestSuite.defaultTest.options.provider = await resolveProvider(constructedTestSuite.defaultTest.options.provider, providerMap, {
+			env: testSuiteConfig.env,
 			basePath: state.basePath
 		});
 	}
-	for (const test of constructedTestSuite.tests || []) {
+	constructedTestSuite.tests = (constructedTestSuite.tests || []).map(cloneTestForResolve);
+	for (const test of constructedTestSuite.tests) {
 		if (test.options?.provider && !isApiProvider(test.options.provider)) test.options.provider = await resolveProvider(test.options.provider, providerMap, {
-			env: testSuite.env,
+			env: testSuiteConfig.env,
 			basePath: state.basePath
 		});
-		if (test.assert) for (const assertion of test.assert) {
+		for (const assertion of test.assert || []) {
 			if (assertion.type === "assert-set" || typeof assertion.provider === "function") continue;
 			if (assertion.provider && !isApiProvider(assertion.provider)) assertion.provider = await resolveProvider(assertion.provider, providerMap, {
-				env: testSuite.env,
+				env: testSuiteConfig.env,
 				basePath: state.basePath
 			});
 		}
 	}
 	if (options.cache === false) disableCache();
-	const parsedProviderPromptMap = readProviderPromptMap(testSuite, constructedTestSuite.prompts);
-	const unifiedConfig = {
-		...testSuite,
-		prompts: constructedTestSuite.prompts
-	};
-	const evalRecord = testSuite.writeLatestResults ? await Eval.create(unifiedConfig, constructedTestSuite.prompts) : new Eval(unifiedConfig);
+	const parsedProviderPromptMap = readProviderPromptMap(testSuiteConfig, constructedTestSuite.prompts);
+	const unifiedConfig = createSerializableUnifiedConfig(testSuiteConfig, constructedTestSuite.prompts);
+	const author = getAuthor(suiteAuthor);
+	const evalRecord = testSuiteConfig.writeLatestResults ? await Eval.create(unifiedConfig, constructedTestSuite.prompts, { author }) : new Eval(unifiedConfig, { author });
 	const ret = await evaluate$1({
 		...constructedTestSuite,
 		providerPromptMap: parsedProviderPromptMap
 	}, evalRecord, {
 		eventSource: "library",
-		isRedteam: Boolean(testSuite.redteam),
+		isRedteam: Boolean(testSuiteConfig.redteam),
 		...options
 	});
-	if (testSuite.writeLatestResults && testSuite.sharing) if (isSharingEnabled(ret)) try {
+	if (testSuiteConfig.writeLatestResults && testSuiteConfig.sharing) if (isSharingEnabled(ret)) try {
 		const shareableUrl = await createShareableUrl(ret, { silent: true });
 		if (shareableUrl) {
 			ret.shareableUrl = shareableUrl;
@@ -14289,9 +15404,9 @@ async function evaluate(testSuite, options = {}) {
 		logger.warn(`Failed to create shareable URL: ${error}`);
 	}
 	else logger.debug("Sharing requested but not enabled (check cloud config or sharing settings)");
-	if (testSuite.outputPath) {
-		if (typeof testSuite.outputPath === "string") await writeOutput(testSuite.outputPath, evalRecord, null);
-		else if (Array.isArray(testSuite.outputPath)) await writeMultipleOutputs(testSuite.outputPath, evalRecord, null);
+	if (testSuiteConfig.outputPath) {
+		if (typeof testSuiteConfig.outputPath === "string") await writeOutput(testSuiteConfig.outputPath, evalRecord, null);
+		else if (Array.isArray(testSuiteConfig.outputPath)) await writeMultipleOutputs(testSuiteConfig.outputPath, evalRecord, null);
 	}
 	return ret;
 }
@@ -14320,6 +15435,6 @@ var src_default = {
 	redteam
 };
 //#endregion
-export { AssertionOrSetSchema, AssertionSchema, AssertionSetSchema, AssertionTypeSchema, AtomicTestCaseSchema, BaseAssertionTypesSchema, BaseTokenUsageSchema, CommandLineOptionsSchema, CompletedPromptSchema, CompletionTokenDetailsSchema, ConversationMessageSchema, DerivedMetricSchema, EvalResultsFilterMode, EvaluateOptionsSchema, GradingConfigSchema, InputsSchema, NotPrefixedAssertionTypesSchema, OutputConfigSchema, OutputFileExtension, PartialGenerationError, PluginConfigSchema, PolicyObjectSchema, ProvidersSchema, ResultFailureReason, ScenarioSchema, SpecialAssertionTypesSchema, StrategyConfigSchema, TestCaseSchema, TestCaseWithVarsFileSchema, TestCasesWithMetadataPromptSchema, TestCasesWithMetadataSchema, TestGeneratorConfigSchema, TestSuiteConfigSchema, TestSuiteSchema, UnifiedConfigSchema, VarsSchema, assertions_default as assertions, cache_exports as cache, src_default as default, evaluate, generateTable, guardrails, isApiProvider, isGradingResult, isProviderOptions, isResultFailureReason, loadApiProvider, redteam };
+export { AssertionOrSetSchema, AssertionSchema, AssertionSetSchema, AssertionTypeSchema, AtomicTestCaseSchema, BaseAssertionTypesSchema, BaseTokenUsageSchema, CommandLineOptionsSchema, CompletedPromptSchema, CompletionTokenDetailsSchema, ConversationMessageSchema, DerivedMetricSchema, DocumentMediaInjectionPlacementSchema, DocumentMediaInjectionPlacementValues, DocxInjectionPlacementSchema, DocxInjectionPlacementValues, EvalResultsFilterMode, EvaluateOptionsSchema, GradingConfigSchema, InputConfigSchema, InputDefinitionObjectSchema, InputDefinitionSchema, InputTypeSchema, InputTypeValues, InputsSchema, NotPrefixedAssertionTypesSchema, OutputConfigSchema, OutputFileExtension, PartialGenerationError, PluginConfigSchema, PolicyObjectSchema, ProvidersSchema, ResultFailureReason, ScenarioSchema, SpecialAssertionTypesSchema, StrategyConfigSchema, TestCaseSchema, TestCaseWithVarsFileSchema, TestCasesWithMetadataPromptSchema, TestCasesWithMetadataSchema, TestGeneratorConfigSchema, TestSuiteConfigSchema, TestSuiteSchema, UnifiedConfigSchema, VarsSchema, assertions_default as assertions, buildInputPromptDescription, cache_exports as cache, src_default as default, evaluate, generateTable, getInputDescription, getInputType, guardrails, isApiProvider, isGradingResult, isProviderOptions, isResultFailureReason, isTransformFunction, loadApiProvider, normalizeInputDefinition, normalizeInputs, redteam };
 
 //# sourceMappingURL=index.js.map