promptfoo 0.121.5 → 0.121.7

package/dist/src/index.js

@@ -1,34 +1,47 @@
 import { C as getEnvFloat, D as getMaxEvalTimeMs, E as getEvalTimeoutMs, O as isCI, S as getEnvBool, T as getEnvString, a as logger, b as summarizeEvaluateResultForLogging, g as getAjv, h as extractJsonObjects, k as state, m as extractFirstJsonObject, n as globalLogCallback, o as setLogCallback, r as isDebugEnabled, s as setLogLevel, t as getLogLevel, v as orderKeys, w as getEnvInt, y as safeJsonStringify } from "./logger-Ct2S6Yx-.js";
 import { t as invariant } from "./invariant-Ddh24eXh.js";
-import { A as TERMINAL_MAX_WIDTH, F as HUMAN_ASSERTION_TYPE, M as getShareApiBaseUrl, N as getShareViewBaseUrl, P as FILE_METADATA_KEY, S as cloudConfig, _ as parseChatPrompt, a as getCurrentTimestamp, j as getDefaultShareViewBaseUrl, m as isPromptfooSampleTarget, n as fetchWithRetries, o as sleep, r as fetchWithTimeout, t as fetchWithProxy, u as REQUEST_TIMEOUT_MS, x as CloudConfig } from "./fetch-D9xxyC1p.js";
-import { n as VERSION } from "./version-CbuBKu2U.js";
-import { $ as categoryAliases, A as isApiProvider, At as CompletionTokenDetailsSchema, B as StrategyConfigSchema, C as TestGeneratorConfigSchema, Ct as TELECOM_PLUGINS, D as VarsSchema, E as UnifiedConfigSchema, F as ProvidersSchema, G as DEFAULT_STRATEGIES, H as isUuid, I as ConversationMessageSchema, J as STRATEGY_COLLECTION_MAPPINGS, L as PartialGenerationError, M as RedteamConfigSchema, N as PromptSchema, O as isGradingResult, Q as Severity, R as PluginConfigSchema, S as TestCasesWithMetadataSchema, St as TEEN_SAFETY_PLUGINS, T as TestSuiteSchema, Tt as CODING_AGENT_CORE_PLUGINS, Y as getDefaultNFanout, Z as isFanoutStrategy, _ as ScenarioSchema, _t as PII_PLUGINS, a as AtomicTestCaseSchema, at as DATASET_EXEMPT_PLUGINS, b as TestCaseWithVarsFileSchema, bt as REMOTE_ONLY_PLUGIN_IDS, c as CompletedPromptSchema, ct as FOUNDATION_PLUGINS, d as EvaluateOptionsSchema, dt as LLAMA_GUARD_ENABLED_CATEGORIES, f as GradingConfigSchema, ft as LLAMA_GUARD_REPLICATE_PROVIDER, g as ResultFailureReason, gt as PHARMACY_PLUGINS, h as OutputFileExtension, ht as MULTI_INPUT_VAR, i as AssertionTypeSchema, it as CANARY_BREAKING_STRATEGY_IDS, j as isProviderOptions, jt as InputsSchema, k as isResultFailureReason, kt as BaseTokenUsageSchema, l as DerivedMetricSchema, lt as HARM_PLUGINS, m as OutputConfigSchema, mt as MULTI_INPUT_EXCLUDED_PLUGINS, n as AssertionSchema, nt as ALIASED_PLUGIN_MAPPINGS, o as BaseAssertionTypesSchema, ot as DEFAULT_PLUGINS, p as NotPrefixedAssertionTypesSchema, pt as MEDICAL_PLUGINS, q as STRATEGY_COLLECTIONS, r as AssertionSetSchema, rt as BIAS_PLUGINS, s as CommandLineOptionsSchema, st as FINANCIAL_PLUGINS, t as AssertionOrSetSchema, tt as riskCategorySeverityMap, u as EvalResultsFilterMode, ut as INSURANCE_PLUGINS, v as SpecialAssertionTypesSchema, vt as PLUGIN_CATEGORIES, w as TestSuiteConfigSchema, wt as UNALIGNED_PROVIDER_HARM_PLUGINS, x as TestCasesWithMetadataPromptSchema, y as TestCaseSchema, yt as REDTEAM_PROVIDER_HARM_PLUGINS, z as PolicyObjectSchema } from "./types-DNRZVOue.js";
-import { c as setUserEmail, i as getUserEmail, o as isLoggedIntoCloud, r as getAuthor, s as promptForEmailUnverified, t as checkEmailStatusAndMaybeExit } from "./accounts-CaLNYnf7.js";
-import { r as importModule, t as getDirectory } from "./esm-BRkfNsYs.js";
-import { a as getNunjucksEngine, i as extractVariablesFromTemplates, r as extractVariablesFromTemplate, t as renderEnvOnlyInObject } from "./render-DlscvAUJ.js";
-import { t as providerRegistry } from "./providerRegistry-DoACwqhD.js";
-import { i as getRemoteGenerationUrl, l as shouldGenerateRemote, o as getRemoteHealthUrl, r as promptYesNo, s as neverGenerateRemote } from "./server-gyd6d4Hc.js";
-import { $ as getMaxCharsPerMessageModifierValue, B as collectFileMetadata, D as validateStrategies, E as loadStrategy, F as getSessionId, G as redteamProviderManager, H as runExtensionHook, I as getShortPluginId, J as createProviderRateLimitOptions, K as TokenUsageTracker, L as isBasicRefusal, N as extractPromptFromTags, O as pluginMatchesStrategyTargets, P as extractVariablesFromJson, Q as getGeneratedPromptOverLimit, T as Strategies, U as isPackagePath, V as renderPrompt, W as loadFromPackage, X as PromptfooHarmfulCompletionProvider, Z as MAX_CHARS_PER_MESSAGE_MODIFIER_KEY, a as resolveProviderConfigs, at as getOrgContext, ct as isCloudProvider, d as MCPProvider, et as throwIfTargetPromptExceedsMaxChars, g as AIStudioChatProvider, h as GoogleLiveProvider, i as resolveProvider, it as getEvalConfigFromCloud, j as extractGoalFromPrompt, k as checkExfilTracking, lt as resolveTeamId, m as VertexChatProvider, n as loadApiProvider, nt as checkCloudPermissions, ot as getPluginSeverityOverridesFromCloud, q as createRateLimitRegistry, r as loadApiProviders, rt as getCloudDatabaseId, t as getProviderIds } from "./providers-CJh7iriU.js";
-import { r as runPython } from "./pythonUtils-DMO68Jg7.js";
-import { i as isJavascriptFile } from "./fileExtensions-8CjoL7vB.js";
-import { A as readFilters, M as loadFunction, N as parseFileUrl, O as maybeLoadToolsFromExternalFile, T as maybeLoadFromExternalFile, _ as isProviderAllowed, a as setupEnv, b as normalizeProviderRef, c as filterRuntimeVars, d as checkProviderApiKeys, f as doesProviderRefMatch, g as isOpenAiProvider, h as isGoogleProvider, i as fetchCsvFromGoogleSheet, j as readOutput, l as getTestCaseDeduplicationKey, m as isAnthropicProvider, n as writeMultipleOutputs, o as deduplicateTestCases, p as getProviderDescription, r as writeOutput, s as extractRuntimeVars, t as printBorder, u as resultIsForTestCase, w as maybeLoadConfigFromExternalFile } from "./util-Dub0f_ej.js";
-import { a as createEmptyTokenUsage, i as createEmptyAssertions, n as accumulateResponseTokenUsage, o as normalizeTokenUsage, r as accumulateTokenUsage, t as accumulateAssertionTokenUsage } from "./tokenUsageUtils-4c780gFd.js";
-import { t as getBlobByHash } from "./blobs-BW4U31ue.js";
-import { a as evalsTable, c as evalsToTagsTable, d as tagsTable, i as evalResultsTable, l as promptsTable, m as getDbSignalPath, o as evalsToDatasetsTable, p as getDb, r as datasetsTable, s as evalsToPromptsTable } from "./tables-WgdUZ8Ck.js";
-import { n as isBlobStorageEnabled, t as extractAndStoreBinaryData } from "./extractor-BM3jRERL.js";
-import { c as isNonTransientHttpStatus, i as getCache, n as disableCache, o as withCacheNamespace, r as fetchWithCache, s as NON_TRANSIENT_HTTP_STATUSES, t as cache_exports } from "./cache-Bzttsk0X.js";
-import { t as OpenAiChatCompletionProvider } from "./chat-Cx_LkwvZ.js";
-import { h as validateFunctionCall } from "./transform-_DpNB4qp.js";
-import { l as validateFunctionCall$1 } from "./util-BlFVL0UF.js";
-import { i as getProcessShim, n as transform, t as TransformInputType } from "./transform-eGiUAv86.js";
-import { t as telemetry } from "./telemetry-CgdVGV8N.js";
-import { t as ellipsize } from "./text-DDQP0tuQ.js";
-import { t as getTraceStore } from "./store-BPkzEyFM.js";
-import { n as sha256, t as randomSequence } from "./createHash-Un4Q_huE.js";
-import { n as runRuby } from "./rubyUtils-BI0p46eZ.js";
-import { $ as SELECT_BEST_PROMPT, A as DivergentRepetitionPlugin, B as sampleArray, C as getPiiLeakTestsForCategory, Ct as DEFAULT_ANTHROPIC_MODEL, D as HarmbenchPlugin, Dt as withProviderCallExecutionContext, E as ImitationPlugin, Et as getGradingProvider, F as AegisPlugin, G as matchesLlmRubric, H as matchesClosedQa, I as RedteamGraderBase, J as doRemoteGrading, K as matchesPiScore, L as RedteamPluginBase, M as CrossSessionLeakPlugin, N as ContractPlugin, O as HallucinationPlugin, P as BeavertailsPlugin, Q as DEFAULT_WEB_SEARCH_PROMPT, R as getCustomPolicies, S as PlinyPlugin, St as getDefaultProviders, T as IntentPlugin, Tt as getAndCheckProvider, U as matchesFactuality, V as fetchHuggingFaceDataset, W as matchesGEval, X as readPrompts, Y as processPrompts, Z as readProviderPromptMap, _ as PoliticsPlugin, _t as tryParse, a as UnverifiableClaimsPlugin, at as CONTEXT_RECALL_ATTRIBUTED_TOKEN, b as isValidPolicyObject, bt as loadFromJavaScriptFile, c as ToolDiscoveryPlugin, ct as CONTEXT_RELEVANCE_BAD, d as TeenSafetyDangerousContentPlugin, dt as cosineSimilarity, et as SUGGEST_PROMPTS_SYSTEM_MESSAGE, f as TeenSafetyAgeRestrictedGoodsAndServicesPlugin, ft as dotProduct, g as PromptExtractionPlugin, gt as splitIntoSentences, h as RbacPlugin, ht as normalizeMatcherTokenUsage, i as VLGuardPlugin, it as CONTEXT_RECALL, j as DebugAccessPlugin, k as ExcessiveAgencyPlugin, l as TeenSafetyHarmfulBodyIdealsPlugin, lt as loadRubricPrompt, m as ShellInjectionPlugin, mt as fail, n as getGraderById, nt as CONTEXT_FAITHFULNESS_LONGFORM, o as UnsafeBenchPlugin, ot as CONTEXT_RECALL_NOT_ATTRIBUTED_TOKEN, p as SqlInjectionPlugin, pt as euclideanDistance, q as matchesTrajectoryGoalSuccess, r as VLSUPlugin, rt as CONTEXT_FAITHFULNESS_NLI_STATEMENTS, s as ToxicChatPlugin, st as CONTEXT_RELEVANCE, t as GRADERS, tt as ANSWER_RELEVANCY_GENERATE, u as TeenSafetyDangerousRoleplayPlugin, ut as renderLlmRubricPrompt, v as PolicyPlugin, vt as coerceString, w as OverreliancePlugin, wt as callProviderWithContext, x as makeInlinePolicyIdSync, xt as processFileReference, y as determinePolicyTypeFromId, yt as getFinalTest, z as retryWithDeduplication } from "./graders-BoUqsCEm.js";
-import { i as generateIdFromPrompt, t as hashPrompt } from "./utils-o8S5huU2.js";
-import { t as EvalResult } from "./evalResult-pSvGWFMo.js";
+import { $ as riskCategorySeverityMap, A as RedteamConfigSchema, At as DocumentMediaInjectionPlacementValues, B as isUuid, Bt as getInputDescription, C as TestGeneratorConfigSchema, Ct as CODING_AGENT_CORE_PLUGINS, D as VarsSchema, Dt as BaseTokenUsageSchema, E as UnifiedConfigSchema, F as PartialGenerationError, Ft as InputDefinitionSchema, G as STRATEGY_COLLECTIONS, Gt as isProviderOptions, Ht as normalizeInputDefinition, I as PluginConfigSchema, It as InputTypeSchema, K as STRATEGY_COLLECTION_MAPPINGS, L as PolicyObjectSchema, Lt as InputTypeValues, Mt as DocxInjectionPlacementValues, N as ProvidersSchema, Nt as InputConfigSchema, O as isGradingResult, Ot as CompletionTokenDetailsSchema, P as ConversationMessageSchema, Pt as InputDefinitionObjectSchema, R as StrategyConfigSchema, Rt as InputsSchema, S as TestCasesWithMetadataSchema, St as UNALIGNED_PROVIDER_HARM_PLUGINS, T as TestSuiteSchema, U as DEFAULT_STRATEGIES, Ut as normalizeInputs, Vt as getInputType, Wt as isApiProvider, X as Severity, Y as isFanoutStrategy, Z as categoryAliases, _ as ScenarioSchema, _t as REDTEAM_PROVIDER_HARM_PLUGINS, a as AtomicTestCaseSchema, at as FINANCIAL_PLUGINS, b as TestCaseWithVarsFileSchema, bt as TEEN_SAFETY_PLUGINS, c as CompletedPromptSchema, ct as INSURANCE_PLUGINS, d as EvaluateOptionsSchema, dt as MEDICAL_PLUGINS, et as ALIASED_PLUGIN_MAPPINGS, f as GradingConfigSchema, ft as MULTI_INPUT_EXCLUDED_PLUGINS, g as ResultFailureReason, gt as PLUGIN_CATEGORIES, h as OutputFileExtension, ht as PII_PLUGINS, i as AssertionTypeSchema, it as DEFAULT_PLUGINS, j as PromptSchema, jt as DocxInjectionPlacementSchema, k as isResultFailureReason, kt as DocumentMediaInjectionPlacementSchema, l as DerivedMetricSchema, lt as LLAMA_GUARD_ENABLED_CATEGORIES, m as OutputConfigSchema, mt as PHARMACY_PLUGINS, n as AssertionSchema, nt as CANARY_BREAKING_STRATEGY_IDS, o as BaseAssertionTypesSchema, ot as FOUNDATION_PLUGINS, p as NotPrefixedAssertionTypesSchema, pt as MULTI_INPUT_VAR, q as getDefaultNFanout, r as AssertionSetSchema, rt as DATASET_EXEMPT_PLUGINS, s as CommandLineOptionsSchema, st as HARM_PLUGINS, t as AssertionOrSetSchema, tt as BIAS_PLUGINS, u as EvalResultsFilterMode, ut as LLAMA_GUARD_REPLICATE_PROVIDER, v as SpecialAssertionTypesSchema, vt as REMOTE_ONLY_PLUGIN_IDS, w as TestSuiteConfigSchema, wt as CODING_AGENT_PLUGINS, x as TestCasesWithMetadataPromptSchema, xt as TELECOM_PLUGINS, y as TestCaseSchema, zt as buildInputPromptDescription } from "./types-D6glLbdF.js";
+import { F as getShareApiBaseUrl, I as getShareViewBaseUrl, L as FILE_METADATA_KEY, N as TERMINAL_MAX_WIDTH, P as getDefaultShareViewBaseUrl, R as HUMAN_ASSERTION_TYPE, T as cloudConfig, _ as isPromptfooSampleTarget, b as parseChatPrompt, c as getCurrentTimestamp, l as sleep, n as fetchWithRetries, p as REQUEST_TIMEOUT_MS, r as fetchWithTimeout, t as fetchWithProxy, w as CloudConfig } from "./fetch-It34O8Ur.js";
+import { n as VERSION } from "./version-lpHV_53E.js";
+import { i as isJavascriptFile } from "./fileExtensions-CXRfY3Ss.js";
+import { c as setUserEmail, i as getUserEmail, o as isLoggedIntoCloud, r as getAuthor, s as promptForEmailUnverified, t as checkEmailStatusAndMaybeExit } from "./accounts-Ca7WIoPY.js";
+import { r as importModule, t as getDirectory } from "./esm-BTK1W7lG.js";
+import { a as extractVariablesFromTemplates, i as extractVariablesFromTemplate, o as getNunjucksEngine, r as analyzeTemplateReference, t as renderEnvOnlyInObject } from "./render-DFfDeYUK.js";
+import { t as providerRegistry } from "./providerRegistry-DHcFiVWX.js";
+import { a as getRemoteHealthUrl, l as shouldGenerateRemote, n as getRemoteGenerationExplicitlyDisabledError, r as getRemoteGenerationUrl, s as neverGenerateRemote } from "./remoteGeneration-DsaSwmG2.js";
+import { r as promptYesNo } from "./server-CMJD10J4.js";
+import { a as getCloudDatabaseId, c as getPluginSeverityOverridesFromCloud, d as isCloudProvider, i as checkCloudPermissions, o as getEvalConfigFromCloud, p as resolveTeamId, s as getOrgContext } from "./storage-CD-GWAdx.js";
+import { r as runPython } from "./pythonUtils-D0BYebvX.js";
+import { A as readFilters, M as loadFunction, N as parseFileUrl, O as maybeLoadToolsFromExternalFile, T as maybeLoadFromExternalFile, _ as isProviderAllowed, a as setupEnv, b as normalizeProviderRef, c as filterRuntimeVars, d as checkProviderApiKeys, f as doesProviderRefMatch, g as isOpenAiProvider, h as isGoogleProvider, i as fetchCsvFromGoogleSheet, j as readOutput, l as getTestCaseDeduplicationKey, m as isAnthropicProvider, n as writeMultipleOutputs, o as deduplicateTestCases, p as getProviderDescription, r as writeOutput, s as extractRuntimeVars, t as printBorder, u as resultIsForTestCase, w as maybeLoadConfigFromExternalFile } from "./util-Bx677_k2.js";
+import { n as sha256, t as randomSequence } from "./createHash-BtbSX3mj.js";
+import { c as NON_TRANSIENT_HTTP_STATUSES, i as getCache, l as isNonTransientHttpStatus, n as disableCache, r as fetchWithCache, s as withCacheNamespace, t as cache_exports } from "./cache-DIXbtkNO.js";
+import { t as OpenAiChatCompletionProvider } from "./chat-Dabu84Br.js";
+import { h as validateFunctionCall } from "./transform-DmvYBRll.js";
+import { l as validateFunctionCall$1 } from "./util-LKTmNsMQ.js";
+import { _ as AIStudioChatProvider, a as resolveProvider, f as MCPProvider, g as GoogleLiveProvider, h as VertexChatProvider, n as loadApiProvider, o as resolveProviderConfigs, r as loadApiProviders, t as getProviderIds } from "./providers-DVYRZP4E.js";
+import { a as createEmptyTokenUsage, i as createEmptyAssertions, n as accumulateResponseTokenUsage, o as normalizeTokenUsage, r as accumulateTokenUsage, t as accumulateAssertionTokenUsage } from "./tokenUsageUtils-CmnQ0G2m.js";
+import { t as ellipsize } from "./text-DwYK5EBn.js";
+import { t as telemetry } from "./telemetry-DwX9XUN5.js";
+import { a as evalsTable, c as evalsToTagsTable, d as tagsTable, i as evalResultsTable, l as promptsTable, m as getDbSignalPath, o as evalsToDatasetsTable, p as getDb, r as datasetsTable, s as evalsToPromptsTable } from "./tables-CRSXQ2Ke.js";
+import { t as getBlobByHash } from "./blobs-Dwef1Ao1.js";
+import { t as getProcessShim } from "./processShim-BcGzU8fY.js";
+import { n as loadFromPackage, t as isPackagePath } from "./packageParser-CgE-ziRo.js";
+import { n as runRuby } from "./rubyUtils-CnlW8AYb.js";
+import { n as materializeInputVariablesWithMetadata, t as buildPromptInputDescriptions } from "./inputVariables-DUGMb9Ka.js";
+import { a as extractPromptFromTags, c as isBasicRefusal, i as extractMaterializedVariablesFromJsonWithMetadata, n as extractGoalFromPrompt, o as getSessionId, r as extractInputVarsFromPrompt, s as getShortPluginId } from "./util-DNl96nNs.js";
+import { n as PromptfooHarmfulCompletionProvider } from "./promptfoo-Cm4hiy1Y.js";
+import { $ as readProviderPromptMap, A as ExcessiveAgencyPlugin, At as withProviderCallExecutionContext, B as retryWithDeduplication, C as PlinyPlugin, Ct as processFileReference, D as ImitationPlugin, Dt as getAndCheckProvider, E as IntentPlugin, Et as callProviderWithContext, F as BeavertailsPlugin, G as matchesFactuality, H as fetchHuggingFaceDataset, I as AegisPlugin, J as matchesPiScore, K as matchesGEval, L as RedteamGraderBase, M as DebugAccessPlugin, N as CrossSessionLeakPlugin, O as HarmbenchPlugin, Ot as getGradingProvider, P as ContractPlugin, Q as readPrompts, R as RedteamPluginBase, S as makeInlinePolicyIdSync, St as loadFromJavaScriptFile, T as OverreliancePlugin, Tt as DEFAULT_ANTHROPIC_MODEL, U as isGraderFailure, V as sampleArray, W as matchesClosedQa, X as doRemoteGrading, Y as matchesTrajectoryGoalSuccess, Z as processPrompts, _ as PromptExtractionPlugin, _t as normalizeMatcherTokenUsage, a as VLGuardPlugin, at as CONTEXT_FAITHFULNESS_NLI_STATEMENTS, b as determinePolicyTypeFromId, bt as coerceString, c as ToxicChatPlugin, ct as CONTEXT_RECALL_NOT_ATTRIBUTED_TOKEN, d as TeenSafetyDangerousRoleplayPlugin, dt as loadRubricPrompt, et as DEFAULT_WEB_SEARCH_PROMPT, f as TeenSafetyDangerousContentPlugin, ft as renderLlmRubricPrompt, g as RbacPlugin, gt as fail, h as ShellInjectionPlugin, ht as euclideanDistance, i as VLSUPlugin, it as CONTEXT_FAITHFULNESS_LONGFORM, j as DivergentRepetitionPlugin, k as HallucinationPlugin, kt as getProviderCallExecutionContext, l as ToolDiscoveryPlugin, lt as CONTEXT_RELEVANCE, m as SqlInjectionPlugin, mt as dotProduct, n as getGraderById, nt as SUGGEST_PROMPTS_SYSTEM_MESSAGE, o as UnverifiableClaimsPlugin, ot as CONTEXT_RECALL, p as TeenSafetyAgeRestrictedGoodsAndServicesPlugin, pt as cosineSimilarity, q as matchesLlmRubric, rt as ANSWER_RELEVANCY_GENERATE, s as UnsafeBenchPlugin, st as CONTEXT_RECALL_ATTRIBUTED_TOKEN, t as GRADERS, tt as SELECT_BEST_PROMPT, u as TeenSafetyHarmfulBodyIdealsPlugin, ut as CONTEXT_RELEVANCE_BAD, v as PoliticsPlugin, vt as splitIntoSentences, w as getPiiLeakTestsForCategory, wt as getDefaultProviders, x as isValidPolicyObject, xt as getFinalTest, y as PolicyPlugin, yt as tryParse, z as getCustomPolicies } from "./graders-BX0f2tvS.js";
+import { f as redteamProviderManager, g as createProviderRateLimitOptions, h as createRateLimitRegistry, m as TokenUsageTracker } from "./shared-D6IjElRI.js";
+import { i as generateIdFromPrompt, t as hashPrompt } from "./utils-BGY69tk_.js";
+import { a as getTransformLabel, i as getTransformErrorMessage, n as TRANSFORM_KEYS, o as transform, r as TransformInputType, t as INLINE_FUNCTION_LABEL } from "./transform-CGt7Kt3y2.js";
+import { t as getTraceStore } from "./store-HpopRVzl.js";
+import { n as isBlobStorageEnabled, t as extractAndStoreBinaryData } from "./extractor-CxRtnaHl.js";
+import { i as throwIfTargetPromptExceedsMaxChars, n as getGeneratedPromptOverLimit, r as getMaxCharsPerMessageModifierValue, t as MAX_CHARS_PER_MESSAGE_MODIFIER_KEY } from "./promptLength-B9nZEfO6.js";
+import { n as checkExfilTracking } from "./indirectWebPwn-CbjUG0rh.js";
+import { n as getFirstStringAttribute, r as getToolNameFromAttributes, t as TOOL_ARGUMENT_ATTRIBUTE_KEYS } from "./toolAttributes-DJ9ZEKXD.js";
+import { i as filterFiniteScores, n as renderPrompt, r as runExtensionHook, t as collectFileMetadata } from "./evaluatorHelpers-DuqFFfq7.js";
+import { r as sanitizeProvider, t as EvalResult } from "./evalResult-2RRJvFyB.js";
+import { i as pluginMatchesStrategyTargets, n as loadStrategy, r as validateStrategies, t as Strategies } from "./strategies-CiSeroPH.js";
 import * as fs$2 from "fs";
 import fs, { createWriteStream } from "fs";
 import * as path$2 from "path";
@@ -51,14 +64,15 @@ import { DiagConsoleLogger, DiagLogLevel, diag, propagation } from "@opentelemet
 import readline from "readline";
 import { parse as parse$1 } from "csv-parse/sync";
 import { globSync } from "glob";
-import { and, desc, eq, inArray, sql } from "drizzle-orm";
 import { XMLParser } from "fast-xml-parser";
-import cliProgress from "cli-progress";
+import { and, desc, eq, inArray, sql } from "drizzle-orm";
 import { URL } from "url";
 import { parse as parse$2 } from "parse5";
 import { distance } from "fastest-levenshtein";
+import cliProgress from "cli-progress";
 import * as rouge from "js-rouge";
 import { isDeepStrictEqual } from "node:util";
+import { LRUCache } from "lru-cache";
 import "debounce";
 import { ExportResultCode, W3CTraceContextPropagator } from "@opentelemetry/core";
 import { OTLPTraceExporter } from "@opentelemetry/exporter-trace-otlp-http";
@@ -429,17 +443,20 @@ async function resolveContext(assertion, test, output, prompt, fallbackContext,
 			contextValue = test.vars.context;
 		}
 	} else if (fallbackContext) contextValue = fallbackContext;
-	if (assertion.contextTransform) try {
-		const outputForTransform = providerResponse?.providerTransformedOutput ?? output;
-		const transformed = await transform(assertion.contextTransform, outputForTransform, {
-			vars: test.vars,
-			prompt: { label: prompt },
-			...providerResponse && providerResponse.metadata && { metadata: providerResponse.metadata }
-		});
-		invariant(typeof transformed === "string" || Array.isArray(transformed) && transformed.every((item) => typeof item === "string"), `contextTransform must return a string or array of strings. Got ${typeof transformed}. Check your transform expression: ${assertion.contextTransform}`);
-		contextValue = transformed;
-	} catch (error) {
-		throw new Error(`Failed to transform context using expression '${assertion.contextTransform}': ${error instanceof Error ? error.message : String(error)}`);
+	if (assertion.contextTransform) {
+		const getLabel = () => getTransformLabel(assertion.contextTransform);
+		try {
+			const outputForTransform = providerResponse?.providerTransformedOutput ?? output;
+			const transformed = await transform(assertion.contextTransform, outputForTransform, {
+				vars: test.vars,
+				prompt: { label: prompt },
+				...providerResponse && providerResponse.metadata && { metadata: providerResponse.metadata }
+			});
+			invariant(typeof transformed === "string" || Array.isArray(transformed) && transformed.every((item) => typeof item === "string"), () => `contextTransform must return a string or array of strings. Got ${typeof transformed}. Check your transform expression: ${getLabel()}`);
+			contextValue = transformed;
+		} catch (error) {
+			throw new Error(`Failed to transform context using expression '${getLabel()}': ${getTransformErrorMessage(error)}`);
+		}
 	}
 	invariant(typeof contextValue === "string" && contextValue.length > 0 || Array.isArray(contextValue) && contextValue.length > 0 && contextValue.every((item) => typeof item === "string" && item.length > 0), "Context is required for context-based assertions. Provide either a \"context\" variable (string or array of strings) in your test case or use \"contextTransform\" to extract context from the provider response.");
 	return contextValue;
@@ -782,7 +799,7 @@ async function startOtlpReceiverIfNeeded(testSuite) {
 		telemetry.record("feature_used", { feature: "tracing" });
 		try {
 			logger.debug("[EvaluatorTracing] Tracing configuration detected, starting OTLP receiver");
-			const { startOTLPReceiver } = await import("./otlpReceiver-CcdIikOu.js");
+			const { startOTLPReceiver } = await import("./otlpReceiver-BXjcRqAM.js");
 			const port = testSuite.tracing.otlp.http.port || 4318;
 			const host = testSuite.tracing.otlp.http.host || "127.0.0.1";
 			const acceptFormats = normalizeOtlpAcceptFormats(testSuite.tracing.otlp.http.acceptFormats);
@@ -806,7 +823,7 @@ async function startOtlpReceiverIfNeeded(testSuite) {
 async function stopOtlpReceiverIfNeeded() {
 	if (otlpReceiverStarted) try {
 		logger.debug("[EvaluatorTracing] Stopping OTLP receiver");
-		const { stopOTLPReceiver } = await import("./otlpReceiver-CcdIikOu.js");
+		const { stopOTLPReceiver } = await import("./otlpReceiver-BXjcRqAM.js");
 		await stopOTLPReceiver();
 		otlpReceiverStarted = false;
 		logger.info("[EvaluatorTracing] OTLP receiver stopped successfully");
@@ -841,7 +858,7 @@ async function generateTraceContextIfNeeded(test, evaluateOptions, testIdx, prom
 	}
 	if (!tracingEnabled) return null;
 	logger.debug("[EvaluatorTracing] Importing trace store");
-	const { getTraceStore } = await import("./store-BPkzEyFM.js").then((n) => n.n);
+	const { getTraceStore } = await import("./store-HpopRVzl.js").then((n) => n.n);
 	const traceStore = getTraceStore();
 	const traceId = generateTraceId();
 	const spanId = generateSpanId();
@@ -1473,27 +1490,67 @@ const handleIsValidFunctionCall = ({ assertion, output, provider, test }) => {
 };
 //#endregion
 //#region src/assertions/geval.ts
-const handleGEval = async ({ assertion, renderedValue, prompt, outputString, test, providerCallContext }) => {
-	invariant(typeof renderedValue === "string" || Array.isArray(renderedValue), "G-Eval assertion type must have a string or array of strings value");
+const handleGEval = async ({ assertion, inverse, renderedValue, prompt, outputString, test, providerCallContext }) => {
+	invariant(typeof renderedValue === "string" || Array.isArray(renderedValue) && renderedValue.every((value) => typeof value === "string"), "G-Eval assertion type must have a string or array of strings value");
 	const threshold = assertion.threshold ?? .7;
 	if (Array.isArray(renderedValue)) {
-		const scores = [];
-		const reasons = [];
-		for (const value of renderedValue) {
+		if (renderedValue.length === 0) return {
+			assertion,
+			pass: false,
+			score: 0,
+			reason: "G-Eval assertion requires at least one criterion string in the value array."
+		};
+		const responses = [];
+		let failure;
+		for (const [index, value] of renderedValue.entries()) {
 			const resp = await matchesGEval(value, prompt || "", outputString, threshold, test.options, providerCallContext);
-			scores.push(resp.score);
-			reasons.push(resp.reason);
+			responses.push(resp);
+			if (isGraderFailure(resp)) {
+				failure = {
+					index,
+					resp
+				};
+				break;
+			}
 		}
-		const scoresSum = scores.reduce((a, b) => a + b, 0);
+		const tokensUsed = createEmptyTokenUsage();
+		for (const r of responses) accumulateTokenUsage(tokensUsed, r.tokensUsed);
+		if (failure) {
+			const criterion = renderedValue[failure.index];
+			return {
+				assertion,
+				pass: false,
+				score: 0,
+				reason: `G-Eval criterion ${failure.index + 1}/${renderedValue.length} (${JSON.stringify(criterion)}) failed: ${failure.resp.reason}`,
+				tokensUsed,
+				metadata: failure.resp.metadata
+			};
+		}
+		const averageScore = responses.reduce((acc, r) => acc + r.score, 0) / responses.length;
+		const combinedReason = responses.map((r) => r.reason).join("\n\n");
 		return {
 			assertion,
-			pass: scoresSum / scores.length >= threshold,
-			score: scoresSum / scores.length,
-			reason: reasons.join("\n\n")
+			pass: averageScore >= threshold !== inverse,
+			score: inverse ? 1 - averageScore : averageScore,
+			reason: combinedReason,
+			tokensUsed
 		};
-	} else return {
+	}
+	const resp = await matchesGEval(renderedValue, prompt || "", outputString, threshold, test.options, providerCallContext);
+	if (isGraderFailure(resp)) return {
+		assertion,
+		pass: false,
+		score: 0,
+		reason: resp.reason,
+		tokensUsed: resp.tokensUsed,
+		metadata: resp.metadata
+	};
+	const passed = resp.score >= threshold !== inverse;
+	return {
 		assertion,
-		...await matchesGEval(renderedValue, prompt || "", outputString, threshold, test.options, providerCallContext)
+		...resp,
+		pass: passed,
+		score: inverse ? 1 - resp.score : resp.score
 	};
 };
 //#endregion
@@ -2496,45 +2553,6 @@ function matchesPattern(spanName, pattern) {
 }
 //#endregion
 //#region src/assertions/trajectoryUtils.ts
-const TOOL_ATTRIBUTE_KEYS = [
-	"tool.name",
-	"tool_name",
-	"tool",
-	"function.name",
-	"function_name",
-	"gen_ai.tool.name",
-	"codex.mcp.tool",
-	"agent.tool",
-	"agent.tool_name",
-	"agent.toolName"
-];
-const TOOL_ARGUMENT_ATTRIBUTE_KEYS = [
-	"tool.arguments",
-	"tool.args",
-	"tool.input",
-	"tool_arguments",
-	"tool_args",
-	"tool_input",
-	"function.arguments",
-	"function.args",
-	"function.input",
-	"function_arguments",
-	"function_args",
-	"gen_ai.tool.arguments",
-	"gen_ai.tool.args",
-	"gen_ai.tool.input",
-	"gen_ai.tool.call.arguments",
-	"gen_ai.tool.call.args",
-	"agent.tool.arguments",
-	"agent.tool.args",
-	"agent.tool.input",
-	"codex.mcp.arguments",
-	"codex.mcp.args",
-	"codex.mcp.input",
-	"arguments",
-	"args",
-	"input"
-];
 const COMMAND_ATTRIBUTE_KEYS = [
 	"codex.command",
 	"command",
@@ -2547,16 +2565,15 @@ const SEARCH_ATTRIBUTE_KEYS = [
 	"search_query"
 ];
 const GENERIC_QUERY_ATTRIBUTE_KEYS = ["query"];
+const COMMAND_TOOL_NAMES = new Set([
+	"exec_command",
+	"local_shell",
+	"shell"
+]);
 const SEARCH_SPAN_NAME_PATTERN = /(^|[\s._:/-])(search|find|lookup|retriev(?:e|al))($|[\s._:/-])/i;
 const MAX_JUDGE_SUMMARY_STEPS = 24;
 const JUDGE_SUMMARY_HEAD_STEPS = 12;
 const JUDGE_SUMMARY_TAIL_STEPS = 12;
-function getStringAttribute(attributes, keys) {
-	for (const key of keys) {
-		const value = attributes[key];
-		if (typeof value === "string" && value.trim()) return value.trim();
-	}
-}
 function normalizeStructuredAttribute(value) {
 	if (value === void 0 || value === null) return;
 	if (typeof value === "string") {
@@ -2588,9 +2605,12 @@ function getTrajectoryStepStatus(step) {
 function getCommandExecutable(command) {
 	return command.trim().split(/\s+/)[0] || void 0;
 }
+function isCommandToolName(toolName) {
+	return !!toolName && COMMAND_TOOL_NAMES.has(toolName.trim().toLowerCase());
+}
 function extractToolName(span) {
 	const attributes = span.attributes || {};
-	const directMatch = getStringAttribute(attributes, TOOL_ATTRIBUTE_KEYS);
+	const directMatch = getToolNameFromAttributes(attributes);
 	if (directMatch) return directMatch;
 	for (const [key, value] of Object.entries(attributes)) {
 		if (typeof value !== "string" || !value.trim()) continue;
@@ -2615,21 +2635,31 @@ function extractToolArgs(span) {
 		if (value !== void 0) return value;
 	}
 }
-function extractCommand(span) {
+function extractCommand(span, toolName = extractToolName(span), getToolArgs = () => extractToolArgs(span)) {
 	const attributes = span.attributes || {};
-	const directMatch = getStringAttribute(attributes, COMMAND_ATTRIBUTE_KEYS);
+	const directMatch = getFirstStringAttribute(attributes, COMMAND_ATTRIBUTE_KEYS);
 	if (directMatch) return directMatch;
 	for (const [key, value] of Object.entries(attributes)) {
 		if (typeof value !== "string" || !value.trim()) continue;
 		if (/command/i.test(key) && !/output|result/i.test(key)) return value.trim();
 	}
+	const toolArgs = getToolArgs();
+	if (isCommandToolName(toolName) && toolArgs && typeof toolArgs === "object") {
+		const args = toolArgs;
+		const command = args.cmd ?? args.command;
+		if (typeof command === "string" && command.trim()) return command.trim();
+		if (Array.isArray(command)) {
+			const joined = command.map((part) => String(part).trim()).filter(Boolean).join(" ");
+			if (joined) return joined;
+		}
+	}
 	if (span.name.startsWith("exec ")) return span.name.slice(5).trim();
 }
 function extractSearchQuery(span) {
 	const attributes = span.attributes || {};
-	const directMatch = getStringAttribute(attributes, SEARCH_ATTRIBUTE_KEYS);
+	const directMatch = getFirstStringAttribute(attributes, SEARCH_ATTRIBUTE_KEYS);
 	if (directMatch) return directMatch;
-	const genericQuery = getStringAttribute(attributes, GENERIC_QUERY_ATTRIBUTE_KEYS);
+	const genericQuery = getFirstStringAttribute(attributes, GENERIC_QUERY_ATTRIBUTE_KEYS);
 	if (genericQuery && isSearchLikeSpan(span)) return genericQuery;
 	if (span.name.startsWith("search ")) return span.name.slice(7).replace(/^"|"$/g, "").trim();
 }
@@ -2653,17 +2683,34 @@ function extractTrajectorySteps(trace) {
 		return left.index - right.index;
 	}).map(({ span }) => {
 		const toolName = extractToolName(span);
-		const command = extractCommand(span);
+		let toolArgs;
+		let hasExtractedToolArgs = false;
+		const getToolArgs = () => {
+			if (!hasExtractedToolArgs) {
+				toolArgs = extractToolArgs(span);
+				hasExtractedToolArgs = true;
+			}
+			return toolArgs;
+		};
+		const command = extractCommand(span, toolName, getToolArgs);
 		const searchQuery = extractSearchQuery(span);
 		let type = "span";
 		let name = span.name;
 		const aliases = new Set([span.name]);
 		let args;
-		if (toolName) {
+		if (command && isCommandToolName(toolName)) {
+			type = "command";
+			name = command;
+			aliases.add(command);
+			args = getToolArgs();
+			if (toolName) aliases.add(toolName);
+			const executable = getCommandExecutable(command);
+			if (executable) aliases.add(executable);
+		} else if (toolName) {
 			type = "tool";
 			name = toolName;
 			aliases.add(toolName);
-			args = extractToolArgs(span);
+			args = getToolArgs();
 		} else if (command) {
 			type = "command";
 			name = command;
@@ -3850,13 +3897,13 @@ function resolveSequenceValue(value) {
 	}
 	throw new Error("trajectory:tool-sequence assertion must have an array or object value");
 }
-function isRecord(value) {
+function isRecord$1(value) {
 	return typeof value === "object" && value !== null && !Array.isArray(value);
 }
 function matchesExpectedArgsPartial(actual, expected) {
 	if (Array.isArray(expected)) return Array.isArray(actual) && actual.length === expected.length && expected.every((item, index) => matchesExpectedArgsPartial(actual[index], item));
-	if (isRecord(expected)) {
-		if (!isRecord(actual)) return false;
+	if (isRecord$1(expected)) {
+		if (!isRecord$1(actual)) return false;
 		return Object.entries(expected).every(([key, expectedValue]) => Object.prototype.hasOwnProperty.call(actual, key) && matchesExpectedArgsPartial(actual[key], expectedValue));
 	}
 	return isDeepStrictEqual(actual, expected);
@@ -4227,7 +4274,7 @@ async function loadTraceData(traceId) {
 	let stableObservations = 0;
 	let latestTrace = null;
 	for (let attempt = 0; attempt < maxAttempts; attempt++) {
-		latestTrace = await traceStore.getTrace(traceId);
+		latestTrace = await traceStore.getTrace(traceId, { sanitizeAttributes: false });
 		const spanCount = latestTrace?.spans?.length ?? 0;
 		if (spanCount > 0) {
 			stableObservations = spanCount === lastSpanCount ? stableObservations + 1 : 1;
@@ -4280,7 +4327,7 @@ const ASSERTION_HANDLERS = {
 	"llm-rubric": handleLlmRubric,
 	meteor: async (params) => {
 		try {
-			const { handleMeteorAssertion } = await import("./meteor-BKTM-7KS.js");
+			const { handleMeteorAssertion } = await import("./meteor-Dce-_zGQ.js");
 			return handleMeteorAssertion(params);
 		} catch (error) {
 			if (error instanceof Error && (error.message.includes("Cannot find module") || error.message.includes("natural\" package is required"))) return {
@@ -4416,7 +4463,7 @@ async function runAssertion({ prompt, provider, assertion, test, vars, latencyMs
 			};
 		}
 		else if (filePath.endsWith(".rb")) try {
-			const { runRuby } = await import("./rubyUtils-BI0p46eZ.js").then((n) => n.t);
+			const { runRuby } = await import("./rubyUtils-CnlW8AYb.js").then((n) => n.t);
 			valueFromScript = await runRuby(filePath, functionName || "get_assert", [output, context]);
 			logger.debug(`Ruby script ${filePath} output: ${valueFromScript}`);
 		} catch (error) {
@@ -4533,7 +4580,8 @@ async function runAssertions({ assertScoringFunction, latencyMs, prompt, provide
 		logger.debug(`Failed to preload trace data for assertions: ${error}`);
 		preloadedTraceData = null;
 	}
-	await async.forEachOfLimit(asserts, ASSERTIONS_MAX_CONCURRENCY, async ({ assertion, assertResult, index }) => {
+	const concurrency = getProviderCallExecutionContext()?.providerCallQueue ? 1 : ASSERTIONS_MAX_CONCURRENCY;
+	await async.forEachOfLimit(asserts, concurrency, async ({ assertion, assertResult, index }) => {
 		if (assertion.type.startsWith("select-") || assertion.type === "max-score") return;
 		const result = await runAssertion({
 			prompt,
@@ -4660,7 +4708,8 @@ var CIProgressReporter = class {
 	}
 	updateTotalTests(newTotal) {
 		this.totalTests = Math.max(newTotal, 1);
-		this.highestPercentageSeen = Math.floor(this.completedTests / this.totalTests * 100);
+		const percentage = Math.floor(this.completedTests / this.totalTests * 100);
+		this.highestPercentageSeen = percentage;
 	}
 	finish() {
 		if (this.intervalId) {
@@ -4833,6 +4882,10 @@ function getDefaultOtelConfig() {
 }
 //#endregion
 //#region src/tracing/localSpanExporter.ts
+const MISSING_TRACE_RETRY_DELAY_MS = 50;
+function delay(ms) {
+	return new Promise((resolve) => setTimeout(resolve, ms));
+}
 /**
 * A span exporter that writes spans to the local TraceStore (SQLite).
 * This allows OTEL spans to be stored locally for analysis in the promptfoo UI.
@@ -4874,7 +4927,7 @@ var LocalSpanExporter = class {
 		}
 		let firstError;
 		for (const [traceId, spanDataList] of spansByTrace) try {
-			const result = await traceStore.addSpans(traceId, spanDataList, { skipTraceCheck: false });
+			const result = await this.addSpansWithTraceRetry(traceStore, traceId, spanDataList);
 			if (result.stored) logger.debug(`[LocalSpanExporter] Added ${spanDataList.length} spans to trace ${traceId}`);
 			else logger.debug(`[LocalSpanExporter] Skipping ${spanDataList.length} spans for orphan trace ${traceId}: ${result.reason}`);
 		} catch (error) {
@@ -4886,6 +4939,16 @@ var LocalSpanExporter = class {
 		}
 		return firstError;
 	}
+	async addSpansWithTraceRetry(traceStore, traceId, spans) {
+		const options = {
+			skipTraceCheck: false,
+			warnIfMissingTrace: false
+		};
+		const result = await traceStore.addSpans(traceId, spans, options);
+		if (result.stored) return result;
+		await delay(MISSING_TRACE_RETRY_DELAY_MS);
+		return traceStore.addSpans(traceId, spans, options);
+	}
 	/**
 	* Convert an OTEL ReadableSpan to our SpanData format.
 	*/
@@ -5168,6 +5231,15 @@ function isPromptAllowed(prompt, allowedPrompts) {
 }
 //#endregion
 //#region src/evaluator.ts
+const CONVERSATION_VAR_NAME = "_conversation";
+const promptUsesConversationVariableCache = new LRUCache({ max: 1024 });
+function promptUsesConversationVariable(prompt) {
+	const cached = promptUsesConversationVariableCache.get(prompt.raw);
+	if (cached !== void 0) return cached;
+	const { referenced, parsed } = analyzeTemplateReference(prompt.raw, CONVERSATION_VAR_NAME);
+	if (parsed) promptUsesConversationVariableCache.set(prompt.raw, referenced);
+	return referenced;
+}
 /**
 * Manages a single progress bar for the evaluation
 */
@@ -5367,6 +5439,18 @@ function hasProviderGroupedAssertion(assertion) {
 function shouldDeferGradingForTest(test) {
 	return Boolean(test.assert?.some(hasProviderGroupedAssertion));
 }
+function logGroupedGradingStatus({ concurrency, hasEvalStepTimeout, runEvalOptions, shouldGroupGradingByProvider, usesConversationVar }) {
+	if (!runEvalOptions.some(({ test }) => shouldDeferGradingForTest(test))) return;
+	if (shouldGroupGradingByProvider) {
+		logger.info("Grouping model-graded assertions by provider to minimize local-model reload overhead.");
+		return;
+	}
+	if (concurrency !== 1) return;
+	const reasons = [];
+	if (hasEvalStepTimeout) reasons.push("per-eval-step timeout is configured");
+	if (usesConversationVar) reasons.push("conversation variables require per-row ordering");
+	if (reasons.length > 0) logger.info(`Serial grading grouping disabled because ${reasons.join(" and ")}; model-graded judges may reload between rows.`);
+}
 function applyGradingResult(row, checkResult) {
 	if (!checkResult.pass) {
 		row.error = checkResult.reason;
@@ -5381,14 +5465,29 @@ function applyGradingResult(row, checkResult) {
 	if (checkResult.tokensUsed) accumulateAssertionTokenUsage(row.tokenUsage.assertions, checkResult.tokensUsed);
 	row.gradingResult = checkResult;
 }
-function applyGradingError(row, error) {
-	const errorMessage = error instanceof Error ? error.stack ?? error.message : String(error);
-	logger.error("Assertion grading failed during eval", {
-		error: errorMessage,
-		promptIdx: row.promptIdx,
-		testIdx: row.testIdx
-	});
-	row.error = errorMessage;
+const ABORTED_GRADING_PREFIX = "Aborted: ";
+function isAbortShapedError(error) {
+	return error instanceof Error && (error.name === "AbortError" || error.name === "AbortException");
+}
+function applyGradingError(row, error, abortSignal) {
+	const errorAsError = error instanceof Error ? error : void 0;
+	if (Boolean(abortSignal?.aborted) && isAbortShapedError(error)) {
+		const shortMessage = errorAsError?.message ?? String(error);
+		logger.debug("Assertion grading aborted", {
+			error: shortMessage,
+			promptIdx: row.promptIdx,
+			testIdx: row.testIdx
+		});
+		row.error = `${ABORTED_GRADING_PREFIX}${shortMessage}`;
+	} else {
+		const fullMessage = errorAsError ? errorAsError.stack ?? errorAsError.message : String(error);
+		logger.error("Assertion grading failed during eval", {
+			error: fullMessage,
+			promptIdx: row.promptIdx,
+			testIdx: row.testIdx
+		});
+		row.error = fullMessage;
+	}
 	row.failureReason = ResultFailureReason.ERROR;
 	row.success = false;
 	row.score = 0;
@@ -5420,7 +5519,7 @@ function createRunEvalState({ provider, prompt, test }) {
 	};
 }
 function attachConversationVar({ conversations, conversationKey, prompt, test, vars }) {
-	const usesConversation = prompt.raw.includes("_conversation");
+	const usesConversation = promptUsesConversationVariable(prompt);
 	if (!getEnvBool("PROMPTFOO_DISABLE_CONVERSATION_VAR") && !test.options?.disableConversationVar && usesConversation) vars._conversation = conversations?.[conversationKey] || [];
 }
 function createRunEvalSetup({ provider, prompt, promptConfig, vars }) {
@@ -5667,7 +5766,7 @@ async function gradeRunEvalResponse({ abortSignal, deferGrading, evalId, latency
 			assertScoringFunction: test.assertScoringFunction,
 			traceId
 		}).then((checkResult) => applyGradingResult(ret, checkResult))).catch((error) => {
-			applyGradingError(ret, error);
+			applyGradingError(ret, error, abortSignal);
 		});
 		deferredGradingPromises.set(ret, gradingPromise);
 		return;
@@ -6214,7 +6313,7 @@ async function resolveDefaultTestProvider(defaultTest, testCase) {
 	const defaultProvider = defaultTest.provider;
 	if (isApiProvider(defaultProvider)) return defaultProvider;
 	if (typeof defaultProvider === "object" && defaultProvider.id) {
-		const { loadApiProvider } = await import("./providers-CCE2COJi2.js");
+		const { loadApiProvider } = await import("./providers-DVYRZP4E.js").then((n) => n.i);
 		return loadApiProvider(typeof defaultProvider.id === "function" ? defaultProvider.id() : defaultProvider.id, { options: defaultProvider });
 	}
 	return defaultProvider;
@@ -6374,7 +6473,7 @@ function buildRepeatCacheContextByTestIdx(runEvalOptions) {
 async function filterCompletedResumeSteps(runEvalOptions, evalRecord) {
 	if (!state.resume || !evalRecord.persisted) return;
 	try {
-		const { default: EvalResult } = await import("./evalResult-pSvGWFMo.js").then((n) => n.n);
+		const { default: EvalResult } = await import("./evalResult-2RRJvFyB.js").then((n) => n.n);
 		const completedPairs = await EvalResult.getCompletedIndexPairs(evalRecord.id, { excludeErrors: state.retryMode });
 		const originalCount = runEvalOptions.length;
 		for (let i = runEvalOptions.length - 1; i >= 0; i--) {
@@ -6388,14 +6487,14 @@ async function filterCompletedResumeSteps(runEvalOptions, evalRecord) {
 	}
 }
 function adjustConcurrencyForSerialFeatures({ concurrency, prompts, tests }) {
-	const usesConversationVar = prompts.some((p) => p.raw.includes("_conversation"));
+	const usesConversationVar = prompts.some(promptUsesConversationVariable);
 	if (concurrency <= 1) return {
 		concurrency,
 		usesConversationVar
 	};
 	const usesStoreOutputAs = tests.some((t) => t.options?.storeOutputAs);
 	if (usesConversationVar) {
-		logger.info(`Setting concurrency to 1 because the ${chalk.cyan("_conversation")} variable is used.`);
+		logger.info(`Setting concurrency to 1 because the ${chalk.cyan(CONVERSATION_VAR_NAME)} variable is used.`);
 		return {
 			concurrency: 1,
 			usesConversationVar
@@ -6625,7 +6724,8 @@ var Evaluator = class {
 		};
 		this.conversations = {};
 		this.registers = {};
-		this.fileWriters = (Array.isArray(evalRecord.config.outputPath) ? evalRecord.config.outputPath.filter((p) => p.endsWith(".jsonl")) : evalRecord.config.outputPath?.endsWith(".jsonl") ? [evalRecord.config.outputPath] : []).map((p) => new JsonlFileWriter(p));
+		const jsonlFiles = Array.isArray(evalRecord.config.outputPath) ? evalRecord.config.outputPath.filter((p) => p.endsWith(".jsonl")) : evalRecord.config.outputPath?.endsWith(".jsonl") ? [evalRecord.config.outputPath] : [];
+		this.fileWriters = jsonlFiles.map((p) => new JsonlFileWriter(p));
 		this.rateLimitRegistry = createRateLimitRegistry({ maxConcurrency: options.maxConcurrency || 4 });
 		this.rateLimitRegistry.on("ratelimit:hit", (data) => {
 			logger.debug(`[Scheduler] Rate limit hit for ${data.rateLimitKey}`, {
@@ -6745,6 +6845,25 @@ var Evaluator = class {
 			this.trackCompletedRow(evalStep, row, context);
 			context.numComplete++;
 			const promptEvalCount = reservePromptEvalCount(context, row.promptIdx);
+			if (context.testSuite.extensions?.length) try {
+				const afterEachOut = await runExtensionHook(context.testSuite.extensions, "afterEach", {
+					test: evalStep.test,
+					result: {
+						...row,
+						namedScores: { ...row.namedScores },
+						metadata: { ...row.metadata },
+						response: row.response ? {
+							...row.response,
+							metadata: { ...row.response.metadata }
+						} : row.response
+					}
+				});
+				row.namedScores = filterFiniteScores(afterEachOut.result.namedScores);
+				row.metadata = afterEachOut.result.metadata;
+				if (row.response && afterEachOut.result.response) row.response.metadata = afterEachOut.result.response.metadata;
+			} catch (error) {
+				logger.error(`afterEach extension hook failed, persisting row without hook modifications`, { error });
+			}
 			await this.persistEvalRow(row);
 			if (this.abortIfTargetUnavailable(row, context)) break;
 			const metrics = context.prompts[row.promptIdx].metrics;
@@ -6756,10 +6875,6 @@ var Evaluator = class {
 				promptEvalCount,
 				row
 			});
-			await runExtensionHook(context.testSuite.extensions, "afterEach", {
-				test: evalStep.test,
-				result: row
-			});
 			context.options.progressCallback?.(context.numComplete, context.runEvalOptionsLength, index, evalStep, metrics);
 		}
 	}
@@ -6914,7 +7029,15 @@ var Evaluator = class {
 				})) break;
 			}
 		} catch (error) {
-			await flushGroupedRows();
+			const pendingRowCount = groupedRows.reduce((sum, entry) => sum + entry.rows.length, 0);
+			try {
+				await flushGroupedRows();
+			} catch (flushError) {
+				logger.warn("Failed to flush grouped rows after error; target outputs may be lost", {
+					error: flushError instanceof Error ? flushError.message : String(flushError),
+					pendingRowCount
+				});
+			}
 			throw error;
 		}
 		await flushGroupedRows();
@@ -7350,6 +7473,13 @@ var Evaluator = class {
 		if (!this.options.silent) {
 			if (serialRunEvalOptions.length > 0) logger.info(`Running ${serialRunEvalOptions.length} test cases serially...`);
 			if (concurrentRunEvalOptions.length > 0) logger.info(`Running ${concurrentRunEvalOptions.length} test cases (up to ${concurrency} at a time)...`);
+			logGroupedGradingStatus({
+				concurrency,
+				hasEvalStepTimeout,
+				runEvalOptions,
+				shouldGroupGradingByProvider,
+				usesConversationVar
+			});
 		}
 		if (this.options.showProgressBar && progressBarManager) {
 			await progressBarManager.initialize(runEvalOptions, concurrency, 0);
@@ -8353,7 +8483,7 @@ var Eval = class Eval {
 		const evalInstance = new Eval(eval_.config, {
 			id: eval_.id,
 			createdAt: new Date(eval_.createdAt),
-			author: eval_.author || void 0,
+			author: eval_.author,
 			description: eval_.description || void 0,
 			prompts: eval_.prompts || [],
 			datasetId,
@@ -8376,7 +8506,7 @@ var Eval = class Eval {
 		return (await getDb().select().from(evalsTable).limit(limit).orderBy(desc(evalsTable.createdAt)).all()).map((e) => new Eval(e.config, {
 			id: e.id,
 			createdAt: new Date(e.createdAt),
-			author: e.author || void 0,
+			author: e.author,
 			description: e.description || void 0,
 			prompts: e.prompts || [],
 			persisted: true
@@ -8391,7 +8521,7 @@ var Eval = class Eval {
 		return (await getDb().select().from(evalsTable).orderBy(desc(evalsTable.createdAt)).limit(limit).offset(offset).all()).map((e) => new Eval(e.config, {
 			id: e.id,
 			createdAt: new Date(e.createdAt),
-			author: e.author || void 0,
+			author: e.author,
 			description: e.description || void 0,
 			prompts: e.prompts || [],
 			persisted: true
@@ -8406,7 +8536,7 @@ var Eval = class Eval {
 	static async create(config, renderedPrompts, opts) {
 		const createdAt = opts?.createdAt || /* @__PURE__ */ new Date();
 		const evalId = opts?.id || createEvalId(createdAt);
-		const author = opts?.author || getUserEmail();
+		const author = opts && "author" in opts ? opts.author ?? null : getAuthor();
 		const db = getDb();
 		const datasetId = sha256(JSON.stringify(config.tests || []));
 		db.transaction(() => {
@@ -8468,7 +8598,7 @@ var Eval = class Eval {
 		});
 		return new Eval(config, {
 			id: evalId,
-			author: opts?.author,
+			author,
 			createdAt,
 			persisted: true,
 			runtimeOptions: sanitizeRuntimeOptions(opts?.runtimeOptions)
@@ -8478,7 +8608,7 @@ var Eval = class Eval {
 		const createdAt = opts?.createdAt || /* @__PURE__ */ new Date();
 		this.createdAt = createdAt.getTime();
 		this.id = opts?.id || createEvalId(createdAt);
-		this.author = opts?.author;
+		this.author = opts?.author ?? null;
 		this.config = config;
 		this.results = [];
 		this.prompts = opts?.prompts || [];
@@ -9008,7 +9138,7 @@ var Eval = class Eval {
 		newConfig.description = copyDescription;
 		const newPrompts = structuredClone(this.prompts);
 		const newVars = this.vars ? structuredClone(this.vars) : [];
-		const author = getUserEmail();
+		const author = getAuthor();
 		const db = getDb();
 		let copiedCount = 0;
 		db.transaction(() => {
@@ -11402,25 +11532,45 @@ var AlignedHarmfulPlugin = class extends RedteamPluginBase {
 	getAssertions(_prompt) {
 		return getHarmfulAssertions(this.harmCategory);
 	}
-	promptsToTestCases(prompts) {
+	async promptsToTestCases(prompts) {
 		const hasMultipleInputs = this.config.inputs && Object.keys(this.config.inputs).length > 0;
 		const harmCategoryLabel = HARM_PLUGINS[this.harmCategory] || this.harmCategory;
-		return prompts.map(({ __prompt }) => {
+		const pluginId = getShortPluginId(this.harmCategory);
+		return Promise.all([...prompts].sort((a, b) => a.__prompt.localeCompare(b.__prompt)).map(async ({ __prompt }, materializationIndex) => {
 			const vars = { [this.injectVar]: __prompt };
-			if (hasMultipleInputs) try {
-				const parsed = JSON.parse(__prompt);
-				Object.assign(vars, extractVariablesFromJson(parsed, this.config.inputs));
-			} catch {}
+			let inputMaterialization;
+			if (hasMultipleInputs) {
+				let parsed;
+				try {
+					parsed = JSON.parse(__prompt);
+				} catch (error) {
+					logger.debug("[AlignedHarmful] Could not parse prompt as JSON for multi-input mode", { error });
+				}
+				if (parsed) try {
+					const materializedVars = await extractMaterializedVariablesFromJsonWithMetadata(parsed, this.config.inputs, {
+						materializationIndex,
+						pluginId,
+						provider: this.provider,
+						purpose: this.purpose
+					});
+					Object.assign(vars, materializedVars.vars);
+					inputMaterialization = materializedVars.metadata;
+				} catch (error) {
+					logger.debug("[AlignedHarmful] Failed to materialize prompt inputs", { error });
+					throw error;
+				}
+			}
 			return {
 				vars,
 				metadata: {
 					harmCategory: harmCategoryLabel,
-					pluginId: getShortPluginId(this.harmCategory),
-					pluginConfig: this.config
+					pluginId,
+					pluginConfig: this.config,
+					...inputMaterialization ? { inputMaterialization } : {}
 				},
 				assert: getHarmfulAssertions(this.harmCategory)
 			};
-		});
+		}));
 	}
 };
 //#endregion
@@ -11429,20 +11579,37 @@ var AlignedHarmfulPlugin = class extends RedteamPluginBase {
 * Extract content from <Prompt> tags and parse JSON if inputs are defined.
 * Returns the processed prompt and any additional vars extracted from JSON.
 */
-function processPromptForInputs(prompt, _injectVar, inputs) {
+async function processPromptForInputs(prompt, inputs, plugin, provider, purpose, materializationIndex) {
 	let processedPrompt = prompt.trim();
 	const additionalVars = {};
+	let additionalMetadata;
 	const extractedPrompt = extractPromptFromTags(processedPrompt);
 	if (extractedPrompt) processedPrompt = extractedPrompt;
-	if (inputs && Object.keys(inputs).length > 0) try {
-		const parsed = JSON.parse(processedPrompt);
-		Object.assign(additionalVars, extractVariablesFromJson(parsed, inputs));
-	} catch {
-		logger.debug("[Harmful] Could not parse prompt as JSON for multi-input mode");
+	if (inputs && Object.keys(inputs).length > 0) {
+		let parsed;
+		try {
+			parsed = JSON.parse(processedPrompt);
+		} catch (error) {
+			logger.debug("[Harmful] Could not parse prompt as JSON for multi-input mode", { error });
+		}
+		if (parsed) try {
+			const materializedVars = await extractMaterializedVariablesFromJsonWithMetadata(parsed, inputs, {
+				materializationIndex,
+				pluginId: plugin,
+				provider,
+				purpose
+			});
+			Object.assign(additionalVars, materializedVars.vars);
+			additionalMetadata = materializedVars.metadata;
+		} catch (error) {
+			logger.debug("[Harmful] Failed to materialize prompt inputs", { error });
+			throw error;
+		}
 	}
 	return {
 		processedPrompt,
-		additionalVars
+		additionalVars,
+		additionalMetadata
 	};
 }
 async function getHarmfulTests({ purpose, injectVar, n, delayMs = 0, config }, plugin) {
@@ -11463,15 +11630,19 @@ async function getHarmfulTests({ purpose, injectVar, n, delayMs = 0, config }, p
 	};
 	const allPrompts = await retryWithDeduplication(generatePrompts, n);
 	const inputs = config?.inputs;
-	return sampleArray(allPrompts, n).map((prompt) => {
-		const { processedPrompt, additionalVars } = processPromptForInputs(prompt, injectVar, inputs);
+	return Promise.all(sampleArray(allPrompts, n).map(async (prompt, materializationIndex) => {
+		const { processedPrompt, additionalVars, additionalMetadata } = await processPromptForInputs(prompt, inputs, plugin, unalignedProvider, purpose, materializationIndex);
 		const testCase = createTestCase(injectVar, processedPrompt, plugin);
 		if (Object.keys(additionalVars).length > 0) testCase.vars = {
 			...testCase.vars,
 			...additionalVars
 		};
+		if (additionalMetadata) testCase.metadata = {
+			...testCase.metadata,
+			inputMaterialization: additionalMetadata
+		};
 		return testCase;
-	});
+	}));
 }
 //#endregion
 //#region src/redteam/plugins/teenSafety/graderExamples.ts
@@ -11789,7 +11960,7 @@ const MAX_CHARS_RETRY_MODIFIER_KEY = "__maxCharsPerMessageRetry";
 function computeModifiersFromConfig(config) {
 	const modifiers = { ...config?.modifiers };
 	if (config?.language && typeof config.language === "string") modifiers.language = config.language;
-	if (config?.inputs && Object.keys(config.inputs).length > 0) modifiers.__outputFormat = `Output each test case as JSON wrapped in <Prompt> tags: <Prompt>{${Object.entries(config.inputs).map(([k, description]) => `"${k}": "${description}"`).join(", ")}}</Prompt>`;
+	if (config?.inputs && Object.keys(config.inputs).length > 0) modifiers.__outputFormat = `Output each test case as JSON wrapped in <Prompt> tags: <Prompt>{${Object.entries(buildPromptInputDescriptions(config.inputs) ?? {}).map(([k, description]) => `"${k}": "${description}"`).join(", ")}}</Prompt>`;
 	const maxCharsModifier = getMaxCharsPerMessageModifierValue(config?.maxCharsPerMessage);
 	if (maxCharsModifier) modifiers[MAX_CHARS_PER_MESSAGE_MODIFIER_KEY] = maxCharsModifier;
 	return modifiers;
@@ -11955,6 +12126,31 @@ async function fetchRemoteTestCases(key, purpose, injectVar, n, config) {
 		return [];
 	}
 }
+async function materializeRemoteTestCaseInputs({ config, injectVar, pluginId, provider, purpose, testCases }) {
+	const inputs = config.inputs;
+	if (!inputs || Object.keys(inputs).length === 0) return testCases;
+	return Promise.all(testCases.map(async (testCase, materializationIndex) => {
+		const inputVars = extractInputVarsFromPrompt(String(testCase.vars?.[injectVar] ?? ""), inputs);
+		if (!inputVars) return testCase;
+		const materializedVars = await materializeInputVariablesWithMetadata(inputVars, inputs, {
+			materializationIndex,
+			pluginId,
+			provider,
+			purpose
+		});
+		return {
+			...testCase,
+			vars: {
+				...testCase.vars || {},
+				...materializedVars.vars
+			},
+			metadata: {
+				...testCase.metadata || {},
+				...materializedVars.metadata ? { inputMaterialization: materializedVars.metadata } : {}
+			}
+		};
+	}));
+}
 function createPluginFactory(PluginClass, key, validate) {
 	return {
 		key,
@@ -11965,13 +12161,21 @@ function createPluginFactory(PluginClass, key, validate) {
 				logger.debug(`Using local redteam generation for ${key}`);
 				return new PluginClass(provider, purpose, injectVar, configWithDefaults).generateTests(n, delayMs);
 			}
-			const testCases = await fetchRemoteTestCases(key, purpose, injectVar, n, configWithDefaults ?? {});
+			const pluginId = getShortPluginId(key);
+			const testCases = await materializeRemoteTestCaseInputs({
+				config: configWithDefaults ?? {},
+				injectVar,
+				pluginId,
+				provider,
+				purpose,
+				testCases: await fetchRemoteTestCases(key, purpose, injectVar, n, configWithDefaults ?? {})
+			});
 			const computedModifiers = computeModifiersFromConfig(configWithDefaults);
 			return testCases.map((testCase) => ({
 				...testCase,
 				metadata: {
 					...testCase.metadata,
-					pluginId: getShortPluginId(key),
+					pluginId,
 					pluginConfig: {
 						...configWithDefaults,
 						modifiers: computedModifiers
@@ -12028,7 +12232,7 @@ const pluginFactories = [
 		key: category,
 		action: async (params) => {
 			if (neverGenerateRemote()) {
-				logger.error(`${category} plugin requires remote generation to be enabled`);
+				logger.error(getRemoteGenerationExplicitlyDisabledError(`${category} plugin`));
 				return [];
 			}
 			const testCases = await getHarmfulTests(params, category);
@@ -12051,13 +12255,21 @@ const piiPlugins = PII_PLUGINS.map((category) => ({
 	key: category,
 	action: async (params) => {
 		if (shouldGenerateRemote()) {
-			const testCases = await fetchRemoteTestCases(category, params.purpose, params.injectVar, params.n, params.config ?? {});
+			const pluginId = getShortPluginId(category);
+			const testCases = await materializeRemoteTestCaseInputs({
+				config: params.config ?? {},
+				injectVar: params.injectVar,
+				pluginId,
+				provider: params.provider,
+				purpose: params.purpose,
+				testCases: await fetchRemoteTestCases(category, params.purpose, params.injectVar, params.n, params.config ?? {})
+			});
 			const computedModifiers = computeModifiersFromConfig(params.config);
 			return testCases.map((testCase) => ({
 				...testCase,
 				metadata: {
 					...testCase.metadata,
-					pluginId: getShortPluginId(category),
+					pluginId,
 					pluginConfig: {
 						...params.config,
 						modifiers: computedModifiers
@@ -12079,16 +12291,24 @@ const biasPlugins = BIAS_PLUGINS.map((category) => ({
 	key: category,
 	action: async (params) => {
 		if (neverGenerateRemote()) {
-			logger.error(`${category} plugin requires remote generation to be enabled`);
+			logger.error(getRemoteGenerationExplicitlyDisabledError(`${category} plugin`));
 			return [];
 		}
-		const testCases = await fetchRemoteTestCases(category, params.purpose, params.injectVar, params.n, params.config ?? {});
+		const pluginId = getShortPluginId(category);
+		const testCases = await materializeRemoteTestCaseInputs({
+			config: params.config ?? {},
+			injectVar: params.injectVar,
+			pluginId,
+			provider: params.provider,
+			purpose: params.purpose,
+			testCases: await fetchRemoteTestCases(category, params.purpose, params.injectVar, params.n, params.config ?? {})
+		});
 		const computedModifiers = computeModifiersFromConfig(params.config);
 		return testCases.map((testCase) => ({
 			...testCase,
 			metadata: {
 				...testCase.metadata,
-				pluginId: getShortPluginId(category),
+				pluginId,
 				pluginConfig: {
 					...params.config,
 					modifiers: computedModifiers
@@ -12101,19 +12321,27 @@ function createRemotePlugin(key, validate) {
 	return {
 		key,
 		validate,
-		action: async ({ purpose, injectVar, n, config }) => {
+		action: async ({ provider, purpose, injectVar, n, config }) => {
 			const configWithDefaults = applyDefaultRemotePluginConfig(key, config);
 			if (neverGenerateRemote()) {
-				logger.error(`${key} plugin requires remote generation to be enabled`);
+				logger.error(getRemoteGenerationExplicitlyDisabledError(`${key} plugin`));
 				return [];
 			}
-			const testCases = await fetchRemoteTestCases(key, purpose, injectVar, n, configWithDefaults ?? {});
+			const pluginId = getShortPluginId(key);
+			const testCases = await materializeRemoteTestCaseInputs({
+				config: configWithDefaults ?? {},
+				injectVar,
+				pluginId,
+				provider,
+				purpose,
+				testCases: await fetchRemoteTestCases(key, purpose, injectVar, n, configWithDefaults ?? {})
+			});
 			const computedModifiers = computeModifiersFromConfig(configWithDefaults);
 			const testsWithMetadata = testCases.map((testCase) => ({
 				...testCase,
 				metadata: {
 					...testCase.metadata,
-					pluginId: getShortPluginId(key),
+					pluginId,
 					pluginConfig: {
 						...configWithDefaults,
 						modifiers: computedModifiers
@@ -12183,6 +12411,37 @@ function getPolicyText(metadata) {
 		return typeof policyObject.text === "string" && policyObject.text.length > 0 ? policyObject.text : void 0;
 	}
 }
+async function rematerializeStrategyInputVars(testCase, injectVar, provider, purpose, materializationIndex) {
+	const inputs = testCase.metadata?.pluginConfig?.inputs;
+	const inputMaterialization = testCase.metadata?.inputMaterialization;
+	if (!inputs || Object.keys(inputs).length === 0 || !testCase.vars?.[injectVar]) return {
+		inputMaterialization,
+		vars: testCase.vars
+	};
+	try {
+		const materializedVars = await extractMaterializedVariablesFromJsonWithMetadata(JSON.parse(String(testCase.vars[injectVar])), inputs, {
+			materializationIndex,
+			pluginId: String(testCase.metadata?.pluginId || "unknown-plugin"),
+			provider,
+			purpose
+		});
+		return {
+			inputMaterialization: materializedVars.metadata ? {
+				...inputMaterialization,
+				...materializedVars.metadata
+			} : inputMaterialization,
+			vars: {
+				...testCase.vars,
+				...materializedVars.vars
+			}
+		};
+	} catch {
+		return {
+			inputMaterialization,
+			vars: testCase.vars
+		};
+	}
+}
 /**
 * Gets the severity level for a plugin based on its ID and configuration.
 * @param pluginId - The ID of the plugin.
@@ -12324,6 +12583,7 @@ const categories = {
 	foundation: FOUNDATION_PLUGINS,
 	harmful: Object.keys(HARM_PLUGINS),
 	"coding-agent:core": CODING_AGENT_CORE_PLUGINS,
+	"coding-agent:all": CODING_AGENT_PLUGINS,
 	bias: BIAS_PLUGINS,
 	pii: PII_PLUGINS,
 	medical: MEDICAL_PLUGINS,
@@ -12418,7 +12678,7 @@ function addLanguageToPluginMetadata(test, lang, plugin, maxCharsPerMessage, tes
 * @param injectVar - The variable to inject.
 * @returns An array of new test cases generated by strategies.
 */
-async function applyStrategies(testCases, strategies, injectVar, excludeTargetOutputFromAgenticAttackGeneration, maxCharsPerMessage) {
+async function applyStrategies(testCases, strategies, injectVar, provider, purpose, excludeTargetOutputFromAgenticAttackGeneration, maxCharsPerMessage) {
 	const newTestCases = [];
 	const strategyResults = {};
 	for (const strategy of strategies) {
@@ -12474,14 +12734,8 @@ async function applyStrategies(testCases, strategies, injectVar, excludeTargetOu
 			}
 		}
 		resultTestCases = filterOversizedTestCases(resultTestCases, injectVar, `Strategy ${strategy.id}`, maxCharsPerMessage);
-		newTestCases.push(...resultTestCases.map((t) => {
-			const inputs = t?.metadata?.pluginConfig?.inputs;
-			let updatedVars = t.vars;
-			if (inputs && Object.keys(inputs).length > 0 && t.vars?.[injectVar]) try {
-				const parsed = JSON.parse(String(t.vars[injectVar]));
-				updatedVars = { ...t.vars };
-				Object.assign(updatedVars, extractVariablesFromJson(parsed, inputs));
-			} catch {}
+		newTestCases.push(...await Promise.all(resultTestCases.map(async (t, materializationIndex) => {
+			const { inputMaterialization, vars } = await rematerializeStrategyInputVars(t, injectVar, provider, purpose, materializationIndex);
 			const strategyConfig = {
 				...strategy.config || {},
 				...maxCharsPerMessage ? { maxCharsPerMessage } : {},
@@ -12489,16 +12743,17 @@ async function applyStrategies(testCases, strategies, injectVar, excludeTargetOu
 			};
 			return {
 				...t,
-				vars: updatedVars,
+				vars,
 				metadata: {
 					...t?.metadata || {},
 					...strategy.id !== "retry" && { strategyId: t?.metadata?.strategyId || strategy.id },
 					...t?.metadata?.pluginId && { pluginId: t.metadata.pluginId },
 					...t?.metadata?.pluginConfig && { pluginConfig: t.metadata.pluginConfig },
+					...inputMaterialization && { inputMaterialization },
 					...Object.keys(strategyConfig).length > 0 && { strategyConfig }
 				}
 			};
-		}));
+		})));
 		const displayId = strategy.id === "layer" && Array.isArray(strategy.config?.steps) ? `layer(${strategy.config.steps.map((st) => typeof st === "string" ? st : st.id).join("→")})` : strategy.id;
 		const languagesInResults = new Set(strategyTestCases.map((t) => getLanguageForTestCase(t)).filter((lang) => lang !== void 0));
 		const applyNumTestsCap = (calculatedRequested) => {
@@ -12953,7 +13208,7 @@ async function synthesize({ abortSignal, delay, entities: entitiesOverride, inje
 			targetIds,
 			...retryStrategy.config
 		};
-		const { testCases: retryTestCases, strategyResults: retryResults } = await applyStrategies(pluginTestCases, [retryStrategy], injectVar, void 0, maxCharsPerMessage);
+		const { testCases: retryTestCases, strategyResults: retryResults } = await applyStrategies(pluginTestCases, [retryStrategy], injectVar, redteamProvider, purpose, void 0, maxCharsPerMessage);
 		pluginTestCases.push(...retryTestCases);
 		Object.assign(strategyResults, retryResults);
 		if (showProgressBar) progressBar?.increment(retryTestCases.length);
@@ -12961,7 +13216,7 @@ async function synthesize({ abortSignal, delay, entities: entitiesOverride, inje
 	checkAbort();
 	const nonBasicStrategies = strategies.filter((s) => !["basic", "retry"].includes(s.id));
 	if (showProgressBar && nonBasicStrategies.length > 0) progressBar?.update({ task: "Applying strategies" });
-	const { testCases: strategyTestCases, strategyResults: otherStrategyResults } = await applyStrategies(pluginTestCases, nonBasicStrategies, injectVar, excludeTargetOutputFromAgenticAttackGeneration, maxCharsPerMessage);
+	const { testCases: strategyTestCases, strategyResults: otherStrategyResults } = await applyStrategies(pluginTestCases, nonBasicStrategies, injectVar, redteamProvider, purpose, excludeTargetOutputFromAgenticAttackGeneration, maxCharsPerMessage);
 	Object.assign(strategyResults, otherStrategyResults);
 	if (showProgressBar && strategyTestCases.length > 0) progressBar?.increment(strategyTestCases.length);
 	const finalTestCases = [...includeBasicTests ? pluginTestCases : [], ...strategyTestCases];
@@ -13854,6 +14109,10 @@ function stripAuthFromUrl(urlString) {
 	}
 }
 async function handleEmailCollection(evalRecord) {
+	if (evalRecord.author) {
+		logger.debug(`[Share] Skipping email collection because author is already set`, { evalId: evalRecord.id });
+		return;
+	}
 	if (!process.stdout.isTTY || isCI() || getEnvBool("PROMPTFOO_DISABLE_SHARE_EMAIL_REQUEST")) return;
 	let email = getUserEmail();
 	if (!email) {
@@ -13997,6 +14256,14 @@ function shouldShareResults(opts) {
 	return cloudConfig.isEnabled() && sharing !== false;
 }
 //#endregion
+//#region src/commands/eval/redteamWarning.ts
+function warnIfRedteamConfigHasNoTests(config, testSuite) {
+	if (config.redteam && (!testSuite.tests || testSuite.tests.length === 0) && (!testSuite.scenarios || testSuite.scenarios.length === 0)) logger.warn(chalk.yellow(dedent`
+        Warning: Config file has a redteam section but no test cases.
+        Did you mean to run ${chalk.bold("promptfoo redteam generate")} instead?
+        `));
+}
+//#endregion
 //#region src/util/formatDuration.ts
 /**
 * Formats a duration in seconds into a human-readable string
@@ -14437,14 +14704,11 @@ async function doEval(cmdObj, defaultConfig, defaultConfigPath, evaluateOptions)
 			state.resume = true;
 			state.retryMode = true;
 		} else ({config, testSuite, basePath: _basePath, commandLineOptions} = await resolveConfigs(cmdObj, defaultConfig));
-		if (!cmdObj.envPath && commandLineOptions?.envPath) {
+		if ((!cmdObj.envPath || cmdObj.envPath.length === 0) && commandLineOptions?.envPath) {
 			logger.debug(`Loading additional environment from config: ${commandLineOptions.envPath}`);
 			setupEnv(commandLineOptions.envPath);
 		}
-		if (config.redteam && (!testSuite.tests || testSuite.tests.length === 0) && (!testSuite.scenarios || testSuite.scenarios.length === 0)) logger.warn(chalk.yellow(dedent`
-        Warning: Config file has a redteam section but no test cases.
-        Did you mean to run ${chalk.bold("promptfoo redteam generate")} instead?
-        `));
+		warnIfRedteamConfigHasNoTests(config, testSuite);
 		if (config.redteam && Array.isArray(config.providers) && config.providers.length > 0 && typeof config.providers[0] === "object" && config.providers[0].id === "http") {
 			const maybeUrl = config.providers[0]?.config?.url;
 			if (typeof maybeUrl === "string" && maybeUrl.includes("promptfoo.app")) telemetry.record("feature_used", { feature: "redteam_run_with_example" });
@@ -14552,7 +14816,14 @@ async function doEval(cmdObj, defaultConfig, defaultConfigPath, evaluateOptions)
         ${z.prettifyError(testSuiteSchema.error)}
 
       Please review your promptfooconfig.yaml configuration.`));
-		const evalRecord = resumeEval ? resumeEval : cmdObj.write ? await Eval.create(config, testSuite.prompts, { runtimeOptions: options }) : new Eval(config, { runtimeOptions: options });
+		const author = getAuthor();
+		const evalRecord = resumeEval ? resumeEval : cmdObj.write ? await Eval.create(config, testSuite.prompts, {
+			author,
+			runtimeOptions: options
+		}) : new Eval(config, {
+			author,
+			runtimeOptions: options
+		});
 		const abortController = new AbortController();
 		const previousAbortSignal = evaluateOptions.abortSignal;
 		evaluateOptions.abortSignal = previousAbortSignal ? AbortSignal.any([previousAbortSignal, abortController.signal]) : abortController.signal;
@@ -14954,6 +15225,12 @@ async function doRedteamRun(options) {
 	return evalResult;
 }
 //#endregion
+//#region src/types/transform.ts
+/** Runtime type guard for `TransformFunction` values. */
+function isTransformFunction(value) {
+	return typeof value === "function";
+}
+//#endregion
 //#region src/index.ts
 /**
 * Shallow-clone a test case so the caller can swap in resolved ApiProvider
@@ -14975,66 +15252,148 @@ function cloneTestForResolve(test) {
 	if (test.assert) cloned.assert = test.assert.map((assertion) => ({ ...assertion }));
 	return cloned;
 }
+function toSerializableProviderRef(provider) {
+	if (isApiProvider(provider)) return sanitizeProvider(provider);
+	if (Array.isArray(provider)) return provider.map(toSerializableProviderRef);
+	return provider;
+}
+function isRecord(value) {
+	return Boolean(value && typeof value === "object" && !Array.isArray(value));
+}
+function withSerializableProvider(record) {
+	if (!isApiProvider(record.provider)) return record;
+	return {
+		...record,
+		provider: sanitizeProvider(record.provider)
+	};
+}
+/**
+* Function-valued transforms are first-class at runtime but are silently dropped
+* by `JSON.stringify`. Persisted eval configs (drizzle-stored) must never retain
+* a function reference, so replace every `transform`-like field with a
+* `[inline function]: name` marker. Non-function values pass through unchanged.
+*
+* `droppedRef.value` is flipped to `true` the first time a function is replaced
+* so the caller can emit a single warning instead of logging per field.
+*/
+function replaceFunctionTransforms(record, droppedRef) {
+	let result;
+	for (const key of TRANSFORM_KEYS) {
+		const value = record[key];
+		if (!isTransformFunction(value)) continue;
+		if (!result) result = { ...record };
+		result[key] = value.name ? `${INLINE_FUNCTION_LABEL}: ${value.name}` : INLINE_FUNCTION_LABEL;
+		droppedRef.value = true;
+	}
+	return result ?? record;
+}
+function toSerializableAssertion(assertion, droppedRef) {
+	if (!isRecord(assertion)) return assertion;
+	let sanitizedAssertion = withSerializableProvider(assertion);
+	sanitizedAssertion = replaceFunctionTransforms(sanitizedAssertion, droppedRef);
+	if (Array.isArray(assertion.assert)) sanitizedAssertion = {
+		...sanitizedAssertion,
+		assert: assertion.assert.map((a) => toSerializableAssertion(a, droppedRef))
+	};
+	return sanitizedAssertion;
+}
+function toSerializableTestCase(test, droppedRef) {
+	if (!isRecord(test)) return test;
+	let sanitizedTest = withSerializableProvider(test);
+	if (isRecord(test.options)) {
+		let options = withSerializableProvider(test.options);
+		options = replaceFunctionTransforms(options, droppedRef);
+		if (options !== test.options) sanitizedTest = {
+			...sanitizedTest,
+			options
+		};
+	}
+	if (Array.isArray(test.assert)) sanitizedTest = {
+		...sanitizedTest,
+		assert: test.assert.map((a) => toSerializableAssertion(a, droppedRef))
+	};
+	return sanitizedTest;
+}
+function toSerializableScenario(scenario, droppedRef) {
+	if (!isRecord(scenario)) return scenario;
+	if (!Array.isArray(scenario.tests)) return scenario;
+	return {
+		...scenario,
+		tests: scenario.tests.map((t) => toSerializableTestCase(t, droppedRef))
+	};
+}
+function createSerializableUnifiedConfig(testSuite, prompts) {
+	const droppedRef = { value: false };
+	const config = {
+		...testSuite,
+		providers: toSerializableProviderRef(testSuite.providers),
+		defaultTest: toSerializableTestCase(testSuite.defaultTest, droppedRef),
+		tests: Array.isArray(testSuite.tests) ? testSuite.tests.map((t) => toSerializableTestCase(t, droppedRef)) : testSuite.tests,
+		scenarios: Array.isArray(testSuite.scenarios) ? testSuite.scenarios.map((s) => toSerializableScenario(s, droppedRef)) : testSuite.scenarios,
+		prompts
+	};
+	if (droppedRef.value && testSuite.writeLatestResults) logger.warn("Function-valued transform(s) in testSuite were replaced with \"[inline function]\" markers in the persisted config. Re-running the saved eval will not invoke them; use string expressions or file:// references if you need the config to round-trip.");
+	return config;
+}
 async function evaluate(testSuite, options = {}) {
-	if (testSuite.writeLatestResults) await runDbMigrations();
-	const loadedProviders = await loadApiProviders(testSuite.providers, { env: testSuite.env });
+	const { author: suiteAuthor, ...testSuiteConfig } = testSuite;
+	if (testSuiteConfig.writeLatestResults) await runDbMigrations();
+	const loadedProviders = await loadApiProviders(testSuiteConfig.providers, { env: testSuiteConfig.env });
 	const providerMap = {};
 	for (const p of loadedProviders) {
 		providerMap[p.id()] = p;
 		if (p.label) providerMap[p.label] = p;
 	}
-	let resolvedDefaultTest = testSuite.defaultTest;
-	if (typeof testSuite.defaultTest === "string" && testSuite.defaultTest.startsWith("file://")) resolvedDefaultTest = await maybeLoadFromExternalFile(testSuite.defaultTest);
+	let resolvedDefaultTest = testSuiteConfig.defaultTest;
+	if (typeof testSuiteConfig.defaultTest === "string" && testSuiteConfig.defaultTest.startsWith("file://")) resolvedDefaultTest = await maybeLoadFromExternalFile(testSuiteConfig.defaultTest);
 	const constructedTestSuite = {
-		...testSuite,
+		...testSuiteConfig,
 		defaultTest: resolvedDefaultTest,
-		scenarios: testSuite.scenarios,
+		scenarios: testSuiteConfig.scenarios,
 		providers: loadedProviders,
-		tests: await readTests(testSuite.tests),
-		nunjucksFilters: await readFilters(testSuite.nunjucksFilters || {}),
-		prompts: await processPrompts(testSuite.prompts)
+		tests: await readTests(testSuiteConfig.tests),
+		nunjucksFilters: await readFilters(testSuiteConfig.nunjucksFilters || {}),
+		prompts: await processPrompts(testSuiteConfig.prompts)
 	};
 	if (typeof constructedTestSuite.defaultTest === "object" && constructedTestSuite.defaultTest) {
 		constructedTestSuite.defaultTest = cloneTestForResolve(constructedTestSuite.defaultTest);
 		if (constructedTestSuite.defaultTest.provider && !isApiProvider(constructedTestSuite.defaultTest.provider)) constructedTestSuite.defaultTest.provider = await resolveProvider(constructedTestSuite.defaultTest.provider, providerMap, {
-			env: testSuite.env,
+			env: testSuiteConfig.env,
 			basePath: state.basePath
 		});
 		if (constructedTestSuite.defaultTest.options?.provider && !isApiProvider(constructedTestSuite.defaultTest.options.provider)) constructedTestSuite.defaultTest.options.provider = await resolveProvider(constructedTestSuite.defaultTest.options.provider, providerMap, {
-			env: testSuite.env,
+			env: testSuiteConfig.env,
 			basePath: state.basePath
 		});
 	}
 	constructedTestSuite.tests = (constructedTestSuite.tests || []).map(cloneTestForResolve);
 	for (const test of constructedTestSuite.tests) {
 		if (test.options?.provider && !isApiProvider(test.options.provider)) test.options.provider = await resolveProvider(test.options.provider, providerMap, {
-			env: testSuite.env,
+			env: testSuiteConfig.env,
 			basePath: state.basePath
 		});
 		for (const assertion of test.assert || []) {
 			if (assertion.type === "assert-set" || typeof assertion.provider === "function") continue;
 			if (assertion.provider && !isApiProvider(assertion.provider)) assertion.provider = await resolveProvider(assertion.provider, providerMap, {
-				env: testSuite.env,
+				env: testSuiteConfig.env,
 				basePath: state.basePath
 			});
 		}
 	}
 	if (options.cache === false) disableCache();
-	const parsedProviderPromptMap = readProviderPromptMap(testSuite, constructedTestSuite.prompts);
-	const unifiedConfig = {
-		...testSuite,
-		prompts: constructedTestSuite.prompts
-	};
-	const evalRecord = testSuite.writeLatestResults ? await Eval.create(unifiedConfig, constructedTestSuite.prompts) : new Eval(unifiedConfig);
+	const parsedProviderPromptMap = readProviderPromptMap(testSuiteConfig, constructedTestSuite.prompts);
+	const unifiedConfig = createSerializableUnifiedConfig(testSuiteConfig, constructedTestSuite.prompts);
+	const author = getAuthor(suiteAuthor);
+	const evalRecord = testSuiteConfig.writeLatestResults ? await Eval.create(unifiedConfig, constructedTestSuite.prompts, { author }) : new Eval(unifiedConfig, { author });
 	const ret = await evaluate$1({
 		...constructedTestSuite,
 		providerPromptMap: parsedProviderPromptMap
 	}, evalRecord, {
 		eventSource: "library",
-		isRedteam: Boolean(testSuite.redteam),
+		isRedteam: Boolean(testSuiteConfig.redteam),
 		...options
 	});
-	if (testSuite.writeLatestResults && testSuite.sharing) if (isSharingEnabled(ret)) try {
+	if (testSuiteConfig.writeLatestResults && testSuiteConfig.sharing) if (isSharingEnabled(ret)) try {
 		const shareableUrl = await createShareableUrl(ret, { silent: true });
 		if (shareableUrl) {
 			ret.shareableUrl = shareableUrl;
@@ -15045,9 +15404,9 @@ async function evaluate(testSuite, options = {}) {
 		logger.warn(`Failed to create shareable URL: ${error}`);
 	}
 	else logger.debug("Sharing requested but not enabled (check cloud config or sharing settings)");
-	if (testSuite.outputPath) {
-		if (typeof testSuite.outputPath === "string") await writeOutput(testSuite.outputPath, evalRecord, null);
-		else if (Array.isArray(testSuite.outputPath)) await writeMultipleOutputs(testSuite.outputPath, evalRecord, null);
+	if (testSuiteConfig.outputPath) {
+		if (typeof testSuiteConfig.outputPath === "string") await writeOutput(testSuiteConfig.outputPath, evalRecord, null);
+		else if (Array.isArray(testSuiteConfig.outputPath)) await writeMultipleOutputs(testSuiteConfig.outputPath, evalRecord, null);
 	}
 	return ret;
 }
@@ -15076,6 +15435,6 @@ var src_default = {
 	redteam
 };
 //#endregion
-export { AssertionOrSetSchema, AssertionSchema, AssertionSetSchema, AssertionTypeSchema, AtomicTestCaseSchema, BaseAssertionTypesSchema, BaseTokenUsageSchema, CommandLineOptionsSchema, CompletedPromptSchema, CompletionTokenDetailsSchema, ConversationMessageSchema, DerivedMetricSchema, EvalResultsFilterMode, EvaluateOptionsSchema, GradingConfigSchema, InputsSchema, NotPrefixedAssertionTypesSchema, OutputConfigSchema, OutputFileExtension, PartialGenerationError, PluginConfigSchema, PolicyObjectSchema, ProvidersSchema, ResultFailureReason, ScenarioSchema, SpecialAssertionTypesSchema, StrategyConfigSchema, TestCaseSchema, TestCaseWithVarsFileSchema, TestCasesWithMetadataPromptSchema, TestCasesWithMetadataSchema, TestGeneratorConfigSchema, TestSuiteConfigSchema, TestSuiteSchema, UnifiedConfigSchema, VarsSchema, assertions_default as assertions, cache_exports as cache, src_default as default, evaluate, generateTable, guardrails, isApiProvider, isGradingResult, isProviderOptions, isResultFailureReason, loadApiProvider, redteam };
+export { AssertionOrSetSchema, AssertionSchema, AssertionSetSchema, AssertionTypeSchema, AtomicTestCaseSchema, BaseAssertionTypesSchema, BaseTokenUsageSchema, CommandLineOptionsSchema, CompletedPromptSchema, CompletionTokenDetailsSchema, ConversationMessageSchema, DerivedMetricSchema, DocumentMediaInjectionPlacementSchema, DocumentMediaInjectionPlacementValues, DocxInjectionPlacementSchema, DocxInjectionPlacementValues, EvalResultsFilterMode, EvaluateOptionsSchema, GradingConfigSchema, InputConfigSchema, InputDefinitionObjectSchema, InputDefinitionSchema, InputTypeSchema, InputTypeValues, InputsSchema, NotPrefixedAssertionTypesSchema, OutputConfigSchema, OutputFileExtension, PartialGenerationError, PluginConfigSchema, PolicyObjectSchema, ProvidersSchema, ResultFailureReason, ScenarioSchema, SpecialAssertionTypesSchema, StrategyConfigSchema, TestCaseSchema, TestCaseWithVarsFileSchema, TestCasesWithMetadataPromptSchema, TestCasesWithMetadataSchema, TestGeneratorConfigSchema, TestSuiteConfigSchema, TestSuiteSchema, UnifiedConfigSchema, VarsSchema, assertions_default as assertions, buildInputPromptDescription, cache_exports as cache, src_default as default, evaluate, generateTable, getInputDescription, getInputType, guardrails, isApiProvider, isGradingResult, isProviderOptions, isResultFailureReason, isTransformFunction, loadApiProvider, normalizeInputDefinition, normalizeInputs, redteam };
 
 //# sourceMappingURL=index.js.map