promptfoo 0.121.5 → 0.121.8

package/dist/src/index.cjs

@@ -2,78 +2,93 @@ Object.defineProperties(exports, {
 	__esModule: { value: true },
 	[Symbol.toStringTag]: { value: "Module" }
 });
-const require_logger = require("./logger-COuQb2xB.cjs");
-const require_invariant = require("./invariant-kfQ8Bu82.cjs");
-const require_fetch = require("./fetch-NuqXW1Xb.cjs");
-const require_version = require("./version-D9zu9FWB.cjs");
-const require_types = require("./types-CgG2rKiW.cjs");
-const require_accounts = require("./accounts-BIFntVWB.cjs");
-const require_esm = require("./esm-B_rGuPTo.cjs");
-const require_render = require("./render-tG6ir9_g.cjs");
-const require_providerRegistry = require("./providerRegistry-BESeALrr.cjs");
-const require_server = require("./server-BEECpeGG.cjs");
-const require_providers = require("./providers-eDShy16E.cjs");
-const require_pythonUtils = require("./pythonUtils-CoLaCwNY.cjs");
-const require_fileExtensions = require("./fileExtensions-D9h-8Wxg.cjs");
-const require_util = require("./util-DvpHnLt0.cjs");
-const require_tokenUsageUtils = require("./tokenUsageUtils-C9odhsbW.cjs");
-const require_blobs = require("./blobs-B95F_7vE.cjs");
-const require_tables = require("./tables-CXbaZ9y1.cjs");
-const require_extractor = require("./extractor-Dxr2J_wK.cjs");
-const require_cache = require("./cache-DGg-yTZG.cjs");
-const require_chat = require("./chat-vYqqv1gP.cjs");
-const require_transform = require("./transform-Dg4LcO1Y.cjs");
-const require_util$1 = require("./util-A5_ZsQUn.cjs");
-const require_transform$1 = require("./transform-C9izGX54.cjs");
-const require_telemetry = require("./telemetry-CEQxGnMZ.cjs");
-const require_text = require("./text-BiNME7QG.cjs");
-const require_store = require("./store-2K0kDi80.cjs");
-const require_createHash = require("./createHash-VvBIc-AW.cjs");
-const require_rubyUtils = require("./rubyUtils-DoifqkiA.cjs");
-const require_graders = require("./graders-Bw1wk_21.cjs");
-const require_utils = require("./utils-DkVeShIB.cjs");
-const require_evalResult = require("./evalResult-Cx-8OWkb.cjs");
+const require_rolldown_runtime = require("./rolldown-runtime-D_mwlA32.cjs");
+const require_logger = require("./logger-cfNpzI4o.cjs");
+const require_invariant = require("./invariant-QtnLD03y.cjs");
+const require_types = require("./types-CxJvaY2S.cjs");
+const require_fetch = require("./fetch-Cfm4EuOB.cjs");
+const require_version = require("./version-VzUqOBZk.cjs");
+const require_fileExtensions = require("./fileExtensions-BhdwzYaD.cjs");
+const require_accounts = require("./accounts-BVz5gHLK.cjs");
+const require_esm = require("./esm-BIKakvNa.cjs");
+const require_render = require("./render-BNTrbmBw.cjs");
+const require_providerRegistry = require("./providerRegistry-CZO_w7ue.cjs");
+const require_remoteGeneration = require("./remoteGeneration-BpIYlb_O.cjs");
+const require_server = require("./server-BHOEL8p8.cjs");
+const require_storage = require("./storage-A4WnAeN3.cjs");
+const require_pythonUtils = require("./pythonUtils-Cokhluq3.cjs");
+const require_util = require("./util-COnzevXE.cjs");
+const require_createHash = require("./createHash-CSiqnK5P.cjs");
+const require_cache = require("./cache-h5MWOBZI.cjs");
+const require_chat = require("./chat-DSyYuTYT.cjs");
+const require_transform = require("./transform-DJkt81VY.cjs");
+const require_util$1 = require("./util-NYQvo1C7.cjs");
+const require_providers = require("./providers-V6RBuieY.cjs");
+const require_tokenUsageUtils = require("./tokenUsageUtils-_B-P8IAi.cjs");
+const require_text = require("./text-nywWsRBM.cjs");
+const require_telemetry = require("./telemetry-C_ImlCOk.cjs");
+const require_tables = require("./tables-DBIJU0WE.cjs");
+const require_blobs = require("./blobs-B0977K1O.cjs");
+const require_processShim = require("./processShim-DSY9BV2T.cjs");
+const require_packageParser = require("./packageParser-QoCS1FMl.cjs");
+const require_rubyUtils = require("./rubyUtils-CqUWBZAt.cjs");
+const require_inputVariables = require("./inputVariables-Dq9W-Z3a.cjs");
+const require_util$2 = require("./util-DNtUsus_.cjs");
+const require_promptfoo = require("./promptfoo-ClChwT74.cjs");
+const require_graders = require("./graders-CBQ2s6gz.cjs");
+const require_shared = require("./shared-aHWko3P1.cjs");
+const require_utils = require("./utils-Ve6kuJsa.cjs");
+const require_transform$1 = require("./transform-BnSXWmU_2.cjs");
+const require_store = require("./store-B2NDDooM.cjs");
+const require_extractor = require("./extractor-bV_NOoaz.cjs");
+const require_promptLength = require("./promptLength-BbBbDHNj.cjs");
+const require_indirectWebPwn = require("./indirectWebPwn-D7NA9Nsv.cjs");
+const require_toolAttributes = require("./toolAttributes-BAjwcBf0.cjs");
+const require_evaluatorHelpers = require("./evaluatorHelpers-D06I9WFL.cjs");
+const require_evalResult = require("./evalResult-DHXs-9TL.cjs");
+const require_strategies = require("./strategies-CCcnEbFO.cjs");
 let fs = require("fs");
-fs = require_logger.__toESM(fs);
+fs = require_rolldown_runtime.__toESM(fs, 1);
 let path = require("path");
-path = require_logger.__toESM(path);
+path = require_rolldown_runtime.__toESM(path, 1);
 let async = require("async");
-async = require_logger.__toESM(async);
+async = require_rolldown_runtime.__toESM(async, 1);
 let js_yaml = require("js-yaml");
-js_yaml = require_logger.__toESM(js_yaml);
+js_yaml = require_rolldown_runtime.__toESM(js_yaml, 1);
 let node_async_hooks = require("node:async_hooks");
 let chalk = require("chalk");
-chalk = require_logger.__toESM(chalk);
+chalk = require_rolldown_runtime.__toESM(chalk, 1);
 let os = require("os");
-os = require_logger.__toESM(os);
+os = require_rolldown_runtime.__toESM(os, 1);
 let dedent = require("dedent");
-dedent = require_logger.__toESM(dedent);
+dedent = require_rolldown_runtime.__toESM(dedent, 1);
 let zod = require("zod");
 let fs_promises = require("fs/promises");
-fs_promises = require_logger.__toESM(fs_promises);
+fs_promises = require_rolldown_runtime.__toESM(fs_promises, 1);
 let util = require("util");
-util = require_logger.__toESM(util);
+util = require_rolldown_runtime.__toESM(util, 1);
 let _inquirer_input = require("@inquirer/input");
-_inquirer_input = require_logger.__toESM(_inquirer_input);
+_inquirer_input = require_rolldown_runtime.__toESM(_inquirer_input, 1);
 require("node:path");
 require("node:url");
 let crypto$1 = require("crypto");
-crypto$1 = require_logger.__toESM(crypto$1);
+crypto$1 = require_rolldown_runtime.__toESM(crypto$1, 1);
 let _opentelemetry_api = require("@opentelemetry/api");
 let readline = require("readline");
-readline = require_logger.__toESM(readline);
+readline = require_rolldown_runtime.__toESM(readline, 1);
 let csv_parse_sync = require("csv-parse/sync");
 let glob = require("glob");
-let drizzle_orm = require("drizzle-orm");
 let fast_xml_parser = require("fast-xml-parser");
-let cli_progress = require("cli-progress");
-cli_progress = require_logger.__toESM(cli_progress);
+let drizzle_orm = require("drizzle-orm");
 let url = require("url");
 let parse5 = require("parse5");
 let fastest_levenshtein = require("fastest-levenshtein");
+let cli_progress = require("cli-progress");
+cli_progress = require_rolldown_runtime.__toESM(cli_progress, 1);
 let js_rouge = require("js-rouge");
-js_rouge = require_logger.__toESM(js_rouge);
+js_rouge = require_rolldown_runtime.__toESM(js_rouge, 1);
 let node_util = require("node:util");
+let lru_cache = require("lru-cache");
 require("debounce");
 let _opentelemetry_core = require("@opentelemetry/core");
 let _opentelemetry_exporter_trace_otlp_http = require("@opentelemetry/exporter-trace-otlp-http");
@@ -82,15 +97,15 @@ let _opentelemetry_sdk_trace_node = require("@opentelemetry/sdk-trace-node");
 let _opentelemetry_semantic_conventions = require("@opentelemetry/semantic-conventions");
 let drizzle_orm_better_sqlite3_migrator = require("drizzle-orm/better-sqlite3/migrator");
 let process$1 = require("process");
-process$1 = require_logger.__toESM(process$1);
+process$1 = require_rolldown_runtime.__toESM(process$1, 1);
 let _apidevtools_json_schema_ref_parser = require("@apidevtools/json-schema-ref-parser");
-_apidevtools_json_schema_ref_parser = require_logger.__toESM(_apidevtools_json_schema_ref_parser);
+_apidevtools_json_schema_ref_parser = require_rolldown_runtime.__toESM(_apidevtools_json_schema_ref_parser, 1);
 let cli_table3 = require("cli-table3");
-cli_table3 = require_logger.__toESM(cli_table3);
+cli_table3 = require_rolldown_runtime.__toESM(cli_table3, 1);
 let chokidar = require("chokidar");
-chokidar = require_logger.__toESM(chokidar);
+chokidar = require_rolldown_runtime.__toESM(chokidar, 1);
 let ora = require("ora");
-ora = require_logger.__toESM(ora);
+ora = require_rolldown_runtime.__toESM(ora, 1);
 require("@inquirer/confirm");
 //#region src/external/matchers/conversationRelevancyTemplate.ts
 var ConversationRelevancyTemplate = class {
@@ -449,17 +464,20 @@ async function resolveContext(assertion, test, output, prompt, fallbackContext,
 			contextValue = test.vars.context;
 		}
 	} else if (fallbackContext) contextValue = fallbackContext;
-	if (assertion.contextTransform) try {
-		const outputForTransform = providerResponse?.providerTransformedOutput ?? output;
-		const transformed = await require_transform$1.transform(assertion.contextTransform, outputForTransform, {
-			vars: test.vars,
-			prompt: { label: prompt },
-			...providerResponse && providerResponse.metadata && { metadata: providerResponse.metadata }
-		});
-		require_invariant.invariant(typeof transformed === "string" || Array.isArray(transformed) && transformed.every((item) => typeof item === "string"), `contextTransform must return a string or array of strings. Got ${typeof transformed}. Check your transform expression: ${assertion.contextTransform}`);
-		contextValue = transformed;
-	} catch (error) {
-		throw new Error(`Failed to transform context using expression '${assertion.contextTransform}': ${error instanceof Error ? error.message : String(error)}`);
+	if (assertion.contextTransform) {
+		const getLabel = () => require_transform$1.getTransformLabel(assertion.contextTransform);
+		try {
+			const outputForTransform = providerResponse?.providerTransformedOutput ?? output;
+			const transformed = await require_transform$1.transform(assertion.contextTransform, outputForTransform, {
+				vars: test.vars,
+				prompt: { label: prompt },
+				...providerResponse && providerResponse.metadata && { metadata: providerResponse.metadata }
+			});
+			require_invariant.invariant(typeof transformed === "string" || Array.isArray(transformed) && transformed.every((item) => typeof item === "string"), () => `contextTransform must return a string or array of strings. Got ${typeof transformed}. Check your transform expression: ${getLabel()}`);
+			contextValue = transformed;
+		} catch (error) {
+			throw new Error(`Failed to transform context using expression '${getLabel()}': ${require_transform$1.getTransformErrorMessage(error)}`);
+		}
 	}
 	require_invariant.invariant(typeof contextValue === "string" && contextValue.length > 0 || Array.isArray(contextValue) && contextValue.length > 0 && contextValue.every((item) => typeof item === "string" && item.length > 0), "Context is required for context-based assertions. Provide either a \"context\" variable (string or array of strings) in your test case or use \"contextTransform\" to extract context from the provider response.");
 	return contextValue;
@@ -740,7 +758,7 @@ async function calculateProviderSimilarity(finalProvider, expected, output, metr
 	return calculateSimilarityScore(expectedEmbedding.embedding, outputEmbedding.embedding, metric, tokensUsed);
 }
 async function matchesSimilarity(expected, output, threshold, inverse = false, grading, metric = "cosine") {
-	if (metric === "cosine" && require_logger.state.config?.redteam && require_server.shouldGenerateRemote({ requireEmbeddingProvider: true })) try {
+	if (metric === "cosine" && require_logger.state.config?.redteam && require_remoteGeneration.shouldGenerateRemote({ requireEmbeddingProvider: true })) try {
 		return await require_graders.doRemoteGrading({
 			task: "similar",
 			expected,
@@ -802,7 +820,7 @@ async function startOtlpReceiverIfNeeded(testSuite) {
 		require_telemetry.telemetry.record("feature_used", { feature: "tracing" });
 		try {
 			require_logger.logger.debug("[EvaluatorTracing] Tracing configuration detected, starting OTLP receiver");
-			const { startOTLPReceiver } = await Promise.resolve().then(() => require("./otlpReceiver-DNSQj6bf.cjs"));
+			const { startOTLPReceiver } = await Promise.resolve().then(() => require("./otlpReceiver-CvJdBGSc.cjs"));
 			const port = testSuite.tracing.otlp.http.port || 4318;
 			const host = testSuite.tracing.otlp.http.host || "127.0.0.1";
 			const acceptFormats = normalizeOtlpAcceptFormats(testSuite.tracing.otlp.http.acceptFormats);
@@ -826,7 +844,7 @@ async function startOtlpReceiverIfNeeded(testSuite) {
 async function stopOtlpReceiverIfNeeded() {
 	if (otlpReceiverStarted) try {
 		require_logger.logger.debug("[EvaluatorTracing] Stopping OTLP receiver");
-		const { stopOTLPReceiver } = await Promise.resolve().then(() => require("./otlpReceiver-DNSQj6bf.cjs"));
+		const { stopOTLPReceiver } = await Promise.resolve().then(() => require("./otlpReceiver-CvJdBGSc.cjs"));
 		await stopOTLPReceiver();
 		otlpReceiverStarted = false;
 		require_logger.logger.info("[EvaluatorTracing] OTLP receiver stopped successfully");
@@ -861,7 +879,7 @@ async function generateTraceContextIfNeeded(test, evaluateOptions, testIdx, prom
 	}
 	if (!tracingEnabled) return null;
 	require_logger.logger.debug("[EvaluatorTracing] Importing trace store");
-	const { getTraceStore } = await Promise.resolve().then(() => require("./store-uQZ4AjPe.cjs"));
+	const { getTraceStore } = await Promise.resolve().then(() => require("./store-B2NDDooM.cjs")).then((n) => n.store_exports);
 	const traceStore = getTraceStore();
 	const traceId = generateTraceId();
 	const spanId = generateSpanId();
@@ -1493,27 +1511,67 @@ const handleIsValidFunctionCall = ({ assertion, output, provider, test }) => {
 };
 //#endregion
 //#region src/assertions/geval.ts
-const handleGEval = async ({ assertion, renderedValue, prompt, outputString, test, providerCallContext }) => {
-	require_invariant.invariant(typeof renderedValue === "string" || Array.isArray(renderedValue), "G-Eval assertion type must have a string or array of strings value");
+const handleGEval = async ({ assertion, inverse, renderedValue, prompt, outputString, test, providerCallContext }) => {
+	require_invariant.invariant(typeof renderedValue === "string" || Array.isArray(renderedValue) && renderedValue.every((value) => typeof value === "string"), "G-Eval assertion type must have a string or array of strings value");
 	const threshold = assertion.threshold ?? .7;
 	if (Array.isArray(renderedValue)) {
-		const scores = [];
-		const reasons = [];
-		for (const value of renderedValue) {
+		if (renderedValue.length === 0) return {
+			assertion,
+			pass: false,
+			score: 0,
+			reason: "G-Eval assertion requires at least one criterion string in the value array."
+		};
+		const responses = [];
+		let failure;
+		for (const [index, value] of renderedValue.entries()) {
 			const resp = await require_graders.matchesGEval(value, prompt || "", outputString, threshold, test.options, providerCallContext);
-			scores.push(resp.score);
-			reasons.push(resp.reason);
+			responses.push(resp);
+			if (require_graders.isGraderFailure(resp)) {
+				failure = {
+					index,
+					resp
+				};
+				break;
+			}
+		}
+		const tokensUsed = require_tokenUsageUtils.createEmptyTokenUsage();
+		for (const r of responses) require_tokenUsageUtils.accumulateTokenUsage(tokensUsed, r.tokensUsed);
+		if (failure) {
+			const criterion = renderedValue[failure.index];
+			return {
+				assertion,
+				pass: false,
+				score: 0,
+				reason: `G-Eval criterion ${failure.index + 1}/${renderedValue.length} (${JSON.stringify(criterion)}) failed: ${failure.resp.reason}`,
+				tokensUsed,
+				metadata: failure.resp.metadata
+			};
 		}
-		const scoresSum = scores.reduce((a, b) => a + b, 0);
+		const averageScore = responses.reduce((acc, r) => acc + r.score, 0) / responses.length;
+		const combinedReason = responses.map((r) => r.reason).join("\n\n");
 		return {
 			assertion,
-			pass: scoresSum / scores.length >= threshold,
-			score: scoresSum / scores.length,
-			reason: reasons.join("\n\n")
+			pass: averageScore >= threshold !== inverse,
+			score: inverse ? 1 - averageScore : averageScore,
+			reason: combinedReason,
+			tokensUsed
 		};
-	} else return {
+	}
+	const resp = await require_graders.matchesGEval(renderedValue, prompt || "", outputString, threshold, test.options, providerCallContext);
+	if (require_graders.isGraderFailure(resp)) return {
+		assertion,
+		pass: false,
+		score: 0,
+		reason: resp.reason,
+		tokensUsed: resp.tokensUsed,
+		metadata: resp.metadata
+	};
+	const passed = resp.score >= threshold !== inverse;
+	return {
 		assertion,
-		...await require_graders.matchesGEval(renderedValue, prompt || "", outputString, threshold, test.options, providerCallContext)
+		...resp,
+		pass: passed,
+		score: inverse ? 1 - resp.score : resp.score
 	};
 };
 //#endregion
@@ -2024,7 +2082,7 @@ const handleJavascript = async ({ assertion, renderedValue, valueFromScript, ass
 		let result;
 		if (typeof valueFromScript === "undefined") {
 			const functionBody = renderedValue.includes("\n") ? renderedValue : buildFunctionBody(renderedValue);
-			result = await validateResult(new Function("output", "context", "process", functionBody)(output, assertionValueContext, require_transform$1.getProcessShim()));
+			result = await validateResult(new Function("output", "context", "process", functionBody)(output, assertionValueContext, require_processShim.getProcessShim()));
 		} else {
 			require_invariant.invariant(typeof valueFromScript === "boolean" || typeof valueFromScript === "number" || typeof valueFromScript === "object", `Javascript assertion script must return a boolean, number, or object (${assertion.value})`);
 			result = await validateResult(valueFromScript);
@@ -2516,45 +2574,6 @@ function matchesPattern(spanName, pattern) {
 }
 //#endregion
 //#region src/assertions/trajectoryUtils.ts
-const TOOL_ATTRIBUTE_KEYS = [
-	"tool.name",
-	"tool_name",
-	"tool",
-	"function.name",
-	"function_name",
-	"gen_ai.tool.name",
-	"codex.mcp.tool",
-	"agent.tool",
-	"agent.tool_name",
-	"agent.toolName"
-];
-const TOOL_ARGUMENT_ATTRIBUTE_KEYS = [
-	"tool.arguments",
-	"tool.args",
-	"tool.input",
-	"tool_arguments",
-	"tool_args",
-	"tool_input",
-	"function.arguments",
-	"function.args",
-	"function.input",
-	"function_arguments",
-	"function_args",
-	"gen_ai.tool.arguments",
-	"gen_ai.tool.args",
-	"gen_ai.tool.input",
-	"gen_ai.tool.call.arguments",
-	"gen_ai.tool.call.args",
-	"agent.tool.arguments",
-	"agent.tool.args",
-	"agent.tool.input",
-	"codex.mcp.arguments",
-	"codex.mcp.args",
-	"codex.mcp.input",
-	"arguments",
-	"args",
-	"input"
-];
 const COMMAND_ATTRIBUTE_KEYS = [
 	"codex.command",
 	"command",
@@ -2567,16 +2586,15 @@ const SEARCH_ATTRIBUTE_KEYS = [
 	"search_query"
 ];
 const GENERIC_QUERY_ATTRIBUTE_KEYS = ["query"];
+const COMMAND_TOOL_NAMES = new Set([
+	"exec_command",
+	"local_shell",
+	"shell"
+]);
 const SEARCH_SPAN_NAME_PATTERN = /(^|[\s._:/-])(search|find|lookup|retriev(?:e|al))($|[\s._:/-])/i;
 const MAX_JUDGE_SUMMARY_STEPS = 24;
 const JUDGE_SUMMARY_HEAD_STEPS = 12;
 const JUDGE_SUMMARY_TAIL_STEPS = 12;
-function getStringAttribute(attributes, keys) {
-	for (const key of keys) {
-		const value = attributes[key];
-		if (typeof value === "string" && value.trim()) return value.trim();
-	}
-}
 function normalizeStructuredAttribute(value) {
 	if (value === void 0 || value === null) return;
 	if (typeof value === "string") {
@@ -2608,9 +2626,12 @@ function getTrajectoryStepStatus(step) {
 function getCommandExecutable(command) {
 	return command.trim().split(/\s+/)[0] || void 0;
 }
+function isCommandToolName(toolName) {
+	return !!toolName && COMMAND_TOOL_NAMES.has(toolName.trim().toLowerCase());
+}
 function extractToolName(span) {
 	const attributes = span.attributes || {};
-	const directMatch = getStringAttribute(attributes, TOOL_ATTRIBUTE_KEYS);
+	const directMatch = require_toolAttributes.getToolNameFromAttributes(attributes);
 	if (directMatch) return directMatch;
 	for (const [key, value] of Object.entries(attributes)) {
 		if (typeof value !== "string" || !value.trim()) continue;
@@ -2624,7 +2645,7 @@ function extractToolName(span) {
 }
 function extractToolArgs(span) {
 	const attributes = span.attributes || {};
-	for (const key of TOOL_ARGUMENT_ATTRIBUTE_KEYS) {
+	for (const key of require_toolAttributes.TOOL_ARGUMENT_ATTRIBUTE_KEYS) {
 		const value = normalizeStructuredAttribute(attributes[key]);
 		if (value !== void 0) return value;
 	}
@@ -2635,21 +2656,31 @@ function extractToolArgs(span) {
 		if (value !== void 0) return value;
 	}
 }
-function extractCommand(span) {
+function extractCommand(span, toolName = extractToolName(span), getToolArgs = () => extractToolArgs(span)) {
 	const attributes = span.attributes || {};
-	const directMatch = getStringAttribute(attributes, COMMAND_ATTRIBUTE_KEYS);
+	const directMatch = require_toolAttributes.getFirstStringAttribute(attributes, COMMAND_ATTRIBUTE_KEYS);
 	if (directMatch) return directMatch;
 	for (const [key, value] of Object.entries(attributes)) {
 		if (typeof value !== "string" || !value.trim()) continue;
 		if (/command/i.test(key) && !/output|result/i.test(key)) return value.trim();
 	}
+	const toolArgs = getToolArgs();
+	if (isCommandToolName(toolName) && toolArgs && typeof toolArgs === "object") {
+		const args = toolArgs;
+		const command = args.cmd ?? args.command;
+		if (typeof command === "string" && command.trim()) return command.trim();
+		if (Array.isArray(command)) {
+			const joined = command.map((part) => String(part).trim()).filter(Boolean).join(" ");
+			if (joined) return joined;
+		}
+	}
 	if (span.name.startsWith("exec ")) return span.name.slice(5).trim();
 }
 function extractSearchQuery(span) {
 	const attributes = span.attributes || {};
-	const directMatch = getStringAttribute(attributes, SEARCH_ATTRIBUTE_KEYS);
+	const directMatch = require_toolAttributes.getFirstStringAttribute(attributes, SEARCH_ATTRIBUTE_KEYS);
 	if (directMatch) return directMatch;
-	const genericQuery = getStringAttribute(attributes, GENERIC_QUERY_ATTRIBUTE_KEYS);
+	const genericQuery = require_toolAttributes.getFirstStringAttribute(attributes, GENERIC_QUERY_ATTRIBUTE_KEYS);
 	if (genericQuery && isSearchLikeSpan(span)) return genericQuery;
 	if (span.name.startsWith("search ")) return span.name.slice(7).replace(/^"|"$/g, "").trim();
 }
@@ -2673,17 +2704,34 @@ function extractTrajectorySteps(trace) {
 		return left.index - right.index;
 	}).map(({ span }) => {
 		const toolName = extractToolName(span);
-		const command = extractCommand(span);
+		let toolArgs;
+		let hasExtractedToolArgs = false;
+		const getToolArgs = () => {
+			if (!hasExtractedToolArgs) {
+				toolArgs = extractToolArgs(span);
+				hasExtractedToolArgs = true;
+			}
+			return toolArgs;
+		};
+		const command = extractCommand(span, toolName, getToolArgs);
 		const searchQuery = extractSearchQuery(span);
 		let type = "span";
 		let name = span.name;
 		const aliases = new Set([span.name]);
 		let args;
-		if (toolName) {
+		if (command && isCommandToolName(toolName)) {
+			type = "command";
+			name = command;
+			aliases.add(command);
+			args = getToolArgs();
+			if (toolName) aliases.add(toolName);
+			const executable = getCommandExecutable(command);
+			if (executable) aliases.add(executable);
+		} else if (toolName) {
 			type = "tool";
 			name = toolName;
 			aliases.add(toolName);
-			args = extractToolArgs(span);
+			args = getToolArgs();
 		} else if (command) {
 			type = "command";
 			name = command;
@@ -2859,7 +2907,7 @@ const handleRedteam = async ({ assertion, baseType, test, prompt, outputString,
 				if (match) evalId = match[1];
 			}
 		}
-		const tracking = await require_providers.checkExfilTracking(webPageUuid, evalId);
+		const tracking = await require_indirectWebPwn.checkExfilTracking(webPageUuid, evalId);
 		if (tracking) gradingContext = {
 			...gradingContext,
 			wasExfiltrated: tracking.wasExfiltrated,
@@ -2925,7 +2973,7 @@ function handleIsRefusal(params) {
 			assertion
 		};
 	}
-	const pass = require_providers.isBasicRefusal(output) !== inverse;
+	const pass = require_util$2.isBasicRefusal(output) !== inverse;
 	return {
 		pass,
 		score: pass ? 1 : 0,
@@ -3870,13 +3918,13 @@ function resolveSequenceValue(value) {
 	}
 	throw new Error("trajectory:tool-sequence assertion must have an array or object value");
 }
-function isRecord(value) {
+function isRecord$1(value) {
 	return typeof value === "object" && value !== null && !Array.isArray(value);
 }
 function matchesExpectedArgsPartial(actual, expected) {
 	if (Array.isArray(expected)) return Array.isArray(actual) && actual.length === expected.length && expected.every((item, index) => matchesExpectedArgsPartial(actual[index], item));
-	if (isRecord(expected)) {
-		if (!isRecord(actual)) return false;
+	if (isRecord$1(expected)) {
+		if (!isRecord$1(actual)) return false;
 		return Object.entries(expected).every(([key, expectedValue]) => Object.prototype.hasOwnProperty.call(actual, key) && matchesExpectedArgsPartial(actual[key], expectedValue));
 	}
 	return (0, node_util.isDeepStrictEqual)(actual, expected);
@@ -4233,7 +4281,7 @@ function assertionMayNeedTraceContext(assertion) {
 	if (assertionUsesTrace(assertion)) return true;
 	if (assertion.type === "assert-set") return assertion.assert.some(assertionMayNeedTraceContext);
 	if (assertion.type.startsWith("promptfoo:redteam:coding-agent:")) return true;
-	return typeof assertion.value === "string" ? assertion.value.startsWith("file://") || require_providers.isPackagePath(assertion.value) : false;
+	return typeof assertion.value === "string" ? assertion.value.startsWith("file://") || require_packageParser.isPackagePath(assertion.value) : false;
 }
 function hasTraceAwareAssertions(assertions) {
 	return Boolean(assertions?.some(assertionMayNeedTraceContext));
@@ -4247,7 +4295,7 @@ async function loadTraceData(traceId) {
 	let stableObservations = 0;
 	let latestTrace = null;
 	for (let attempt = 0; attempt < maxAttempts; attempt++) {
-		latestTrace = await traceStore.getTrace(traceId);
+		latestTrace = await traceStore.getTrace(traceId, { sanitizeAttributes: false });
 		const spanCount = latestTrace?.spans?.length ?? 0;
 		if (spanCount > 0) {
 			stableObservations = spanCount === lastSpanCount ? stableObservations + 1 : 1;
@@ -4300,7 +4348,7 @@ const ASSERTION_HANDLERS = {
 	"llm-rubric": handleLlmRubric,
 	meteor: async (params) => {
 		try {
-			const { handleMeteorAssertion } = await Promise.resolve().then(() => require("./meteor-BBGcGeCa.cjs"));
+			const { handleMeteorAssertion } = await Promise.resolve().then(() => require("./meteor-CR226f7Z.cjs"));
 			return handleMeteorAssertion(params);
 		} catch (error) {
 			if (error instanceof Error && (error.message.includes("Cannot find module") || error.message.includes("natural\" package is required"))) return {
@@ -4436,7 +4484,7 @@ async function runAssertion({ prompt, provider, assertion, test, vars, latencyMs
 			};
 		}
 		else if (filePath.endsWith(".rb")) try {
-			const { runRuby } = await Promise.resolve().then(() => require("./rubyUtils-CO-tuszQ.cjs"));
+			const { runRuby } = await Promise.resolve().then(() => require("./rubyUtils-CqUWBZAt.cjs")).then((n) => n.rubyUtils_exports);
 			valueFromScript = await runRuby(filePath, functionName || "get_assert", [output, context]);
 			require_logger.logger.debug(`Ruby script ${filePath} output: ${valueFromScript}`);
 		} catch (error) {
@@ -4448,9 +4496,9 @@ async function runAssertion({ prompt, provider, assertion, test, vars, latencyMs
 			};
 		}
 		else renderedValue = require_graders.processFileReference(renderedValue);
-	} else if (require_providers.isPackagePath(renderedValue)) {
+	} else if (require_packageParser.isPackagePath(renderedValue)) {
 		const basePath = require_logger.state.basePath || "";
-		const requiredModule = await require_providers.loadFromPackage(renderedValue, basePath);
+		const requiredModule = await require_packageParser.loadFromPackage(renderedValue, basePath);
 		if (typeof requiredModule !== "function") throw new Error(`Assertion malformed: ${renderedValue} must be a function. Received: ${typeof requiredModule}`);
 		valueFromScript = await Promise.resolve(requiredModule(output, context));
 	} else renderedValue = nunjucks.renderString(renderedValue, resolvedVars);
@@ -4553,7 +4601,8 @@ async function runAssertions({ assertScoringFunction, latencyMs, prompt, provide
 		require_logger.logger.debug(`Failed to preload trace data for assertions: ${error}`);
 		preloadedTraceData = null;
 	}
-	await async.default.forEachOfLimit(asserts, ASSERTIONS_MAX_CONCURRENCY, async ({ assertion, assertResult, index }) => {
+	const concurrency = require_graders.getProviderCallExecutionContext()?.providerCallQueue ? 1 : ASSERTIONS_MAX_CONCURRENCY;
+	await async.default.forEachOfLimit(asserts, concurrency, async ({ assertion, assertResult, index }) => {
 		if (assertion.type.startsWith("select-") || assertion.type === "max-score") return;
 		const result = await runAssertion({
 			prompt,
@@ -4680,7 +4729,8 @@ var CIProgressReporter = class {
 	}
 	updateTotalTests(newTotal) {
 		this.totalTests = Math.max(newTotal, 1);
-		this.highestPercentageSeen = Math.floor(this.completedTests / this.totalTests * 100);
+		const percentage = Math.floor(this.completedTests / this.totalTests * 100);
+		this.highestPercentageSeen = percentage;
 	}
 	finish() {
 		if (this.intervalId) {
@@ -4853,6 +4903,10 @@ function getDefaultOtelConfig() {
 }
 //#endregion
 //#region src/tracing/localSpanExporter.ts
+const MISSING_TRACE_RETRY_DELAY_MS = 50;
+function delay(ms) {
+	return new Promise((resolve) => setTimeout(resolve, ms));
+}
 /**
 * A span exporter that writes spans to the local TraceStore (SQLite).
 * This allows OTEL spans to be stored locally for analysis in the promptfoo UI.
@@ -4894,7 +4948,7 @@ var LocalSpanExporter = class {
 		}
 		let firstError;
 		for (const [traceId, spanDataList] of spansByTrace) try {
-			const result = await traceStore.addSpans(traceId, spanDataList, { skipTraceCheck: false });
+			const result = await this.addSpansWithTraceRetry(traceStore, traceId, spanDataList);
 			if (result.stored) require_logger.logger.debug(`[LocalSpanExporter] Added ${spanDataList.length} spans to trace ${traceId}`);
 			else require_logger.logger.debug(`[LocalSpanExporter] Skipping ${spanDataList.length} spans for orphan trace ${traceId}: ${result.reason}`);
 		} catch (error) {
@@ -4906,6 +4960,16 @@ var LocalSpanExporter = class {
 		}
 		return firstError;
 	}
+	async addSpansWithTraceRetry(traceStore, traceId, spans) {
+		const options = {
+			skipTraceCheck: false,
+			warnIfMissingTrace: false
+		};
+		const result = await traceStore.addSpans(traceId, spans, options);
+		if (result.stored) return result;
+		await delay(MISSING_TRACE_RETRY_DELAY_MS);
+		return traceStore.addSpans(traceId, spans, options);
+	}
 	/**
 	* Convert an OTEL ReadableSpan to our SpanData format.
 	*/
@@ -5188,6 +5252,15 @@ function isPromptAllowed(prompt, allowedPrompts) {
 }
 //#endregion
 //#region src/evaluator.ts
+const CONVERSATION_VAR_NAME = "_conversation";
+const promptUsesConversationVariableCache = new lru_cache.LRUCache({ max: 1024 });
+function promptUsesConversationVariable(prompt) {
+	const cached = promptUsesConversationVariableCache.get(prompt.raw);
+	if (cached !== void 0) return cached;
+	const { referenced, parsed } = require_render.analyzeTemplateReference(prompt.raw, CONVERSATION_VAR_NAME);
+	if (parsed) promptUsesConversationVariableCache.set(prompt.raw, referenced);
+	return referenced;
+}
 /**
 * Manages a single progress bar for the evaluation
 */
@@ -5387,6 +5460,18 @@ function hasProviderGroupedAssertion(assertion) {
 function shouldDeferGradingForTest(test) {
 	return Boolean(test.assert?.some(hasProviderGroupedAssertion));
 }
+function logGroupedGradingStatus({ concurrency, hasEvalStepTimeout, runEvalOptions, shouldGroupGradingByProvider, usesConversationVar }) {
+	if (!runEvalOptions.some(({ test }) => shouldDeferGradingForTest(test))) return;
+	if (shouldGroupGradingByProvider) {
+		require_logger.logger.info("Grouping model-graded assertions by provider to minimize local-model reload overhead.");
+		return;
+	}
+	if (concurrency !== 1) return;
+	const reasons = [];
+	if (hasEvalStepTimeout) reasons.push("per-eval-step timeout is configured");
+	if (usesConversationVar) reasons.push("conversation variables require per-row ordering");
+	if (reasons.length > 0) require_logger.logger.info(`Serial grading grouping disabled because ${reasons.join(" and ")}; model-graded judges may reload between rows.`);
+}
 function applyGradingResult(row, checkResult) {
 	if (!checkResult.pass) {
 		row.error = checkResult.reason;
@@ -5401,14 +5486,29 @@ function applyGradingResult(row, checkResult) {
 	if (checkResult.tokensUsed) require_tokenUsageUtils.accumulateAssertionTokenUsage(row.tokenUsage.assertions, checkResult.tokensUsed);
 	row.gradingResult = checkResult;
 }
-function applyGradingError(row, error) {
-	const errorMessage = error instanceof Error ? error.stack ?? error.message : String(error);
-	require_logger.logger.error("Assertion grading failed during eval", {
-		error: errorMessage,
-		promptIdx: row.promptIdx,
-		testIdx: row.testIdx
-	});
-	row.error = errorMessage;
+const ABORTED_GRADING_PREFIX = "Aborted: ";
+function isAbortShapedError(error) {
+	return error instanceof Error && (error.name === "AbortError" || error.name === "AbortException");
+}
+function applyGradingError(row, error, abortSignal) {
+	const errorAsError = error instanceof Error ? error : void 0;
+	if (Boolean(abortSignal?.aborted) && isAbortShapedError(error)) {
+		const shortMessage = errorAsError?.message ?? String(error);
+		require_logger.logger.debug("Assertion grading aborted", {
+			error: shortMessage,
+			promptIdx: row.promptIdx,
+			testIdx: row.testIdx
+		});
+		row.error = `${ABORTED_GRADING_PREFIX}${shortMessage}`;
+	} else {
+		const fullMessage = errorAsError ? errorAsError.stack ?? errorAsError.message : String(error);
+		require_logger.logger.error("Assertion grading failed during eval", {
+			error: fullMessage,
+			promptIdx: row.promptIdx,
+			testIdx: row.testIdx
+		});
+		row.error = fullMessage;
+	}
 	row.failureReason = require_types.ResultFailureReason.ERROR;
 	row.success = false;
 	row.score = 0;
@@ -5420,7 +5520,7 @@ function getNonTransientTargetStatus(row) {
 }
 function createRunEvalState({ provider, prompt, test }) {
 	const vars = structuredClone(test.vars || {});
-	const fileMetadata = require_providers.collectFileMetadata(vars);
+	const fileMetadata = require_evaluatorHelpers.collectFileMetadata(vars);
 	const conversationKey = `${provider.label || provider.id()}:${prompt.id}${test.metadata?.conversationId ? `:${test.metadata.conversationId}` : ""}`;
 	const setup = createRunEvalSetup({
 		provider,
@@ -5440,7 +5540,7 @@ function createRunEvalState({ provider, prompt, test }) {
 	};
 }
 function attachConversationVar({ conversations, conversationKey, prompt, test, vars }) {
-	const usesConversation = prompt.raw.includes("_conversation");
+	const usesConversation = promptUsesConversationVariable(prompt);
 	if (!require_logger.getEnvBool("PROMPTFOO_DISABLE_CONVERSATION_VAR") && !test.options?.disableConversationVar && usesConversation) vars._conversation = conversations?.[conversationKey] || [];
 }
 function createRunEvalSetup({ provider, prompt, promptConfig, vars }) {
@@ -5459,8 +5559,8 @@ function createRunEvalSetup({ provider, prompt, promptConfig, vars }) {
 	};
 }
 async function renderRunEvalPrompt({ filters, isRedteam, provider, promptForRender, test, testSuite, vars }) {
-	const renderedPrompt = await require_providers.renderPrompt(promptForRender, vars, filters, provider, shouldSkipRedteamInjectVar(test, testSuite, isRedteam) ? [getRedteamInjectVar(test, promptForRender, testSuite)] : void 0);
-	if (isRedteam) require_providers.throwIfTargetPromptExceedsMaxChars(renderedPrompt, testSuite?.redteam?.maxCharsPerMessage);
+	const renderedPrompt = await require_evaluatorHelpers.renderPrompt(promptForRender, vars, filters, provider, shouldSkipRedteamInjectVar(test, testSuite, isRedteam) ? [getRedteamInjectVar(test, promptForRender, testSuite)] : void 0);
+	if (isRedteam) require_promptLength.throwIfTargetPromptExceedsMaxChars(renderedPrompt, testSuite?.redteam?.maxCharsPerMessage);
 	const setup = createRunEvalSetup({
 		provider,
 		prompt: promptForRender,
@@ -5527,7 +5627,7 @@ async function callActiveProvider({ abortSignal, evalId, filters, promptForRende
 	});
 	const callApiOptions = abortSignal ? { abortSignal } : void 0;
 	const callApi = () => activeProvider.callApi(renderedPrompt, callApiContext, callApiOptions);
-	const response = rateLimitRegistry ? await rateLimitRegistry.execute(activeProvider, callApi, require_providers.createProviderRateLimitOptions()) : await callApi();
+	const response = rateLimitRegistry ? await rateLimitRegistry.execute(activeProvider, callApi, require_shared.createProviderRateLimitOptions()) : await callApi();
 	require_logger.logger.debug(`Provider response properties: ${Object.keys(response).join(", ")}`);
 	require_logger.logger.debug(`Provider response cached property explicitly: ${response.cached}`);
 	return response;
@@ -5605,7 +5705,7 @@ function createEvaluateResult({ fileMetadata, latencyMs, prompt, promptIdx, rend
 	};
 	if (!ret.metadata?.sessionIds && !ret.metadata?.sessionId) {
 		ret.metadata ??= {};
-		ret.metadata.sessionId = require_providers.getSessionId(response, { vars });
+		ret.metadata.sessionId = require_util$2.getSessionId(response, { vars });
 	}
 	return ret;
 }
@@ -5613,7 +5713,7 @@ function trackProviderUsage(provider, response) {
 	if (!response.tokenUsage) return;
 	const providerId = provider.id();
 	const trackingId = provider.constructor?.name ? `${providerId} (${provider.constructor.name})` : providerId;
-	require_providers.TokenUsageTracker.getInstance().trackUsage(trackingId, response.tokenUsage);
+	require_shared.TokenUsageTracker.getInstance().trackUsage(trackingId, response.tokenUsage);
 }
 async function applyRunEvalResponseOutcome({ abortSignal, deferGrading, evalId, isRedteam, latencyMs, prompt, promptIdx, provider, providerCallQueue, rateLimitRegistry, renderedPrompt, response, ret, test, testIdx, traceContext, vars }) {
 	if (response.error) {
@@ -5687,7 +5787,7 @@ async function gradeRunEvalResponse({ abortSignal, deferGrading, evalId, latency
 			assertScoringFunction: test.assertScoringFunction,
 			traceId
 		}).then((checkResult) => applyGradingResult(ret, checkResult))).catch((error) => {
-			applyGradingError(ret, error);
+			applyGradingError(ret, error, abortSignal);
 		});
 		deferredGradingPromises.set(ret, gradingPromise);
 		return;
@@ -6234,7 +6334,7 @@ async function resolveDefaultTestProvider(defaultTest, testCase) {
 	const defaultProvider = defaultTest.provider;
 	if (require_types.isApiProvider(defaultProvider)) return defaultProvider;
 	if (typeof defaultProvider === "object" && defaultProvider.id) {
-		const { loadApiProvider } = await Promise.resolve().then(() => require("./providers-C7lNVBjX.cjs"));
+		const { loadApiProvider } = await Promise.resolve().then(() => require("./providers-V6RBuieY.cjs")).then((n) => n.providers_exports);
 		return loadApiProvider(typeof defaultProvider.id === "function" ? defaultProvider.id() : defaultProvider.id, { options: defaultProvider });
 	}
 	return defaultProvider;
@@ -6394,7 +6494,7 @@ function buildRepeatCacheContextByTestIdx(runEvalOptions) {
 async function filterCompletedResumeSteps(runEvalOptions, evalRecord) {
 	if (!require_logger.state.resume || !evalRecord.persisted) return;
 	try {
-		const { default: EvalResult } = await Promise.resolve().then(() => require("./evalResult-BBJAHAtw.cjs"));
+		const { default: EvalResult } = await Promise.resolve().then(() => require("./evalResult-DHXs-9TL.cjs")).then((n) => n.evalResult_exports);
 		const completedPairs = await EvalResult.getCompletedIndexPairs(evalRecord.id, { excludeErrors: require_logger.state.retryMode });
 		const originalCount = runEvalOptions.length;
 		for (let i = runEvalOptions.length - 1; i >= 0; i--) {
@@ -6408,14 +6508,14 @@ async function filterCompletedResumeSteps(runEvalOptions, evalRecord) {
 	}
 }
 function adjustConcurrencyForSerialFeatures({ concurrency, prompts, tests }) {
-	const usesConversationVar = prompts.some((p) => p.raw.includes("_conversation"));
+	const usesConversationVar = prompts.some(promptUsesConversationVariable);
 	if (concurrency <= 1) return {
 		concurrency,
 		usesConversationVar
 	};
 	const usesStoreOutputAs = tests.some((t) => t.options?.storeOutputAs);
 	if (usesConversationVar) {
-		require_logger.logger.info(`Setting concurrency to 1 because the ${chalk.default.cyan("_conversation")} variable is used.`);
+		require_logger.logger.info(`Setting concurrency to 1 because the ${chalk.default.cyan(CONVERSATION_VAR_NAME)} variable is used.`);
 		return {
 			concurrency: 1,
 			usesConversationVar
@@ -6645,8 +6745,9 @@ var Evaluator = class {
 		};
 		this.conversations = {};
 		this.registers = {};
-		this.fileWriters = (Array.isArray(evalRecord.config.outputPath) ? evalRecord.config.outputPath.filter((p) => p.endsWith(".jsonl")) : evalRecord.config.outputPath?.endsWith(".jsonl") ? [evalRecord.config.outputPath] : []).map((p) => new JsonlFileWriter(p));
-		this.rateLimitRegistry = require_providers.createRateLimitRegistry({ maxConcurrency: options.maxConcurrency || 4 });
+		const jsonlFiles = Array.isArray(evalRecord.config.outputPath) ? evalRecord.config.outputPath.filter((p) => p.endsWith(".jsonl")) : evalRecord.config.outputPath?.endsWith(".jsonl") ? [evalRecord.config.outputPath] : [];
+		this.fileWriters = jsonlFiles.map((p) => new JsonlFileWriter(p));
+		this.rateLimitRegistry = require_shared.createRateLimitRegistry({ maxConcurrency: options.maxConcurrency || 4 });
 		this.rateLimitRegistry.on("ratelimit:hit", (data) => {
 			require_logger.logger.debug(`[Scheduler] Rate limit hit for ${data.rateLimitKey}`, {
 				retryAfterMs: data.retryAfterMs,
@@ -6672,7 +6773,7 @@ var Evaluator = class {
 				current: data.current
 			});
 		});
-		require_providers.redteamProviderManager.setRateLimitRegistry(this.rateLimitRegistry);
+		require_shared.redteamProviderManager.setRateLimitRegistry(this.rateLimitRegistry);
 	}
 	/**
 	* Updates metrics and stats after a comparison assertion (select-best or max-score).
@@ -6750,7 +6851,7 @@ var Evaluator = class {
 		});
 	}
 	async runEvalStepAfterBeforeEach(evalStep, { deferGrading, onRowsReady, providerCallQueue, testSuite }) {
-		evalStep.test = (await require_providers.runExtensionHook(testSuite.extensions, "beforeEach", { test: evalStep.test })).test;
+		evalStep.test = (await require_evaluatorHelpers.runExtensionHook(testSuite.extensions, "beforeEach", { test: evalStep.test })).test;
 		const rows = await runEvalInternal({
 			...evalStep,
 			deferGrading,
@@ -6765,6 +6866,25 @@ var Evaluator = class {
 			this.trackCompletedRow(evalStep, row, context);
 			context.numComplete++;
 			const promptEvalCount = reservePromptEvalCount(context, row.promptIdx);
+			if (context.testSuite.extensions?.length) try {
+				const afterEachOut = await require_evaluatorHelpers.runExtensionHook(context.testSuite.extensions, "afterEach", {
+					test: evalStep.test,
+					result: {
+						...row,
+						namedScores: { ...row.namedScores },
+						metadata: { ...row.metadata },
+						response: row.response ? {
+							...row.response,
+							metadata: { ...row.response.metadata }
+						} : row.response
+					}
+				});
+				row.namedScores = require_evaluatorHelpers.filterFiniteScores(afterEachOut.result.namedScores);
+				row.metadata = afterEachOut.result.metadata;
+				if (row.response && afterEachOut.result.response) row.response.metadata = afterEachOut.result.response.metadata;
+			} catch (error) {
+				require_logger.logger.error(`afterEach extension hook failed, persisting row without hook modifications`, { error });
+			}
 			await this.persistEvalRow(row);
 			if (this.abortIfTargetUnavailable(row, context)) break;
 			const metrics = context.prompts[row.promptIdx].metrics;
@@ -6776,10 +6896,6 @@ var Evaluator = class {
 				promptEvalCount,
 				row
 			});
-			await require_providers.runExtensionHook(context.testSuite.extensions, "afterEach", {
-				test: evalStep.test,
-				result: row
-			});
 			context.options.progressCallback?.(context.numComplete, context.runEvalOptionsLength, index, evalStep, metrics);
 		}
 	}
@@ -6934,7 +7050,15 @@ var Evaluator = class {
 				})) break;
 			}
 		} catch (error) {
-			await flushGroupedRows();
+			const pendingRowCount = groupedRows.reduce((sum, entry) => sum + entry.rows.length, 0);
+			try {
+				await flushGroupedRows();
+			} catch (flushError) {
+				require_logger.logger.warn("Failed to flush grouped rows after error; target outputs may be lost", {
+					error: flushError instanceof Error ? flushError.message : String(flushError),
+					pendingRowCount
+				});
+			}
 			throw error;
 		}
 		await flushGroupedRows();
@@ -7204,7 +7328,7 @@ var Evaluator = class {
 	async runAfterAllExtensions(testSuite) {
 		if (!testSuite.extensions?.length) return;
 		const resultsForExtension = (await this.evalRecord.getResults()).map((result) => "toEvaluateResult" in result ? result.toEvaluateResult() : result);
-		await require_providers.runExtensionHook(testSuite.extensions, "afterAll", {
+		await require_evaluatorHelpers.runExtensionHook(testSuite.extensions, "afterAll", {
 			prompts: this.evalRecord.prompts,
 			results: resultsForExtension,
 			suite: testSuite,
@@ -7290,7 +7414,7 @@ var Evaluator = class {
 		const rowsWithSelectBestAssertion = /* @__PURE__ */ new Set();
 		const rowsWithMaxScoreAssertion = /* @__PURE__ */ new Set();
 		ensureDefaultTestForExtensions(testSuite);
-		testSuite = (await require_providers.runExtensionHook(testSuite.extensions, "beforeAll", { suite: testSuite })).suite;
+		testSuite = (await require_evaluatorHelpers.runExtensionHook(testSuite.extensions, "beforeAll", { suite: testSuite })).suite;
 		if (!await maybeAddGeneratedPrompts(testSuite, options)) return this.evalRecord;
 		prompts.push(...buildCompletedPrompts(testSuite, this.evalRecord));
 		const promptIndexMap = buildPromptIndexMap(prompts);
@@ -7370,6 +7494,13 @@ var Evaluator = class {
 		if (!this.options.silent) {
 			if (serialRunEvalOptions.length > 0) require_logger.logger.info(`Running ${serialRunEvalOptions.length} test cases serially...`);
 			if (concurrentRunEvalOptions.length > 0) require_logger.logger.info(`Running ${concurrentRunEvalOptions.length} test cases (up to ${concurrency} at a time)...`);
+			logGroupedGradingStatus({
+				concurrency,
+				hasEvalStepTimeout,
+				runEvalOptions,
+				shouldGroupGradingByProvider,
+				usesConversationVar
+			});
 		}
 		if (this.options.showProgressBar && progressBarManager) {
 			await progressBarManager.initialize(runEvalOptions, concurrency, 0);
@@ -7461,7 +7592,7 @@ var Evaluator = class {
 				});
 			}
 			this.rateLimitRegistry?.dispose();
-			require_providers.redteamProviderManager.setRateLimitRegistry(void 0);
+			require_shared.redteamProviderManager.setRateLimitRegistry(void 0);
 			require_logger.state.maxConcurrency = void 0;
 		}
 	}
@@ -8369,7 +8500,7 @@ var Eval = class Eval {
 		const evalInstance = new Eval(eval_.config, {
 			id: eval_.id,
 			createdAt: new Date(eval_.createdAt),
-			author: eval_.author || void 0,
+			author: eval_.author,
 			description: eval_.description || void 0,
 			prompts: eval_.prompts || [],
 			datasetId,
@@ -8392,7 +8523,7 @@ var Eval = class Eval {
 		return (await require_tables.getDb().select().from(require_tables.evalsTable).limit(limit).orderBy((0, drizzle_orm.desc)(require_tables.evalsTable.createdAt)).all()).map((e) => new Eval(e.config, {
 			id: e.id,
 			createdAt: new Date(e.createdAt),
-			author: e.author || void 0,
+			author: e.author,
 			description: e.description || void 0,
 			prompts: e.prompts || [],
 			persisted: true
@@ -8407,7 +8538,7 @@ var Eval = class Eval {
 		return (await require_tables.getDb().select().from(require_tables.evalsTable).orderBy((0, drizzle_orm.desc)(require_tables.evalsTable.createdAt)).limit(limit).offset(offset).all()).map((e) => new Eval(e.config, {
 			id: e.id,
 			createdAt: new Date(e.createdAt),
-			author: e.author || void 0,
+			author: e.author,
 			description: e.description || void 0,
 			prompts: e.prompts || [],
 			persisted: true
@@ -8422,7 +8553,7 @@ var Eval = class Eval {
 	static async create(config, renderedPrompts, opts) {
 		const createdAt = opts?.createdAt || /* @__PURE__ */ new Date();
 		const evalId = opts?.id || createEvalId(createdAt);
-		const author = opts?.author || require_accounts.getUserEmail();
+		const author = opts && "author" in opts ? opts.author ?? null : require_accounts.getAuthor();
 		const db = require_tables.getDb();
 		const datasetId = require_createHash.sha256(JSON.stringify(config.tests || []));
 		db.transaction(() => {
@@ -8484,7 +8615,7 @@ var Eval = class Eval {
 		});
 		return new Eval(config, {
 			id: evalId,
-			author: opts?.author,
+			author,
 			createdAt,
 			persisted: true,
 			runtimeOptions: sanitizeRuntimeOptions(opts?.runtimeOptions)
@@ -8494,7 +8625,7 @@ var Eval = class Eval {
 		const createdAt = opts?.createdAt || /* @__PURE__ */ new Date();
 		this.createdAt = createdAt.getTime();
 		this.id = opts?.id || createEvalId(createdAt);
-		this.author = opts?.author;
+		this.author = opts?.author ?? null;
 		this.config = config;
 		this.results = [];
 		this.prompts = opts?.prompts || [];
@@ -9024,7 +9155,7 @@ var Eval = class Eval {
 		newConfig.description = copyDescription;
 		const newPrompts = structuredClone(this.prompts);
 		const newVars = this.vars ? structuredClone(this.vars) : [];
-		const author = require_accounts.getUserEmail();
+		const author = require_accounts.getAuthor();
 		const db = require_tables.getDb();
 		let copiedCount = 0;
 		db.transaction(() => {
@@ -10941,7 +11072,7 @@ async function fetchRemoteGeneration(task, prompts) {
 			version: require_version.VERSION,
 			email: require_accounts.getUserEmail()
 		};
-		const response = await require_cache.fetchWithCache(require_server.getRemoteGenerationUrl(), {
+		const response = await require_cache.fetchWithCache(require_remoteGeneration.getRemoteGenerationUrl(), {
 			method: "POST",
 			headers: { "Content-Type": "application/json" },
 			body: JSON.stringify(body)
@@ -10976,7 +11107,7 @@ function formatPrompts(prompts) {
 //#endregion
 //#region src/redteam/extraction/entities.ts
 async function extractEntities(provider, prompts) {
-	if (require_server.shouldGenerateRemote()) try {
+	if (require_remoteGeneration.shouldGenerateRemote()) try {
 		return await fetchRemoteGeneration("entities", prompts);
 	} catch (error) {
 		require_logger.logger.warn(`[Entity Extraction] Failed, returning 0 entities. Error using remote generation: ${error}`);
@@ -11023,7 +11154,7 @@ async function extractSystemPurpose(provider, prompts) {
 		require_logger.logger.debug("[purpose] No meaningful prompts provided, returning default purpose");
 		return DEFAULT_PURPOSE;
 	}
-	if (!require_server.neverGenerateRemote()) try {
+	if (!require_remoteGeneration.neverGenerateRemote()) try {
 		return await fetchRemoteGeneration("purpose", prompts);
 	} catch (error) {
 		require_logger.logger.warn(`[purpose] Error using remote generation, returning empty string: ${error}`);
@@ -11295,7 +11426,7 @@ function createTestCase(injectVar, output, harmCategory) {
 		vars: { [injectVar]: output.trim() },
 		metadata: {
 			harmCategory: harmCategoryLabel,
-			pluginId: require_providers.getShortPluginId(harmCategory)
+			pluginId: require_util$2.getShortPluginId(harmCategory)
 		},
 		assert: getHarmfulAssertions(harmCategory)
 	};
@@ -11418,25 +11549,45 @@ var AlignedHarmfulPlugin = class extends require_graders.RedteamPluginBase {
 	getAssertions(_prompt) {
 		return getHarmfulAssertions(this.harmCategory);
 	}
-	promptsToTestCases(prompts) {
+	async promptsToTestCases(prompts) {
 		const hasMultipleInputs = this.config.inputs && Object.keys(this.config.inputs).length > 0;
 		const harmCategoryLabel = require_types.HARM_PLUGINS[this.harmCategory] || this.harmCategory;
-		return prompts.map(({ __prompt }) => {
+		const pluginId = require_util$2.getShortPluginId(this.harmCategory);
+		return Promise.all([...prompts].sort((a, b) => a.__prompt.localeCompare(b.__prompt)).map(async ({ __prompt }, materializationIndex) => {
 			const vars = { [this.injectVar]: __prompt };
-			if (hasMultipleInputs) try {
-				const parsed = JSON.parse(__prompt);
-				Object.assign(vars, require_providers.extractVariablesFromJson(parsed, this.config.inputs));
-			} catch {}
+			let inputMaterialization;
+			if (hasMultipleInputs) {
+				let parsed;
+				try {
+					parsed = JSON.parse(__prompt);
+				} catch (error) {
+					require_logger.logger.debug("[AlignedHarmful] Could not parse prompt as JSON for multi-input mode", { error });
+				}
+				if (parsed) try {
+					const materializedVars = await require_util$2.extractMaterializedVariablesFromJsonWithMetadata(parsed, this.config.inputs, {
+						materializationIndex,
+						pluginId,
+						provider: this.provider,
+						purpose: this.purpose
+					});
+					Object.assign(vars, materializedVars.vars);
+					inputMaterialization = materializedVars.metadata;
+				} catch (error) {
+					require_logger.logger.debug("[AlignedHarmful] Failed to materialize prompt inputs", { error });
+					throw error;
+				}
+			}
 			return {
 				vars,
 				metadata: {
 					harmCategory: harmCategoryLabel,
-					pluginId: require_providers.getShortPluginId(this.harmCategory),
-					pluginConfig: this.config
+					pluginId,
+					pluginConfig: this.config,
+					...inputMaterialization ? { inputMaterialization } : {}
 				},
 				assert: getHarmfulAssertions(this.harmCategory)
 			};
-		});
+		}));
 	}
 };
 //#endregion
@@ -11445,25 +11596,42 @@ var AlignedHarmfulPlugin = class extends require_graders.RedteamPluginBase {
 * Extract content from <Prompt> tags and parse JSON if inputs are defined.
 * Returns the processed prompt and any additional vars extracted from JSON.
 */
-function processPromptForInputs(prompt, _injectVar, inputs) {
+async function processPromptForInputs(prompt, inputs, plugin, provider, purpose, materializationIndex) {
 	let processedPrompt = prompt.trim();
 	const additionalVars = {};
-	const extractedPrompt = require_providers.extractPromptFromTags(processedPrompt);
+	let additionalMetadata;
+	const extractedPrompt = require_util$2.extractPromptFromTags(processedPrompt);
 	if (extractedPrompt) processedPrompt = extractedPrompt;
-	if (inputs && Object.keys(inputs).length > 0) try {
-		const parsed = JSON.parse(processedPrompt);
-		Object.assign(additionalVars, require_providers.extractVariablesFromJson(parsed, inputs));
-	} catch {
-		require_logger.logger.debug("[Harmful] Could not parse prompt as JSON for multi-input mode");
+	if (inputs && Object.keys(inputs).length > 0) {
+		let parsed;
+		try {
+			parsed = JSON.parse(processedPrompt);
+		} catch (error) {
+			require_logger.logger.debug("[Harmful] Could not parse prompt as JSON for multi-input mode", { error });
+		}
+		if (parsed) try {
+			const materializedVars = await require_util$2.extractMaterializedVariablesFromJsonWithMetadata(parsed, inputs, {
+				materializationIndex,
+				pluginId: plugin,
+				provider,
+				purpose
+			});
+			Object.assign(additionalVars, materializedVars.vars);
+			additionalMetadata = materializedVars.metadata;
+		} catch (error) {
+			require_logger.logger.debug("[Harmful] Failed to materialize prompt inputs", { error });
+			throw error;
+		}
 	}
 	return {
 		processedPrompt,
-		additionalVars
+		additionalVars,
+		additionalMetadata
 	};
 }
 async function getHarmfulTests({ purpose, injectVar, n, delayMs = 0, config }, plugin) {
 	const maxHarmfulTests = require_logger.getEnvInt("PROMPTFOO_MAX_HARMFUL_TESTS_PER_REQUEST", 5);
-	const unalignedProvider = new require_providers.PromptfooHarmfulCompletionProvider({
+	const unalignedProvider = new require_promptfoo.PromptfooHarmfulCompletionProvider({
 		purpose,
 		n: Math.min(n, maxHarmfulTests),
 		harmCategory: plugin,
@@ -11479,15 +11647,19 @@ async function getHarmfulTests({ purpose, injectVar, n, delayMs = 0, config }, p
 	};
 	const allPrompts = await require_graders.retryWithDeduplication(generatePrompts, n);
 	const inputs = config?.inputs;
-	return require_graders.sampleArray(allPrompts, n).map((prompt) => {
-		const { processedPrompt, additionalVars } = processPromptForInputs(prompt, injectVar, inputs);
+	return Promise.all(require_graders.sampleArray(allPrompts, n).map(async (prompt, materializationIndex) => {
+		const { processedPrompt, additionalVars, additionalMetadata } = await processPromptForInputs(prompt, inputs, plugin, unalignedProvider, purpose, materializationIndex);
 		const testCase = createTestCase(injectVar, processedPrompt, plugin);
 		if (Object.keys(additionalVars).length > 0) testCase.vars = {
 			...testCase.vars,
 			...additionalVars
 		};
+		if (additionalMetadata) testCase.metadata = {
+			...testCase.metadata,
+			inputMaterialization: additionalMetadata
+		};
 		return testCase;
-	});
+	}));
 }
 //#endregion
 //#region src/redteam/plugins/teenSafety/graderExamples.ts
@@ -11805,9 +11977,9 @@ const MAX_CHARS_RETRY_MODIFIER_KEY = "__maxCharsPerMessageRetry";
 function computeModifiersFromConfig(config) {
 	const modifiers = { ...config?.modifiers };
 	if (config?.language && typeof config.language === "string") modifiers.language = config.language;
-	if (config?.inputs && Object.keys(config.inputs).length > 0) modifiers.__outputFormat = `Output each test case as JSON wrapped in <Prompt> tags: <Prompt>{${Object.entries(config.inputs).map(([k, description]) => `"${k}": "${description}"`).join(", ")}}</Prompt>`;
-	const maxCharsModifier = require_providers.getMaxCharsPerMessageModifierValue(config?.maxCharsPerMessage);
-	if (maxCharsModifier) modifiers[require_providers.MAX_CHARS_PER_MESSAGE_MODIFIER_KEY] = maxCharsModifier;
+	if (config?.inputs && Object.keys(config.inputs).length > 0) modifiers.__outputFormat = `Output each test case as JSON wrapped in <Prompt> tags: <Prompt>{${Object.entries(require_inputVariables.buildPromptInputDescriptions(config.inputs) ?? {}).map(([k, description]) => `"${k}": "${description}"`).join(", ")}}</Prompt>`;
+	const maxCharsModifier = require_promptLength.getMaxCharsPerMessageModifierValue(config?.maxCharsPerMessage);
+	if (maxCharsModifier) modifiers[require_promptLength.MAX_CHARS_PER_MESSAGE_MODIFIER_KEY] = maxCharsModifier;
 	return modifiers;
 }
 function applyDefaultGraderExamples(key, config) {
@@ -11831,7 +12003,7 @@ function isValidMaxCharsPerMessage(limit) {
 }
 function getMaxCharsPerMessageFromConfig(config) {
 	if (isValidMaxCharsPerMessage(config?.maxCharsPerMessage)) return config.maxCharsPerMessage;
-	const maxCharsModifier = (config?.modifiers)?.[require_providers.MAX_CHARS_PER_MESSAGE_MODIFIER_KEY];
+	const maxCharsModifier = (config?.modifiers)?.[require_promptLength.MAX_CHARS_PER_MESSAGE_MODIFIER_KEY];
 	if (typeof maxCharsModifier !== "string") return;
 	const match = /must be (\d+) characters or fewer\./.exec(maxCharsModifier);
 	if (!match) return;
@@ -11915,7 +12087,7 @@ function withMaxCharsRetries(pluginFactory) {
 				const rejectedPromptLengths = [];
 				let rejectedPromptLimit;
 				for (const testCase of generatedTestCases) {
-					const violation = require_providers.getGeneratedPromptOverLimit(String(testCase.vars?.[params.injectVar] ?? ""), maxCharsPerMessage);
+					const violation = require_promptLength.getGeneratedPromptOverLimit(String(testCase.vars?.[params.injectVar] ?? ""), maxCharsPerMessage);
 					if (violation) {
 						rejectedPromptLengths.push(violation.length);
 						rejectedPromptLimit = violation.limit;
@@ -11932,16 +12104,16 @@ function withMaxCharsRetries(pluginFactory) {
 }
 async function fetchRemoteTestCases(key, purpose, injectVar, n, config) {
 	require_invariant.invariant(!require_logger.getEnvBool("PROMPTFOO_DISABLE_REDTEAM_REMOTE_GENERATION"), "fetchRemoteTestCases should never be called when remote generation is disabled");
-	const remoteHealth = await checkRemoteHealth(require_server.getRemoteHealthUrl());
+	const remoteHealth = await checkRemoteHealth(require_remoteGeneration.getRemoteHealthUrl());
 	if (remoteHealth.status !== "OK") {
 		require_logger.logger.error(`Error generating test cases for ${key}: ${remoteHealth.message}`);
 		return [];
 	}
 	const { graderExamples, ...configForRemote } = config ?? {};
-	const maxCharsModifier = require_providers.getMaxCharsPerMessageModifierValue(config?.maxCharsPerMessage);
+	const maxCharsModifier = require_promptLength.getMaxCharsPerMessageModifierValue(config?.maxCharsPerMessage);
 	if (maxCharsModifier) configForRemote.modifiers = {
 		...configForRemote.modifiers ?? {},
-		[require_providers.MAX_CHARS_PER_MESSAGE_MODIFIER_KEY]: maxCharsModifier
+		[require_promptLength.MAX_CHARS_PER_MESSAGE_MODIFIER_KEY]: maxCharsModifier
 	};
 	const body = JSON.stringify({
 		config: configForRemote,
@@ -11954,7 +12126,7 @@ async function fetchRemoteTestCases(key, purpose, injectVar, n, config) {
 		email: require_accounts.getUserEmail()
 	});
 	try {
-		const { data, status, statusText } = await require_cache.fetchWithCache(require_server.getRemoteGenerationUrl(), {
+		const { data, status, statusText } = await require_cache.fetchWithCache(require_remoteGeneration.getRemoteGenerationUrl(), {
 			method: "POST",
 			headers: { "Content-Type": "application/json" },
 			body
@@ -11963,6 +12135,7 @@ async function fetchRemoteTestCases(key, purpose, injectVar, n, config) {
 			require_logger.logger.error(`Error generating test cases for ${key}: ${statusText} ${JSON.stringify(data)}`);
 			return [];
 		}
+		if (require_promptfoo.requiresRemoteMaterialization(config?.inputs)) require_promptfoo.assertRemoteMaterializationHandled(data, `Remote plugin generation for ${key}`);
 		const ret = data.result;
 		require_logger.logger.debug(`Received remote generation for ${key}:\n${JSON.stringify(ret)}`);
 		return ret;
@@ -11977,17 +12150,18 @@ function createPluginFactory(PluginClass, key, validate) {
 		validate,
 		action: async ({ provider, purpose, injectVar, n, delayMs, config }) => {
 			const configWithDefaults = applyDefaultGraderExamples(key, config);
-			if (PluginClass.canGenerateRemote === false || !require_server.shouldGenerateRemote()) {
+			if (PluginClass.canGenerateRemote === false || !require_remoteGeneration.shouldGenerateRemote()) {
 				require_logger.logger.debug(`Using local redteam generation for ${key}`);
 				return new PluginClass(provider, purpose, injectVar, configWithDefaults).generateTests(n, delayMs);
 			}
+			const pluginId = require_util$2.getShortPluginId(key);
 			const testCases = await fetchRemoteTestCases(key, purpose, injectVar, n, configWithDefaults ?? {});
 			const computedModifiers = computeModifiersFromConfig(configWithDefaults);
 			return testCases.map((testCase) => ({
 				...testCase,
 				metadata: {
 					...testCase.metadata,
-					pluginId: require_providers.getShortPluginId(key),
+					pluginId,
 					pluginConfig: {
 						...configWithDefaults,
 						modifiers: computedModifiers
@@ -12043,8 +12217,8 @@ const pluginFactories = [
 	...unalignedHarmCategories.map((category) => ({
 		key: category,
 		action: async (params) => {
-			if (require_server.neverGenerateRemote()) {
-				require_logger.logger.error(`${category} plugin requires remote generation to be enabled`);
+			if (require_remoteGeneration.neverGenerateRemote()) {
+				require_logger.logger.error(require_remoteGeneration.getRemoteGenerationExplicitlyDisabledError(`${category} plugin`));
 				return [];
 			}
 			const testCases = await getHarmfulTests(params, category);
@@ -12053,7 +12227,7 @@ const pluginFactories = [
 				...testCase,
 				metadata: {
 					...testCase.metadata,
-					pluginId: require_providers.getShortPluginId(category),
+					pluginId: require_util$2.getShortPluginId(category),
 					pluginConfig: {
 						...params.config,
 						modifiers: computedModifiers
@@ -12066,14 +12240,15 @@ const pluginFactories = [
 const piiPlugins = require_types.PII_PLUGINS.map((category) => ({
 	key: category,
 	action: async (params) => {
-		if (require_server.shouldGenerateRemote()) {
+		if (require_remoteGeneration.shouldGenerateRemote()) {
+			const pluginId = require_util$2.getShortPluginId(category);
 			const testCases = await fetchRemoteTestCases(category, params.purpose, params.injectVar, params.n, params.config ?? {});
 			const computedModifiers = computeModifiersFromConfig(params.config);
 			return testCases.map((testCase) => ({
 				...testCase,
 				metadata: {
 					...testCase.metadata,
-					pluginId: require_providers.getShortPluginId(category),
+					pluginId,
 					pluginConfig: {
 						...params.config,
 						modifiers: computedModifiers
@@ -12086,7 +12261,7 @@ const piiPlugins = require_types.PII_PLUGINS.map((category) => ({
 			...testCase,
 			metadata: {
 				...testCase.metadata,
-				pluginId: require_providers.getShortPluginId(category)
+				pluginId: require_util$2.getShortPluginId(category)
 			}
 		}));
 	}
@@ -12094,17 +12269,18 @@ const piiPlugins = require_types.PII_PLUGINS.map((category) => ({
 const biasPlugins = require_types.BIAS_PLUGINS.map((category) => ({
 	key: category,
 	action: async (params) => {
-		if (require_server.neverGenerateRemote()) {
-			require_logger.logger.error(`${category} plugin requires remote generation to be enabled`);
+		if (require_remoteGeneration.neverGenerateRemote()) {
+			require_logger.logger.error(require_remoteGeneration.getRemoteGenerationExplicitlyDisabledError(`${category} plugin`));
 			return [];
 		}
+		const pluginId = require_util$2.getShortPluginId(category);
 		const testCases = await fetchRemoteTestCases(category, params.purpose, params.injectVar, params.n, params.config ?? {});
 		const computedModifiers = computeModifiersFromConfig(params.config);
 		return testCases.map((testCase) => ({
 			...testCase,
 			metadata: {
 				...testCase.metadata,
-				pluginId: require_providers.getShortPluginId(category),
+				pluginId,
 				pluginConfig: {
 					...params.config,
 					modifiers: computedModifiers
@@ -12119,17 +12295,18 @@ function createRemotePlugin(key, validate) {
 		validate,
 		action: async ({ purpose, injectVar, n, config }) => {
 			const configWithDefaults = applyDefaultRemotePluginConfig(key, config);
-			if (require_server.neverGenerateRemote()) {
-				require_logger.logger.error(`${key} plugin requires remote generation to be enabled`);
+			if (require_remoteGeneration.neverGenerateRemote()) {
+				require_logger.logger.error(require_remoteGeneration.getRemoteGenerationExplicitlyDisabledError(`${key} plugin`));
 				return [];
 			}
+			const pluginId = require_util$2.getShortPluginId(key);
 			const testCases = await fetchRemoteTestCases(key, purpose, injectVar, n, configWithDefaults ?? {});
 			const computedModifiers = computeModifiersFromConfig(configWithDefaults);
 			const testsWithMetadata = testCases.map((testCase) => ({
 				...testCase,
 				metadata: {
 					...testCase.metadata,
-					pluginId: require_providers.getShortPluginId(key),
+					pluginId,
 					pluginConfig: {
 						...configWithDefaults,
 						modifiers: computedModifiers
@@ -12190,6 +12367,15 @@ async function validateSharpDependency(strategies, plugins, checkSharp = isSharp
 }
 //#endregion
 //#region src/redteam/index.ts
+const MATERIALIZED_MULTI_INPUT_PROMPT_METADATA_KEY = "__promptfooMaterializedMultiInputPrompt";
+function getMaterializedMultiInputPromptSnapshot(metadata) {
+	const snapshot = metadata?.[MATERIALIZED_MULTI_INPUT_PROMPT_METADATA_KEY];
+	return typeof snapshot === "string" ? snapshot : void 0;
+}
+function getMaterializedMultiInputPromptMetadata(vars) {
+	const prompt = vars?.[require_types.MULTI_INPUT_VAR];
+	return typeof prompt === "string" ? { [MATERIALIZED_MULTI_INPUT_PROMPT_METADATA_KEY]: prompt } : void 0;
+}
 function getPolicyText(metadata) {
 	if (!metadata || metadata.policy === void 0 || metadata.policy === null) return;
 	const policyValue = metadata.policy;
@@ -12199,6 +12385,44 @@ function getPolicyText(metadata) {
 		return typeof policyObject.text === "string" && policyObject.text.length > 0 ? policyObject.text : void 0;
 	}
 }
+async function rematerializeStrategyInputVars(testCase, injectVar, provider, purpose, materializationIndex) {
+	const inputs = testCase.metadata?.pluginConfig?.inputs;
+	const inputMaterialization = testCase.metadata?.inputMaterialization;
+	const materializedPromptSnapshot = getMaterializedMultiInputPromptSnapshot(testCase.metadata);
+	const currentInjectVar = testCase.vars?.[injectVar];
+	if (!inputs || Object.keys(inputs).length === 0 || !currentInjectVar) return {
+		inputMaterialization,
+		vars: testCase.vars
+	};
+	const promptChangedSinceMaterialization = typeof currentInjectVar === "string" && typeof materializedPromptSnapshot === "string" && currentInjectVar !== materializedPromptSnapshot;
+	if (Boolean(inputMaterialization) && Object.keys(inputs).every((key) => Object.prototype.hasOwnProperty.call(testCase.vars ?? {}, key)) && !promptChangedSinceMaterialization) return {
+		inputMaterialization,
+		vars: testCase.vars
+	};
+	try {
+		const materializedVars = await require_util$2.extractMaterializedVariablesFromJsonWithMetadata(JSON.parse(String(currentInjectVar)), inputs, {
+			materializationIndex,
+			pluginId: String(testCase.metadata?.pluginId || "unknown-plugin"),
+			provider,
+			purpose
+		});
+		return {
+			inputMaterialization: materializedVars.metadata ? {
+				...inputMaterialization,
+				...materializedVars.metadata
+			} : inputMaterialization,
+			vars: {
+				...testCase.vars,
+				...materializedVars.vars
+			}
+		};
+	} catch {
+		return {
+			inputMaterialization,
+			vars: testCase.vars
+		};
+	}
+}
 /**
 * Gets the severity level for a plugin based on its ID and configuration.
 * @param pluginId - The ID of the plugin.
@@ -12207,7 +12431,7 @@ function getPolicyText(metadata) {
 */
 function getPluginSeverity(pluginId, pluginConfig) {
 	if (pluginConfig?.severity) return pluginConfig.severity;
-	const shortId = require_providers.getShortPluginId(pluginId);
+	const shortId = require_util$2.getShortPluginId(pluginId);
 	return shortId in require_types.riskCategorySeverityMap ? require_types.riskCategorySeverityMap[shortId] : require_types.Severity.Low;
 }
 const POLICY_PREVIEW_MAX_LENGTH = 20;
@@ -12332,14 +12556,15 @@ function buildRedteamModifiers({ maxCharsPerMessage, pluginConfig, testGeneratio
 		...testGenerationInstructions ? { testGenerationInstructions } : {},
 		...pluginConfig?.modifiers ?? {}
 	};
-	const maxCharsPerMessageModifier = require_providers.getMaxCharsPerMessageModifierValue(maxCharsPerMessage ?? pluginConfig?.maxCharsPerMessage);
-	if (maxCharsPerMessageModifier) modifiers[require_providers.MAX_CHARS_PER_MESSAGE_MODIFIER_KEY] = maxCharsPerMessageModifier;
+	const maxCharsPerMessageModifier = require_promptLength.getMaxCharsPerMessageModifierValue(maxCharsPerMessage ?? pluginConfig?.maxCharsPerMessage);
+	if (maxCharsPerMessageModifier) modifiers[require_promptLength.MAX_CHARS_PER_MESSAGE_MODIFIER_KEY] = maxCharsPerMessageModifier;
 	return modifiers;
 }
 const categories = {
 	foundation: require_types.FOUNDATION_PLUGINS,
 	harmful: Object.keys(require_types.HARM_PLUGINS),
 	"coding-agent:core": require_types.CODING_AGENT_CORE_PLUGINS,
+	"coding-agent:all": require_types.CODING_AGENT_PLUGINS,
 	bias: require_types.BIAS_PLUGINS,
 	pii: require_types.PII_PLUGINS,
 	medical: require_types.MEDICAL_PLUGINS,
@@ -12369,7 +12594,7 @@ function getLanguageForTestCase(test) {
 function filterOversizedTestCases(testCases, injectVar, sourceLabel, maxCharsPerMessage) {
 	return testCases.filter((testCase) => {
 		const testCaseMaxCharsPerMessage = maxCharsPerMessage ?? (testCase.metadata?.strategyConfig)?.maxCharsPerMessage ?? (testCase.metadata?.pluginConfig)?.maxCharsPerMessage;
-		const violation = require_providers.getGeneratedPromptOverLimit(String(testCase.vars?.[injectVar] ?? ""), testCaseMaxCharsPerMessage);
+		const violation = require_promptLength.getGeneratedPromptOverLimit(String(testCase.vars?.[injectVar] ?? ""), testCaseMaxCharsPerMessage);
 		if (!violation) return true;
 		require_logger.logger.warn(`[${sourceLabel}] Dropping generated test case that exceeds maxCharsPerMessage=${violation.limit} (${violation.length} chars)`);
 		return false;
@@ -12407,7 +12632,8 @@ function addLanguageToPluginMetadata(test, lang, plugin, maxCharsPerMessage, tes
 				...test.metadata?.modifiers,
 				...languageToAdd
 			},
-			...languageToAdd
+			...languageToAdd,
+			...getMaterializedMultiInputPromptMetadata(test.vars)
 		}
 	};
 }
@@ -12434,18 +12660,18 @@ function addLanguageToPluginMetadata(test, lang, plugin, maxCharsPerMessage, tes
 * @param injectVar - The variable to inject.
 * @returns An array of new test cases generated by strategies.
 */
-async function applyStrategies(testCases, strategies, injectVar, excludeTargetOutputFromAgenticAttackGeneration, maxCharsPerMessage) {
+async function applyStrategies(testCases, strategies, injectVar, provider, purpose, excludeTargetOutputFromAgenticAttackGeneration, maxCharsPerMessage) {
 	const newTestCases = [];
 	const strategyResults = {};
 	for (const strategy of strategies) {
 		require_logger.logger.debug(`Generating ${strategy.id} tests`);
 		let strategyAction;
-		if (strategy.id.startsWith("file://")) strategyAction = (await require_providers.loadStrategy(strategy.id)).action;
+		if (strategy.id.startsWith("file://")) strategyAction = (await require_strategies.loadStrategy(strategy.id)).action;
 		else {
-			let builtinStrategy = require_providers.Strategies.find((s) => s.id === strategy.id);
+			let builtinStrategy = require_strategies.Strategies.find((s) => s.id === strategy.id);
 			if (!builtinStrategy && strategy.id.includes(":")) {
 				const baseStrategyId = strategy.id.split(":")[0];
-				builtinStrategy = require_providers.Strategies.find((s) => s.id === baseStrategyId);
+				builtinStrategy = require_strategies.Strategies.find((s) => s.id === baseStrategyId);
 			}
 			if (!builtinStrategy) {
 				require_logger.logger.warn(`Strategy ${strategy.id} not registered, skipping`);
@@ -12455,7 +12681,7 @@ async function applyStrategies(testCases, strategies, injectVar, excludeTargetOu
 		}
 		const targetPlugins = strategy.config?.plugins;
 		const applicableTestCases = testCases.filter((t) => {
-			if (!require_providers.pluginMatchesStrategyTargets(t, strategy.id, targetPlugins)) return false;
+			if (!require_strategies.pluginMatchesStrategyTargets(t, strategy.id, targetPlugins)) return false;
 			if (t.metadata?.retry === true) {
 				require_logger.logger.debug(`Skipping ${strategy.id} for retry test (plugin: ${t.metadata?.pluginId}) - retry tests are not transformed`);
 				return false;
@@ -12490,14 +12716,8 @@ async function applyStrategies(testCases, strategies, injectVar, excludeTargetOu
 			}
 		}
 		resultTestCases = filterOversizedTestCases(resultTestCases, injectVar, `Strategy ${strategy.id}`, maxCharsPerMessage);
-		newTestCases.push(...resultTestCases.map((t) => {
-			const inputs = t?.metadata?.pluginConfig?.inputs;
-			let updatedVars = t.vars;
-			if (inputs && Object.keys(inputs).length > 0 && t.vars?.[injectVar]) try {
-				const parsed = JSON.parse(String(t.vars[injectVar]));
-				updatedVars = { ...t.vars };
-				Object.assign(updatedVars, require_providers.extractVariablesFromJson(parsed, inputs));
-			} catch {}
+		newTestCases.push(...await Promise.all(resultTestCases.map(async (t, materializationIndex) => {
+			const { inputMaterialization, vars } = await rematerializeStrategyInputVars(t, injectVar, provider, purpose, materializationIndex);
 			const strategyConfig = {
 				...strategy.config || {},
 				...maxCharsPerMessage ? { maxCharsPerMessage } : {},
@@ -12505,16 +12725,18 @@ async function applyStrategies(testCases, strategies, injectVar, excludeTargetOu
 			};
 			return {
 				...t,
-				vars: updatedVars,
+				vars,
 				metadata: {
 					...t?.metadata || {},
 					...strategy.id !== "retry" && { strategyId: t?.metadata?.strategyId || strategy.id },
 					...t?.metadata?.pluginId && { pluginId: t.metadata.pluginId },
 					...t?.metadata?.pluginConfig && { pluginConfig: t.metadata.pluginConfig },
-					...Object.keys(strategyConfig).length > 0 && { strategyConfig }
+					...inputMaterialization && { inputMaterialization },
+					...Object.keys(strategyConfig).length > 0 && { strategyConfig },
+					...getMaterializedMultiInputPromptMetadata(vars)
 				}
 			};
-		}));
+		})));
 		const displayId = strategy.id === "layer" && Array.isArray(strategy.config?.steps) ? `layer(${strategy.config.steps.map((st) => typeof st === "string" ? st : st.id).join("→")})` : strategy.id;
 		const languagesInResults = new Set(strategyTestCases.map((t) => getLanguageForTestCase(t)).filter((lang) => lang !== void 0));
 		const applyNumTestsCap = (calculatedRequested) => {
@@ -12672,10 +12894,10 @@ async function synthesize({ abortSignal, delay, entities: entitiesOverride, inje
 		seen.add(key);
 		return true;
 	});
-	const needsGoalExtraction = strategies.some((s) => require_providers.Strategies.find((def) => def.id === s.id)?.requiresGoalExtraction);
-	await require_providers.validateStrategies(strategies);
+	const needsGoalExtraction = strategies.some((s) => require_strategies.Strategies.find((def) => def.id === s.id)?.requiresGoalExtraction);
+	await require_strategies.validateStrategies(strategies);
 	await validateSharpDependency(strategies, plugins);
-	const redteamProvider = await require_providers.redteamProviderManager.getProvider({ provider });
+	const redteamProvider = await require_shared.redteamProviderManager.getProvider({ provider });
 	const { effectiveStrategyCount, includeBasicTests, totalPluginTests, totalTests } = calculateTotalTests(plugins, strategies, language);
 	require_logger.logger.info(`Synthesizing test cases for ${prompts.length} ${prompts.length === 1 ? "prompt" : "prompts"}...\nUsing plugins:\n\n${chalk.default.yellow(plugins.map((p) => {
 		const pluginLanguageConfig = p.config?.language ?? language;
@@ -12779,8 +13001,8 @@ async function synthesize({ abortSignal, delay, entities: entitiesOverride, inje
 	};
 	require_logger.logger.debug("Validating plugins...");
 	plugins = [...new Set(expandedPlugins)].filter(validatePlugin).sort();
-	if (require_server.shouldGenerateRemote()) {
-		const healthUrl = require_server.getRemoteHealthUrl();
+	if (require_remoteGeneration.shouldGenerateRemote()) {
+		const healthUrl = require_remoteGeneration.getRemoteHealthUrl();
 		if (healthUrl) {
 			require_logger.logger.debug(`Checking Promptfoo API health at ${healthUrl}...`);
 			const healthResult = await checkRemoteHealth(healthUrl);
@@ -12857,14 +13079,21 @@ async function synthesize({ abortSignal, delay, entities: entitiesOverride, inje
 				}
 			});
 			const languageResults = await Promise.allSettled(languagePromises);
-			for (const result of languageResults) if (result.status === "fulfilled") {
+			for (const [index, result] of languageResults.entries()) if (result.status === "fulfilled") {
 				const { lang, tests, requested, generated } = result.value;
 				allPluginTests.push(...tests);
 				resultsPerLanguage[lang || "default"] = {
 					requested,
 					generated
 				};
-			} else require_logger.logger.warn(`[Language Processing] Error generating tests for ${plugin.id}: ${result.reason}`);
+			} else {
+				const lang = languages[index];
+				require_logger.logger.warn(`[Language Processing] Error generating tests for ${plugin.id}: ${result.reason}`);
+				resultsPerLanguage[lang || "default"] = {
+					requested: plugin.numTests,
+					generated: 0
+				};
+			}
 			require_logger.logger.debug(`[Language Processing] Total tests generated for ${plugin.id}: ${allPluginTests.length} (across ${languages.length} language(s))`);
 			if (!Array.isArray(allPluginTests) || allPluginTests.length === 0) require_logger.logger.warn(`Failed to generate tests for ${plugin.id}`);
 			else {
@@ -12875,7 +13104,7 @@ async function synthesize({ abortSignal, delay, entities: entitiesOverride, inje
 						const promptVar = testCase.vars?.[injectVar];
 						const prompt = Array.isArray(promptVar) ? promptVar[0] : String(promptVar);
 						const policy = getPolicyText(testCase.metadata);
-						const extractedGoal = await require_providers.extractGoalFromPrompt(prompt, purpose, plugin.id, policy);
+						const extractedGoal = await require_util$2.extractGoalFromPrompt(prompt, purpose, plugin.id, policy);
 						testCase.metadata.goal = extractedGoal;
 					}
 				}
@@ -12898,49 +13127,73 @@ async function synthesize({ abortSignal, delay, entities: entitiesOverride, inje
 				generated: allPluginTests.length
 			};
 		} else if (plugin.id.startsWith("file://")) try {
-			const customTests = await new CustomPlugin(redteamProvider, purpose, injectVar, plugin.id, resolvePluginConfigWithMaxChars(plugin.config, maxCharsPerMessage)).generateTests(plugin.numTests, delay);
-			const testCasesWithMetadata = filterOversizedTestCases(customTests.map((t) => {
-				const includePluginConfig = !(t.metadata && Object.hasOwn(t.metadata, "pluginConfig") && t.metadata.pluginConfig === void 0);
-				const pluginConfigWithMaxChars = {
+			const languageConfig = plugin.config?.language ?? language;
+			const languages = Array.isArray(languageConfig) ? languageConfig : languageConfig ? [languageConfig] : [void 0];
+			const allCustomTests = [];
+			const resultsPerLanguage = {};
+			const languagePromises = languages.map(async (lang) => {
+				const resolvedConfig = {
 					...resolvePluginConfigWithMaxChars(plugin.config, maxCharsPerMessage),
-					...t.metadata?.pluginConfig ?? {}
+					...lang ? { language: lang } : {},
+					...hasMultipleInputs ? { inputs } : {}
 				};
-				const modifiers = {
-					...buildRedteamModifiers({
+				const customPluginConfig = {
+					...resolvedConfig,
+					modifiers: buildRedteamModifiers({
 						maxCharsPerMessage,
-						pluginConfig: pluginConfigWithMaxChars,
+						pluginConfig: resolvedConfig,
 						testGenerationInstructions
-					}),
-					...t.metadata?.modifiers
+					})
 				};
+				const testCasesWithMetadata = filterOversizedTestCases((await new CustomPlugin(redteamProvider, purpose, injectVar, plugin.id, customPluginConfig).generateTests(plugin.numTests, delay)).map((t) => addLanguageToPluginMetadata(t, lang, plugin, maxCharsPerMessage, testGenerationInstructions)), injectVar, `Custom plugin ${plugin.id}`, maxCharsPerMessage);
 				return {
-					...t,
-					metadata: {
-						...t.metadata || {},
-						pluginId: plugin.id,
-						...includePluginConfig && { pluginConfig: pluginConfigWithMaxChars },
-						severity: plugin.severity || getPluginSeverity(plugin.id, resolvePluginConfig(plugin.config)),
-						modifiers
-					}
+					lang,
+					tests: testCasesWithMetadata,
+					requested: plugin.numTests,
+					generated: testCasesWithMetadata.length
+				};
+			});
+			const languageResults = await Promise.allSettled(languagePromises);
+			for (const [index, result] of languageResults.entries()) if (result.status === "fulfilled") {
+				const { lang, tests, requested, generated } = result.value;
+				allCustomTests.push(...tests);
+				resultsPerLanguage[lang || "default"] = {
+					requested,
+					generated
 				};
-			}), injectVar, `Custom plugin ${plugin.id}`, maxCharsPerMessage);
+			} else {
+				const lang = languages[index];
+				require_logger.logger.warn(`[Language Processing] Error generating tests for custom plugin ${plugin.id}: ${result.reason}`);
+				resultsPerLanguage[lang || "default"] = {
+					requested: plugin.numTests,
+					generated: 0
+				};
+			}
 			if (needsGoalExtraction) {
-				require_logger.logger.debug(`Extracting goal for ${testCasesWithMetadata.length} custom tests from ${plugin.id}...`);
-				for (const testCase of testCasesWithMetadata) {
+				require_logger.logger.debug(`Extracting goal for ${allCustomTests.length} custom tests from ${plugin.id}...`);
+				for (const testCase of allCustomTests) {
 					const promptVar = testCase.vars?.[injectVar];
 					const prompt = Array.isArray(promptVar) ? promptVar[0] : String(promptVar);
 					const policy = getPolicyText(testCase.metadata);
-					const extractedGoal = await require_providers.extractGoalFromPrompt(prompt, purpose, plugin.id, policy);
+					const extractedGoal = await require_util$2.extractGoalFromPrompt(prompt, purpose, plugin.id, policy);
 					testCase.metadata.goal = extractedGoal;
 				}
 			}
-			testCases.push(...testCasesWithMetadata);
-			require_logger.logger.debug(`Added ${customTests.length} custom test cases from ${plugin.id}`);
-			const displayId = getPluginDisplayId(plugin);
-			pluginResults[displayId] = {
-				requested: plugin.numTests,
-				generated: testCasesWithMetadata.length
+			testCases.push(...allCustomTests);
+			require_logger.logger.debug(`Added ${allCustomTests.length} custom test cases from ${plugin.id}`);
+			const baseDisplayId = getPluginDisplayId(plugin);
+			if (languages.filter((lang) => lang !== void 0).length > 1) for (const [langKey, result] of Object.entries(resultsPerLanguage)) {
+				const displayId = langKey === "en" ? baseDisplayId : `(${langKey}) ${baseDisplayId}`;
+				pluginResults[displayId] = {
+					requested: result.requested,
+					generated: result.generated
+				};
+			}
+			else pluginResults[baseDisplayId] = {
+				requested: plugin.numTests * languages.length,
+				generated: allCustomTests.length
 			};
+			progressBar?.increment(plugin.numTests * languages.length);
 		} catch (e) {
 			require_logger.logger.error(`Error generating tests for custom plugin ${plugin.id}: ${e}`);
 			const displayId = getPluginDisplayId(plugin);
@@ -12969,7 +13222,7 @@ async function synthesize({ abortSignal, delay, entities: entitiesOverride, inje
 			targetIds,
 			...retryStrategy.config
 		};
-		const { testCases: retryTestCases, strategyResults: retryResults } = await applyStrategies(pluginTestCases, [retryStrategy], injectVar, void 0, maxCharsPerMessage);
+		const { testCases: retryTestCases, strategyResults: retryResults } = await applyStrategies(pluginTestCases, [retryStrategy], injectVar, redteamProvider, purpose, void 0, maxCharsPerMessage);
 		pluginTestCases.push(...retryTestCases);
 		Object.assign(strategyResults, retryResults);
 		if (showProgressBar) progressBar?.increment(retryTestCases.length);
@@ -12977,7 +13230,7 @@ async function synthesize({ abortSignal, delay, entities: entitiesOverride, inje
 	checkAbort();
 	const nonBasicStrategies = strategies.filter((s) => !["basic", "retry"].includes(s.id));
 	if (showProgressBar && nonBasicStrategies.length > 0) progressBar?.update({ task: "Applying strategies" });
-	const { testCases: strategyTestCases, strategyResults: otherStrategyResults } = await applyStrategies(pluginTestCases, nonBasicStrategies, injectVar, excludeTargetOutputFromAgenticAttackGeneration, maxCharsPerMessage);
+	const { testCases: strategyTestCases, strategyResults: otherStrategyResults } = await applyStrategies(pluginTestCases, nonBasicStrategies, injectVar, redteamProvider, purpose, excludeTargetOutputFromAgenticAttackGeneration, maxCharsPerMessage);
 	Object.assign(strategyResults, otherStrategyResults);
 	if (showProgressBar && strategyTestCases.length > 0) progressBar?.increment(strategyTestCases.length);
 	const finalTestCases = [...includeBasicTests ? pluginTestCases : [], ...strategyTestCases];
@@ -13104,7 +13357,7 @@ async function doGenerateRedteam(options) {
 		redteamConfig = resolved.config.redteam;
 		commandLineOptions = resolved.commandLineOptions;
 		resolvedConfig = resolved.config;
-		await require_providers.checkCloudPermissions(resolved.config);
+		await require_storage.checkCloudPermissions(resolved.config);
 		if (redteamConfig && resolved.testSuite.tests && resolved.testSuite.tests.length > 0) require_logger.logger.warn(chalk.default.yellow(dedent.default`
             ⚠️  Warning: Found both 'tests' section and 'redteam' configuration in your config file.
 
@@ -13119,8 +13372,8 @@ async function doGenerateRedteam(options) {
           `));
 		try {
 			const providerId = require_providers.getProviderIds(resolved.config.providers)[0];
-			if (require_providers.isCloudProvider(providerId)) {
-				const overrides = await require_providers.getPluginSeverityOverridesFromCloud(require_providers.getCloudDatabaseId(providerId));
+			if (require_storage.isCloudProvider(providerId)) {
+				const overrides = await require_storage.getPluginSeverityOverridesFromCloud(require_storage.getCloudDatabaseId(providerId));
 				if (overrides) {
 					pluginSeverityOverrides = new Map(Object.entries(overrides.severities));
 					pluginSeverityOverridesId = overrides.id;
@@ -13138,7 +13391,7 @@ async function doGenerateRedteam(options) {
 		require_logger.logger.info(chalk.default.red(`\nCan't generate without configuration - run ${chalk.default.yellow.bold(promptfooCommand("redteam init"))} first`));
 		return null;
 	}
-	if (!require_server.neverGenerateRemote()) {
+	if (!require_remoteGeneration.neverGenerateRemote()) {
 		let hasValidEmail = false;
 		while (!hasValidEmail) {
 			const { emailNeedsValidation } = await require_accounts.promptForEmailUnverified();
@@ -13202,7 +13455,7 @@ async function doGenerateRedteam(options) {
 	}
 	const policyPluginsWithRefs = plugins.filter((plugin) => plugin.config?.policy && require_graders.isValidPolicyObject(plugin.config?.policy) && require_graders.determinePolicyTypeFromId(plugin.config.policy.id) === "reusable");
 	if (policyPluginsWithRefs.length > 0) {
-		const teamId = (await require_providers.resolveTeamId()).id;
+		const teamId = (await require_storage.resolveTeamId()).id;
 		const policiesById = await require_graders.getCustomPolicies(policyPluginsWithRefs, teamId);
 		for (const policyPlugin of policyPluginsWithRefs) {
 			const policyId = policyPlugin.config.policy.id;
@@ -13774,7 +14027,7 @@ async function sendChunkedResults(evalRecord, url$5, options = {}) {
 	const isVerbose = require_logger.isDebugEnabled();
 	const { silent = false } = options;
 	require_logger.logger.debug(`Starting chunked results upload to ${url$5}`);
-	await require_providers.checkCloudPermissions(evalRecord.config);
+	await require_storage.checkCloudPermissions(evalRecord.config);
 	const inlineBlobs = require_extractor.isBlobStorageEnabled() && require_logger.getEnvBool("PROMPTFOO_SHARE_INLINE_BLOBS", !require_fetch.cloudConfig.isEnabled());
 	const inlineCache = inlineBlobs ? createBlobInlineCache() : null;
 	let sampleResults = (await evalRecord.fetchResultsBatched(100).next()).value ?? [];
@@ -13870,6 +14123,10 @@ function stripAuthFromUrl(urlString) {
 	}
 }
 async function handleEmailCollection(evalRecord) {
+	if (evalRecord.author) {
+		require_logger.logger.debug(`[Share] Skipping email collection because author is already set`, { evalId: evalRecord.id });
+		return;
+	}
 	if (!process.stdout.isTTY || require_logger.isCI() || require_logger.getEnvBool("PROMPTFOO_DISABLE_SHARE_EMAIL_REQUEST")) return;
 	let email = require_accounts.getUserEmail();
 	if (!email) {
@@ -13912,7 +14169,7 @@ async function createShareableUrl(evalRecord, options = {}) {
 		return null;
 	}
 	if (!silent) {
-		const orgContext = await require_providers.getOrgContext();
+		const orgContext = await require_storage.getOrgContext();
 		if (orgContext) {
 			const teamSuffix = orgContext.teamName ? ` > ${orgContext.teamName}` : "";
 			require_logger.logger.info(`${chalk.default.dim("Sharing to:")} ${chalk.default.cyan(orgContext.organizationName)}${teamSuffix}`);
@@ -14013,6 +14270,14 @@ function shouldShareResults(opts) {
 	return require_fetch.cloudConfig.isEnabled() && sharing !== false;
 }
 //#endregion
+//#region src/commands/eval/redteamWarning.ts
+function warnIfRedteamConfigHasNoTests(config, testSuite) {
+	if (config.redteam && (!testSuite.tests || testSuite.tests.length === 0) && (!testSuite.scenarios || testSuite.scenarios.length === 0)) require_logger.logger.warn(chalk.default.yellow(dedent.default`
+        Warning: Config file has a redteam section but no test cases.
+        Did you mean to run ${chalk.default.bold("promptfoo redteam generate")} instead?
+        `));
+}
+//#endregion
 //#region src/util/formatDuration.ts
 /**
 * Formats a duration in seconds into a human-readable string
@@ -14359,7 +14624,7 @@ async function doEval(cmdObj, defaultConfig, defaultConfigPath, evaluateOptions)
 		const cloudConfigId = uuidConfigArgs[0];
 		if (cmdObj.watch) throw new Error("--watch is not supported when using a cloud config UUID with -c. Use a local config file path for watch mode.");
 		try {
-			defaultConfig = await require_providers.getEvalConfigFromCloud(cloudConfigId);
+			defaultConfig = await require_storage.getEvalConfigFromCloud(cloudConfigId);
 		} catch (error) {
 			const reason = error instanceof Error ? error.message : String(error);
 			throw new Error(`Failed to load cloud eval config "${cloudConfigId}". ${reason}. Cloud UUID inputs do not fall back to local file paths. Check authentication and that the UUID exists.`);
@@ -14453,14 +14718,11 @@ async function doEval(cmdObj, defaultConfig, defaultConfigPath, evaluateOptions)
 			require_logger.state.resume = true;
 			require_logger.state.retryMode = true;
 		} else ({config, testSuite, basePath: _basePath, commandLineOptions} = await resolveConfigs(cmdObj, defaultConfig));
-		if (!cmdObj.envPath && commandLineOptions?.envPath) {
+		if ((!cmdObj.envPath || cmdObj.envPath.length === 0) && commandLineOptions?.envPath) {
 			require_logger.logger.debug(`Loading additional environment from config: ${commandLineOptions.envPath}`);
 			require_util.setupEnv(commandLineOptions.envPath);
 		}
-		if (config.redteam && (!testSuite.tests || testSuite.tests.length === 0) && (!testSuite.scenarios || testSuite.scenarios.length === 0)) require_logger.logger.warn(chalk.default.yellow(dedent.default`
-        Warning: Config file has a redteam section but no test cases.
-        Did you mean to run ${chalk.default.bold("promptfoo redteam generate")} instead?
-        `));
+		warnIfRedteamConfigHasNoTests(config, testSuite);
 		if (config.redteam && Array.isArray(config.providers) && config.providers.length > 0 && typeof config.providers[0] === "object" && config.providers[0].id === "http") {
 			const maybeUrl = config.providers[0]?.config?.url;
 			if (typeof maybeUrl === "string" && maybeUrl.includes("promptfoo.app")) require_telemetry.telemetry.record("feature_used", { feature: "redteam_run_with_example" });
@@ -14508,7 +14770,7 @@ async function doEval(cmdObj, defaultConfig, defaultConfigPath, evaluateOptions)
 			};
 			testSuite.tests = await filterTests(testSuite, filterOptions);
 		}
-		if (!require_server.neverGenerateRemote() && config.redteam && config.redteam.plugins && config.redteam.plugins.length > 0 && testSuite.tests && testSuite.tests.length > 0) {
+		if (!require_remoteGeneration.neverGenerateRemote() && config.redteam && config.redteam.plugins && config.redteam.plugins.length > 0 && testSuite.tests && testSuite.tests.length > 0) {
 			let hasValidEmail = false;
 			while (!hasValidEmail) {
 				const { emailNeedsValidation } = await require_accounts.promptForEmailUnverified();
@@ -14526,7 +14788,7 @@ async function doEval(cmdObj, defaultConfig, defaultConfigPath, evaluateOptions)
 			process.exitCode = 1;
 			return new Eval({}, { persisted: false });
 		}
-		await require_providers.checkCloudPermissions(config);
+		await require_storage.checkCloudPermissions(config);
 		const options = {
 			...evaluateOptions,
 			showProgressBar: require_logger.getLogLevel() === "debug" ? false : cmdObj.progressBar === void 0 ? evaluateOptions.showProgressBar === void 0 ? true : evaluateOptions.showProgressBar : cmdObj.progressBar !== false,
@@ -14568,7 +14830,14 @@ async function doEval(cmdObj, defaultConfig, defaultConfigPath, evaluateOptions)
         ${zod.z.prettifyError(testSuiteSchema.error)}
 
       Please review your promptfooconfig.yaml configuration.`));
-		const evalRecord = resumeEval ? resumeEval : cmdObj.write ? await Eval.create(config, testSuite.prompts, { runtimeOptions: options }) : new Eval(config, { runtimeOptions: options });
+		const author = require_accounts.getAuthor();
+		const evalRecord = resumeEval ? resumeEval : cmdObj.write ? await Eval.create(config, testSuite.prompts, {
+			author,
+			runtimeOptions: options
+		}) : new Eval(config, {
+			author,
+			runtimeOptions: options
+		});
 		const abortController = new AbortController();
 		const previousAbortSignal = evaluateOptions.abortSignal;
 		evaluateOptions.abortSignal = previousAbortSignal ? AbortSignal.any([previousAbortSignal, abortController.signal]) : abortController.signal;
@@ -14679,7 +14948,7 @@ async function doEval(cmdObj, defaultConfig, defaultConfigPath, evaluateOptions)
 		const paths = (Array.isArray(outputPath) ? outputPath : [outputPath]).filter((p) => typeof p === "string" && p.length > 0 && !p.endsWith(".jsonl"));
 		const isRedteam = Boolean(config.redteam);
 		const duration = Math.round((Date.now() - startTime) / 1e3);
-		const tracker = require_providers.TokenUsageTracker.getInstance();
+		const tracker = require_shared.TokenUsageTracker.getInstance();
 		const targetErrorStatus = await evalRecord.findTargetErrorStatus();
 		const summaryLines = generateEvalSummary({
 			evalId: evalRecord.id,
@@ -14709,7 +14978,7 @@ async function doEval(cmdObj, defaultConfig, defaultConfigPath, evaluateOptions)
 		} else for (const line of summaryLines) require_logger.logger.info(line);
 		let shareableUrl = null;
 		if (sharePromise != null) {
-			const orgContext = await require_providers.getOrgContext();
+			const orgContext = await require_storage.getOrgContext();
 			const orgSuffix = orgContext ? ` to ${orgContext.organizationName}${orgContext.teamName ? ` > ${orgContext.teamName}` : ""}` : "";
 			if (process.stdout.isTTY && !require_logger.isCI()) {
 				const spinner = (0, ora.default)({
@@ -14884,7 +15153,7 @@ async function doRedteamRun(options) {
 		redteamPath = path.join(configDir, "redteam.yaml");
 	}
 	try {
-		const healthUrl = require_server.getRemoteHealthUrl();
+		const healthUrl = require_remoteGeneration.getRemoteHealthUrl();
 		if (healthUrl) {
 			require_logger.logger.debug(`Checking Promptfoo API health at ${healthUrl}...`);
 			const healthResult = await checkRemoteHealth(healthUrl);
@@ -14970,6 +15239,12 @@ async function doRedteamRun(options) {
 	return evalResult;
 }
 //#endregion
+//#region src/types/transform.ts
+/** Runtime type guard for `TransformFunction` values. */
+function isTransformFunction(value) {
+	return typeof value === "function";
+}
+//#endregion
 //#region src/index.ts
 /**
 * Shallow-clone a test case so the caller can swap in resolved ApiProvider
@@ -14991,66 +15266,148 @@ function cloneTestForResolve(test) {
 	if (test.assert) cloned.assert = test.assert.map((assertion) => ({ ...assertion }));
 	return cloned;
 }
+function toSerializableProviderRef(provider) {
+	if (require_types.isApiProvider(provider)) return require_evalResult.sanitizeProvider(provider);
+	if (Array.isArray(provider)) return provider.map(toSerializableProviderRef);
+	return provider;
+}
+function isRecord(value) {
+	return Boolean(value && typeof value === "object" && !Array.isArray(value));
+}
+function withSerializableProvider(record) {
+	if (!require_types.isApiProvider(record.provider)) return record;
+	return {
+		...record,
+		provider: require_evalResult.sanitizeProvider(record.provider)
+	};
+}
+/**
+* Function-valued transforms are first-class at runtime but are silently dropped
+* by `JSON.stringify`. Persisted eval configs (drizzle-stored) must never retain
+* a function reference, so replace every `transform`-like field with a
+* `[inline function]: name` marker. Non-function values pass through unchanged.
+*
+* `droppedRef.value` is flipped to `true` the first time a function is replaced
+* so the caller can emit a single warning instead of logging per field.
+*/
+function replaceFunctionTransforms(record, droppedRef) {
+	let result;
+	for (const key of require_transform$1.TRANSFORM_KEYS) {
+		const value = record[key];
+		if (!isTransformFunction(value)) continue;
+		if (!result) result = { ...record };
+		result[key] = value.name ? `${require_transform$1.INLINE_FUNCTION_LABEL}: ${value.name}` : require_transform$1.INLINE_FUNCTION_LABEL;
+		droppedRef.value = true;
+	}
+	return result ?? record;
+}
+function toSerializableAssertion(assertion, droppedRef) {
+	if (!isRecord(assertion)) return assertion;
+	let sanitizedAssertion = withSerializableProvider(assertion);
+	sanitizedAssertion = replaceFunctionTransforms(sanitizedAssertion, droppedRef);
+	if (Array.isArray(assertion.assert)) sanitizedAssertion = {
+		...sanitizedAssertion,
+		assert: assertion.assert.map((a) => toSerializableAssertion(a, droppedRef))
+	};
+	return sanitizedAssertion;
+}
+function toSerializableTestCase(test, droppedRef) {
+	if (!isRecord(test)) return test;
+	let sanitizedTest = withSerializableProvider(test);
+	if (isRecord(test.options)) {
+		let options = withSerializableProvider(test.options);
+		options = replaceFunctionTransforms(options, droppedRef);
+		if (options !== test.options) sanitizedTest = {
+			...sanitizedTest,
+			options
+		};
+	}
+	if (Array.isArray(test.assert)) sanitizedTest = {
+		...sanitizedTest,
+		assert: test.assert.map((a) => toSerializableAssertion(a, droppedRef))
+	};
+	return sanitizedTest;
+}
+function toSerializableScenario(scenario, droppedRef) {
+	if (!isRecord(scenario)) return scenario;
+	if (!Array.isArray(scenario.tests)) return scenario;
+	return {
+		...scenario,
+		tests: scenario.tests.map((t) => toSerializableTestCase(t, droppedRef))
+	};
+}
+function createSerializableUnifiedConfig(testSuite, prompts) {
+	const droppedRef = { value: false };
+	const config = {
+		...testSuite,
+		providers: toSerializableProviderRef(testSuite.providers),
+		defaultTest: toSerializableTestCase(testSuite.defaultTest, droppedRef),
+		tests: Array.isArray(testSuite.tests) ? testSuite.tests.map((t) => toSerializableTestCase(t, droppedRef)) : testSuite.tests,
+		scenarios: Array.isArray(testSuite.scenarios) ? testSuite.scenarios.map((s) => toSerializableScenario(s, droppedRef)) : testSuite.scenarios,
+		prompts
+	};
+	if (droppedRef.value && testSuite.writeLatestResults) require_logger.logger.warn("Function-valued transform(s) in testSuite were replaced with \"[inline function]\" markers in the persisted config. Re-running the saved eval will not invoke them; use string expressions or file:// references if you need the config to round-trip.");
+	return config;
+}
 async function evaluate(testSuite, options = {}) {
-	if (testSuite.writeLatestResults) await runDbMigrations();
-	const loadedProviders = await require_providers.loadApiProviders(testSuite.providers, { env: testSuite.env });
+	const { author: suiteAuthor, ...testSuiteConfig } = testSuite;
+	if (testSuiteConfig.writeLatestResults) await runDbMigrations();
+	const loadedProviders = await require_providers.loadApiProviders(testSuiteConfig.providers, { env: testSuiteConfig.env });
 	const providerMap = {};
 	for (const p of loadedProviders) {
 		providerMap[p.id()] = p;
 		if (p.label) providerMap[p.label] = p;
 	}
-	let resolvedDefaultTest = testSuite.defaultTest;
-	if (typeof testSuite.defaultTest === "string" && testSuite.defaultTest.startsWith("file://")) resolvedDefaultTest = await require_util.maybeLoadFromExternalFile(testSuite.defaultTest);
+	let resolvedDefaultTest = testSuiteConfig.defaultTest;
+	if (typeof testSuiteConfig.defaultTest === "string" && testSuiteConfig.defaultTest.startsWith("file://")) resolvedDefaultTest = await require_util.maybeLoadFromExternalFile(testSuiteConfig.defaultTest);
 	const constructedTestSuite = {
-		...testSuite,
+		...testSuiteConfig,
 		defaultTest: resolvedDefaultTest,
-		scenarios: testSuite.scenarios,
+		scenarios: testSuiteConfig.scenarios,
 		providers: loadedProviders,
-		tests: await readTests(testSuite.tests),
-		nunjucksFilters: await require_util.readFilters(testSuite.nunjucksFilters || {}),
-		prompts: await require_graders.processPrompts(testSuite.prompts)
+		tests: await readTests(testSuiteConfig.tests),
+		nunjucksFilters: await require_util.readFilters(testSuiteConfig.nunjucksFilters || {}),
+		prompts: await require_graders.processPrompts(testSuiteConfig.prompts)
 	};
 	if (typeof constructedTestSuite.defaultTest === "object" && constructedTestSuite.defaultTest) {
 		constructedTestSuite.defaultTest = cloneTestForResolve(constructedTestSuite.defaultTest);
 		if (constructedTestSuite.defaultTest.provider && !require_types.isApiProvider(constructedTestSuite.defaultTest.provider)) constructedTestSuite.defaultTest.provider = await require_providers.resolveProvider(constructedTestSuite.defaultTest.provider, providerMap, {
-			env: testSuite.env,
+			env: testSuiteConfig.env,
 			basePath: require_logger.state.basePath
 		});
 		if (constructedTestSuite.defaultTest.options?.provider && !require_types.isApiProvider(constructedTestSuite.defaultTest.options.provider)) constructedTestSuite.defaultTest.options.provider = await require_providers.resolveProvider(constructedTestSuite.defaultTest.options.provider, providerMap, {
-			env: testSuite.env,
+			env: testSuiteConfig.env,
 			basePath: require_logger.state.basePath
 		});
 	}
 	constructedTestSuite.tests = (constructedTestSuite.tests || []).map(cloneTestForResolve);
 	for (const test of constructedTestSuite.tests) {
 		if (test.options?.provider && !require_types.isApiProvider(test.options.provider)) test.options.provider = await require_providers.resolveProvider(test.options.provider, providerMap, {
-			env: testSuite.env,
+			env: testSuiteConfig.env,
 			basePath: require_logger.state.basePath
 		});
 		for (const assertion of test.assert || []) {
 			if (assertion.type === "assert-set" || typeof assertion.provider === "function") continue;
 			if (assertion.provider && !require_types.isApiProvider(assertion.provider)) assertion.provider = await require_providers.resolveProvider(assertion.provider, providerMap, {
-				env: testSuite.env,
+				env: testSuiteConfig.env,
 				basePath: require_logger.state.basePath
 			});
 		}
 	}
 	if (options.cache === false) require_cache.disableCache();
-	const parsedProviderPromptMap = require_graders.readProviderPromptMap(testSuite, constructedTestSuite.prompts);
-	const unifiedConfig = {
-		...testSuite,
-		prompts: constructedTestSuite.prompts
-	};
-	const evalRecord = testSuite.writeLatestResults ? await Eval.create(unifiedConfig, constructedTestSuite.prompts) : new Eval(unifiedConfig);
+	const parsedProviderPromptMap = require_graders.readProviderPromptMap(testSuiteConfig, constructedTestSuite.prompts);
+	const unifiedConfig = createSerializableUnifiedConfig(testSuiteConfig, constructedTestSuite.prompts);
+	const author = require_accounts.getAuthor(suiteAuthor);
+	const evalRecord = testSuiteConfig.writeLatestResults ? await Eval.create(unifiedConfig, constructedTestSuite.prompts, { author }) : new Eval(unifiedConfig, { author });
 	const ret = await evaluate$1({
 		...constructedTestSuite,
 		providerPromptMap: parsedProviderPromptMap
 	}, evalRecord, {
 		eventSource: "library",
-		isRedteam: Boolean(testSuite.redteam),
+		isRedteam: Boolean(testSuiteConfig.redteam),
 		...options
 	});
-	if (testSuite.writeLatestResults && testSuite.sharing) if (isSharingEnabled(ret)) try {
+	if (testSuiteConfig.writeLatestResults && testSuiteConfig.sharing) if (isSharingEnabled(ret)) try {
 		const shareableUrl = await createShareableUrl(ret, { silent: true });
 		if (shareableUrl) {
 			ret.shareableUrl = shareableUrl;
@@ -15061,9 +15418,9 @@ async function evaluate(testSuite, options = {}) {
 		require_logger.logger.warn(`Failed to create shareable URL: ${error}`);
 	}
 	else require_logger.logger.debug("Sharing requested but not enabled (check cloud config or sharing settings)");
-	if (testSuite.outputPath) {
-		if (typeof testSuite.outputPath === "string") await require_util.writeOutput(testSuite.outputPath, evalRecord, null);
-		else if (Array.isArray(testSuite.outputPath)) await require_util.writeMultipleOutputs(testSuite.outputPath, evalRecord, null);
+	if (testSuiteConfig.outputPath) {
+		if (typeof testSuiteConfig.outputPath === "string") await require_util.writeOutput(testSuiteConfig.outputPath, evalRecord, null);
+		else if (Array.isArray(testSuiteConfig.outputPath)) await require_util.writeMultipleOutputs(testSuiteConfig.outputPath, evalRecord, null);
 	}
 	return ret;
 }
@@ -15075,7 +15432,7 @@ const redteam = {
 	},
 	Graders: require_graders.GRADERS,
 	Plugins,
-	Strategies: require_providers.Strategies,
+	Strategies: require_strategies.Strategies,
 	Base: {
 		Plugin: require_graders.RedteamPluginBase,
 		Grader: require_graders.RedteamGraderBase
@@ -15104,9 +15461,18 @@ exports.CompletedPromptSchema = require_types.CompletedPromptSchema;
 exports.CompletionTokenDetailsSchema = require_types.CompletionTokenDetailsSchema;
 exports.ConversationMessageSchema = require_types.ConversationMessageSchema;
 exports.DerivedMetricSchema = require_types.DerivedMetricSchema;
+exports.DocumentMediaInjectionPlacementSchema = require_types.DocumentMediaInjectionPlacementSchema;
+exports.DocumentMediaInjectionPlacementValues = require_types.DocumentMediaInjectionPlacementValues;
+exports.DocxInjectionPlacementSchema = require_types.DocxInjectionPlacementSchema;
+exports.DocxInjectionPlacementValues = require_types.DocxInjectionPlacementValues;
 exports.EvalResultsFilterMode = require_types.EvalResultsFilterMode;
 exports.EvaluateOptionsSchema = require_types.EvaluateOptionsSchema;
 exports.GradingConfigSchema = require_types.GradingConfigSchema;
+exports.InputConfigSchema = require_types.InputConfigSchema;
+exports.InputDefinitionObjectSchema = require_types.InputDefinitionObjectSchema;
+exports.InputDefinitionSchema = require_types.InputDefinitionSchema;
+exports.InputTypeSchema = require_types.InputTypeSchema;
+exports.InputTypeValues = require_types.InputTypeValues;
 exports.InputsSchema = require_types.InputsSchema;
 exports.NotPrefixedAssertionTypesSchema = require_types.NotPrefixedAssertionTypesSchema;
 exports.OutputConfigSchema = require_types.OutputConfigSchema;
@@ -15129,6 +15495,7 @@ exports.TestSuiteSchema = require_types.TestSuiteSchema;
 exports.UnifiedConfigSchema = require_types.UnifiedConfigSchema;
 exports.VarsSchema = require_types.VarsSchema;
 exports.assertions = assertions_default;
+exports.buildInputPromptDescription = require_types.buildInputPromptDescription;
 Object.defineProperty(exports, "cache", {
 	enumerable: true,
 	get: function() {
@@ -15138,12 +15505,17 @@ Object.defineProperty(exports, "cache", {
 exports.default = src_default;
 exports.evaluate = evaluate;
 exports.generateTable = generateTable;
+exports.getInputDescription = require_types.getInputDescription;
+exports.getInputType = require_types.getInputType;
 exports.guardrails = guardrails;
 exports.isApiProvider = require_types.isApiProvider;
 exports.isGradingResult = require_types.isGradingResult;
 exports.isProviderOptions = require_types.isProviderOptions;
 exports.isResultFailureReason = require_types.isResultFailureReason;
+exports.isTransformFunction = isTransformFunction;
 exports.loadApiProvider = require_providers.loadApiProvider;
+exports.normalizeInputDefinition = require_types.normalizeInputDefinition;
+exports.normalizeInputs = require_types.normalizeInputs;
 exports.redteam = redteam;
 
 //# sourceMappingURL=index.cjs.map