promptfoo 0.121.4 → 0.121.7

package/dist/src/index.cjs

@@ -2,77 +2,93 @@ Object.defineProperties(exports, {
 	__esModule: { value: true },
 	[Symbol.toStringTag]: { value: "Module" }
 });
-const require_logger = require("./logger-COuQb2xB.cjs");
-const require_invariant = require("./invariant-kfQ8Bu82.cjs");
-const require_esm = require("./esm-CtEPLdAj.cjs");
-const require_pythonUtils = require("./pythonUtils-tAJvvpS-.cjs");
-const require_fileExtensions = require("./fileExtensions-bYh77CN8.cjs");
-const require_transform = require("./transform-CUnzlsbn.cjs");
-const require_graders = require("./graders--zknU_uk.cjs");
-const require_types = require("./types-CeaeaZdP.cjs");
-const require_util = require("./util-Bxn8emtE.cjs");
-const require_render = require("./render-DfQSFxGE.cjs");
-const require_fetch = require("./fetch-CJU5ELPa.cjs");
-const require_cache = require("./cache-BIyPcp5v.cjs");
-const require_providers = require("./providers-1eKkXBKp.cjs");
-const require_utils = require("./utils-B05gLxER.cjs");
-const require_createHash = require("./createHash-B7KvgoOD.cjs");
-const require_chat = require("./chat-CclRbxGf.cjs");
-const require_tokenUsageUtils = require("./tokenUsageUtils-Bb7DkZPz.cjs");
-const require_transform$1 = require("./transform-CTeuTR3S.cjs");
-const require_util$1 = require("./util-CN3SrLT4.cjs");
-const require_providerRegistry = require("./providerRegistry-CQMdTmHP.cjs");
-const require_server = require("./server-Dx2TyCH2.cjs");
-const require_accounts = require("./accounts-Dy17bs4D.cjs");
-const require_blobs = require("./blobs-D2FAd1Q5.cjs");
-const require_tables = require("./tables-DUfh1F7Z.cjs");
-const require_extractor = require("./extractor-WVPOrH43.cjs");
-const require_telemetry = require("./telemetry-Dsw_faFj.cjs");
-const require_text = require("./text-BVi-cLPJ.cjs");
-const require_store = require("./store-DAAyxcy6.cjs");
-const require_rubyUtils = require("./rubyUtils-B6eljPuh.cjs");
-const require_evalResult = require("./evalResult-KZqXl4XP.cjs");
+const require_rolldown_runtime = require("./rolldown-runtime-D_mwlA32.cjs");
+const require_logger = require("./logger-cfNpzI4o.cjs");
+const require_invariant = require("./invariant-QtnLD03y.cjs");
+const require_types = require("./types-CxJvaY2S.cjs");
+const require_fetch = require("./fetch-Dw4XZHjj.cjs");
+const require_version = require("./version-BWCSaByA.cjs");
+const require_fileExtensions = require("./fileExtensions-BhdwzYaD.cjs");
+const require_accounts = require("./accounts-CmWzeD2d.cjs");
+const require_esm = require("./esm-BIKakvNa.cjs");
+const require_render = require("./render-BNTrbmBw.cjs");
+const require_providerRegistry = require("./providerRegistry-CZO_w7ue.cjs");
+const require_remoteGeneration = require("./remoteGeneration-DS9N3pgB.cjs");
+const require_server = require("./server-B8rqV126.cjs");
+const require_storage = require("./storage-CA-v9V2v.cjs");
+const require_pythonUtils = require("./pythonUtils-Cokhluq3.cjs");
+const require_util = require("./util-SPsvFONY.cjs");
+const require_createHash = require("./createHash-CSiqnK5P.cjs");
+const require_cache = require("./cache-CPGUA4Yl.cjs");
+const require_chat = require("./chat-DxTDQ83C.cjs");
+const require_transform = require("./transform-DhNkAUs8.cjs");
+const require_util$1 = require("./util-BSIuSLVK.cjs");
+const require_providers = require("./providers-BDVVIQM6.cjs");
+const require_tokenUsageUtils = require("./tokenUsageUtils-_B-P8IAi.cjs");
+const require_text = require("./text-nywWsRBM.cjs");
+const require_telemetry = require("./telemetry-CJ7FnCsc.cjs");
+const require_tables = require("./tables-DBIJU0WE.cjs");
+const require_blobs = require("./blobs-B0977K1O.cjs");
+const require_processShim = require("./processShim-DSY9BV2T.cjs");
+const require_packageParser = require("./packageParser-QoCS1FMl.cjs");
+const require_rubyUtils = require("./rubyUtils-CqUWBZAt.cjs");
+const require_inputVariables = require("./inputVariables-Dq9W-Z3a.cjs");
+const require_util$2 = require("./util-CN8om2rz.cjs");
+const require_promptfoo = require("./promptfoo-b-baRMj-.cjs");
+const require_graders = require("./graders-ClrU2fnd.cjs");
+const require_shared = require("./shared-WkgnDkcg.cjs");
+const require_utils = require("./utils-Ve6kuJsa.cjs");
+const require_transform$1 = require("./transform-BnSXWmU_2.cjs");
+const require_store = require("./store-B2NDDooM.cjs");
+const require_extractor = require("./extractor-BdxEtt3J.cjs");
+const require_promptLength = require("./promptLength-BbBbDHNj.cjs");
+const require_indirectWebPwn = require("./indirectWebPwn-BJ22AbQa.cjs");
+const require_toolAttributes = require("./toolAttributes-BAjwcBf0.cjs");
+const require_evaluatorHelpers = require("./evaluatorHelpers-CRqTvSux.cjs");
+const require_evalResult = require("./evalResult-CvtS8h8u.cjs");
+const require_strategies = require("./strategies-7CS3Alao.cjs");
 let fs = require("fs");
-fs = require_logger.__toESM(fs);
+fs = require_rolldown_runtime.__toESM(fs, 1);
 let path = require("path");
-path = require_logger.__toESM(path);
+path = require_rolldown_runtime.__toESM(path, 1);
 let async = require("async");
-async = require_logger.__toESM(async);
+async = require_rolldown_runtime.__toESM(async, 1);
 let js_yaml = require("js-yaml");
-js_yaml = require_logger.__toESM(js_yaml);
+js_yaml = require_rolldown_runtime.__toESM(js_yaml, 1);
 let node_async_hooks = require("node:async_hooks");
-require("node:path");
-require("node:url");
 let chalk = require("chalk");
-chalk = require_logger.__toESM(chalk);
+chalk = require_rolldown_runtime.__toESM(chalk, 1);
 let os = require("os");
-os = require_logger.__toESM(os);
-let util = require("util");
-util = require_logger.__toESM(util);
+os = require_rolldown_runtime.__toESM(os, 1);
 let dedent = require("dedent");
-dedent = require_logger.__toESM(dedent);
-let fs_promises = require("fs/promises");
-fs_promises = require_logger.__toESM(fs_promises);
-let glob = require("glob");
+dedent = require_rolldown_runtime.__toESM(dedent, 1);
 let zod = require("zod");
-let csv_parse_sync = require("csv-parse/sync");
-let fast_xml_parser = require("fast-xml-parser");
+let fs_promises = require("fs/promises");
+fs_promises = require_rolldown_runtime.__toESM(fs_promises, 1);
+let util = require("util");
+util = require_rolldown_runtime.__toESM(util, 1);
+let _inquirer_input = require("@inquirer/input");
+_inquirer_input = require_rolldown_runtime.__toESM(_inquirer_input, 1);
+require("node:path");
+require("node:url");
 let crypto$1 = require("crypto");
-crypto$1 = require_logger.__toESM(crypto$1);
+crypto$1 = require_rolldown_runtime.__toESM(crypto$1, 1);
 let _opentelemetry_api = require("@opentelemetry/api");
-let _inquirer_input = require("@inquirer/input");
-_inquirer_input = require_logger.__toESM(_inquirer_input);
 let readline = require("readline");
-readline = require_logger.__toESM(readline);
+readline = require_rolldown_runtime.__toESM(readline, 1);
+let csv_parse_sync = require("csv-parse/sync");
+let glob = require("glob");
+let fast_xml_parser = require("fast-xml-parser");
 let drizzle_orm = require("drizzle-orm");
-let cli_progress = require("cli-progress");
-cli_progress = require_logger.__toESM(cli_progress);
 let url = require("url");
-let jsdom = require("jsdom");
+let parse5 = require("parse5");
 let fastest_levenshtein = require("fastest-levenshtein");
+let cli_progress = require("cli-progress");
+cli_progress = require_rolldown_runtime.__toESM(cli_progress, 1);
 let js_rouge = require("js-rouge");
-js_rouge = require_logger.__toESM(js_rouge);
+js_rouge = require_rolldown_runtime.__toESM(js_rouge, 1);
 let node_util = require("node:util");
+let lru_cache = require("lru-cache");
 require("debounce");
 let _opentelemetry_core = require("@opentelemetry/core");
 let _opentelemetry_exporter_trace_otlp_http = require("@opentelemetry/exporter-trace-otlp-http");
@@ -81,15 +97,15 @@ let _opentelemetry_sdk_trace_node = require("@opentelemetry/sdk-trace-node");
 let _opentelemetry_semantic_conventions = require("@opentelemetry/semantic-conventions");
 let drizzle_orm_better_sqlite3_migrator = require("drizzle-orm/better-sqlite3/migrator");
 let process$1 = require("process");
-process$1 = require_logger.__toESM(process$1);
+process$1 = require_rolldown_runtime.__toESM(process$1, 1);
 let _apidevtools_json_schema_ref_parser = require("@apidevtools/json-schema-ref-parser");
-_apidevtools_json_schema_ref_parser = require_logger.__toESM(_apidevtools_json_schema_ref_parser);
+_apidevtools_json_schema_ref_parser = require_rolldown_runtime.__toESM(_apidevtools_json_schema_ref_parser, 1);
 let cli_table3 = require("cli-table3");
-cli_table3 = require_logger.__toESM(cli_table3);
+cli_table3 = require_rolldown_runtime.__toESM(cli_table3, 1);
 let chokidar = require("chokidar");
-chokidar = require_logger.__toESM(chokidar);
+chokidar = require_rolldown_runtime.__toESM(chokidar, 1);
 let ora = require("ora");
-ora = require_logger.__toESM(ora);
+ora = require_rolldown_runtime.__toESM(ora, 1);
 require("@inquirer/confirm");
 //#region src/external/matchers/conversationRelevancyTemplate.ts
 var ConversationRelevancyTemplate = class {
@@ -262,6 +278,505 @@ const handleConversationRelevance = async ({ assertion, outputString, prompt, pr
 	};
 };
 //#endregion
+//#region src/matchers/classification.ts
+/**
+*
+* @param expected Expected classification. If undefined, matches any classification.
+* @param output Text to classify.
+* @param threshold Value between 0 and 1. If the expected classification is undefined, the threshold is the minimum score for any classification. If the expected classification is defined, the threshold is the minimum score for that classification.
+* @param grading
+* @returns Pass if the output matches the classification with a score greater than or equal to the threshold.
+*/
+async function matchesClassification(expected, output, threshold, grading) {
+	const resp = await (await require_graders.getAndCheckProvider("classification", grading?.provider, null, "classification check")).callClassificationApi(output);
+	if (!resp.classification) return require_graders.fail(resp.error || "Unknown error fetching classification");
+	let score;
+	if (expected === void 0) {
+		const scores = Object.values(resp.classification);
+		if (scores.length === 0) return {
+			pass: false,
+			score: 0,
+			reason: "No classification scores returned"
+		};
+		score = Math.max(...scores);
+	} else score = resp.classification[expected] || 0;
+	if (score >= threshold - Number.EPSILON) {
+		const reason = expected === void 0 ? `Maximum classification score ${score.toFixed(2)} >= ${threshold}` : `Classification ${expected} has score ${score.toFixed(2)} >= ${threshold}`;
+		return {
+			pass: true,
+			score,
+			reason
+		};
+	}
+	return {
+		pass: false,
+		score,
+		reason: expected === void 0 ? `Maximum classification score ${score.toFixed(2)} < ${threshold}` : `Classification ${expected} has score ${score.toFixed(2)} < ${threshold}`
+	};
+}
+//#endregion
+//#region src/matchers/comparison.ts
+async function matchesSelectBest(criteria, outputs, grading, vars, providerCallContext) {
+	require_invariant.invariant(outputs.length >= 2, "select-best assertion must have at least two outputs to compare between");
+	const resp = await require_graders.callProviderWithContext(await require_graders.getAndCheckProvider("text", grading?.provider, (await require_graders.getDefaultProviders()).gradingProvider, "select-best check"), await require_graders.renderLlmRubricPrompt(await require_graders.loadRubricPrompt(grading?.rubricPrompt, require_graders.SELECT_BEST_PROMPT), {
+		criteria,
+		outputs: outputs.map((o) => require_graders.tryParse(o)),
+		...vars || {}
+	}), "select-best", {
+		criteria,
+		outputs: outputs.map((o) => require_graders.tryParse(o)),
+		...vars || {}
+	}, providerCallContext);
+	if (resp.error || !resp.output) return Array.from({ length: outputs.length }, () => require_graders.fail(resp.error || "No output", resp.tokenUsage));
+	require_invariant.invariant(typeof resp.output === "string", "select-best produced malformed response");
+	const firstIntegerMatch = resp.output.trim().match(/\d+/);
+	const verdict = firstIntegerMatch ? Number.parseInt(firstIntegerMatch[0], 10) : NaN;
+	if (Number.isNaN(verdict) || verdict < 0 || verdict >= outputs.length) return Array.from({ length: outputs.length }, () => require_graders.fail(`Invalid select-best verdict: ${verdict}`, resp.tokenUsage));
+	const tokensUsed = require_graders.normalizeMatcherTokenUsage(resp.tokenUsage);
+	return outputs.map((_output, index) => {
+		if (index === verdict) return {
+			pass: true,
+			score: 1,
+			reason: `Output selected as the best: ${criteria}`,
+			tokensUsed
+		};
+		else return {
+			pass: false,
+			score: 0,
+			reason: `Output not selected: ${criteria}`,
+			tokensUsed
+		};
+	});
+}
+async function selectMaxScore(outputs, resultsWithGradingResults, assertion) {
+	require_invariant.invariant(outputs.length >= 2, "max-score assertion must have at least two outputs to compare between");
+	const value = assertion.value || {};
+	const options = {
+		method: typeof value === "object" && "method" in value ? value.method : "average",
+		weights: typeof value === "object" && "weights" in value ? value.weights : {},
+		threshold: typeof value === "object" && "threshold" in value ? value.threshold : void 0
+	};
+	const scores = resultsWithGradingResults.map((result, index) => {
+		const relevantResults = (result.gradingResult?.componentResults || []).filter((r) => r.assertion && r.assertion.type !== "max-score" && r.assertion.type !== "select-best");
+		if (relevantResults.length === 0) throw new Error("max-score requires at least one other assertion (besides max-score or select-best) to aggregate scores from");
+		let totalWeightedScore = 0;
+		let totalWeight = 0;
+		relevantResults.forEach((componentResult) => {
+			const assertionType = componentResult.assertion?.type || "unknown";
+			const weight = options.weights[assertionType] === void 0 ? 1 : options.weights[assertionType];
+			const score = componentResult.score || 0;
+			totalWeightedScore += score * weight;
+			totalWeight += weight;
+		});
+		let aggregateScore;
+		if (options.method === "sum") aggregateScore = totalWeightedScore;
+		else aggregateScore = totalWeight > 0 ? totalWeightedScore / totalWeight : 0;
+		return {
+			index,
+			score: aggregateScore,
+			componentCount: relevantResults.length,
+			totalWeight
+		};
+	});
+	let maxScore = -Infinity;
+	let winnerIndex = 0;
+	for (let i = 0; i < scores.length; i++) if (scores[i].score > maxScore) {
+		maxScore = scores[i].score;
+		winnerIndex = i;
+	}
+	const meetsThreshold = options.threshold === void 0 || maxScore >= options.threshold;
+	return scores.map(({ index, score, componentCount, totalWeight }) => {
+		const isWinner = index === winnerIndex && meetsThreshold;
+		return {
+			pass: isWinner,
+			score: isWinner ? 1 : 0,
+			reason: isWinner ? `Selected as highest scoring output (score: ${score.toFixed(3)})` : score === maxScore && !meetsThreshold ? `Not selected - score ${score.toFixed(3)} below threshold ${options.threshold}` : `Not selected (score: ${score.toFixed(3)}, max: ${maxScore.toFixed(3)})`,
+			namedScores: {
+				maxScore: score,
+				assertionCount: componentCount,
+				totalWeight
+			}
+		};
+	});
+}
+//#endregion
+//#region src/matchers/moderation.ts
+async function matchesModeration({ userPrompt, assistantResponse, categories = [] }, grading) {
+	if (!assistantResponse) return {
+		pass: true,
+		score: 1,
+		reason: "No output to moderate"
+	};
+	const defaultProviders = await require_graders.getDefaultProviders();
+	const defaultModerationProvider = !require_logger.getEnvString("OPENAI_API_KEY") && (require_logger.getEnvString("REPLICATE_API_KEY") || require_logger.getEnvString("REPLICATE_API_TOKEN")) ? await require_providers.loadApiProvider(require_types.LLAMA_GUARD_REPLICATE_PROVIDER) : defaultProviders.moderationProvider;
+	const moderationProvider = await require_graders.getAndCheckProvider("moderation", grading?.provider, defaultModerationProvider, "moderation check");
+	require_invariant.invariant(moderationProvider, "Moderation provider must be defined");
+	const resp = await moderationProvider.callModerationApi(userPrompt, assistantResponse);
+	if (resp.error) return {
+		pass: false,
+		score: 0,
+		reason: `Moderation API error: ${resp.error}`
+	};
+	const { flags } = resp;
+	if (!flags || flags.length === 0) return {
+		pass: true,
+		score: 1,
+		reason: "No moderation flags detected"
+	};
+	const filteredFlags = categories.length === 0 ? flags : flags.filter((flag) => categories.includes(flag.code));
+	if (filteredFlags.length > 0) return {
+		pass: false,
+		score: 0,
+		reason: `Moderation flags detected: ${filteredFlags.map((flag) => flag.description).join(", ")}`
+	};
+	return {
+		pass: true,
+		score: 1,
+		reason: "No relevant moderation flags detected"
+	};
+}
+//#endregion
+//#region src/assertions/contextUtils.ts
+/**
+* Resolves the context value for context-based assertions.
+* Supports extracting context from test variables or transforming from output.
+* Can return either a single context string or an array of context chunks.
+*
+* @param assertion - The assertion configuration
+* @param test - The test case
+* @param output - The provider output (after provider transform, before test transform)
+* @param prompt - The prompt text
+* @param fallbackContext - Optional fallback context (e.g., prompt for context-recall)
+* @param providerResponse - Optional full provider response for contextTransform
+* @returns The resolved context string or array of strings
+* @throws Error if context cannot be resolved or transform fails
+*/
+async function resolveContext(assertion, test, output, prompt, fallbackContext, providerResponse) {
+	let contextValue;
+	if (test.vars?.context) {
+		if (typeof test.vars.context === "string") contextValue = test.vars.context;
+		else if (Array.isArray(test.vars.context)) {
+			const invalidEntry = [...test.vars.context.entries()].find(([, v]) => typeof v !== "string");
+			if (invalidEntry) {
+				const [idx, val] = invalidEntry;
+				require_invariant.invariant(false, `Invalid context: expected an array of strings, but found ${typeof val} at index ${idx}`);
+			}
+			contextValue = test.vars.context;
+		}
+	} else if (fallbackContext) contextValue = fallbackContext;
+	if (assertion.contextTransform) {
+		const getLabel = () => require_transform$1.getTransformLabel(assertion.contextTransform);
+		try {
+			const outputForTransform = providerResponse?.providerTransformedOutput ?? output;
+			const transformed = await require_transform$1.transform(assertion.contextTransform, outputForTransform, {
+				vars: test.vars,
+				prompt: { label: prompt },
+				...providerResponse && providerResponse.metadata && { metadata: providerResponse.metadata }
+			});
+			require_invariant.invariant(typeof transformed === "string" || Array.isArray(transformed) && transformed.every((item) => typeof item === "string"), () => `contextTransform must return a string or array of strings. Got ${typeof transformed}. Check your transform expression: ${getLabel()}`);
+			contextValue = transformed;
+		} catch (error) {
+			throw new Error(`Failed to transform context using expression '${getLabel()}': ${require_transform$1.getTransformErrorMessage(error)}`);
+		}
+	}
+	require_invariant.invariant(typeof contextValue === "string" && contextValue.length > 0 || Array.isArray(contextValue) && contextValue.length > 0 && contextValue.every((item) => typeof item === "string" && item.length > 0), "Context is required for context-based assertions. Provide either a \"context\" variable (string or array of strings) in your test case or use \"contextTransform\" to extract context from the provider response.");
+	return contextValue;
+}
+/**
+* Serializes context (string or string[]) to a single string for prompts.
+* Joins chunks with double newlines to preserve separation.
+*/
+function serializeContext(context) {
+	return Array.isArray(context) ? context.join("\n\n") : context;
+}
+//#endregion
+//#region src/matchers/rag.ts
+async function matchesAnswerRelevance(input, output, threshold, grading, providerCallContext) {
+	const defaults = await require_graders.getDefaultProviders();
+	const embeddingProvider = await require_graders.getAndCheckProvider("embedding", grading?.provider, defaults.embeddingProvider, "answer relevancy check");
+	const textProvider = await require_graders.getAndCheckProvider("text", grading?.provider, defaults.gradingProvider, "answer relevancy check");
+	const tokensUsed = require_graders.normalizeMatcherTokenUsage(void 0);
+	const rubricPrompt = await require_graders.loadRubricPrompt(grading?.rubricPrompt, require_graders.ANSWER_RELEVANCY_GENERATE);
+	const parsedOutput = require_graders.tryParse(output);
+	const promptText = await require_graders.renderLlmRubricPrompt(rubricPrompt, { answer: parsedOutput });
+	const candidateQuestions = [];
+	for (let i = 0; i < 3; i++) {
+		const resp = await require_graders.callProviderWithContext(textProvider, promptText, "answer-relevance", { answer: parsedOutput }, providerCallContext);
+		require_tokenUsageUtils.accumulateTokenUsage(tokensUsed, resp.tokenUsage);
+		if (resp.error || !resp.output) return require_graders.fail(resp.error || "No output", tokensUsed);
+		require_invariant.invariant(typeof resp.output === "string", "answer relevancy check produced malformed response");
+		candidateQuestions.push(resp.output);
+	}
+	require_invariant.invariant(typeof embeddingProvider.callEmbeddingApi === "function", `Provider ${embeddingProvider.id()} must implement callEmbeddingApi for similarity check`);
+	const inputEmbeddingResp = await embeddingProvider.callEmbeddingApi(input);
+	require_tokenUsageUtils.accumulateTokenUsage(tokensUsed, inputEmbeddingResp.tokenUsage);
+	if (inputEmbeddingResp.error || !inputEmbeddingResp.embedding) return require_graders.fail(inputEmbeddingResp.error || "No embedding", tokensUsed);
+	const inputEmbedding = inputEmbeddingResp.embedding;
+	const similarities = [];
+	const questionsWithScores = [];
+	for (const question of candidateQuestions) {
+		const resp = await embeddingProvider.callEmbeddingApi(question);
+		require_tokenUsageUtils.accumulateTokenUsage(tokensUsed, resp.tokenUsage);
+		if (resp.error || !resp.embedding) return require_graders.fail(resp.error || "No embedding", tokensUsed);
+		const questionSimilarity = require_graders.cosineSimilarity(inputEmbedding, resp.embedding);
+		similarities.push(questionSimilarity);
+		questionsWithScores.push({
+			question,
+			similarity: questionSimilarity
+		});
+	}
+	const similarity = similarities.reduce((a, b) => a + b, 0) / similarities.length;
+	const pass = similarity >= threshold - Number.EPSILON;
+	const greaterThanReason = `Relevance ${similarity.toFixed(2)} is greater than threshold ${threshold}`;
+	const lessThanReason = `Relevance ${similarity.toFixed(2)} is less than threshold ${threshold}`;
+	const metadata = {
+		generatedQuestions: questionsWithScores,
+		averageSimilarity: similarity,
+		threshold
+	};
+	if (pass) return {
+		pass: true,
+		score: similarity,
+		reason: greaterThanReason,
+		tokensUsed,
+		metadata
+	};
+	return {
+		pass: false,
+		score: similarity,
+		reason: lessThanReason,
+		tokensUsed,
+		metadata
+	};
+}
+async function matchesContextRecall(context, groundTruth, threshold, grading, vars, providerCallContext) {
+	const textProvider = await require_graders.getAndCheckProvider("text", grading?.provider, (await require_graders.getDefaultProviders()).gradingProvider, "context recall check");
+	const contextString = serializeContext(context);
+	const resp = await require_graders.callProviderWithContext(textProvider, await require_graders.renderLlmRubricPrompt(await require_graders.loadRubricPrompt(grading?.rubricPrompt, require_graders.CONTEXT_RECALL), {
+		context: contextString,
+		groundTruth,
+		...vars || {}
+	}), "context-recall", {
+		context: contextString,
+		groundTruth,
+		...vars || {}
+	}, providerCallContext);
+	if (resp.error || !resp.output) return require_graders.fail(resp.error || "No output", resp.tokenUsage);
+	require_invariant.invariant(typeof resp.output === "string", "context-recall produced malformed response");
+	const attributedTokenLower = require_graders.CONTEXT_RECALL_ATTRIBUTED_TOKEN.toLowerCase();
+	const notAttributedTokenLower = require_graders.CONTEXT_RECALL_NOT_ATTRIBUTED_TOKEN.toLowerCase();
+	const sentences = require_graders.splitIntoSentences(resp.output).filter((line) => {
+		const lowerLine = line.toLowerCase();
+		return lowerLine.includes(attributedTokenLower) || lowerLine.includes(notAttributedTokenLower);
+	});
+	const sentenceAttributions = [];
+	let numerator = 0;
+	for (const sentence of sentences) {
+		const lowerSentence = sentence.toLowerCase();
+		const isAttributed = !lowerSentence.includes(notAttributedTokenLower) && lowerSentence.includes(attributedTokenLower);
+		if (isAttributed) numerator++;
+		const sentenceMatch = sentence.match(/^\d+\.\s*([^\.]+\.)/);
+		const cleanSentence = sentenceMatch ? sentenceMatch[1].trim() : sentence.split(".")[0].trim();
+		sentenceAttributions.push({
+			sentence: cleanSentence,
+			attributed: isAttributed
+		});
+	}
+	const score = sentences.length > 0 ? numerator / sentences.length : 0;
+	const pass = score >= threshold - Number.EPSILON;
+	const metadata = {
+		sentenceAttributions,
+		totalSentences: sentences.length,
+		attributedSentences: numerator,
+		score
+	};
+	return {
+		pass,
+		score,
+		reason: pass ? `Recall ${score.toFixed(2)} is >= ${threshold}` : `Recall ${score.toFixed(2)} is < ${threshold}`,
+		tokensUsed: require_graders.normalizeMatcherTokenUsage(resp.tokenUsage),
+		metadata
+	};
+}
+async function matchesContextRelevance(question, context, threshold, grading, providerCallContext) {
+	const textProvider = await require_graders.getAndCheckProvider("text", grading?.provider, (await require_graders.getDefaultProviders()).gradingProvider, "context relevance check");
+	const contextString = serializeContext(context);
+	const resp = await require_graders.callProviderWithContext(textProvider, await require_graders.renderLlmRubricPrompt(await require_graders.loadRubricPrompt(grading?.rubricPrompt, require_graders.CONTEXT_RELEVANCE), {
+		context: contextString,
+		query: question
+	}), "context-relevance", {
+		context: contextString,
+		query: question
+	}, providerCallContext);
+	if (resp.error || !resp.output) return require_graders.fail(resp.error || "No output", resp.tokenUsage);
+	require_invariant.invariant(typeof resp.output === "string", "context-relevance produced malformed response");
+	const contextUnits = Array.isArray(context) ? context.filter((chunk) => chunk.trim().length > 0) : require_graders.splitIntoSentences(context);
+	const totalContextUnits = contextUnits.length;
+	const extractedSentences = require_graders.splitIntoSentences(resp.output);
+	const relevantSentences = [];
+	const insufficientInformation = resp.output.includes(require_graders.CONTEXT_RELEVANCE_BAD);
+	let numerator = 0;
+	if (insufficientInformation) numerator = 0;
+	else {
+		const uniqueRelevantSentences = [...new Set(extractedSentences)];
+		numerator = Math.min(uniqueRelevantSentences.length, totalContextUnits);
+		relevantSentences.push(...uniqueRelevantSentences);
+	}
+	const score = totalContextUnits > 0 ? numerator / totalContextUnits : 0;
+	const pass = score >= threshold - Number.EPSILON;
+	const metadata = {
+		extractedSentences: relevantSentences,
+		totalContextUnits,
+		totalContextSentences: totalContextUnits,
+		contextUnits,
+		relevantSentenceCount: numerator,
+		insufficientInformation,
+		score
+	};
+	return {
+		pass,
+		score,
+		reason: pass ? `Context relevance ${score.toFixed(2)} is >= ${threshold}` : `Context relevance ${score.toFixed(2)} is < ${threshold}`,
+		tokensUsed: require_graders.normalizeMatcherTokenUsage(resp.tokenUsage),
+		metadata
+	};
+}
+async function matchesContextFaithfulness(query, output, context, threshold, grading, vars, providerCallContext) {
+	const textProvider = await require_graders.getAndCheckProvider("text", grading?.provider, (await require_graders.getDefaultProviders()).gradingProvider, "faithfulness check");
+	const tokensUsed = require_graders.normalizeMatcherTokenUsage(void 0);
+	if (grading?.rubricPrompt) require_invariant.invariant(Array.isArray(grading.rubricPrompt), "rubricPrompt must be an array");
+	const rawLongformPrompt = typeof grading?.rubricPrompt?.[0] === "string" ? grading?.rubricPrompt?.[0] : grading?.rubricPrompt?.[0]?.content;
+	const rawNliPrompt = typeof grading?.rubricPrompt?.[1] === "string" ? grading?.rubricPrompt?.[1] : grading?.rubricPrompt?.[1]?.content;
+	const longformPrompt = await require_graders.loadRubricPrompt(rawLongformPrompt, require_graders.CONTEXT_FAITHFULNESS_LONGFORM);
+	const nliPrompt = await require_graders.loadRubricPrompt(rawNliPrompt, require_graders.CONTEXT_FAITHFULNESS_NLI_STATEMENTS);
+	let promptText = await require_graders.renderLlmRubricPrompt(longformPrompt, {
+		question: query,
+		answer: require_graders.tryParse(output),
+		...vars || {}
+	});
+	let resp = await require_graders.callProviderWithContext(textProvider, promptText, "context-faithfulness-longform", {
+		question: query,
+		answer: require_graders.tryParse(output),
+		...vars || {}
+	}, providerCallContext);
+	require_tokenUsageUtils.accumulateTokenUsage(tokensUsed, resp.tokenUsage);
+	if (resp.error || !resp.output) return require_graders.fail(resp.error || "No output", tokensUsed);
+	require_invariant.invariant(typeof resp.output === "string", "context-faithfulness produced malformed response");
+	const contextString = serializeContext(context);
+	const statements = require_graders.splitIntoSentences(resp.output);
+	promptText = await require_graders.renderLlmRubricPrompt(nliPrompt, {
+		context: contextString,
+		statements,
+		...vars || {}
+	});
+	resp = await require_graders.callProviderWithContext(textProvider, promptText, "context-faithfulness-nli", {
+		context: contextString,
+		statements,
+		...vars || {}
+	}, providerCallContext);
+	require_tokenUsageUtils.accumulateTokenUsage(tokensUsed, resp.tokenUsage);
+	if (resp.error || !resp.output) return require_graders.fail(resp.error || "No output", tokensUsed);
+	require_invariant.invariant(typeof resp.output === "string", "context-faithfulness produced malformed response");
+	let finalAnswer = "Final verdict for each statement in order:";
+	finalAnswer = finalAnswer.toLowerCase();
+	let verdicts = resp.output.toLowerCase().trim();
+	let score = 0;
+	if (statements.length > 0) if (verdicts.includes(finalAnswer)) {
+		verdicts = verdicts.slice(verdicts.indexOf(finalAnswer) + finalAnswer.length);
+		const parsedVerdicts = verdicts.split(".").filter((answer) => answer.trim() !== "");
+		if (parsedVerdicts.length > 0) score = 1 - parsedVerdicts.filter((answer) => !answer.includes("yes")).length / statements.length;
+	} else {
+		const noVerdictCount = verdicts.split("verdict: no").length - 1;
+		if (noVerdictCount + (verdicts.split("verdict: yes").length - 1) > 0) score = 1 - noVerdictCount / statements.length;
+	}
+	score = Math.min(1, Math.max(0, score));
+	const pass = score >= threshold - Number.EPSILON;
+	return {
+		pass,
+		score,
+		reason: pass ? `Faithfulness ${score.toFixed(2)} is >= ${threshold}` : `Faithfulness ${score.toFixed(2)} is < ${threshold}`,
+		tokensUsed
+	};
+}
+//#endregion
+//#region src/matchers/similarity.ts
+function calculateSimilarityScore(expectedEmbedding, outputEmbedding, metric, tokensUsed) {
+	switch (metric) {
+		case "cosine": return require_graders.cosineSimilarity(expectedEmbedding, outputEmbedding);
+		case "dot_product": return require_graders.dotProduct(expectedEmbedding, outputEmbedding);
+		case "euclidean": return require_graders.euclideanDistance(expectedEmbedding, outputEmbedding);
+		default: return require_graders.fail(`Unsupported metric: ${metric}`, tokensUsed);
+	}
+}
+function buildSimilarityResult(similarity, threshold, inverse, metric, tokensUsed) {
+	if (metric === "euclidean") {
+		const distance = similarity;
+		const pass = inverse ? distance >= threshold - Number.EPSILON : distance <= threshold + Number.EPSILON;
+		const normalizedScore = 1 / (1 + distance);
+		const score = inverse ? 1 - normalizedScore : normalizedScore;
+		const belowThresholdReason = `Distance ${distance.toFixed(2)} is less than or equal to threshold ${threshold}`;
+		const aboveThresholdReason = `Distance ${distance.toFixed(2)} is greater than threshold ${threshold}`;
+		return {
+			pass,
+			score,
+			reason: pass ? inverse ? aboveThresholdReason : belowThresholdReason : inverse ? belowThresholdReason : aboveThresholdReason,
+			tokensUsed
+		};
+	}
+	const pass = inverse ? similarity <= threshold + Number.EPSILON : similarity >= threshold - Number.EPSILON;
+	const score = inverse ? 1 - similarity : similarity;
+	const greaterThanReason = `Similarity ${similarity.toFixed(2)} is greater than or equal to threshold ${threshold}`;
+	const lessThanReason = `Similarity ${similarity.toFixed(2)} is less than threshold ${threshold}`;
+	return {
+		pass,
+		score,
+		reason: pass ? inverse ? lessThanReason : greaterThanReason : inverse ? greaterThanReason : lessThanReason,
+		tokensUsed
+	};
+}
+async function calculateProviderSimilarity(finalProvider, expected, output, metric, tokensUsed) {
+	if (metric === "cosine" && "callSimilarityApi" in finalProvider) {
+		const similarityResp = await finalProvider.callSimilarityApi(expected, output);
+		require_tokenUsageUtils.accumulateTokenUsage(tokensUsed, similarityResp.tokenUsage);
+		if (similarityResp.error) return require_graders.fail(similarityResp.error, tokensUsed);
+		if (similarityResp.similarity == null) return require_graders.fail("Unknown error fetching similarity", tokensUsed);
+		if (!Number.isFinite(similarityResp.similarity)) return require_graders.fail(`Invalid similarity score: ${similarityResp.similarity}`, tokensUsed);
+		return similarityResp.similarity;
+	}
+	const callEmbeddingApi = "callEmbeddingApi" in finalProvider ? finalProvider.callEmbeddingApi : void 0;
+	if (typeof callEmbeddingApi !== "function") {
+		if ("callSimilarityApi" in finalProvider) return require_graders.fail(`Provider ${finalProvider.id()} only supports cosine similarity via callSimilarityApi`, tokensUsed);
+		throw new Error("Provider must implement callSimilarityApi or callEmbeddingApi");
+	}
+	const [expectedEmbedding, outputEmbedding] = await Promise.all([callEmbeddingApi.call(finalProvider, expected), callEmbeddingApi.call(finalProvider, output)]);
+	const mergedUsage = require_graders.normalizeMatcherTokenUsage(void 0);
+	require_tokenUsageUtils.accumulateTokenUsage(mergedUsage, expectedEmbedding.tokenUsage);
+	require_tokenUsageUtils.accumulateTokenUsage(mergedUsage, outputEmbedding.tokenUsage);
+	require_tokenUsageUtils.accumulateTokenUsage(tokensUsed, mergedUsage);
+	if (expectedEmbedding.error || outputEmbedding.error) return require_graders.fail(expectedEmbedding.error || outputEmbedding.error || "Unknown error fetching embeddings", tokensUsed);
+	if (!expectedEmbedding.embedding || !outputEmbedding.embedding) return require_graders.fail("Embedding not found", tokensUsed);
+	return calculateSimilarityScore(expectedEmbedding.embedding, outputEmbedding.embedding, metric, tokensUsed);
+}
+async function matchesSimilarity(expected, output, threshold, inverse = false, grading, metric = "cosine") {
+	if (metric === "cosine" && require_logger.state.config?.redteam && require_remoteGeneration.shouldGenerateRemote({ requireEmbeddingProvider: true })) try {
+		return await require_graders.doRemoteGrading({
+			task: "similar",
+			expected,
+			output,
+			threshold,
+			inverse
+		});
+	} catch (error) {
+		return require_graders.fail(`Could not perform remote grading: ${error}`);
+	}
+	const defaults = await require_graders.getDefaultProviders();
+	const finalProvider = await require_graders.getAndCheckProvider("embedding", grading?.provider, defaults.embeddingProvider, "similarity check");
+	const tokensUsed = require_graders.normalizeMatcherTokenUsage(void 0);
+	const similarity = await calculateProviderSimilarity(finalProvider, expected, output, metric, tokensUsed);
+	if (typeof similarity !== "number") return similarity;
+	return buildSimilarityResult(similarity, threshold, inverse, metric, tokensUsed);
+}
+//#endregion
 //#region src/tracing/evaluatorTracing.ts
 let otlpReceiverStarted = false;
 const DEFAULT_OTLP_ACCEPT_FORMATS = ["json", "protobuf"];
@@ -305,7 +820,7 @@ async function startOtlpReceiverIfNeeded(testSuite) {
 		require_telemetry.telemetry.record("feature_used", { feature: "tracing" });
 		try {
 			require_logger.logger.debug("[EvaluatorTracing] Tracing configuration detected, starting OTLP receiver");
-			const { startOTLPReceiver } = await Promise.resolve().then(() => require("./otlpReceiver-BfcVq2Nq.cjs"));
+			const { startOTLPReceiver } = await Promise.resolve().then(() => require("./otlpReceiver-CvJdBGSc.cjs"));
 			const port = testSuite.tracing.otlp.http.port || 4318;
 			const host = testSuite.tracing.otlp.http.host || "127.0.0.1";
 			const acceptFormats = normalizeOtlpAcceptFormats(testSuite.tracing.otlp.http.acceptFormats);
@@ -329,7 +844,7 @@ async function startOtlpReceiverIfNeeded(testSuite) {
 async function stopOtlpReceiverIfNeeded() {
 	if (otlpReceiverStarted) try {
 		require_logger.logger.debug("[EvaluatorTracing] Stopping OTLP receiver");
-		const { stopOTLPReceiver } = await Promise.resolve().then(() => require("./otlpReceiver-BfcVq2Nq.cjs"));
+		const { stopOTLPReceiver } = await Promise.resolve().then(() => require("./otlpReceiver-CvJdBGSc.cjs"));
 		await stopOTLPReceiver();
 		otlpReceiverStarted = false;
 		require_logger.logger.info("[EvaluatorTracing] OTLP receiver stopped successfully");
@@ -364,7 +879,7 @@ async function generateTraceContextIfNeeded(test, evaluateOptions, testIdx, prom
 	}
 	if (!tracingEnabled) return null;
 	require_logger.logger.debug("[EvaluatorTracing] Importing trace store");
-	const { getTraceStore } = await Promise.resolve().then(() => require("./store-CWOSz6D_.cjs"));
+	const { getTraceStore } = await Promise.resolve().then(() => require("./store-B2NDDooM.cjs")).then((n) => n.store_exports);
 	const traceStore = getTraceStore();
 	const traceId = generateTraceId();
 	const spanId = generateSpanId();
@@ -406,7 +921,7 @@ const handleAnswerRelevance = async ({ assertion, output, prompt, test, provider
 	require_invariant.invariant(prompt, "answer-relevance assertion type must have a prompt");
 	return {
 		assertion,
-		...await require_graders.matchesAnswerRelevance(typeof test?.vars?.query === "string" ? test.vars.query : prompt, output, assertion.threshold ?? 0, test.options, providerCallContext)
+		...await matchesAnswerRelevance(typeof test?.vars?.query === "string" ? test.vars.query : prompt, output, assertion.threshold ?? 0, test.options, providerCallContext)
 	};
 };
 //#endregion
@@ -662,7 +1177,7 @@ function handleBleuScore({ assertion, inverse, outputString, renderedValue }) {
 //#region src/assertions/classifier.ts
 async function handleClassifier({ assertion, renderedValue, outputString, test, inverse }) {
 	require_invariant.invariant(typeof renderedValue === "string" || typeof renderedValue === "undefined", "\"classifier\" assertion type must have a string value or be undefined");
-	const classificationResult = await require_graders.matchesClassification(renderedValue, outputString, assertion.threshold ?? 1, test.options);
+	const classificationResult = await matchesClassification(renderedValue, outputString, assertion.threshold ?? 1, test.options);
 	if (inverse) {
 		classificationResult.pass = !classificationResult.pass;
 		classificationResult.score = 1 - classificationResult.score;
@@ -674,38 +1189,84 @@ async function handleClassifier({ assertion, renderedValue, outputString, test,
 }
 //#endregion
 //#region src/assertions/contains.ts
+/**
+* Advance over separators between parsed fields.
+*
+* Contains-any values allow whitespace around comma delimiters, and historical
+* parsing ignored repeated commas rather than producing empty fields.
+*/
+function skipWhitespaceAndCommas(value, startIndex) {
+	let i = startIndex;
+	while (i < value.length) {
+		i = skipWhitespace(value, i);
+		if (value[i] !== ",") break;
+		i++;
+	}
+	return i;
+}
+/**
+* Advance over whitespace while preserving comma delimiter handling for callers.
+*/
+function skipWhitespace(value, startIndex) {
+	let i = startIndex;
+	while (i < value.length && /\s/.test(value[i])) i++;
+	return i;
+}
+/**
+* Parse a quoted field using the assertion parser's CSV-like escape rules.
+*
+* Supports backslash-escaped quotes/backslashes and doubled quotes, and rejects
+* unterminated fields so malformed assertion values do not silently pass.
+*/
+function parseQuotedField(value, startIndex) {
+	let i = startIndex + 1;
+	let field = "";
+	let terminated = false;
+	while (i < value.length) if (value[i] === "\\" && i + 1 < value.length && ["\"", "\\"].includes(value[i + 1])) {
+		field += value[i + 1];
+		i += 2;
+	} else if (value[i] === "\"" && i + 1 < value.length && value[i + 1] === "\"") {
+		field += "\"";
+		i += 2;
+	} else if (value[i] === "\"") {
+		i++;
+		terminated = true;
+		break;
+	} else {
+		field += value[i];
+		i++;
+	}
+	require_invariant.invariant(terminated, "Unterminated quoted field in contains assertion value");
+	return {
+		field,
+		nextIndex: i
+	};
+}
+/**
+* Parse an unquoted field up to the next comma, trimming surrounding whitespace.
+*/
+function parseUnquotedField(value, startIndex) {
+	let i = startIndex;
+	while (i < value.length && value[i] !== ",") i++;
+	return {
+		field: value.substring(startIndex, i).trim(),
+		nextIndex: i
+	};
+}
+/**
+* Split a contains-any string into fields while preserving quoted commas.
+*/
 function parseCommaSeparatedValues(value) {
 	const results = [];
 	let i = 0;
 	while (i < value.length) {
-		while (i < value.length && /\s/.test(value[i])) i++;
+		i = skipWhitespaceAndCommas(value, i);
 		if (i >= value.length) break;
-		if (value[i] === ",") {
-			i++;
-			continue;
-		}
-		if (value[i] === "\"") {
-			i++;
-			let field = "";
-			while (i < value.length) if (value[i] === "\\" && i + 1 < value.length && (value[i + 1] === "\"" || value[i + 1] === "\\")) {
-				field += value[i + 1];
-				i += 2;
-			} else if (value[i] === "\"" && i + 1 < value.length && value[i + 1] === "\"") {
-				field += "\"";
-				i += 2;
-			} else if (value[i] === "\"") {
-				i++;
-				break;
-			} else {
-				field += value[i];
-				i++;
-			}
-			results.push(field);
-		} else {
-			const start = i;
-			while (i < value.length && value[i] !== ",") i++;
-			results.push(value.substring(start, i).trim());
-		}
+		const isQuotedField = value[i] === "\"";
+		const parsed = isQuotedField ? parseQuotedField(value, i) : parseUnquotedField(value, i);
+		results.push(parsed.field);
+		i = isQuotedField ? skipWhitespace(value, parsed.nextIndex) : parsed.nextIndex;
+		require_invariant.invariant(!isQuotedField || i >= value.length || value[i] === ",", "Expected comma after quoted field in contains assertion value");
 	}
 	return results;
 }
@@ -803,10 +1364,10 @@ async function handleContextFaithfulness({ assertion, test, output, prompt, prov
 	require_invariant.invariant(test.vars, "context-faithfulness assertion requires a test with variables");
 	require_invariant.invariant(typeof test.vars.query === "string", "context-faithfulness assertion requires a \"query\" variable with the user question");
 	require_invariant.invariant(typeof output === "string", "context-faithfulness assertion requires string output from the provider");
-	const context = await require_graders.resolveContext(assertion, test, output, prompt, void 0, providerResponse);
+	const context = await resolveContext(assertion, test, output, prompt, void 0, providerResponse);
 	return {
 		assertion,
-		...await require_graders.matchesContextFaithfulness(test.vars.query, output, context, assertion.threshold ?? 0, test.options, test.vars, providerCallContext),
+		...await matchesContextFaithfulness(test.vars.query, output, context, assertion.threshold ?? 0, test.options, test.vars, providerCallContext),
 		metadata: { context }
 	};
 }
@@ -825,8 +1386,8 @@ async function handleContextFaithfulness({ assertion, test, output, prompt, prov
 const handleContextRecall = async ({ assertion, renderedValue, prompt, test, output, providerResponse, providerCallContext }) => {
 	require_invariant.invariant(typeof renderedValue === "string", "context-recall assertion requires a string value (expected answer or fact to verify)");
 	require_invariant.invariant(prompt, "context-recall assertion requires a prompt");
-	const context = await require_graders.resolveContext(assertion, test, output, prompt, prompt, providerResponse);
-	const result = await require_graders.matchesContextRecall(context, renderedValue, assertion.threshold ?? 0, test.options, test.vars, providerCallContext);
+	const context = await resolveContext(assertion, test, output, prompt, prompt, providerResponse);
+	const result = await matchesContextRecall(context, renderedValue, assertion.threshold ?? 0, test.options, test.vars, providerCallContext);
 	return {
 		assertion,
 		...result,
@@ -851,8 +1412,8 @@ const handleContextRecall = async ({ assertion, renderedValue, prompt, test, out
 const handleContextRelevance = async ({ assertion, test, output, prompt, providerResponse, providerCallContext }) => {
 	require_invariant.invariant(test.vars, "context-relevance assertion requires a test with variables");
 	require_invariant.invariant(typeof test.vars.query === "string", "context-relevance assertion requires a \"query\" variable with the user question");
-	const context = await require_graders.resolveContext(assertion, test, output, prompt, void 0, providerResponse);
-	const result = await require_graders.matchesContextRelevance(test.vars.query, context, assertion.threshold ?? 0, test.options, providerCallContext);
+	const context = await resolveContext(assertion, test, output, prompt, void 0, providerResponse);
+	const result = await matchesContextRelevance(test.vars.query, context, assertion.threshold ?? 0, test.options, providerCallContext);
 	return {
 		assertion,
 		...result,
@@ -930,7 +1491,7 @@ function handleFinishReason({ assertion, inverse = false, renderedValue, provide
 //#region src/assertions/functionToolCall.ts
 const handleIsValidFunctionCall = ({ assertion, output, provider, test }) => {
 	try {
-		if (provider instanceof require_providers.AIStudioChatProvider || provider instanceof require_providers.GoogleLiveProvider || provider instanceof require_providers.VertexChatProvider) require_transform$1.validateFunctionCall(output, provider.config?.tools, test.vars);
+		if (provider instanceof require_providers.AIStudioChatProvider || provider instanceof require_providers.GoogleLiveProvider || provider instanceof require_providers.VertexChatProvider) require_transform.validateFunctionCall(output, provider.config?.tools, test.vars);
 		else if (provider instanceof require_chat.OpenAiChatCompletionProvider) require_util$1.validateFunctionCall(output, provider.config.functions, test.vars);
 		else throw new Error(`Provider does not have functionality for checking function call.`);
 		return {
@@ -950,27 +1511,67 @@ const handleIsValidFunctionCall = ({ assertion, output, provider, test }) => {
 };
 //#endregion
 //#region src/assertions/geval.ts
-const handleGEval = async ({ assertion, renderedValue, prompt, outputString, test, providerCallContext }) => {
-	require_invariant.invariant(typeof renderedValue === "string" || Array.isArray(renderedValue), "G-Eval assertion type must have a string or array of strings value");
+const handleGEval = async ({ assertion, inverse, renderedValue, prompt, outputString, test, providerCallContext }) => {
+	require_invariant.invariant(typeof renderedValue === "string" || Array.isArray(renderedValue) && renderedValue.every((value) => typeof value === "string"), "G-Eval assertion type must have a string or array of strings value");
 	const threshold = assertion.threshold ?? .7;
 	if (Array.isArray(renderedValue)) {
-		const scores = [];
-		const reasons = [];
-		for (const value of renderedValue) {
+		if (renderedValue.length === 0) return {
+			assertion,
+			pass: false,
+			score: 0,
+			reason: "G-Eval assertion requires at least one criterion string in the value array."
+		};
+		const responses = [];
+		let failure;
+		for (const [index, value] of renderedValue.entries()) {
 			const resp = await require_graders.matchesGEval(value, prompt || "", outputString, threshold, test.options, providerCallContext);
-			scores.push(resp.score);
-			reasons.push(resp.reason);
+			responses.push(resp);
+			if (require_graders.isGraderFailure(resp)) {
+				failure = {
+					index,
+					resp
+				};
+				break;
+			}
+		}
+		const tokensUsed = require_tokenUsageUtils.createEmptyTokenUsage();
+		for (const r of responses) require_tokenUsageUtils.accumulateTokenUsage(tokensUsed, r.tokensUsed);
+		if (failure) {
+			const criterion = renderedValue[failure.index];
+			return {
+				assertion,
+				pass: false,
+				score: 0,
+				reason: `G-Eval criterion ${failure.index + 1}/${renderedValue.length} (${JSON.stringify(criterion)}) failed: ${failure.resp.reason}`,
+				tokensUsed,
+				metadata: failure.resp.metadata
+			};
 		}
-		const scoresSum = scores.reduce((a, b) => a + b, 0);
+		const averageScore = responses.reduce((acc, r) => acc + r.score, 0) / responses.length;
+		const combinedReason = responses.map((r) => r.reason).join("\n\n");
 		return {
 			assertion,
-			pass: scoresSum / scores.length >= threshold,
-			score: scoresSum / scores.length,
-			reason: reasons.join("\n\n")
+			pass: averageScore >= threshold !== inverse,
+			score: inverse ? 1 - averageScore : averageScore,
+			reason: combinedReason,
+			tokensUsed
 		};
-	} else return {
+	}
+	const resp = await require_graders.matchesGEval(renderedValue, prompt || "", outputString, threshold, test.options, providerCallContext);
+	if (require_graders.isGraderFailure(resp)) return {
+		assertion,
+		pass: false,
+		score: 0,
+		reason: resp.reason,
+		tokensUsed: resp.tokensUsed,
+		metadata: resp.metadata
+	};
+	const passed = resp.score >= threshold !== inverse;
+	return {
 		assertion,
-		...await require_graders.matchesGEval(renderedValue, prompt || "", outputString, threshold, test.options, providerCallContext)
+		...resp,
+		pass: passed,
+		score: inverse ? 1 - resp.score : resp.score
 	};
 };
 //#endregion
@@ -1110,6 +1711,43 @@ const handleGuardrails = async ({ assertion, inverse, providerResponse }) => {
 };
 //#endregion
 //#region src/assertions/html.ts
+const LITERAL_WRAPPER_PATTERNS = {
+	html: /<html(?=[\s>/])/,
+	head: /<head(?=[\s>/])/,
+	body: /<body(?=[\s>/])/
+};
+function isWrapperTagName(tagName) {
+	return tagName === "html" || tagName === "head" || tagName === "body";
+}
+function isTextNode(node) {
+	return node.nodeName === "#text";
+}
+function isElementNode(node) {
+	return "tagName" in node;
+}
+function hasSourceCodeLocation(element) {
+	return "sourceCodeLocation" in element && element.sourceCodeLocation !== null && element.sourceCodeLocation !== void 0;
+}
+function getChildNodes(node) {
+	return "childNodes" in node ? node.childNodes : [];
+}
+function findFirstElement(root, predicate) {
+	const stack = [root];
+	while (stack.length > 0) {
+		const current = stack.pop();
+		if (isElementNode(current) && predicate(current)) return current;
+		const children = getChildNodes(current);
+		for (let i = children.length - 1; i >= 0; i--) stack.push(children[i]);
+	}
+}
+function hasTopLevelText(parentNode) {
+	return parentNode.childNodes.some((node) => isTextNode(node) && Boolean(node.value.trim()));
+}
+function isUserProvidedElement(element, inputLowercase) {
+	const tagName = element.tagName.toLowerCase();
+	if (isWrapperTagName(tagName)) return LITERAL_WRAPPER_PATTERNS[tagName].test(inputLowercase) && hasSourceCodeLocation(element);
+	return VALID_HTML_ELEMENTS.has(tagName) || tagName.includes("-");
+}
 const HTML_PATTERNS = {
 	openingTag: /<[a-zA-Z][a-zA-Z0-9-]*(?:\s[^>]*)?>/,
 	closingTag: /<\/[a-zA-Z][a-zA-Z0-9-]*\s*>/,
@@ -1265,37 +1903,21 @@ function validateHtml(htmlString) {
 		isValid: false,
 		reason: "Output appears to be XML, not HTML"
 	};
-	try {
-		const { document } = new jsdom.JSDOM(trimmed, { contentType: "text/html" }).window;
-		if (document.body && !trimmed.toLowerCase().includes("<body")) {
-			if (Array.from(document.body.childNodes).some((node) => node.nodeType === 3 && node.textContent?.trim())) return {
-				isValid: false,
-				reason: "Output must be wrapped in HTML tags"
-			};
-		}
-		const allElements = document.querySelectorAll("*");
-		if (!Array.from(allElements).find((element) => {
-			const tagName = element.tagName.toLowerCase();
-			if ([
-				"html",
-				"head",
-				"body"
-			].includes(tagName) && !trimmed.toLowerCase().includes(`<${tagName}`)) return false;
-			return VALID_HTML_ELEMENTS.has(tagName) || tagName.includes("-");
-		})) return {
-			isValid: false,
-			reason: "Output does not contain recognized HTML elements"
-		};
-		return {
-			isValid: true,
-			reason: "Output is valid HTML"
-		};
-	} catch (error) {
-		return {
-			isValid: false,
-			reason: `HTML parsing failed: ${error instanceof Error ? error.message : "Unknown error"}`
-		};
-	}
+	const document = (0, parse5.parse)(trimmed, { sourceCodeLocationInfo: true });
+	const inputLowercase = trimmed.toLowerCase();
+	const body = findFirstElement(document, (element) => element.tagName === "body");
+	if (!(body !== void 0 && LITERAL_WRAPPER_PATTERNS.body.test(inputLowercase) && hasSourceCodeLocation(body)) && body && hasTopLevelText(body)) return {
+		isValid: false,
+		reason: "Output must be wrapped in HTML tags"
+	};
+	if (!findFirstElement(document, (element) => isUserProvidedElement(element, inputLowercase))) return {
+		isValid: false,
+		reason: "Output does not contain recognized HTML elements"
+	};
+	return {
+		isValid: true,
+		reason: "Output is valid HTML"
+	};
 }
 const handleContainsHtml = ({ assertion, outputString, inverse }) => {
 	const pass = containsHtml(outputString) !== inverse;
@@ -1460,7 +2082,7 @@ const handleJavascript = async ({ assertion, renderedValue, valueFromScript, ass
 		let result;
 		if (typeof valueFromScript === "undefined") {
 			const functionBody = renderedValue.includes("\n") ? renderedValue : buildFunctionBody(renderedValue);
-			result = await validateResult(new Function("output", "context", "process", functionBody)(output, assertionValueContext, require_transform.getProcessShim()));
+			result = await validateResult(new Function("output", "context", "process", functionBody)(output, assertionValueContext, require_processShim.getProcessShim()));
 		} else {
 			require_invariant.invariant(typeof valueFromScript === "boolean" || typeof valueFromScript === "number" || typeof valueFromScript === "object", `Javascript assertion script must return a boolean, number, or object (${assertion.value})`);
 			result = await validateResult(valueFromScript);
@@ -1667,7 +2289,7 @@ const handleModeration = async ({ assertion, test, outputString, providerRespons
 		const parsedPrompt = require_fetch.parseChatPrompt(promptToModerate, null);
 		if (parsedPrompt && parsedPrompt.length > 0) promptToModerate = getLastModerationPrompt(parsedPrompt) ?? promptToModerate;
 	} catch {}
-	const moderationResult = await require_graders.matchesModeration({
+	const moderationResult = await matchesModeration({
 		userPrompt: promptToModerate,
 		assistantResponse: outputString,
 		categories: Array.isArray(assertion.value) ? assertion.value : []
@@ -1952,45 +2574,6 @@ function matchesPattern(spanName, pattern) {
 }
 //#endregion
 //#region src/assertions/trajectoryUtils.ts
-const TOOL_ATTRIBUTE_KEYS = [
-	"tool.name",
-	"tool_name",
-	"tool",
-	"function.name",
-	"function_name",
-	"gen_ai.tool.name",
-	"codex.mcp.tool",
-	"agent.tool",
-	"agent.tool_name",
-	"agent.toolName"
-];
-const TOOL_ARGUMENT_ATTRIBUTE_KEYS = [
-	"tool.arguments",
-	"tool.args",
-	"tool.input",
-	"tool_arguments",
-	"tool_args",
-	"tool_input",
-	"function.arguments",
-	"function.args",
-	"function.input",
-	"function_arguments",
-	"function_args",
-	"gen_ai.tool.arguments",
-	"gen_ai.tool.args",
-	"gen_ai.tool.input",
-	"gen_ai.tool.call.arguments",
-	"gen_ai.tool.call.args",
-	"agent.tool.arguments",
-	"agent.tool.args",
-	"agent.tool.input",
-	"codex.mcp.arguments",
-	"codex.mcp.args",
-	"codex.mcp.input",
-	"arguments",
-	"args",
-	"input"
-];
 const COMMAND_ATTRIBUTE_KEYS = [
 	"codex.command",
 	"command",
@@ -2003,16 +2586,15 @@ const SEARCH_ATTRIBUTE_KEYS = [
 	"search_query"
 ];
 const GENERIC_QUERY_ATTRIBUTE_KEYS = ["query"];
+const COMMAND_TOOL_NAMES = new Set([
+	"exec_command",
+	"local_shell",
+	"shell"
+]);
 const SEARCH_SPAN_NAME_PATTERN = /(^|[\s._:/-])(search|find|lookup|retriev(?:e|al))($|[\s._:/-])/i;
 const MAX_JUDGE_SUMMARY_STEPS = 24;
 const JUDGE_SUMMARY_HEAD_STEPS = 12;
 const JUDGE_SUMMARY_TAIL_STEPS = 12;
-function getStringAttribute(attributes, keys) {
-	for (const key of keys) {
-		const value = attributes[key];
-		if (typeof value === "string" && value.trim()) return value.trim();
-	}
-}
 function normalizeStructuredAttribute(value) {
 	if (value === void 0 || value === null) return;
 	if (typeof value === "string") {
@@ -2044,9 +2626,12 @@ function getTrajectoryStepStatus(step) {
 function getCommandExecutable(command) {
 	return command.trim().split(/\s+/)[0] || void 0;
 }
+function isCommandToolName(toolName) {
+	return !!toolName && COMMAND_TOOL_NAMES.has(toolName.trim().toLowerCase());
+}
 function extractToolName(span) {
 	const attributes = span.attributes || {};
-	const directMatch = getStringAttribute(attributes, TOOL_ATTRIBUTE_KEYS);
+	const directMatch = require_toolAttributes.getToolNameFromAttributes(attributes);
 	if (directMatch) return directMatch;
 	for (const [key, value] of Object.entries(attributes)) {
 		if (typeof value !== "string" || !value.trim()) continue;
@@ -2060,7 +2645,7 @@ function extractToolName(span) {
 }
 function extractToolArgs(span) {
 	const attributes = span.attributes || {};
-	for (const key of TOOL_ARGUMENT_ATTRIBUTE_KEYS) {
+	for (const key of require_toolAttributes.TOOL_ARGUMENT_ATTRIBUTE_KEYS) {
 		const value = normalizeStructuredAttribute(attributes[key]);
 		if (value !== void 0) return value;
 	}
@@ -2071,21 +2656,31 @@ function extractToolArgs(span) {
 		if (value !== void 0) return value;
 	}
 }
-function extractCommand(span) {
+function extractCommand(span, toolName = extractToolName(span), getToolArgs = () => extractToolArgs(span)) {
 	const attributes = span.attributes || {};
-	const directMatch = getStringAttribute(attributes, COMMAND_ATTRIBUTE_KEYS);
+	const directMatch = require_toolAttributes.getFirstStringAttribute(attributes, COMMAND_ATTRIBUTE_KEYS);
 	if (directMatch) return directMatch;
 	for (const [key, value] of Object.entries(attributes)) {
 		if (typeof value !== "string" || !value.trim()) continue;
 		if (/command/i.test(key) && !/output|result/i.test(key)) return value.trim();
 	}
+	const toolArgs = getToolArgs();
+	if (isCommandToolName(toolName) && toolArgs && typeof toolArgs === "object") {
+		const args = toolArgs;
+		const command = args.cmd ?? args.command;
+		if (typeof command === "string" && command.trim()) return command.trim();
+		if (Array.isArray(command)) {
+			const joined = command.map((part) => String(part).trim()).filter(Boolean).join(" ");
+			if (joined) return joined;
+		}
+	}
 	if (span.name.startsWith("exec ")) return span.name.slice(5).trim();
 }
 function extractSearchQuery(span) {
 	const attributes = span.attributes || {};
-	const directMatch = getStringAttribute(attributes, SEARCH_ATTRIBUTE_KEYS);
+	const directMatch = require_toolAttributes.getFirstStringAttribute(attributes, SEARCH_ATTRIBUTE_KEYS);
 	if (directMatch) return directMatch;
-	const genericQuery = getStringAttribute(attributes, GENERIC_QUERY_ATTRIBUTE_KEYS);
+	const genericQuery = require_toolAttributes.getFirstStringAttribute(attributes, GENERIC_QUERY_ATTRIBUTE_KEYS);
 	if (genericQuery && isSearchLikeSpan(span)) return genericQuery;
 	if (span.name.startsWith("search ")) return span.name.slice(7).replace(/^"|"$/g, "").trim();
 }
@@ -2109,17 +2704,34 @@ function extractTrajectorySteps(trace) {
 		return left.index - right.index;
 	}).map(({ span }) => {
 		const toolName = extractToolName(span);
-		const command = extractCommand(span);
+		let toolArgs;
+		let hasExtractedToolArgs = false;
+		const getToolArgs = () => {
+			if (!hasExtractedToolArgs) {
+				toolArgs = extractToolArgs(span);
+				hasExtractedToolArgs = true;
+			}
+			return toolArgs;
+		};
+		const command = extractCommand(span, toolName, getToolArgs);
 		const searchQuery = extractSearchQuery(span);
 		let type = "span";
 		let name = span.name;
 		const aliases = new Set([span.name]);
 		let args;
-		if (toolName) {
+		if (command && isCommandToolName(toolName)) {
+			type = "command";
+			name = command;
+			aliases.add(command);
+			args = getToolArgs();
+			if (toolName) aliases.add(toolName);
+			const executable = getCommandExecutable(command);
+			if (executable) aliases.add(executable);
+		} else if (toolName) {
 			type = "tool";
 			name = toolName;
 			aliases.add(toolName);
-			args = extractToolArgs(span);
+			args = getToolArgs();
 		} else if (command) {
 			type = "command";
 			name = command;
@@ -2295,7 +2907,7 @@ const handleRedteam = async ({ assertion, baseType, test, prompt, outputString,
 				if (match) evalId = match[1];
 			}
 		}
-		const tracking = await require_providers.checkExfilTracking(webPageUuid, evalId);
+		const tracking = await require_indirectWebPwn.checkExfilTracking(webPageUuid, evalId);
 		if (tracking) gradingContext = {
 			...gradingContext,
 			wasExfiltrated: tracking.wasExfiltrated,
@@ -2361,7 +2973,7 @@ function handleIsRefusal(params) {
 			assertion
 		};
 	}
-	const pass = require_providers.isBasicRefusal(output) !== inverse;
+	const pass = require_util$2.isBasicRefusal(output) !== inverse;
 	return {
 		pass,
 		score: pass ? 1 : 0,
@@ -2400,11 +3012,10 @@ function handleRougeScore({ baseType, assertion, renderedValue, outputString, in
 	const rougeMethod = js_rouge[baseType[baseType.length - 1]];
 	const score = rougeMethod(outputString, renderedValue, {});
 	const threshold = assertion.threshold ?? .75;
-	const pass = score >= threshold != inverse;
 	return {
-		pass,
+		pass: score >= threshold !== inverse,
 		score: inverse ? 1 - score : score,
-		reason: pass ? `${baseType.toUpperCase()} score ${score.toFixed(2)} is greater than or equal to threshold ${threshold}` : `${baseType.toUpperCase()} score ${score.toFixed(2)} is less than threshold ${threshold}`,
+		reason: `${baseType.toUpperCase()} score ${score.toFixed(2)} is ${score >= threshold ? "greater than or equal to" : "less than"} threshold ${threshold}`,
 		assertion
 	};
 }
@@ -2466,10 +3077,196 @@ const handleRuby = async ({ assertion, renderedValue, valueFromScript, assertion
 	}
 };
 //#endregion
+//#region src/providers/webSearchUtils.ts
+function hasTool(provider, predicate) {
+	return Array.isArray(provider.config?.tools) && provider.config.tools.some(predicate);
+}
+function getProviderId(provider) {
+	if (typeof provider.id !== "function") return null;
+	try {
+		return provider.id();
+	} catch (err) {
+		require_logger.logger.debug(`Failed to read provider id: ${err}`);
+		return null;
+	}
+}
+function isOpenAiResponsesProvider(provider, id) {
+	return id.includes("openai:responses") || provider.constructor?.name === "OpenAiResponsesProvider";
+}
+/**
+* Check if a provider has web search capabilities
+* @param provider The provider to check
+* @returns true if the provider supports web search
+*/
+function hasWebSearchCapability(provider) {
+	if (!provider) return false;
+	const id = getProviderId(provider);
+	if (!id) return false;
+	if (id.includes("perplexity")) return true;
+	if ((id.includes("google") || id.includes("gemini") || id.includes("vertex")) && hasTool(provider, (t) => t.googleSearch !== void 0)) return true;
+	if (id.includes("xai") && provider.config?.search_parameters?.mode === "on") return true;
+	if (isOpenAiResponsesProvider(provider, id) && hasTool(provider, (t) => t.type === "web_search_preview")) return true;
+	if (id.startsWith("openai:codex") && (provider.config?.web_search_mode === "live" || provider.config?.web_search_mode === "cached" || provider.config?.web_search_enabled === true)) return true;
+	if (id.includes("anthropic") && hasTool(provider, (t) => t.type === "web_search_20250305")) return true;
+	return false;
+}
+/**
+* Load a provider with web search capabilities.
+* Tries multiple providers in order of preference until one succeeds.
+* Uses the latest and most capable models from each provider with specific checkpoint IDs.
+*
+* @param preferAnthropic Whether to try Anthropic first (true) or OpenAI first (false)
+* @returns A provider with web search capabilities or null
+*/
+async function loadWebSearchProvider(preferAnthropic = false) {
+	const loadAnthropicWebSearch = async () => {
+		try {
+			return await require_providers.loadApiProvider("anthropic:messages:claude-opus-4-6", { options: { config: { tools: [{
+				type: "web_search_20250305",
+				name: "web_search",
+				max_uses: 5
+			}] } } });
+		} catch (err) {
+			require_logger.logger.debug(`Failed to load Anthropic web search provider: ${err}`);
+			return null;
+		}
+	};
+	const loadOpenAIWebSearch = async () => {
+		try {
+			return await require_providers.loadApiProvider("openai:responses:gpt-5.4-2026-03-05", { options: { config: { tools: [{ type: "web_search_preview" }] } } });
+		} catch (err) {
+			require_logger.logger.debug(`Failed to load OpenAI web search provider: ${err}`);
+			return null;
+		}
+	};
+	const loadPerplexity = async () => {
+		try {
+			return await require_providers.loadApiProvider("perplexity:sonar-pro");
+		} catch (err) {
+			require_logger.logger.debug(`Failed to load Perplexity provider: ${err}`);
+			return null;
+		}
+	};
+	const loadGoogleWebSearch = async () => {
+		try {
+			return await require_providers.loadApiProvider("google:gemini-3-pro-preview", { options: { config: { tools: [{ googleSearch: {} }] } } });
+		} catch (err) {
+			require_logger.logger.debug(`Failed to load Google web search provider: ${err}`);
+			return null;
+		}
+	};
+	const loadVertexWebSearch = async () => {
+		try {
+			return await require_providers.loadApiProvider("vertex:gemini-3-pro-preview", { options: { config: { tools: [{ googleSearch: {} }] } } });
+		} catch (err) {
+			require_logger.logger.debug(`Failed to load Vertex web search provider: ${err}`);
+			return null;
+		}
+	};
+	const loadXaiWebSearch = async () => {
+		try {
+			return await require_providers.loadApiProvider("xai:grok-4-1-fast-reasoning", { options: { config: { search_parameters: { mode: "on" } } } });
+		} catch (err) {
+			require_logger.logger.debug(`Failed to load xAI web search provider: ${err}`);
+			return null;
+		}
+	};
+	const providers = preferAnthropic ? [
+		loadAnthropicWebSearch,
+		loadOpenAIWebSearch,
+		loadPerplexity,
+		loadGoogleWebSearch,
+		loadVertexWebSearch,
+		loadXaiWebSearch
+	] : [
+		loadOpenAIWebSearch,
+		loadAnthropicWebSearch,
+		loadPerplexity,
+		loadGoogleWebSearch,
+		loadVertexWebSearch,
+		loadXaiWebSearch
+	];
+	for (const getProvider of providers) {
+		const provider = await getProvider();
+		if (provider && hasWebSearchCapability(provider)) {
+			require_logger.logger.info(`Using ${getProviderId(provider) ?? "loaded provider"} as web search provider`);
+			return provider;
+		}
+		if (provider) require_logger.logger.debug(`Loaded provider ${getProviderId(provider) ?? "unknown"} does not support web search`);
+	}
+	return null;
+}
+//#endregion
+//#region src/matchers/search.ts
+async function matchesSearchRubric(rubric, llmOutput, grading, vars, assertion, _provider, providerCallContext) {
+	if (!grading) throw new Error("Cannot grade output without grading config. Specify --grader option or grading config.");
+	const defaultProviders = await require_graders.getDefaultProviders();
+	const defaultSearchProviders = [
+		defaultProviders.webSearchProvider,
+		defaultProviders.llmRubricProvider,
+		defaultProviders.gradingProvider
+	];
+	let searchProvider = (grading.provider ? await require_graders.getGradingProvider("text", grading.provider, null) : null) || defaultSearchProviders.find((provider) => Boolean(provider));
+	if (!hasWebSearchCapability(searchProvider)) {
+		const webSearchDefault = defaultSearchProviders.find((provider) => hasWebSearchCapability(provider));
+		if (webSearchDefault) searchProvider = webSearchDefault;
+	}
+	if (!hasWebSearchCapability(searchProvider)) {
+		const webSearchProvider = await loadWebSearchProvider(true);
+		if (webSearchProvider) searchProvider = webSearchProvider;
+	}
+	if (!searchProvider || !hasWebSearchCapability(searchProvider)) throw new Error(`search-rubric assertion requires a grading provider with web search capabilities. Use --grader with a web search provider (e.g., anthropic:messages:${require_graders.DEFAULT_ANTHROPIC_MODEL}, openai:responses:o4-mini with tools configured, perplexity:sonar) or configure one in defaultTest.options.provider`);
+	const prompt = await require_graders.renderLlmRubricPrompt(await require_graders.loadRubricPrompt(grading?.rubricPrompt, require_graders.DEFAULT_WEB_SEARCH_PROMPT), {
+		output: require_graders.tryParse(llmOutput),
+		rubric,
+		...vars || {}
+	});
+	const resp = await require_graders.callProviderWithContext(searchProvider, prompt, "search-rubric", {
+		output: require_graders.tryParse(llmOutput),
+		rubric,
+		...vars || {}
+	}, providerCallContext);
+	if (resp.error || !resp.output) return {
+		pass: false,
+		score: 0,
+		reason: `Search rubric evaluation failed: ${resp.error || "No output"}`,
+		tokensUsed: resp.tokenUsage,
+		assertion
+	};
+	try {
+		const result = require_logger.extractFirstJsonObject(String(resp.output));
+		let pass = result.pass ?? false;
+		const score = typeof result.score === "number" ? result.score : pass ? 1 : 0;
+		if (assertion?.threshold !== void 0) pass = pass && score >= assertion.threshold;
+		return {
+			pass,
+			score,
+			reason: result.reason || "No reason provided",
+			tokensUsed: resp.tokenUsage,
+			assertion,
+			metadata: {
+				searchResults: result.searchResults || [],
+				searchProvider: searchProvider.id()
+			}
+		};
+	} catch (err) {
+		require_logger.logger.warn(`[search-rubric] Could not parse structured JSON from provider response, falling back to substring matching: ${err.message}`);
+		const outputLower = String(resp.output).toLowerCase();
+		const pass = outputLower.includes("\"pass\":true") || outputLower.includes("\"pass\": true");
+		return {
+			pass,
+			score: pass ? 1 : 0,
+			reason: resp.output,
+			tokensUsed: resp.tokenUsage,
+			assertion
+		};
+	}
+}
+//#endregion
 //#region src/assertions/searchRubric.ts
 async function handleSearchRubric({ assertion, baseType: _baseType, inverse, provider, providerCallContext, renderedValue, test, providerResponse }) {
 	if (renderedValue == null) throw new Error("search-rubric assertion type must have a string value");
-	const result = await require_graders.matchesSearchRubric(String(renderedValue), providerResponse.output, test.options, test.vars, assertion, provider, providerCallContext);
+	const result = await matchesSearchRubric(String(renderedValue), providerResponse.output, test.options, test.vars, assertion, provider, providerCallContext);
 	if (inverse) {
 		result.pass = !result.pass;
 		result.reason = result.pass ? `Output does not require web search verification: ${result.reason}` : `Output requires web search verification: ${result.reason}`;
@@ -2500,7 +3297,7 @@ const handleSimilar = async ({ assertion, renderedValue, outputString, inverse,
 	if (Array.isArray(renderedValue)) {
 		let minScore = Number.POSITIVE_INFINITY;
 		for (const value of renderedValue) {
-			const result = await require_graders.matchesSimilarity(value, outputString, threshold, inverse, test.options, metric);
+			const result = await matchesSimilarity(value, outputString, threshold, inverse, test.options, metric);
 			if (result.pass) return {
 				assertion,
 				...result
@@ -2515,7 +3312,7 @@ const handleSimilar = async ({ assertion, renderedValue, outputString, inverse,
 		};
 	} else return {
 		assertion,
-		...await require_graders.matchesSimilarity(renderedValue, outputString, threshold, inverse, test.options, metric)
+		...await matchesSimilarity(renderedValue, outputString, threshold, inverse, test.options, metric)
 	};
 };
 //#endregion
@@ -3121,13 +3918,13 @@ function resolveSequenceValue(value) {
 	}
 	throw new Error("trajectory:tool-sequence assertion must have an array or object value");
 }
-function isRecord(value) {
+function isRecord$1(value) {
 	return typeof value === "object" && value !== null && !Array.isArray(value);
 }
 function matchesExpectedArgsPartial(actual, expected) {
 	if (Array.isArray(expected)) return Array.isArray(actual) && actual.length === expected.length && expected.every((item, index) => matchesExpectedArgsPartial(actual[index], item));
-	if (isRecord(expected)) {
-		if (!isRecord(actual)) return false;
+	if (isRecord$1(expected)) {
+		if (!isRecord$1(actual)) return false;
 		return Object.entries(expected).every(([key, expectedValue]) => Object.prototype.hasOwnProperty.call(actual, key) && matchesExpectedArgsPartial(actual[key], expectedValue));
 	}
 	return (0, node_util.isDeepStrictEqual)(actual, expected);
@@ -3484,7 +4281,7 @@ function assertionMayNeedTraceContext(assertion) {
 	if (assertionUsesTrace(assertion)) return true;
 	if (assertion.type === "assert-set") return assertion.assert.some(assertionMayNeedTraceContext);
 	if (assertion.type.startsWith("promptfoo:redteam:coding-agent:")) return true;
-	return typeof assertion.value === "string" ? assertion.value.startsWith("file://") || require_providers.isPackagePath(assertion.value) : false;
+	return typeof assertion.value === "string" ? assertion.value.startsWith("file://") || require_packageParser.isPackagePath(assertion.value) : false;
 }
 function hasTraceAwareAssertions(assertions) {
 	return Boolean(assertions?.some(assertionMayNeedTraceContext));
@@ -3498,7 +4295,7 @@ async function loadTraceData(traceId) {
 	let stableObservations = 0;
 	let latestTrace = null;
 	for (let attempt = 0; attempt < maxAttempts; attempt++) {
-		latestTrace = await traceStore.getTrace(traceId);
+		latestTrace = await traceStore.getTrace(traceId, { sanitizeAttributes: false });
 		const spanCount = latestTrace?.spans?.length ?? 0;
 		if (spanCount > 0) {
 			stableObservations = spanCount === lastSpanCount ? stableObservations + 1 : 1;
@@ -3551,7 +4348,7 @@ const ASSERTION_HANDLERS = {
 	"llm-rubric": handleLlmRubric,
 	meteor: async (params) => {
 		try {
-			const { handleMeteorAssertion } = await Promise.resolve().then(() => require("./meteor-Co1VQ1u5.cjs"));
+			const { handleMeteorAssertion } = await Promise.resolve().then(() => require("./meteor-CR226f7Z.cjs"));
 			return handleMeteorAssertion(params);
 		} catch (error) {
 			if (error instanceof Error && (error.message.includes("Cannot find module") || error.message.includes("natural\" package is required"))) return {
@@ -3633,7 +4430,7 @@ async function runAssertion({ prompt, provider, assertion, test, vars, latencyMs
 	const { cost, logProbs, output: originalOutput } = providerResponse;
 	let output = originalOutput;
 	require_invariant.invariant(assertion.type, `Assertion must have a type: ${JSON.stringify(assertion)}`);
-	if (assertion.transform) output = await require_transform.transform(assertion.transform, output, {
+	if (assertion.transform) output = await require_transform$1.transform(assertion.transform, output, {
 		vars: resolvedVars,
 		prompt: { label: prompt },
 		...providerResponse && providerResponse.metadata && { metadata: providerResponse.metadata }
@@ -3687,7 +4484,7 @@ async function runAssertion({ prompt, provider, assertion, test, vars, latencyMs
 			};
 		}
 		else if (filePath.endsWith(".rb")) try {
-			const { runRuby } = await Promise.resolve().then(() => require("./rubyUtils-DUbq4tff.cjs"));
+			const { runRuby } = await Promise.resolve().then(() => require("./rubyUtils-CqUWBZAt.cjs")).then((n) => n.rubyUtils_exports);
 			valueFromScript = await runRuby(filePath, functionName || "get_assert", [output, context]);
 			require_logger.logger.debug(`Ruby script ${filePath} output: ${valueFromScript}`);
 		} catch (error) {
@@ -3699,9 +4496,9 @@ async function runAssertion({ prompt, provider, assertion, test, vars, latencyMs
 			};
 		}
 		else renderedValue = require_graders.processFileReference(renderedValue);
-	} else if (require_providers.isPackagePath(renderedValue)) {
+	} else if (require_packageParser.isPackagePath(renderedValue)) {
 		const basePath = require_logger.state.basePath || "";
-		const requiredModule = await require_providers.loadFromPackage(renderedValue, basePath);
+		const requiredModule = await require_packageParser.loadFromPackage(renderedValue, basePath);
 		if (typeof requiredModule !== "function") throw new Error(`Assertion malformed: ${renderedValue} must be a function. Received: ${typeof requiredModule}`);
 		valueFromScript = await Promise.resolve(requiredModule(output, context));
 	} else renderedValue = nunjucks.renderString(renderedValue, resolvedVars);
@@ -3804,7 +4601,8 @@ async function runAssertions({ assertScoringFunction, latencyMs, prompt, provide
 		require_logger.logger.debug(`Failed to preload trace data for assertions: ${error}`);
 		preloadedTraceData = null;
 	}
-	await async.default.forEachOfLimit(asserts, ASSERTIONS_MAX_CONCURRENCY, async ({ assertion, assertResult, index }) => {
+	const concurrency = require_graders.getProviderCallExecutionContext()?.providerCallQueue ? 1 : ASSERTIONS_MAX_CONCURRENCY;
+	await async.default.forEachOfLimit(asserts, concurrency, async ({ assertion, assertResult, index }) => {
 		if (assertion.type.startsWith("select-") || assertion.type === "max-score") return;
 		const result = await runAssertion({
 			prompt,
@@ -3840,7 +4638,7 @@ async function runAssertions({ assertScoringFunction, latencyMs, prompt, provide
 async function runCompareAssertion(test, assertion, outputs, context) {
 	require_invariant.invariant(typeof assertion.value === "string", "select-best must have a string value");
 	test = require_graders.getFinalTest(test, assertion);
-	return (await require_graders.matchesSelectBest(assertion.value, outputs, test.options, test.vars, context)).map((result) => ({
+	return (await matchesSelectBest(assertion.value, outputs, test.options, test.vars, context)).map((result) => ({
 		...result,
 		assertion
 	}));
@@ -3857,17 +4655,17 @@ async function readAssertions(filePath) {
 var assertions_default = {
 	runAssertion,
 	runAssertions,
-	matchesSimilarity: require_graders.matchesSimilarity,
-	matchesClassification: require_graders.matchesClassification,
+	matchesSimilarity,
+	matchesClassification,
 	matchesLlmRubric: require_graders.matchesLlmRubric,
 	matchesFactuality: require_graders.matchesFactuality,
 	matchesClosedQa: require_graders.matchesClosedQa,
-	matchesAnswerRelevance: require_graders.matchesAnswerRelevance,
-	matchesContextRecall: require_graders.matchesContextRecall,
-	matchesContextRelevance: require_graders.matchesContextRelevance,
-	matchesContextFaithfulness: require_graders.matchesContextFaithfulness,
-	matchesComparisonBoolean: require_graders.matchesSelectBest,
-	matchesModeration: require_graders.matchesModeration,
+	matchesAnswerRelevance,
+	matchesContextRecall,
+	matchesContextRelevance,
+	matchesContextFaithfulness,
+	matchesComparisonBoolean: matchesSelectBest,
+	matchesModeration,
 	matchesConversationRelevance
 };
 //#endregion
@@ -3931,7 +4729,8 @@ var CIProgressReporter = class {
 	}
 	updateTotalTests(newTotal) {
 		this.totalTests = Math.max(newTotal, 1);
-		this.highestPercentageSeen = Math.floor(this.completedTests / this.totalTests * 100);
+		const percentage = Math.floor(this.completedTests / this.totalTests * 100);
+		this.highestPercentageSeen = percentage;
 	}
 	finish() {
 		if (this.intervalId) {
@@ -4104,6 +4903,10 @@ function getDefaultOtelConfig() {
 }
 //#endregion
 //#region src/tracing/localSpanExporter.ts
+const MISSING_TRACE_RETRY_DELAY_MS = 50;
+function delay(ms) {
+	return new Promise((resolve) => setTimeout(resolve, ms));
+}
 /**
 * A span exporter that writes spans to the local TraceStore (SQLite).
 * This allows OTEL spans to be stored locally for analysis in the promptfoo UI.
@@ -4145,7 +4948,7 @@ var LocalSpanExporter = class {
 		}
 		let firstError;
 		for (const [traceId, spanDataList] of spansByTrace) try {
-			const result = await traceStore.addSpans(traceId, spanDataList, { skipTraceCheck: false });
+			const result = await this.addSpansWithTraceRetry(traceStore, traceId, spanDataList);
 			if (result.stored) require_logger.logger.debug(`[LocalSpanExporter] Added ${spanDataList.length} spans to trace ${traceId}`);
 			else require_logger.logger.debug(`[LocalSpanExporter] Skipping ${spanDataList.length} spans for orphan trace ${traceId}: ${result.reason}`);
 		} catch (error) {
@@ -4157,6 +4960,16 @@ var LocalSpanExporter = class {
 		}
 		return firstError;
 	}
+	async addSpansWithTraceRetry(traceStore, traceId, spans) {
+		const options = {
+			skipTraceCheck: false,
+			warnIfMissingTrace: false
+		};
+		const result = await traceStore.addSpans(traceId, spans, options);
+		if (result.stored) return result;
+		await delay(MISSING_TRACE_RETRY_DELAY_MS);
+		return traceStore.addSpans(traceId, spans, options);
+	}
 	/**
 	* Convert an OTEL ReadableSpan to our SpanData format.
 	*/
@@ -4242,7 +5055,7 @@ function initializeOtel(config) {
 	require_logger.logger.debug("[OtelSdk] Registered W3C Trace Context propagator");
 	const resource = (0, _opentelemetry_resources.resourceFromAttributes)({
 		[_opentelemetry_semantic_conventions.ATTR_SERVICE_NAME]: config.serviceName,
-		[_opentelemetry_semantic_conventions.ATTR_SERVICE_VERSION]: require_fetch.VERSION
+		[_opentelemetry_semantic_conventions.ATTR_SERVICE_VERSION]: require_version.VERSION
 	});
 	const spanProcessors = [];
 	if (config.localExport) {
@@ -4439,6 +5252,15 @@ function isPromptAllowed(prompt, allowedPrompts) {
 }
 //#endregion
 //#region src/evaluator.ts
+const CONVERSATION_VAR_NAME = "_conversation";
+const promptUsesConversationVariableCache = new lru_cache.LRUCache({ max: 1024 });
+function promptUsesConversationVariable(prompt) {
+	const cached = promptUsesConversationVariableCache.get(prompt.raw);
+	if (cached !== void 0) return cached;
+	const { referenced, parsed } = require_render.analyzeTemplateReference(prompt.raw, CONVERSATION_VAR_NAME);
+	if (parsed) promptUsesConversationVariableCache.set(prompt.raw, referenced);
+	return referenced;
+}
 /**
 * Manages a single progress bar for the evaluation
 */
@@ -4638,6 +5460,18 @@ function hasProviderGroupedAssertion(assertion) {
 function shouldDeferGradingForTest(test) {
 	return Boolean(test.assert?.some(hasProviderGroupedAssertion));
 }
+function logGroupedGradingStatus({ concurrency, hasEvalStepTimeout, runEvalOptions, shouldGroupGradingByProvider, usesConversationVar }) {
+	if (!runEvalOptions.some(({ test }) => shouldDeferGradingForTest(test))) return;
+	if (shouldGroupGradingByProvider) {
+		require_logger.logger.info("Grouping model-graded assertions by provider to minimize local-model reload overhead.");
+		return;
+	}
+	if (concurrency !== 1) return;
+	const reasons = [];
+	if (hasEvalStepTimeout) reasons.push("per-eval-step timeout is configured");
+	if (usesConversationVar) reasons.push("conversation variables require per-row ordering");
+	if (reasons.length > 0) require_logger.logger.info(`Serial grading grouping disabled because ${reasons.join(" and ")}; model-graded judges may reload between rows.`);
+}
 function applyGradingResult(row, checkResult) {
 	if (!checkResult.pass) {
 		row.error = checkResult.reason;
@@ -4652,14 +5486,29 @@ function applyGradingResult(row, checkResult) {
 	if (checkResult.tokensUsed) require_tokenUsageUtils.accumulateAssertionTokenUsage(row.tokenUsage.assertions, checkResult.tokensUsed);
 	row.gradingResult = checkResult;
 }
-function applyGradingError(row, error) {
-	const errorMessage = error instanceof Error ? error.stack ?? error.message : String(error);
-	require_logger.logger.error("Assertion grading failed during eval", {
-		error: errorMessage,
-		promptIdx: row.promptIdx,
-		testIdx: row.testIdx
-	});
-	row.error = errorMessage;
+const ABORTED_GRADING_PREFIX = "Aborted: ";
+function isAbortShapedError(error) {
+	return error instanceof Error && (error.name === "AbortError" || error.name === "AbortException");
+}
+function applyGradingError(row, error, abortSignal) {
+	const errorAsError = error instanceof Error ? error : void 0;
+	if (Boolean(abortSignal?.aborted) && isAbortShapedError(error)) {
+		const shortMessage = errorAsError?.message ?? String(error);
+		require_logger.logger.debug("Assertion grading aborted", {
+			error: shortMessage,
+			promptIdx: row.promptIdx,
+			testIdx: row.testIdx
+		});
+		row.error = `${ABORTED_GRADING_PREFIX}${shortMessage}`;
+	} else {
+		const fullMessage = errorAsError ? errorAsError.stack ?? errorAsError.message : String(error);
+		require_logger.logger.error("Assertion grading failed during eval", {
+			error: fullMessage,
+			promptIdx: row.promptIdx,
+			testIdx: row.testIdx
+		});
+		row.error = fullMessage;
+	}
 	row.failureReason = require_types.ResultFailureReason.ERROR;
 	row.success = false;
 	row.score = 0;
@@ -4671,7 +5520,7 @@ function getNonTransientTargetStatus(row) {
 }
 function createRunEvalState({ provider, prompt, test }) {
 	const vars = structuredClone(test.vars || {});
-	const fileMetadata = require_providers.collectFileMetadata(vars);
+	const fileMetadata = require_evaluatorHelpers.collectFileMetadata(vars);
 	const conversationKey = `${provider.label || provider.id()}:${prompt.id}${test.metadata?.conversationId ? `:${test.metadata.conversationId}` : ""}`;
 	const setup = createRunEvalSetup({
 		provider,
@@ -4691,7 +5540,7 @@ function createRunEvalState({ provider, prompt, test }) {
 	};
 }
 function attachConversationVar({ conversations, conversationKey, prompt, test, vars }) {
-	const usesConversation = prompt.raw.includes("_conversation");
+	const usesConversation = promptUsesConversationVariable(prompt);
 	if (!require_logger.getEnvBool("PROMPTFOO_DISABLE_CONVERSATION_VAR") && !test.options?.disableConversationVar && usesConversation) vars._conversation = conversations?.[conversationKey] || [];
 }
 function createRunEvalSetup({ provider, prompt, promptConfig, vars }) {
@@ -4710,8 +5559,8 @@ function createRunEvalSetup({ provider, prompt, promptConfig, vars }) {
 	};
 }
 async function renderRunEvalPrompt({ filters, isRedteam, provider, promptForRender, test, testSuite, vars }) {
-	const renderedPrompt = await require_providers.renderPrompt(promptForRender, vars, filters, provider, shouldSkipRedteamInjectVar(test, testSuite, isRedteam) ? [getRedteamInjectVar(test, promptForRender, testSuite)] : void 0);
-	if (isRedteam) require_providers.throwIfTargetPromptExceedsMaxChars(renderedPrompt, testSuite?.redteam?.maxCharsPerMessage);
+	const renderedPrompt = await require_evaluatorHelpers.renderPrompt(promptForRender, vars, filters, provider, shouldSkipRedteamInjectVar(test, testSuite, isRedteam) ? [getRedteamInjectVar(test, promptForRender, testSuite)] : void 0);
+	if (isRedteam) require_promptLength.throwIfTargetPromptExceedsMaxChars(renderedPrompt, testSuite?.redteam?.maxCharsPerMessage);
 	const setup = createRunEvalSetup({
 		provider,
 		prompt: promptForRender,
@@ -4778,7 +5627,7 @@ async function callActiveProvider({ abortSignal, evalId, filters, promptForRende
 	});
 	const callApiOptions = abortSignal ? { abortSignal } : void 0;
 	const callApi = () => activeProvider.callApi(renderedPrompt, callApiContext, callApiOptions);
-	const response = rateLimitRegistry ? await rateLimitRegistry.execute(activeProvider, callApi, require_providers.createProviderRateLimitOptions()) : await callApi();
+	const response = rateLimitRegistry ? await rateLimitRegistry.execute(activeProvider, callApi, require_shared.createProviderRateLimitOptions()) : await callApi();
 	require_logger.logger.debug(`Provider response properties: ${Object.keys(response).join(", ")}`);
 	require_logger.logger.debug(`Provider response cached property explicitly: ${response.cached}`);
 	return response;
@@ -4856,7 +5705,7 @@ function createEvaluateResult({ fileMetadata, latencyMs, prompt, promptIdx, rend
 	};
 	if (!ret.metadata?.sessionIds && !ret.metadata?.sessionId) {
 		ret.metadata ??= {};
-		ret.metadata.sessionId = require_providers.getSessionId(response, { vars });
+		ret.metadata.sessionId = require_util$2.getSessionId(response, { vars });
 	}
 	return ret;
 }
@@ -4864,7 +5713,7 @@ function trackProviderUsage(provider, response) {
 	if (!response.tokenUsage) return;
 	const providerId = provider.id();
 	const trackingId = provider.constructor?.name ? `${providerId} (${provider.constructor.name})` : providerId;
-	require_providers.TokenUsageTracker.getInstance().trackUsage(trackingId, response.tokenUsage);
+	require_shared.TokenUsageTracker.getInstance().trackUsage(trackingId, response.tokenUsage);
 }
 async function applyRunEvalResponseOutcome({ abortSignal, deferGrading, evalId, isRedteam, latencyMs, prompt, promptIdx, provider, providerCallQueue, rateLimitRegistry, renderedPrompt, response, ret, test, testIdx, traceContext, vars }) {
 	if (response.error) {
@@ -4938,7 +5787,7 @@ async function gradeRunEvalResponse({ abortSignal, deferGrading, evalId, latency
 			assertScoringFunction: test.assertScoringFunction,
 			traceId
 		}).then((checkResult) => applyGradingResult(ret, checkResult))).catch((error) => {
-			applyGradingError(ret, error);
+			applyGradingError(ret, error, abortSignal);
 		});
 		deferredGradingPromises.set(ret, gradingPromise);
 		return;
@@ -4960,13 +5809,13 @@ async function gradeRunEvalResponse({ abortSignal, deferGrading, evalId, latency
 }
 async function transformRunEvalResponse({ evalId, prompt, promptIdx, provider, response, test, testIdx, vars }) {
 	const processedResponse = { ...response };
-	if (provider.transform) processedResponse.output = await require_transform.transform(provider.transform, processedResponse.output, {
+	if (provider.transform) processedResponse.output = await require_transform$1.transform(provider.transform, processedResponse.output, {
 		vars,
 		prompt
 	});
 	const providerTransformedOutput = processedResponse.output;
 	const testTransform = test.options?.transform || test.options?.postprocess;
-	if (testTransform) processedResponse.output = await require_transform.transform(testTransform, processedResponse.output, {
+	if (testTransform) processedResponse.output = await require_transform$1.transform(testTransform, processedResponse.output, {
 		vars,
 		prompt,
 		...response && response.metadata && { metadata: response.metadata }
@@ -5418,10 +6267,10 @@ async function prepareTestVariables(tests, testSuite) {
 async function applyInputTransform(testCase, inputTransformDefault) {
 	const inputTransform = testCase.options?.transformVars || inputTransformDefault;
 	if (!inputTransform) return;
-	const transformedVars = await require_transform.transform(inputTransform, testCase.vars, {
+	const transformedVars = await require_transform$1.transform(inputTransform, testCase.vars, {
 		prompt: {},
 		uuid: crypto.randomUUID()
-	}, true, require_transform.TransformInputType.VARS);
+	}, true, require_transform$1.TransformInputType.VARS);
 	require_invariant.invariant(typeof transformedVars === "object", "Transform function did not return a valid object");
 	testCase.vars = {
 		...testCase.vars,
@@ -5485,7 +6334,7 @@ async function resolveDefaultTestProvider(defaultTest, testCase) {
 	const defaultProvider = defaultTest.provider;
 	if (require_types.isApiProvider(defaultProvider)) return defaultProvider;
 	if (typeof defaultProvider === "object" && defaultProvider.id) {
-		const { loadApiProvider } = await Promise.resolve().then(() => require("./providers-DV3ax9e_.cjs"));
+		const { loadApiProvider } = await Promise.resolve().then(() => require("./providers-BDVVIQM6.cjs")).then((n) => n.providers_exports);
 		return loadApiProvider(typeof defaultProvider.id === "function" ? defaultProvider.id() : defaultProvider.id, { options: defaultProvider });
 	}
 	return defaultProvider;
@@ -5645,7 +6494,7 @@ function buildRepeatCacheContextByTestIdx(runEvalOptions) {
 async function filterCompletedResumeSteps(runEvalOptions, evalRecord) {
 	if (!require_logger.state.resume || !evalRecord.persisted) return;
 	try {
-		const { default: EvalResult } = await Promise.resolve().then(() => require("./evalResult-Dvc-iucu.cjs"));
+		const { default: EvalResult } = await Promise.resolve().then(() => require("./evalResult-CvtS8h8u.cjs")).then((n) => n.evalResult_exports);
 		const completedPairs = await EvalResult.getCompletedIndexPairs(evalRecord.id, { excludeErrors: require_logger.state.retryMode });
 		const originalCount = runEvalOptions.length;
 		for (let i = runEvalOptions.length - 1; i >= 0; i--) {
@@ -5659,14 +6508,14 @@ async function filterCompletedResumeSteps(runEvalOptions, evalRecord) {
 	}
 }
 function adjustConcurrencyForSerialFeatures({ concurrency, prompts, tests }) {
-	const usesConversationVar = prompts.some((p) => p.raw.includes("_conversation"));
+	const usesConversationVar = prompts.some(promptUsesConversationVariable);
 	if (concurrency <= 1) return {
 		concurrency,
 		usesConversationVar
 	};
 	const usesStoreOutputAs = tests.some((t) => t.options?.storeOutputAs);
 	if (usesConversationVar) {
-		require_logger.logger.info(`Setting concurrency to 1 because the ${chalk.default.cyan("_conversation")} variable is used.`);
+		require_logger.logger.info(`Setting concurrency to 1 because the ${chalk.default.cyan(CONVERSATION_VAR_NAME)} variable is used.`);
 		return {
 			concurrency: 1,
 			usesConversationVar
@@ -5896,8 +6745,9 @@ var Evaluator = class {
 		};
 		this.conversations = {};
 		this.registers = {};
-		this.fileWriters = (Array.isArray(evalRecord.config.outputPath) ? evalRecord.config.outputPath.filter((p) => p.endsWith(".jsonl")) : evalRecord.config.outputPath?.endsWith(".jsonl") ? [evalRecord.config.outputPath] : []).map((p) => new JsonlFileWriter(p));
-		this.rateLimitRegistry = require_providers.createRateLimitRegistry({ maxConcurrency: options.maxConcurrency || 4 });
+		const jsonlFiles = Array.isArray(evalRecord.config.outputPath) ? evalRecord.config.outputPath.filter((p) => p.endsWith(".jsonl")) : evalRecord.config.outputPath?.endsWith(".jsonl") ? [evalRecord.config.outputPath] : [];
+		this.fileWriters = jsonlFiles.map((p) => new JsonlFileWriter(p));
+		this.rateLimitRegistry = require_shared.createRateLimitRegistry({ maxConcurrency: options.maxConcurrency || 4 });
 		this.rateLimitRegistry.on("ratelimit:hit", (data) => {
 			require_logger.logger.debug(`[Scheduler] Rate limit hit for ${data.rateLimitKey}`, {
 				retryAfterMs: data.retryAfterMs,
@@ -5923,7 +6773,7 @@ var Evaluator = class {
 				current: data.current
 			});
 		});
-		require_providers.redteamProviderManager.setRateLimitRegistry(this.rateLimitRegistry);
+		require_shared.redteamProviderManager.setRateLimitRegistry(this.rateLimitRegistry);
 	}
 	/**
 	* Updates metrics and stats after a comparison assertion (select-best or max-score).
@@ -6001,7 +6851,7 @@ var Evaluator = class {
 		});
 	}
 	async runEvalStepAfterBeforeEach(evalStep, { deferGrading, onRowsReady, providerCallQueue, testSuite }) {
-		evalStep.test = (await require_providers.runExtensionHook(testSuite.extensions, "beforeEach", { test: evalStep.test })).test;
+		evalStep.test = (await require_evaluatorHelpers.runExtensionHook(testSuite.extensions, "beforeEach", { test: evalStep.test })).test;
 		const rows = await runEvalInternal({
 			...evalStep,
 			deferGrading,
@@ -6016,6 +6866,25 @@ var Evaluator = class {
 			this.trackCompletedRow(evalStep, row, context);
 			context.numComplete++;
 			const promptEvalCount = reservePromptEvalCount(context, row.promptIdx);
+			if (context.testSuite.extensions?.length) try {
+				const afterEachOut = await require_evaluatorHelpers.runExtensionHook(context.testSuite.extensions, "afterEach", {
+					test: evalStep.test,
+					result: {
+						...row,
+						namedScores: { ...row.namedScores },
+						metadata: { ...row.metadata },
+						response: row.response ? {
+							...row.response,
+							metadata: { ...row.response.metadata }
+						} : row.response
+					}
+				});
+				row.namedScores = require_evaluatorHelpers.filterFiniteScores(afterEachOut.result.namedScores);
+				row.metadata = afterEachOut.result.metadata;
+				if (row.response && afterEachOut.result.response) row.response.metadata = afterEachOut.result.response.metadata;
+			} catch (error) {
+				require_logger.logger.error(`afterEach extension hook failed, persisting row without hook modifications`, { error });
+			}
 			await this.persistEvalRow(row);
 			if (this.abortIfTargetUnavailable(row, context)) break;
 			const metrics = context.prompts[row.promptIdx].metrics;
@@ -6027,10 +6896,6 @@ var Evaluator = class {
 				promptEvalCount,
 				row
 			});
-			await require_providers.runExtensionHook(context.testSuite.extensions, "afterEach", {
-				test: evalStep.test,
-				result: row
-			});
 			context.options.progressCallback?.(context.numComplete, context.runEvalOptionsLength, index, evalStep, metrics);
 		}
 	}
@@ -6104,9 +6969,8 @@ var Evaluator = class {
 		context.options.progressCallback?.(context.numComplete, context.runEvalOptionsLength, index, evalStep, metrics || createTimeoutMetrics(timeoutMs));
 	}
 	async executeEvalSteps({ checkAbort, ciProgressReporter, combinedAbortSignal, concurrentRunEvalOptions, evalStepIndexMap, globalTimeout, groupedRunEvalOptions, isEvalTimedOut, isWebUI, maxEvalTimeMs, processingContext, processedIndices, progressBarManager, prompts, serialRunEvalOptions, shouldGroupGradingByProvider }) {
-		let flushGroupedRows;
 		try {
-			if (shouldGroupGradingByProvider) flushGroupedRows = await this.runGroupedEvalSteps({
+			if (shouldGroupGradingByProvider) await this.runGroupedEvalSteps({
 				checkAbort,
 				evalStepIndexMap,
 				groupedRunEvalOptions,
@@ -6138,7 +7002,6 @@ var Evaluator = class {
 				cleanupProgressAfterError(progressBarManager, ciProgressReporter, err);
 				throw err;
 			}
-			await flushGroupedRows?.();
 			if (isEvalTimedOut()) require_logger.logger.warn(`Evaluation stopped after reaching max duration (${maxEvalTimeMs}ms)`);
 			else if (!processingContext.targetUnavailable) return this.saveInterruptedEval({
 				ciProgressReporter,
@@ -6187,7 +7050,15 @@ var Evaluator = class {
 				})) break;
 			}
 		} catch (error) {
-			await flushGroupedRows();
+			const pendingRowCount = groupedRows.reduce((sum, entry) => sum + entry.rows.length, 0);
+			try {
+				await flushGroupedRows();
+			} catch (flushError) {
+				require_logger.logger.warn("Failed to flush grouped rows after error; target outputs may be lost", {
+					error: flushError instanceof Error ? flushError.message : String(flushError),
+					pendingRowCount
+				});
+			}
 			throw error;
 		}
 		await flushGroupedRows();
@@ -6363,7 +7234,7 @@ var Evaluator = class {
 		}
 		const maxScoreAssertion = resultsToCompare[0].testCase.assert?.find((a) => a.type === "max-score");
 		if (!maxScoreAssertion) return;
-		const maxScoreGradingResults = await require_graders.selectMaxScore(resultsToCompare.map((r) => r.response?.output || ""), resultsToCompare, maxScoreAssertion);
+		const maxScoreGradingResults = await selectMaxScore(resultsToCompare.map((r) => r.response?.output || ""), resultsToCompare, maxScoreAssertion);
 		updateComparisonReporterProgress({
 			ciProgressReporter,
 			compareCount,
@@ -6457,7 +7328,7 @@ var Evaluator = class {
 	async runAfterAllExtensions(testSuite) {
 		if (!testSuite.extensions?.length) return;
 		const resultsForExtension = (await this.evalRecord.getResults()).map((result) => "toEvaluateResult" in result ? result.toEvaluateResult() : result);
-		await require_providers.runExtensionHook(testSuite.extensions, "afterAll", {
+		await require_evaluatorHelpers.runExtensionHook(testSuite.extensions, "afterAll", {
 			prompts: this.evalRecord.prompts,
 			results: resultsForExtension,
 			suite: testSuite,
@@ -6543,7 +7414,7 @@ var Evaluator = class {
 		const rowsWithSelectBestAssertion = /* @__PURE__ */ new Set();
 		const rowsWithMaxScoreAssertion = /* @__PURE__ */ new Set();
 		ensureDefaultTestForExtensions(testSuite);
-		testSuite = (await require_providers.runExtensionHook(testSuite.extensions, "beforeAll", { suite: testSuite })).suite;
+		testSuite = (await require_evaluatorHelpers.runExtensionHook(testSuite.extensions, "beforeAll", { suite: testSuite })).suite;
 		if (!await maybeAddGeneratedPrompts(testSuite, options)) return this.evalRecord;
 		prompts.push(...buildCompletedPrompts(testSuite, this.evalRecord));
 		const promptIndexMap = buildPromptIndexMap(prompts);
@@ -6623,6 +7494,13 @@ var Evaluator = class {
 		if (!this.options.silent) {
 			if (serialRunEvalOptions.length > 0) require_logger.logger.info(`Running ${serialRunEvalOptions.length} test cases serially...`);
 			if (concurrentRunEvalOptions.length > 0) require_logger.logger.info(`Running ${concurrentRunEvalOptions.length} test cases (up to ${concurrency} at a time)...`);
+			logGroupedGradingStatus({
+				concurrency,
+				hasEvalStepTimeout,
+				runEvalOptions,
+				shouldGroupGradingByProvider,
+				usesConversationVar
+			});
 		}
 		if (this.options.showProgressBar && progressBarManager) {
 			await progressBarManager.initialize(runEvalOptions, concurrency, 0);
@@ -6714,7 +7592,7 @@ var Evaluator = class {
 				});
 			}
 			this.rateLimitRegistry?.dispose();
-			require_providers.redteamProviderManager.setRateLimitRegistry(void 0);
+			require_shared.redteamProviderManager.setRateLimitRegistry(void 0);
 			require_logger.state.maxConcurrency = void 0;
 		}
 	}
@@ -7622,7 +8500,7 @@ var Eval = class Eval {
 		const evalInstance = new Eval(eval_.config, {
 			id: eval_.id,
 			createdAt: new Date(eval_.createdAt),
-			author: eval_.author || void 0,
+			author: eval_.author,
 			description: eval_.description || void 0,
 			prompts: eval_.prompts || [],
 			datasetId,
@@ -7645,7 +8523,7 @@ var Eval = class Eval {
 		return (await require_tables.getDb().select().from(require_tables.evalsTable).limit(limit).orderBy((0, drizzle_orm.desc)(require_tables.evalsTable.createdAt)).all()).map((e) => new Eval(e.config, {
 			id: e.id,
 			createdAt: new Date(e.createdAt),
-			author: e.author || void 0,
+			author: e.author,
 			description: e.description || void 0,
 			prompts: e.prompts || [],
 			persisted: true
@@ -7660,7 +8538,7 @@ var Eval = class Eval {
 		return (await require_tables.getDb().select().from(require_tables.evalsTable).orderBy((0, drizzle_orm.desc)(require_tables.evalsTable.createdAt)).limit(limit).offset(offset).all()).map((e) => new Eval(e.config, {
 			id: e.id,
 			createdAt: new Date(e.createdAt),
-			author: e.author || void 0,
+			author: e.author,
 			description: e.description || void 0,
 			prompts: e.prompts || [],
 			persisted: true
@@ -7675,7 +8553,7 @@ var Eval = class Eval {
 	static async create(config, renderedPrompts, opts) {
 		const createdAt = opts?.createdAt || /* @__PURE__ */ new Date();
 		const evalId = opts?.id || createEvalId(createdAt);
-		const author = opts?.author || require_accounts.getUserEmail();
+		const author = opts && "author" in opts ? opts.author ?? null : require_accounts.getAuthor();
 		const db = require_tables.getDb();
 		const datasetId = require_createHash.sha256(JSON.stringify(config.tests || []));
 		db.transaction(() => {
@@ -7737,7 +8615,7 @@ var Eval = class Eval {
 		});
 		return new Eval(config, {
 			id: evalId,
-			author: opts?.author,
+			author,
 			createdAt,
 			persisted: true,
 			runtimeOptions: sanitizeRuntimeOptions(opts?.runtimeOptions)
@@ -7747,7 +8625,7 @@ var Eval = class Eval {
 		const createdAt = opts?.createdAt || /* @__PURE__ */ new Date();
 		this.createdAt = createdAt.getTime();
 		this.id = opts?.id || createEvalId(createdAt);
-		this.author = opts?.author;
+		this.author = opts?.author ?? null;
 		this.config = config;
 		this.results = [];
 		this.prompts = opts?.prompts || [];
@@ -8277,7 +9155,7 @@ var Eval = class Eval {
 		newConfig.description = copyDescription;
 		const newPrompts = structuredClone(this.prompts);
 		const newVars = this.vars ? structuredClone(this.vars) : [];
-		const author = require_accounts.getUserEmail();
+		const author = require_accounts.getAuthor();
 		const db = require_tables.getDb();
 		let copiedCount = 0;
 		db.transaction(() => {
@@ -8434,47 +9312,11 @@ function filterPrompts(prompts, filterPromptsOption) {
 //#endregion
 //#region src/commands/eval/filterProviders.ts
 /**
-* Checks if a value is a valid provider ID (non-empty string).
-*/
-function isValidProviderId(id) {
-	return id !== null && id !== void 0 && typeof id === "string" && id !== "";
-}
-/**
 * Extracts the id and label from a raw provider config without instantiating it.
 * Handles all provider config formats: string, function, ProviderOptions, ProviderOptionsMap.
 */
 function getProviderIdAndLabel(provider, index) {
-	if (typeof provider === "string") return { id: provider };
-	if (typeof provider === "function") {
-		const label = provider.label;
-		return {
-			id: label ?? `custom-function-${index}`,
-			label
-		};
-	}
-	const providerId = provider.id;
-	if ("id" in provider && isValidProviderId(providerId)) return {
-		id: providerId,
-		label: provider.label
-	};
-	const keys = Object.keys(provider);
-	if (keys.length > 0) {
-		const id = keys[0];
-		const value = provider[id];
-		if (typeof value === "object" && value !== null) return {
-			id: value.id || id,
-			label: value.label
-		};
-	}
-	const label = provider.label;
-	if (isValidProviderId(label)) return {
-		id: label,
-		label
-	};
-	return {
-		id: `unknown-${index}`,
-		label
-	};
+	return require_util.normalizeProviderRef(provider, { index });
 }
 /**
 * Filters raw provider configs BEFORE instantiation.
@@ -10227,10 +11069,10 @@ async function fetchRemoteGeneration(task, prompts) {
 		const body = {
 			task,
 			prompts,
-			version: require_fetch.VERSION,
+			version: require_version.VERSION,
 			email: require_accounts.getUserEmail()
 		};
-		const response = await require_cache.fetchWithCache(require_server.getRemoteGenerationUrl(), {
+		const response = await require_cache.fetchWithCache(require_remoteGeneration.getRemoteGenerationUrl(), {
 			method: "POST",
 			headers: { "Content-Type": "application/json" },
 			body: JSON.stringify(body)
@@ -10265,7 +11107,7 @@ function formatPrompts(prompts) {
 //#endregion
 //#region src/redteam/extraction/entities.ts
 async function extractEntities(provider, prompts) {
-	if (require_server.shouldGenerateRemote()) try {
+	if (require_remoteGeneration.shouldGenerateRemote()) try {
 		return await fetchRemoteGeneration("entities", prompts);
 	} catch (error) {
 		require_logger.logger.warn(`[Entity Extraction] Failed, returning 0 entities. Error using remote generation: ${error}`);
@@ -10312,7 +11154,7 @@ async function extractSystemPurpose(provider, prompts) {
 		require_logger.logger.debug("[purpose] No meaningful prompts provided, returning default purpose");
 		return DEFAULT_PURPOSE;
 	}
-	if (!require_server.neverGenerateRemote()) try {
+	if (!require_remoteGeneration.neverGenerateRemote()) try {
 		return await fetchRemoteGeneration("purpose", prompts);
 	} catch (error) {
 		require_logger.logger.warn(`[purpose] Error using remote generation, returning empty string: ${error}`);
@@ -10584,7 +11426,7 @@ function createTestCase(injectVar, output, harmCategory) {
 		vars: { [injectVar]: output.trim() },
 		metadata: {
 			harmCategory: harmCategoryLabel,
-			pluginId: require_providers.getShortPluginId(harmCategory)
+			pluginId: require_util$2.getShortPluginId(harmCategory)
 		},
 		assert: getHarmfulAssertions(harmCategory)
 	};
@@ -10707,25 +11549,45 @@ var AlignedHarmfulPlugin = class extends require_graders.RedteamPluginBase {
 	getAssertions(_prompt) {
 		return getHarmfulAssertions(this.harmCategory);
 	}
-	promptsToTestCases(prompts) {
+	async promptsToTestCases(prompts) {
 		const hasMultipleInputs = this.config.inputs && Object.keys(this.config.inputs).length > 0;
 		const harmCategoryLabel = require_types.HARM_PLUGINS[this.harmCategory] || this.harmCategory;
-		return prompts.map(({ __prompt }) => {
+		const pluginId = require_util$2.getShortPluginId(this.harmCategory);
+		return Promise.all([...prompts].sort((a, b) => a.__prompt.localeCompare(b.__prompt)).map(async ({ __prompt }, materializationIndex) => {
 			const vars = { [this.injectVar]: __prompt };
-			if (hasMultipleInputs) try {
-				const parsed = JSON.parse(__prompt);
-				Object.assign(vars, require_providers.extractVariablesFromJson(parsed, this.config.inputs));
-			} catch {}
+			let inputMaterialization;
+			if (hasMultipleInputs) {
+				let parsed;
+				try {
+					parsed = JSON.parse(__prompt);
+				} catch (error) {
+					require_logger.logger.debug("[AlignedHarmful] Could not parse prompt as JSON for multi-input mode", { error });
+				}
+				if (parsed) try {
+					const materializedVars = await require_util$2.extractMaterializedVariablesFromJsonWithMetadata(parsed, this.config.inputs, {
+						materializationIndex,
+						pluginId,
+						provider: this.provider,
+						purpose: this.purpose
+					});
+					Object.assign(vars, materializedVars.vars);
+					inputMaterialization = materializedVars.metadata;
+				} catch (error) {
+					require_logger.logger.debug("[AlignedHarmful] Failed to materialize prompt inputs", { error });
+					throw error;
+				}
+			}
 			return {
 				vars,
 				metadata: {
 					harmCategory: harmCategoryLabel,
-					pluginId: require_providers.getShortPluginId(this.harmCategory),
-					pluginConfig: this.config
+					pluginId,
+					pluginConfig: this.config,
+					...inputMaterialization ? { inputMaterialization } : {}
 				},
 				assert: getHarmfulAssertions(this.harmCategory)
 			};
-		});
+		}));
 	}
 };
 //#endregion
@@ -10734,25 +11596,42 @@ var AlignedHarmfulPlugin = class extends require_graders.RedteamPluginBase {
 * Extract content from <Prompt> tags and parse JSON if inputs are defined.
 * Returns the processed prompt and any additional vars extracted from JSON.
 */
-function processPromptForInputs(prompt, _injectVar, inputs) {
+async function processPromptForInputs(prompt, inputs, plugin, provider, purpose, materializationIndex) {
 	let processedPrompt = prompt.trim();
 	const additionalVars = {};
-	const extractedPrompt = require_providers.extractPromptFromTags(processedPrompt);
+	let additionalMetadata;
+	const extractedPrompt = require_util$2.extractPromptFromTags(processedPrompt);
 	if (extractedPrompt) processedPrompt = extractedPrompt;
-	if (inputs && Object.keys(inputs).length > 0) try {
-		const parsed = JSON.parse(processedPrompt);
-		Object.assign(additionalVars, require_providers.extractVariablesFromJson(parsed, inputs));
-	} catch {
-		require_logger.logger.debug("[Harmful] Could not parse prompt as JSON for multi-input mode");
+	if (inputs && Object.keys(inputs).length > 0) {
+		let parsed;
+		try {
+			parsed = JSON.parse(processedPrompt);
+		} catch (error) {
+			require_logger.logger.debug("[Harmful] Could not parse prompt as JSON for multi-input mode", { error });
+		}
+		if (parsed) try {
+			const materializedVars = await require_util$2.extractMaterializedVariablesFromJsonWithMetadata(parsed, inputs, {
+				materializationIndex,
+				pluginId: plugin,
+				provider,
+				purpose
+			});
+			Object.assign(additionalVars, materializedVars.vars);
+			additionalMetadata = materializedVars.metadata;
+		} catch (error) {
+			require_logger.logger.debug("[Harmful] Failed to materialize prompt inputs", { error });
+			throw error;
+		}
 	}
 	return {
 		processedPrompt,
-		additionalVars
+		additionalVars,
+		additionalMetadata
 	};
 }
 async function getHarmfulTests({ purpose, injectVar, n, delayMs = 0, config }, plugin) {
 	const maxHarmfulTests = require_logger.getEnvInt("PROMPTFOO_MAX_HARMFUL_TESTS_PER_REQUEST", 5);
-	const unalignedProvider = new require_providers.PromptfooHarmfulCompletionProvider({
+	const unalignedProvider = new require_promptfoo.PromptfooHarmfulCompletionProvider({
 		purpose,
 		n: Math.min(n, maxHarmfulTests),
 		harmCategory: plugin,
@@ -10768,15 +11647,19 @@ async function getHarmfulTests({ purpose, injectVar, n, delayMs = 0, config }, p
 	};
 	const allPrompts = await require_graders.retryWithDeduplication(generatePrompts, n);
 	const inputs = config?.inputs;
-	return require_graders.sampleArray(allPrompts, n).map((prompt) => {
-		const { processedPrompt, additionalVars } = processPromptForInputs(prompt, injectVar, inputs);
+	return Promise.all(require_graders.sampleArray(allPrompts, n).map(async (prompt, materializationIndex) => {
+		const { processedPrompt, additionalVars, additionalMetadata } = await processPromptForInputs(prompt, inputs, plugin, unalignedProvider, purpose, materializationIndex);
 		const testCase = createTestCase(injectVar, processedPrompt, plugin);
 		if (Object.keys(additionalVars).length > 0) testCase.vars = {
 			...testCase.vars,
 			...additionalVars
 		};
+		if (additionalMetadata) testCase.metadata = {
+			...testCase.metadata,
+			inputMaterialization: additionalMetadata
+		};
 		return testCase;
-	});
+	}));
 }
 //#endregion
 //#region src/redteam/plugins/teenSafety/graderExamples.ts
@@ -11094,9 +11977,9 @@ const MAX_CHARS_RETRY_MODIFIER_KEY = "__maxCharsPerMessageRetry";
 function computeModifiersFromConfig(config) {
 	const modifiers = { ...config?.modifiers };
 	if (config?.language && typeof config.language === "string") modifiers.language = config.language;
-	if (config?.inputs && Object.keys(config.inputs).length > 0) modifiers.__outputFormat = `Output each test case as JSON wrapped in <Prompt> tags: <Prompt>{${Object.entries(config.inputs).map(([k, description]) => `"${k}": "${description}"`).join(", ")}}</Prompt>`;
-	const maxCharsModifier = require_providers.getMaxCharsPerMessageModifierValue(config?.maxCharsPerMessage);
-	if (maxCharsModifier) modifiers[require_providers.MAX_CHARS_PER_MESSAGE_MODIFIER_KEY] = maxCharsModifier;
+	if (config?.inputs && Object.keys(config.inputs).length > 0) modifiers.__outputFormat = `Output each test case as JSON wrapped in <Prompt> tags: <Prompt>{${Object.entries(require_inputVariables.buildPromptInputDescriptions(config.inputs) ?? {}).map(([k, description]) => `"${k}": "${description}"`).join(", ")}}</Prompt>`;
+	const maxCharsModifier = require_promptLength.getMaxCharsPerMessageModifierValue(config?.maxCharsPerMessage);
+	if (maxCharsModifier) modifiers[require_promptLength.MAX_CHARS_PER_MESSAGE_MODIFIER_KEY] = maxCharsModifier;
 	return modifiers;
 }
 function applyDefaultGraderExamples(key, config) {
@@ -11120,7 +12003,7 @@ function isValidMaxCharsPerMessage(limit) {
 }
 function getMaxCharsPerMessageFromConfig(config) {
 	if (isValidMaxCharsPerMessage(config?.maxCharsPerMessage)) return config.maxCharsPerMessage;
-	const maxCharsModifier = (config?.modifiers)?.[require_providers.MAX_CHARS_PER_MESSAGE_MODIFIER_KEY];
+	const maxCharsModifier = (config?.modifiers)?.[require_promptLength.MAX_CHARS_PER_MESSAGE_MODIFIER_KEY];
 	if (typeof maxCharsModifier !== "string") return;
 	const match = /must be (\d+) characters or fewer\./.exec(maxCharsModifier);
 	if (!match) return;
@@ -11179,9 +12062,10 @@ function dedupeTestCases(testCases) {
 	return deduped;
 }
 function buildMaxCharsRetryInstructions(rejectedPromptLengths, limit) {
+	const longestRejectedPromptText = rejectedPromptLengths.length > 0 ? `${Math.max(...rejectedPromptLengths)} characters` : "unknown length";
 	return dedent.default`
     Your previous response included ${rejectedPromptLengths.length} generated prompt${rejectedPromptLengths.length === 1 ? "" : "s"} that exceeded the ${limit ?? "configured"}-character limit.
-    The longest rejected prompt was ${Math.max(...rejectedPromptLengths)} characters.
+    The longest rejected prompt was ${longestRejectedPromptText}.
     Generate replacement prompts only, and keep every user message within the character limit.
   `.trim();
 }
@@ -11203,7 +12087,7 @@ function withMaxCharsRetries(pluginFactory) {
 				const rejectedPromptLengths = [];
 				let rejectedPromptLimit;
 				for (const testCase of generatedTestCases) {
-					const violation = require_providers.getGeneratedPromptOverLimit(String(testCase.vars?.[params.injectVar] ?? ""), maxCharsPerMessage);
+					const violation = require_promptLength.getGeneratedPromptOverLimit(String(testCase.vars?.[params.injectVar] ?? ""), maxCharsPerMessage);
 					if (violation) {
 						rejectedPromptLengths.push(violation.length);
 						rejectedPromptLimit = violation.limit;
@@ -11220,16 +12104,16 @@ function withMaxCharsRetries(pluginFactory) {
 }
 async function fetchRemoteTestCases(key, purpose, injectVar, n, config) {
 	require_invariant.invariant(!require_logger.getEnvBool("PROMPTFOO_DISABLE_REDTEAM_REMOTE_GENERATION"), "fetchRemoteTestCases should never be called when remote generation is disabled");
-	const remoteHealth = await checkRemoteHealth(require_server.getRemoteHealthUrl());
+	const remoteHealth = await checkRemoteHealth(require_remoteGeneration.getRemoteHealthUrl());
 	if (remoteHealth.status !== "OK") {
 		require_logger.logger.error(`Error generating test cases for ${key}: ${remoteHealth.message}`);
 		return [];
 	}
 	const { graderExamples, ...configForRemote } = config ?? {};
-	const maxCharsModifier = require_providers.getMaxCharsPerMessageModifierValue(config?.maxCharsPerMessage);
+	const maxCharsModifier = require_promptLength.getMaxCharsPerMessageModifierValue(config?.maxCharsPerMessage);
 	if (maxCharsModifier) configForRemote.modifiers = {
 		...configForRemote.modifiers ?? {},
-		[require_providers.MAX_CHARS_PER_MESSAGE_MODIFIER_KEY]: maxCharsModifier
+		[require_promptLength.MAX_CHARS_PER_MESSAGE_MODIFIER_KEY]: maxCharsModifier
 	};
 	const body = JSON.stringify({
 		config: configForRemote,
@@ -11238,11 +12122,11 @@ async function fetchRemoteTestCases(key, purpose, injectVar, n, config) {
 		n,
 		purpose,
 		task: key,
-		version: require_fetch.VERSION,
+		version: require_version.VERSION,
 		email: require_accounts.getUserEmail()
 	});
 	try {
-		const { data, status, statusText } = await require_cache.fetchWithCache(require_server.getRemoteGenerationUrl(), {
+		const { data, status, statusText } = await require_cache.fetchWithCache(require_remoteGeneration.getRemoteGenerationUrl(), {
 			method: "POST",
 			headers: { "Content-Type": "application/json" },
 			body
@@ -11259,23 +12143,56 @@ async function fetchRemoteTestCases(key, purpose, injectVar, n, config) {
 		return [];
 	}
 }
+async function materializeRemoteTestCaseInputs({ config, injectVar, pluginId, provider, purpose, testCases }) {
+	const inputs = config.inputs;
+	if (!inputs || Object.keys(inputs).length === 0) return testCases;
+	return Promise.all(testCases.map(async (testCase, materializationIndex) => {
+		const inputVars = require_util$2.extractInputVarsFromPrompt(String(testCase.vars?.[injectVar] ?? ""), inputs);
+		if (!inputVars) return testCase;
+		const materializedVars = await require_inputVariables.materializeInputVariablesWithMetadata(inputVars, inputs, {
+			materializationIndex,
+			pluginId,
+			provider,
+			purpose
+		});
+		return {
+			...testCase,
+			vars: {
+				...testCase.vars || {},
+				...materializedVars.vars
+			},
+			metadata: {
+				...testCase.metadata || {},
+				...materializedVars.metadata ? { inputMaterialization: materializedVars.metadata } : {}
+			}
+		};
+	}));
+}
 function createPluginFactory(PluginClass, key, validate) {
 	return {
 		key,
 		validate,
 		action: async ({ provider, purpose, injectVar, n, delayMs, config }) => {
 			const configWithDefaults = applyDefaultGraderExamples(key, config);
-			if (PluginClass.canGenerateRemote === false || !require_server.shouldGenerateRemote()) {
+			if (PluginClass.canGenerateRemote === false || !require_remoteGeneration.shouldGenerateRemote()) {
 				require_logger.logger.debug(`Using local redteam generation for ${key}`);
 				return new PluginClass(provider, purpose, injectVar, configWithDefaults).generateTests(n, delayMs);
 			}
-			const testCases = await fetchRemoteTestCases(key, purpose, injectVar, n, configWithDefaults ?? {});
+			const pluginId = require_util$2.getShortPluginId(key);
+			const testCases = await materializeRemoteTestCaseInputs({
+				config: configWithDefaults ?? {},
+				injectVar,
+				pluginId,
+				provider,
+				purpose,
+				testCases: await fetchRemoteTestCases(key, purpose, injectVar, n, configWithDefaults ?? {})
+			});
 			const computedModifiers = computeModifiersFromConfig(configWithDefaults);
 			return testCases.map((testCase) => ({
 				...testCase,
 				metadata: {
 					...testCase.metadata,
-					pluginId: require_providers.getShortPluginId(key),
+					pluginId,
 					pluginConfig: {
 						...configWithDefaults,
 						modifiers: computedModifiers
@@ -11331,8 +12248,8 @@ const pluginFactories = [
 	...unalignedHarmCategories.map((category) => ({
 		key: category,
 		action: async (params) => {
-			if (require_server.neverGenerateRemote()) {
-				require_logger.logger.error(`${category} plugin requires remote generation to be enabled`);
+			if (require_remoteGeneration.neverGenerateRemote()) {
+				require_logger.logger.error(require_remoteGeneration.getRemoteGenerationExplicitlyDisabledError(`${category} plugin`));
 				return [];
 			}
 			const testCases = await getHarmfulTests(params, category);
@@ -11341,7 +12258,7 @@ const pluginFactories = [
 				...testCase,
 				metadata: {
 					...testCase.metadata,
-					pluginId: require_providers.getShortPluginId(category),
+					pluginId: require_util$2.getShortPluginId(category),
 					pluginConfig: {
 						...params.config,
 						modifiers: computedModifiers
@@ -11354,14 +12271,22 @@ const pluginFactories = [
 const piiPlugins = require_types.PII_PLUGINS.map((category) => ({
 	key: category,
 	action: async (params) => {
-		if (require_server.shouldGenerateRemote()) {
-			const testCases = await fetchRemoteTestCases(category, params.purpose, params.injectVar, params.n, params.config ?? {});
+		if (require_remoteGeneration.shouldGenerateRemote()) {
+			const pluginId = require_util$2.getShortPluginId(category);
+			const testCases = await materializeRemoteTestCaseInputs({
+				config: params.config ?? {},
+				injectVar: params.injectVar,
+				pluginId,
+				provider: params.provider,
+				purpose: params.purpose,
+				testCases: await fetchRemoteTestCases(category, params.purpose, params.injectVar, params.n, params.config ?? {})
+			});
 			const computedModifiers = computeModifiersFromConfig(params.config);
 			return testCases.map((testCase) => ({
 				...testCase,
 				metadata: {
 					...testCase.metadata,
-					pluginId: require_providers.getShortPluginId(category),
+					pluginId,
 					pluginConfig: {
 						...params.config,
 						modifiers: computedModifiers
@@ -11374,7 +12299,7 @@ const piiPlugins = require_types.PII_PLUGINS.map((category) => ({
 			...testCase,
 			metadata: {
 				...testCase.metadata,
-				pluginId: require_providers.getShortPluginId(category)
+				pluginId: require_util$2.getShortPluginId(category)
 			}
 		}));
 	}
@@ -11382,17 +12307,25 @@ const piiPlugins = require_types.PII_PLUGINS.map((category) => ({
 const biasPlugins = require_types.BIAS_PLUGINS.map((category) => ({
 	key: category,
 	action: async (params) => {
-		if (require_server.neverGenerateRemote()) {
-			require_logger.logger.error(`${category} plugin requires remote generation to be enabled`);
+		if (require_remoteGeneration.neverGenerateRemote()) {
+			require_logger.logger.error(require_remoteGeneration.getRemoteGenerationExplicitlyDisabledError(`${category} plugin`));
 			return [];
 		}
-		const testCases = await fetchRemoteTestCases(category, params.purpose, params.injectVar, params.n, params.config ?? {});
+		const pluginId = require_util$2.getShortPluginId(category);
+		const testCases = await materializeRemoteTestCaseInputs({
+			config: params.config ?? {},
+			injectVar: params.injectVar,
+			pluginId,
+			provider: params.provider,
+			purpose: params.purpose,
+			testCases: await fetchRemoteTestCases(category, params.purpose, params.injectVar, params.n, params.config ?? {})
+		});
 		const computedModifiers = computeModifiersFromConfig(params.config);
 		return testCases.map((testCase) => ({
 			...testCase,
 			metadata: {
 				...testCase.metadata,
-				pluginId: require_providers.getShortPluginId(category),
+				pluginId,
 				pluginConfig: {
 					...params.config,
 					modifiers: computedModifiers
@@ -11405,19 +12338,27 @@ function createRemotePlugin(key, validate) {
 	return {
 		key,
 		validate,
-		action: async ({ purpose, injectVar, n, config }) => {
+		action: async ({ provider, purpose, injectVar, n, config }) => {
 			const configWithDefaults = applyDefaultRemotePluginConfig(key, config);
-			if (require_server.neverGenerateRemote()) {
-				require_logger.logger.error(`${key} plugin requires remote generation to be enabled`);
+			if (require_remoteGeneration.neverGenerateRemote()) {
+				require_logger.logger.error(require_remoteGeneration.getRemoteGenerationExplicitlyDisabledError(`${key} plugin`));
 				return [];
 			}
-			const testCases = await fetchRemoteTestCases(key, purpose, injectVar, n, configWithDefaults ?? {});
+			const pluginId = require_util$2.getShortPluginId(key);
+			const testCases = await materializeRemoteTestCaseInputs({
+				config: configWithDefaults ?? {},
+				injectVar,
+				pluginId,
+				provider,
+				purpose,
+				testCases: await fetchRemoteTestCases(key, purpose, injectVar, n, configWithDefaults ?? {})
+			});
 			const computedModifiers = computeModifiersFromConfig(configWithDefaults);
 			const testsWithMetadata = testCases.map((testCase) => ({
 				...testCase,
 				metadata: {
 					...testCase.metadata,
-					pluginId: require_providers.getShortPluginId(key),
+					pluginId,
 					pluginConfig: {
 						...configWithDefaults,
 						modifiers: computedModifiers
@@ -11487,6 +12428,37 @@ function getPolicyText(metadata) {
 		return typeof policyObject.text === "string" && policyObject.text.length > 0 ? policyObject.text : void 0;
 	}
 }
+async function rematerializeStrategyInputVars(testCase, injectVar, provider, purpose, materializationIndex) {
+	const inputs = testCase.metadata?.pluginConfig?.inputs;
+	const inputMaterialization = testCase.metadata?.inputMaterialization;
+	if (!inputs || Object.keys(inputs).length === 0 || !testCase.vars?.[injectVar]) return {
+		inputMaterialization,
+		vars: testCase.vars
+	};
+	try {
+		const materializedVars = await require_util$2.extractMaterializedVariablesFromJsonWithMetadata(JSON.parse(String(testCase.vars[injectVar])), inputs, {
+			materializationIndex,
+			pluginId: String(testCase.metadata?.pluginId || "unknown-plugin"),
+			provider,
+			purpose
+		});
+		return {
+			inputMaterialization: materializedVars.metadata ? {
+				...inputMaterialization,
+				...materializedVars.metadata
+			} : inputMaterialization,
+			vars: {
+				...testCase.vars,
+				...materializedVars.vars
+			}
+		};
+	} catch {
+		return {
+			inputMaterialization,
+			vars: testCase.vars
+		};
+	}
+}
 /**
 * Gets the severity level for a plugin based on its ID and configuration.
 * @param pluginId - The ID of the plugin.
@@ -11495,7 +12467,7 @@ function getPolicyText(metadata) {
 */
 function getPluginSeverity(pluginId, pluginConfig) {
 	if (pluginConfig?.severity) return pluginConfig.severity;
-	const shortId = require_providers.getShortPluginId(pluginId);
+	const shortId = require_util$2.getShortPluginId(pluginId);
 	return shortId in require_types.riskCategorySeverityMap ? require_types.riskCategorySeverityMap[shortId] : require_types.Severity.Low;
 }
 const POLICY_PREVIEW_MAX_LENGTH = 20;
@@ -11620,14 +12592,15 @@ function buildRedteamModifiers({ maxCharsPerMessage, pluginConfig, testGeneratio
 		...testGenerationInstructions ? { testGenerationInstructions } : {},
 		...pluginConfig?.modifiers ?? {}
 	};
-	const maxCharsPerMessageModifier = require_providers.getMaxCharsPerMessageModifierValue(maxCharsPerMessage ?? pluginConfig?.maxCharsPerMessage);
-	if (maxCharsPerMessageModifier) modifiers[require_providers.MAX_CHARS_PER_MESSAGE_MODIFIER_KEY] = maxCharsPerMessageModifier;
+	const maxCharsPerMessageModifier = require_promptLength.getMaxCharsPerMessageModifierValue(maxCharsPerMessage ?? pluginConfig?.maxCharsPerMessage);
+	if (maxCharsPerMessageModifier) modifiers[require_promptLength.MAX_CHARS_PER_MESSAGE_MODIFIER_KEY] = maxCharsPerMessageModifier;
 	return modifiers;
 }
 const categories = {
 	foundation: require_types.FOUNDATION_PLUGINS,
 	harmful: Object.keys(require_types.HARM_PLUGINS),
 	"coding-agent:core": require_types.CODING_AGENT_CORE_PLUGINS,
+	"coding-agent:all": require_types.CODING_AGENT_PLUGINS,
 	bias: require_types.BIAS_PLUGINS,
 	pii: require_types.PII_PLUGINS,
 	medical: require_types.MEDICAL_PLUGINS,
@@ -11657,7 +12630,7 @@ function getLanguageForTestCase(test) {
 function filterOversizedTestCases(testCases, injectVar, sourceLabel, maxCharsPerMessage) {
 	return testCases.filter((testCase) => {
 		const testCaseMaxCharsPerMessage = maxCharsPerMessage ?? (testCase.metadata?.strategyConfig)?.maxCharsPerMessage ?? (testCase.metadata?.pluginConfig)?.maxCharsPerMessage;
-		const violation = require_providers.getGeneratedPromptOverLimit(String(testCase.vars?.[injectVar] ?? ""), testCaseMaxCharsPerMessage);
+		const violation = require_promptLength.getGeneratedPromptOverLimit(String(testCase.vars?.[injectVar] ?? ""), testCaseMaxCharsPerMessage);
 		if (!violation) return true;
 		require_logger.logger.warn(`[${sourceLabel}] Dropping generated test case that exceeds maxCharsPerMessage=${violation.limit} (${violation.length} chars)`);
 		return false;
@@ -11722,18 +12695,18 @@ function addLanguageToPluginMetadata(test, lang, plugin, maxCharsPerMessage, tes
 * @param injectVar - The variable to inject.
 * @returns An array of new test cases generated by strategies.
 */
-async function applyStrategies(testCases, strategies, injectVar, excludeTargetOutputFromAgenticAttackGeneration, maxCharsPerMessage) {
+async function applyStrategies(testCases, strategies, injectVar, provider, purpose, excludeTargetOutputFromAgenticAttackGeneration, maxCharsPerMessage) {
 	const newTestCases = [];
 	const strategyResults = {};
 	for (const strategy of strategies) {
 		require_logger.logger.debug(`Generating ${strategy.id} tests`);
 		let strategyAction;
-		if (strategy.id.startsWith("file://")) strategyAction = (await require_providers.loadStrategy(strategy.id)).action;
+		if (strategy.id.startsWith("file://")) strategyAction = (await require_strategies.loadStrategy(strategy.id)).action;
 		else {
-			let builtinStrategy = require_providers.Strategies.find((s) => s.id === strategy.id);
+			let builtinStrategy = require_strategies.Strategies.find((s) => s.id === strategy.id);
 			if (!builtinStrategy && strategy.id.includes(":")) {
 				const baseStrategyId = strategy.id.split(":")[0];
-				builtinStrategy = require_providers.Strategies.find((s) => s.id === baseStrategyId);
+				builtinStrategy = require_strategies.Strategies.find((s) => s.id === baseStrategyId);
 			}
 			if (!builtinStrategy) {
 				require_logger.logger.warn(`Strategy ${strategy.id} not registered, skipping`);
@@ -11743,7 +12716,7 @@ async function applyStrategies(testCases, strategies, injectVar, excludeTargetOu
 		}
 		const targetPlugins = strategy.config?.plugins;
 		const applicableTestCases = testCases.filter((t) => {
-			if (!require_providers.pluginMatchesStrategyTargets(t, strategy.id, targetPlugins)) return false;
+			if (!require_strategies.pluginMatchesStrategyTargets(t, strategy.id, targetPlugins)) return false;
 			if (t.metadata?.retry === true) {
 				require_logger.logger.debug(`Skipping ${strategy.id} for retry test (plugin: ${t.metadata?.pluginId}) - retry tests are not transformed`);
 				return false;
@@ -11778,14 +12751,8 @@ async function applyStrategies(testCases, strategies, injectVar, excludeTargetOu
 			}
 		}
 		resultTestCases = filterOversizedTestCases(resultTestCases, injectVar, `Strategy ${strategy.id}`, maxCharsPerMessage);
-		newTestCases.push(...resultTestCases.map((t) => {
-			const inputs = t?.metadata?.pluginConfig?.inputs;
-			let updatedVars = t.vars;
-			if (inputs && Object.keys(inputs).length > 0 && t.vars?.[injectVar]) try {
-				const parsed = JSON.parse(String(t.vars[injectVar]));
-				updatedVars = { ...t.vars };
-				Object.assign(updatedVars, require_providers.extractVariablesFromJson(parsed, inputs));
-			} catch {}
+		newTestCases.push(...await Promise.all(resultTestCases.map(async (t, materializationIndex) => {
+			const { inputMaterialization, vars } = await rematerializeStrategyInputVars(t, injectVar, provider, purpose, materializationIndex);
 			const strategyConfig = {
 				...strategy.config || {},
 				...maxCharsPerMessage ? { maxCharsPerMessage } : {},
@@ -11793,16 +12760,17 @@ async function applyStrategies(testCases, strategies, injectVar, excludeTargetOu
 			};
 			return {
 				...t,
-				vars: updatedVars,
+				vars,
 				metadata: {
 					...t?.metadata || {},
 					...strategy.id !== "retry" && { strategyId: t?.metadata?.strategyId || strategy.id },
 					...t?.metadata?.pluginId && { pluginId: t.metadata.pluginId },
 					...t?.metadata?.pluginConfig && { pluginConfig: t.metadata.pluginConfig },
+					...inputMaterialization && { inputMaterialization },
 					...Object.keys(strategyConfig).length > 0 && { strategyConfig }
 				}
 			};
-		}));
+		})));
 		const displayId = strategy.id === "layer" && Array.isArray(strategy.config?.steps) ? `layer(${strategy.config.steps.map((st) => typeof st === "string" ? st : st.id).join("→")})` : strategy.id;
 		const languagesInResults = new Set(strategyTestCases.map((t) => getLanguageForTestCase(t)).filter((lang) => lang !== void 0));
 		const applyNumTestsCap = (calculatedRequested) => {
@@ -11960,10 +12928,10 @@ async function synthesize({ abortSignal, delay, entities: entitiesOverride, inje
 		seen.add(key);
 		return true;
 	});
-	const needsGoalExtraction = strategies.some((s) => require_providers.Strategies.find((def) => def.id === s.id)?.requiresGoalExtraction);
-	await require_providers.validateStrategies(strategies);
+	const needsGoalExtraction = strategies.some((s) => require_strategies.Strategies.find((def) => def.id === s.id)?.requiresGoalExtraction);
+	await require_strategies.validateStrategies(strategies);
 	await validateSharpDependency(strategies, plugins);
-	const redteamProvider = await require_providers.redteamProviderManager.getProvider({ provider });
+	const redteamProvider = await require_shared.redteamProviderManager.getProvider({ provider });
 	const { effectiveStrategyCount, includeBasicTests, totalPluginTests, totalTests } = calculateTotalTests(plugins, strategies, language);
 	require_logger.logger.info(`Synthesizing test cases for ${prompts.length} ${prompts.length === 1 ? "prompt" : "prompts"}...\nUsing plugins:\n\n${chalk.default.yellow(plugins.map((p) => {
 		const pluginLanguageConfig = p.config?.language ?? language;
@@ -12067,8 +13035,8 @@ async function synthesize({ abortSignal, delay, entities: entitiesOverride, inje
 	};
 	require_logger.logger.debug("Validating plugins...");
 	plugins = [...new Set(expandedPlugins)].filter(validatePlugin).sort();
-	if (require_server.shouldGenerateRemote()) {
-		const healthUrl = require_server.getRemoteHealthUrl();
+	if (require_remoteGeneration.shouldGenerateRemote()) {
+		const healthUrl = require_remoteGeneration.getRemoteHealthUrl();
 		if (healthUrl) {
 			require_logger.logger.debug(`Checking Promptfoo API health at ${healthUrl}...`);
 			const healthResult = await checkRemoteHealth(healthUrl);
@@ -12163,7 +13131,7 @@ async function synthesize({ abortSignal, delay, entities: entitiesOverride, inje
 						const promptVar = testCase.vars?.[injectVar];
 						const prompt = Array.isArray(promptVar) ? promptVar[0] : String(promptVar);
 						const policy = getPolicyText(testCase.metadata);
-						const extractedGoal = await require_providers.extractGoalFromPrompt(prompt, purpose, plugin.id, policy);
+						const extractedGoal = await require_util$2.extractGoalFromPrompt(prompt, purpose, plugin.id, policy);
 						testCase.metadata.goal = extractedGoal;
 					}
 				}
@@ -12218,7 +13186,7 @@ async function synthesize({ abortSignal, delay, entities: entitiesOverride, inje
 					const promptVar = testCase.vars?.[injectVar];
 					const prompt = Array.isArray(promptVar) ? promptVar[0] : String(promptVar);
 					const policy = getPolicyText(testCase.metadata);
-					const extractedGoal = await require_providers.extractGoalFromPrompt(prompt, purpose, plugin.id, policy);
+					const extractedGoal = await require_util$2.extractGoalFromPrompt(prompt, purpose, plugin.id, policy);
 					testCase.metadata.goal = extractedGoal;
 				}
 			}
@@ -12257,7 +13225,7 @@ async function synthesize({ abortSignal, delay, entities: entitiesOverride, inje
 			targetIds,
 			...retryStrategy.config
 		};
-		const { testCases: retryTestCases, strategyResults: retryResults } = await applyStrategies(pluginTestCases, [retryStrategy], injectVar, void 0, maxCharsPerMessage);
+		const { testCases: retryTestCases, strategyResults: retryResults } = await applyStrategies(pluginTestCases, [retryStrategy], injectVar, redteamProvider, purpose, void 0, maxCharsPerMessage);
 		pluginTestCases.push(...retryTestCases);
 		Object.assign(strategyResults, retryResults);
 		if (showProgressBar) progressBar?.increment(retryTestCases.length);
@@ -12265,7 +13233,7 @@ async function synthesize({ abortSignal, delay, entities: entitiesOverride, inje
 	checkAbort();
 	const nonBasicStrategies = strategies.filter((s) => !["basic", "retry"].includes(s.id));
 	if (showProgressBar && nonBasicStrategies.length > 0) progressBar?.update({ task: "Applying strategies" });
-	const { testCases: strategyTestCases, strategyResults: otherStrategyResults } = await applyStrategies(pluginTestCases, nonBasicStrategies, injectVar, excludeTargetOutputFromAgenticAttackGeneration, maxCharsPerMessage);
+	const { testCases: strategyTestCases, strategyResults: otherStrategyResults } = await applyStrategies(pluginTestCases, nonBasicStrategies, injectVar, redteamProvider, purpose, excludeTargetOutputFromAgenticAttackGeneration, maxCharsPerMessage);
 	Object.assign(strategyResults, otherStrategyResults);
 	if (showProgressBar && strategyTestCases.length > 0) progressBar?.increment(strategyTestCases.length);
 	const finalTestCases = [...includeBasicTests ? pluginTestCases : [], ...strategyTestCases];
@@ -12317,7 +13285,7 @@ function handleFailedPlugins(failedPlugins, strict) {
 }
 function getConfigHash(configPath) {
 	const content = fs.readFileSync(configPath, "utf8");
-	return (0, crypto$1.createHash)("md5").update(`${require_fetch.VERSION}:${content}`).digest("hex");
+	return (0, crypto$1.createHash)("md5").update(`${require_version.VERSION}:${content}`).digest("hex");
 }
 function createHeaderComments({ title, timestampLabel, author, cloudHost, testCasesCount, plugins, strategies, isUpdate = false }) {
 	const sectionLabel = isUpdate ? "Changes:" : "Test Configuration:";
@@ -12392,7 +13360,7 @@ async function doGenerateRedteam(options) {
 		redteamConfig = resolved.config.redteam;
 		commandLineOptions = resolved.commandLineOptions;
 		resolvedConfig = resolved.config;
-		await require_providers.checkCloudPermissions(resolved.config);
+		await require_storage.checkCloudPermissions(resolved.config);
 		if (redteamConfig && resolved.testSuite.tests && resolved.testSuite.tests.length > 0) require_logger.logger.warn(chalk.default.yellow(dedent.default`
             ⚠️  Warning: Found both 'tests' section and 'redteam' configuration in your config file.
 
@@ -12407,8 +13375,8 @@ async function doGenerateRedteam(options) {
           `));
 		try {
 			const providerId = require_providers.getProviderIds(resolved.config.providers)[0];
-			if (require_providers.isCloudProvider(providerId)) {
-				const overrides = await require_providers.getPluginSeverityOverridesFromCloud(require_providers.getCloudDatabaseId(providerId));
+			if (require_storage.isCloudProvider(providerId)) {
+				const overrides = await require_storage.getPluginSeverityOverridesFromCloud(require_storage.getCloudDatabaseId(providerId));
 				if (overrides) {
 					pluginSeverityOverrides = new Map(Object.entries(overrides.severities));
 					pluginSeverityOverridesId = overrides.id;
@@ -12426,7 +13394,7 @@ async function doGenerateRedteam(options) {
 		require_logger.logger.info(chalk.default.red(`\nCan't generate without configuration - run ${chalk.default.yellow.bold(promptfooCommand("redteam init"))} first`));
 		return null;
 	}
-	if (!require_server.neverGenerateRemote()) {
+	if (!require_remoteGeneration.neverGenerateRemote()) {
 		let hasValidEmail = false;
 		while (!hasValidEmail) {
 			const { emailNeedsValidation } = await require_accounts.promptForEmailUnverified();
@@ -12490,7 +13458,7 @@ async function doGenerateRedteam(options) {
 	}
 	const policyPluginsWithRefs = plugins.filter((plugin) => plugin.config?.policy && require_graders.isValidPolicyObject(plugin.config?.policy) && require_graders.determinePolicyTypeFromId(plugin.config.policy.id) === "reusable");
 	if (policyPluginsWithRefs.length > 0) {
-		const teamId = (await require_providers.resolveTeamId()).id;
+		const teamId = (await require_storage.resolveTeamId()).id;
 		const policiesById = await require_graders.getCustomPolicies(policyPluginsWithRefs, teamId);
 		for (const policyPlugin of policyPluginsWithRefs) {
 			const policyId = policyPlugin.config.policy.id;
@@ -13062,7 +14030,7 @@ async function sendChunkedResults(evalRecord, url$5, options = {}) {
 	const isVerbose = require_logger.isDebugEnabled();
 	const { silent = false } = options;
 	require_logger.logger.debug(`Starting chunked results upload to ${url$5}`);
-	await require_providers.checkCloudPermissions(evalRecord.config);
+	await require_storage.checkCloudPermissions(evalRecord.config);
 	const inlineBlobs = require_extractor.isBlobStorageEnabled() && require_logger.getEnvBool("PROMPTFOO_SHARE_INLINE_BLOBS", !require_fetch.cloudConfig.isEnabled());
 	const inlineCache = inlineBlobs ? createBlobInlineCache() : null;
 	let sampleResults = (await evalRecord.fetchResultsBatched(100).next()).value ?? [];
@@ -13158,6 +14126,10 @@ function stripAuthFromUrl(urlString) {
 	}
 }
 async function handleEmailCollection(evalRecord) {
+	if (evalRecord.author) {
+		require_logger.logger.debug(`[Share] Skipping email collection because author is already set`, { evalId: evalRecord.id });
+		return;
+	}
 	if (!process.stdout.isTTY || require_logger.isCI() || require_logger.getEnvBool("PROMPTFOO_DISABLE_SHARE_EMAIL_REQUEST")) return;
 	let email = require_accounts.getUserEmail();
 	if (!email) {
@@ -13200,7 +14172,7 @@ async function createShareableUrl(evalRecord, options = {}) {
 		return null;
 	}
 	if (!silent) {
-		const orgContext = await require_providers.getOrgContext();
+		const orgContext = await require_storage.getOrgContext();
 		if (orgContext) {
 			const teamSuffix = orgContext.teamName ? ` > ${orgContext.teamName}` : "";
 			require_logger.logger.info(`${chalk.default.dim("Sharing to:")} ${chalk.default.cyan(orgContext.organizationName)}${teamSuffix}`);
@@ -13230,7 +14202,7 @@ function generateTable(evaluateTable, tableCellMaxLength = 250, maxRows = 25) {
 	for (const row of evaluateTable.body.slice(0, maxRows)) table.push([...row.vars.map((v) => require_text.ellipsize(v, tableCellMaxLength)), ...row.outputs.map(({ pass, text, failureReason: failureType }) => {
 		text = require_text.ellipsize(text, tableCellMaxLength);
 		if (pass) return chalk.default.green("[PASS] ") + text;
-		else return chalk.default.red(failureType === require_types.ResultFailureReason.ASSERT ? "[FAIL] " : "[ERROR] ") + text.split("---").map((c, idx) => idx === 0 ? chalk.default.red.bold(c) : c).join("---");
+		return chalk.default.red(failureType === require_types.ResultFailureReason.ASSERT ? "[FAIL] " : "[ERROR] ") + text.split("---").map((c, idx) => idx === 0 ? chalk.default.red.bold(c) : c).join("---");
 	})]);
 	return table.toString();
 }
@@ -13301,6 +14273,14 @@ function shouldShareResults(opts) {
 	return require_fetch.cloudConfig.isEnabled() && sharing !== false;
 }
 //#endregion
+//#region src/commands/eval/redteamWarning.ts
+function warnIfRedteamConfigHasNoTests(config, testSuite) {
+	if (config.redteam && (!testSuite.tests || testSuite.tests.length === 0) && (!testSuite.scenarios || testSuite.scenarios.length === 0)) require_logger.logger.warn(chalk.default.yellow(dedent.default`
+        Warning: Config file has a redteam section but no test cases.
+        Did you mean to run ${chalk.default.bold("promptfoo redteam generate")} instead?
+        `));
+}
+//#endregion
 //#region src/util/formatDuration.ts
 /**
 * Formats a duration in seconds into a human-readable string
@@ -13321,6 +14301,115 @@ function formatDuration(seconds) {
 }
 //#endregion
 //#region src/commands/eval/summary.ts
+function getCompletionMessage({ completionType, evalId, shareableUrl, wasAborted, writeToDatabase, activelySharing }) {
+	if (wasAborted) {
+		const idSuffix = writeToDatabase ? ` (ID: ${chalk.default.cyan(evalId)})` : "";
+		return `${chalk.default.red("✗")} ${completionType} aborted${idSuffix}`;
+	}
+	if (writeToDatabase && shareableUrl) return `${chalk.default.green("✓")} ${completionType} complete: ${shareableUrl}`;
+	if (writeToDatabase && activelySharing) return `${chalk.default.green("✓")} ${completionType} complete`;
+	if (writeToDatabase) return `${chalk.default.green("✓")} ${completionType} complete (ID: ${chalk.default.cyan(evalId)})`;
+	return `${chalk.default.green("✓")} ${completionType} complete`;
+}
+function getAbortSummaryLines(targetErrorStatus) {
+	if (targetErrorStatus == null) return [];
+	return [
+		"",
+		chalk.default.red.bold("Scan stopped: Target is unavailable and will not recover on retry."),
+		chalk.default.red(`  Target returned HTTP ${targetErrorStatus}`),
+		"",
+		chalk.default.yellow("Possible causes:"),
+		chalk.default.yellow("  • Invalid API key or authentication (401/403)"),
+		chalk.default.yellow("  • Target endpoint does not exist (404)"),
+		chalk.default.yellow("  • Server does not support the request (501)"),
+		"",
+		chalk.default.cyan("To fix: Check your target configuration and credentials.")
+	];
+}
+function getGuidanceLines({ writeToDatabase, shareableUrl, wantsToShare, activelySharing, hasExplicitDisable, cloudEnabled }) {
+	if (!writeToDatabase || shareableUrl || wantsToShare || activelySharing) return [];
+	const lines = ["", `» View results: ${chalk.default.green.bold("promptfoo view")}`];
+	if (!hasExplicitDisable) lines.push(cloudEnabled ? `» Create shareable URL: ${chalk.default.green.bold("promptfoo share")}` : `» Share with your team: ${chalk.default.green.bold("https://promptfoo.app")}`);
+	lines.push(`» Feedback: ${chalk.default.green.bold("https://promptfoo.dev/feedback")}`);
+	return lines;
+}
+function buildUsageDetails(usage, total) {
+	const parts = [];
+	if (usage.prompt && usage.prompt > 0) parts.push(`${usage.prompt.toLocaleString()} prompt`);
+	if (usage.completion && usage.completion > 0) parts.push(`${usage.completion.toLocaleString()} completion`);
+	if (usage.cached && usage.cached > 0) parts.push(usage.cached === total && parts.length === 0 ? "cached" : `${usage.cached.toLocaleString()} cached`);
+	if (usage.completionDetails?.reasoning && usage.completionDetails.reasoning > 0) parts.push(`${usage.completionDetails.reasoning.toLocaleString()} reasoning`);
+	return parts;
+}
+function getTokenUsageLines(tokenUsage, isRedteam, tracker) {
+	const hasEvalTokens = (tokenUsage.total || 0) > 0 || (tokenUsage.prompt || 0) + (tokenUsage.completion || 0) > 0;
+	const hasGradingTokens = tokenUsage.assertions && (tokenUsage.assertions.total || 0) > 0;
+	if (!hasEvalTokens && !hasGradingTokens) return [];
+	const combinedTotal = (tokenUsage.prompt || 0) + (tokenUsage.completion || 0);
+	const evalTokens = {
+		prompt: tokenUsage.prompt || 0,
+		completion: tokenUsage.completion || 0,
+		total: tokenUsage.total || combinedTotal,
+		cached: tokenUsage.cached || 0,
+		numRequests: tokenUsage.numRequests || 0,
+		completionDetails: tokenUsage.completionDetails || {
+			reasoning: 0,
+			acceptedPrediction: 0,
+			rejectedPrediction: 0
+		}
+	};
+	const lines = [`${chalk.default.bold("Total Tokens:")} ${chalk.default.white.bold((evalTokens.total + (tokenUsage.assertions?.total || 0)).toLocaleString())}`];
+	if (isRedteam && tokenUsage.numRequests) lines.push(`  ${chalk.default.gray("Probes:")} ${chalk.default.white(tokenUsage.numRequests.toLocaleString())}`);
+	if (evalTokens.total > 0) {
+		const evalParts = buildUsageDetails(evalTokens, evalTokens.total);
+		lines.push(`  ${chalk.default.gray("Eval:")} ${chalk.default.white(evalTokens.total.toLocaleString())} (${evalParts.join(", ")})`);
+	}
+	if (tokenUsage.assertions?.total && tokenUsage.assertions.total > 0) {
+		const gradingParts = buildUsageDetails(tokenUsage.assertions, tokenUsage.assertions.total);
+		lines.push(`  ${chalk.default.gray("Grading:")} ${chalk.default.white(tokenUsage.assertions.total.toLocaleString())} (${gradingParts.join(", ")})`);
+	}
+	lines.push(...getProviderUsageLines(tracker));
+	return lines;
+}
+function getProviderUsageLines(tracker) {
+	const providerIds = tracker.getProviderIds();
+	if (providerIds.length <= 1) return [];
+	const sortedProviders = providerIds.map((id) => ({
+		id,
+		usage: tracker.getProviderUsage(id)
+	})).filter((p) => p.usage != null).sort((a, b) => (b.usage.total || 0) - (a.usage.total || 0));
+	const lines = ["", chalk.default.bold("Providers:")];
+	for (const { id, usage } of sortedProviders) {
+		if ((usage.total || 0) === 0 && (usage.prompt || 0) + (usage.completion || 0) === 0) continue;
+		const displayTotal = usage.total || (usage.prompt || 0) + (usage.completion || 0);
+		const displayId = id.includes(" (") ? id.substring(0, id.indexOf(" (")) : id;
+		const details = buildUsageDetails(usage, displayTotal);
+		const requestInfo = `${usage.numRequests || 0} requests`;
+		const separator = details.length > 0 ? "; " : "";
+		lines.push(`  ${chalk.default.gray(`${displayId}:`)} ${chalk.default.white(displayTotal.toLocaleString())} (${requestInfo}${separator}${details.join(", ")})`);
+	}
+	return lines;
+}
+function formatResultPercentage(count, totalTests) {
+	const percentage = totalTests === 0 ? 0 : count / totalTests * 100;
+	return percentage === 0 || percentage === 100 ? `${percentage.toFixed(0)}%` : `${percentage.toFixed(2)}%`;
+}
+function formatResultLine(count, label, icon, iconColor, totalTests) {
+	return `  ${icon ? `${iconColor(icon)} ` : ""}${chalk.default.white.bold(count.toLocaleString())} ${chalk.default.white(label)} ${chalk.default.gray(`(${formatResultPercentage(count, totalTests)})`)}`;
+}
+function getResultsLines({ successes, failures, errors, duration, maxConcurrency }) {
+	const totalTests = successes + failures + errors;
+	const errorLabel = errors === 1 ? "error" : "errors";
+	return [
+		"",
+		chalk.default.bold("Results:"),
+		formatResultLine(successes, "passed", successes > 0 ? "✓" : void 0, chalk.default.green, totalTests),
+		formatResultLine(failures, "failed", failures > 0 ? "✗" : void 0, chalk.default.red, totalTests),
+		formatResultLine(errors, errorLabel, errors > 0 ? "✗" : void 0, chalk.default.red, totalTests),
+		chalk.default.gray(`Duration: ${formatDuration(duration)} (concurrency: ${maxConcurrency})`),
+		""
+	];
+}
 /**
 * Generate formatted evaluation summary output for CLI display.
 *
@@ -13359,115 +14448,28 @@ function formatDuration(seconds) {
 * ```
 */
 function generateEvalSummary(params) {
-	const { evalId, isRedteam, writeToDatabase, shareableUrl, wantsToShare, hasExplicitDisable, cloudEnabled, activelySharing = false, tokenUsage, successes, failures, errors, duration, maxConcurrency, tracker, targetErrorStatus } = params;
-	const lines = [];
-	const completionType = isRedteam ? "Red team" : "Eval";
-	const wasAborted = targetErrorStatus != null;
-	let completionMessage;
-	if (wasAborted) {
-		completionMessage = `${chalk.default.red("✗")} ${completionType} aborted`;
-		if (writeToDatabase) completionMessage += ` (ID: ${chalk.default.cyan(evalId)})`;
-	} else if (writeToDatabase && shareableUrl) completionMessage = `${chalk.default.green("✓")} ${completionType} complete: ${shareableUrl}`;
-	else if (writeToDatabase && activelySharing) completionMessage = `${chalk.default.green("✓")} ${completionType} complete`;
-	else if (writeToDatabase) completionMessage = `${chalk.default.green("✓")} ${completionType} complete (ID: ${chalk.default.cyan(evalId)})`;
-	else completionMessage = `${chalk.default.green("✓")} ${completionType} complete`;
-	lines.push(completionMessage);
-	if (wasAborted && targetErrorStatus != null) {
-		lines.push("");
-		lines.push(chalk.default.red.bold("Scan stopped: Target is unavailable and will not recover on retry."));
-		lines.push(chalk.default.red(`  Target returned HTTP ${targetErrorStatus}`));
-		lines.push("");
-		lines.push(chalk.default.yellow("Possible causes:"));
-		lines.push(chalk.default.yellow("  • Invalid API key or authentication (401/403)"));
-		lines.push(chalk.default.yellow("  • Target endpoint does not exist (404)"));
-		lines.push(chalk.default.yellow("  • Server does not support the request (501)"));
-		lines.push("");
-		lines.push(chalk.default.cyan("To fix: Check your target configuration and credentials."));
-	}
-	if (writeToDatabase && !shareableUrl && !wantsToShare && !activelySharing) {
-		lines.push("");
-		lines.push(`» View results: ${chalk.default.green.bold("promptfoo view")}`);
-		if (!hasExplicitDisable) if (cloudEnabled) lines.push(`» Create shareable URL: ${chalk.default.green.bold("promptfoo share")}`);
-		else lines.push(`» Share with your team: ${chalk.default.green.bold("https://promptfoo.app")}`);
-		lines.push(`» Feedback: ${chalk.default.green.bold("https://promptfoo.dev/feedback")}`);
-	}
-	lines.push("");
-	const hasEvalTokens = (tokenUsage.total || 0) > 0 || (tokenUsage.prompt || 0) + (tokenUsage.completion || 0) > 0;
-	const hasGradingTokens = tokenUsage.assertions && (tokenUsage.assertions.total || 0) > 0;
-	if (hasEvalTokens || hasGradingTokens) {
-		const combinedTotal = (tokenUsage.prompt || 0) + (tokenUsage.completion || 0);
-		const evalTokens = {
-			prompt: tokenUsage.prompt || 0,
-			completion: tokenUsage.completion || 0,
-			total: tokenUsage.total || combinedTotal,
-			cached: tokenUsage.cached || 0,
-			completionDetails: tokenUsage.completionDetails || {
-				reasoning: 0,
-				acceptedPrediction: 0,
-				rejectedPrediction: 0
-			}
-		};
-		const grandTotal = evalTokens.total + (tokenUsage.assertions?.total || 0);
-		lines.push(`${chalk.default.bold("Total Tokens:")} ${chalk.default.white.bold(grandTotal.toLocaleString())}`);
-		if (isRedteam && tokenUsage.numRequests) lines.push(`  ${chalk.default.gray("Probes:")} ${chalk.default.white(tokenUsage.numRequests.toLocaleString())}`);
-		if (evalTokens.total > 0) {
-			const evalParts = [];
-			if (evalTokens.prompt > 0) evalParts.push(`${evalTokens.prompt.toLocaleString()} prompt`);
-			if (evalTokens.completion > 0) evalParts.push(`${evalTokens.completion.toLocaleString()} completion`);
-			if (evalTokens.cached > 0) if (evalTokens.cached === evalTokens.total && evalParts.length === 0) evalParts.push("cached");
-			else evalParts.push(`${evalTokens.cached.toLocaleString()} cached`);
-			if (evalTokens.completionDetails?.reasoning && evalTokens.completionDetails.reasoning > 0) evalParts.push(`${evalTokens.completionDetails.reasoning.toLocaleString()} reasoning`);
-			lines.push(`  ${chalk.default.gray("Eval:")} ${chalk.default.white(evalTokens.total.toLocaleString())} (${evalParts.join(", ")})`);
-		}
-		if (tokenUsage.assertions && tokenUsage.assertions.total && tokenUsage.assertions.total > 0) {
-			const gradingParts = [];
-			if (tokenUsage.assertions.prompt && tokenUsage.assertions.prompt > 0) gradingParts.push(`${tokenUsage.assertions.prompt.toLocaleString()} prompt`);
-			if (tokenUsage.assertions.completion && tokenUsage.assertions.completion > 0) gradingParts.push(`${tokenUsage.assertions.completion.toLocaleString()} completion`);
-			if (tokenUsage.assertions.cached && tokenUsage.assertions.cached > 0) if (tokenUsage.assertions.cached === tokenUsage.assertions.total && gradingParts.length === 0) gradingParts.push("cached");
-			else gradingParts.push(`${tokenUsage.assertions.cached.toLocaleString()} cached`);
-			if (tokenUsage.assertions.completionDetails?.reasoning && tokenUsage.assertions.completionDetails.reasoning > 0) gradingParts.push(`${tokenUsage.assertions.completionDetails.reasoning.toLocaleString()} reasoning`);
-			lines.push(`  ${chalk.default.gray("Grading:")} ${chalk.default.white(tokenUsage.assertions.total.toLocaleString())} (${gradingParts.join(", ")})`);
-		}
-		const providerIds = tracker.getProviderIds();
-		if (providerIds.length > 1) {
-			lines.push("");
-			lines.push(chalk.default.bold("Providers:"));
-			const sortedProviders = providerIds.map((id) => ({
-				id,
-				usage: tracker.getProviderUsage(id)
-			})).filter((p) => p.usage != null).sort((a, b) => (b.usage.total || 0) - (a.usage.total || 0));
-			for (const { id, usage } of sortedProviders) if ((usage.total || 0) > 0 || (usage.prompt || 0) + (usage.completion || 0) > 0) {
-				const displayTotal = usage.total || (usage.prompt || 0) + (usage.completion || 0);
-				const displayId = id.includes(" (") ? id.substring(0, id.indexOf(" (")) : id;
-				const details = [];
-				if (usage.prompt && usage.prompt > 0) details.push(`${usage.prompt.toLocaleString()} prompt`);
-				if (usage.completion && usage.completion > 0) details.push(`${usage.completion.toLocaleString()} completion`);
-				if (usage.cached && usage.cached > 0) if (usage.cached === displayTotal && details.length === 0) details.push("cached");
-				else details.push(`${usage.cached.toLocaleString()} cached`);
-				if (usage.completionDetails?.reasoning && usage.completionDetails.reasoning > 0) details.push(`${usage.completionDetails.reasoning.toLocaleString()} reasoning`);
-				const breakdown = ` (${`${usage.numRequests || 0} requests`}${details.length > 0 ? "; " : ""}${details.join(", ")})`;
-				lines.push(`  ${chalk.default.gray(displayId + ":")} ${chalk.default.white(displayTotal.toLocaleString())}${breakdown}`);
-			}
-		}
-	}
-	lines.push("");
-	const totalTests = successes + failures + errors;
-	const formatResultPercentage = (count) => {
-		const percentage = totalTests === 0 ? 0 : count / totalTests * 100;
-		return percentage === 0 || percentage === 100 ? `${percentage.toFixed(0)}%` : `${percentage.toFixed(2)}%`;
-	};
-	const formatResultLine = (count, label, icon, iconColor) => {
-		return `  ${icon ? `${iconColor(icon)} ` : ""}${chalk.default.white.bold(count.toLocaleString())} ${chalk.default.white(label)} ${chalk.default.gray(`(${formatResultPercentage(count)})`)}`;
-	};
-	const errorLabel = errors === 1 ? "error" : "errors";
-	lines.push(chalk.default.bold("Results:"));
-	lines.push(formatResultLine(successes, "passed", successes > 0 ? "✓" : void 0, chalk.default.green));
-	lines.push(formatResultLine(failures, "failed", failures > 0 ? "✗" : void 0, chalk.default.red));
-	lines.push(formatResultLine(errors, errorLabel, errors > 0 ? "✗" : void 0, chalk.default.red));
-	const durationDisplay = formatDuration(duration);
-	lines.push(chalk.default.gray(`Duration: ${durationDisplay} (concurrency: ${maxConcurrency})`));
-	lines.push("");
-	return lines;
+	return [
+		getCompletionMessage({
+			completionType: params.isRedteam ? "Red team" : "Eval",
+			evalId: params.evalId,
+			shareableUrl: params.shareableUrl,
+			wasAborted: params.targetErrorStatus != null,
+			writeToDatabase: params.writeToDatabase,
+			activelySharing: params.activelySharing ?? false
+		}),
+		...getAbortSummaryLines(params.targetErrorStatus),
+		...getGuidanceLines({
+			writeToDatabase: params.writeToDatabase,
+			shareableUrl: params.shareableUrl,
+			wantsToShare: params.wantsToShare,
+			activelySharing: params.activelySharing ?? false,
+			hasExplicitDisable: params.hasExplicitDisable,
+			cloudEnabled: params.cloudEnabled
+		}),
+		"",
+		...getTokenUsageLines(params.tokenUsage, params.isRedteam, params.tracker),
+		...getResultsLines(params)
+	];
 }
 //#endregion
 //#region src/commands/retry.ts
@@ -13625,7 +14627,7 @@ async function doEval(cmdObj, defaultConfig, defaultConfigPath, evaluateOptions)
 		const cloudConfigId = uuidConfigArgs[0];
 		if (cmdObj.watch) throw new Error("--watch is not supported when using a cloud config UUID with -c. Use a local config file path for watch mode.");
 		try {
-			defaultConfig = await require_providers.getEvalConfigFromCloud(cloudConfigId);
+			defaultConfig = await require_storage.getEvalConfigFromCloud(cloudConfigId);
 		} catch (error) {
 			const reason = error instanceof Error ? error.message : String(error);
 			throw new Error(`Failed to load cloud eval config "${cloudConfigId}". ${reason}. Cloud UUID inputs do not fall back to local file paths. Check authentication and that the UUID exists.`);
@@ -13719,14 +14721,11 @@ async function doEval(cmdObj, defaultConfig, defaultConfigPath, evaluateOptions)
 			require_logger.state.resume = true;
 			require_logger.state.retryMode = true;
 		} else ({config, testSuite, basePath: _basePath, commandLineOptions} = await resolveConfigs(cmdObj, defaultConfig));
-		if (!cmdObj.envPath && commandLineOptions?.envPath) {
+		if ((!cmdObj.envPath || cmdObj.envPath.length === 0) && commandLineOptions?.envPath) {
 			require_logger.logger.debug(`Loading additional environment from config: ${commandLineOptions.envPath}`);
 			require_util.setupEnv(commandLineOptions.envPath);
 		}
-		if (config.redteam && (!testSuite.tests || testSuite.tests.length === 0) && (!testSuite.scenarios || testSuite.scenarios.length === 0)) require_logger.logger.warn(chalk.default.yellow(dedent.default`
-        Warning: Config file has a redteam section but no test cases.
-        Did you mean to run ${chalk.default.bold("promptfoo redteam generate")} instead?
-        `));
+		warnIfRedteamConfigHasNoTests(config, testSuite);
 		if (config.redteam && Array.isArray(config.providers) && config.providers.length > 0 && typeof config.providers[0] === "object" && config.providers[0].id === "http") {
 			const maybeUrl = config.providers[0]?.config?.url;
 			if (typeof maybeUrl === "string" && maybeUrl.includes("promptfoo.app")) require_telemetry.telemetry.record("feature_used", { feature: "redteam_run_with_example" });
@@ -13774,7 +14773,7 @@ async function doEval(cmdObj, defaultConfig, defaultConfigPath, evaluateOptions)
 			};
 			testSuite.tests = await filterTests(testSuite, filterOptions);
 		}
-		if (!require_server.neverGenerateRemote() && config.redteam && config.redteam.plugins && config.redteam.plugins.length > 0 && testSuite.tests && testSuite.tests.length > 0) {
+		if (!require_remoteGeneration.neverGenerateRemote() && config.redteam && config.redteam.plugins && config.redteam.plugins.length > 0 && testSuite.tests && testSuite.tests.length > 0) {
 			let hasValidEmail = false;
 			while (!hasValidEmail) {
 				const { emailNeedsValidation } = await require_accounts.promptForEmailUnverified();
@@ -13792,7 +14791,7 @@ async function doEval(cmdObj, defaultConfig, defaultConfigPath, evaluateOptions)
 			process.exitCode = 1;
 			return new Eval({}, { persisted: false });
 		}
-		await require_providers.checkCloudPermissions(config);
+		await require_storage.checkCloudPermissions(config);
 		const options = {
 			...evaluateOptions,
 			showProgressBar: require_logger.getLogLevel() === "debug" ? false : cmdObj.progressBar === void 0 ? evaluateOptions.showProgressBar === void 0 ? true : evaluateOptions.showProgressBar : cmdObj.progressBar !== false,
@@ -13834,7 +14833,14 @@ async function doEval(cmdObj, defaultConfig, defaultConfigPath, evaluateOptions)
         ${zod.z.prettifyError(testSuiteSchema.error)}
 
       Please review your promptfooconfig.yaml configuration.`));
-		const evalRecord = resumeEval ? resumeEval : cmdObj.write ? await Eval.create(config, testSuite.prompts, { runtimeOptions: options }) : new Eval(config, { runtimeOptions: options });
+		const author = require_accounts.getAuthor();
+		const evalRecord = resumeEval ? resumeEval : cmdObj.write ? await Eval.create(config, testSuite.prompts, {
+			author,
+			runtimeOptions: options
+		}) : new Eval(config, {
+			author,
+			runtimeOptions: options
+		});
 		const abortController = new AbortController();
 		const previousAbortSignal = evaluateOptions.abortSignal;
 		evaluateOptions.abortSignal = previousAbortSignal ? AbortSignal.any([previousAbortSignal, abortController.signal]) : abortController.signal;
@@ -13945,7 +14951,7 @@ async function doEval(cmdObj, defaultConfig, defaultConfigPath, evaluateOptions)
 		const paths = (Array.isArray(outputPath) ? outputPath : [outputPath]).filter((p) => typeof p === "string" && p.length > 0 && !p.endsWith(".jsonl"));
 		const isRedteam = Boolean(config.redteam);
 		const duration = Math.round((Date.now() - startTime) / 1e3);
-		const tracker = require_providers.TokenUsageTracker.getInstance();
+		const tracker = require_shared.TokenUsageTracker.getInstance();
 		const targetErrorStatus = await evalRecord.findTargetErrorStatus();
 		const summaryLines = generateEvalSummary({
 			evalId: evalRecord.id,
@@ -13975,7 +14981,7 @@ async function doEval(cmdObj, defaultConfig, defaultConfigPath, evaluateOptions)
 		} else for (const line of summaryLines) require_logger.logger.info(line);
 		let shareableUrl = null;
 		if (sharePromise != null) {
-			const orgContext = await require_providers.getOrgContext();
+			const orgContext = await require_storage.getOrgContext();
 			const orgSuffix = orgContext ? ` to ${orgContext.organizationName}${orgContext.teamName ? ` > ${orgContext.teamName}` : ""}` : "";
 			if (process.stdout.isTTY && !require_logger.isCI()) {
 				const spinner = (0, ora.default)({
@@ -14150,7 +15156,7 @@ async function doRedteamRun(options) {
 		redteamPath = path.join(configDir, "redteam.yaml");
 	}
 	try {
-		const healthUrl = require_server.getRemoteHealthUrl();
+		const healthUrl = require_remoteGeneration.getRemoteHealthUrl();
 		if (healthUrl) {
 			require_logger.logger.debug(`Checking Promptfoo API health at ${healthUrl}...`);
 			const healthResult = await checkRemoteHealth(healthUrl);
@@ -14236,65 +15242,175 @@ async function doRedteamRun(options) {
 	return evalResult;
 }
 //#endregion
+//#region src/types/transform.ts
+/** Runtime type guard for `TransformFunction` values. */
+function isTransformFunction(value) {
+	return typeof value === "function";
+}
+//#endregion
 //#region src/index.ts
+/**
+* Shallow-clone a test case so the caller can swap in resolved ApiProvider
+* instances on `options.provider` / `assert[].provider` without leaking those
+* mutations back to the input. The input may alias the unified config written
+* to the Eval record, and a live SDK client (e.g. Bedrock's BedrockRuntime,
+* Anthropic's client) holds circular references that break drizzle's JSON
+* serialization on `evalRecord.save()`. Fixes #8687.
+*
+* Detaches only `options` and `assert[]`. Other reference fields (`provider`,
+* `vars`, `metadata`, `providerOutput`) remain aliased — callers must reassign
+* those by reference rather than mutating in place. `assert-set` children are
+* not deep-cloned because the resolve loop skips `assert-set`; if that ever
+* changes, extend this helper.
+*/
+function cloneTestForResolve(test) {
+	const cloned = { ...test };
+	if (test.options) cloned.options = { ...test.options };
+	if (test.assert) cloned.assert = test.assert.map((assertion) => ({ ...assertion }));
+	return cloned;
+}
+function toSerializableProviderRef(provider) {
+	if (require_types.isApiProvider(provider)) return require_evalResult.sanitizeProvider(provider);
+	if (Array.isArray(provider)) return provider.map(toSerializableProviderRef);
+	return provider;
+}
+function isRecord(value) {
+	return Boolean(value && typeof value === "object" && !Array.isArray(value));
+}
+function withSerializableProvider(record) {
+	if (!require_types.isApiProvider(record.provider)) return record;
+	return {
+		...record,
+		provider: require_evalResult.sanitizeProvider(record.provider)
+	};
+}
+/**
+* Function-valued transforms are first-class at runtime but are silently dropped
+* by `JSON.stringify`. Persisted eval configs (drizzle-stored) must never retain
+* a function reference, so replace every `transform`-like field with a
+* `[inline function]: name` marker. Non-function values pass through unchanged.
+*
+* `droppedRef.value` is flipped to `true` the first time a function is replaced
+* so the caller can emit a single warning instead of logging per field.
+*/
+function replaceFunctionTransforms(record, droppedRef) {
+	let result;
+	for (const key of require_transform$1.TRANSFORM_KEYS) {
+		const value = record[key];
+		if (!isTransformFunction(value)) continue;
+		if (!result) result = { ...record };
+		result[key] = value.name ? `${require_transform$1.INLINE_FUNCTION_LABEL}: ${value.name}` : require_transform$1.INLINE_FUNCTION_LABEL;
+		droppedRef.value = true;
+	}
+	return result ?? record;
+}
+function toSerializableAssertion(assertion, droppedRef) {
+	if (!isRecord(assertion)) return assertion;
+	let sanitizedAssertion = withSerializableProvider(assertion);
+	sanitizedAssertion = replaceFunctionTransforms(sanitizedAssertion, droppedRef);
+	if (Array.isArray(assertion.assert)) sanitizedAssertion = {
+		...sanitizedAssertion,
+		assert: assertion.assert.map((a) => toSerializableAssertion(a, droppedRef))
+	};
+	return sanitizedAssertion;
+}
+function toSerializableTestCase(test, droppedRef) {
+	if (!isRecord(test)) return test;
+	let sanitizedTest = withSerializableProvider(test);
+	if (isRecord(test.options)) {
+		let options = withSerializableProvider(test.options);
+		options = replaceFunctionTransforms(options, droppedRef);
+		if (options !== test.options) sanitizedTest = {
+			...sanitizedTest,
+			options
+		};
+	}
+	if (Array.isArray(test.assert)) sanitizedTest = {
+		...sanitizedTest,
+		assert: test.assert.map((a) => toSerializableAssertion(a, droppedRef))
+	};
+	return sanitizedTest;
+}
+function toSerializableScenario(scenario, droppedRef) {
+	if (!isRecord(scenario)) return scenario;
+	if (!Array.isArray(scenario.tests)) return scenario;
+	return {
+		...scenario,
+		tests: scenario.tests.map((t) => toSerializableTestCase(t, droppedRef))
+	};
+}
+function createSerializableUnifiedConfig(testSuite, prompts) {
+	const droppedRef = { value: false };
+	const config = {
+		...testSuite,
+		providers: toSerializableProviderRef(testSuite.providers),
+		defaultTest: toSerializableTestCase(testSuite.defaultTest, droppedRef),
+		tests: Array.isArray(testSuite.tests) ? testSuite.tests.map((t) => toSerializableTestCase(t, droppedRef)) : testSuite.tests,
+		scenarios: Array.isArray(testSuite.scenarios) ? testSuite.scenarios.map((s) => toSerializableScenario(s, droppedRef)) : testSuite.scenarios,
+		prompts
+	};
+	if (droppedRef.value && testSuite.writeLatestResults) require_logger.logger.warn("Function-valued transform(s) in testSuite were replaced with \"[inline function]\" markers in the persisted config. Re-running the saved eval will not invoke them; use string expressions or file:// references if you need the config to round-trip.");
+	return config;
+}
 async function evaluate(testSuite, options = {}) {
-	if (testSuite.writeLatestResults) await runDbMigrations();
-	const loadedProviders = await require_providers.loadApiProviders(testSuite.providers, { env: testSuite.env });
+	const { author: suiteAuthor, ...testSuiteConfig } = testSuite;
+	if (testSuiteConfig.writeLatestResults) await runDbMigrations();
+	const loadedProviders = await require_providers.loadApiProviders(testSuiteConfig.providers, { env: testSuiteConfig.env });
 	const providerMap = {};
 	for (const p of loadedProviders) {
 		providerMap[p.id()] = p;
 		if (p.label) providerMap[p.label] = p;
 	}
-	let resolvedDefaultTest = testSuite.defaultTest;
-	if (typeof testSuite.defaultTest === "string" && testSuite.defaultTest.startsWith("file://")) resolvedDefaultTest = await require_util.maybeLoadFromExternalFile(testSuite.defaultTest);
+	let resolvedDefaultTest = testSuiteConfig.defaultTest;
+	if (typeof testSuiteConfig.defaultTest === "string" && testSuiteConfig.defaultTest.startsWith("file://")) resolvedDefaultTest = await require_util.maybeLoadFromExternalFile(testSuiteConfig.defaultTest);
 	const constructedTestSuite = {
-		...testSuite,
+		...testSuiteConfig,
 		defaultTest: resolvedDefaultTest,
-		scenarios: testSuite.scenarios,
+		scenarios: testSuiteConfig.scenarios,
 		providers: loadedProviders,
-		tests: await readTests(testSuite.tests),
-		nunjucksFilters: await require_util.readFilters(testSuite.nunjucksFilters || {}),
-		prompts: await require_graders.processPrompts(testSuite.prompts)
-	};
-	if (typeof constructedTestSuite.defaultTest === "object") {
-		if (constructedTestSuite.defaultTest?.provider && !require_types.isApiProvider(constructedTestSuite.defaultTest.provider)) constructedTestSuite.defaultTest.provider = await require_providers.resolveProvider(constructedTestSuite.defaultTest.provider, providerMap, {
-			env: testSuite.env,
+		tests: await readTests(testSuiteConfig.tests),
+		nunjucksFilters: await require_util.readFilters(testSuiteConfig.nunjucksFilters || {}),
+		prompts: await require_graders.processPrompts(testSuiteConfig.prompts)
+	};
+	if (typeof constructedTestSuite.defaultTest === "object" && constructedTestSuite.defaultTest) {
+		constructedTestSuite.defaultTest = cloneTestForResolve(constructedTestSuite.defaultTest);
+		if (constructedTestSuite.defaultTest.provider && !require_types.isApiProvider(constructedTestSuite.defaultTest.provider)) constructedTestSuite.defaultTest.provider = await require_providers.resolveProvider(constructedTestSuite.defaultTest.provider, providerMap, {
+			env: testSuiteConfig.env,
 			basePath: require_logger.state.basePath
 		});
-		if (constructedTestSuite.defaultTest?.options?.provider && !require_types.isApiProvider(constructedTestSuite.defaultTest.options.provider)) constructedTestSuite.defaultTest.options.provider = await require_providers.resolveProvider(constructedTestSuite.defaultTest.options.provider, providerMap, {
-			env: testSuite.env,
+		if (constructedTestSuite.defaultTest.options?.provider && !require_types.isApiProvider(constructedTestSuite.defaultTest.options.provider)) constructedTestSuite.defaultTest.options.provider = await require_providers.resolveProvider(constructedTestSuite.defaultTest.options.provider, providerMap, {
+			env: testSuiteConfig.env,
 			basePath: require_logger.state.basePath
 		});
 	}
-	for (const test of constructedTestSuite.tests || []) {
+	constructedTestSuite.tests = (constructedTestSuite.tests || []).map(cloneTestForResolve);
+	for (const test of constructedTestSuite.tests) {
 		if (test.options?.provider && !require_types.isApiProvider(test.options.provider)) test.options.provider = await require_providers.resolveProvider(test.options.provider, providerMap, {
-			env: testSuite.env,
+			env: testSuiteConfig.env,
 			basePath: require_logger.state.basePath
 		});
-		if (test.assert) for (const assertion of test.assert) {
+		for (const assertion of test.assert || []) {
 			if (assertion.type === "assert-set" || typeof assertion.provider === "function") continue;
 			if (assertion.provider && !require_types.isApiProvider(assertion.provider)) assertion.provider = await require_providers.resolveProvider(assertion.provider, providerMap, {
-				env: testSuite.env,
+				env: testSuiteConfig.env,
 				basePath: require_logger.state.basePath
 			});
 		}
 	}
 	if (options.cache === false) require_cache.disableCache();
-	const parsedProviderPromptMap = require_graders.readProviderPromptMap(testSuite, constructedTestSuite.prompts);
-	const unifiedConfig = {
-		...testSuite,
-		prompts: constructedTestSuite.prompts
-	};
-	const evalRecord = testSuite.writeLatestResults ? await Eval.create(unifiedConfig, constructedTestSuite.prompts) : new Eval(unifiedConfig);
+	const parsedProviderPromptMap = require_graders.readProviderPromptMap(testSuiteConfig, constructedTestSuite.prompts);
+	const unifiedConfig = createSerializableUnifiedConfig(testSuiteConfig, constructedTestSuite.prompts);
+	const author = require_accounts.getAuthor(suiteAuthor);
+	const evalRecord = testSuiteConfig.writeLatestResults ? await Eval.create(unifiedConfig, constructedTestSuite.prompts, { author }) : new Eval(unifiedConfig, { author });
 	const ret = await evaluate$1({
 		...constructedTestSuite,
 		providerPromptMap: parsedProviderPromptMap
 	}, evalRecord, {
 		eventSource: "library",
-		isRedteam: Boolean(testSuite.redteam),
+		isRedteam: Boolean(testSuiteConfig.redteam),
 		...options
 	});
-	if (testSuite.writeLatestResults && testSuite.sharing) if (isSharingEnabled(ret)) try {
+	if (testSuiteConfig.writeLatestResults && testSuiteConfig.sharing) if (isSharingEnabled(ret)) try {
 		const shareableUrl = await createShareableUrl(ret, { silent: true });
 		if (shareableUrl) {
 			ret.shareableUrl = shareableUrl;
@@ -14305,9 +15421,9 @@ async function evaluate(testSuite, options = {}) {
 		require_logger.logger.warn(`Failed to create shareable URL: ${error}`);
 	}
 	else require_logger.logger.debug("Sharing requested but not enabled (check cloud config or sharing settings)");
-	if (testSuite.outputPath) {
-		if (typeof testSuite.outputPath === "string") await require_util.writeOutput(testSuite.outputPath, evalRecord, null);
-		else if (Array.isArray(testSuite.outputPath)) await require_util.writeMultipleOutputs(testSuite.outputPath, evalRecord, null);
+	if (testSuiteConfig.outputPath) {
+		if (typeof testSuiteConfig.outputPath === "string") await require_util.writeOutput(testSuiteConfig.outputPath, evalRecord, null);
+		else if (Array.isArray(testSuiteConfig.outputPath)) await require_util.writeMultipleOutputs(testSuiteConfig.outputPath, evalRecord, null);
 	}
 	return ret;
 }
@@ -14319,7 +15435,7 @@ const redteam = {
 	},
 	Graders: require_graders.GRADERS,
 	Plugins,
-	Strategies: require_providers.Strategies,
+	Strategies: require_strategies.Strategies,
 	Base: {
 		Plugin: require_graders.RedteamPluginBase,
 		Grader: require_graders.RedteamGraderBase
@@ -14348,9 +15464,18 @@ exports.CompletedPromptSchema = require_types.CompletedPromptSchema;
 exports.CompletionTokenDetailsSchema = require_types.CompletionTokenDetailsSchema;
 exports.ConversationMessageSchema = require_types.ConversationMessageSchema;
 exports.DerivedMetricSchema = require_types.DerivedMetricSchema;
+exports.DocumentMediaInjectionPlacementSchema = require_types.DocumentMediaInjectionPlacementSchema;
+exports.DocumentMediaInjectionPlacementValues = require_types.DocumentMediaInjectionPlacementValues;
+exports.DocxInjectionPlacementSchema = require_types.DocxInjectionPlacementSchema;
+exports.DocxInjectionPlacementValues = require_types.DocxInjectionPlacementValues;
 exports.EvalResultsFilterMode = require_types.EvalResultsFilterMode;
 exports.EvaluateOptionsSchema = require_types.EvaluateOptionsSchema;
 exports.GradingConfigSchema = require_types.GradingConfigSchema;
+exports.InputConfigSchema = require_types.InputConfigSchema;
+exports.InputDefinitionObjectSchema = require_types.InputDefinitionObjectSchema;
+exports.InputDefinitionSchema = require_types.InputDefinitionSchema;
+exports.InputTypeSchema = require_types.InputTypeSchema;
+exports.InputTypeValues = require_types.InputTypeValues;
 exports.InputsSchema = require_types.InputsSchema;
 exports.NotPrefixedAssertionTypesSchema = require_types.NotPrefixedAssertionTypesSchema;
 exports.OutputConfigSchema = require_types.OutputConfigSchema;
@@ -14373,6 +15498,7 @@ exports.TestSuiteSchema = require_types.TestSuiteSchema;
 exports.UnifiedConfigSchema = require_types.UnifiedConfigSchema;
 exports.VarsSchema = require_types.VarsSchema;
 exports.assertions = assertions_default;
+exports.buildInputPromptDescription = require_types.buildInputPromptDescription;
 Object.defineProperty(exports, "cache", {
 	enumerable: true,
 	get: function() {
@@ -14382,12 +15508,17 @@ Object.defineProperty(exports, "cache", {
 exports.default = src_default;
 exports.evaluate = evaluate;
 exports.generateTable = generateTable;
+exports.getInputDescription = require_types.getInputDescription;
+exports.getInputType = require_types.getInputType;
 exports.guardrails = guardrails;
 exports.isApiProvider = require_types.isApiProvider;
 exports.isGradingResult = require_types.isGradingResult;
 exports.isProviderOptions = require_types.isProviderOptions;
 exports.isResultFailureReason = require_types.isResultFailureReason;
+exports.isTransformFunction = isTransformFunction;
 exports.loadApiProvider = require_providers.loadApiProvider;
+exports.normalizeInputDefinition = require_types.normalizeInputDefinition;
+exports.normalizeInputs = require_types.normalizeInputs;
 exports.redteam = redteam;
 
 //# sourceMappingURL=index.cjs.map