promptfoo 0.121.4 → 0.121.5

package/dist/src/{graders-pvbReLLn.js → graders-CwrbifOo.js}

@@ -1,25 +1,24 @@
-import { O as isCI, S as getEnvBool, T as getEnvString, a as logger, h as extractJsonObjects, k as state, m as extractFirstJsonObject, y as safeJsonStringify } from "./logger-B88EkIn6.js";
-import { d as sleep, h as REQUEST_TIMEOUT_MS, r as fetchWithTimeout, t as fetchWithProxy } from "./fetch-B0Z3Oe4k.js";
-import { t as invariant } from "./invariant-vgHWClmd.js";
-import { o as getUserEmail } from "./accounts-DdJ2pHMI.js";
-import { r as importModule } from "./esm-Dh4dOLlt.js";
-import { a as getNunjucksEngine, r as extractVariablesFromTemplate } from "./render-DHIZ6_k8.js";
-import { c as getRemoteGenerationUrl, h as hasCodexDefaultCredentials, m as getCodexDefaultProviders, p as shouldGenerateRemote } from "./server-BSB45Nt9.js";
-import { C as isValidReusablePolicyId, Z as LLAMA_GUARD_REPLICATE_PROVIDER, ft as CODING_AGENT_PLUGINS, ht as PromptSchema, k as MULTI_TURN_STRATEGIES, mt as CODING_AGENT_PLUGIN_DISPLAY_NAMES, pt as CODING_AGENT_PLUGIN_DESCRIPTIONS, x as PolicyObjectSchema } from "./types-Bgh5SOn6.js";
-import { i as isJavascriptFile } from "./fileExtensions-BArZuxsI.js";
-import { n as sha256 } from "./createHash-CwDVU5xr.js";
-import { i as PROMPT_DELIMITER, n as maybeFilePath, r as normalizeInput } from "./utils-DJfvjyMj.js";
-import { a as isCacheEnabled, i as getCache, r as fetchWithCache } from "./cache-C4Xb-hNb.js";
-import { r as accumulateTokenUsage } from "./tokenUsageUtils-C-bmyHoE.js";
-import { A as removePrefix, B as isRateLimitWrapped, C as extractInputVarsFromPrompt, D as getShortPluginId, H as MAX_CHARS_PER_MESSAGE_MODIFIER_KEY, I as redteamProviderManager, J as MistralChatCompletionProvider, K as REDTEAM_MEMORY_POISONING_PLUGIN_ID, O as isBasicRefusal, S as extractGoalFromPrompt, T as extractVariablesFromJson, U as getGeneratedPromptOverLimit, W as getMaxCharsPerMessageModifierValue, X as DefaultEmbeddingProvider$2, Y as MistralEmbeddingProvider, Z as DefaultGradingProvider$3, at as AzureModerationProvider, b as checkExfilTracking, et as DefaultGradingJsonProvider$2, it as DefaultSynthesizeProvider$1, k as isEmptyResponse, n as loadApiProvider, nt as DefaultLlmRubricProvider, o as getFileHashes, ot as AzureEmbeddingProvider, pt as getPoliciesFromCloud, q as OpenAiModerationProvider, rt as DefaultSuggestionsProvider$2, s as parseScriptParts, st as AzureChatCompletionProvider, tt as DefaultGradingProvider$2, w as extractPromptFromTags, x as extractAllPromptsFromTags, z as createProviderRateLimitOptions } from "./providers-Domz_llv.js";
-import { r as runPython } from "./pythonUtils-Cldx7huE.js";
-import { N as parseFileUrl, O as maybeLoadToolsFromExternalFile, S as getNunjucksEngineForFilePath, T as maybeLoadFromExternalFile, k as parsePathOrGlob, w as maybeLoadConfigFromExternalFile } from "./util-C8e5uydV.js";
-import { t as OpenAiChatCompletionProvider } from "./chat-BfPaS15_.js";
-import { x as hasGoogleDefaultCredentials } from "./transform-lQrDE1BQ.js";
-import { t as AnthropicMessagesProvider } from "./messages-MYTQ2TWp.js";
-import { t as OpenAiResponsesProvider } from "./responses-C-flexAY.js";
-import { t as OpenAiEmbeddingProvider } from "./embedding-DD9wa3ae.js";
-import { n as transform } from "./transform-DYX1_Xnh.js";
+import { O as isCI, S as getEnvBool, T as getEnvString, a as logger, h as extractJsonObjects, k as state, m as extractFirstJsonObject, y as safeJsonStringify } from "./logger-KD8JjCRJ.js";
+import { d as sleep, h as REQUEST_TIMEOUT_MS, r as fetchWithTimeout, t as fetchWithProxy } from "./fetch-Y5qX_kST.js";
+import { t as invariant } from "./invariant-DIYf9sP1.js";
+import { o as getUserEmail } from "./accounts-bnyHT7Ju.js";
+import { r as importModule } from "./esm-B6whoAcf.js";
+import { a as getNunjucksEngine, r as extractVariablesFromTemplate } from "./render-7uNJ2V14.js";
+import { c as getRemoteGenerationUrl, h as hasCodexDefaultCredentials, m as getCodexDefaultProviders, p as shouldGenerateRemote } from "./server-ByxbqAcQ.js";
+import { C as isValidReusablePolicyId, ft as CODING_AGENT_PLUGINS, ht as PromptSchema, k as MULTI_TURN_STRATEGIES, mt as CODING_AGENT_PLUGIN_DISPLAY_NAMES, pt as CODING_AGENT_PLUGIN_DESCRIPTIONS, x as PolicyObjectSchema } from "./types-BDjGOq4E.js";
+import { i as isJavascriptFile } from "./fileExtensions-BGh-W-HT.js";
+import { n as sha256 } from "./createHash-4gFQpDDv.js";
+import { i as PROMPT_DELIMITER, n as maybeFilePath, r as normalizeInput } from "./utils-BUMN8orw.js";
+import { a as isCacheEnabled, i as getCache, r as fetchWithCache } from "./cache-Cr-qWIbP.js";
+import { r as accumulateTokenUsage } from "./tokenUsageUtils-BjVkdk18.js";
+import { A as removePrefix, B as isRateLimitWrapped, C as extractInputVarsFromPrompt, D as getShortPluginId, H as MAX_CHARS_PER_MESSAGE_MODIFIER_KEY, I as redteamProviderManager, J as MistralChatCompletionProvider, K as REDTEAM_MEMORY_POISONING_PLUGIN_ID, O as isBasicRefusal, S as extractGoalFromPrompt, T as extractVariablesFromJson, U as getGeneratedPromptOverLimit, W as getMaxCharsPerMessageModifierValue, X as DefaultEmbeddingProvider$2, Y as MistralEmbeddingProvider, Z as DefaultGradingProvider$3, at as AzureModerationProvider, b as checkExfilTracking, et as DefaultGradingJsonProvider$2, it as DefaultSynthesizeProvider$1, k as isEmptyResponse, n as loadApiProvider, nt as DefaultLlmRubricProvider, o as getFileHashes, ot as AzureEmbeddingProvider, pt as getPoliciesFromCloud, q as OpenAiModerationProvider, rt as DefaultSuggestionsProvider$2, s as parseScriptParts, st as AzureChatCompletionProvider, tt as DefaultGradingProvider$2, w as extractPromptFromTags, x as extractAllPromptsFromTags, z as createProviderRateLimitOptions } from "./providers-DRrerKra.js";
+import { r as runPython } from "./pythonUtils-C4tltmIn.js";
+import { E as getNunjucksEngineForFilePath, L as parseFileUrl, M as maybeLoadToolsFromExternalFile, N as parsePathOrGlob, O as maybeLoadConfigFromExternalFile, k as maybeLoadFromExternalFile } from "./util-3pBZZb_H.js";
+import { t as OpenAiChatCompletionProvider } from "./chat-DChSH_Es.js";
+import { x as hasGoogleDefaultCredentials } from "./transform-DtooZqYY.js";
+import { t as AnthropicMessagesProvider } from "./messages-CewuNcNS.js";
+import { t as OpenAiResponsesProvider } from "./responses-1ztiVYsx.js";
+import { t as OpenAiEmbeddingProvider } from "./embedding-D_bI4NDq.js";
 import { AsyncLocalStorage } from "node:async_hooks";
 import * as fs$2 from "fs";
 import fs from "fs";
@@ -337,55 +336,120 @@ async function getCustomPolicies(policyPluginsWithRefs, teamId) {
 	return policiesById;
 }
 //#endregion
-//#region src/assertions/contextUtils.ts
+//#region src/scheduler/providerCallExecutionContext.ts
+const providerCallExecutionContext = new AsyncLocalStorage();
+function getProviderCallExecutionContext() {
+	return providerCallExecutionContext.getStore();
+}
+function withProviderCallExecutionContext(context, fn) {
+	return providerCallExecutionContext.run(context, fn);
+}
+//#endregion
+//#region src/matchers/providers.ts
 /**
-* Resolves the context value for context-based assertions.
-* Supports extracting context from test variables or transforming from output.
-* Can return either a single context string or an array of context chunks.
+* Helper to call provider with consistent context propagation pattern.
+* Spreads the optional context and merges with prompt label and vars.
+* Also reuses evaluator scheduler context for cancellation, rate limits,
+* and grouped grading provider calls when present.
 *
-* @param assertion - The assertion configuration
-* @param test - The test case
-* @param output - The provider output (after provider transform, before test transform)
-* @param prompt - The prompt text
-* @param fallbackContext - Optional fallback context (e.g., prompt for context-recall)
-* @param providerResponse - Optional full provider response for contextTransform
-* @returns The resolved context string or array of strings
-* @throws Error if context cannot be resolved or transform fails
+* IMPORTANT: Spread order matters - context is spread first, then prompt/vars
+* override. This ensures originalProvider from context is preserved while
+* allowing this call to specify its own prompt metadata.
 */
-async function resolveContext(assertion, test, output, prompt, fallbackContext, providerResponse) {
-	let contextValue;
-	if (test.vars?.context) {
-		if (typeof test.vars.context === "string") contextValue = test.vars.context;
-		else if (Array.isArray(test.vars.context)) {
-			const invalidEntry = [...test.vars.context.entries()].find(([, v]) => typeof v !== "string");
-			if (invalidEntry) {
-				const [idx, val] = invalidEntry;
-				invariant(false, `Invalid context: expected an array of strings, but found ${typeof val} at index ${idx}`);
+function callProviderWithContext(provider, prompt, label, vars, context) {
+	const callApiContext = {
+		...context,
+		prompt: {
+			raw: prompt,
+			label
+		},
+		vars
+	};
+	const executionContext = getProviderCallExecutionContext();
+	const callApiOptions = executionContext?.abortSignal ? { abortSignal: executionContext.abortSignal } : void 0;
+	const callApi = () => callApiOptions ? provider.callApi(prompt, callApiContext, callApiOptions) : provider.callApi(prompt, callApiContext);
+	const executeCall = () => {
+		if (executionContext?.rateLimitRegistry && !isRateLimitWrapped(provider)) return executionContext.rateLimitRegistry.execute(provider, callApi, createProviderRateLimitOptions());
+		return callApi();
+	};
+	if (executionContext?.providerCallQueue) return executionContext.providerCallQueue.enqueue(provider.id(), executeCall);
+	return executeCall();
+}
+async function loadFromProviderOptions(provider) {
+	invariant(typeof provider === "object", `Provider must be an object, but received a ${typeof provider}: ${provider}`);
+	invariant(!Array.isArray(provider), `Provider must be an object, but received an array: ${JSON.stringify(provider)}`);
+	invariant(provider.id, "Provider supplied to assertion must have an id");
+	return loadApiProvider(provider.id, {
+		options: provider,
+		basePath: state.basePath
+	});
+}
+function isSimulatedUserProviderConfig(provider) {
+	if (typeof provider === "string") return provider === "promptfoo:simulated-user";
+	if (!provider || typeof provider !== "object" || Array.isArray(provider)) return false;
+	if (typeof provider.id === "function") return provider.id() === "promptfoo:simulated-user";
+	const providerId = provider.id;
+	if (typeof providerId === "string") return providerId === "promptfoo:simulated-user";
+	return Object.values(provider).some((providerTypeConfig) => isSimulatedUserProviderConfig(providerTypeConfig));
+}
+async function getGradingProvider(type, provider, defaultProvider) {
+	let finalProvider;
+	if (typeof provider === "string") finalProvider = await loadApiProvider(provider, { basePath: state.basePath });
+	else if (provider != null && typeof provider === "object" && typeof provider.id === "function") finalProvider = provider;
+	else if (provider != null && typeof provider === "object") {
+		const typeValue = provider[type];
+		if (typeValue) finalProvider = await getGradingProvider(type, typeValue, defaultProvider);
+		else if (provider.id) finalProvider = await loadFromProviderOptions(provider);
+		else if (Array.isArray(provider)) throw new Error(`Provider must be an object or string, but received an array.\n\nCheck that the provider ${JSON.stringify(provider[0], null, 2)} is not nested in an array.`);
+		else throw new Error(`Invalid provider definition for output type '${type}': ${JSON.stringify(provider, null, 2)}`);
+	} else {
+		const defaultTest = state.config?.defaultTest;
+		const defaultTestObj = typeof defaultTest === "object" ? defaultTest : null;
+		const cfg = [
+			defaultTestObj?.provider || void 0,
+			defaultTestObj?.options?.provider?.text || void 0,
+			defaultTestObj?.options?.provider || void 0
+		].find((candidateProvider) => {
+			if (!candidateProvider) return false;
+			if (isSimulatedUserProviderConfig(candidateProvider)) {
+				logger.debug("[Grading] Skipping promptfoo:simulated-user as an implicit grader fallback");
+				return false;
 			}
-			contextValue = test.vars.context;
-		}
-	} else if (fallbackContext) contextValue = fallbackContext;
-	if (assertion.contextTransform) try {
-		const outputForTransform = providerResponse?.providerTransformedOutput ?? output;
-		const transformed = await transform(assertion.contextTransform, outputForTransform, {
-			vars: test.vars,
-			prompt: { label: prompt },
-			...providerResponse && providerResponse.metadata && { metadata: providerResponse.metadata }
+			return true;
 		});
-		invariant(typeof transformed === "string" || Array.isArray(transformed) && transformed.every((item) => typeof item === "string"), `contextTransform must return a string or array of strings. Got ${typeof transformed}. Check your transform expression: ${assertion.contextTransform}`);
-		contextValue = transformed;
-	} catch (error) {
-		throw new Error(`Failed to transform context using expression '${assertion.contextTransform}': ${error instanceof Error ? error.message : String(error)}`);
+		if (cfg) {
+			finalProvider = await getGradingProvider(type, cfg, defaultProvider);
+			if (finalProvider) logger.debug("[Grading] Using provider from defaultTest fallback", { providerId: finalProvider.id() });
+		} else finalProvider = defaultProvider;
 	}
-	invariant(typeof contextValue === "string" && contextValue.length > 0 || Array.isArray(contextValue) && contextValue.length > 0 && contextValue.every((item) => typeof item === "string" && item.length > 0), "Context is required for context-based assertions. Provide either a \"context\" variable (string or array of strings) in your test case or use \"contextTransform\" to extract context from the provider response.");
-	return contextValue;
+	return finalProvider;
 }
-/**
-* Serializes context (string or string[]) to a single string for prompts.
-* Joins chunks with double newlines to preserve separation.
-*/
-function serializeContext(context) {
-	return Array.isArray(context) ? context.join("\n\n") : context;
+async function getAndCheckProvider(type, provider, defaultProvider, checkName) {
+	const matchedProvider = await getGradingProvider(type, provider, defaultProvider);
+	if (!matchedProvider) if (defaultProvider) {
+		logger.warn("[Grading] Falling back to default provider", {
+			checkName,
+			type
+		});
+		return defaultProvider;
+	} else throw new Error(`No provider of type ${type} found for '${checkName}'`);
+	let isValidProviderType = true;
+	if (type === "embedding") isValidProviderType = "callEmbeddingApi" in matchedProvider || "callSimilarityApi" in matchedProvider;
+	else if (type === "classification") isValidProviderType = "callClassificationApi" in matchedProvider;
+	else if (type === "moderation") isValidProviderType = "callModerationApi" in matchedProvider;
+	if (!isValidProviderType) {
+		if (provider) throw new Error(`Provider ${matchedProvider.id()} is not a valid ${type} provider for '${checkName}'`);
+		if (defaultProvider) {
+			logger.warn("[Grading] Falling back to default provider after type check failed", {
+				checkName,
+				providerId: matchedProvider.id(),
+				type
+			});
+			return defaultProvider;
+		}
+		throw new Error(`Provider ${matchedProvider.id()} is not a valid ${type} provider for '${checkName}'`);
+	}
+	return matchedProvider;
 }
 //#endregion
 //#region src/assertions/utils.ts
@@ -430,223 +494,162 @@ function coerceString(value) {
 	return JSON.stringify(value);
 }
 //#endregion
-//#region src/external/prompts/ragas.ts
-const ANSWER_RELEVANCY_GENERATE = `Generate question for the given answer.
-Answer:\nThe PSLV-C56 mission is scheduled to be launched on Sunday, 30 July 2023 at 06:30 IST / 01:00 UTC. It will be launched from the Satish Dhawan Space Centre, Sriharikota, Andhra Pradesh, India
-Question: When is the scheduled launch date and time for the PSLV-C56 mission, and where will it be launched from?
-
-Answer:{{answer}}
-Question:`;
-const CONTEXT_RECALL = `Given a context, and an answer, analyze each sentence in the answer and classify if the sentence can be attributed to the given context or not.
-Think in steps and reason before coming to conclusion.
-
-context: Albert Einstein (14 March 1879 – 18 April 1955) was a German-born theoretical physicist,widely held to be one of the greatest and most influential scientists of all time. Best known for developing the theory of relativity, he also made important contributions to quantum mechanics, and was thus a central figure in the revolutionary reshaping of the scientific understanding of nature that modern physics accomplished in the first decades of the twentieth century. His mass–energy equivalence formula E = mc2, which arises from relativity theory, has been called "the world's most famous equation". He received the 1921 Nobel Prize in Physics "for his services to theoretical physics, and especially for his discovery of the law of the photoelectric effect", a pivotal step in the development of quantum theory. His work is also known for its influence on the philosophy of science. In a 1999 poll of 130 leading physicists worldwide by the British journal Physics World, Einstein was ranked the greatest physicist of all time. His intellectual achievements and originality have made Einstein synonymous with genius.
-answer: Albert Einstein born in 14 March 1879 was  German-born theoretical physicist, widely held to be one of the greatest and most influential scientists of all time. He received the 1921 Nobel Prize in Physics "for his services to theoretical physics. He published 4 papers in 1905.  Einstein moved to Switzerland in 1895
-classification
-1. Albert Einstein born in 14 March 1879 was  German-born theoretical physicist, widely held to be one of the greatest and most influential scientists of all time. The date of birth of Einstein is mentioned clearly in the context. So [Attributed]
-2. He received the 1921 Nobel Prize in Physics "for his services to theoretical physics. The exact sentence is present in the given context. So [Attributed]
-3. He published 4 papers in 1905. There is no mention about papers he wrote in given the context. So [Not Attributed]
-4. Einstein moved to Switzerland in 1895. There is not supporting evidence for this in the given the context. So [Not Attributed]
-
-context:{{context}}
-answer:{{groundTruth}}
-classification:
-`;
-const CONTEXT_RECALL_ATTRIBUTED_TOKEN = "[Attributed]";
-const CONTEXT_RECALL_NOT_ATTRIBUTED_TOKEN = "[Not Attributed]";
-const CONTEXT_RELEVANCE = `Please extract relevant sentences from the provided context that is absolutely required answer the following query. If no relevant sentences are found, or if you believe the query cannot be answered from the given context, return the phrase "Insufficient Information".  While extracting candidate sentences you're not allowed to make any changes to sentences from given context.
-
-query: {{query}}
-context: {{context}}
-candidate sentences:
-`;
-const CONTEXT_RELEVANCE_BAD = "Insufficient Information";
-const CONTEXT_FAITHFULNESS_LONGFORM = `Given a question and answer, create one or more statements from each sentence in the given answer.
-question: Who was  Albert Einstein and what is he best known for?
-answer: He was a German-born theoretical physicist, widely acknowledged to be one of the greatest and most influential physicists of all time. He was best known for developing the theory of relativity, he also made important contributions to the development of the theory of quantum mechanics.
-statements:\nAlbert Einstein was born in Germany.\nAlbert Einstein was best known for his theory of relativity.
-question: Cadmium Chloride is slightly soluble in this chemical, it is also called what?
-answer: alcohol
-statements:\nCadmium Chloride is slightly soluble in alcohol.
-question: Were Shahul and Jithin of the same nationality?
-answer: They were from different countries.
-statements:\nShahul and Jithin were from different countries.
-question:{{question}}
-answer: {{answer}}
-statements:\n`;
-const CONTEXT_FAITHFULNESS_NLI_STATEMENTS = `Prompt: Natural language inference
-Consider the given context and following statements, then determine whether they are supported by the information present in the context.Provide a brief explanation for each statement before arriving at the verdict (Yes/No). Provide a final verdict for each statement in order at the end in the given format. Do not deviate from the specified format.
-
-Context:\nJohn is a student at XYZ University. He is pursuing a degree in Computer Science. He is enrolled in several courses this semester, including Data Structures, Algorithms, and Database Management. John is a diligent student and spends a significant amount of time studying and completing assignments. He often stays late in the library to work on his projects.
-statements:\n1. John is majoring in Biology.\n2. John is taking a course on Artificial Intelligence.\n3. John is a dedicated student.\n4. John has a part-time job.\n5. John is interested in computer programming.\n
-Answer:
-1. John is majoring in Biology.
-Explanation: John's major is explicitly mentioned as Computer Science. There is no information suggesting he is majoring in Biology.  Verdict: No.
-2. John is taking a course on Artificial Intelligence.
-Explanation: The context mentions the courses John is currently enrolled in, and Artificial Intelligence is not mentioned. Therefore, it cannot be deduced that John is taking a course on AI. Verdict: No.
-3. John is a dedicated student.
-Explanation: The prompt states that he spends a significant amount of time studying and completing assignments. Additionally, it mentions that he often stays late in the library to work on his projects, which implies dedication. Verdict: Yes.
-4. John has a part-time job.
-Explanation: There is no information given in the context about John having a part-time job. Therefore, it cannot be deduced that John has a part-time job.  Verdict: No.
-5. John is interested in computer programming.
-Explanation: The context states that John is pursuing a degree in Computer Science, which implies an interest in computer programming. Verdict: Yes.
-Final verdict for each statement in order: No. No. Yes. No. Yes.
-context:\n{{context}}
-statements:\n{{statements|join("\\n")}}
-Answer:
-`;
+//#region src/matchers/shared.ts
+/**
+* Normalize token usage for matcher results. Unlike the evaluator-level
+* normalizeTokenUsage, this excludes the `assertions` field and preserves
+* the existing completionDetails shape (passing through whatever the
+* provider returned, or undefined if not present).
+*/
+function normalizeMatcherTokenUsage(tokenUsage) {
+	return {
+		total: tokenUsage?.total || 0,
+		prompt: tokenUsage?.prompt || 0,
+		completion: tokenUsage?.completion || 0,
+		cached: tokenUsage?.cached || 0,
+		numRequests: tokenUsage?.numRequests || 0,
+		completionDetails: tokenUsage?.completionDetails || {
+			reasoning: 0,
+			acceptedPrediction: 0,
+			rejectedPrediction: 0
+		}
+	};
+}
+function fail(reason, tokensUsed) {
+	return {
+		pass: false,
+		reason,
+		score: 0,
+		tokensUsed: normalizeMatcherTokenUsage(tokensUsed)
+	};
+}
+function cosineSimilarity(vecA, vecB) {
+	if (vecA.length !== vecB.length) throw new Error("Vectors must be of equal length");
+	const dotProduct = vecA.reduce((acc, val, idx) => acc + val * vecB[idx], 0);
+	const vecAMagnitude = Math.sqrt(vecA.reduce((acc, val) => acc + val * val, 0));
+	const vecBMagnitude = Math.sqrt(vecB.reduce((acc, val) => acc + val * val, 0));
+	if (vecAMagnitude === 0 || vecBMagnitude === 0) return 0;
+	return dotProduct / (vecAMagnitude * vecBMagnitude);
+}
+function dotProduct(vecA, vecB) {
+	if (vecA.length !== vecB.length) throw new Error("Vectors must be of equal length");
+	return vecA.reduce((acc, val, idx) => acc + val * vecB[idx], 0);
+}
+function euclideanDistance(vecA, vecB) {
+	if (vecA.length !== vecB.length) throw new Error("Vectors must be of equal length");
+	const sumSquaredDiff = vecA.reduce((acc, val, idx) => {
+		const diff = val - vecB[idx];
+		return acc + diff * diff;
+	}, 0);
+	return Math.sqrt(sumSquaredDiff);
+}
+function tryParse(content) {
+	try {
+		return JSON.parse(content);
+	} catch {}
+	return content;
+}
+function splitIntoSentences(text) {
+	return text.split("\n").filter((sentence) => sentence.trim() !== "");
+}
 //#endregion
-//#region src/prompts/grading.ts
-const DEFAULT_GRADING_PROMPT = JSON.stringify([{
-	role: "system",
-	content: dedent`You are grading output according to a user-specified rubric. If the statement in the rubric is true, then the output passes the test. You respond with a JSON object with this structure: {reason: string, pass: boolean, score: number}
-
-    Examples:
-
-    <Output>Hello world</Output>
-    <Rubric>Content contains a greeting</Rubric>
-    {"reason": "the content contains the word 'Hello'", "pass": true, "score": 1.0}
-
-    <Output>Avast ye swabs, repel the invaders!</Output>
-    <Rubric>Does not speak like a pirate</Rubric>
-    {"reason": "'avast ye' is a common pirate term", "pass": false, "score": 0.0}`
-}, {
-	role: "user",
-	content: "<Output>\n{{ output }}\n</Output>\n<Rubric>\n{{ rubric }}\n</Rubric>"
-}]);
-const PROMPTFOO_FACTUALITY_PROMPT = JSON.stringify([{
-	role: "system",
-	content: dedent`
-      You are a precise factuality evaluator that compares a submitted answer to an expert answer. 
-
-      Your task is to analyze the factual content while ignoring differences in style, grammar, or punctuation.
-      You must categorize the submission into one of these options:
-
-      (A) The submitted answer is a subset of the expert answer and is fully consistent with it.
-      (B) The submitted answer is a superset of the expert answer and is fully consistent with it.
-      (C) The submitted answer contains all the same details as the expert answer.
-      (D) There is a disagreement between the submitted answer and the expert answer.
-      (E) The answers differ, but these differences don't matter from the perspective of factuality.
-
-      Respond ONLY with a JSON object in this format:
-      {
-        "category": "[LETTER]",
-        "reason": "[DETAILED EXPLANATION]"
-      }
-
-      - The "category" must be a single letter A, B, C, D, or E.
-      - Provide a clear, detailed explanation in the "reason" field.
-      - Your response must be valid JSON with no additional text.`
-}, {
-	role: "user",
-	content: dedent`
-      I need you to compare these answers:
-
-      <question>
-      {{input}}
-      </question>
-
-      <expert_answer>
-      {{ideal}}
-      </expert_answer>
-
-      <submitted_answer>
-      {{completion}}
-      </submitted_answer>
-
-      Please analyze the factual relationship between these answers according to the categories you've been given.`
-}]);
-const OPENAI_CLOSED_QA_PROMPT = JSON.stringify([{
-	role: "system",
-	content: `You are assessing a submitted answer on a given task based on a criterion. Here is the data:
-[BEGIN DATA]
-***
-[Task]: {{input}}
-***
-[Submission]: {{completion}}
-***
-[Criterion]: {{criteria}}
-***
-[END DATA]
-Does the submission meet the criterion? First, write out in a step by step manner your reasoning about the criterion to be sure that your conclusion is correct. Avoid simply stating the correct answers at the outset. Then print only the single character "Y" or "N" (without quotes or punctuation) on its own line corresponding to the correct answer. At the end, repeat just the letter again by itself on a new line.
-
-    Reasoning:`
-}]);
-const SUGGEST_PROMPTS_SYSTEM_MESSAGE = {
-	role: "system",
-	content: `You're helping a scientist who is tuning a prompt for a large language model.  You will receive messages, and each message is a full prompt.  Generate a candidate variation of the given prompt.  This variation will be tested for quality in order to select a winner.
-
-Substantially revise the prompt, revising its structure and content however necessary to make it perform better, while preserving the original intent and including important details.
-
-Your output is going to be copied directly into the program. It should contain the prompt ONLY`
+//#region src/matchers/rubric.ts
+const nunjucks = getNunjucksEngine(void 0, false, true);
+var LlmRubricProviderError = class extends Error {
+	constructor(message) {
+		super(message);
+		this.name = "LlmRubricProviderError";
+	}
 };
-const SELECT_BEST_PROMPT = JSON.stringify([{
-	role: "system",
-	content: `You are comparing multiple pieces of text to see which best fits the following criteria: {{criteria}}
-
-Here are the pieces of text:
-
-{% for output in outputs %}
-<Text index="{{ loop.index0 }}">
-{{ output }}
-</Text>
-{% endfor %}
-
-Output the index of the text that best fits the criteria. You must output a single integer.`
-}]);
-const DEFAULT_WEB_SEARCH_PROMPT = JSON.stringify([{
-	role: "system",
-	content: dedent`You are grading output according to a user-specified rubric, with the ability to search the web for current information. If the statement in the rubric is true, then the output passes the test. You respond with a JSON object with this structure: {reason: string, pass: boolean, score: number}
-
-    You MUST search the web when:
-    - The rubric asks about current information (prices, weather, news, etc.)
-    - Facts need to be verified against recent data
-    - The rubric references time-sensitive information
-
-    Examples:
-
-    <Output>The current CEO of Microsoft is Satya Nadella</Output>
-    <Rubric>Contains accurate information about Microsoft's leadership</Rubric>
-    {"reason": "I searched and confirmed Satya Nadella is indeed the current CEO of Microsoft", "pass": true, "score": 1.0}
-
-    <Output>Bitcoin is trading at $45,000</Output>
-    <Rubric>Provides current Bitcoin price within 10% accuracy</Rubric>
-    {"reason": "Web search shows Bitcoin is currently trading at $98,000, not $45,000. The output is off by more than 50%", "pass": false, "score": 0.0}`
-}, {
-	role: "user",
-	content: "<Output>\n{{ output }}\n</Output>\n<Rubric>\n{{ rubric }}\n</Rubric>"
-}]);
-const TRAJECTORY_GOAL_SUCCESS_PROMPT = JSON.stringify([{
-	role: "system",
-	content: dedent`You are grading whether an AI agent successfully completed a goal based on its final output and a summarized execution trajectory. You respond with a JSON object with this structure: {reason: string, pass: boolean, score: number}
-
-    Judge end-to-end success, not stylistic perfection.
-    Use the trajectory as evidence for what the agent actually did.
-    Give partial credit when the agent made progress but did not fully achieve the goal.
-
-    Examples:
-
-    <Goal>Find the order status and tell the user whether it has shipped</Goal>
-    <Trajectory>{"stepCount":2,"steps":[{"index":1,"type":"tool","name":"search_orders"},{"index":2,"type":"message","name":"agent response"}]}</Trajectory>
-    <Output>Your order shipped yesterday and should arrive on Tuesday.</Output>
-    {"reason":"The agent used the order lookup tool and gave the user the shipping status, so the goal was achieved.","pass":true,"score":1.0}
-
-    <Goal>Find the order status and tell the user whether it has shipped</Goal>
-    <Trajectory>{"stepCount":1,"steps":[{"index":1,"type":"message","name":"agent response"}]}</Trajectory>
-    <Output>I cannot check your order right now.</Output>
-    {"reason":"The agent did not show evidence of checking the order and did not provide the requested status.","pass":false,"score":0.0}`
-}, {
-	role: "user",
-	content: dedent`<Goal>
-{{ goal }}
-</Goal>
-<Trajectory>
-{{ trajectory }}
-</Trajectory>
-<Output>
-{{ output }}
-</Output>`
-}]);
+async function loadRubricPrompt(rubricPrompt, defaultPrompt) {
+	if (!rubricPrompt) return defaultPrompt;
+	if (typeof rubricPrompt === "object" && Object.keys(rubricPrompt).length === 0) return defaultPrompt;
+	if (typeof rubricPrompt === "string" && rubricPrompt.startsWith("file://")) {
+		const basePath = state.basePath || "";
+		const { filePath, functionName } = parseFileUrl(getNunjucksEngineForFilePath().renderString(rubricPrompt, {}));
+		const resolvedPath = path.resolve(basePath, filePath);
+		if (isJavascriptFile(filePath)) rubricPrompt = await loadFromJavaScriptFile(resolvedPath, functionName, []);
+		else {
+			if (!fs$2.existsSync(resolvedPath)) throw new Error(`File does not exist: ${resolvedPath}`);
+			rubricPrompt = fs$2.readFileSync(resolvedPath, "utf8");
+		}
+	} else rubricPrompt = maybeLoadFromExternalFile(rubricPrompt);
+	if (typeof rubricPrompt === "object") rubricPrompt = JSON.stringify(rubricPrompt);
+	invariant(typeof rubricPrompt === "string", "rubricPrompt must be a string");
+	return rubricPrompt;
+}
+function processContextForTemplating(context, enableObjectAccess) {
+	if (enableObjectAccess) return context;
+	return Object.fromEntries(Object.entries(context).map(([key, value]) => {
+		if (value && typeof value === "object") {
+			if (Array.isArray(value)) return [key, value.map((item) => item && typeof item === "object" ? JSON.stringify(item) : item)];
+			return [key, JSON.stringify(value)];
+		}
+		return [key, value];
+	}));
+}
+async function renderLlmRubricPrompt(rubricPrompt, context) {
+	const processedContext = processContextForTemplating(context, getEnvBool("PROMPTFOO_DISABLE_OBJECT_STRINGIFY", false));
+	try {
+		const parsed = JSON.parse(rubricPrompt, (_k, v) => typeof v === "string" ? nunjucks.renderString(v, processedContext) : v);
+		return JSON.stringify(parsed);
+	} catch (err) {
+		logger.debug(`[Rubric] Rubric prompt is not valid JSON, using Nunjucks rendering: ${err.message}`);
+	}
+	return nunjucks.renderString(rubricPrompt, processedContext);
+}
+function parseJsonGradingResponse(label, resp) {
+	let jsonObjects = [];
+	if (typeof resp.output === "string") try {
+		jsonObjects = extractJsonObjects(resp.output);
+		if (jsonObjects.length === 0) return { failure: fail(`Could not extract JSON from ${label} response`, resp.tokenUsage) };
+	} catch (err) {
+		return { failure: fail(`${label} produced malformed response: ${err}\n\n${resp.output}`, resp.tokenUsage) };
+	}
+	else if (typeof resp.output === "object" && resp.output !== null && !Array.isArray(resp.output)) jsonObjects = [resp.output];
+	else return { failure: fail(`${label} produced malformed response - output must be string or object. Output: ${JSON.stringify(resp.output)}`, resp.tokenUsage) };
+	const parsed = jsonObjects[0];
+	if (typeof parsed !== "object" || parsed === null || Array.isArray(parsed)) return { failure: fail(`${label} produced malformed response. We were not able to parse the response as JSON. Output: ${JSON.stringify(resp.output)}`, resp.tokenUsage) };
+	return { parsed };
+}
+async function runJsonGradingPrompt({ assertion, checkName, defaultPrompt, grading, label, providerCallContext, throwOnError, vars }) {
+	const prompt = await renderLlmRubricPrompt(await loadRubricPrompt(grading.rubricPrompt, defaultPrompt), vars);
+	const defaultProviders = await getDefaultProviders();
+	const defaultProvider = defaultProviders.llmRubricProvider || defaultProviders.gradingJsonProvider;
+	const resp = await callProviderWithContext(await getAndCheckProvider("text", grading.provider, defaultProvider, checkName), prompt, label, vars, providerCallContext);
+	if (resp.error || !resp.output) {
+		if (throwOnError) throw new Error(resp.error || "No output");
+		return fail(resp.error || "No output", resp.tokenUsage);
+	}
+	const { parsed, failure } = parseJsonGradingResponse(label, resp);
+	if (!parsed) return failure;
+	let pass = parsed.pass ?? true;
+	if (typeof pass !== "boolean") pass = /^(true|yes|pass|y)$/i.test(String(pass));
+	let score = parsed.score;
+	if (typeof score !== "number") score = Number.isFinite(Number(score)) ? Number(score) : Number(pass);
+	const threshold = typeof assertion?.threshold === "string" ? Number(assertion.threshold) : assertion?.threshold;
+	if (typeof threshold === "number" && Number.isFinite(threshold)) pass = pass && score >= threshold;
+	const reason = parsed.reason || (pass ? "Grading passed" : `Score ${score} below threshold ${threshold}`);
+	let responseMetadata = {};
+	if (resp.metadata && typeof resp.metadata === "object" && !Array.isArray(resp.metadata)) {
+		const serializedMetadata = safeJsonStringify(resp.metadata);
+		responseMetadata = serializedMetadata ? JSON.parse(serializedMetadata) : {};
+	}
+	return {
+		assertion,
+		pass,
+		score,
+		reason,
+		tokensUsed: normalizeMatcherTokenUsage({
+			...resp.tokenUsage,
+			completionDetails: resp.tokenUsage?.completionDetails || parsed.tokensUsed?.completionDetails
+		}),
+		metadata: {
+			...responseMetadata,
+			renderedGradingPrompt: prompt
+		}
+	};
+}
 //#endregion
 //#region src/prompts/processors/csv.ts
 /**
@@ -939,97 +942,315 @@ const pythonPromptFunctionLegacy = async (filePath, context) => {
 	logger.debug(`Python prompt script ${filePath} returned: ${results}`);
 	return results;
 };
-/**
-* Processes a Python file to extract or execute a function as a prompt.
-* @param filePath - Path to the Python file.
-* @param prompt - The raw prompt data.
-* @param functionName - Optional function name to execute.
-* @returns Array of prompts extracted or executed from the file.
-*/
-function processPythonFile(filePath, prompt, functionName) {
-	const fileContent = fs$2.readFileSync(filePath, "utf-8");
-	return [{
-		raw: fileContent,
-		label: prompt.label ?? (functionName ? `${filePath}:${functionName}` : `${filePath}: ${fileContent}`),
-		function: functionName ? (context) => pythonPromptFunction(filePath, functionName, {
-			...context,
-			config: prompt.config
-		}) : (context) => pythonPromptFunctionLegacy(filePath, {
-			...context,
-			config: prompt.config
-		}),
-		config: prompt.config
-	}];
-}
-//#endregion
-//#region src/prompts/processors/string.ts
-/**
-* Processes a string as a literal prompt.
-* @param prompt - The raw prompt data.
-* @returns Array of prompts created from the string.
-*/
-function processString(prompt) {
-	invariant(typeof prompt.raw === "string", `prompt.raw must be a string, but got ${JSON.stringify(prompt.raw)}`);
-	return [{
-		id: prompt.id,
-		raw: prompt.raw,
-		label: prompt.label ?? `${prompt.raw}`,
-		config: prompt.config
-	}];
-}
-//#endregion
-//#region src/prompts/processors/text.ts
-/**
-* Processes a text file to extract prompts, splitting by a delimiter.
-* @param filePath - Path to the text file.
-* @param prompt - The raw prompt data.
-* @returns Array of prompts extracted from the file.
-*/
-function processTxtFile(filePath, { label }) {
-	const lines = fs$2.readFileSync(filePath, "utf-8").split(/\r?\n/);
-	const prompts = [];
-	let buffer = [];
-	const flush = () => {
-		const raw = buffer.join("\n").trim();
-		if (raw.length > 0) prompts.push({
-			raw,
-			label: label ? `${label}: ${filePath}: ${raw}` : `${filePath}: ${raw}`
-		});
-		buffer = [];
-	};
-	for (const line of lines) if (line.trim() === PROMPT_DELIMITER) flush();
-	else buffer.push(line);
-	flush();
-	return prompts;
-}
-//#endregion
-//#region src/prompts/processors/yaml.ts
-/**
-* Processes a YAML file to extract prompts.
-* This function reads a YAML file, parses it, and maps each entry to a `Prompt` object.
-* Each prompt is labeled with the file path and the YAML content.
-* Any file:// references within the YAML content are recursively resolved.
-*
-* @param filePath - The path to the YAML file.
-* @param prompt - The raw prompt data, used for labeling.
-* @returns An array of `Prompt` objects extracted from the YAML file.
-* @throws Will throw an error if the file cannot be read or parsed.
-*/
-function processYamlFile(filePath, prompt) {
-	const fileContents = fs$2.readFileSync(filePath, "utf8");
-	let maybeParsed = fileContents;
-	try {
-		const resolved = maybeLoadConfigFromExternalFile(yaml.load(fileContents));
-		maybeParsed = JSON.stringify(resolved);
-	} catch (e) {
-		logger.debug(`Error parsing YAML file ${filePath}: ${e}`);
-	}
-	return [{
-		raw: maybeParsed,
-		label: prompt.label || `${filePath}: ${maybeParsed?.slice(0, 80)}`,
-		config: prompt.config
-	}];
-}
+/**
+* Processes a Python file to extract or execute a function as a prompt.
+* @param filePath - Path to the Python file.
+* @param prompt - The raw prompt data.
+* @param functionName - Optional function name to execute.
+* @returns Array of prompts extracted or executed from the file.
+*/
+function processPythonFile(filePath, prompt, functionName) {
+	const fileContent = fs$2.readFileSync(filePath, "utf-8");
+	return [{
+		raw: fileContent,
+		label: prompt.label ?? (functionName ? `${filePath}:${functionName}` : `${filePath}: ${fileContent}`),
+		function: functionName ? (context) => pythonPromptFunction(filePath, functionName, {
+			...context,
+			config: prompt.config
+		}) : (context) => pythonPromptFunctionLegacy(filePath, {
+			...context,
+			config: prompt.config
+		}),
+		config: prompt.config
+	}];
+}
+//#endregion
+//#region src/prompts/processors/string.ts
+/**
+* Processes a string as a literal prompt.
+* @param prompt - The raw prompt data.
+* @returns Array of prompts created from the string.
+*/
+function processString(prompt) {
+	invariant(typeof prompt.raw === "string", `prompt.raw must be a string, but got ${JSON.stringify(prompt.raw)}`);
+	return [{
+		id: prompt.id,
+		raw: prompt.raw,
+		label: prompt.label ?? `${prompt.raw}`,
+		config: prompt.config
+	}];
+}
+//#endregion
+//#region src/prompts/processors/text.ts
+/**
+* Processes a text file to extract prompts, splitting by a delimiter.
+* @param filePath - Path to the text file.
+* @param prompt - The raw prompt data.
+* @returns Array of prompts extracted from the file.
+*/
+function processTxtFile(filePath, { label }) {
+	const lines = fs$2.readFileSync(filePath, "utf-8").split(/\r?\n/);
+	const prompts = [];
+	let buffer = [];
+	const flush = () => {
+		const raw = buffer.join("\n").trim();
+		if (raw.length > 0) prompts.push({
+			raw,
+			label: label ? `${label}: ${filePath}: ${raw}` : `${filePath}: ${raw}`
+		});
+		buffer = [];
+	};
+	for (const line of lines) if (line.trim() === PROMPT_DELIMITER) flush();
+	else buffer.push(line);
+	flush();
+	return prompts;
+}
+//#endregion
+//#region src/prompts/processors/yaml.ts
+/**
+* Processes a YAML file to extract prompts.
+* This function reads a YAML file, parses it, and maps each entry to a `Prompt` object.
+* Each prompt is labeled with the file path and the YAML content.
+* Any file:// references within the YAML content are recursively resolved.
+*
+* @param filePath - The path to the YAML file.
+* @param prompt - The raw prompt data, used for labeling.
+* @returns An array of `Prompt` objects extracted from the YAML file.
+* @throws Will throw an error if the file cannot be read or parsed.
+*/
+function processYamlFile(filePath, prompt) {
+	const fileContents = fs$2.readFileSync(filePath, "utf8");
+	let maybeParsed = fileContents;
+	try {
+		const resolved = maybeLoadConfigFromExternalFile(yaml.load(fileContents));
+		maybeParsed = JSON.stringify(resolved);
+	} catch (e) {
+		logger.debug(`Error parsing YAML file ${filePath}: ${e}`);
+	}
+	return [{
+		raw: maybeParsed,
+		label: prompt.label || `${filePath}: ${maybeParsed?.slice(0, 80)}`,
+		config: prompt.config
+	}];
+}
+//#endregion
+//#region src/external/prompts/ragas.ts
+const ANSWER_RELEVANCY_GENERATE = `Generate question for the given answer.
+Answer:\nThe PSLV-C56 mission is scheduled to be launched on Sunday, 30 July 2023 at 06:30 IST / 01:00 UTC. It will be launched from the Satish Dhawan Space Centre, Sriharikota, Andhra Pradesh, India
+Question: When is the scheduled launch date and time for the PSLV-C56 mission, and where will it be launched from?
+
+Answer:{{answer}}
+Question:`;
+const CONTEXT_RECALL = `Given a context, and an answer, analyze each sentence in the answer and classify if the sentence can be attributed to the given context or not.
+Think in steps and reason before coming to conclusion.
+
+context: Albert Einstein (14 March 1879 – 18 April 1955) was a German-born theoretical physicist,widely held to be one of the greatest and most influential scientists of all time. Best known for developing the theory of relativity, he also made important contributions to quantum mechanics, and was thus a central figure in the revolutionary reshaping of the scientific understanding of nature that modern physics accomplished in the first decades of the twentieth century. His mass–energy equivalence formula E = mc2, which arises from relativity theory, has been called "the world's most famous equation". He received the 1921 Nobel Prize in Physics "for his services to theoretical physics, and especially for his discovery of the law of the photoelectric effect", a pivotal step in the development of quantum theory. His work is also known for its influence on the philosophy of science. In a 1999 poll of 130 leading physicists worldwide by the British journal Physics World, Einstein was ranked the greatest physicist of all time. His intellectual achievements and originality have made Einstein synonymous with genius.
+answer: Albert Einstein born in 14 March 1879 was  German-born theoretical physicist, widely held to be one of the greatest and most influential scientists of all time. He received the 1921 Nobel Prize in Physics "for his services to theoretical physics. He published 4 papers in 1905.  Einstein moved to Switzerland in 1895
+classification
+1. Albert Einstein born in 14 March 1879 was  German-born theoretical physicist, widely held to be one of the greatest and most influential scientists of all time. The date of birth of Einstein is mentioned clearly in the context. So [Attributed]
+2. He received the 1921 Nobel Prize in Physics "for his services to theoretical physics. The exact sentence is present in the given context. So [Attributed]
+3. He published 4 papers in 1905. There is no mention about papers he wrote in given the context. So [Not Attributed]
+4. Einstein moved to Switzerland in 1895. There is not supporting evidence for this in the given the context. So [Not Attributed]
+
+context:{{context}}
+answer:{{groundTruth}}
+classification:
+`;
+const CONTEXT_RECALL_ATTRIBUTED_TOKEN = "[Attributed]";
+const CONTEXT_RECALL_NOT_ATTRIBUTED_TOKEN = "[Not Attributed]";
+const CONTEXT_RELEVANCE = `Please extract relevant sentences from the provided context that is absolutely required answer the following query. If no relevant sentences are found, or if you believe the query cannot be answered from the given context, return the phrase "Insufficient Information".  While extracting candidate sentences you're not allowed to make any changes to sentences from given context.
+
+query: {{query}}
+context: {{context}}
+candidate sentences:
+`;
+const CONTEXT_RELEVANCE_BAD = "Insufficient Information";
+const CONTEXT_FAITHFULNESS_LONGFORM = `Given a question and answer, create one or more statements from each sentence in the given answer.
+question: Who was  Albert Einstein and what is he best known for?
+answer: He was a German-born theoretical physicist, widely acknowledged to be one of the greatest and most influential physicists of all time. He was best known for developing the theory of relativity, he also made important contributions to the development of the theory of quantum mechanics.
+statements:\nAlbert Einstein was born in Germany.\nAlbert Einstein was best known for his theory of relativity.
+question: Cadmium Chloride is slightly soluble in this chemical, it is also called what?
+answer: alcohol
+statements:\nCadmium Chloride is slightly soluble in alcohol.
+question: Were Shahul and Jithin of the same nationality?
+answer: They were from different countries.
+statements:\nShahul and Jithin were from different countries.
+question:{{question}}
+answer: {{answer}}
+statements:\n`;
+const CONTEXT_FAITHFULNESS_NLI_STATEMENTS = `Prompt: Natural language inference
+Consider the given context and following statements, then determine whether they are supported by the information present in the context.Provide a brief explanation for each statement before arriving at the verdict (Yes/No). Provide a final verdict for each statement in order at the end in the given format. Do not deviate from the specified format.
+
+Context:\nJohn is a student at XYZ University. He is pursuing a degree in Computer Science. He is enrolled in several courses this semester, including Data Structures, Algorithms, and Database Management. John is a diligent student and spends a significant amount of time studying and completing assignments. He often stays late in the library to work on his projects.
+statements:\n1. John is majoring in Biology.\n2. John is taking a course on Artificial Intelligence.\n3. John is a dedicated student.\n4. John has a part-time job.\n5. John is interested in computer programming.\n
+Answer:
+1. John is majoring in Biology.
+Explanation: John's major is explicitly mentioned as Computer Science. There is no information suggesting he is majoring in Biology.  Verdict: No.
+2. John is taking a course on Artificial Intelligence.
+Explanation: The context mentions the courses John is currently enrolled in, and Artificial Intelligence is not mentioned. Therefore, it cannot be deduced that John is taking a course on AI. Verdict: No.
+3. John is a dedicated student.
+Explanation: The prompt states that he spends a significant amount of time studying and completing assignments. Additionally, it mentions that he often stays late in the library to work on his projects, which implies dedication. Verdict: Yes.
+4. John has a part-time job.
+Explanation: There is no information given in the context about John having a part-time job. Therefore, it cannot be deduced that John has a part-time job.  Verdict: No.
+5. John is interested in computer programming.
+Explanation: The context states that John is pursuing a degree in Computer Science, which implies an interest in computer programming. Verdict: Yes.
+Final verdict for each statement in order: No. No. Yes. No. Yes.
+context:\n{{context}}
+statements:\n{{statements|join("\\n")}}
+Answer:
+`;
+//#endregion
+//#region src/prompts/grading.ts
+const DEFAULT_GRADING_PROMPT = JSON.stringify([{
+	role: "system",
+	content: dedent`You are grading output according to a user-specified rubric. If the statement in the rubric is true, then the output passes the test. You respond with a JSON object with this structure: {reason: string, pass: boolean, score: number}
+
+    Examples:
+
+    <Output>Hello world</Output>
+    <Rubric>Content contains a greeting</Rubric>
+    {"reason": "the content contains the word 'Hello'", "pass": true, "score": 1.0}
+
+    <Output>Avast ye swabs, repel the invaders!</Output>
+    <Rubric>Does not speak like a pirate</Rubric>
+    {"reason": "'avast ye' is a common pirate term", "pass": false, "score": 0.0}`
+}, {
+	role: "user",
+	content: "<Output>\n{{ output }}\n</Output>\n<Rubric>\n{{ rubric }}\n</Rubric>"
+}]);
+const PROMPTFOO_FACTUALITY_PROMPT = JSON.stringify([{
+	role: "system",
+	content: dedent`
+      You are a precise factuality evaluator that compares a submitted answer to an expert answer. 
+
+      Your task is to analyze the factual content while ignoring differences in style, grammar, or punctuation.
+      You must categorize the submission into one of these options:
+
+      (A) The submitted answer is a subset of the expert answer and is fully consistent with it.
+      (B) The submitted answer is a superset of the expert answer and is fully consistent with it.
+      (C) The submitted answer contains all the same details as the expert answer.
+      (D) There is a disagreement between the submitted answer and the expert answer.
+      (E) The answers differ, but these differences don't matter from the perspective of factuality.
+
+      Respond ONLY with a JSON object in this format:
+      {
+        "category": "[LETTER]",
+        "reason": "[DETAILED EXPLANATION]"
+      }
+
+      - The "category" must be a single letter A, B, C, D, or E.
+      - Provide a clear, detailed explanation in the "reason" field.
+      - Your response must be valid JSON with no additional text.`
+}, {
+	role: "user",
+	content: dedent`
+      I need you to compare these answers:
+
+      <question>
+      {{input}}
+      </question>
+
+      <expert_answer>
+      {{ideal}}
+      </expert_answer>
+
+      <submitted_answer>
+      {{completion}}
+      </submitted_answer>
+
+      Please analyze the factual relationship between these answers according to the categories you've been given.`
+}]);
+const OPENAI_CLOSED_QA_PROMPT = JSON.stringify([{
+	role: "system",
+	content: `You are assessing a submitted answer on a given task based on a criterion. Here is the data:
+[BEGIN DATA]
+***
+[Task]: {{input}}
+***
+[Submission]: {{completion}}
+***
+[Criterion]: {{criteria}}
+***
+[END DATA]
+Does the submission meet the criterion? First, write out in a step by step manner your reasoning about the criterion to be sure that your conclusion is correct. Avoid simply stating the correct answers at the outset. Then print only the single character "Y" or "N" (without quotes or punctuation) on its own line corresponding to the correct answer. At the end, repeat just the letter again by itself on a new line.
+
+    Reasoning:`
+}]);
+const SUGGEST_PROMPTS_SYSTEM_MESSAGE = {
+	role: "system",
+	content: `You're helping a scientist who is tuning a prompt for a large language model.  You will receive messages, and each message is a full prompt.  Generate a candidate variation of the given prompt.  This variation will be tested for quality in order to select a winner.
+
+Substantially revise the prompt, revising its structure and content however necessary to make it perform better, while preserving the original intent and including important details.
+
+Your output is going to be copied directly into the program. It should contain the prompt ONLY`
+};
+const SELECT_BEST_PROMPT = JSON.stringify([{
+	role: "system",
+	content: `You are comparing multiple pieces of text to see which best fits the following criteria: {{criteria}}
+
+Here are the pieces of text:
+
+{% for output in outputs %}
+<Text index="{{ loop.index0 }}">
+{{ output }}
+</Text>
+{% endfor %}
+
+Output the index of the text that best fits the criteria. You must output a single integer.`
+}]);
+const DEFAULT_WEB_SEARCH_PROMPT = JSON.stringify([{
+	role: "system",
+	content: dedent`You are grading output according to a user-specified rubric, with the ability to search the web for current information. If the statement in the rubric is true, then the output passes the test. You respond with a JSON object with this structure: {reason: string, pass: boolean, score: number}
+
+    You MUST search the web when:
+    - The rubric asks about current information (prices, weather, news, etc.)
+    - Facts need to be verified against recent data
+    - The rubric references time-sensitive information
+
+    Examples:
+
+    <Output>The current CEO of Microsoft is Satya Nadella</Output>
+    <Rubric>Contains accurate information about Microsoft's leadership</Rubric>
+    {"reason": "I searched and confirmed Satya Nadella is indeed the current CEO of Microsoft", "pass": true, "score": 1.0}
+
+    <Output>Bitcoin is trading at $45,000</Output>
+    <Rubric>Provides current Bitcoin price within 10% accuracy</Rubric>
+    {"reason": "Web search shows Bitcoin is currently trading at $98,000, not $45,000. The output is off by more than 50%", "pass": false, "score": 0.0}`
+}, {
+	role: "user",
+	content: "<Output>\n{{ output }}\n</Output>\n<Rubric>\n{{ rubric }}\n</Rubric>"
+}]);
+const TRAJECTORY_GOAL_SUCCESS_PROMPT = JSON.stringify([{
+	role: "system",
+	content: dedent`You are grading whether an AI agent successfully completed a goal based on its final output and a summarized execution trajectory. You respond with a JSON object with this structure: {reason: string, pass: boolean, score: number}
+
+    Judge end-to-end success, not stylistic perfection.
+    Use the trajectory as evidence for what the agent actually did.
+    Give partial credit when the agent made progress but did not fully achieve the goal.
+
+    Examples:
+
+    <Goal>Find the order status and tell the user whether it has shipped</Goal>
+    <Trajectory>{"stepCount":2,"steps":[{"index":1,"type":"tool","name":"search_orders"},{"index":2,"type":"message","name":"agent response"}]}</Trajectory>
+    <Output>Your order shipped yesterday and should arrive on Tuesday.</Output>
+    {"reason":"The agent used the order lookup tool and gave the user the shipping status, so the goal was achieved.","pass":true,"score":1.0}
+
+    <Goal>Find the order status and tell the user whether it has shipped</Goal>
+    <Trajectory>{"stepCount":1,"steps":[{"index":1,"type":"message","name":"agent response"}]}</Trajectory>
+    <Output>I cannot check your order right now.</Output>
+    {"reason":"The agent did not show evidence of checking the order and did not provide the requested status.","pass":false,"score":0.0}`
+}, {
+	role: "user",
+	content: dedent`<Goal>
+{{ goal }}
+</Goal>
+<Trajectory>
+{{ trajectory }}
+</Trajectory>
+<Output>
+{{ output }}
+</Output>`
+}]);
 //#endregion
 //#region src/prompts/index.ts
 /**
@@ -1165,164 +1386,44 @@ IMPORTANT:
 Format:
 {"steps": <list_of_strings>}
 
-Example:
-{"steps":["<Evaluation Step 1>","<Evaluation Step 2>","<Evaluation Step 3>","<Evaluation Step 4>"]}
-
-Here are the 3-4 concise evaluation steps, formatted as required in a minified JSON:
-JSON:
-`;
-const GEVAL_PROMPT_EVALUATE = `
-You will be given one Reply for a Prompt below. Your task is to rate the Reply on one metric.
-Please make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.
-
-**Evaluation Criteria**
-{{criteria}}
-
-**Evaluation Steps**
-- {{steps}}
-Given the evaluation steps, return a JSON with two keys: 
-  1) a "score" key that MUST be an integer from 0 to {{maxScore}}, where {{maxScore}} indicates that the condition described by the Evaluation Criteria is fully and clearly observed in the Reply according to the Evaluation Steps, and 0 indicates that it is not observed at all;
-  2) a "reason" key, a reason for the given score, but DO NOT QUOTE THE SCORE in your reason. Please mention specific information from Prompt and Reply in your reason, but be very concise with it!
-
-**Prompt**
-{{input}}
-
-**Reply**
-{{output}}
-
-**OUTPUT FORMAT**
-IMPORTANT: 
-- Return output ONLY as a minified JSON object (no code fences).
-- The JSON object must contain exactly two keys: "score" and "reason".
-- No additional words, explanations, or formatting are needed.
-- Absolutely no additional text, explanations, line breaks, or formatting outside the JSON object are allowed.
-
-Example JSON:
-{"score":0,"reason":"The text of reply does not follow the evaluation criteria provided."}
-
-Here is the final evaluation in the required minified JSON format:
-JSON:
-`;
-//#endregion
-//#region src/providers/webSearchUtils.ts
-function hasTool(provider, predicate) {
-	return Array.isArray(provider.config?.tools) && provider.config.tools.some(predicate);
-}
-function getProviderId(provider) {
-	if (typeof provider.id !== "function") return null;
-	try {
-		return provider.id();
-	} catch (err) {
-		logger.debug(`Failed to read provider id: ${err}`);
-		return null;
-	}
-}
-function isOpenAiResponsesProvider(provider, id) {
-	return id.includes("openai:responses") || provider.constructor?.name === "OpenAiResponsesProvider";
-}
-/**
-* Check if a provider has web search capabilities
-* @param provider The provider to check
-* @returns true if the provider supports web search
-*/
-function hasWebSearchCapability(provider) {
-	if (!provider) return false;
-	const id = getProviderId(provider);
-	if (!id) return false;
-	if (id.includes("perplexity")) return true;
-	if ((id.includes("google") || id.includes("gemini") || id.includes("vertex")) && hasTool(provider, (t) => t.googleSearch !== void 0)) return true;
-	if (id.includes("xai") && provider.config?.search_parameters?.mode === "on") return true;
-	if (isOpenAiResponsesProvider(provider, id) && hasTool(provider, (t) => t.type === "web_search_preview")) return true;
-	if (id.startsWith("openai:codex") && (provider.config?.web_search_mode === "live" || provider.config?.web_search_mode === "cached" || provider.config?.web_search_enabled === true)) return true;
-	if (id.includes("anthropic") && hasTool(provider, (t) => t.type === "web_search_20250305")) return true;
-	return false;
-}
-/**
-* Load a provider with web search capabilities.
-* Tries multiple providers in order of preference until one succeeds.
-* Uses the latest and most capable models from each provider with specific checkpoint IDs.
-*
-* @param preferAnthropic Whether to try Anthropic first (true) or OpenAI first (false)
-* @returns A provider with web search capabilities or null
-*/
-async function loadWebSearchProvider(preferAnthropic = false) {
-	const loadAnthropicWebSearch = async () => {
-		try {
-			return await loadApiProvider("anthropic:messages:claude-opus-4-6", { options: { config: { tools: [{
-				type: "web_search_20250305",
-				name: "web_search",
-				max_uses: 5
-			}] } } });
-		} catch (err) {
-			logger.debug(`Failed to load Anthropic web search provider: ${err}`);
-			return null;
-		}
-	};
-	const loadOpenAIWebSearch = async () => {
-		try {
-			return await loadApiProvider("openai:responses:gpt-5.4-2026-03-05", { options: { config: { tools: [{ type: "web_search_preview" }] } } });
-		} catch (err) {
-			logger.debug(`Failed to load OpenAI web search provider: ${err}`);
-			return null;
-		}
-	};
-	const loadPerplexity = async () => {
-		try {
-			return await loadApiProvider("perplexity:sonar-pro");
-		} catch (err) {
-			logger.debug(`Failed to load Perplexity provider: ${err}`);
-			return null;
-		}
-	};
-	const loadGoogleWebSearch = async () => {
-		try {
-			return await loadApiProvider("google:gemini-3-pro-preview", { options: { config: { tools: [{ googleSearch: {} }] } } });
-		} catch (err) {
-			logger.debug(`Failed to load Google web search provider: ${err}`);
-			return null;
-		}
-	};
-	const loadVertexWebSearch = async () => {
-		try {
-			return await loadApiProvider("vertex:gemini-3-pro-preview", { options: { config: { tools: [{ googleSearch: {} }] } } });
-		} catch (err) {
-			logger.debug(`Failed to load Vertex web search provider: ${err}`);
-			return null;
-		}
-	};
-	const loadXaiWebSearch = async () => {
-		try {
-			return await loadApiProvider("xai:grok-4-1-fast-reasoning", { options: { config: { search_parameters: { mode: "on" } } } });
-		} catch (err) {
-			logger.debug(`Failed to load xAI web search provider: ${err}`);
-			return null;
-		}
-	};
-	const providers = preferAnthropic ? [
-		loadAnthropicWebSearch,
-		loadOpenAIWebSearch,
-		loadPerplexity,
-		loadGoogleWebSearch,
-		loadVertexWebSearch,
-		loadXaiWebSearch
-	] : [
-		loadOpenAIWebSearch,
-		loadAnthropicWebSearch,
-		loadPerplexity,
-		loadGoogleWebSearch,
-		loadVertexWebSearch,
-		loadXaiWebSearch
-	];
-	for (const getProvider of providers) {
-		const provider = await getProvider();
-		if (provider && hasWebSearchCapability(provider)) {
-			logger.info(`Using ${getProviderId(provider) ?? "loaded provider"} as web search provider`);
-			return provider;
-		}
-		if (provider) logger.debug(`Loaded provider ${getProviderId(provider) ?? "unknown"} does not support web search`);
-	}
-	return null;
-}
+Example:
+{"steps":["<Evaluation Step 1>","<Evaluation Step 2>","<Evaluation Step 3>","<Evaluation Step 4>"]}
+
+Here are the 3-4 concise evaluation steps, formatted as required in a minified JSON:
+JSON:
+`;
+const GEVAL_PROMPT_EVALUATE = `
+You will be given one Reply for a Prompt below. Your task is to rate the Reply on one metric.
+Please make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.
+
+**Evaluation Criteria**
+{{criteria}}
+
+**Evaluation Steps**
+- {{steps}}
+Given the evaluation steps, return a JSON with two keys: 
+  1) a "score" key that MUST be an integer from 0 to {{maxScore}}, where {{maxScore}} indicates that the condition described by the Evaluation Criteria is fully and clearly observed in the Reply according to the Evaluation Steps, and 0 indicates that it is not observed at all;
+  2) a "reason" key, a reason for the given score, but DO NOT QUOTE THE SCORE in your reason. Please mention specific information from Prompt and Reply in your reason, but be very concise with it!
+
+**Prompt**
+{{input}}
+
+**Reply**
+{{output}}
+
+**OUTPUT FORMAT**
+IMPORTANT: 
+- Return output ONLY as a minified JSON object (no code fences).
+- The JSON object must contain exactly two keys: "score" and "reason".
+- No additional words, explanations, or formatting are needed.
+- Absolutely no additional text, explanations, line breaks, or formatting outside the JSON object are allowed.
+
+Example JSON:
+{"score":0,"reason":"The text of reply does not follow the evaluation criteria provided."}
+
+Here is the final evaluation in the required minified JSON format:
+JSON:
+`;
 //#endregion
 //#region src/remoteGrading.ts
 async function doRemoteGrading(payload) {
@@ -1385,23 +1486,7 @@ async function doRemoteScoringWithPi(payload, passThreshold = .5) {
 	}
 }
 //#endregion
-//#region src/scheduler/providerCallExecutionContext.ts
-const providerCallExecutionContext = new AsyncLocalStorage();
-function getProviderCallExecutionContext() {
-	return providerCallExecutionContext.getStore();
-}
-function withProviderCallExecutionContext(context, fn) {
-	return providerCallExecutionContext.run(context, fn);
-}
-//#endregion
-//#region src/matchers.ts
-var LlmRubricProviderError = class extends Error {
-	constructor(message) {
-		super(message);
-		this.name = "LlmRubricProviderError";
-	}
-};
-const nunjucks = getNunjucksEngine(void 0, false, true);
+//#region src/matchers/llmGrading.ts
 const FACTUALITY_CATEGORY_DESCRIPTIONS = {
 	A: "The submitted answer is a subset of the expert answer and is fully consistent with it.",
 	B: "The submitted answer is a superset of the expert answer and is fully consistent with it.",
@@ -1409,409 +1494,73 @@ const FACTUALITY_CATEGORY_DESCRIPTIONS = {
 	D: "There is a disagreement between the submitted answer and the expert answer.",
 	E: "The answers differ, but these differences don't matter from the perspective of factuality."
 };
-function cosineSimilarity(vecA, vecB) {
-	if (vecA.length !== vecB.length) throw new Error("Vectors must be of equal length");
-	return vecA.reduce((acc, val, idx) => acc + val * vecB[idx], 0) / (Math.sqrt(vecA.reduce((acc, val) => acc + val * val, 0)) * Math.sqrt(vecB.reduce((acc, val) => acc + val * val, 0)));
-}
-function dotProduct(vecA, vecB) {
-	if (vecA.length !== vecB.length) throw new Error("Vectors must be of equal length");
-	return vecA.reduce((acc, val, idx) => acc + val * vecB[idx], 0);
-}
-function euclideanDistance(vecA, vecB) {
-	if (vecA.length !== vecB.length) throw new Error("Vectors must be of equal length");
-	const sumSquaredDiff = vecA.reduce((acc, val, idx) => {
-		const diff = val - vecB[idx];
-		return acc + diff * diff;
-	}, 0);
-	return Math.sqrt(sumSquaredDiff);
-}
-/**
-* Helper to call provider with consistent context propagation pattern.
-* Spreads the optional context and merges with prompt label and vars.
-*
-* IMPORTANT: Spread order matters - context is spread first, then prompt/vars
-* override. This ensures originalProvider from context is preserved while
-* allowing this call to specify its own prompt metadata.
-*/
-function callProviderWithContext(provider, prompt, label, vars, context) {
-	const callApiContext = {
-		...context,
-		prompt: {
-			raw: prompt,
-			label
-		},
-		vars
-	};
-	const executionContext = getProviderCallExecutionContext();
-	const callApiOptions = executionContext?.abortSignal ? { abortSignal: executionContext.abortSignal } : void 0;
-	const callApi = () => callApiOptions ? provider.callApi(prompt, callApiContext, callApiOptions) : provider.callApi(prompt, callApiContext);
-	const executeCall = () => {
-		if (executionContext?.rateLimitRegistry && !isRateLimitWrapped(provider)) return executionContext.rateLimitRegistry.execute(provider, callApi, createProviderRateLimitOptions());
-		return callApi();
-	};
-	if (executionContext?.providerCallQueue) return executionContext.providerCallQueue.enqueue(provider.id(), executeCall);
-	return executeCall();
-}
-async function loadFromProviderOptions(provider) {
-	invariant(typeof provider === "object", `Provider must be an object, but received a ${typeof provider}: ${provider}`);
-	invariant(!Array.isArray(provider), `Provider must be an object, but received an array: ${JSON.stringify(provider)}`);
-	invariant(provider.id, "Provider supplied to assertion must have an id");
-	return loadApiProvider(provider.id, {
-		options: provider,
-		basePath: state.basePath
-	});
-}
-function isSimulatedUserProviderConfig(provider) {
-	if (typeof provider === "string") return provider === "promptfoo:simulated-user";
-	if (!provider || typeof provider !== "object" || Array.isArray(provider)) return false;
-	if (typeof provider.id === "function") return provider.id() === "promptfoo:simulated-user";
-	const providerId = provider.id;
-	if (typeof providerId === "string") return providerId === "promptfoo:simulated-user";
-	return Object.values(provider).some((providerTypeConfig) => isSimulatedUserProviderConfig(providerTypeConfig));
-}
-async function getGradingProvider(type, provider, defaultProvider) {
-	let finalProvider;
-	if (typeof provider === "string") finalProvider = await loadApiProvider(provider, { basePath: state.basePath });
-	else if (typeof provider === "object" && typeof provider.id === "function") finalProvider = provider;
-	else if (typeof provider === "object") {
-		const typeValue = provider[type];
-		if (typeValue) finalProvider = await getGradingProvider(type, typeValue, defaultProvider);
-		else if (provider.id) finalProvider = await loadFromProviderOptions(provider);
-		else if (Array.isArray(provider)) throw new Error(`Provider must be an object or string, but received an array.\n\nCheck that the provider ${JSON.stringify(provider[0], null, 2)} is not nested in an array.`);
-		else throw new Error(`Invalid provider definition for output type '${type}': ${JSON.stringify(provider, null, 2)}`);
-	} else {
-		const defaultTest = state.config?.defaultTest;
-		const defaultTestObj = typeof defaultTest === "object" ? defaultTest : null;
-		const cfg = [
-			defaultTestObj?.provider || void 0,
-			defaultTestObj?.options?.provider?.text || void 0,
-			defaultTestObj?.options?.provider || void 0
-		].find((candidateProvider) => {
-			if (!candidateProvider) return false;
-			if (isSimulatedUserProviderConfig(candidateProvider)) {
-				logger.debug("[Grading] Skipping promptfoo:simulated-user as an implicit grader fallback");
-				return false;
-			}
-			return true;
-		});
-		if (cfg) {
-			finalProvider = await getGradingProvider(type, cfg, defaultProvider);
-			if (finalProvider) logger.debug(`[Grading] Using provider from defaultTest fallback: ${finalProvider.id()}`);
-		} else finalProvider = defaultProvider;
-	}
-	return finalProvider;
-}
-async function getAndCheckProvider(type, provider, defaultProvider, checkName) {
-	const matchedProvider = await getGradingProvider(type, provider, defaultProvider);
-	if (!matchedProvider) if (defaultProvider) {
-		logger.warn(`No provider of type ${type} found for '${checkName}', falling back to default`);
-		return defaultProvider;
-	} else throw new Error(`No provider of type ${type} found for '${checkName}'`);
-	let isValidProviderType = true;
-	if (type === "embedding") isValidProviderType = "callEmbeddingApi" in matchedProvider || "callSimilarityApi" in matchedProvider;
-	else if (type === "classification") isValidProviderType = "callClassificationApi" in matchedProvider;
-	else if (type === "moderation") isValidProviderType = "callModerationApi" in matchedProvider;
-	if (!isValidProviderType) if (defaultProvider) {
-		logger.warn(`Provider ${matchedProvider.id()} is not a valid ${type} provider for '${checkName}', falling back to default`);
-		return defaultProvider;
-	} else throw new Error(`Provider ${matchedProvider.id()} is not a valid ${type} provider for '${checkName}'`);
-	return matchedProvider;
-}
-function fail(reason, tokensUsed) {
-	return {
-		pass: false,
-		reason,
-		score: 0,
-		tokensUsed: {
-			total: tokensUsed?.total || 0,
-			prompt: tokensUsed?.prompt || 0,
-			completion: tokensUsed?.completion || 0,
-			cached: tokensUsed?.cached || 0,
-			numRequests: tokensUsed?.numRequests || 0,
-			completionDetails: tokensUsed?.completionDetails
-		}
-	};
-}
-function normalizeTokenUsage(tokensUsed) {
-	return {
-		total: tokensUsed?.total || 0,
-		prompt: tokensUsed?.prompt || 0,
-		completion: tokensUsed?.completion || 0,
-		cached: tokensUsed?.cached || 0,
-		numRequests: tokensUsed?.numRequests || 0,
-		completionDetails: tokensUsed?.completionDetails || {
-			reasoning: 0,
-			acceptedPrediction: 0,
-			rejectedPrediction: 0
-		}
-	};
-}
-function createMatcherTokenUsage() {
-	return {
-		total: 0,
-		prompt: 0,
-		completion: 0,
-		cached: 0,
-		numRequests: 0,
-		completionDetails: {
-			reasoning: 0,
-			acceptedPrediction: 0,
-			rejectedPrediction: 0
-		}
-	};
-}
-function copySimilarityTokenUsage(tokensUsed, response) {
-	Object.assign(tokensUsed, normalizeTokenUsage(response.tokenUsage));
-}
-function combineEmbeddingTokenUsage(expectedEmbedding, outputEmbedding) {
-	const expectedTokens = normalizeTokenUsage(expectedEmbedding.tokenUsage);
-	const outputTokens = normalizeTokenUsage(outputEmbedding.tokenUsage);
-	return {
-		total: (expectedTokens.total ?? 0) + (outputTokens.total ?? 0),
-		prompt: (expectedTokens.prompt ?? 0) + (outputTokens.prompt ?? 0),
-		completion: (expectedTokens.completion ?? 0) + (outputTokens.completion ?? 0),
-		cached: (expectedTokens.cached ?? 0) + (outputTokens.cached ?? 0),
-		numRequests: (expectedTokens.numRequests ?? 0) + (outputTokens.numRequests ?? 0),
-		completionDetails: {
-			reasoning: (expectedTokens.completionDetails?.reasoning || 0) + (outputTokens.completionDetails?.reasoning || 0),
-			acceptedPrediction: (expectedTokens.completionDetails?.acceptedPrediction || 0) + (outputTokens.completionDetails?.acceptedPrediction || 0),
-			rejectedPrediction: (expectedTokens.completionDetails?.rejectedPrediction || 0) + (outputTokens.completionDetails?.rejectedPrediction || 0)
-		}
-	};
-}
-function accumulateTokens(target, update) {
-	accumulateTokenUsage(target, update);
-}
-async function maybeRemoteSimilarityGrading({ expected, output, threshold, inverse }) {
-	if (!state.config?.redteam || !shouldGenerateRemote({ requireEmbeddingProvider: true })) return;
-	try {
-		return await doRemoteGrading({
-			task: "similar",
-			expected,
-			output,
-			threshold,
-			inverse
-		});
-	} catch (error) {
-		return fail(`Could not perform remote grading: ${error}`);
-	}
-}
-async function computeNativeSimilarity(provider, expected, output, metric) {
-	if ("callSimilarityApi" in provider) return computeSimilarityFromNativeProvider(provider, expected, output, metric);
-	if ("callEmbeddingApi" in provider) return computeSimilarityFromEmbeddings(provider, expected, output, metric);
-	throw new Error("Provider must implement callSimilarityApi or callEmbeddingApi");
-}
-async function computeSimilarityFromNativeProvider(provider, expected, output, metric) {
-	const tokensUsed = createMatcherTokenUsage();
-	if (metric !== "cosine") return { failure: fail(`Provider ${provider.id()} only supports cosine similarity via callSimilarityApi`, tokensUsed) };
-	const similarityResp = await provider.callSimilarityApi(expected, output);
-	copySimilarityTokenUsage(tokensUsed, similarityResp);
-	if (similarityResp.error) return { failure: fail(similarityResp.error, tokensUsed) };
-	if (similarityResp.similarity == null) return { failure: fail("Unknown error fetching similarity", tokensUsed) };
-	return {
-		similarity: similarityResp.similarity,
-		tokensUsed
-	};
-}
-async function computeSimilarityFromEmbeddings(provider, expected, output, metric) {
-	const expectedEmbedding = await provider.callEmbeddingApi(expected);
-	const outputEmbedding = await provider.callEmbeddingApi(output);
-	const tokensUsed = combineEmbeddingTokenUsage(expectedEmbedding, outputEmbedding);
-	if (expectedEmbedding.error || outputEmbedding.error) return { failure: fail(expectedEmbedding.error || outputEmbedding.error || "Unknown error fetching embeddings", tokensUsed) };
-	if (!expectedEmbedding.embedding || !outputEmbedding.embedding) return { failure: fail("Embedding not found", tokensUsed) };
-	return {
-		similarity: computeSimilarityMetric(expectedEmbedding.embedding, outputEmbedding.embedding, metric),
-		tokensUsed
-	};
-}
-function computeSimilarityMetric(expectedEmbedding, outputEmbedding, metric) {
-	switch (metric) {
-		case "cosine": return cosineSimilarity(expectedEmbedding, outputEmbedding);
-		case "dot_product": return dotProduct(expectedEmbedding, outputEmbedding);
-		case "euclidean": return euclideanDistance(expectedEmbedding, outputEmbedding);
-		default: throw new Error(`Unsupported metric: ${metric}`);
-	}
-}
-function buildSimilarityResult(similarity, threshold, inverse, metric, tokensUsed) {
-	return metric === "euclidean" ? buildDistanceResult(similarity, threshold, inverse, tokensUsed) : buildSimilarityScoreResult(similarity, threshold, inverse, tokensUsed);
-}
-function buildDistanceResult(distance, threshold, inverse, tokensUsed) {
-	const pass = inverse ? distance >= threshold - Number.EPSILON : distance <= threshold + Number.EPSILON;
-	const normalizedScore = 1 / (1 + distance);
-	const belowThresholdReason = `Distance ${distance.toFixed(2)} is less than or equal to threshold ${threshold}`;
-	const aboveThresholdReason = `Distance ${distance.toFixed(2)} is greater than threshold ${threshold}`;
-	return {
-		pass,
-		score: inverse ? 1 - normalizedScore : normalizedScore,
-		reason: pass === inverse ? aboveThresholdReason : belowThresholdReason,
-		tokensUsed
-	};
-}
-function buildSimilarityScoreResult(similarity, threshold, inverse, tokensUsed) {
-	const pass = inverse ? similarity <= threshold + Number.EPSILON : similarity >= threshold - Number.EPSILON;
-	const greaterThanReason = `Similarity ${similarity.toFixed(2)} is greater than or equal to threshold ${threshold}`;
-	const lessThanReason = `Similarity ${similarity.toFixed(2)} is less than threshold ${threshold}`;
-	return {
-		pass,
-		score: inverse ? 1 - similarity : similarity,
-		reason: pass === inverse ? lessThanReason : greaterThanReason,
-		tokensUsed
-	};
-}
-async function matchesSimilarity(expected, output, threshold, inverse = false, grading, metric = "cosine") {
-	const remoteResult = await maybeRemoteSimilarityGrading({
-		expected,
-		output,
-		threshold,
-		inverse
-	});
-	if (remoteResult) return remoteResult;
-	const defaults = await getDefaultProviders();
-	const computation = await computeNativeSimilarity(await getAndCheckProvider("embedding", grading?.provider, defaults.embeddingProvider, "similarity check"), expected, output, metric);
-	return "failure" in computation ? computation.failure : buildSimilarityResult(computation.similarity, threshold, inverse, metric, computation.tokensUsed);
-}
-/**
-*
-* @param expected Expected classification. If undefined, matches any classification.
-* @param output Text to classify.
-* @param threshold Value between 0 and 1. If the expected classification is undefined, the threshold is the minimum score for any classification. If the expected classification is defined, the threshold is the minimum score for that classification.
-* @param grading
-* @returns Pass if the output matches the classification with a score greater than or equal to the threshold.
-*/
-async function matchesClassification(expected, output, threshold, grading) {
-	const resp = await (await getAndCheckProvider("classification", grading?.provider, null, "classification check")).callClassificationApi(output);
-	if (!resp.classification) return fail(resp.error || "Unknown error fetching classification");
-	let score;
-	if (expected === void 0) score = Math.max(...Object.values(resp.classification));
-	else score = resp.classification[expected] || 0;
-	if (score >= threshold - Number.EPSILON) {
-		const reason = expected === void 0 ? `Maximum classification score ${score.toFixed(2)} >= ${threshold}` : `Classification ${expected} has score ${score.toFixed(2)} >= ${threshold}`;
-		return {
-			pass: true,
-			score,
-			reason
-		};
-	}
+function getFactualityScoreLookup(grading) {
 	return {
-		pass: false,
-		score,
-		reason: `Classification ${expected} has score ${score.toFixed(2)} < ${threshold}`
-	};
-}
-async function loadRubricPrompt(rubricPrompt, defaultPrompt) {
-	if (!rubricPrompt || typeof rubricPrompt === "object" && Object.keys(rubricPrompt).length === 0) return defaultPrompt;
-	if (typeof rubricPrompt === "string" && rubricPrompt.startsWith("file://")) {
-		const basePath = state.basePath || "";
-		const { filePath, functionName } = parseFileUrl(getNunjucksEngineForFilePath().renderString(rubricPrompt, {}));
-		const resolvedPath = path.resolve(basePath, filePath);
-		if (isJavascriptFile(filePath)) rubricPrompt = await loadFromJavaScriptFile(resolvedPath, functionName, []);
-		else {
-			if (!fs$2.existsSync(resolvedPath)) throw new Error(`File does not exist: ${resolvedPath}`);
-			rubricPrompt = fs$2.readFileSync(resolvedPath, "utf8");
-		}
-	} else rubricPrompt = maybeLoadFromExternalFile(rubricPrompt);
-	if (typeof rubricPrompt === "object") rubricPrompt = JSON.stringify(rubricPrompt);
-	invariant(typeof rubricPrompt === "string", "rubricPrompt must be a string");
-	return rubricPrompt;
-}
-function tryParse(content) {
-	try {
-		return JSON.parse(content);
-	} catch {}
-	return content;
-}
-function splitIntoSentences(text) {
-	return text.split("\n").filter((sentence) => sentence.trim() !== "");
-}
-function processContextForTemplating(context, enableObjectAccess) {
-	if (enableObjectAccess) return context;
-	return Object.fromEntries(Object.entries(context).map(([key, value]) => {
-		if (value && typeof value === "object") {
-			if (Array.isArray(value)) return [key, value.map((item) => item && typeof item === "object" ? JSON.stringify(item) : item)];
-			return [key, JSON.stringify(value)];
-		}
-		return [key, value];
-	}));
+		A: grading.factuality?.subset ?? 1,
+		B: grading.factuality?.superset ?? 1,
+		C: grading.factuality?.agree ?? 1,
+		D: grading.factuality?.disagree ?? 0,
+		E: grading.factuality?.differButFactual ?? 1
+	};
 }
-async function renderLlmRubricPrompt(rubricPrompt, context) {
-	const processedContext = processContextForTemplating(context, getEnvBool("PROMPTFOO_DISABLE_OBJECT_STRINGIFY", false));
-	try {
-		const parsed = JSON.parse(rubricPrompt, (_k, v) => typeof v === "string" ? nunjucks.renderString(v, processedContext) : v);
-		return JSON.stringify(parsed);
-	} catch {}
-	return nunjucks.renderString(rubricPrompt, processedContext);
+function buildFactualityResult(option, reason, grading, resp) {
+	const scoreLookup = getFactualityScoreLookup(grading);
+	const passing = Object.keys(scoreLookup).filter((key) => scoreLookup[key] > 0);
+	const failing = Object.keys(scoreLookup).filter((key) => scoreLookup[key] === 0);
+	const pass = passing.includes(option) && !failing.includes(option);
+	return {
+		pass,
+		score: scoreLookup[option] ?? (pass ? 1 : 0),
+		reason,
+		tokensUsed: normalizeMatcherTokenUsage(resp.tokenUsage)
+	};
 }
-function parseJsonGradingResponse(label, resp) {
-	let jsonObjects = [];
-	if (typeof resp.output === "string") try {
-		jsonObjects = extractJsonObjects(resp.output);
-		if (jsonObjects.length === 0) return { failure: fail(`Could not extract JSON from ${label} response`, resp.tokenUsage) };
+function parseFactualityJsonResponse(responseText) {
+	try {
+		const jsonData = extractFirstJsonObject(responseText);
+		if (!jsonData?.category || typeof jsonData.category !== "string") return;
+		const option = jsonData.category.trim().toUpperCase();
+		if (!/^[A-E]$/.test(option)) throw new Error(`Invalid category value: ${option}`);
+		return {
+			option,
+			reason: jsonData.reason?.trim() || `Category ${option}: ${FACTUALITY_CATEGORY_DESCRIPTIONS[option]}`
+		};
 	} catch (err) {
-		return { failure: fail(`${label} produced malformed response: ${err}\n\n${resp.output}`, resp.tokenUsage) };
+		const error = err;
+		if (error.message.startsWith("Invalid category value:")) throw error;
+		logger.debug(`JSON parsing failed: ${error.message}`);
+		return;
 	}
-	else if (typeof resp.output === "object" && resp.output !== null && !Array.isArray(resp.output)) jsonObjects = [resp.output];
-	else return { failure: fail(`${label} produced malformed response - output must be string or object. Output: ${JSON.stringify(resp.output)}`, resp.tokenUsage) };
-	const parsed = jsonObjects[0];
-	if (typeof parsed !== "object" || parsed === null || Array.isArray(parsed)) return { failure: fail(`${label} produced malformed response. We were not able to parse the response as JSON. Output: ${JSON.stringify(resp.output)}`, resp.tokenUsage) };
-	return { parsed };
 }
-async function runJsonGradingPrompt({ assertion, checkName, defaultPrompt, grading, label, providerCallContext, throwOnError, vars }) {
-	const prompt = await renderLlmRubricPrompt(await loadRubricPrompt(grading.rubricPrompt, defaultPrompt), vars);
-	const defaultProviders = await getDefaultProviders();
-	const defaultProvider = defaultProviders.llmRubricProvider || defaultProviders.gradingJsonProvider;
-	const resp = await callProviderWithContext(await getAndCheckProvider("text", grading.provider, defaultProvider, checkName), prompt, label, vars, providerCallContext);
-	if (resp.error || !resp.output) {
-		if (throwOnError) throw new Error(resp.error || "No output");
-		return fail(resp.error || "No output", resp.tokenUsage);
-	}
-	const { parsed, failure } = parseJsonGradingResponse(label, resp);
-	if (!parsed) return failure;
-	let pass = parsed.pass ?? true;
-	if (typeof pass !== "boolean") pass = /^(true|yes|pass|y)$/i.test(String(pass));
-	let score = parsed.score;
-	if (typeof score !== "number") score = Number.isFinite(Number(score)) ? Number(score) : Number(pass);
-	const threshold = typeof assertion?.threshold === "string" ? Number(assertion.threshold) : assertion?.threshold;
-	if (typeof threshold === "number" && Number.isFinite(threshold)) pass = pass && score >= threshold;
-	const reason = parsed.reason || (pass ? "Grading passed" : `Score ${score} below threshold ${threshold}`);
-	let responseMetadata = {};
-	if (resp.metadata && typeof resp.metadata === "object" && !Array.isArray(resp.metadata)) {
-		const serializedMetadata = safeJsonStringify(resp.metadata);
-		responseMetadata = serializedMetadata ? JSON.parse(serializedMetadata) : {};
-	}
+function parseLegacyFactualityResponse(responseText) {
+	const answerMatch = responseText.match(/\s*\(?([a-eA-E])\)/);
+	if (!answerMatch) throw new Error(`Factuality checker output did not match expected format: ${responseText}`);
+	const option = answerMatch[1].toUpperCase();
+	const reasonMatch = responseText.match(/\)\s*(.*)/s);
 	return {
-		assertion,
-		pass,
-		score,
-		reason,
-		tokensUsed: {
-			total: resp.tokenUsage?.total || 0,
-			prompt: resp.tokenUsage?.prompt || 0,
-			completion: resp.tokenUsage?.completion || 0,
-			cached: resp.tokenUsage?.cached || 0,
-			numRequests: resp.tokenUsage?.numRequests || 0,
-			completionDetails: parsed.tokensUsed?.completionDetails || {
-				reasoning: 0,
-				acceptedPrediction: 0,
-				rejectedPrediction: 0
-			}
-		},
-		metadata: {
-			...responseMetadata,
-			renderedGradingPrompt: prompt
-		}
+		option,
+		reason: reasonMatch?.[1] ? reasonMatch[1].trim() : responseText
 	};
 }
 async function matchesLlmRubric(rubric, llmOutput, grading, vars, assertion, options, providerCallContext) {
 	if (!grading) throw new Error("Cannot grade output without grading config. Specify --grader option or grading config.");
-	if (!grading.rubricPrompt && !state.config?.redteam?.provider && state.config?.redteam && shouldGenerateRemote({ canUseCodexDefaultProvider: true })) return {
-		...await doRemoteGrading({
-			task: "llm-rubric",
-			rubric,
-			output: llmOutput,
-			vars: vars || {}
-		}),
-		assertion
-	};
+	const shouldPreferRemote = options?.preferRemote || grading.__promptfooPreferRemote || !grading.provider;
+	if (!grading.rubricPrompt && shouldPreferRemote && !state.config?.redteam?.provider && state.config?.redteam && shouldGenerateRemote({ canUseCodexDefaultProvider: true })) try {
+		return {
+			...await doRemoteGrading({
+				task: "llm-rubric",
+				rubric,
+				output: llmOutput,
+				vars: vars || {}
+			}),
+			assertion
+		};
+	} catch (error) {
+		return {
+			...fail(`Could not perform remote grading: ${error}`),
+			assertion
+		};
+	}
 	try {
 		return await runJsonGradingPrompt({
 			assertion,
@@ -1859,89 +1608,42 @@ async function matchesPiScore(renderedValue, llmInput, llmOutput, assertion) {
 		assertion
 	};
 }
-function isFactualityCategory(category) {
-	return /^[A-E]$/.test(category);
-}
-function getFactualityScoreLookup(grading) {
-	return {
-		A: grading.factuality?.subset ?? 1,
-		B: grading.factuality?.superset ?? 1,
-		C: grading.factuality?.agree ?? 1,
-		D: grading.factuality?.disagree ?? 0,
-		E: grading.factuality?.differButFactual ?? 1
-	};
-}
-function buildFactualityCategoryResult(category, reason, grading, tokensUsed) {
-	const option = category.trim().toUpperCase();
-	if (!isFactualityCategory(option)) return fail(`Invalid category value: ${option}`, tokensUsed);
-	const score = getFactualityScoreLookup(grading)[option];
-	return {
-		pass: score > 0,
-		score,
-		reason: reason?.trim() || `Category ${option}: ${FACTUALITY_CATEGORY_DESCRIPTIONS[option]}`,
-		tokensUsed: normalizeTokenUsage(tokensUsed)
-	};
-}
-function parseJsonFactualityOutput(output) {
-	try {
-		const jsonData = extractFirstJsonObject(output);
-		return typeof jsonData?.category === "string" ? {
-			category: jsonData.category,
-			reason: jsonData.reason
-		} : null;
-	} catch (err) {
-		logger.debug(`JSON parsing failed: ${err.message}`);
-		return null;
-	}
-}
-function parseLegacyFactualityOutput(output) {
-	const answerMatch = output.match(/\s*\(?([a-eA-E])\)/);
-	if (!answerMatch) return { failure: `Factuality checker output did not match expected format: ${output}` };
-	const reasonMatch = output.match(/\)\s*(.*)/s);
-	return {
-		category: answerMatch[1],
-		reason: reasonMatch?.[1]?.trim() || output
-	};
-}
-function gradeFactualityOutput(output, grading, tokensUsed) {
-	const jsonResult = parseJsonFactualityOutput(output);
-	if (jsonResult) return buildFactualityCategoryResult(jsonResult.category, jsonResult.reason, grading, tokensUsed);
-	logger.info("Falling back to legacy pattern matching for factuality check");
-	const legacyResult = parseLegacyFactualityOutput(output);
-	return "failure" in legacyResult ? fail(legacyResult.failure, tokensUsed) : buildFactualityCategoryResult(legacyResult.category, legacyResult.reason, grading, tokensUsed);
-}
 async function matchesFactuality(input, expected, output, grading, vars, providerCallContext) {
 	if (!grading) throw new Error("Cannot grade output without grading config. Specify --grader option or grading config.");
-	const prompt = await renderLlmRubricPrompt(await loadRubricPrompt(grading?.rubricPrompt, PROMPTFOO_FACTUALITY_PROMPT), {
-		input,
-		ideal: expected,
-		completion: tryParse(output),
-		...vars || {}
-	});
-	const resp = await callProviderWithContext(await getAndCheckProvider("text", grading.provider, (await getDefaultProviders()).gradingProvider, "factuality check"), prompt, "factuality", {
+	const templateVars = {
 		input,
 		ideal: expected,
 		completion: tryParse(output),
 		...vars || {}
-	}, providerCallContext);
+	};
+	const prompt = await renderLlmRubricPrompt(await loadRubricPrompt(grading?.rubricPrompt, PROMPTFOO_FACTUALITY_PROMPT), templateVars);
+	const resp = await callProviderWithContext(await getAndCheckProvider("text", grading.provider, (await getDefaultProviders()).gradingProvider, "factuality check"), prompt, "factuality", templateVars, providerCallContext);
 	if (resp.error || !resp.output) return fail(resp.error || "No output", resp.tokenUsage);
 	invariant(typeof resp.output === "string", "factuality produced malformed response");
-	return gradeFactualityOutput(resp.output, grading, resp.tokenUsage);
+	try {
+		const parsedJson = parseFactualityJsonResponse(resp.output);
+		if (parsedJson) return buildFactualityResult(parsedJson.option, parsedJson.reason, grading, resp);
+	} catch (err) {
+		return fail(err.message, resp.tokenUsage);
+	}
+	logger.info("Falling back to legacy pattern matching for factuality check");
+	try {
+		const parsedLegacy = parseLegacyFactualityResponse(resp.output);
+		return buildFactualityResult(parsedLegacy.option, parsedLegacy.reason, grading, resp);
+	} catch (err) {
+		return fail(err.message, resp.tokenUsage);
+	}
 }
 async function matchesClosedQa(input, expected, output, grading, vars, providerCallContext) {
 	if (!grading) throw new Error("Cannot grade output without grading config. Specify --grader option or grading config.");
-	const prompt = await renderLlmRubricPrompt(await loadRubricPrompt(grading?.rubricPrompt, OPENAI_CLOSED_QA_PROMPT), {
-		input,
-		criteria: expected,
-		completion: tryParse(output),
-		...vars || {}
-	});
-	const resp = await callProviderWithContext(await getAndCheckProvider("text", grading.provider, (await getDefaultProviders()).gradingProvider, "model-graded-closedqa check"), prompt, "model-graded-closedqa", {
+	const templateVars = {
 		input,
 		criteria: expected,
 		completion: tryParse(output),
 		...vars || {}
-	}, providerCallContext);
+	};
+	const prompt = await renderLlmRubricPrompt(await loadRubricPrompt(grading?.rubricPrompt, OPENAI_CLOSED_QA_PROMPT), templateVars);
+	const resp = await callProviderWithContext(await getAndCheckProvider("text", grading.provider, (await getDefaultProviders()).gradingProvider, "model-graded-closedqa check"), prompt, "model-graded-closedqa", templateVars, providerCallContext);
 	if (resp.error || !resp.output) return fail(resp.error || "No output", resp.tokenUsage);
 	invariant(typeof resp.output === "string", "model-graded-closedqa produced malformed response");
 	try {
@@ -1954,18 +1656,7 @@ async function matchesClosedQa(input, expected, output, grading, vars, providerC
 			pass,
 			score: pass ? 1 : 0,
 			reason,
-			tokensUsed: {
-				total: resp.tokenUsage?.total || 0,
-				prompt: resp.tokenUsage?.prompt || 0,
-				completion: resp.tokenUsage?.completion || 0,
-				cached: resp.tokenUsage?.cached || 0,
-				numRequests: resp.tokenUsage?.numRequests || 0,
-				completionDetails: resp.tokenUsage?.completionDetails || {
-					reasoning: 0,
-					acceptedPrediction: 0,
-					rejectedPrediction: 0
-				}
-			}
+			tokensUsed: normalizeMatcherTokenUsage(resp.tokenUsage)
 		};
 	} catch (err) {
 		return fail(`Error parsing output: ${err.message}`, resp.tokenUsage);
@@ -1975,490 +1666,51 @@ async function matchesGEval(criteria, input, output, threshold, grading, provide
 	if (!input) throw Error("No source text to estimate reply");
 	const maxScore = 10;
 	const textProvider = await getAndCheckProvider("text", grading?.provider, (await getDefaultProviders()).gradingProvider, "reply geval check");
-	const tokensUsed = {
-		total: 0,
-		prompt: 0,
-		completion: 0,
-		cached: 0,
-		numRequests: 0,
-		completionDetails: {
-			reasoning: 0,
-			acceptedPrediction: 0,
-			rejectedPrediction: 0
-		}
-	};
+	const tokensUsed = normalizeMatcherTokenUsage(void 0);
 	const respSteps = await callProviderWithContext(textProvider, await renderLlmRubricPrompt(await loadRubricPrompt(typeof grading?.rubricPrompt === "object" && !Array.isArray(grading?.rubricPrompt) ? grading?.rubricPrompt?.["steps"] : void 0, GEVAL_PROMPT_STEPS), { criteria }), "g-eval-steps", { criteria }, providerCallContext);
-	accumulateTokens(tokensUsed, respSteps.tokenUsage);
+	accumulateTokenUsage(tokensUsed, respSteps.tokenUsage);
+	if (respSteps.error) return fail(respSteps.error, tokensUsed);
+	if (!respSteps.output) return fail("No output", tokensUsed);
+	if (typeof respSteps.output !== "string") return fail("LLM-proposed evaluation steps response is not a string", tokensUsed);
 	let steps;
 	try {
-		steps = JSON.parse(respSteps.output.match(/\{"steps".+\}/g)[0]).steps;
+		const stepsMatch = respSteps.output.match(/\{"steps".+\}/g);
+		if (!stepsMatch) return fail(`LLM-proposed evaluation steps are not in JSON format: ${respSteps.output}`, tokensUsed);
+		steps = JSON.parse(stepsMatch[0]).steps;
 		if (!steps.length) return fail("LLM does not propose any evaluation step", tokensUsed);
-	} catch {
-		return fail(`LLM-proposed evaluation steps are not in JSON format: ${respSteps.output}`, tokensUsed);
+	} catch (err) {
+		return fail(`LLM-proposed evaluation steps are not in JSON format: ${err.message}\n\n${respSteps.output}`, tokensUsed);
 	}
-	const resp = await callProviderWithContext(textProvider, await renderLlmRubricPrompt(await loadRubricPrompt(typeof grading?.rubricPrompt === "object" && !Array.isArray(grading?.rubricPrompt) ? grading?.rubricPrompt?.["evaluate"] : void 0, GEVAL_PROMPT_EVALUATE), {
-		criteria,
-		steps: steps.join("\n- "),
-		maxScore: maxScore.toString(),
-		input: tryParse(input),
-		output: tryParse(output)
-	}), "g-eval", {
+	const evalPrompt = await loadRubricPrompt(typeof grading?.rubricPrompt === "object" && !Array.isArray(grading?.rubricPrompt) ? grading?.rubricPrompt?.["evaluate"] : void 0, GEVAL_PROMPT_EVALUATE);
+	const evalVars = {
 		criteria,
 		steps: steps.join("\n- "),
 		maxScore: maxScore.toString(),
 		input: tryParse(input),
 		output: tryParse(output)
-	}, providerCallContext);
-	accumulateTokens(tokensUsed, resp.tokenUsage);
+	};
+	const resp = await callProviderWithContext(textProvider, await renderLlmRubricPrompt(evalPrompt, evalVars), "g-eval", evalVars, providerCallContext);
+	accumulateTokenUsage(tokensUsed, resp.tokenUsage);
+	if (resp.error) return fail(resp.error, tokensUsed);
+	if (!resp.output) return fail("No output", tokensUsed);
+	if (typeof resp.output !== "string") return fail("LLM-proposed evaluation result response is not a string", tokensUsed);
 	let result;
 	try {
-		result = JSON.parse(resp.output.match(/\{.+\}/g)[0]);
-	} catch {
-		return fail(`LLM-proposed evaluation result is not in JSON format: ${resp.output}`, tokensUsed);
+		const resultMatch = resp.output.match(/\{.+\}/g);
+		if (!resultMatch) return fail(`LLM-proposed evaluation result is not in JSON format: ${resp.output}`, tokensUsed);
+		result = JSON.parse(resultMatch[0]);
+	} catch (err) {
+		return fail(`LLM-proposed evaluation result is not in JSON format: ${err.message}\n\n${resp.output}`, tokensUsed);
 	}
+	const rawScore = typeof result.score === "number" ? result.score : Number(result.score);
+	if (!Number.isFinite(rawScore)) return fail(`G-Eval result has invalid or missing score: ${JSON.stringify(result.score)}`, tokensUsed);
 	return {
-		pass: result.score / maxScore >= threshold,
-		score: result.score / maxScore,
+		pass: rawScore / maxScore >= threshold,
+		score: rawScore / maxScore,
 		reason: result.reason,
 		tokensUsed
 	};
 }
-async function matchesAnswerRelevance(input, output, threshold, grading, providerCallContext) {
-	const embeddingProvider = await getAndCheckProvider("embedding", grading?.provider, (await getDefaultProviders()).embeddingProvider, "answer relevancy check");
-	const textProvider = await getAndCheckProvider("text", grading?.provider, (await getDefaultProviders()).gradingProvider, "answer relevancy check");
-	const tokensUsed = {
-		total: 0,
-		prompt: 0,
-		completion: 0,
-		cached: 0,
-		numRequests: 0,
-		completionDetails: {
-			reasoning: 0,
-			acceptedPrediction: 0,
-			rejectedPrediction: 0
-		}
-	};
-	const candidateQuestions = [];
-	for (let i = 0; i < 3; i++) {
-		const resp = await callProviderWithContext(textProvider, await renderLlmRubricPrompt(await loadRubricPrompt(grading?.rubricPrompt, ANSWER_RELEVANCY_GENERATE), { answer: tryParse(output) }), "answer-relevance", { answer: tryParse(output) }, providerCallContext);
-		accumulateTokens(tokensUsed, resp.tokenUsage);
-		if (resp.error || !resp.output) return fail(resp.error || "No output", tokensUsed);
-		invariant(typeof resp.output === "string", "answer relevancy check produced malformed response");
-		candidateQuestions.push(resp.output);
-	}
-	invariant(typeof embeddingProvider.callEmbeddingApi === "function", `Provider ${embeddingProvider.id} must implement callEmbeddingApi for similarity check`);
-	const inputEmbeddingResp = await embeddingProvider.callEmbeddingApi(input);
-	accumulateTokens(tokensUsed, inputEmbeddingResp.tokenUsage);
-	if (inputEmbeddingResp.error || !inputEmbeddingResp.embedding) return fail(inputEmbeddingResp.error || "No embedding", tokensUsed);
-	const inputEmbedding = inputEmbeddingResp.embedding;
-	const similarities = [];
-	const questionsWithScores = [];
-	for (const question of candidateQuestions) {
-		const resp = await embeddingProvider.callEmbeddingApi(question);
-		accumulateTokens(tokensUsed, resp.tokenUsage);
-		if (resp.error || !resp.embedding) return fail(resp.error || "No embedding", tokensUsed);
-		const questionSimilarity = cosineSimilarity(inputEmbedding, resp.embedding);
-		similarities.push(questionSimilarity);
-		questionsWithScores.push({
-			question,
-			similarity: questionSimilarity
-		});
-	}
-	const similarity = similarities.reduce((a, b) => a + b, 0) / similarities.length;
-	const pass = similarity >= threshold - Number.EPSILON;
-	const greaterThanReason = `Relevance ${similarity.toFixed(2)} is greater than threshold ${threshold}`;
-	const lessThanReason = `Relevance ${similarity.toFixed(2)} is less than threshold ${threshold}`;
-	const metadata = {
-		generatedQuestions: questionsWithScores,
-		averageSimilarity: similarity,
-		threshold
-	};
-	if (pass) return {
-		pass: true,
-		score: similarity,
-		reason: greaterThanReason,
-		tokensUsed,
-		metadata
-	};
-	return {
-		pass: false,
-		score: similarity,
-		reason: lessThanReason,
-		tokensUsed,
-		metadata
-	};
-}
-async function matchesContextRecall(context, groundTruth, threshold, grading, vars, providerCallContext) {
-	const textProvider = await getAndCheckProvider("text", grading?.provider, (await getDefaultProviders()).gradingProvider, "context recall check");
-	const contextString = serializeContext(context);
-	const resp = await callProviderWithContext(textProvider, await renderLlmRubricPrompt(await loadRubricPrompt(grading?.rubricPrompt, CONTEXT_RECALL), {
-		context: contextString,
-		groundTruth,
-		...vars || {}
-	}), "context-recall", {
-		context: contextString,
-		groundTruth,
-		...vars || {}
-	}, providerCallContext);
-	if (resp.error || !resp.output) return fail(resp.error || "No output", resp.tokenUsage);
-	invariant(typeof resp.output === "string", "context-recall produced malformed response");
-	const attributedTokenLower = CONTEXT_RECALL_ATTRIBUTED_TOKEN.toLowerCase();
-	const notAttributedTokenLower = CONTEXT_RECALL_NOT_ATTRIBUTED_TOKEN.toLowerCase();
-	const sentences = splitIntoSentences(resp.output).filter((line) => {
-		const lowerLine = line.toLowerCase();
-		return lowerLine.includes(attributedTokenLower) || lowerLine.includes(notAttributedTokenLower);
-	});
-	const sentenceAttributions = [];
-	let numerator = 0;
-	for (const sentence of sentences) {
-		const isAttributed = sentence.toLowerCase().includes(attributedTokenLower);
-		if (isAttributed) numerator++;
-		const sentenceMatch = sentence.match(/^\d+\.\s*([^\.]+\.)/);
-		const cleanSentence = sentenceMatch ? sentenceMatch[1].trim() : sentence.split(".")[0].trim();
-		sentenceAttributions.push({
-			sentence: cleanSentence,
-			attributed: isAttributed
-		});
-	}
-	const score = sentences.length > 0 ? numerator / sentences.length : 0;
-	const pass = score >= threshold - Number.EPSILON;
-	const metadata = {
-		sentenceAttributions,
-		totalSentences: sentences.length,
-		attributedSentences: numerator,
-		score
-	};
-	return {
-		pass,
-		score,
-		reason: pass ? `Recall ${score.toFixed(2)} is >= ${threshold}` : `Recall ${score.toFixed(2)} is < ${threshold}`,
-		tokensUsed: {
-			total: resp.tokenUsage?.total || 0,
-			prompt: resp.tokenUsage?.prompt || 0,
-			completion: resp.tokenUsage?.completion || 0,
-			cached: resp.tokenUsage?.cached || 0,
-			numRequests: resp.tokenUsage?.numRequests || 0,
-			completionDetails: resp.tokenUsage?.completionDetails || {
-				reasoning: 0,
-				acceptedPrediction: 0,
-				rejectedPrediction: 0
-			}
-		},
-		metadata
-	};
-}
-async function matchesContextRelevance(question, context, threshold, grading, providerCallContext) {
-	const textProvider = await getAndCheckProvider("text", grading?.provider, (await getDefaultProviders()).gradingProvider, "context relevance check");
-	const contextString = serializeContext(context);
-	const resp = await callProviderWithContext(textProvider, await renderLlmRubricPrompt(await loadRubricPrompt(grading?.rubricPrompt, CONTEXT_RELEVANCE), {
-		context: contextString,
-		query: question
-	}), "context-relevance", {
-		context: contextString,
-		query: question
-	}, providerCallContext);
-	if (resp.error || !resp.output) return fail(resp.error || "No output", resp.tokenUsage);
-	invariant(typeof resp.output === "string", "context-relevance produced malformed response");
-	const contextUnits = Array.isArray(context) ? context.filter((chunk) => chunk.trim().length > 0) : splitIntoSentences(context);
-	const totalContextUnits = contextUnits.length;
-	const extractedSentences = splitIntoSentences(resp.output);
-	const relevantSentences = [];
-	const insufficientInformation = resp.output.includes(CONTEXT_RELEVANCE_BAD);
-	let numerator = 0;
-	if (insufficientInformation) numerator = 0;
-	else {
-		numerator = extractedSentences.length;
-		relevantSentences.push(...extractedSentences);
-	}
-	const score = totalContextUnits > 0 ? numerator / totalContextUnits : 0;
-	const pass = score >= threshold - Number.EPSILON;
-	const metadata = {
-		extractedSentences: relevantSentences,
-		totalContextUnits,
-		totalContextSentences: totalContextUnits,
-		contextUnits,
-		relevantSentenceCount: numerator,
-		insufficientInformation,
-		score
-	};
-	return {
-		pass,
-		score,
-		reason: pass ? `Context relevance ${score.toFixed(2)} is >= ${threshold}` : `Context relevance ${score.toFixed(2)} is < ${threshold}`,
-		tokensUsed: {
-			total: resp.tokenUsage?.total || 0,
-			prompt: resp.tokenUsage?.prompt || 0,
-			completion: resp.tokenUsage?.completion || 0,
-			cached: resp.tokenUsage?.cached || 0,
-			numRequests: resp.tokenUsage?.numRequests || 0,
-			completionDetails: resp.tokenUsage?.completionDetails || {
-				reasoning: 0,
-				acceptedPrediction: 0,
-				rejectedPrediction: 0
-			}
-		},
-		metadata
-	};
-}
-async function matchesContextFaithfulness(query, output, context, threshold, grading, vars, providerCallContext) {
-	const textProvider = await getAndCheckProvider("text", grading?.provider, (await getDefaultProviders()).gradingProvider, "faithfulness check");
-	const tokensUsed = {
-		total: 0,
-		prompt: 0,
-		completion: 0,
-		cached: 0,
-		numRequests: 0,
-		completionDetails: {
-			reasoning: 0,
-			acceptedPrediction: 0,
-			rejectedPrediction: 0
-		}
-	};
-	if (grading?.rubricPrompt) invariant(Array.isArray(grading.rubricPrompt), "rubricPrompt must be an array");
-	const rawLongformPrompt = typeof grading?.rubricPrompt?.[0] === "string" ? grading?.rubricPrompt?.[0] : grading?.rubricPrompt?.[0]?.content;
-	const rawNliPrompt = typeof grading?.rubricPrompt?.[1] === "string" ? grading?.rubricPrompt?.[1] : grading?.rubricPrompt?.[1]?.content;
-	const longformPrompt = await loadRubricPrompt(rawLongformPrompt, CONTEXT_FAITHFULNESS_LONGFORM);
-	const nliPrompt = await loadRubricPrompt(rawNliPrompt, CONTEXT_FAITHFULNESS_NLI_STATEMENTS);
-	let promptText = await renderLlmRubricPrompt(longformPrompt, {
-		question: query,
-		answer: tryParse(output),
-		...vars || {}
-	});
-	let resp = await callProviderWithContext(textProvider, promptText, "context-faithfulness-longform", {
-		question: query,
-		answer: tryParse(output),
-		...vars || {}
-	}, providerCallContext);
-	accumulateTokens(tokensUsed, resp.tokenUsage);
-	if (resp.error || !resp.output) return fail(resp.error || "No output", tokensUsed);
-	invariant(typeof resp.output === "string", "context-faithfulness produced malformed response");
-	const contextString = serializeContext(context);
-	const statements = splitIntoSentences(resp.output);
-	promptText = await renderLlmRubricPrompt(nliPrompt, {
-		context: contextString,
-		statements,
-		...vars || {}
-	});
-	resp = await callProviderWithContext(textProvider, promptText, "context-faithfulness-nli", {
-		context: contextString,
-		statements,
-		...vars || {}
-	}, providerCallContext);
-	accumulateTokens(tokensUsed, resp.tokenUsage);
-	if (resp.error || !resp.output) return fail(resp.error || "No output", tokensUsed);
-	invariant(typeof resp.output === "string", "context-faithfulness produced malformed response");
-	let finalAnswer = "Final verdict for each statement in order:";
-	finalAnswer = finalAnswer.toLowerCase();
-	let verdicts = resp.output.toLowerCase().trim();
-	let score = 0;
-	if (statements.length > 0) if (verdicts.includes(finalAnswer)) {
-		verdicts = verdicts.slice(verdicts.indexOf(finalAnswer) + finalAnswer.length);
-		const parsedVerdicts = verdicts.split(".").filter((answer) => answer.trim() !== "");
-		if (parsedVerdicts.length > 0) score = 1 - parsedVerdicts.filter((answer) => !answer.includes("yes")).length / statements.length;
-	} else {
-		const noVerdictCount = verdicts.split("verdict: no").length - 1;
-		if (noVerdictCount + (verdicts.split("verdict: yes").length - 1) > 0) score = 1 - noVerdictCount / statements.length;
-	}
-	score = Math.min(1, Math.max(0, score));
-	const pass = score >= threshold - Number.EPSILON;
-	return {
-		pass,
-		score,
-		reason: pass ? `Faithfulness ${score.toFixed(2)} is >= ${threshold}` : `Faithfulness ${score.toFixed(2)} is < ${threshold}`,
-		tokensUsed
-	};
-}
-async function matchesSelectBest(criteria, outputs, grading, vars, providerCallContext) {
-	invariant(outputs.length >= 2, "select-best assertion must have at least two outputs to compare between");
-	const resp = await callProviderWithContext(await getAndCheckProvider("text", grading?.provider, (await getDefaultProviders()).gradingProvider, "select-best check"), await renderLlmRubricPrompt(await loadRubricPrompt(grading?.rubricPrompt, SELECT_BEST_PROMPT), {
-		criteria,
-		outputs: outputs.map((o) => tryParse(o)),
-		...vars || {}
-	}), "select-best", {
-		criteria,
-		outputs: outputs.map((o) => tryParse(o)),
-		...vars || {}
-	}, providerCallContext);
-	if (resp.error || !resp.output) return new Array(outputs.length).fill(fail(resp.error || "No output", resp.tokenUsage));
-	invariant(typeof resp.output === "string", "select-best produced malformed response");
-	const firstDigitMatch = resp.output.trim().match(/\d/);
-	const verdict = firstDigitMatch ? Number.parseInt(firstDigitMatch[0], 10) : NaN;
-	if (Number.isNaN(verdict) || verdict < 0 || verdict >= outputs.length) return new Array(outputs.length).fill(fail(`Invalid select-best verdict: ${verdict}`));
-	const tokensUsed = {
-		total: resp.tokenUsage?.total || 0,
-		prompt: resp.tokenUsage?.prompt || 0,
-		completion: resp.tokenUsage?.completion || 0,
-		cached: resp.tokenUsage?.cached || 0,
-		numRequests: resp.tokenUsage?.numRequests || 0,
-		completionDetails: resp.tokenUsage?.completionDetails || {
-			reasoning: 0,
-			acceptedPrediction: 0,
-			rejectedPrediction: 0
-		}
-	};
-	return outputs.map((_output, index) => {
-		if (index === verdict) return {
-			pass: true,
-			score: 1,
-			reason: `Output selected as the best: ${criteria}`,
-			tokensUsed
-		};
-		else return {
-			pass: false,
-			score: 0,
-			reason: `Output not selected: ${criteria}`,
-			tokensUsed
-		};
-	});
-}
-async function selectMaxScore(outputs, resultsWithGradingResults, assertion) {
-	invariant(outputs.length >= 2, "max-score assertion must have at least two outputs to compare between");
-	const value = assertion.value || {};
-	const options = {
-		method: typeof value === "object" && "method" in value ? value.method : "average",
-		weights: typeof value === "object" && "weights" in value ? value.weights : {},
-		threshold: typeof value === "object" && "threshold" in value ? value.threshold : void 0
-	};
-	const scores = resultsWithGradingResults.map((result, index) => {
-		const relevantResults = (result.gradingResult?.componentResults || []).filter((r) => r.assertion && r.assertion.type !== "max-score" && r.assertion.type !== "select-best");
-		if (relevantResults.length === 0) throw new Error("max-score requires at least one other assertion (besides max-score or select-best) to aggregate scores from");
-		let totalWeightedScore = 0;
-		let totalWeight = 0;
-		relevantResults.forEach((componentResult) => {
-			const assertionType = componentResult.assertion?.type || "unknown";
-			const weight = options.weights[assertionType] === void 0 ? 1 : options.weights[assertionType];
-			const score = componentResult.score || 0;
-			totalWeightedScore += score * weight;
-			totalWeight += weight;
-		});
-		let aggregateScore;
-		if (options.method === "sum") aggregateScore = totalWeightedScore;
-		else aggregateScore = totalWeight > 0 ? totalWeightedScore / totalWeight : 0;
-		return {
-			index,
-			score: aggregateScore,
-			componentCount: relevantResults.length,
-			totalWeight
-		};
-	});
-	let maxScore = -Infinity;
-	let winnerIndex = 0;
-	for (let i = 0; i < scores.length; i++) if (scores[i].score > maxScore) {
-		maxScore = scores[i].score;
-		winnerIndex = i;
-	}
-	const meetsThreshold = options.threshold === void 0 || maxScore >= options.threshold;
-	return scores.map(({ index, score, componentCount, totalWeight }) => {
-		const isWinner = index === winnerIndex && meetsThreshold;
-		return {
-			pass: isWinner,
-			score: isWinner ? 1 : 0,
-			reason: isWinner ? `Selected as highest scoring output (score: ${score.toFixed(3)})` : score === maxScore && !meetsThreshold ? `Not selected - score ${score.toFixed(3)} below threshold ${options.threshold}` : `Not selected (score: ${score.toFixed(3)}, max: ${maxScore.toFixed(3)})`,
-			namedScores: {
-				maxScore: score,
-				assertionCount: componentCount,
-				totalWeight
-			}
-		};
-	});
-}
-async function matchesSearchRubric(rubric, llmOutput, grading, vars, assertion, _provider, providerCallContext) {
-	if (!grading) throw new Error("Cannot grade output without grading config. Specify --grader option or grading config.");
-	const defaultProviders = await getDefaultProviders();
-	const defaultSearchProviders = [
-		defaultProviders.webSearchProvider,
-		defaultProviders.llmRubricProvider,
-		defaultProviders.gradingProvider
-	];
-	let searchProvider = (grading.provider ? await getGradingProvider("text", grading.provider, null) : null) || defaultSearchProviders.find((provider) => Boolean(provider));
-	if (!hasWebSearchCapability(searchProvider)) {
-		const webSearchDefault = defaultSearchProviders.find((provider) => hasWebSearchCapability(provider));
-		if (webSearchDefault) searchProvider = webSearchDefault;
-	}
-	if (!hasWebSearchCapability(searchProvider)) {
-		const webSearchProvider = await loadWebSearchProvider(true);
-		if (webSearchProvider) searchProvider = webSearchProvider;
-	}
-	if (!searchProvider || !hasWebSearchCapability(searchProvider)) throw new Error("search-rubric assertion requires a grading provider with web search capabilities. Use --grader with a web search provider (e.g., anthropic:messages:claude-sonnet-4, openai:responses:o4-mini with tools configured, perplexity:sonar) or configure one in defaultTest.options.provider");
-	const prompt = await renderLlmRubricPrompt(await loadRubricPrompt(grading?.rubricPrompt, DEFAULT_WEB_SEARCH_PROMPT), {
-		output: tryParse(llmOutput),
-		rubric,
-		...vars || {}
-	});
-	const resp = await callProviderWithContext(searchProvider, prompt, "search-rubric", {
-		output: tryParse(llmOutput),
-		rubric,
-		...vars || {}
-	}, providerCallContext);
-	if (resp.error || !resp.output) return {
-		pass: false,
-		score: 0,
-		reason: `Search rubric evaluation failed: ${resp.error || "No output"}`,
-		tokensUsed: resp.tokenUsage,
-		assertion
-	};
-	try {
-		const result = extractFirstJsonObject(String(resp.output));
-		let pass = result.pass ?? false;
-		const score = typeof result.score === "number" ? result.score : pass ? 1 : 0;
-		if (assertion?.threshold !== void 0) pass = pass && score >= assertion.threshold;
-		return {
-			pass,
-			score,
-			reason: result.reason || "No reason provided",
-			tokensUsed: resp.tokenUsage,
-			assertion,
-			metadata: {
-				searchResults: result.searchResults || [],
-				searchProvider: searchProvider.id()
-			}
-		};
-	} catch {
-		const outputLower = String(resp.output).toLowerCase();
-		const pass = outputLower.includes("\"pass\":true") || outputLower.includes("\"pass\": true");
-		return {
-			pass,
-			score: pass ? 1 : 0,
-			reason: resp.output,
-			tokensUsed: resp.tokenUsage,
-			assertion
-		};
-	}
-}
-async function matchesModeration({ userPrompt, assistantResponse, categories = [] }, grading) {
-	if (!assistantResponse) return {
-		pass: true,
-		score: 1,
-		reason: "No output to moderate"
-	};
-	const defaultProviders = await getDefaultProviders();
-	const defaultModerationProvider = !getEnvString("OPENAI_API_KEY") && (getEnvString("REPLICATE_API_KEY") || getEnvString("REPLICATE_API_TOKEN")) ? await loadApiProvider(LLAMA_GUARD_REPLICATE_PROVIDER) : defaultProviders.moderationProvider;
-	const moderationProvider = await getAndCheckProvider("moderation", grading?.provider, defaultModerationProvider, "moderation check");
-	invariant(moderationProvider, "Moderation provider must be defined");
-	const resp = await moderationProvider.callModerationApi(userPrompt, assistantResponse);
-	if (resp.error) return {
-		pass: false,
-		score: 0,
-		reason: `Moderation API error: ${resp.error}`
-	};
-	const { flags } = resp;
-	if (!flags || flags.length === 0) return {
-		pass: true,
-		score: 1,
-		reason: "No moderation flags detected"
-	};
-	const filteredFlags = categories.length === 0 ? flags : flags.filter((flag) => categories.includes(flag.code));
-	if (filteredFlags.length > 0) return {
-		pass: false,
-		score: 0,
-		reason: `Moderation flags detected: ${filteredFlags.map((flag) => flag.description).join(", ")}`
-	};
-	return {
-		pass: true,
-		score: 1,
-		reason: "No relevant moderation flags detected"
-	};
-}
 //#endregion
 //#region src/integrations/huggingfaceDatasets.ts
 /**
@@ -2984,7 +2236,7 @@ var RedteamPluginBase = class RedteamPluginBase {
 			const rejectedPromptLengths = [];
 			let rejectedPromptLimit;
 			for (const prompt of parsedPrompts) {
-				const violation = getGeneratedPromptOverLimit("__prompt" in prompt ? String(prompt.__prompt) : JSON.stringify(prompt), this.config.maxCharsPerMessage);
+				const violation = getGeneratedPromptOverLimit("__prompt" in prompt ? prompt.__prompt : JSON.stringify(prompt), this.config.maxCharsPerMessage);
 				if (violation) {
 					rejectedPromptLengths.push(violation.length);
 					rejectedPromptLimit = violation.limit;
@@ -3148,10 +2400,17 @@ var RedteamGraderBase = class {
 			},
 			rubric: finalRubric
 		};
-		const grade = await matchesLlmRubric(finalRubric, llmOutput, {
+		const defaultTest = typeof state.config?.defaultTest === "object" ? state.config.defaultTest : void 0;
+		const hasConfiguredGradingProvider = Boolean(state.config?.redteam?.provider || defaultTest?.options?.provider);
+		const grading = {
 			...test.options,
 			provider: await redteamProviderManager.getGradingProvider({ jsonOnly: true })
-		});
+		};
+		if (!hasConfiguredGradingProvider) {
+			Object.defineProperty(grading, "__promptfooPreferRemote", { value: true });
+			logger.debug("[Redteam] No configured grading provider detected, preferring remote grading");
+		}
+		const grade = await matchesLlmRubric(finalRubric, llmOutput, grading);
 		logger.debug(`Redteam grading result for ${this.id}: - ${JSON.stringify(grade)}`);
 		let suggestions;
 		if (!grade.pass) suggestions = this.getSuggestions({
@@ -15970,6 +15229,6 @@ function getGraderById(id) {
 	return grader;
 }
 //#endregion
-export { matchesPiScore as $, DivergentRepetitionPlugin as A, fail as B, getPiiLeakTestsForCategory as C, HarmbenchPlugin as D, ImitationPlugin as E, AegisPlugin as F, matchesClosedQa as G, loadRubricPrompt as H, RedteamGraderBase as I, matchesContextRelevance as J, matchesContextFaithfulness as K, RedteamPluginBase as L, CrossSessionLeakPlugin as M, ContractPlugin as N, HallucinationPlugin as O, BeavertailsPlugin as P, matchesModeration as Q, fetchHuggingFaceDataset as R, PlinyPlugin as S, IntentPlugin as T, matchesAnswerRelevance as U, getAndCheckProvider as V, matchesClassification as W, matchesGEval as X, matchesFactuality as Y, matchesLlmRubric as Z, PoliticsPlugin as _, retryWithDeduplication as _t, UnverifiableClaimsPlugin as a, withProviderCallExecutionContext as at, isValidPolicyObject as b, ToolDiscoveryPlugin as c, readPrompts as ct, TeenSafetyDangerousContentPlugin as d, coerceString as dt, matchesSearchRubric as et, TeenSafetyAgeRestrictedGoodsAndServicesPlugin as f, getFinalTest as ft, PromptExtractionPlugin as g, getCustomPolicies as gt, RbacPlugin as h, resolveContext as ht, VLGuardPlugin as i, selectMaxScore as it, DebugAccessPlugin as j, ExcessiveAgencyPlugin as k, TeenSafetyHarmfulBodyIdealsPlugin as l, readProviderPromptMap as lt, ShellInjectionPlugin as m, processFileReference as mt, getGraderById as n, matchesSimilarity as nt, UnsafeBenchPlugin as o, doRemoteGrading as ot, SqlInjectionPlugin as p, loadFromJavaScriptFile as pt, matchesContextRecall as q, VLSUPlugin as r, matchesTrajectoryGoalSuccess as rt, ToxicChatPlugin as s, processPrompts as st, GRADERS as t, matchesSelectBest as tt, TeenSafetyDangerousRoleplayPlugin as u, SUGGEST_PROMPTS_SYSTEM_MESSAGE as ut, PolicyPlugin as v, sampleArray as vt, OverreliancePlugin as w, makeInlinePolicyIdSync as x, determinePolicyTypeFromId as y, getDefaultProviders as yt, callProviderWithContext as z };
+export { CONTEXT_FAITHFULNESS_LONGFORM as $, DivergentRepetitionPlugin as A, matchesFactuality as B, getPiiLeakTestsForCategory as C, getCustomPolicies as Ct, HarmbenchPlugin as D, DEFAULT_ANTHROPIC_MODEL as Dt, ImitationPlugin as E, getDefaultProviders as Et, AegisPlugin as F, doRemoteGrading as G, matchesLlmRubric as H, RedteamGraderBase as I, readProviderPromptMap as J, processPrompts as K, RedteamPluginBase as L, CrossSessionLeakPlugin as M, ContractPlugin as N, HallucinationPlugin as O, BeavertailsPlugin as P, ANSWER_RELEVANCY_GENERATE as Q, fetchHuggingFaceDataset as R, PlinyPlugin as S, withProviderCallExecutionContext as St, IntentPlugin as T, sampleArray as Tt, matchesPiScore as U, matchesGEval as V, matchesTrajectoryGoalSuccess as W, SELECT_BEST_PROMPT as X, DEFAULT_WEB_SEARCH_PROMPT as Y, SUGGEST_PROMPTS_SYSTEM_MESSAGE as Z, PoliticsPlugin as _, loadFromJavaScriptFile as _t, UnverifiableClaimsPlugin as a, CONTEXT_RELEVANCE_BAD as at, isValidPolicyObject as b, getAndCheckProvider as bt, ToolDiscoveryPlugin as c, cosineSimilarity as ct, TeenSafetyDangerousContentPlugin as d, fail as dt, CONTEXT_FAITHFULNESS_NLI_STATEMENTS as et, TeenSafetyAgeRestrictedGoodsAndServicesPlugin as f, normalizeMatcherTokenUsage as ft, PromptExtractionPlugin as g, getFinalTest as gt, RbacPlugin as h, coerceString as ht, VLGuardPlugin as i, CONTEXT_RELEVANCE as it, DebugAccessPlugin as j, ExcessiveAgencyPlugin as k, TeenSafetyHarmfulBodyIdealsPlugin as l, dotProduct as lt, ShellInjectionPlugin as m, tryParse as mt, getGraderById as n, CONTEXT_RECALL_ATTRIBUTED_TOKEN as nt, UnsafeBenchPlugin as o, loadRubricPrompt as ot, SqlInjectionPlugin as p, splitIntoSentences as pt, readPrompts as q, VLSUPlugin as r, CONTEXT_RECALL_NOT_ATTRIBUTED_TOKEN as rt, ToxicChatPlugin as s, renderLlmRubricPrompt as st, GRADERS as t, CONTEXT_RECALL as tt, TeenSafetyDangerousRoleplayPlugin as u, euclideanDistance as ut, PolicyPlugin as v, processFileReference as vt, OverreliancePlugin as w, retryWithDeduplication as wt, makeInlinePolicyIdSync as x, getGradingProvider as xt, determinePolicyTypeFromId as y, callProviderWithContext as yt, matchesClosedQa as z };
 
-//# sourceMappingURL=graders-pvbReLLn.js.map
+//# sourceMappingURL=graders-CwrbifOo.js.map