promptfoo 0.121.1 → 0.121.3

package/dist/src/{graders-R9rYUM0d.js → graders-BXAJ0sbS.js}

@@ -1,24 +1,24 @@
-import { C as isCI, _ as getEnvBool, b as getEnvString, i as logger, l as extractFirstJsonObject, m as safeJsonStringify, u as extractJsonObjects, w as state } from "./logger-DLcq4dWf.js";
-import { d as sleep, p as REQUEST_TIMEOUT_MS, r as fetchWithTimeout, t as fetchWithProxy } from "./fetch-4M3YRaqL.js";
+import { C as getEnvString, D as state, E as isCI, _ as safeJsonStringify, a as logger, b as getEnvBool, f as extractFirstJsonObject, p as extractJsonObjects } from "./logger-BnkjG2jt.js";
+import { d as sleep, p as REQUEST_TIMEOUT_MS, r as fetchWithTimeout, t as fetchWithProxy } from "./fetch-BiYv2BZc.js";
 import { t as invariant } from "./invariant-vgHWClmd.js";
-import { o as getUserEmail } from "./accounts-BgNJDBE6.js";
-import { c as getRemoteGenerationUrl, p as shouldGenerateRemote } from "./server-OAs3nBRT.js";
-import { r as importModule } from "./esm-CaIwzWR5.js";
-import { C as isValidReusablePolicyId, X as LLAMA_GUARD_REPLICATE_PROVIDER, k as MULTI_TURN_STRATEGIES, lt as PromptSchema, x as PolicyObjectSchema } from "./types-CN_TZ2GJ.js";
+import { o as getUserEmail } from "./accounts-Xatc0RYb.js";
+import { c as getRemoteGenerationUrl, p as shouldGenerateRemote } from "./server-Cns05F1j.js";
+import { r as importModule } from "./esm-CKWP3u_P.js";
+import { C as isValidReusablePolicyId, X as LLAMA_GUARD_REPLICATE_PROVIDER, k as MULTI_TURN_STRATEGIES, lt as PromptSchema, x as PolicyObjectSchema } from "./types-Cd3ygw8W.js";
 import { i as isJavascriptFile } from "./fileExtensions-LcDYkU4v.js";
 import { n as sha256 } from "./createHash-CTQmL3G2.js";
-import { i as PROMPT_DELIMITER, n as maybeFilePath, r as normalizeInput } from "./utils-dLokC-eR.js";
-import { a as isCacheEnabled, i as getCache, r as fetchWithCache } from "./cache-D3eqDYGU.js";
+import { i as PROMPT_DELIMITER, n as maybeFilePath, r as normalizeInput } from "./utils-DOjD4dTC.js";
+import { a as isCacheEnabled, i as getCache, r as fetchWithCache } from "./cache-HP0NP4k3.js";
 import { r as accumulateTokenUsage } from "./tokenUsageUtils-BDGe-iyI.js";
-import { $ as DefaultSynthesizeProvider$1, A as removePrefix, C as extractInputVarsFromPrompt, D as getShortPluginId, G as DefaultEmbeddingProvider$2, H as OpenAiModerationProvider, I as redteamProviderManager, K as DefaultGradingProvider$3, O as isBasicRefusal, Q as DefaultSuggestionsProvider$2, S as extractGoalFromPrompt, T as extractVariablesFromJson, U as MistralChatCompletionProvider, V as REDTEAM_MEMORY_POISONING_PLUGIN_ID, W as MistralEmbeddingProvider, X as DefaultGradingProvider$2, Y as DefaultGradingJsonProvider$2, Z as DefaultLlmRubricProvider, b as checkExfilTracking, ct as getPoliciesFromCloud, et as AzureModerationProvider, k as isEmptyResponse, n as loadApiProvider, nt as AzureChatCompletionProvider, o as getFileHashes, s as parseScriptParts, tt as AzureEmbeddingProvider, w as extractPromptFromTags, x as extractAllPromptsFromTags } from "./providers-B3HvufyI.js";
-import { r as runPython } from "./pythonUtils-D6fwaDSg.js";
-import { I as getNunjucksEngine, O as maybeLoadToolsFromExternalFile, P as extractVariablesFromTemplate, R as parseFileUrl, S as getNunjucksEngineForFilePath, T as maybeLoadFromExternalFile, k as parsePathOrGlob, w as maybeLoadConfigFromExternalFile } from "./util-5cB-L7U3.js";
-import { t as OpenAiChatCompletionProvider } from "./chat-BKm79wib.js";
-import { v as hasGoogleDefaultCredentials } from "./transform-DGxXocjk.js";
-import { t as AnthropicMessagesProvider } from "./messages-D8EA0oDc.js";
-import { t as OpenAiResponsesProvider } from "./responses-jxdehPkC.js";
-import { n as OpenAiEmbeddingProvider } from "./completion-6Mx_iXxK.js";
-import { n as transform } from "./transform-aa6tmVpZ.js";
+import { $ as DefaultSynthesizeProvider$1, A as removePrefix, C as extractInputVarsFromPrompt, D as getShortPluginId, G as DefaultEmbeddingProvider$2, H as OpenAiModerationProvider, I as redteamProviderManager, K as DefaultGradingProvider$3, O as isBasicRefusal, Q as DefaultSuggestionsProvider$2, S as extractGoalFromPrompt, T as extractVariablesFromJson, U as MistralChatCompletionProvider, V as REDTEAM_MEMORY_POISONING_PLUGIN_ID, W as MistralEmbeddingProvider, X as DefaultGradingProvider$2, Y as DefaultGradingJsonProvider$2, Z as DefaultLlmRubricProvider, b as checkExfilTracking, ct as getPoliciesFromCloud, et as AzureModerationProvider, k as isEmptyResponse, n as loadApiProvider, nt as AzureChatCompletionProvider, o as getFileHashes, s as parseScriptParts, tt as AzureEmbeddingProvider, w as extractPromptFromTags, x as extractAllPromptsFromTags } from "./providers-DvddrgxL.js";
+import { r as runPython } from "./pythonUtils-Bzwbgpbg.js";
+import { I as getNunjucksEngine, O as maybeLoadToolsFromExternalFile, P as extractVariablesFromTemplate, R as parseFileUrl, S as getNunjucksEngineForFilePath, T as maybeLoadFromExternalFile, k as parsePathOrGlob, w as maybeLoadConfigFromExternalFile } from "./util-BLvy9qfE.js";
+import { t as OpenAiChatCompletionProvider } from "./chat-B0iaWhoh.js";
+import { b as hasGoogleDefaultCredentials } from "./transform-B2-jIv68.js";
+import { t as AnthropicMessagesProvider } from "./messages-biC_ex-p.js";
+import { t as OpenAiResponsesProvider } from "./responses-CgNyTPsY.js";
+import { n as OpenAiEmbeddingProvider } from "./completion-BCimtq-h.js";
+import { n as transform } from "./transform-BzK09Q_9.js";
 import * as fs$1 from "fs";
 import fs from "fs";
 import path from "path";
@@ -588,6 +588,37 @@ const DEFAULT_WEB_SEARCH_PROMPT = JSON.stringify([{
 	role: "user",
 	content: "<Output>\n{{ output }}\n</Output>\n<Rubric>\n{{ rubric }}\n</Rubric>"
 }]);
+const TRAJECTORY_GOAL_SUCCESS_PROMPT = JSON.stringify([{
+	role: "system",
+	content: dedent`You are grading whether an AI agent successfully completed a goal based on its final output and a summarized execution trajectory. You respond with a JSON object with this structure: {reason: string, pass: boolean, score: number}
+
+    Judge end-to-end success, not stylistic perfection.
+    Use the trajectory as evidence for what the agent actually did.
+    Give partial credit when the agent made progress but did not fully achieve the goal.
+
+    Examples:
+
+    <Goal>Find the order status and tell the user whether it has shipped</Goal>
+    <Trajectory>{"stepCount":2,"steps":[{"index":1,"type":"tool","name":"search_orders"},{"index":2,"type":"message","name":"agent response"}]}</Trajectory>
+    <Output>Your order shipped yesterday and should arrive on Tuesday.</Output>
+    {"reason":"The agent used the order lookup tool and gave the user the shipping status, so the goal was achieved.","pass":true,"score":1.0}
+
+    <Goal>Find the order status and tell the user whether it has shipped</Goal>
+    <Trajectory>{"stepCount":1,"steps":[{"index":1,"type":"message","name":"agent response"}]}</Trajectory>
+    <Output>I cannot check your order right now.</Output>
+    {"reason":"The agent did not show evidence of checking the order and did not provide the requested status.","pass":false,"score":0.0}`
+}, {
+	role: "user",
+	content: dedent`<Goal>
+{{ goal }}
+</Goal>
+<Trajectory>
+{{ trajectory }}
+</Trajectory>
+<Output>
+{{ output }}
+</Output>`
+}]);
 //#endregion
 //#region src/prompts/processors/csv.ts
 /**
@@ -1578,45 +1609,31 @@ async function renderLlmRubricPrompt(rubricPrompt, context) {
 	} catch {}
 	return nunjucks.renderString(rubricPrompt, processedContext);
 }
-async function matchesLlmRubric(rubric, llmOutput, grading, vars, assertion, options, providerCallContext) {
-	if (!grading) throw new Error("Cannot grade output without grading config. Specify --grader option or grading config.");
-	if (!grading.rubricPrompt && !state.config?.redteam?.provider && state.config?.redteam && shouldGenerateRemote()) return {
-		...await doRemoteGrading({
-			task: "llm-rubric",
-			rubric,
-			output: llmOutput,
-			vars: vars || {}
-		}),
-		assertion
-	};
-	const prompt = await renderLlmRubricPrompt(await loadRubricPrompt(grading?.rubricPrompt, DEFAULT_GRADING_PROMPT), {
-		output: tryParse(llmOutput),
-		rubric,
-		...vars || {}
-	});
-	const defaultProviders = await getDefaultProviders();
-	const defaultProvider = defaultProviders.llmRubricProvider || defaultProviders.gradingJsonProvider;
-	const resp = await callProviderWithContext(await getAndCheckProvider("text", grading.provider, defaultProvider, "llm-rubric check"), prompt, "llm-rubric", {
-		output: tryParse(llmOutput),
-		rubric,
-		...vars || {}
-	}, providerCallContext);
-	if (resp.error || !resp.output) {
-		if (options?.throwOnError) throw new LlmRubricProviderError(resp.error || "No output");
-		return fail(resp.error || "No output", resp.tokenUsage);
-	}
+function parseJsonGradingResponse(label, resp) {
 	let jsonObjects = [];
 	if (typeof resp.output === "string") try {
 		jsonObjects = extractJsonObjects(resp.output);
-		if (jsonObjects.length === 0) return fail("Could not extract JSON from llm-rubric response", resp.tokenUsage);
+		if (jsonObjects.length === 0) return { failure: fail(`Could not extract JSON from ${label} response`, resp.tokenUsage) };
 	} catch (err) {
-		return fail(`llm-rubric produced malformed response: ${err}\n\n${resp.output}`, resp.tokenUsage);
+		return { failure: fail(`${label} produced malformed response: ${err}\n\n${resp.output}`, resp.tokenUsage) };
 	}
 	else if (typeof resp.output === "object") jsonObjects = [resp.output];
-	else return fail(`llm-rubric produced malformed response - output must be string or object. Output: ${JSON.stringify(resp.output)}`, resp.tokenUsage);
-	if (!Array.isArray(jsonObjects) || jsonObjects.length === 0) return fail(`llm-rubric produced malformed response - We were not able to parse the response as JSON. Output: ${JSON.stringify(resp.output)}`, resp.tokenUsage);
+	else return { failure: fail(`${label} produced malformed response - output must be string or object. Output: ${JSON.stringify(resp.output)}`, resp.tokenUsage) };
 	const parsed = jsonObjects[0];
-	if (typeof parsed !== "object" || parsed === null || parsed === void 0) return fail(`llm-rubric produced malformed response. We were not able to parse the response as JSON. Output: ${JSON.stringify(resp.output)}`, resp.tokenUsage);
+	if (typeof parsed !== "object" || parsed === null || parsed === void 0) return { failure: fail(`${label} produced malformed response. We were not able to parse the response as JSON. Output: ${JSON.stringify(resp.output)}`, resp.tokenUsage) };
+	return { parsed };
+}
+async function runJsonGradingPrompt({ assertion, checkName, defaultPrompt, grading, label, providerCallContext, throwOnError, vars }) {
+	const prompt = await renderLlmRubricPrompt(await loadRubricPrompt(grading.rubricPrompt, defaultPrompt), vars);
+	const defaultProviders = await getDefaultProviders();
+	const defaultProvider = defaultProviders.llmRubricProvider || defaultProviders.gradingJsonProvider;
+	const resp = await callProviderWithContext(await getAndCheckProvider("text", grading.provider, defaultProvider, checkName), prompt, label, vars, providerCallContext);
+	if (resp.error || !resp.output) {
+		if (throwOnError) throw new Error(resp.error || "No output");
+		return fail(resp.error || "No output", resp.tokenUsage);
+	}
+	const { parsed, failure } = parseJsonGradingResponse(label, resp);
+	if (!parsed) return failure;
 	let pass = parsed.pass ?? true;
 	if (typeof pass !== "boolean") pass = /^(true|yes|pass|y)$/i.test(String(pass));
 	let score = parsed.score;
@@ -1644,6 +1661,54 @@ async function matchesLlmRubric(rubric, llmOutput, grading, vars, assertion, opt
 		metadata: { renderedGradingPrompt: prompt }
 	};
 }
+async function matchesLlmRubric(rubric, llmOutput, grading, vars, assertion, options, providerCallContext) {
+	if (!grading) throw new Error("Cannot grade output without grading config. Specify --grader option or grading config.");
+	if (!grading.rubricPrompt && !state.config?.redteam?.provider && state.config?.redteam && shouldGenerateRemote()) return {
+		...await doRemoteGrading({
+			task: "llm-rubric",
+			rubric,
+			output: llmOutput,
+			vars: vars || {}
+		}),
+		assertion
+	};
+	try {
+		return await runJsonGradingPrompt({
+			assertion,
+			checkName: "llm-rubric check",
+			defaultPrompt: DEFAULT_GRADING_PROMPT,
+			grading,
+			label: "llm-rubric",
+			providerCallContext,
+			throwOnError: options?.throwOnError,
+			vars: {
+				output: tryParse(llmOutput),
+				rubric,
+				...vars || {}
+			}
+		});
+	} catch (error) {
+		if (options?.throwOnError) throw new LlmRubricProviderError(error.message || "No output");
+		throw error;
+	}
+}
+async function matchesTrajectoryGoalSuccess(goal, trajectory, llmOutput, grading, vars, assertion, providerCallContext) {
+	if (!grading) throw new Error("Cannot grade output without grading config. Specify --grader option or grading config.");
+	return runJsonGradingPrompt({
+		assertion,
+		checkName: "trajectory:goal-success check",
+		defaultPrompt: TRAJECTORY_GOAL_SUCCESS_PROMPT,
+		grading,
+		label: "trajectory:goal-success",
+		providerCallContext,
+		vars: {
+			...vars || {},
+			goal,
+			output: tryParse(llmOutput),
+			trajectory
+		}
+	});
+}
 async function matchesPiScore(renderedValue, llmInput, llmOutput, assertion) {
 	return {
 		...await doRemoteScoringWithPi({
@@ -2151,7 +2216,7 @@ async function selectMaxScore(outputs, resultsWithGradingResults, assertion) {
 		let totalWeight = 0;
 		relevantResults.forEach((componentResult) => {
 			const assertionType = componentResult.assertion?.type || "unknown";
-			const weight = options.weights[assertionType] !== void 0 ? options.weights[assertionType] : 1;
+			const weight = options.weights[assertionType] === void 0 ? 1 : options.weights[assertionType];
 			const score = componentResult.score || 0;
 			totalWeightedScore += score * weight;
 			totalWeight += weight;
@@ -2401,9 +2466,9 @@ async function fetchHuggingFaceDataset(datasetPath, limit) {
 		while (true) {
 			const requestParams = new URLSearchParams(queryParams);
 			requestParams.set("offset", offset.toString());
-			const remainingUserLimit = userLimit !== void 0 ? Math.max(userLimit - offset, 0) : void 0;
-			const remainingDatasetRows = totalRows !== void 0 ? Math.max(totalRows - offset, 0) : void 0;
-			const requestedLength = remainingUserLimit !== void 0 ? Math.min(pageSize, remainingUserLimit) : remainingDatasetRows !== void 0 ? Math.min(pageSize, remainingDatasetRows) : pageSize;
+			const remainingUserLimit = userLimit === void 0 ? void 0 : Math.max(userLimit - offset, 0);
+			const remainingDatasetRows = totalRows === void 0 ? void 0 : Math.max(totalRows - offset, 0);
+			const requestedLength = remainingUserLimit === void 0 ? remainingDatasetRows === void 0 ? pageSize : Math.min(pageSize, remainingDatasetRows) : Math.min(pageSize, remainingUserLimit);
 			if (requestedLength <= 0) {
 				logger.debug(`[HF Dataset] No remaining rows to fetch for ${owner}/${repo} (offset ${offset})`);
 				break;
@@ -13461,6 +13526,6 @@ function getGraderById(id) {
 	return grader;
 }
 //#endregion
-export { selectMaxScore as $, BeavertailsPlugin as A, matchesClassification as B, HarmbenchPlugin as C, DebugAccessPlugin as D, DivergentRepetitionPlugin as E, callProviderWithContext as F, matchesFactuality as G, matchesContextFaithfulness as H, fail as I, matchesModeration as J, matchesGEval as K, getAndCheckProvider as L, RedteamGraderBase as M, RedteamPluginBase as N, CrossSessionLeakPlugin as O, fetchHuggingFaceDataset as P, matchesSimilarity as Q, loadRubricPrompt as R, ImitationPlugin as S, ExcessiveAgencyPlugin as T, matchesContextRecall as U, matchesClosedQa as V, matchesContextRelevance as W, matchesSearchRubric as X, matchesPiScore as Y, matchesSelectBest as Z, makeInlinePolicyIdSync as _, UnverifiableClaimsPlugin as a, coerceString as at, OverreliancePlugin as b, ToolDiscoveryPlugin as c, processFileReference as ct, RbacPlugin as d, retryWithDeduplication as dt, doRemoteGrading as et, PromptExtractionPlugin as f, sampleArray as ft, isValidPolicyObject as g, determinePolicyTypeFromId as h, VLGuardPlugin as i, SUGGEST_PROMPTS_SYSTEM_MESSAGE as it, AegisPlugin as j, ContractPlugin as k, SqlInjectionPlugin as l, resolveContext as lt, PolicyPlugin as m, DefaultSuggestionsProvider as mt, getGraderById as n, readPrompts as nt, UnsafeBenchPlugin as o, getFinalTest as ot, PoliticsPlugin as p, getDefaultProviders as pt, matchesLlmRubric as q, VLSUPlugin as r, readProviderPromptMap as rt, ToxicChatPlugin as s, loadFromJavaScriptFile as st, GRADERS as t, processPrompts as tt, ShellInjectionPlugin as u, getCustomPolicies as ut, PlinyPlugin as v, HallucinationPlugin as w, IntentPlugin as x, getPiiLeakTestsForCategory as y, matchesAnswerRelevance as z };
+export { matchesTrajectoryGoalSuccess as $, BeavertailsPlugin as A, matchesClassification as B, HarmbenchPlugin as C, DebugAccessPlugin as D, DivergentRepetitionPlugin as E, callProviderWithContext as F, matchesFactuality as G, matchesContextFaithfulness as H, fail as I, matchesModeration as J, matchesGEval as K, getAndCheckProvider as L, RedteamGraderBase as M, RedteamPluginBase as N, CrossSessionLeakPlugin as O, fetchHuggingFaceDataset as P, matchesSimilarity as Q, loadRubricPrompt as R, ImitationPlugin as S, ExcessiveAgencyPlugin as T, matchesContextRecall as U, matchesClosedQa as V, matchesContextRelevance as W, matchesSearchRubric as X, matchesPiScore as Y, matchesSelectBest as Z, makeInlinePolicyIdSync as _, UnverifiableClaimsPlugin as a, SUGGEST_PROMPTS_SYSTEM_MESSAGE as at, OverreliancePlugin as b, ToolDiscoveryPlugin as c, loadFromJavaScriptFile as ct, RbacPlugin as d, getCustomPolicies as dt, selectMaxScore as et, PromptExtractionPlugin as f, retryWithDeduplication as ft, isValidPolicyObject as g, determinePolicyTypeFromId as h, DefaultSuggestionsProvider as ht, VLGuardPlugin as i, readProviderPromptMap as it, AegisPlugin as j, ContractPlugin as k, SqlInjectionPlugin as l, processFileReference as lt, PolicyPlugin as m, getDefaultProviders as mt, getGraderById as n, processPrompts as nt, UnsafeBenchPlugin as o, coerceString as ot, PoliticsPlugin as p, sampleArray as pt, matchesLlmRubric as q, VLSUPlugin as r, readPrompts as rt, ToxicChatPlugin as s, getFinalTest as st, GRADERS as t, doRemoteGrading as tt, ShellInjectionPlugin as u, resolveContext as ut, PlinyPlugin as v, HallucinationPlugin as w, IntentPlugin as x, getPiiLeakTestsForCategory as y, matchesAnswerRelevance as z };
 
-//# sourceMappingURL=graders-R9rYUM0d.js.map
+//# sourceMappingURL=graders-BXAJ0sbS.js.map

package/dist/src/graders-BxfEguVY.js

@@ -0,0 +1,32 @@
+import "./logger-BnkjG2jt.js";
+import "./fetch-BiYv2BZc.js";
+import "./accounts-Xatc0RYb.js";
+import "./server-Cns05F1j.js";
+import "./tables-BEIFz2tM.js";
+import "./esm-CKWP3u_P.js";
+import "./types-Cd3ygw8W.js";
+import "./utils-DOjD4dTC.js";
+import "./store-VB0GP46K.js";
+import "./cache-HP0NP4k3.js";
+import "./blobs-BUWmKWzo.js";
+import "./extractor-C8XwivI9.js";
+import "./providers-DvddrgxL.js";
+import "./telemetry-DPXLd7UE.js";
+import "./genaiTracer-70Z8BIuV.js";
+import "./pythonUtils-Bzwbgpbg.js";
+import "./util-BLvy9qfE.js";
+import "./chat-B0iaWhoh.js";
+import "./transform-B2-jIv68.js";
+import "./messages-biC_ex-p.js";
+import "./util-DbVG-yZU.js";
+import { n as getGraderById } from "./graders-BXAJ0sbS.js";
+import "./responses-CgNyTPsY.js";
+import "./openai-D6wITiVn.js";
+import "./util-BtoGs5Cb.js";
+import "./completion-BCimtq-h.js";
+import "./transform-BzK09Q_9.js";
+import "./base-Cz2ZC_iA.js";
+import "./image-B8b6f36E.js";
+import "./providerRegistry-BkzVH5Ba.js";
+import "./rubyUtils-DECSbsfY.js";
+export { getGraderById };

package/dist/src/graders-CzVMbEnv.js

@@ -0,0 +1,34 @@
+#!/usr/bin/env node
+import "./logger-BcJBzSSA.js";
+import "./fetch-DoVRJZhJ.js";
+import "./accounts-CFLK3mnD.js";
+import "./cloud-z8KZpUoa.js";
+import "./telemetry-BnH5VJAU.js";
+import "./types-CIhFeUC4.js";
+import "./server-DZ9MtCn0.js";
+import "./providers-Ch6Mr0gn.js";
+import "./cache-DSqR6ezl.js";
+import "./util-Bm3E9jpK.js";
+import "./esm-7UIl0pPM.js";
+import "./pythonUtils-wIqk7zAf.js";
+import "./transform-ljLYHEPh.js";
+import { n as getGraderById } from "./graders-RjHF8VfG.js";
+import "./utils-DEuL4VNB.js";
+import "./genaiTracer-C1rxGO8Q.js";
+import "./chat-BEwdgGEg.js";
+import "./transform-DrleutM3.js";
+import "./messages-DJNo37Ko.js";
+import "./util-DMFeUvLz.js";
+import "./responses-BKP_WYis.js";
+import "./openai-BMcwgD5C.js";
+import "./util-DM2rTn_6.js";
+import "./completion-DoYy49ti.js";
+import "./blobs-B1JriOyi.js";
+import "./tables-DmzvLbeZ.js";
+import "./extractor-CAZ2G3Kh.js";
+import "./store-P8OKm19S.js";
+import "./base-CKjwebIH.js";
+import "./image-PoF6DN3x.js";
+import "./providerRegistry-B9lh-_tx.js";
+import "./rubyUtils-CiVfln3g.js";
+export { getGraderById };

package/dist/src/{graders-CpdqD9PI.js → graders-DG7mhg-b.js}

@@ -1,24 +1,24 @@
-import { C as isCI, _ as getEnvBool, b as getEnvString, i as logger, l as extractFirstJsonObject, m as safeJsonStringify, u as extractJsonObjects, w as state } from "./logger-CT3IKMKA.js";
+import { C as getEnvString, D as state, E as isCI, _ as safeJsonStringify, a as logger, b as getEnvBool, f as extractFirstJsonObject, p as extractJsonObjects } from "./logger-DO8_zM18.js";
 import { t as invariant } from "./invariant-Ddh24eXh.js";
-import { r as importModule } from "./esm-Cd1AjG1D.js";
-import { r as runPython } from "./pythonUtils-D5nxkQ0P.js";
+import { r as importModule } from "./esm-SUNIX1x3.js";
+import { r as runPython } from "./pythonUtils-Cpo0Ez1p.js";
 import { i as isJavascriptFile } from "./fileExtensions-DnqA1y9x.js";
-import { n as transform } from "./transform-DECvGmzp.js";
-import { B as isValidReusablePolicyId, G as MULTI_TURN_STRATEGIES, R as PolicyObjectSchema, St as PromptSchema, ut as LLAMA_GUARD_REPLICATE_PROVIDER } from "./types-CLKiCBW3.js";
-import { _ as extractVariablesFromTemplate, a as getNunjucksEngineForFilePath, c as maybeLoadFromExternalFile, d as maybeLoadToolsFromExternalFile, f as parsePathOrGlob, s as maybeLoadConfigFromExternalFile, x as parseFileUrl, y as getNunjucksEngine } from "./util-Dlz_Wvgm.js";
-import { d as sleep, p as REQUEST_TIMEOUT_MS, r as fetchWithTimeout, t as fetchWithProxy } from "./fetch-60Gzydls.js";
-import { a as isCacheEnabled, i as getCache, r as fetchWithCache } from "./cache-8XhNqPKW.js";
-import { $ as DefaultSynthesizeProvider$1, G as DefaultEmbeddingProvider$2, H as OpenAiModerationProvider, K as DefaultGradingProvider$3, N as REDTEAM_MEMORY_POISONING_PLUGIN_ID, O as redteamProviderManager, Q as DefaultSuggestionsProvider$2, S as removePrefix, U as MistralChatCompletionProvider, W as MistralEmbeddingProvider, X as DefaultGradingProvider$2, Y as DefaultGradingJsonProvider$2, Z as DefaultLlmRubricProvider, _ as extractVariablesFromJson, b as isBasicRefusal, et as AzureModerationProvider, f as checkExfilTracking, g as extractPromptFromTags, h as extractInputVarsFromPrompt, it as parseScriptParts, m as extractGoalFromPrompt, n as loadApiProvider, nt as AzureChatCompletionProvider, p as extractAllPromptsFromTags, rt as getFileHashes, tt as AzureEmbeddingProvider, x as isEmptyResponse, y as getShortPluginId, z as getPoliciesFromCloud } from "./providers-BKRJTjBz.js";
-import { a as PROMPT_DELIMITER, n as maybeFilePath, r as normalizeInput } from "./utils-XiOAgly5.js";
+import { n as transform } from "./transform-BqPkNPYm.js";
+import { B as isValidReusablePolicyId, G as MULTI_TURN_STRATEGIES, R as PolicyObjectSchema, St as PromptSchema, ut as LLAMA_GUARD_REPLICATE_PROVIDER } from "./types-q8GXGF65.js";
+import { _ as extractVariablesFromTemplate, a as getNunjucksEngineForFilePath, c as maybeLoadFromExternalFile, d as maybeLoadToolsFromExternalFile, f as parsePathOrGlob, s as maybeLoadConfigFromExternalFile, x as parseFileUrl, y as getNunjucksEngine } from "./util-CMMkIxfU.js";
+import { d as sleep, p as REQUEST_TIMEOUT_MS, r as fetchWithTimeout, t as fetchWithProxy } from "./fetch-CVAtKnI3.js";
+import { a as isCacheEnabled, i as getCache, r as fetchWithCache } from "./cache-CaT5tPgo.js";
+import { $ as DefaultSynthesizeProvider$1, G as DefaultEmbeddingProvider$2, H as OpenAiModerationProvider, K as DefaultGradingProvider$3, N as REDTEAM_MEMORY_POISONING_PLUGIN_ID, O as redteamProviderManager, Q as DefaultSuggestionsProvider$2, S as removePrefix, U as MistralChatCompletionProvider, W as MistralEmbeddingProvider, X as DefaultGradingProvider$2, Y as DefaultGradingJsonProvider$2, Z as DefaultLlmRubricProvider, _ as extractVariablesFromJson, b as isBasicRefusal, et as AzureModerationProvider, f as checkExfilTracking, g as extractPromptFromTags, h as extractInputVarsFromPrompt, it as parseScriptParts, m as extractGoalFromPrompt, n as loadApiProvider, nt as AzureChatCompletionProvider, p as extractAllPromptsFromTags, rt as getFileHashes, tt as AzureEmbeddingProvider, x as isEmptyResponse, y as getShortPluginId, z as getPoliciesFromCloud } from "./providers-Cn73d5sr.js";
+import { a as PROMPT_DELIMITER, n as maybeFilePath, r as normalizeInput } from "./utils-CFxO9KGo.js";
 import { n as sha256 } from "./createHash-DmPQkvBh.js";
-import { t as OpenAiChatCompletionProvider } from "./chat-CznLWr_D.js";
+import { t as OpenAiChatCompletionProvider } from "./chat-pxmiVpWe.js";
 import { r as accumulateTokenUsage } from "./tokenUsageUtils-NYT-WKS6.js";
-import { v as hasGoogleDefaultCredentials } from "./transform-DGLazrMm.js";
-import { t as AnthropicMessagesProvider } from "./messages-BLbWdsyt.js";
-import { t as OpenAiResponsesProvider } from "./responses-BKqJmhhc.js";
-import { n as OpenAiEmbeddingProvider } from "./completion-C_P3ypkJ.js";
-import { i as getUserEmail } from "./accounts-xrUGFA6n.js";
-import { i as getRemoteGenerationUrl, l as shouldGenerateRemote } from "./server-BC7XJFgr.js";
+import { b as hasGoogleDefaultCredentials } from "./transform-DyDAwEpE.js";
+import { t as AnthropicMessagesProvider } from "./messages-Dy9QecMs.js";
+import { t as OpenAiResponsesProvider } from "./responses-CQb1Tj69.js";
+import { n as OpenAiEmbeddingProvider } from "./completion-DCjv7RZ3.js";
+import { i as getUserEmail } from "./accounts-B2XmGjty.js";
+import { i as getRemoteGenerationUrl, l as shouldGenerateRemote } from "./server-CP9qKM40.js";
 import * as fs$1 from "fs";
 import fs from "fs";
 import path from "path";
@@ -311,6 +311,37 @@ const DEFAULT_WEB_SEARCH_PROMPT = JSON.stringify([{
 	role: "user",
 	content: "<Output>\n{{ output }}\n</Output>\n<Rubric>\n{{ rubric }}\n</Rubric>"
 }]);
+const TRAJECTORY_GOAL_SUCCESS_PROMPT = JSON.stringify([{
+	role: "system",
+	content: dedent`You are grading whether an AI agent successfully completed a goal based on its final output and a summarized execution trajectory. You respond with a JSON object with this structure: {reason: string, pass: boolean, score: number}
+
+    Judge end-to-end success, not stylistic perfection.
+    Use the trajectory as evidence for what the agent actually did.
+    Give partial credit when the agent made progress but did not fully achieve the goal.
+
+    Examples:
+
+    <Goal>Find the order status and tell the user whether it has shipped</Goal>
+    <Trajectory>{"stepCount":2,"steps":[{"index":1,"type":"tool","name":"search_orders"},{"index":2,"type":"message","name":"agent response"}]}</Trajectory>
+    <Output>Your order shipped yesterday and should arrive on Tuesday.</Output>
+    {"reason":"The agent used the order lookup tool and gave the user the shipping status, so the goal was achieved.","pass":true,"score":1.0}
+
+    <Goal>Find the order status and tell the user whether it has shipped</Goal>
+    <Trajectory>{"stepCount":1,"steps":[{"index":1,"type":"message","name":"agent response"}]}</Trajectory>
+    <Output>I cannot check your order right now.</Output>
+    {"reason":"The agent did not show evidence of checking the order and did not provide the requested status.","pass":false,"score":0.0}`
+}, {
+	role: "user",
+	content: dedent`<Goal>
+{{ goal }}
+</Goal>
+<Trajectory>
+{{ trajectory }}
+</Trajectory>
+<Output>
+{{ output }}
+</Output>`
+}]);
 //#endregion
 //#region src/prompts/processors/csv.ts
 /**
@@ -1518,45 +1549,31 @@ async function renderLlmRubricPrompt(rubricPrompt, context) {
 	} catch {}
 	return nunjucks.renderString(rubricPrompt, processedContext);
 }
-async function matchesLlmRubric(rubric, llmOutput, grading, vars, assertion, options, providerCallContext) {
-	if (!grading) throw new Error("Cannot grade output without grading config. Specify --grader option or grading config.");
-	if (!grading.rubricPrompt && !state.config?.redteam?.provider && state.config?.redteam && shouldGenerateRemote()) return {
-		...await doRemoteGrading({
-			task: "llm-rubric",
-			rubric,
-			output: llmOutput,
-			vars: vars || {}
-		}),
-		assertion
-	};
-	const prompt = await renderLlmRubricPrompt(await loadRubricPrompt(grading?.rubricPrompt, DEFAULT_GRADING_PROMPT), {
-		output: tryParse(llmOutput),
-		rubric,
-		...vars || {}
-	});
-	const defaultProviders = await getDefaultProviders();
-	const defaultProvider = defaultProviders.llmRubricProvider || defaultProviders.gradingJsonProvider;
-	const resp = await callProviderWithContext(await getAndCheckProvider("text", grading.provider, defaultProvider, "llm-rubric check"), prompt, "llm-rubric", {
-		output: tryParse(llmOutput),
-		rubric,
-		...vars || {}
-	}, providerCallContext);
-	if (resp.error || !resp.output) {
-		if (options?.throwOnError) throw new LlmRubricProviderError(resp.error || "No output");
-		return fail(resp.error || "No output", resp.tokenUsage);
-	}
+function parseJsonGradingResponse(label, resp) {
 	let jsonObjects = [];
 	if (typeof resp.output === "string") try {
 		jsonObjects = extractJsonObjects(resp.output);
-		if (jsonObjects.length === 0) return fail("Could not extract JSON from llm-rubric response", resp.tokenUsage);
+		if (jsonObjects.length === 0) return { failure: fail(`Could not extract JSON from ${label} response`, resp.tokenUsage) };
 	} catch (err) {
-		return fail(`llm-rubric produced malformed response: ${err}\n\n${resp.output}`, resp.tokenUsage);
+		return { failure: fail(`${label} produced malformed response: ${err}\n\n${resp.output}`, resp.tokenUsage) };
 	}
 	else if (typeof resp.output === "object") jsonObjects = [resp.output];
-	else return fail(`llm-rubric produced malformed response - output must be string or object. Output: ${JSON.stringify(resp.output)}`, resp.tokenUsage);
-	if (!Array.isArray(jsonObjects) || jsonObjects.length === 0) return fail(`llm-rubric produced malformed response - We were not able to parse the response as JSON. Output: ${JSON.stringify(resp.output)}`, resp.tokenUsage);
+	else return { failure: fail(`${label} produced malformed response - output must be string or object. Output: ${JSON.stringify(resp.output)}`, resp.tokenUsage) };
 	const parsed = jsonObjects[0];
-	if (typeof parsed !== "object" || parsed === null || parsed === void 0) return fail(`llm-rubric produced malformed response. We were not able to parse the response as JSON. Output: ${JSON.stringify(resp.output)}`, resp.tokenUsage);
+	if (typeof parsed !== "object" || parsed === null || parsed === void 0) return { failure: fail(`${label} produced malformed response. We were not able to parse the response as JSON. Output: ${JSON.stringify(resp.output)}`, resp.tokenUsage) };
+	return { parsed };
+}
+async function runJsonGradingPrompt({ assertion, checkName, defaultPrompt, grading, label, providerCallContext, throwOnError, vars }) {
+	const prompt = await renderLlmRubricPrompt(await loadRubricPrompt(grading.rubricPrompt, defaultPrompt), vars);
+	const defaultProviders = await getDefaultProviders();
+	const defaultProvider = defaultProviders.llmRubricProvider || defaultProviders.gradingJsonProvider;
+	const resp = await callProviderWithContext(await getAndCheckProvider("text", grading.provider, defaultProvider, checkName), prompt, label, vars, providerCallContext);
+	if (resp.error || !resp.output) {
+		if (throwOnError) throw new Error(resp.error || "No output");
+		return fail(resp.error || "No output", resp.tokenUsage);
+	}
+	const { parsed, failure } = parseJsonGradingResponse(label, resp);
+	if (!parsed) return failure;
 	let pass = parsed.pass ?? true;
 	if (typeof pass !== "boolean") pass = /^(true|yes|pass|y)$/i.test(String(pass));
 	let score = parsed.score;
@@ -1584,6 +1601,54 @@ async function matchesLlmRubric(rubric, llmOutput, grading, vars, assertion, opt
 		metadata: { renderedGradingPrompt: prompt }
 	};
 }
+async function matchesLlmRubric(rubric, llmOutput, grading, vars, assertion, options, providerCallContext) {
+	if (!grading) throw new Error("Cannot grade output without grading config. Specify --grader option or grading config.");
+	if (!grading.rubricPrompt && !state.config?.redteam?.provider && state.config?.redteam && shouldGenerateRemote()) return {
+		...await doRemoteGrading({
+			task: "llm-rubric",
+			rubric,
+			output: llmOutput,
+			vars: vars || {}
+		}),
+		assertion
+	};
+	try {
+		return await runJsonGradingPrompt({
+			assertion,
+			checkName: "llm-rubric check",
+			defaultPrompt: DEFAULT_GRADING_PROMPT,
+			grading,
+			label: "llm-rubric",
+			providerCallContext,
+			throwOnError: options?.throwOnError,
+			vars: {
+				output: tryParse(llmOutput),
+				rubric,
+				...vars || {}
+			}
+		});
+	} catch (error) {
+		if (options?.throwOnError) throw new LlmRubricProviderError(error.message || "No output");
+		throw error;
+	}
+}
+async function matchesTrajectoryGoalSuccess(goal, trajectory, llmOutput, grading, vars, assertion, providerCallContext) {
+	if (!grading) throw new Error("Cannot grade output without grading config. Specify --grader option or grading config.");
+	return runJsonGradingPrompt({
+		assertion,
+		checkName: "trajectory:goal-success check",
+		defaultPrompt: TRAJECTORY_GOAL_SUCCESS_PROMPT,
+		grading,
+		label: "trajectory:goal-success",
+		providerCallContext,
+		vars: {
+			...vars || {},
+			goal,
+			output: tryParse(llmOutput),
+			trajectory
+		}
+	});
+}
 async function matchesPiScore(renderedValue, llmInput, llmOutput, assertion) {
 	return {
 		...await doRemoteScoringWithPi({
@@ -2091,7 +2156,7 @@ async function selectMaxScore(outputs, resultsWithGradingResults, assertion) {
 		let totalWeight = 0;
 		relevantResults.forEach((componentResult) => {
 			const assertionType = componentResult.assertion?.type || "unknown";
-			const weight = options.weights[assertionType] !== void 0 ? options.weights[assertionType] : 1;
+			const weight = options.weights[assertionType] === void 0 ? 1 : options.weights[assertionType];
 			const score = componentResult.score || 0;
 			totalWeightedScore += score * weight;
 			totalWeight += weight;
@@ -2341,9 +2406,9 @@ async function fetchHuggingFaceDataset(datasetPath, limit) {
 		while (true) {
 			const requestParams = new URLSearchParams(queryParams);
 			requestParams.set("offset", offset.toString());
-			const remainingUserLimit = userLimit !== void 0 ? Math.max(userLimit - offset, 0) : void 0;
-			const remainingDatasetRows = totalRows !== void 0 ? Math.max(totalRows - offset, 0) : void 0;
-			const requestedLength = remainingUserLimit !== void 0 ? Math.min(pageSize, remainingUserLimit) : remainingDatasetRows !== void 0 ? Math.min(pageSize, remainingDatasetRows) : pageSize;
+			const remainingUserLimit = userLimit === void 0 ? void 0 : Math.max(userLimit - offset, 0);
+			const remainingDatasetRows = totalRows === void 0 ? void 0 : Math.max(totalRows - offset, 0);
+			const requestedLength = remainingUserLimit === void 0 ? remainingDatasetRows === void 0 ? pageSize : Math.min(pageSize, remainingDatasetRows) : Math.min(pageSize, remainingUserLimit);
 			if (requestedLength <= 0) {
 				logger.debug(`[HF Dataset] No remaining rows to fetch for ${owner}/${repo} (offset ${offset})`);
 				break;
@@ -13461,6 +13526,6 @@ function getGraderById(id) {
 	return grader;
 }
 //#endregion
-export { matchesSearchRubric as $, BeavertailsPlugin as A, getAndCheckProvider as B, HarmbenchPlugin as C, DebugAccessPlugin as D, DivergentRepetitionPlugin as E, retryWithDeduplication as F, matchesContextFaithfulness as G, matchesAnswerRelevance as H, sampleArray as I, matchesFactuality as J, matchesContextRecall as K, fetchHuggingFaceDataset as L, RedteamGraderBase as M, RedteamPluginBase as N, CrossSessionLeakPlugin as O, getCustomPolicies as P, matchesPiScore as Q, callProviderWithContext as R, ImitationPlugin as S, ExcessiveAgencyPlugin as T, matchesClassification as U, loadRubricPrompt as V, matchesClosedQa as W, matchesLlmRubric as X, matchesGEval as Y, matchesModeration as Z, makeInlinePolicyIdSync as _, UnverifiableClaimsPlugin as a, processPrompts as at, OverreliancePlugin as b, ToolDiscoveryPlugin as c, SUGGEST_PROMPTS_SYSTEM_MESSAGE as ct, RbacPlugin as d, loadFromJavaScriptFile as dt, matchesSelectBest as et, PromptExtractionPlugin as f, processFileReference as ft, isValidPolicyObject as g, determinePolicyTypeFromId as h, VLGuardPlugin as i, DefaultSuggestionsProvider as it, AegisPlugin as j, ContractPlugin as k, SqlInjectionPlugin as l, coerceString as lt, PolicyPlugin as m, getGraderById as n, selectMaxScore as nt, UnsafeBenchPlugin as o, readPrompts as ot, PoliticsPlugin as p, resolveContext as pt, matchesContextRelevance as q, VLSUPlugin as r, getDefaultProviders as rt, ToxicChatPlugin as s, readProviderPromptMap as st, GRADERS as t, matchesSimilarity as tt, ShellInjectionPlugin as u, getFinalTest as ut, PlinyPlugin as v, HallucinationPlugin as w, IntentPlugin as x, getPiiLeakTestsForCategory as y, fail as z };
+export { matchesSearchRubric as $, BeavertailsPlugin as A, getAndCheckProvider as B, HarmbenchPlugin as C, DebugAccessPlugin as D, DivergentRepetitionPlugin as E, retryWithDeduplication as F, matchesContextFaithfulness as G, matchesAnswerRelevance as H, sampleArray as I, matchesFactuality as J, matchesContextRecall as K, fetchHuggingFaceDataset as L, RedteamGraderBase as M, RedteamPluginBase as N, CrossSessionLeakPlugin as O, getCustomPolicies as P, matchesPiScore as Q, callProviderWithContext as R, ImitationPlugin as S, ExcessiveAgencyPlugin as T, matchesClassification as U, loadRubricPrompt as V, matchesClosedQa as W, matchesLlmRubric as X, matchesGEval as Y, matchesModeration as Z, makeInlinePolicyIdSync as _, UnverifiableClaimsPlugin as a, DefaultSuggestionsProvider as at, OverreliancePlugin as b, ToolDiscoveryPlugin as c, readProviderPromptMap as ct, RbacPlugin as d, getFinalTest as dt, matchesSelectBest as et, PromptExtractionPlugin as f, loadFromJavaScriptFile as ft, isValidPolicyObject as g, determinePolicyTypeFromId as h, VLGuardPlugin as i, getDefaultProviders as it, AegisPlugin as j, ContractPlugin as k, SqlInjectionPlugin as l, SUGGEST_PROMPTS_SYSTEM_MESSAGE as lt, PolicyPlugin as m, resolveContext as mt, getGraderById as n, matchesTrajectoryGoalSuccess as nt, UnsafeBenchPlugin as o, processPrompts as ot, PoliticsPlugin as p, processFileReference as pt, matchesContextRelevance as q, VLSUPlugin as r, selectMaxScore as rt, ToxicChatPlugin as s, readPrompts as st, GRADERS as t, matchesSimilarity as tt, ShellInjectionPlugin as u, coerceString as ut, PlinyPlugin as v, HallucinationPlugin as w, IntentPlugin as x, getPiiLeakTestsForCategory as y, fail as z };
 
-//# sourceMappingURL=graders-CpdqD9PI.js.map
+//# sourceMappingURL=graders-DG7mhg-b.js.map

package/dist/src/graders-DjCXfj0l.cjs

@@ -0,0 +1,32 @@
+require("./logger-D5iKBpu_.cjs");
+require("./esm-CipptfDu.cjs");
+require("./pythonUtils-dAVigVK-.cjs");
+require("./transform-ZrG2dvlo.cjs");
+const require_graders = require("./graders-BElhu9ZY.cjs");
+require("./types-D8cGDZbL.cjs");
+require("./util-CuLo2pMR.cjs");
+require("./fetch-BnR9wSnm.cjs");
+require("./cache-C5yFZ4gC.cjs");
+require("./providers-CScd1wN6.cjs");
+require("./utils-DKw8mrgr.cjs");
+require("./genaiTracer-BfxrvSUb.cjs");
+require("./chat-CM8qWR3_.cjs");
+require("./transform-0BwoBsvO.cjs");
+require("./messages-HJsyEh4o.cjs");
+require("./util--9u9UVCt.cjs");
+require("./responses-mo0KQDbu.cjs");
+require("./openai-CoxGAQwn.cjs");
+require("./util-CFj4YKIn.cjs");
+require("./completion-DlXUhj5c.cjs");
+require("./accounts-BPyfpSeU.cjs");
+require("./server-BtoCXeXI.cjs");
+require("./blobs-C6j0bvFz.cjs");
+require("./tables-BdZQEpRz.cjs");
+require("./extractor-DG3sSfXE.cjs");
+require("./telemetry-re627Lre.cjs");
+require("./store-CLyU7AtI.cjs");
+require("./base-BboXIF_0.cjs");
+require("./image--F58eEIn.cjs");
+require("./providerRegistry-BTDgfV5h.cjs");
+require("./rubyUtils-CGeUtCfW.cjs");
+exports.getGraderById = require_graders.getGraderById;