promptfoo 0.121.2 → 0.121.3

package/dist/src/{extractor-D25qpmGX.js → extractor-C8XwivI9.js}

@@ -1,7 +1,7 @@
-import { _ as getEnvBool, i as logger } from "./logger-DLcq4dWf.js";
-import { o as cloudConfig } from "./fetch-HaqdX7U1.js";
-import { c as isLoggedIntoCloud } from "./accounts-B0pgC1oV.js";
-import { i as storeBlob, r as recordBlobReference } from "./blobs-BDbfYdrJ.js";
+import { a as logger, b as getEnvBool } from "./logger-BnkjG2jt.js";
+import { o as cloudConfig } from "./fetch-BiYv2BZc.js";
+import { c as isLoggedIntoCloud } from "./accounts-Xatc0RYb.js";
+import { i as storeBlob, r as recordBlobReference } from "./blobs-BUWmKWzo.js";
 //#region src/blobs/remoteUpload.ts
 function buildRemoteUrl() {
 	const baseUrl = cloudConfig.getApiHost();
@@ -25,7 +25,7 @@ async function uploadBlobRemote(buffer, mimeType, context) {
 	const apiKey = cloudConfig.getApiKey();
 	if (!url || !apiKey) return null;
 	try {
-		const { fetchWithProxy } = await import("./fetch-HaqdX7U1.js").then((n) => n.i);
+		const { fetchWithProxy } = await import("./fetch-BiYv2BZc.js").then((n) => n.i);
 		const response = await fetchWithProxy(url, {
 			method: "POST",
 			headers: {
@@ -371,4 +371,4 @@ async function recordExistingBlobReferences(value, context, location) {
 //#endregion
 export { isBlobStorageEnabled as n, shouldAttemptRemoteBlobUpload as r, extractAndStoreBinaryData as t };
 
-//# sourceMappingURL=extractor-D25qpmGX.js.map
+//# sourceMappingURL=extractor-C8XwivI9.js.map

package/dist/src/{extractor-DReVID0K.js → extractor-CAZ2G3Kh.js}

@@ -1,8 +1,8 @@
 #!/usr/bin/env node
-import { b as getEnvBool, o as logger } from "./logger-KkObSCzq.js";
-import { c as isLoggedIntoCloud } from "./accounts-Bm2D8Db9.js";
-import { a as cloudConfig } from "./cloud-C0dlstV_.js";
-import { i as storeBlob, r as recordBlobReference } from "./blobs-CMHN0Qcz.js";
+import { C as getEnvBool, s as logger } from "./logger-BcJBzSSA.js";
+import { c as isLoggedIntoCloud } from "./accounts-CFLK3mnD.js";
+import { a as cloudConfig } from "./cloud-z8KZpUoa.js";
+import { i as storeBlob, r as recordBlobReference } from "./blobs-B1JriOyi.js";
 //#region src/blobs/remoteUpload.ts
 function buildRemoteUrl() {
 	const baseUrl = cloudConfig.getApiHost();
@@ -26,7 +26,7 @@ async function uploadBlobRemote(buffer, mimeType, context) {
 	const apiKey = cloudConfig.getApiKey();
 	if (!url || !apiKey) return null;
 	try {
-		const { fetchWithProxy } = await import("./fetch-BxNb_Lp3.js");
+		const { fetchWithProxy } = await import("./fetch-UWU706qb.js");
 		const response = await fetchWithProxy(url, {
 			method: "POST",
 			headers: {
@@ -372,4 +372,4 @@ async function recordExistingBlobReferences(value, context, location) {
 //#endregion
 export { isBlobStorageEnabled as n, shouldAttemptRemoteBlobUpload as r, extractAndStoreBinaryData as t };
 
-//# sourceMappingURL=extractor-DReVID0K.js.map
+//# sourceMappingURL=extractor-CAZ2G3Kh.js.map

package/dist/src/{extractor-pYLLi3wS.cjs → extractor-DG3sSfXE.cjs}

@@ -1,7 +1,7 @@
-const require_logger = require("./logger-Cp1GPUjj.cjs");
-const require_fetch = require("./fetch-BPkYtG8K.cjs");
-const require_accounts = require("./accounts-gtkH-5KX.cjs");
-const require_blobs = require("./blobs-D23XLin-.cjs");
+const require_logger = require("./logger-D5iKBpu_.cjs");
+const require_fetch = require("./fetch-BnR9wSnm.cjs");
+const require_accounts = require("./accounts-BPyfpSeU.cjs");
+const require_blobs = require("./blobs-C6j0bvFz.cjs");
 //#region src/blobs/remoteUpload.ts
 function buildRemoteUrl() {
 	const baseUrl = require_fetch.cloudConfig.getApiHost();
@@ -25,7 +25,7 @@ async function uploadBlobRemote(buffer, mimeType, context) {
 	const apiKey = require_fetch.cloudConfig.getApiKey();
 	if (!url || !apiKey) return null;
 	try {
-		const { fetchWithProxy } = await Promise.resolve().then(() => require("./fetch-BPkYtG8K.cjs")).then((n) => n.fetch_exports);
+		const { fetchWithProxy } = await Promise.resolve().then(() => require("./fetch-BnR9wSnm.cjs")).then((n) => n.fetch_exports);
 		const response = await fetchWithProxy(url, {
 			method: "POST",
 			headers: {
@@ -388,4 +388,4 @@ Object.defineProperty(exports, "shouldAttemptRemoteBlobUpload", {
 	}
 });
 
-//# sourceMappingURL=extractor-pYLLi3wS.cjs.map
+//# sourceMappingURL=extractor-DG3sSfXE.cjs.map

package/dist/src/{extractor-C0EVHewb.js → extractor-D_wd8jxt.js}

@@ -1,7 +1,7 @@
-import { _ as getEnvBool, i as logger } from "./logger-CT3IKMKA.js";
-import { o as cloudConfig } from "./fetch-Cwxnd8zz.js";
-import { o as isLoggedIntoCloud } from "./accounts-CiBLOnA7.js";
-import { n as recordBlobReference, r as storeBlob } from "./blobs-CBO20krR.js";
+import { a as logger, b as getEnvBool } from "./logger-DO8_zM18.js";
+import { o as cloudConfig } from "./fetch-CVAtKnI3.js";
+import { o as isLoggedIntoCloud } from "./accounts-B2XmGjty.js";
+import { n as recordBlobReference, r as storeBlob } from "./blobs-DXTl6J3H.js";
 //#region src/blobs/remoteUpload.ts
 function buildRemoteUrl() {
 	const baseUrl = cloudConfig.getApiHost();
@@ -25,7 +25,7 @@ async function uploadBlobRemote(buffer, mimeType, context) {
 	const apiKey = cloudConfig.getApiKey();
 	if (!url || !apiKey) return null;
 	try {
-		const { fetchWithProxy } = await import("./fetch-Cwxnd8zz.js").then((n) => n.i);
+		const { fetchWithProxy } = await import("./fetch-CVAtKnI3.js").then((n) => n.i);
 		const response = await fetchWithProxy(url, {
 			method: "POST",
 			headers: {
@@ -371,4 +371,4 @@ async function recordExistingBlobReferences(value, context, location) {
 //#endregion
 export { isBlobStorageEnabled as n, shouldAttemptRemoteBlobUpload as r, extractAndStoreBinaryData as t };
 
-//# sourceMappingURL=extractor-C0EVHewb.js.map
+//# sourceMappingURL=extractor-D_wd8jxt.js.map

package/dist/src/{fetch-HaqdX7U1.js → fetch-BiYv2BZc.js}

@@ -1,5 +1,5 @@
 import { t as __exportAll } from "./chunk-DEq-mXcV.js";
-import { _ as getEnvBool, b as getEnvString, c as sanitizeUrl, g as getConfigDirectoryPath, i as logger, r as logRequestResponse, w as state, y as getEnvInt } from "./logger-DLcq4dWf.js";
+import { C as getEnvString, D as state, S as getEnvInt, a as logger, b as getEnvBool, d as sanitizeUrl, i as logRequestResponse, y as getConfigDirectoryPath } from "./logger-BnkjG2jt.js";
 import { t as invariant } from "./invariant-vgHWClmd.js";
 import * as fs$1 from "fs";
 import * as path$1 from "path";
@@ -23,7 +23,7 @@ const HUMAN_ASSERTION_TYPE = "human";
 * Application version from package.json.
 * Injected at build time, or read from npm environment in development.
 */
-const VERSION = "0.121.2";
+const VERSION = "0.121.3";
 /**
 * PostHog analytics key.
 * Only populated during production builds via PROMPTFOO_POSTHOG_KEY env var.
@@ -777,4 +777,4 @@ async function fetchWithRetries(url, options = {}, timeout, maxRetries) {
 //#endregion
 export { getDefaultShareViewBaseUrl as A, transformTools as C, R_ENDPOINT as D, EVENTS_ENDPOINT as E, FILE_METADATA_KEY as F, HUMAN_ASSERTION_TYPE as I, getShareViewBaseUrl as M, POSTHOG_KEY as N, TERMINAL_MAX_WIDTH as O, VERSION as P, transformToolChoice as S, CONSENT_ENDPOINT as T, isPromptfooSampleTarget as _, CloudConfig as a, parseChatPrompt as b, writeGlobalConfig as c, sleep as d, LONG_RUNNING_MODEL_TIMEOUT_MS as f, isOpenAIToolChoice as g, isOpenAIToolArray as h, fetch_exports as i, getShareApiBaseUrl as j, getDefaultPort as k, writeGlobalConfigPartial as l, calculateCost as m, fetchWithRetries as n, cloudConfig as o, REQUEST_TIMEOUT_MS as p, fetchWithTimeout as r, readGlobalConfig as s, fetchWithProxy as t, getCurrentTimestamp as u, openaiToolChoiceToBedrock as v, CLOUD_PROVIDER_PREFIX as w, toTitleCase as x, openaiToolsToBedrock as y };
 
-//# sourceMappingURL=fetch-HaqdX7U1.js.map
+//# sourceMappingURL=fetch-BiYv2BZc.js.map

package/dist/src/{fetch-BPkYtG8K.cjs → fetch-BnR9wSnm.cjs}

@@ -1,4 +1,4 @@
-const require_logger = require("./logger-Cp1GPUjj.cjs");
+const require_logger = require("./logger-D5iKBpu_.cjs");
 const require_invariant = require("./invariant-kfQ8Bu82.cjs");
 let fs = require("fs");
 fs = require_logger.__toESM(fs);
@@ -25,7 +25,7 @@ const HUMAN_ASSERTION_TYPE = "human";
 * Application version from package.json.
 * Injected at build time, or read from npm environment in development.
 */
-const VERSION = "0.121.2";
+const VERSION = "0.121.3";
 /**
 * PostHog analytics key.
 * Only populated during production builds via PROMPTFOO_POSTHOG_KEY env var.
@@ -985,4 +985,4 @@ Object.defineProperty(exports, "writeGlobalConfigPartial", {
 	}
 });
 
-//# sourceMappingURL=fetch-BPkYtG8K.cjs.map
+//# sourceMappingURL=fetch-BnR9wSnm.cjs.map

package/dist/src/{fetch-Cwxnd8zz.js → fetch-CVAtKnI3.js}

@@ -1,5 +1,5 @@
 import { t as __exportAll } from "./chunk-DEq-mXcV.js";
-import { _ as getEnvBool, b as getEnvString, c as sanitizeUrl, g as getConfigDirectoryPath, i as logger, r as logRequestResponse, w as state, y as getEnvInt } from "./logger-CT3IKMKA.js";
+import { C as getEnvString, D as state, S as getEnvInt, a as logger, b as getEnvBool, d as sanitizeUrl, i as logRequestResponse, y as getConfigDirectoryPath } from "./logger-DO8_zM18.js";
 import { t as invariant } from "./invariant-Ddh24eXh.js";
 import * as fs$1 from "fs";
 import * as path$1 from "path";
@@ -23,7 +23,7 @@ const HUMAN_ASSERTION_TYPE = "human";
 * Application version from package.json.
 * Injected at build time, or read from npm environment in development.
 */
-const VERSION = "0.121.2";
+const VERSION = "0.121.3";
 /**
 * PostHog analytics key.
 * Only populated during production builds via PROMPTFOO_POSTHOG_KEY env var.
@@ -774,4 +774,4 @@ async function fetchWithRetries(url, options = {}, timeout, maxRetries) {
 //#endregion
 export { getShareApiBaseUrl as A, transformTools as C, R_ENDPOINT as D, EVENTS_ENDPOINT as E, HUMAN_ASSERTION_TYPE as F, POSTHOG_KEY as M, VERSION as N, TERMINAL_MAX_WIDTH as O, FILE_METADATA_KEY as P, transformToolChoice as S, CONSENT_ENDPOINT as T, isPromptfooSampleTarget as _, CloudConfig as a, parseChatPrompt as b, writeGlobalConfig as c, sleep as d, LONG_RUNNING_MODEL_TIMEOUT_MS as f, isOpenAIToolChoice as g, isOpenAIToolArray as h, fetch_exports as i, getShareViewBaseUrl as j, getDefaultShareViewBaseUrl as k, writeGlobalConfigPartial as l, calculateCost as m, fetchWithRetries as n, cloudConfig as o, REQUEST_TIMEOUT_MS as p, fetchWithTimeout as r, readGlobalConfig as s, fetchWithProxy as t, getCurrentTimestamp as u, openaiToolChoiceToBedrock as v, CLOUD_PROVIDER_PREFIX as w, toTitleCase as x, openaiToolsToBedrock as y };
 
-//# sourceMappingURL=fetch-Cwxnd8zz.js.map
+//# sourceMappingURL=fetch-CVAtKnI3.js.map

package/dist/src/{fetch-Dxpd4_sr.js → fetch-DoVRJZhJ.js}

@@ -1,7 +1,7 @@
 #!/usr/bin/env node
-import { C as getEnvString, O as state, S as getEnvInt, a as logRequestResponse, b as getEnvBool, o as logger, u as sanitizeUrl } from "./logger-KkObSCzq.js";
+import { C as getEnvBool, E as getEnvString, T as getEnvInt, j as state, o as logRequestResponse, p as sanitizeUrl, s as logger } from "./logger-BcJBzSSA.js";
 import { t as invariant } from "./invariant-BtWWVVhl.js";
-import { a as cloudConfig } from "./cloud-C0dlstV_.js";
+import { a as cloudConfig } from "./cloud-z8KZpUoa.js";
 import { Agent, ProxyAgent } from "undici";
 import path from "path";
 import yaml from "js-yaml";
@@ -22,7 +22,7 @@ const HUMAN_ASSERTION_TYPE = "human";
 * Application version from package.json.
 * Injected at build time, or read from npm environment in development.
 */
-const VERSION = "0.121.2";
+const VERSION = "0.121.3";
 /**
 * PostHog analytics key.
 * Only populated during production builds via PROMPTFOO_POSTHOG_KEY env var.
@@ -595,4 +595,4 @@ async function fetchWithRetries(url, options = {}, timeout, maxRetries) {
 //#endregion
 export { getShareApiBaseUrl as A, CLOUD_PROVIDER_PREFIX as C, TERMINAL_MAX_WIDTH as D, R_ENDPOINT as E, HUMAN_ASSERTION_TYPE as F, POSTHOG_KEY as M, VERSION as N, getDefaultPort as O, FILE_METADATA_KEY as P, transformTools as S, EVENTS_ENDPOINT as T, openaiToolChoiceToBedrock as _, handleRateLimit as a, toTitleCase as b, getCurrentTimestamp as c, LONG_RUNNING_MODEL_TIMEOUT_MS as d, REQUEST_TIMEOUT_MS as f, isPromptfooSampleTarget as g, isOpenAIToolChoice as h, fetchWithTimeout as i, getShareViewBaseUrl as j, getDefaultShareViewBaseUrl as k, sleep as l, isOpenAIToolArray as m, fetchWithProxy as n, isRateLimited as o, calculateCost as p, fetchWithRetries as r, isTransientError as s, clearAgentCache as t, sleepWithAbort as u, openaiToolsToBedrock as v, CONSENT_ENDPOINT as w, transformToolChoice as x, parseChatPrompt as y };
 
-//# sourceMappingURL=fetch-Dxpd4_sr.js.map
+//# sourceMappingURL=fetch-DoVRJZhJ.js.map

package/dist/src/fetch-UWU706qb.js

@@ -0,0 +1,5 @@
+#!/usr/bin/env node
+import "./logger-BcJBzSSA.js";
+import { n as fetchWithProxy } from "./fetch-DoVRJZhJ.js";
+import "./cloud-z8KZpUoa.js";
+export { fetchWithProxy };

package/dist/src/{genaiTracer-DN4dQywX.cjs → genaiTracer-BfxrvSUb.cjs}

@@ -1,4 +1,4 @@
-require("./logger-Cp1GPUjj.cjs");
+require("./logger-D5iKBpu_.cjs");
 let _opentelemetry_api = require("@opentelemetry/api");
 //#region src/tracing/genaiTracer.ts
 const TRACER_NAME = "promptfoo.providers";
@@ -265,4 +265,4 @@ Object.defineProperty(exports, "withGenAISpan", {
 	}
 });
 
-//# sourceMappingURL=genaiTracer-DN4dQywX.cjs.map
+//# sourceMappingURL=genaiTracer-BfxrvSUb.cjs.map

package/dist/src/{graders-DU49_J8Y.cjs → graders-BElhu9ZY.cjs}

@@ -1,24 +1,24 @@
-const require_logger = require("./logger-Cp1GPUjj.cjs");
+const require_logger = require("./logger-D5iKBpu_.cjs");
 const require_invariant = require("./invariant-kfQ8Bu82.cjs");
-const require_esm = require("./esm-CnNt7sI4.cjs");
-const require_pythonUtils = require("./pythonUtils-CTU3Y3lw.cjs");
+const require_esm = require("./esm-CipptfDu.cjs");
+const require_pythonUtils = require("./pythonUtils-dAVigVK-.cjs");
 const require_fileExtensions = require("./fileExtensions-bYh77CN8.cjs");
-const require_transform = require("./transform-CzK1Q0zl.cjs");
-const require_types = require("./types-C_7nyzr1.cjs");
-const require_util = require("./util-B9vlHIIh.cjs");
-const require_fetch = require("./fetch-BPkYtG8K.cjs");
-const require_cache = require("./cache-Dh5WtQps.cjs");
-const require_providers = require("./providers-CgKOSgTR.cjs");
-const require_utils = require("./utils-Cz9qXqII.cjs");
-const require_createHash = require("./createHash-CfZSc0b4.cjs");
-const require_chat = require("./chat-CgF-J-Jj.cjs");
+const require_transform = require("./transform-ZrG2dvlo.cjs");
+const require_types = require("./types-D8cGDZbL.cjs");
+const require_util = require("./util-CuLo2pMR.cjs");
+const require_fetch = require("./fetch-BnR9wSnm.cjs");
+const require_cache = require("./cache-C5yFZ4gC.cjs");
+const require_providers = require("./providers-CScd1wN6.cjs");
+const require_utils = require("./utils-DKw8mrgr.cjs");
+const require_createHash = require("./createHash-BYwImsYv.cjs");
+const require_chat = require("./chat-CM8qWR3_.cjs");
 const require_tokenUsageUtils = require("./tokenUsageUtils-bVa1ga6f.cjs");
-const require_transform$1 = require("./transform-DuHvhZpj.cjs");
-const require_messages = require("./messages-Bs1kC7P4.cjs");
-const require_responses = require("./responses-D8SBTL64.cjs");
-const require_completion = require("./completion-CM6oK8PS.cjs");
-const require_accounts = require("./accounts-gtkH-5KX.cjs");
-const require_server = require("./server-C_7Ax-hA.cjs");
+const require_transform$1 = require("./transform-0BwoBsvO.cjs");
+const require_messages = require("./messages-HJsyEh4o.cjs");
+const require_responses = require("./responses-mo0KQDbu.cjs");
+const require_completion = require("./completion-DlXUhj5c.cjs");
+const require_accounts = require("./accounts-BPyfpSeU.cjs");
+const require_server = require("./server-BtoCXeXI.cjs");
 let fs = require("fs");
 fs = require_logger.__toESM(fs);
 let path = require("path");
@@ -317,6 +317,37 @@ const DEFAULT_WEB_SEARCH_PROMPT = JSON.stringify([{
 	role: "user",
 	content: "<Output>\n{{ output }}\n</Output>\n<Rubric>\n{{ rubric }}\n</Rubric>"
 }]);
+const TRAJECTORY_GOAL_SUCCESS_PROMPT = JSON.stringify([{
+	role: "system",
+	content: dedent.default`You are grading whether an AI agent successfully completed a goal based on its final output and a summarized execution trajectory. You respond with a JSON object with this structure: {reason: string, pass: boolean, score: number}
+
+    Judge end-to-end success, not stylistic perfection.
+    Use the trajectory as evidence for what the agent actually did.
+    Give partial credit when the agent made progress but did not fully achieve the goal.
+
+    Examples:
+
+    <Goal>Find the order status and tell the user whether it has shipped</Goal>
+    <Trajectory>{"stepCount":2,"steps":[{"index":1,"type":"tool","name":"search_orders"},{"index":2,"type":"message","name":"agent response"}]}</Trajectory>
+    <Output>Your order shipped yesterday and should arrive on Tuesday.</Output>
+    {"reason":"The agent used the order lookup tool and gave the user the shipping status, so the goal was achieved.","pass":true,"score":1.0}
+
+    <Goal>Find the order status and tell the user whether it has shipped</Goal>
+    <Trajectory>{"stepCount":1,"steps":[{"index":1,"type":"message","name":"agent response"}]}</Trajectory>
+    <Output>I cannot check your order right now.</Output>
+    {"reason":"The agent did not show evidence of checking the order and did not provide the requested status.","pass":false,"score":0.0}`
+}, {
+	role: "user",
+	content: dedent.default`<Goal>
+{{ goal }}
+</Goal>
+<Trajectory>
+{{ trajectory }}
+</Trajectory>
+<Output>
+{{ output }}
+</Output>`
+}]);
 //#endregion
 //#region src/prompts/processors/csv.ts
 /**
@@ -1524,45 +1555,31 @@ async function renderLlmRubricPrompt(rubricPrompt, context) {
 	} catch {}
 	return nunjucks.renderString(rubricPrompt, processedContext);
 }
-async function matchesLlmRubric(rubric, llmOutput, grading, vars, assertion, options, providerCallContext) {
-	if (!grading) throw new Error("Cannot grade output without grading config. Specify --grader option or grading config.");
-	if (!grading.rubricPrompt && !require_logger.state.config?.redteam?.provider && require_logger.state.config?.redteam && require_server.shouldGenerateRemote()) return {
-		...await doRemoteGrading({
-			task: "llm-rubric",
-			rubric,
-			output: llmOutput,
-			vars: vars || {}
-		}),
-		assertion
-	};
-	const prompt = await renderLlmRubricPrompt(await loadRubricPrompt(grading?.rubricPrompt, DEFAULT_GRADING_PROMPT), {
-		output: tryParse(llmOutput),
-		rubric,
-		...vars || {}
-	});
-	const defaultProviders = await getDefaultProviders();
-	const defaultProvider = defaultProviders.llmRubricProvider || defaultProviders.gradingJsonProvider;
-	const resp = await callProviderWithContext(await getAndCheckProvider("text", grading.provider, defaultProvider, "llm-rubric check"), prompt, "llm-rubric", {
-		output: tryParse(llmOutput),
-		rubric,
-		...vars || {}
-	}, providerCallContext);
-	if (resp.error || !resp.output) {
-		if (options?.throwOnError) throw new LlmRubricProviderError(resp.error || "No output");
-		return fail(resp.error || "No output", resp.tokenUsage);
-	}
+function parseJsonGradingResponse(label, resp) {
 	let jsonObjects = [];
 	if (typeof resp.output === "string") try {
 		jsonObjects = require_logger.extractJsonObjects(resp.output);
-		if (jsonObjects.length === 0) return fail("Could not extract JSON from llm-rubric response", resp.tokenUsage);
+		if (jsonObjects.length === 0) return { failure: fail(`Could not extract JSON from ${label} response`, resp.tokenUsage) };
 	} catch (err) {
-		return fail(`llm-rubric produced malformed response: ${err}\n\n${resp.output}`, resp.tokenUsage);
+		return { failure: fail(`${label} produced malformed response: ${err}\n\n${resp.output}`, resp.tokenUsage) };
 	}
 	else if (typeof resp.output === "object") jsonObjects = [resp.output];
-	else return fail(`llm-rubric produced malformed response - output must be string or object. Output: ${JSON.stringify(resp.output)}`, resp.tokenUsage);
-	if (!Array.isArray(jsonObjects) || jsonObjects.length === 0) return fail(`llm-rubric produced malformed response - We were not able to parse the response as JSON. Output: ${JSON.stringify(resp.output)}`, resp.tokenUsage);
+	else return { failure: fail(`${label} produced malformed response - output must be string or object. Output: ${JSON.stringify(resp.output)}`, resp.tokenUsage) };
 	const parsed = jsonObjects[0];
-	if (typeof parsed !== "object" || parsed === null || parsed === void 0) return fail(`llm-rubric produced malformed response. We were not able to parse the response as JSON. Output: ${JSON.stringify(resp.output)}`, resp.tokenUsage);
+	if (typeof parsed !== "object" || parsed === null || parsed === void 0) return { failure: fail(`${label} produced malformed response. We were not able to parse the response as JSON. Output: ${JSON.stringify(resp.output)}`, resp.tokenUsage) };
+	return { parsed };
+}
+async function runJsonGradingPrompt({ assertion, checkName, defaultPrompt, grading, label, providerCallContext, throwOnError, vars }) {
+	const prompt = await renderLlmRubricPrompt(await loadRubricPrompt(grading.rubricPrompt, defaultPrompt), vars);
+	const defaultProviders = await getDefaultProviders();
+	const defaultProvider = defaultProviders.llmRubricProvider || defaultProviders.gradingJsonProvider;
+	const resp = await callProviderWithContext(await getAndCheckProvider("text", grading.provider, defaultProvider, checkName), prompt, label, vars, providerCallContext);
+	if (resp.error || !resp.output) {
+		if (throwOnError) throw new Error(resp.error || "No output");
+		return fail(resp.error || "No output", resp.tokenUsage);
+	}
+	const { parsed, failure } = parseJsonGradingResponse(label, resp);
+	if (!parsed) return failure;
 	let pass = parsed.pass ?? true;
 	if (typeof pass !== "boolean") pass = /^(true|yes|pass|y)$/i.test(String(pass));
 	let score = parsed.score;
@@ -1590,6 +1607,54 @@ async function matchesLlmRubric(rubric, llmOutput, grading, vars, assertion, opt
 		metadata: { renderedGradingPrompt: prompt }
 	};
 }
+async function matchesLlmRubric(rubric, llmOutput, grading, vars, assertion, options, providerCallContext) {
+	if (!grading) throw new Error("Cannot grade output without grading config. Specify --grader option or grading config.");
+	if (!grading.rubricPrompt && !require_logger.state.config?.redteam?.provider && require_logger.state.config?.redteam && require_server.shouldGenerateRemote()) return {
+		...await doRemoteGrading({
+			task: "llm-rubric",
+			rubric,
+			output: llmOutput,
+			vars: vars || {}
+		}),
+		assertion
+	};
+	try {
+		return await runJsonGradingPrompt({
+			assertion,
+			checkName: "llm-rubric check",
+			defaultPrompt: DEFAULT_GRADING_PROMPT,
+			grading,
+			label: "llm-rubric",
+			providerCallContext,
+			throwOnError: options?.throwOnError,
+			vars: {
+				output: tryParse(llmOutput),
+				rubric,
+				...vars || {}
+			}
+		});
+	} catch (error) {
+		if (options?.throwOnError) throw new LlmRubricProviderError(error.message || "No output");
+		throw error;
+	}
+}
+async function matchesTrajectoryGoalSuccess(goal, trajectory, llmOutput, grading, vars, assertion, providerCallContext) {
+	if (!grading) throw new Error("Cannot grade output without grading config. Specify --grader option or grading config.");
+	return runJsonGradingPrompt({
+		assertion,
+		checkName: "trajectory:goal-success check",
+		defaultPrompt: TRAJECTORY_GOAL_SUCCESS_PROMPT,
+		grading,
+		label: "trajectory:goal-success",
+		providerCallContext,
+		vars: {
+			...vars || {},
+			goal,
+			output: tryParse(llmOutput),
+			trajectory
+		}
+	});
+}
 async function matchesPiScore(renderedValue, llmInput, llmOutput, assertion) {
 	return {
 		...await doRemoteScoringWithPi({
@@ -2097,7 +2162,7 @@ async function selectMaxScore(outputs, resultsWithGradingResults, assertion) {
 		let totalWeight = 0;
 		relevantResults.forEach((componentResult) => {
 			const assertionType = componentResult.assertion?.type || "unknown";
-			const weight = options.weights[assertionType] !== void 0 ? options.weights[assertionType] : 1;
+			const weight = options.weights[assertionType] === void 0 ? 1 : options.weights[assertionType];
 			const score = componentResult.score || 0;
 			totalWeightedScore += score * weight;
 			totalWeight += weight;
@@ -2347,9 +2412,9 @@ async function fetchHuggingFaceDataset(datasetPath, limit) {
 		while (true) {
 			const requestParams = new URLSearchParams(queryParams);
 			requestParams.set("offset", offset.toString());
-			const remainingUserLimit = userLimit !== void 0 ? Math.max(userLimit - offset, 0) : void 0;
-			const remainingDatasetRows = totalRows !== void 0 ? Math.max(totalRows - offset, 0) : void 0;
-			const requestedLength = remainingUserLimit !== void 0 ? Math.min(pageSize, remainingUserLimit) : remainingDatasetRows !== void 0 ? Math.min(pageSize, remainingDatasetRows) : pageSize;
+			const remainingUserLimit = userLimit === void 0 ? void 0 : Math.max(userLimit - offset, 0);
+			const remainingDatasetRows = totalRows === void 0 ? void 0 : Math.max(totalRows - offset, 0);
+			const requestedLength = remainingUserLimit === void 0 ? remainingDatasetRows === void 0 ? pageSize : Math.min(pageSize, remainingDatasetRows) : Math.min(pageSize, remainingUserLimit);
 			if (requestedLength <= 0) {
 				require_logger.logger.debug(`[HF Dataset] No remaining rows to fetch for ${owner}/${repo} (offset ${offset})`);
 				break;
@@ -13821,6 +13886,12 @@ Object.defineProperty(exports, "matchesSimilarity", {
 		return matchesSimilarity;
 	}
 });
+Object.defineProperty(exports, "matchesTrajectoryGoalSuccess", {
+	enumerable: true,
+	get: function() {
+		return matchesTrajectoryGoalSuccess;
+	}
+});
 Object.defineProperty(exports, "processFileReference", {
 	enumerable: true,
 	get: function() {
@@ -13870,4 +13941,4 @@ Object.defineProperty(exports, "selectMaxScore", {
 	}
 });
 
-//# sourceMappingURL=graders-DU49_J8Y.cjs.map
+//# sourceMappingURL=graders-BElhu9ZY.cjs.map