promptfoo 0.121.7 → 0.121.9

package/dist/src/{goat-Ckd3q3AY.js → goat-0HwO0EB2.js}

@@ -1,21 +1,21 @@
 #!/usr/bin/env node
-import { s as logger, x as safeJsonStringify } from "./logger-BbY6ypFL.js";
-import { f as sleep, n as fetchWithProxy } from "./fetch-DXUnXkVU.js";
-import { n as VERSION } from "./version-eRkNuGv8.js";
+import { s as logger, x as safeJsonStringify } from "./logger-h-eiS4iv.js";
+import { f as sleep, n as fetchWithProxy } from "./fetch-C59QM4cf.js";
+import { n as VERSION } from "./version-CnYqmWbz.js";
 import { t as invariant } from "./invariant-B2Rf6avk.js";
-import { c as isLoggedIntoCloud, o as getUserEmail } from "./accounts-CjFnOPmb.js";
-import { o as getNunjucksEngine } from "./render-CSP99NLm.js";
-import { r as getRemoteGenerationUrl, s as neverGenerateRemote } from "./remoteGeneration-B1_XsKXU.js";
+import { c as isLoggedIntoCloud, o as getUserEmail } from "./accounts-Bw2qPqE-.js";
+import { o as getNunjucksEngine } from "./render-jE6RKVgk.js";
+import { r as getRemoteGenerationUrl, s as neverGenerateRemote } from "./remoteGeneration-DW6mPZnO.js";
 import { a as createEmptyTokenUsage, n as accumulateResponseTokenUsage } from "./tokenUsageUtils-CDet74yk.js";
-import { r as materializeInputVariablesWithMetadata } from "./inputVariables-DXFdi7AI.js";
-import { a as extractPromptFromTags, o as getSessionId, r as extractInputVarsFromPrompt } from "./util-kDURhgJW.js";
-import { o as getGraderAssertionValue, p as tryUnblocking, s as getLastMessageContent, t as buildGraderResultAssertion } from "./shared-9WHQ1oNE.js";
-import { i as throwIfTargetPromptExceedsMaxChars } from "./promptLength-4X-Wd8PG.js";
-import { n as checkExfilTracking } from "./indirectWebPwn-uyWdHx04.js";
-import { n as renderPrompt } from "./evaluatorHelpers-BsYP_muT.js";
-import { t as Strategies } from "./strategies-DRJjGTIY.js";
-import { a as fetchTraceContext, i as extractTraceIdFromTraceparent, n as formatTraceForMetadata, r as formatTraceSummary, t as resolveTracingOptions } from "./tracingOptions-BnwKCkSB.js";
-import { t as applyRuntimeTransforms } from "./runtimeTransform-Dgh_D7DU.js";
+import { a as extractPromptFromTags, o as getSessionId, r as extractInputVarsFromPrompt } from "./util-D9grslJP.js";
+import { a as assertRemoteMaterializationHandled, c as isRemoteMaterializationUpgradeError, s as buildRemoteMaterializedInputVariables } from "./promptfoo-wxe28-q9.js";
+import { o as getGraderAssertionValue, p as tryUnblocking, s as getLastMessageContent, t as buildGraderResultAssertion } from "./shared-DsbObxvy.js";
+import { i as throwIfTargetPromptExceedsMaxChars } from "./promptLength-PkjlYFx0.js";
+import { n as checkExfilTracking } from "./indirectWebPwn-BXUoJl4I.js";
+import { n as renderPrompt } from "./evaluatorHelpers-DDG44JeI.js";
+import { t as Strategies } from "./strategies-Ptz_P4P4.js";
+import { a as fetchTraceContext, i as extractTraceIdFromTraceparent, n as formatTraceForMetadata, r as formatTraceSummary, t as resolveTracingOptions } from "./tracingOptions-D2F0D7es.js";
+import { t as applyRuntimeTransforms } from "./runtimeTransform-s14J7yxm.js";
 import { i as getGoalRubric } from "./prompts-C-bqE1Yp.js";
 import chalk from "chalk";
 import dedent from "dedent";
@@ -81,7 +81,7 @@ var GoatProvider = class {
 		let assertToUse;
 		let graderPassed;
 		let storedGraderResult;
-		const { getGraderById } = await import("./graders-BDT7dif6.js");
+		const { getGraderById } = await import("./graders-D4vc1PFi.js");
 		let test;
 		if (context?.test) {
 			test = context?.test;
@@ -192,11 +192,11 @@ var GoatProvider = class {
 			let processedMessage = attackerMessage.content;
 			const extractedPrompt = extractPromptFromTags(attackerMessage.content);
 			if (extractedPrompt) processedMessage = extractedPrompt;
+			if (this.config.inputs) assertRemoteMaterializationHandled(data, "GOAT multi-input generation");
 			const currentInputVars = extractInputVarsFromPrompt(processedMessage, this.config.inputs);
-			const currentRenderInputVars = (currentInputVars && this.config.inputs ? await materializeInputVariablesWithMetadata(currentInputVars, this.config.inputs, {
-				materializationIndex: turn,
-				pluginId: "goat"
-			}) : void 0)?.vars ?? currentInputVars;
+			let materializedInputVars;
+			if (currentInputVars && this.config.inputs) materializedInputVars = buildRemoteMaterializedInputVariables(data, currentInputVars, this.config.inputs);
+			const currentRenderInputVars = materializedInputVars?.vars ?? currentInputVars;
 			if (currentInputVars && this.config.inputs) try {
 				const parsed = JSON.parse(processedMessage);
 				if (typeof parsed.prompt === "string") processedMessage = parsed.prompt;
@@ -438,6 +438,7 @@ var GoatProvider = class {
 				logger.debug("[GOAT] Operation aborted");
 				throw error;
 			}
+			if (isRemoteMaterializationUpgradeError(error)) throw error;
 			logger.error(`[GOAT] An error occurred in GOAT turn ${turn}.  The test will continue to the next turn in the conversation.`, { error: error.message || error });
 		}
 		const finalPrompt = getLastMessageContent(messages, "user") || "";
@@ -464,4 +465,4 @@ var GoatProvider = class {
 //#endregion
 export { GoatProvider as default };
 
-//# sourceMappingURL=goat-Ckd3q3AY.js.map
+//# sourceMappingURL=goat-0HwO0EB2.js.map

package/dist/src/{goat-una6pZGP.cjs → goat-7wQgRcT4.cjs}

@@ -1,21 +1,21 @@
 const require_rolldown_runtime = require("./rolldown-runtime-D_mwlA32.cjs");
-const require_logger = require("./logger-cfNpzI4o.cjs");
+const require_logger = require("./logger-Cm0Nb8YX.cjs");
 const require_invariant = require("./invariant-QtnLD03y.cjs");
-const require_fetch = require("./fetch-Dw4XZHjj.cjs");
-const require_version = require("./version-BWCSaByA.cjs");
-const require_accounts = require("./accounts-CmWzeD2d.cjs");
-const require_render = require("./render-BNTrbmBw.cjs");
-const require_remoteGeneration = require("./remoteGeneration-DS9N3pgB.cjs");
+const require_fetch = require("./fetch-B2HvaVUc.cjs");
+const require_version = require("./version-XjGPawtU.cjs");
+const require_accounts = require("./accounts-DAcQXPwE.cjs");
+const require_render = require("./render-DJ4yipCD.cjs");
+const require_remoteGeneration = require("./remoteGeneration-CD3c_rzc.cjs");
 const require_tokenUsageUtils = require("./tokenUsageUtils-_B-P8IAi.cjs");
-const require_inputVariables = require("./inputVariables-Dq9W-Z3a.cjs");
-const require_util = require("./util-CN8om2rz.cjs");
-const require_shared = require("./shared-WkgnDkcg.cjs");
-const require_promptLength = require("./promptLength-BbBbDHNj.cjs");
-const require_indirectWebPwn = require("./indirectWebPwn-BJ22AbQa.cjs");
-const require_evaluatorHelpers = require("./evaluatorHelpers-CRqTvSux.cjs");
-const require_strategies = require("./strategies-7CS3Alao.cjs");
-const require_tracingOptions = require("./tracingOptions-DrbSFaKy.cjs");
-const require_runtimeTransform = require("./runtimeTransform-ON3YYILw.cjs");
+const require_util = require("./util-3BRss2KA.cjs");
+const require_promptfoo = require("./promptfoo-DAXBmrE6.cjs");
+const require_shared = require("./shared-DlmtkptR.cjs");
+const require_promptLength = require("./promptLength-CAcg7BmO.cjs");
+const require_indirectWebPwn = require("./indirectWebPwn-04ziKbly.cjs");
+const require_evaluatorHelpers = require("./evaluatorHelpers-2gpZ9Q_f.cjs");
+const require_strategies = require("./strategies-BD7US9WB.cjs");
+const require_tracingOptions = require("./tracingOptions-eqmq5sA2.cjs");
+const require_runtimeTransform = require("./runtimeTransform-DJEknTZQ.cjs");
 const require_prompts = require("./prompts-hNvWBD3z.cjs");
 let chalk = require("chalk");
 chalk = require_rolldown_runtime.__toESM(chalk, 1);
@@ -83,7 +83,7 @@ var GoatProvider = class {
 		let assertToUse;
 		let graderPassed;
 		let storedGraderResult;
-		const { getGraderById } = await Promise.resolve().then(() => require("./graders-ClrU2fnd.cjs")).then((n) => n.graders_exports);
+		const { getGraderById } = await Promise.resolve().then(() => require("./graders-CQv1GT5Q.cjs")).then((n) => n.graders_exports);
 		let test;
 		if (context?.test) {
 			test = context?.test;
@@ -194,11 +194,11 @@ var GoatProvider = class {
 			let processedMessage = attackerMessage.content;
 			const extractedPrompt = require_util.extractPromptFromTags(attackerMessage.content);
 			if (extractedPrompt) processedMessage = extractedPrompt;
+			if (this.config.inputs) require_promptfoo.assertRemoteMaterializationHandled(data, "GOAT multi-input generation");
 			const currentInputVars = require_util.extractInputVarsFromPrompt(processedMessage, this.config.inputs);
-			const currentRenderInputVars = (currentInputVars && this.config.inputs ? await require_inputVariables.materializeInputVariablesWithMetadata(currentInputVars, this.config.inputs, {
-				materializationIndex: turn,
-				pluginId: "goat"
-			}) : void 0)?.vars ?? currentInputVars;
+			let materializedInputVars;
+			if (currentInputVars && this.config.inputs) materializedInputVars = require_promptfoo.buildRemoteMaterializedInputVariables(data, currentInputVars, this.config.inputs);
+			const currentRenderInputVars = materializedInputVars?.vars ?? currentInputVars;
 			if (currentInputVars && this.config.inputs) try {
 				const parsed = JSON.parse(processedMessage);
 				if (typeof parsed.prompt === "string") processedMessage = parsed.prompt;
@@ -440,6 +440,7 @@ var GoatProvider = class {
 				require_logger.logger.debug("[GOAT] Operation aborted");
 				throw error;
 			}
+			if (require_promptfoo.isRemoteMaterializationUpgradeError(error)) throw error;
 			require_logger.logger.error(`[GOAT] An error occurred in GOAT turn ${turn}.  The test will continue to the next turn in the conversation.`, { error: error.message || error });
 		}
 		const finalPrompt = require_shared.getLastMessageContent(messages, "user") || "";
@@ -466,4 +467,4 @@ var GoatProvider = class {
 //#endregion
 exports.default = GoatProvider;
 
-//# sourceMappingURL=goat-una6pZGP.cjs.map
+//# sourceMappingURL=goat-7wQgRcT4.cjs.map

package/dist/src/{goat-ghadEDdy.js → goat-BNX0lOFx.js}

@@ -1,19 +1,19 @@
-import { a as logger, y as safeJsonStringify } from "./logger-KD8JjCRJ.js";
-import { m as sleep, t as fetchWithProxy } from "./fetch-BufrQtvR.js";
-import { n as VERSION } from "./version-BK20a4sw.js";
+import { a as logger, y as safeJsonStringify } from "./logger-BjYLHvGm.js";
+import { m as sleep, t as fetchWithProxy } from "./fetch-CBe8pU8Q.js";
+import { n as VERSION } from "./version-BGiEZvX0.js";
 import { t as invariant } from "./invariant-DIYf9sP1.js";
-import { c as isLoggedIntoCloud, o as getUserEmail } from "./accounts-DanM1wq_.js";
-import { o as getNunjucksEngine } from "./render-_6ur1fhE.js";
-import { r as getRemoteGenerationUrl, s as neverGenerateRemote } from "./remoteGeneration-COpWcmWd.js";
+import { c as isLoggedIntoCloud, o as getUserEmail } from "./accounts-BAajbKSh.js";
+import { o as getNunjucksEngine } from "./render-BoxrvB84.js";
+import { r as getRemoteGenerationUrl, s as neverGenerateRemote } from "./remoteGeneration-CoxtxLvB.js";
 import { a as createEmptyTokenUsage, n as accumulateResponseTokenUsage } from "./tokenUsageUtils-BjVkdk18.js";
-import { r as materializeInputVariablesWithMetadata } from "./inputVariables-B0qUChbV.js";
-import { a as extractPromptFromTags, o as getSessionId, r as extractInputVarsFromPrompt } from "./util-Df8YMvS1.js";
-import { o as getGraderAssertionValue, p as tryUnblocking, s as getLastMessageContent, t as buildGraderResultAssertion } from "./shared-7pmVZLNO.js";
-import { i as throwIfTargetPromptExceedsMaxChars } from "./promptLength-0qIHyhA5.js";
-import { n as checkExfilTracking } from "./indirectWebPwn-tNx9OZ35.js";
+import { a as extractPromptFromTags, o as getSessionId, r as extractInputVarsFromPrompt } from "./util-CKZReNsz.js";
+import { a as assertRemoteMaterializationHandled, c as isRemoteMaterializationUpgradeError, s as buildRemoteMaterializedInputVariables } from "./promptfoo-DVKqidd8.js";
+import { o as getGraderAssertionValue, p as tryUnblocking, s as getLastMessageContent, t as buildGraderResultAssertion } from "./shared-yBQATkD5.js";
+import { i as throwIfTargetPromptExceedsMaxChars } from "./promptLength-5B4_P93z.js";
+import { n as checkExfilTracking } from "./indirectWebPwn-DRYpRTx6.js";
 import { n as renderPrompt, t as Strategies } from "./server/index.js";
-import { a as fetchTraceContext, i as extractTraceIdFromTraceparent, n as formatTraceForMetadata, r as formatTraceSummary, t as resolveTracingOptions } from "./tracingOptions-Chi74lOD.js";
-import { t as applyRuntimeTransforms } from "./runtimeTransform-BJOpL9Yc.js";
+import { a as fetchTraceContext, i as extractTraceIdFromTraceparent, n as formatTraceForMetadata, r as formatTraceSummary, t as resolveTracingOptions } from "./tracingOptions-PLePpCmo.js";
+import { t as applyRuntimeTransforms } from "./runtimeTransform-BmhEztlx.js";
 import { i as getGoalRubric } from "./prompts-BYMtqPCw.js";
 import chalk from "chalk";
 import dedent from "dedent";
@@ -79,7 +79,7 @@ var GoatProvider = class {
 		let assertToUse;
 		let graderPassed;
 		let storedGraderResult;
-		const { getGraderById } = await import("./graders-C0nXU_ZP.js").then((n) => n.r);
+		const { getGraderById } = await import("./graders-k8qmhrPu.js").then((n) => n.r);
 		let test;
 		if (context?.test) {
 			test = context?.test;
@@ -190,11 +190,11 @@ var GoatProvider = class {
 			let processedMessage = attackerMessage.content;
 			const extractedPrompt = extractPromptFromTags(attackerMessage.content);
 			if (extractedPrompt) processedMessage = extractedPrompt;
+			if (this.config.inputs) assertRemoteMaterializationHandled(data, "GOAT multi-input generation");
 			const currentInputVars = extractInputVarsFromPrompt(processedMessage, this.config.inputs);
-			const currentRenderInputVars = (currentInputVars && this.config.inputs ? await materializeInputVariablesWithMetadata(currentInputVars, this.config.inputs, {
-				materializationIndex: turn,
-				pluginId: "goat"
-			}) : void 0)?.vars ?? currentInputVars;
+			let materializedInputVars;
+			if (currentInputVars && this.config.inputs) materializedInputVars = buildRemoteMaterializedInputVariables(data, currentInputVars, this.config.inputs);
+			const currentRenderInputVars = materializedInputVars?.vars ?? currentInputVars;
 			if (currentInputVars && this.config.inputs) try {
 				const parsed = JSON.parse(processedMessage);
 				if (typeof parsed.prompt === "string") processedMessage = parsed.prompt;
@@ -436,6 +436,7 @@ var GoatProvider = class {
 				logger.debug("[GOAT] Operation aborted");
 				throw error;
 			}
+			if (isRemoteMaterializationUpgradeError(error)) throw error;
 			logger.error(`[GOAT] An error occurred in GOAT turn ${turn}.  The test will continue to the next turn in the conversation.`, { error: error.message || error });
 		}
 		const finalPrompt = getLastMessageContent(messages, "user") || "";
@@ -462,4 +463,4 @@ var GoatProvider = class {
 //#endregion
 export { GoatProvider as default };
 
-//# sourceMappingURL=goat-ghadEDdy.js.map
+//# sourceMappingURL=goat-BNX0lOFx.js.map

package/dist/src/{goat-Qgurm-NP.js → goat-BOVnOl5S.js}

@@ -1,20 +1,20 @@
-import { a as logger, y as safeJsonStringify } from "./logger-Ct2S6Yx-.js";
+import { a as logger, y as safeJsonStringify } from "./logger-Cs0I2v_y.js";
 import { t as invariant } from "./invariant-Ddh24eXh.js";
-import { l as sleep, t as fetchWithProxy } from "./fetch-It34O8Ur.js";
-import { n as VERSION } from "./version-lpHV_53E.js";
-import { i as getUserEmail, o as isLoggedIntoCloud } from "./accounts-Ca7WIoPY.js";
-import { o as getNunjucksEngine } from "./render-DFfDeYUK.js";
-import { r as getRemoteGenerationUrl, s as neverGenerateRemote } from "./remoteGeneration-DsaSwmG2.js";
+import { l as sleep, t as fetchWithProxy } from "./fetch-CHJu2KON.js";
+import { n as VERSION } from "./version-BNhpIb2U.js";
+import { i as getUserEmail, o as isLoggedIntoCloud } from "./accounts-4Xa5fcHy.js";
+import { o as getNunjucksEngine } from "./render-CTZQAINJ.js";
+import { r as getRemoteGenerationUrl, s as neverGenerateRemote } from "./remoteGeneration-DvWEm3yi.js";
 import { a as createEmptyTokenUsage, n as accumulateResponseTokenUsage } from "./tokenUsageUtils-CmnQ0G2m.js";
-import { n as materializeInputVariablesWithMetadata } from "./inputVariables-DUGMb9Ka.js";
-import { a as extractPromptFromTags, o as getSessionId, r as extractInputVarsFromPrompt } from "./util-DNl96nNs.js";
-import { o as getGraderAssertionValue, p as tryUnblocking, s as getLastMessageContent, t as buildGraderResultAssertion } from "./shared-D6IjElRI.js";
-import { i as throwIfTargetPromptExceedsMaxChars } from "./promptLength-B9nZEfO6.js";
-import { n as checkExfilTracking } from "./indirectWebPwn-CbjUG0rh.js";
-import { n as renderPrompt } from "./evaluatorHelpers-DuqFFfq7.js";
-import { t as Strategies } from "./strategies-CiSeroPH.js";
-import { a as fetchTraceContext, i as extractTraceIdFromTraceparent, n as formatTraceForMetadata, r as formatTraceSummary, t as resolveTracingOptions } from "./tracingOptions-ji2OuXbT.js";
-import { t as applyRuntimeTransforms } from "./runtimeTransform-DigbjU1r.js";
+import { a as extractPromptFromTags, o as getSessionId, r as extractInputVarsFromPrompt } from "./util-D1NKMz55.js";
+import { a as assertRemoteMaterializationHandled, c as isRemoteMaterializationUpgradeError, s as buildRemoteMaterializedInputVariables } from "./promptfoo-CEsoPlBa.js";
+import { o as getGraderAssertionValue, p as tryUnblocking, s as getLastMessageContent, t as buildGraderResultAssertion } from "./shared-BF25a7ms.js";
+import { i as throwIfTargetPromptExceedsMaxChars } from "./promptLength-ho3wbst6.js";
+import { n as checkExfilTracking } from "./indirectWebPwn-1gHqaL91.js";
+import { n as renderPrompt } from "./evaluatorHelpers-C8C8WQTv.js";
+import { t as Strategies } from "./strategies-4Nnr_9xt.js";
+import { a as fetchTraceContext, i as extractTraceIdFromTraceparent, n as formatTraceForMetadata, r as formatTraceSummary, t as resolveTracingOptions } from "./tracingOptions-WpE9IqC8.js";
+import { t as applyRuntimeTransforms } from "./runtimeTransform-Bw9Tttpp.js";
 import { i as getGoalRubric } from "./prompts-DHhQsANy.js";
 import chalk from "chalk";
 import dedent from "dedent";
@@ -80,7 +80,7 @@ var GoatProvider = class {
 		let assertToUse;
 		let graderPassed;
 		let storedGraderResult;
-		const { getGraderById } = await import("./graders-BX0f2tvS.js").then((n) => n.r);
+		const { getGraderById } = await import("./graders-GNRQ1357.js").then((n) => n.r);
 		let test;
 		if (context?.test) {
 			test = context?.test;
@@ -191,11 +191,11 @@ var GoatProvider = class {
 			let processedMessage = attackerMessage.content;
 			const extractedPrompt = extractPromptFromTags(attackerMessage.content);
 			if (extractedPrompt) processedMessage = extractedPrompt;
+			if (this.config.inputs) assertRemoteMaterializationHandled(data, "GOAT multi-input generation");
 			const currentInputVars = extractInputVarsFromPrompt(processedMessage, this.config.inputs);
-			const currentRenderInputVars = (currentInputVars && this.config.inputs ? await materializeInputVariablesWithMetadata(currentInputVars, this.config.inputs, {
-				materializationIndex: turn,
-				pluginId: "goat"
-			}) : void 0)?.vars ?? currentInputVars;
+			let materializedInputVars;
+			if (currentInputVars && this.config.inputs) materializedInputVars = buildRemoteMaterializedInputVariables(data, currentInputVars, this.config.inputs);
+			const currentRenderInputVars = materializedInputVars?.vars ?? currentInputVars;
 			if (currentInputVars && this.config.inputs) try {
 				const parsed = JSON.parse(processedMessage);
 				if (typeof parsed.prompt === "string") processedMessage = parsed.prompt;
@@ -437,6 +437,7 @@ var GoatProvider = class {
 				logger.debug("[GOAT] Operation aborted");
 				throw error;
 			}
+			if (isRemoteMaterializationUpgradeError(error)) throw error;
 			logger.error(`[GOAT] An error occurred in GOAT turn ${turn}.  The test will continue to the next turn in the conversation.`, { error: error.message || error });
 		}
 		const finalPrompt = getLastMessageContent(messages, "user") || "";
@@ -463,4 +464,4 @@ var GoatProvider = class {
 //#endregion
 export { GoatProvider as default };
 
-//# sourceMappingURL=goat-Qgurm-NP.js.map
+//# sourceMappingURL=goat-BOVnOl5S.js.map

package/dist/src/{graders-BGP99PdK.js → graders-BnO-xoa-.js}

@@ -1,32 +1,32 @@
 #!/usr/bin/env node
-import { N as state, O as getEnvString, T as getEnvBool, _ as extractJsonObjects, g as extractFirstJsonObject, j as isCI, s as logger, x as safeJsonStringify } from "./logger-BbY6ypFL.js";
-import { _ as REQUEST_TIMEOUT_MS, f as sleep, i as fetchWithTimeout, n as fetchWithProxy } from "./fetch-DXUnXkVU.js";
+import { N as state, O as getEnvString, T as getEnvBool, _ as extractJsonObjects, g as extractFirstJsonObject, j as isCI, s as logger, x as safeJsonStringify } from "./logger-h-eiS4iv.js";
+import { f as sleep, i as fetchWithTimeout, n as fetchWithProxy, v as getRequestTimeoutMs } from "./fetch-C59QM4cf.js";
 import { t as invariant } from "./invariant-B2Rf6avk.js";
-import { o as getUserEmail } from "./accounts-CjFnOPmb.js";
-import { C as PolicyObjectSchema, M as MULTI_TURN_STRATEGIES, St as buildInputPromptDescription, T as isValidReusablePolicyId, _ as PromptSchema, _t as CODING_AGENT_PLUGINS, p as isApiProvider, vt as CODING_AGENT_PLUGIN_DESCRIPTIONS, yt as CODING_AGENT_PLUGIN_DISPLAY_NAMES } from "./types-BFevViUY.js";
+import { o as getUserEmail } from "./accounts-Bw2qPqE-.js";
+import { C as PolicyObjectSchema, M as MULTI_TURN_STRATEGIES, St as buildInputPromptDescription, T as isValidReusablePolicyId, _ as PromptSchema, _t as CODING_AGENT_PLUGINS, p as isApiProvider, vt as CODING_AGENT_PLUGIN_DESCRIPTIONS, yt as CODING_AGENT_PLUGIN_DISPLAY_NAMES } from "./types-C_e_SykX.js";
 import { i as isJavascriptFile } from "./fileExtensions-D4GCJ67J.js";
-import { r as importModule } from "./esm-Bexx2PFc.js";
-import { i as extractVariablesFromTemplate, o as getNunjucksEngine } from "./render-CSP99NLm.js";
-import { d as hasCodexDefaultCredentials, l as shouldGenerateRemote, r as getRemoteGenerationUrl, u as getCodexDefaultProviders } from "./remoteGeneration-B1_XsKXU.js";
-import { p as getPoliciesFromCloud } from "./storage-BU4qcnOb.js";
+import { r as importModule } from "./esm-CYMWMaSj.js";
+import { i as extractVariablesFromTemplate, o as getNunjucksEngine } from "./render-jE6RKVgk.js";
+import { d as hasCodexDefaultCredentials, l as shouldGenerateRemote, r as getRemoteGenerationUrl, u as getCodexDefaultProviders } from "./remoteGeneration-DW6mPZnO.js";
+import { p as getPoliciesFromCloud } from "./storage-B9lg2RCL.js";
 import { n as sha256 } from "./createHash-CgRvs4Fn.js";
-import { a as fetchWithCache, c as isCacheEnabled, o as getCache } from "./cache-DpPWrkTE.js";
-import { r as runPython } from "./pythonUtils-CgYxeSmO.js";
-import { B as parseFileUrl, F as maybeLoadToolsFromExternalFile, I as parsePathOrGlob, M as maybeLoadFromExternalFile, j as maybeLoadConfigFromExternalFile, k as getNunjucksEngineForFilePath } from "./util-DURocbYR.js";
-import { t as OpenAiChatCompletionProvider } from "./chat-DqUFcWI0.js";
-import { x as hasGoogleDefaultCredentials } from "./transform-EtD4jAWi.js";
-import { C as DefaultSuggestionsProvider$2, D as AzureChatCompletionProvider, E as AzureEmbeddingProvider, S as DefaultLlmRubricProvider, T as AzureModerationProvider, b as DefaultGradingJsonProvider$2, c as OpenAiModerationProvider, g as DefaultGradingProvider$3, h as DefaultEmbeddingProvider$2, l as MistralChatCompletionProvider, n as loadApiProvider, o as getFileHashes, s as parseScriptParts, u as MistralEmbeddingProvider, w as DefaultSynthesizeProvider$1, x as DefaultGradingProvider$2 } from "./providers-B9KzWxAX.js";
-import { t as OpenAiEmbeddingProvider } from "./embedding--UZVe4_7.js";
+import { a as fetchWithCache, c as isCacheEnabled, o as getCache } from "./cache-BuudGL4I.js";
+import { r as runPython } from "./pythonUtils-CLAoc6jg.js";
+import { B as parseFileUrl, F as maybeLoadToolsFromExternalFile, I as parsePathOrGlob, M as maybeLoadFromExternalFile, j as maybeLoadConfigFromExternalFile, k as getNunjucksEngineForFilePath } from "./util-CES4qWd7.js";
+import { t as OpenAiChatCompletionProvider } from "./chat-B8lsx0A0.js";
+import { x as hasGoogleDefaultCredentials } from "./transform-AfC8gqCB.js";
+import { C as DefaultSuggestionsProvider$2, D as AzureChatCompletionProvider, E as AzureEmbeddingProvider, S as DefaultLlmRubricProvider, T as AzureModerationProvider, b as DefaultGradingJsonProvider$2, c as OpenAiModerationProvider, g as DefaultGradingProvider$3, h as DefaultEmbeddingProvider$2, l as MistralChatCompletionProvider, n as loadApiProvider, o as getFileHashes, s as parseScriptParts, u as MistralEmbeddingProvider, w as DefaultSynthesizeProvider$1, x as DefaultGradingProvider$2 } from "./providers-CYBHjFEg.js";
+import { t as OpenAiEmbeddingProvider } from "./embedding-BQBXSWgy.js";
 import { r as accumulateTokenUsage } from "./tokenUsageUtils-CDet74yk.js";
-import { t as AnthropicMessagesProvider } from "./messages-DGUlSNU7.js";
-import { t as OpenAiResponsesProvider } from "./responses-B3W2JvOQ.js";
-import { r as materializeInputVariablesWithMetadata } from "./inputVariables-DXFdi7AI.js";
-import { a as extractPromptFromTags, c as isBasicRefusal, i as extractMaterializedVariablesFromJsonWithMetadata, l as isEmptyResponse, n as extractGoalFromPrompt, r as extractInputVarsFromPrompt, s as getShortPluginId, t as extractAllPromptsFromTags, u as removePrefix } from "./util-kDURhgJW.js";
-import { _ as isRateLimitWrapped, f as redteamProviderManager, g as createProviderRateLimitOptions } from "./shared-9WHQ1oNE.js";
-import { a as PROMPT_DELIMITER, n as maybeFilePath, r as normalizeInput } from "./utils-BFOh20Gb.js";
-import { n as getGeneratedPromptOverLimit, r as getMaxCharsPerMessageModifierValue, t as MAX_CHARS_PER_MESSAGE_MODIFIER_KEY } from "./promptLength-4X-Wd8PG.js";
+import { t as AnthropicMessagesProvider } from "./messages-CQoK4jHx.js";
+import { t as OpenAiResponsesProvider } from "./responses-C0yU3fO6.js";
+import { r as materializeInputVariablesWithMetadata } from "./inputVariables-qJmuSN5i.js";
+import { a as extractPromptFromTags, c as isBasicRefusal, i as extractMaterializedVariablesFromJsonWithMetadata, l as isEmptyResponse, n as extractGoalFromPrompt, r as extractInputVarsFromPrompt, s as getShortPluginId, t as extractAllPromptsFromTags, u as removePrefix } from "./util-D9grslJP.js";
+import { _ as isRateLimitWrapped, f as redteamProviderManager, g as createProviderRateLimitOptions } from "./shared-DsbObxvy.js";
+import { a as PROMPT_DELIMITER, n as maybeFilePath, r as normalizeInput } from "./utils-Cdx3ZJr7.js";
+import { n as getGeneratedPromptOverLimit, r as getMaxCharsPerMessageModifierValue, t as MAX_CHARS_PER_MESSAGE_MODIFIER_KEY } from "./promptLength-PkjlYFx0.js";
 import { t as REDTEAM_MEMORY_POISONING_PLUGIN_ID } from "./constants-DH5XYLKZ.js";
-import { n as checkExfilTracking } from "./indirectWebPwn-uyWdHx04.js";
+import { n as checkExfilTracking } from "./indirectWebPwn-BXUoJl4I.js";
 import { AsyncLocalStorage } from "node:async_hooks";
 import z$1 from "zod";
 import * as fs$2 from "fs";
@@ -276,13 +276,13 @@ const DefaultSuggestionsProvider$1 = new MistralChatCompletionProvider("mistral-
 const DefaultSynthesizeProvider = new MistralChatCompletionProvider("mistral-large-latest");
 //#endregion
 //#region src/providers/openai/defaults.ts
-const DEFAULT_OPENAI_GRADING_MODEL = "gpt-5.4-2026-03-05";
+const DEFAULT_OPENAI_GRADING_MODEL = "gpt-5.5-2026-04-23";
 const DefaultEmbeddingProvider = new OpenAiEmbeddingProvider("text-embedding-3-large");
 const DefaultGradingProvider = new OpenAiChatCompletionProvider(DEFAULT_OPENAI_GRADING_MODEL);
 const DefaultGradingJsonProvider = new OpenAiChatCompletionProvider(DEFAULT_OPENAI_GRADING_MODEL, { config: { response_format: { type: "json_object" } } });
 const DefaultSuggestionsProvider = new OpenAiChatCompletionProvider(DEFAULT_OPENAI_GRADING_MODEL);
 const DefaultModerationProvider = new OpenAiModerationProvider("omni-moderation-latest");
-const DefaultWebSearchProvider = new OpenAiResponsesProvider("gpt-5.4-2026-03-05", { config: { tools: [{ type: "web_search_preview" }] } });
+const DefaultWebSearchProvider = new OpenAiResponsesProvider("gpt-5.5-2026-04-23", { config: { tools: [{ type: "web_search_preview" }] } });
 async function getDefaultProviderPreferences(env) {
 	const hasAnthropicCredentials = Boolean(getEnvString("ANTHROPIC_API_KEY") || env?.ANTHROPIC_API_KEY);
 	const hasOpenAiCredentials = Boolean(getEnvString("OPENAI_API_KEY") || env?.OPENAI_API_KEY);
@@ -1396,7 +1396,7 @@ async function doRemoteGrading(payload) {
 			method: "POST",
 			headers: { "Content-Type": "application/json" },
 			body
-		}, REQUEST_TIMEOUT_MS);
+		}, getRequestTimeoutMs());
 		logger.debug(`Remote grading result: status=${status}, statusText=${statusText}, data=${JSON.stringify(data)}`);
 		if (status !== 200) throw new Error(`Remote grading failed with status ${status}: ${statusText} ${JSON.stringify(data)}`);
 		const { result } = data;
@@ -1439,7 +1439,7 @@ async function doRemoteScoringWithPi(payload, passThreshold = .5) {
 					"x-api-key": apiKey
 				},
 				body
-			}, REQUEST_TIMEOUT_MS);
+			}, getRequestTimeoutMs());
 			return convertPiResultToGradingResult(data, passThreshold);
 		} else throw new Error(`Env var WITHPI_API_KEY must be set. Visit https://docs.withpi.ai for more information.`);
 	} catch (error) {
@@ -8389,7 +8389,7 @@ function describeFilters(config) {
 }
 async function fetchDataset$1(limit, config) {
 	try {
-		const response = await fetchWithTimeout(DATASET_URL, {}, REQUEST_TIMEOUT_MS);
+		const response = await fetchWithTimeout(DATASET_URL, {}, getRequestTimeoutMs());
 		if (!response.ok) throw new Error(`HTTP status: ${response.status} ${response.statusText}`);
 		const records = parse$1(await response.text(), { columns: true });
 		logger.debug(`[harmbench] Parsed ${records.length} entries from CSV`);
@@ -15606,7 +15606,7 @@ var VLSUDatasetManager = class VLSUDatasetManager extends ImageDatasetManager {
 		}
 		logger.debug(`[vlsu] Fetching CSV from ${VLSU_CSV_URL}`);
 		try {
-			const response = await fetchWithCache(VLSU_CSV_URL, {}, REQUEST_TIMEOUT_MS, "text");
+			const response = await fetchWithCache(VLSU_CSV_URL, {}, getRequestTimeoutMs(), "text");
 			if (response.status < 200 || response.status >= 300) throw new Error(`Failed to fetch VLSU CSV: ${response.statusText}`);
 			const csvText = response.data;
 			const records = parse$1(csvText, {
@@ -16165,4 +16165,4 @@ function getGraderById(id) {
 //#endregion
 export { DEFAULT_WEB_SEARCH_PROMPT as $, DivergentRepetitionPlugin as A, sampleArray as B, getPiiLeakTestsForCategory as C, getDefaultProviders as Ct, HarmbenchPlugin as D, getGradingProvider as Dt, ImitationPlugin as E, getAndCheckProvider as Et, AegisPlugin as F, matchesGEval as G, isGraderFailure as H, RedteamGraderBase as I, matchesTrajectoryGoalSuccess as J, matchesLlmRubric as K, RedteamPluginBase as L, CrossSessionLeakPlugin as M, ContractPlugin as N, HallucinationPlugin as O, getProviderCallExecutionContext as Ot, BeavertailsPlugin as P, readProviderPromptMap as Q, getCustomPolicies as R, PlinyPlugin as S, processFileReference as St, IntentPlugin as T, callProviderWithContext as Tt, matchesClosedQa as U, fetchHuggingFaceDataset as V, matchesFactuality as W, processPrompts as X, doRemoteGrading as Y, readPrompts as Z, PoliticsPlugin as _, splitIntoSentences as _t, UnverifiableClaimsPlugin as a, CONTEXT_RECALL as at, isValidPolicyObject as b, getFinalTest as bt, ToolDiscoveryPlugin as c, CONTEXT_RELEVANCE as ct, TeenSafetyDangerousContentPlugin as d, renderLlmRubricPrompt as dt, SELECT_BEST_PROMPT as et, TeenSafetyAgeRestrictedGoodsAndServicesPlugin as f, cosineSimilarity as ft, PromptExtractionPlugin as g, normalizeMatcherTokenUsage as gt, RbacPlugin as h, fail as ht, VLGuardPlugin as i, CONTEXT_FAITHFULNESS_NLI_STATEMENTS as it, DebugAccessPlugin as j, ExcessiveAgencyPlugin as k, withProviderCallExecutionContext as kt, TeenSafetyHarmfulBodyIdealsPlugin as l, CONTEXT_RELEVANCE_BAD as lt, ShellInjectionPlugin as m, euclideanDistance as mt, getGraderById as n, ANSWER_RELEVANCY_GENERATE as nt, UnsafeBenchPlugin as o, CONTEXT_RECALL_ATTRIBUTED_TOKEN as ot, SqlInjectionPlugin as p, dotProduct as pt, matchesPiScore as q, VLSUPlugin as r, CONTEXT_FAITHFULNESS_LONGFORM as rt, ToxicChatPlugin as s, CONTEXT_RECALL_NOT_ATTRIBUTED_TOKEN as st, GRADERS as t, SUGGEST_PROMPTS_SYSTEM_MESSAGE as tt, TeenSafetyDangerousRoleplayPlugin as u, loadRubricPrompt as ut, PolicyPlugin as v, tryParse as vt, OverreliancePlugin as w, DEFAULT_ANTHROPIC_MODEL as wt, makeInlinePolicyIdSync as x, loadFromJavaScriptFile as xt, determinePolicyTypeFromId as y, coerceString as yt, retryWithDeduplication as z };
 
-//# sourceMappingURL=graders-BGP99PdK.js.map
+//# sourceMappingURL=graders-BnO-xoa-.js.map

package/dist/src/{graders-ClrU2fnd.cjs → graders-CQv1GT5Q.cjs}

@@ -1,32 +1,32 @@
 const require_rolldown_runtime = require("./rolldown-runtime-D_mwlA32.cjs");
-const require_logger = require("./logger-cfNpzI4o.cjs");
+const require_logger = require("./logger-Cm0Nb8YX.cjs");
 const require_invariant = require("./invariant-QtnLD03y.cjs");
 const require_types = require("./types-CxJvaY2S.cjs");
-const require_fetch = require("./fetch-Dw4XZHjj.cjs");
+const require_fetch = require("./fetch-B2HvaVUc.cjs");
 const require_fileExtensions = require("./fileExtensions-BhdwzYaD.cjs");
-const require_accounts = require("./accounts-CmWzeD2d.cjs");
-const require_esm = require("./esm-BIKakvNa.cjs");
-const require_render = require("./render-BNTrbmBw.cjs");
-const require_remoteGeneration = require("./remoteGeneration-DS9N3pgB.cjs");
-const require_storage = require("./storage-CA-v9V2v.cjs");
-const require_pythonUtils = require("./pythonUtils-Cokhluq3.cjs");
-const require_util = require("./util-SPsvFONY.cjs");
+const require_accounts = require("./accounts-DAcQXPwE.cjs");
+const require_esm = require("./esm-vv4rR-Dp.cjs");
+const require_render = require("./render-DJ4yipCD.cjs");
+const require_remoteGeneration = require("./remoteGeneration-CD3c_rzc.cjs");
+const require_storage = require("./storage-Cv7QYUPO.cjs");
+const require_pythonUtils = require("./pythonUtils-BjuymNdM.cjs");
+const require_util = require("./util-BuYzkbEz.cjs");
 const require_createHash = require("./createHash-CSiqnK5P.cjs");
-const require_cache = require("./cache-CPGUA4Yl.cjs");
-const require_chat = require("./chat-DxTDQ83C.cjs");
-const require_transform = require("./transform-DhNkAUs8.cjs");
-const require_providers = require("./providers-BDVVIQM6.cjs");
-const require_embedding = require("./embedding-BbrwopfX.cjs");
+const require_cache = require("./cache-WuIsgzMA.cjs");
+const require_chat = require("./chat-DkitQHdt.cjs");
+const require_transform = require("./transform-DKw2FNwg.cjs");
+const require_providers = require("./providers-DP1CDgd5.cjs");
+const require_embedding = require("./embedding-BMqFDnLI.cjs");
 const require_tokenUsageUtils = require("./tokenUsageUtils-_B-P8IAi.cjs");
-const require_messages = require("./messages-DBPir0TQ.cjs");
-const require_responses = require("./responses-1UFFF9N_.cjs");
-const require_inputVariables = require("./inputVariables-Dq9W-Z3a.cjs");
-const require_util$1 = require("./util-CN8om2rz.cjs");
-const require_shared = require("./shared-WkgnDkcg.cjs");
-const require_utils = require("./utils-Ve6kuJsa.cjs");
-const require_promptLength = require("./promptLength-BbBbDHNj.cjs");
+const require_messages = require("./messages-BSFLFXIO.cjs");
+const require_responses = require("./responses-Bs2_3zyr.cjs");
+const require_inputVariables = require("./inputVariables-D5K2vTp4.cjs");
+const require_util$1 = require("./util-3BRss2KA.cjs");
+const require_shared = require("./shared-DlmtkptR.cjs");
+const require_utils = require("./utils-B_HCobst.cjs");
+const require_promptLength = require("./promptLength-CAcg7BmO.cjs");
 const require_constants = require("./constants-a2kYssQk.cjs");
-const require_indirectWebPwn = require("./indirectWebPwn-BJ22AbQa.cjs");
+const require_indirectWebPwn = require("./indirectWebPwn-04ziKbly.cjs");
 let fs = require("fs");
 fs = require_rolldown_runtime.__toESM(fs, 1);
 let path = require("path");
@@ -285,13 +285,13 @@ const DefaultSuggestionsProvider$1 = new require_providers.MistralChatCompletion
 const DefaultSynthesizeProvider = new require_providers.MistralChatCompletionProvider("mistral-large-latest");
 //#endregion
 //#region src/providers/openai/defaults.ts
-const DEFAULT_OPENAI_GRADING_MODEL = "gpt-5.4-2026-03-05";
+const DEFAULT_OPENAI_GRADING_MODEL = "gpt-5.5-2026-04-23";
 const DefaultEmbeddingProvider = new require_embedding.OpenAiEmbeddingProvider("text-embedding-3-large");
 const DefaultGradingProvider = new require_chat.OpenAiChatCompletionProvider(DEFAULT_OPENAI_GRADING_MODEL);
 const DefaultGradingJsonProvider = new require_chat.OpenAiChatCompletionProvider(DEFAULT_OPENAI_GRADING_MODEL, { config: { response_format: { type: "json_object" } } });
 const DefaultSuggestionsProvider = new require_chat.OpenAiChatCompletionProvider(DEFAULT_OPENAI_GRADING_MODEL);
 const DefaultModerationProvider = new require_providers.OpenAiModerationProvider("omni-moderation-latest");
-const DefaultWebSearchProvider = new require_responses.OpenAiResponsesProvider("gpt-5.4-2026-03-05", { config: { tools: [{ type: "web_search_preview" }] } });
+const DefaultWebSearchProvider = new require_responses.OpenAiResponsesProvider("gpt-5.5-2026-04-23", { config: { tools: [{ type: "web_search_preview" }] } });
 async function getDefaultProviderPreferences(env) {
 	const hasAnthropicCredentials = Boolean(require_logger.getEnvString("ANTHROPIC_API_KEY") || env?.ANTHROPIC_API_KEY);
 	const hasOpenAiCredentials = Boolean(require_logger.getEnvString("OPENAI_API_KEY") || env?.OPENAI_API_KEY);
@@ -1405,7 +1405,7 @@ async function doRemoteGrading(payload) {
 			method: "POST",
 			headers: { "Content-Type": "application/json" },
 			body
-		}, require_fetch.REQUEST_TIMEOUT_MS);
+		}, require_fetch.getRequestTimeoutMs());
 		require_logger.logger.debug(`Remote grading result: status=${status}, statusText=${statusText}, data=${JSON.stringify(data)}`);
 		if (status !== 200) throw new Error(`Remote grading failed with status ${status}: ${statusText} ${JSON.stringify(data)}`);
 		const { result } = data;
@@ -1448,7 +1448,7 @@ async function doRemoteScoringWithPi(payload, passThreshold = .5) {
 					"x-api-key": apiKey
 				},
 				body
-			}, require_fetch.REQUEST_TIMEOUT_MS);
+			}, require_fetch.getRequestTimeoutMs());
 			return convertPiResultToGradingResult(data, passThreshold);
 		} else throw new Error(`Env var WITHPI_API_KEY must be set. Visit https://docs.withpi.ai for more information.`);
 	} catch (error) {
@@ -8398,7 +8398,7 @@ function describeFilters(config) {
 }
 async function fetchDataset$1(limit, config) {
 	try {
-		const response = await require_fetch.fetchWithTimeout(DATASET_URL, {}, require_fetch.REQUEST_TIMEOUT_MS);
+		const response = await require_fetch.fetchWithTimeout(DATASET_URL, {}, require_fetch.getRequestTimeoutMs());
 		if (!response.ok) throw new Error(`HTTP status: ${response.status} ${response.statusText}`);
 		const records = (0, csv_parse_sync.parse)(await response.text(), { columns: true });
 		require_logger.logger.debug(`[harmbench] Parsed ${records.length} entries from CSV`);
@@ -15615,7 +15615,7 @@ var VLSUDatasetManager = class VLSUDatasetManager extends ImageDatasetManager {
 		}
 		require_logger.logger.debug(`[vlsu] Fetching CSV from ${VLSU_CSV_URL}`);
 		try {
-			const response = await require_cache.fetchWithCache(VLSU_CSV_URL, {}, require_fetch.REQUEST_TIMEOUT_MS, "text");
+			const response = await require_cache.fetchWithCache(VLSU_CSV_URL, {}, require_fetch.getRequestTimeoutMs(), "text");
 			if (response.status < 200 || response.status >= 300) throw new Error(`Failed to fetch VLSU CSV: ${response.statusText}`);
 			const csvText = response.data;
 			const records = (0, csv_parse_sync.parse)(csvText, {
@@ -16681,4 +16681,4 @@ Object.defineProperty(exports, "withProviderCallExecutionContext", {
 	}
 });
 
-//# sourceMappingURL=graders-ClrU2fnd.cjs.map
+//# sourceMappingURL=graders-CQv1GT5Q.cjs.map

package/dist/src/graders-D4vc1PFi.js

@@ -0,0 +1,3 @@
+#!/usr/bin/env node
+import { n as getGraderById } from "./graders-BnO-xoa-.js";
+export { getGraderById };