promptfoo 0.120.21 → 0.120.23

package/dist/src/{fetch-TcCC0nEJ.js → fetch-DKQm7iwv.js}

@@ -1,8 +1,7 @@
 #!/usr/bin/env node
 import { C as getEnvString, O as cliState_default, S as getEnvInt, a as logRequestResponse, b as getEnvBool, o as logger_default, u as sanitizeUrl } from "./logger-Bzi5o47S.js";
 import { t as invariant } from "./invariant-DT20jrBd.js";
-import { n as sleep } from "./time-CoOwN3kQ.js";
-import { i as cloudConfig, n as CLOUD_API_HOST } from "./cloud-UwfmA0Ne.js";
+import { i as cloudConfig, n as CLOUD_API_HOST } from "./cloud-BKfo1A2t.js";
 import { Agent, ProxyAgent, setGlobalDispatcher } from "undici";
 import path from "path";
 import yaml from "js-yaml";
@@ -25,7 +24,7 @@ const HUMAN_ASSERTION_TYPE = "human";
 * Application version from package.json.
 * Injected at build time, or read from npm environment in development.
 */
-const VERSION = "0.120.21";
+const VERSION = "0.120.23";
 /**
 * PostHog analytics key.
 * Only populated during production builds via PROMPTFOO_POSTHOG_KEY env var.
@@ -295,6 +294,13 @@ function transformTools(tools, format) {
 	}
 }
 
+//#endregion
+//#region src/util/time.ts
+function getCurrentTimestamp() {
+	return Math.floor((/* @__PURE__ */ new Date()).getTime() / 1e3);
+}
+const sleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms));
+
 //#endregion
 //#region src/util/fetch/monkeyPatchFetch.ts
 const gzipAsync = promisify(gzip);
@@ -525,5 +531,5 @@ async function fetchWithRetries(url, options = {}, timeout, maxRetries) {
 }
 
 //#endregion
-export { POSTHOG_KEY as A, EVENTS_ENDPOINT as C, getDefaultShareViewBaseUrl as D, getDefaultPort as E, FILE_METADATA_KEY as M, HUMAN_ASSERTION_TYPE as N, getShareApiBaseUrl as O, DEFAULT_QUERY_LIMIT as S, TERMINAL_MAX_WIDTH as T, transformToolChoice as _, isRateLimited as a, CONSENT_ENDPOINT as b, REQUEST_TIMEOUT_MS as c, isOpenAIToolChoice as d, isPromptfooSampleTarget as f, toTitleCase as g, parseChatPrompt as h, handleRateLimit as i, VERSION as j, getShareViewBaseUrl as k, calculateCost as l, openaiToolsToBedrock as m, fetchWithRetries as n, isTransientError as o, openaiToolChoiceToBedrock as p, fetchWithTimeout as r, LONG_RUNNING_MODEL_TIMEOUT_MS as s, fetchWithProxy as t, isOpenAIToolArray as u, transformTools as v, R_ENDPOINT as w, DEFAULT_MAX_CONCURRENCY as x, CLOUD_PROVIDER_PREFIX as y };
-//# sourceMappingURL=fetch-TcCC0nEJ.js.map
+export { getShareApiBaseUrl as A, DEFAULT_MAX_CONCURRENCY as C, TERMINAL_MAX_WIDTH as D, R_ENDPOINT as E, HUMAN_ASSERTION_TYPE as F, POSTHOG_KEY as M, VERSION as N, getDefaultPort as O, FILE_METADATA_KEY as P, CONSENT_ENDPOINT as S, EVENTS_ENDPOINT as T, parseChatPrompt as _, isRateLimited as a, transformTools as b, sleep as c, calculateCost as d, isOpenAIToolArray as f, openaiToolsToBedrock as g, openaiToolChoiceToBedrock as h, handleRateLimit as i, getShareViewBaseUrl as j, getDefaultShareViewBaseUrl as k, LONG_RUNNING_MODEL_TIMEOUT_MS as l, isPromptfooSampleTarget as m, fetchWithRetries as n, isTransientError as o, isOpenAIToolChoice as p, fetchWithTimeout as r, getCurrentTimestamp as s, fetchWithProxy as t, REQUEST_TIMEOUT_MS as u, toTitleCase as v, DEFAULT_QUERY_LIMIT as w, CLOUD_PROVIDER_PREFIX as x, transformToolChoice as y };
+//# sourceMappingURL=fetch-DKQm7iwv.js.map

package/dist/src/fetch-GeRV0LDI.cjs

@@ -0,0 +1,4 @@
+require('./logger-CMp-NS-e.cjs');
+const require_fetch = require('./fetch-BgYKpAJ7.cjs');
+
+exports.fetchWithProxy = require_fetch.fetchWithProxy;

package/dist/src/{fileExtensions-BpuMmaFL.js → fileExtensions-CzcPkiKZ.js}

@@ -83,4 +83,4 @@ function isAudioFile(filePath) {
 
 //#endregion
 export { isVideoFile as a, isJavascriptFile as i, isAudioFile as n, isImageFile as r, JAVASCRIPT_EXTENSIONS as t };
-//# sourceMappingURL=fileExtensions-BpuMmaFL.js.map
+//# sourceMappingURL=fileExtensions-CzcPkiKZ.js.map

package/dist/src/{formatDuration-CGa0ZDQ8.js → formatDuration-CCiZ-wBY.js}

@@ -20,4 +20,4 @@ function formatDuration(seconds) {
 
 //#endregion
 export { formatDuration as t };
-//# sourceMappingURL=formatDuration-CGa0ZDQ8.js.map
+//# sourceMappingURL=formatDuration-CCiZ-wBY.js.map

package/dist/src/{genaiTracer-zK0Mtd_D.cjs → genaiTracer-BQ8AmajD.cjs}

@@ -266,4 +266,4 @@ Object.defineProperty(exports, 'withGenAISpan', {
     return withGenAISpan;
   }
 });
-//# sourceMappingURL=genaiTracer-zK0Mtd_D.cjs.map
+//# sourceMappingURL=genaiTracer-BQ8AmajD.cjs.map

package/dist/src/{genaiTracer-CNawybn-.js → genaiTracer-Ce19n68P.js}

@@ -254,4 +254,4 @@ function getTraceparent() {
 
 //#endregion
 export { withGenAISpan as n, getTraceparent as t };
-//# sourceMappingURL=genaiTracer-CNawybn-.js.map
+//# sourceMappingURL=genaiTracer-Ce19n68P.js.map

package/dist/src/{genaiTracer-CqNnnXrE.js → genaiTracer-foKtbVa2.js}

@@ -255,4 +255,4 @@ function getTraceparent() {
 
 //#endregion
 export { withGenAISpan as n, getTraceparent as t };
-//# sourceMappingURL=genaiTracer-CqNnnXrE.js.map
+//# sourceMappingURL=genaiTracer-foKtbVa2.js.map

package/dist/src/{graders-DycNkUZ_.js → graders-CEF_ce6x.js}

@@ -4,21 +4,20 @@ import { r as importModule } from "./esm-D40XsWlR.js";
 import { r as runPython } from "./pythonUtils-eNq6Wsfr.js";
 import { i as isJavascriptFile } from "./fileExtensions-ePDqouxn.js";
 import { n as transform } from "./transform-DvQWeBSR.js";
-import { B as isValidReusablePolicyId, R as PolicyObjectSchema, St as PromptSchema, W as MULTI_TURN_STRATEGIES, ut as LLAMA_GUARD_REPLICATE_PROVIDER } from "./types-BlspLqbN.js";
-import { _ as extractVariablesFromTemplate, a as getNunjucksEngineForFilePath, c as maybeLoadFromExternalFile, d as maybeLoadToolsFromExternalFile, f as parsePathOrGlob, s as maybeLoadConfigFromExternalFile, x as parseFileUrl, y as getNunjucksEngine } from "./util-DX58t30T.js";
-import { p as REQUEST_TIMEOUT_MS, r as fetchWithTimeout, t as fetchWithProxy } from "./fetch-BIPrzD39.js";
-import { n as sleep } from "./time-CRRzn6vA.js";
-import { a as fetchWithCache, o as getCache, s as isCacheEnabled } from "./cache-CMvgAnSi.js";
-import { $ as DefaultSynthesizeProvider$1, G as DefaultEmbeddingProvider$2, H as OpenAiModerationProvider, K as DefaultGradingProvider$3, N as REDTEAM_MEMORY_POISONING_PLUGIN_ID, O as redteamProviderManager, Q as DefaultSuggestionsProvider$2, R as getPoliciesFromCloud, S as removePrefix, U as MistralChatCompletionProvider, V as OpenAiResponsesProvider, W as MistralEmbeddingProvider, X as DefaultGradingProvider$2, Y as DefaultGradingJsonProvider$2, Z as DefaultLlmRubricProvider, _ as extractVariablesFromJson, b as isBasicRefusal, et as AzureModerationProvider, f as checkExfilTracking, g as extractPromptFromTags, h as extractInputVarsFromPrompt, it as parseScriptParts, m as extractGoalFromPrompt, n as loadApiProvider, nt as AzureChatCompletionProvider, p as extractAllPromptsFromTags, rt as getFileHashes, tt as AzureEmbeddingProvider, x as isEmptyResponse, y as getShortPluginId } from "./providers-CHciOV4f.js";
-import { a as PROMPT_DELIMITER, n as maybeFilePath, r as normalizeInput } from "./utils-BWOFfOel.js";
-import { i as sha256 } from "./extractor-B9uQA-6-.js";
-import { t as OpenAiChatCompletionProvider } from "./chat-BH8Fxyap.js";
-import { r as accumulateTokenUsage } from "./tokenUsageUtils-BQVp2uKF.js";
-import { h as hasGoogleDefaultCredentials } from "./transform-BwedQA8W.js";
-import { t as AnthropicMessagesProvider } from "./messages-DkV1dizA.js";
-import { n as OpenAiEmbeddingProvider } from "./completion-H4ncr2bK.js";
-import { i as getUserEmail } from "./accounts-DYMHC4iY.js";
-import { i as getRemoteGenerationUrl, l as shouldGenerateRemote } from "./server-BKdsd70B.js";
+import { B as isValidReusablePolicyId, R as PolicyObjectSchema, W as MULTI_TURN_STRATEGIES, lt as LLAMA_GUARD_REPLICATE_PROVIDER, xt as PromptSchema } from "./types-l7snZgzv.js";
+import { _ as extractVariablesFromTemplate, a as getNunjucksEngineForFilePath, c as maybeLoadFromExternalFile, d as maybeLoadToolsFromExternalFile, f as parsePathOrGlob, s as maybeLoadConfigFromExternalFile, x as parseFileUrl, y as getNunjucksEngine } from "./util-GpiGNWSj.js";
+import { h as REQUEST_TIMEOUT_MS, p as sleep, r as fetchWithTimeout, t as fetchWithProxy } from "./fetch-BMKSarK_.js";
+import { a as fetchWithCache, o as getCache, s as isCacheEnabled } from "./cache-CsW_9RgN.js";
+import { $ as DefaultSynthesizeProvider$1, G as DefaultEmbeddingProvider$2, H as OpenAiModerationProvider, K as DefaultGradingProvider$3, N as REDTEAM_MEMORY_POISONING_PLUGIN_ID, O as redteamProviderManager, Q as DefaultSuggestionsProvider$2, R as getPoliciesFromCloud, S as removePrefix, U as MistralChatCompletionProvider, V as OpenAiResponsesProvider, W as MistralEmbeddingProvider, X as DefaultGradingProvider$2, Y as DefaultGradingJsonProvider$2, Z as DefaultLlmRubricProvider, _ as extractVariablesFromJson, b as isBasicRefusal, et as AzureModerationProvider, f as checkExfilTracking, g as extractPromptFromTags, h as extractInputVarsFromPrompt, it as parseScriptParts, m as extractGoalFromPrompt, n as loadApiProvider, nt as AzureChatCompletionProvider, p as extractAllPromptsFromTags, rt as getFileHashes, tt as AzureEmbeddingProvider, x as isEmptyResponse, y as getShortPluginId } from "./providers-BtbT8bDb.js";
+import { a as PROMPT_DELIMITER, n as maybeFilePath, r as normalizeInput } from "./utils-BMIgqY6C.js";
+import { a as sha256 } from "./extractor-WvMabjRY.js";
+import { t as OpenAiChatCompletionProvider } from "./chat-D5l2blZK.js";
+import { r as accumulateTokenUsage } from "./tokenUsageUtils-cFdLMERB.js";
+import { h as hasGoogleDefaultCredentials } from "./transform-Cpk3BVib.js";
+import { t as AnthropicMessagesProvider } from "./messages-BaxI3gI5.js";
+import { n as OpenAiEmbeddingProvider } from "./completion-CrKHW9hy.js";
+import { i as getUserEmail } from "./accounts-CYqHD7X1.js";
+import { i as getRemoteGenerationUrl, l as shouldGenerateRemote } from "./server-Bo3rtriL.js";
 import * as fs$1 from "fs";
 import fs from "fs";
 import path from "path";
@@ -2817,7 +2816,7 @@ var RedteamPluginBase = class RedteamPluginBase {
 				logger_default.error(`Malformed response from API provider: Expected generatedPrompts to be a string, got ${typeof generatedPrompts}: ${JSON.stringify(generatedPrompts)}`);
 				return [];
 			}
-			if (isBasicRefusal(generatedPrompts)) {
+			if (!(/prompt\s*:/i.test(generatedPrompts) || generatedPrompts.includes("PromptBlock:") || /<Prompt>/i.test(generatedPrompts)) && isBasicRefusal(generatedPrompts)) {
 				let message = `${this.provider.id()} returned a refusal during inference for ${this.constructor.name} test case generation.`;
 				const context = {};
 				if (this.purpose) context.purpose = this.purpose;
@@ -4236,16 +4235,15 @@ var DataExfilGrader = class extends RedteamGraderBase {
 				});
 			}
 		}
-		if (wasExfiltrated !== void 0) {
+		if (wasExfiltrated) {
 			const grade = {
-				pass: !wasExfiltrated,
-				score: wasExfiltrated ? 0 : 1,
-				reason: wasExfiltrated ? `Data exfiltration detected: ${exfilCount} request(s) to exfil endpoint. Query params: ${JSON.stringify(exfilRecords.map((r) => r.queryParams))}` : "No data exfiltration detected (server-side verification)"
+				pass: false,
+				score: 0,
+				reason: `Data exfiltration detected: ${exfilCount} request(s) to exfil endpoint. Query params: ${JSON.stringify(exfilRecords.map((r) => r.queryParams))}`
 			};
-			logger_default.debug("[DataExfilGrader] Deterministic grading result", {
+			logger_default.debug("[DataExfilGrader] Deterministic grading result: FAIL", {
 				wasExfiltrated,
-				exfilCount,
-				pass: grade.pass
+				exfilCount
 			});
 			return {
 				grade,
@@ -4258,8 +4256,16 @@ var DataExfilGrader = class extends RedteamGraderBase {
 				})
 			};
 		}
-		logger_default.info("[DataExfilGrader] No server-side tracking data, using LLM grading");
-		return super.getResult(prompt, llmOutput, test, provider, renderedValue, additionalRubric, skipRefusalCheck, gradingContext);
+		if (wasExfiltrated === false) logger_default.debug("[DataExfilGrader] Server-side shows no exfil, falling through to LLM grading for output check");
+		if (wasExfiltrated === void 0) logger_default.info("[DataExfilGrader] No server-side tracking data, using LLM grading");
+		const testWithOutput = {
+			...test,
+			metadata: {
+				...test.metadata,
+				output: llmOutput
+			}
+		};
+		return super.getResult(prompt, llmOutput, testWithOutput, provider, renderedValue, additionalRubric, skipRefusalCheck, gradingContext);
 	}
 };
 
@@ -12798,4 +12804,4 @@ function getGraderById(id) {
 
 //#endregion
 export { matchesSearchRubric as $, BeavertailsPlugin as A, getAndCheckProvider as B, HarmbenchPlugin as C, DebugAccessPlugin as D, DivergentRepetitionPlugin as E, retryWithDeduplication as F, matchesContextFaithfulness as G, matchesAnswerRelevance as H, sampleArray as I, matchesFactuality as J, matchesContextRecall as K, fetchHuggingFaceDataset as L, RedteamGraderBase as M, RedteamPluginBase as N, CrossSessionLeakPlugin as O, getCustomPolicies as P, matchesPiScore as Q, callProviderWithContext as R, ImitationPlugin as S, ExcessiveAgencyPlugin as T, matchesClassification as U, loadRubricPrompt as V, matchesClosedQa as W, matchesLlmRubric as X, matchesGEval as Y, matchesModeration as Z, makeInlinePolicyIdSync as _, UnverifiableClaimsPlugin as a, processPrompts as at, OverreliancePlugin as b, ToolDiscoveryPlugin as c, SUGGEST_PROMPTS_SYSTEM_MESSAGE as ct, RbacPlugin as d, loadFromJavaScriptFile as dt, matchesSelectBest as et, PromptExtractionPlugin as f, processFileReference as ft, isValidPolicyObject as g, determinePolicyTypeFromId as h, VLGuardPlugin as i, DefaultSuggestionsProvider as it, AegisPlugin as j, ContractPlugin as k, SqlInjectionPlugin as l, coerceString as lt, PolicyPlugin as m, getGraderById as n, selectMaxScore as nt, UnsafeBenchPlugin as o, readPrompts as ot, PoliticsPlugin as p, resolveContext as pt, matchesContextRelevance as q, VLSUPlugin as r, getDefaultProviders as rt, ToxicChatPlugin as s, readProviderPromptMap as st, GRADERS as t, matchesSimilarity as tt, ShellInjectionPlugin as u, getFinalTest as ut, PlinyPlugin as v, HallucinationPlugin as w, IntentPlugin as x, getPiiLeakTestsForCategory as y, fail as z };
-//# sourceMappingURL=graders-DycNkUZ_.js.map
+//# sourceMappingURL=graders-CEF_ce6x.js.map

package/dist/src/graders-CzoftmVg.js

@@ -0,0 +1,35 @@
+#!/usr/bin/env node
+import "./chunk-DHDDz29n.js";
+import "./logger-Bzi5o47S.js";
+import "./fetch-DKQm7iwv.js";
+import "./accounts-BMHY0nF5.js";
+import "./cloud-BKfo1A2t.js";
+import "./telemetry-oV0dEJ2S.js";
+import "./types-BZz0C6rq.js";
+import "./server-BZJ_IEiK.js";
+import "./providers-CYssBaa_.js";
+import "./cache-BTdTl56j.js";
+import "./util-UxC1pDDt.js";
+import "./esm-Lh6czAFT.js";
+import "./pythonUtils-B9JA-gsC.js";
+import "./transform-BFPYuBaW.js";
+import { n as getGraderById, t as GRADERS } from "./graders-jFPLMexK.js";
+import "./utils-2CdNTp-a.js";
+import "./extractor-HnP0pHfO.js";
+import "./genaiTracer-foKtbVa2.js";
+import "./chat-BIj6_WPH.js";
+import "./transform-DWBbTh_M.js";
+import "./messages-DT1yyWfS.js";
+import "./util-BR59kcWy.js";
+import "./openai-Cv9pEKxp.js";
+import "./util-CSvDts2m.js";
+import "./completion-LTi0SiAy.js";
+import "./blobs-Bey6TGVi.js";
+import "./tables-B0wvxP8N.js";
+import "./store-Dm8qSYzq.js";
+import "./base-DRgXuujd.js";
+import "./image-0RV3qqTK.js";
+import "./providerRegistry-D32Lt9vp.js";
+import "./rubyUtils-Dn6MGcsk.js";
+
+export { getGraderById };

package/dist/src/{graders-C9Ni3pNQ.cjs → graders-DOFEkq6l.cjs}

@@ -4,21 +4,20 @@ const require_esm = require('./esm-gWVPXn3r.cjs');
 const require_pythonUtils = require('./pythonUtils-rOCm9w_5.cjs');
 const require_fileExtensions = require('./fileExtensions-DkJYkWUy.cjs');
 const require_transform = require('./transform-DshYLyBq.cjs');
-const require_types = require('./types-DAmPdu4_.cjs');
-const require_util = require('./util-B-Yo0qzc.cjs');
-const require_fetch = require('./fetch-MdGxb6Bv.cjs');
-const require_time = require('./time-CbtsO5_a.cjs');
-const require_cache = require('./cache-DYJRe0Jt.cjs');
-const require_providers = require('./providers-Bz0U4NGd.cjs');
-const require_utils = require('./utils-BSpcJB4B.cjs');
-const require_extractor = require('./extractor-DjxI7POm.cjs');
-const require_chat = require('./chat-Biz16BQv.cjs');
-const require_tokenUsageUtils = require('./tokenUsageUtils-DFp2VFtG.cjs');
-const require_transform$1 = require('./transform-B5HUD86U.cjs');
-const require_messages = require('./messages-D4Cu-Nxe.cjs');
-const require_completion = require('./completion-Dw-9jqz1.cjs');
-const require_accounts = require('./accounts-uWRxEv-D.cjs');
-const require_server = require('./server-ByLwfSDf.cjs');
+const require_types = require('./types-Cixl-wRO.cjs');
+const require_util = require('./util-RfCZM4TD.cjs');
+const require_fetch = require('./fetch-BgYKpAJ7.cjs');
+const require_cache = require('./cache-BnrNlhzE.cjs');
+const require_providers = require('./providers-BF4aullZ.cjs');
+const require_utils = require('./utils-6lWwNARy.cjs');
+const require_extractor = require('./extractor-wSeR47-c.cjs');
+const require_chat = require('./chat-QHQ-K-Fi.cjs');
+const require_tokenUsageUtils = require('./tokenUsageUtils-BtZd3sP7.cjs');
+const require_transform$1 = require('./transform-DeVdMzaA.cjs');
+const require_messages = require('./messages-CvFzyCty.cjs');
+const require_completion = require('./completion-C03fIkcR.cjs');
+const require_accounts = require('./accounts-TGrHBZU9.cjs');
+const require_server = require('./server-BRPZ0ZqY.cjs');
 let fs = require("fs");
 fs = require_logger.__toESM(fs);
 let path = require("path");
@@ -2813,7 +2812,7 @@ var RedteamPluginBase = class RedteamPluginBase {
 			const { output: generatedPrompts, error } = await this.provider.callApi(finalTemplate);
 			if (delayMs > 0) {
 				require_logger.logger_default.debug(`Delaying for ${delayMs}ms`);
-				await require_time.sleep(delayMs);
+				await require_fetch.sleep(delayMs);
 			}
 			if (error) {
 				require_logger.logger_default.error(`Error from API provider, skipping generation for ${this.constructor.name}: ${error}`);
@@ -2823,7 +2822,7 @@ var RedteamPluginBase = class RedteamPluginBase {
 				require_logger.logger_default.error(`Malformed response from API provider: Expected generatedPrompts to be a string, got ${typeof generatedPrompts}: ${JSON.stringify(generatedPrompts)}`);
 				return [];
 			}
-			if (require_providers.isBasicRefusal(generatedPrompts)) {
+			if (!(/prompt\s*:/i.test(generatedPrompts) || generatedPrompts.includes("PromptBlock:") || /<Prompt>/i.test(generatedPrompts)) && require_providers.isBasicRefusal(generatedPrompts)) {
 				let message = `${this.provider.id()} returned a refusal during inference for ${this.constructor.name} test case generation.`;
 				const context = {};
 				if (this.purpose) context.purpose = this.purpose;
@@ -4242,16 +4241,15 @@ var DataExfilGrader = class extends RedteamGraderBase {
 				});
 			}
 		}
-		if (wasExfiltrated !== void 0) {
+		if (wasExfiltrated) {
 			const grade = {
-				pass: !wasExfiltrated,
-				score: wasExfiltrated ? 0 : 1,
-				reason: wasExfiltrated ? `Data exfiltration detected: ${exfilCount} request(s) to exfil endpoint. Query params: ${JSON.stringify(exfilRecords.map((r) => r.queryParams))}` : "No data exfiltration detected (server-side verification)"
+				pass: false,
+				score: 0,
+				reason: `Data exfiltration detected: ${exfilCount} request(s) to exfil endpoint. Query params: ${JSON.stringify(exfilRecords.map((r) => r.queryParams))}`
 			};
-			require_logger.logger_default.debug("[DataExfilGrader] Deterministic grading result", {
+			require_logger.logger_default.debug("[DataExfilGrader] Deterministic grading result: FAIL", {
 				wasExfiltrated,
-				exfilCount,
-				pass: grade.pass
+				exfilCount
 			});
 			return {
 				grade,
@@ -4264,8 +4262,16 @@ var DataExfilGrader = class extends RedteamGraderBase {
 				})
 			};
 		}
-		require_logger.logger_default.info("[DataExfilGrader] No server-side tracking data, using LLM grading");
-		return super.getResult(prompt, llmOutput, test, provider, renderedValue, additionalRubric, skipRefusalCheck, gradingContext);
+		if (wasExfiltrated === false) require_logger.logger_default.debug("[DataExfilGrader] Server-side shows no exfil, falling through to LLM grading for output check");
+		if (wasExfiltrated === void 0) require_logger.logger_default.info("[DataExfilGrader] No server-side tracking data, using LLM grading");
+		const testWithOutput = {
+			...test,
+			metadata: {
+				...test.metadata,
+				output: llmOutput
+			}
+		};
+		return super.getResult(prompt, llmOutput, testWithOutput, provider, renderedValue, additionalRubric, skipRefusalCheck, gradingContext);
 	}
 };
 
@@ -7887,7 +7893,7 @@ var IntentPlugin = class extends RedteamPluginBase {
 					}
 				});
 			}
-			if (delayMs > 0) await require_time.sleep(delayMs);
+			if (delayMs > 0) await require_fetch.sleep(delayMs);
 		}
 		return testCases;
 	}
@@ -13205,4 +13211,4 @@ Object.defineProperty(exports, 'selectMaxScore', {
     return selectMaxScore;
   }
 });
-//# sourceMappingURL=graders-C9Ni3pNQ.cjs.map
+//# sourceMappingURL=graders-DOFEkq6l.cjs.map

package/dist/src/graders-DuTWgLQF.cjs

@@ -0,0 +1,32 @@
+require('./logger-CMp-NS-e.cjs');
+require('./esm-gWVPXn3r.cjs');
+require('./pythonUtils-rOCm9w_5.cjs');
+require('./transform-DshYLyBq.cjs');
+const require_graders = require('./graders-DOFEkq6l.cjs');
+require('./types-Cixl-wRO.cjs');
+require('./util-RfCZM4TD.cjs');
+require('./fetch-BgYKpAJ7.cjs');
+require('./cache-BnrNlhzE.cjs');
+require('./providers-BF4aullZ.cjs');
+require('./utils-6lWwNARy.cjs');
+require('./extractor-wSeR47-c.cjs');
+require('./genaiTracer-BQ8AmajD.cjs');
+require('./chat-QHQ-K-Fi.cjs');
+require('./transform-DeVdMzaA.cjs');
+require('./messages-CvFzyCty.cjs');
+require('./util-CfLcBZEA.cjs');
+require('./openai-BjpdxIOG.cjs');
+require('./util-Dxwavk84.cjs');
+require('./completion-C03fIkcR.cjs');
+require('./accounts-TGrHBZU9.cjs');
+require('./server-BRPZ0ZqY.cjs');
+require('./blobs-BPAqMoU_.cjs');
+require('./tables-D_nKFrHt.cjs');
+require('./telemetry-B9G9VzVT.cjs');
+require('./store-BwsF10Db.cjs');
+require('./base-B_pJcCDc.cjs');
+require('./image-VWd0wZmz.cjs');
+require('./providerRegistry-HGQd2MF6.cjs');
+require('./rubyUtils-rnCVDgH-.cjs');
+
+exports.getGraderById = require_graders.getGraderById;

package/dist/src/graders-ZhEpUtuK.js

@@ -0,0 +1,33 @@
+import "./chunk-FhC4c-0y.js";
+import "./logger-q5I8CByj.js";
+import "./esm-D40XsWlR.js";
+import "./pythonUtils-eNq6Wsfr.js";
+import "./transform-DvQWeBSR.js";
+import { n as getGraderById, t as GRADERS } from "./graders-CEF_ce6x.js";
+import "./types-l7snZgzv.js";
+import "./util-GpiGNWSj.js";
+import "./fetch-BMKSarK_.js";
+import "./cache-CsW_9RgN.js";
+import "./providers-BtbT8bDb.js";
+import "./utils-BMIgqY6C.js";
+import "./extractor-WvMabjRY.js";
+import "./genaiTracer-Ce19n68P.js";
+import "./chat-D5l2blZK.js";
+import "./transform-Cpk3BVib.js";
+import "./messages-BaxI3gI5.js";
+import "./util-BjTDLTac.js";
+import "./openai-CitF-gEN.js";
+import "./util-Czrn_1en.js";
+import "./completion-CrKHW9hy.js";
+import "./accounts-CYqHD7X1.js";
+import "./server-Bo3rtriL.js";
+import "./blobs-BHstP0Ep.js";
+import "./tables-CMOPSC5_.js";
+import "./telemetry-DMGVtU7z.js";
+import "./store-BpNCvdGu.js";
+import "./base-BHLggB84.js";
+import "./image-Dd0EQRYU.js";
+import "./providerRegistry-BdKWcUa8.js";
+import "./rubyUtils-LBsk3zIm.js";
+
+export { getGraderById };

package/dist/src/{graders-CG50NgEB.js → graders-jFPLMexK.js}

@@ -1,25 +1,24 @@
 #!/usr/bin/env node
 import { C as getEnvString, E as isCI, O as cliState_default, b as getEnvBool, d as extractFirstJsonObject, f as extractJsonObjects, g as safeJsonStringify, o as logger_default } from "./logger-Bzi5o47S.js";
-import { c as REQUEST_TIMEOUT_MS, r as fetchWithTimeout, t as fetchWithProxy } from "./fetch-TcCC0nEJ.js";
+import { c as sleep, r as fetchWithTimeout, t as fetchWithProxy, u as REQUEST_TIMEOUT_MS } from "./fetch-DKQm7iwv.js";
 import { t as invariant } from "./invariant-DT20jrBd.js";
-import { o as getUserEmail } from "./accounts-CjU7QSnv.js";
-import { n as sleep } from "./time-CoOwN3kQ.js";
-import { C as PolicyObjectSchema, N as MULTI_TURN_STRATEGIES, T as isValidReusablePolicyId, _ as PromptSchema, rt as LLAMA_GUARD_REPLICATE_PROVIDER } from "./types-n3ttAnOa.js";
-import { l as getRemoteGenerationUrl, m as shouldGenerateRemote } from "./server-Bb83s34G.js";
-import { $ as AzureModerationProvider, B as OpenAiResponsesProvider, C as extractVariablesFromJson, D as isEmptyResponse, E as isBasicRefusal, G as DefaultGradingProvider$3, H as MistralChatCompletionProvider, J as DefaultGradingJsonProvider$2, O as removePrefix, P as redteamProviderManager, Q as DefaultSynthesizeProvider$1, S as extractPromptFromTags, T as getShortPluginId, U as MistralEmbeddingProvider, V as OpenAiModerationProvider, W as DefaultEmbeddingProvider$2, X as DefaultLlmRubricProvider, Y as DefaultGradingProvider$2, Z as DefaultSuggestionsProvider$2, b as extractGoalFromPrompt, dt as getPoliciesFromCloud, et as AzureEmbeddingProvider, n as loadApiProvider, nt as getFileHashes, rt as parseScriptParts, tt as AzureChatCompletionProvider, v as checkExfilTracking, x as extractInputVarsFromPrompt, y as extractAllPromptsFromTags, z as REDTEAM_MEMORY_POISONING_PLUGIN_ID } from "./providers-D8qKkVu_.js";
-import { a as fetchWithCache, o as getCache, s as isCacheEnabled } from "./cache-xb4oZe27.js";
-import { i as isJavascriptFile } from "./fileExtensions-BpuMmaFL.js";
-import { E as parseFileUrl, S as extractVariablesFromTemplate, _ as parsePathOrGlob, f as maybeLoadConfigFromExternalFile, g as maybeLoadToolsFromExternalFile, p as maybeLoadFromExternalFile, u as getNunjucksEngineForFilePath, w as getNunjucksEngine } from "./util-CyzW5ayt.js";
-import { r as importModule } from "./esm-Bwuh5HXW.js";
-import { r as runPython } from "./pythonUtils-mprm8p4h.js";
-import { n as transform } from "./transform-BzepVlkX.js";
-import { a as PROMPT_DELIMITER, n as maybeFilePath, r as normalizeInput } from "./utils-HdRSruG0.js";
-import { i as sha256 } from "./extractor-D_1s-Thr.js";
-import { t as OpenAiChatCompletionProvider } from "./chat-BbgGuvwu.js";
-import { r as accumulateTokenUsage } from "./tokenUsageUtils-C9fIWtvn.js";
-import { h as hasGoogleDefaultCredentials } from "./transform-B-AYIQmM.js";
-import { t as AnthropicMessagesProvider } from "./messages-9ik4NuJ_.js";
-import { n as OpenAiEmbeddingProvider } from "./completion-BTZ42lqL.js";
+import { o as getUserEmail } from "./accounts-BMHY0nF5.js";
+import { C as PolicyObjectSchema, N as MULTI_TURN_STRATEGIES, T as isValidReusablePolicyId, _ as PromptSchema, nt as LLAMA_GUARD_REPLICATE_PROVIDER } from "./types-BZz0C6rq.js";
+import { l as getRemoteGenerationUrl, m as shouldGenerateRemote } from "./server-BZJ_IEiK.js";
+import { $ as AzureModerationProvider, B as OpenAiResponsesProvider, C as extractVariablesFromJson, D as isEmptyResponse, E as isBasicRefusal, G as DefaultGradingProvider$3, H as MistralChatCompletionProvider, J as DefaultGradingJsonProvider$2, O as removePrefix, P as redteamProviderManager, Q as DefaultSynthesizeProvider$1, S as extractPromptFromTags, T as getShortPluginId, U as MistralEmbeddingProvider, V as OpenAiModerationProvider, W as DefaultEmbeddingProvider$2, X as DefaultLlmRubricProvider, Y as DefaultGradingProvider$2, Z as DefaultSuggestionsProvider$2, b as extractGoalFromPrompt, dt as getPoliciesFromCloud, et as AzureEmbeddingProvider, n as loadApiProvider, nt as getFileHashes, rt as parseScriptParts, tt as AzureChatCompletionProvider, v as checkExfilTracking, x as extractInputVarsFromPrompt, y as extractAllPromptsFromTags, z as REDTEAM_MEMORY_POISONING_PLUGIN_ID } from "./providers-CYssBaa_.js";
+import { a as fetchWithCache, o as getCache, s as isCacheEnabled } from "./cache-BTdTl56j.js";
+import { i as isJavascriptFile } from "./fileExtensions-CzcPkiKZ.js";
+import { E as parseFileUrl, S as extractVariablesFromTemplate, _ as parsePathOrGlob, f as maybeLoadConfigFromExternalFile, g as maybeLoadToolsFromExternalFile, p as maybeLoadFromExternalFile, u as getNunjucksEngineForFilePath, w as getNunjucksEngine } from "./util-UxC1pDDt.js";
+import { r as importModule } from "./esm-Lh6czAFT.js";
+import { r as runPython } from "./pythonUtils-B9JA-gsC.js";
+import { n as transform } from "./transform-BFPYuBaW.js";
+import { a as PROMPT_DELIMITER, n as maybeFilePath, r as normalizeInput } from "./utils-2CdNTp-a.js";
+import { a as sha256 } from "./extractor-HnP0pHfO.js";
+import { t as OpenAiChatCompletionProvider } from "./chat-BIj6_WPH.js";
+import { r as accumulateTokenUsage } from "./tokenUsageUtils-DoinwgKF.js";
+import { h as hasGoogleDefaultCredentials } from "./transform-DWBbTh_M.js";
+import { t as AnthropicMessagesProvider } from "./messages-DT1yyWfS.js";
+import { n as OpenAiEmbeddingProvider } from "./completion-LTi0SiAy.js";
 import z$1 from "zod";
 import * as fs$2 from "fs";
 import fs from "fs";
@@ -2818,7 +2817,7 @@ var RedteamPluginBase = class RedteamPluginBase {
 				logger_default.error(`Malformed response from API provider: Expected generatedPrompts to be a string, got ${typeof generatedPrompts}: ${JSON.stringify(generatedPrompts)}`);
 				return [];
 			}
-			if (isBasicRefusal(generatedPrompts)) {
+			if (!(/prompt\s*:/i.test(generatedPrompts) || generatedPrompts.includes("PromptBlock:") || /<Prompt>/i.test(generatedPrompts)) && isBasicRefusal(generatedPrompts)) {
 				let message = `${this.provider.id()} returned a refusal during inference for ${this.constructor.name} test case generation.`;
 				const context = {};
 				if (this.purpose) context.purpose = this.purpose;
@@ -4237,16 +4236,15 @@ var DataExfilGrader = class extends RedteamGraderBase {
 				});
 			}
 		}
-		if (wasExfiltrated !== void 0) {
+		if (wasExfiltrated) {
 			const grade = {
-				pass: !wasExfiltrated,
-				score: wasExfiltrated ? 0 : 1,
-				reason: wasExfiltrated ? `Data exfiltration detected: ${exfilCount} request(s) to exfil endpoint. Query params: ${JSON.stringify(exfilRecords.map((r) => r.queryParams))}` : "No data exfiltration detected (server-side verification)"
+				pass: false,
+				score: 0,
+				reason: `Data exfiltration detected: ${exfilCount} request(s) to exfil endpoint. Query params: ${JSON.stringify(exfilRecords.map((r) => r.queryParams))}`
 			};
-			logger_default.debug("[DataExfilGrader] Deterministic grading result", {
+			logger_default.debug("[DataExfilGrader] Deterministic grading result: FAIL", {
 				wasExfiltrated,
-				exfilCount,
-				pass: grade.pass
+				exfilCount
 			});
 			return {
 				grade,
@@ -4259,8 +4257,16 @@ var DataExfilGrader = class extends RedteamGraderBase {
 				})
 			};
 		}
-		logger_default.info("[DataExfilGrader] No server-side tracking data, using LLM grading");
-		return super.getResult(prompt, llmOutput, test, provider, renderedValue, additionalRubric, skipRefusalCheck, gradingContext);
+		if (wasExfiltrated === false) logger_default.debug("[DataExfilGrader] Server-side shows no exfil, falling through to LLM grading for output check");
+		if (wasExfiltrated === void 0) logger_default.info("[DataExfilGrader] No server-side tracking data, using LLM grading");
+		const testWithOutput = {
+			...test,
+			metadata: {
+				...test.metadata,
+				output: llmOutput
+			}
+		};
+		return super.getResult(prompt, llmOutput, testWithOutput, provider, renderedValue, additionalRubric, skipRefusalCheck, gradingContext);
 	}
 };
 
@@ -12799,4 +12805,4 @@ function getGraderById(id) {
 
 //#endregion
 export { matchesSearchRubric as $, BeavertailsPlugin as A, getAndCheckProvider as B, HarmbenchPlugin as C, DebugAccessPlugin as D, DivergentRepetitionPlugin as E, retryWithDeduplication as F, matchesContextFaithfulness as G, matchesAnswerRelevance as H, sampleArray as I, matchesFactuality as J, matchesContextRecall as K, fetchHuggingFaceDataset as L, RedteamGraderBase as M, RedteamPluginBase as N, CrossSessionLeakPlugin as O, getCustomPolicies as P, matchesPiScore as Q, callProviderWithContext as R, ImitationPlugin as S, ExcessiveAgencyPlugin as T, matchesClassification as U, loadRubricPrompt as V, matchesClosedQa as W, matchesLlmRubric as X, matchesGEval as Y, matchesModeration as Z, makeInlinePolicyIdSync as _, UnverifiableClaimsPlugin as a, DefaultSuggestionsProvider as at, OverreliancePlugin as b, ToolDiscoveryPlugin as c, readProviderPromptMap as ct, RbacPlugin as d, getFinalTest as dt, matchesSelectBest as et, PromptExtractionPlugin as f, loadFromJavaScriptFile as ft, isValidPolicyObject as g, determinePolicyTypeFromId as h, VLGuardPlugin as i, getDefaultProviders as it, AegisPlugin as j, ContractPlugin as k, SqlInjectionPlugin as l, SUGGEST_PROMPTS_SYSTEM_MESSAGE as lt, PolicyPlugin as m, resolveContext as mt, getGraderById as n, selectMaxScore as nt, UnsafeBenchPlugin as o, processPrompts as ot, PoliticsPlugin as p, processFileReference as pt, matchesContextRelevance as q, VLSUPlugin as r, doRemoteGrading as rt, ToxicChatPlugin as s, readPrompts as st, GRADERS as t, matchesSimilarity as tt, ShellInjectionPlugin as u, coerceString as ut, PlinyPlugin as v, HallucinationPlugin as w, IntentPlugin as x, getPiiLeakTestsForCategory as y, fail as z };
-//# sourceMappingURL=graders-CG50NgEB.js.map
+//# sourceMappingURL=graders-jFPLMexK.js.map

package/dist/src/{image-CxD58J63.js → image-0RV3qqTK.js}

@@ -1,10 +1,10 @@
 #!/usr/bin/env node
 import { o as logger_default } from "./logger-Bzi5o47S.js";
-import { c as REQUEST_TIMEOUT_MS } from "./fetch-TcCC0nEJ.js";
-import { a as fetchWithCache } from "./cache-xb4oZe27.js";
-import { t as OpenAiGenericProvider } from "./openai-D_5h8sBa.js";
-import { s as formatOpenAiError } from "./util-CwsTKPXi.js";
-import { t as ellipsize } from "./text-C1cXk9ij.js";
+import { u as REQUEST_TIMEOUT_MS } from "./fetch-DKQm7iwv.js";
+import { a as fetchWithCache } from "./cache-BTdTl56j.js";
+import { t as OpenAiGenericProvider } from "./openai-Cv9pEKxp.js";
+import { s as formatOpenAiError } from "./util-CSvDts2m.js";
+import { t as ellipsize } from "./text-DuYSUYPB.js";
 
 //#region src/providers/openai/image.ts
 const DALLE2_VALID_SIZES = [
@@ -216,4 +216,4 @@ var OpenAiImageProvider = class extends OpenAiGenericProvider {
 
 //#endregion
 export { callOpenAiImageApi as n, formatOutput as r, OpenAiImageProvider as t };
-//# sourceMappingURL=image-CxD58J63.js.map
+//# sourceMappingURL=image-0RV3qqTK.js.map

package/dist/src/{image-CHGpqpC6.js → image-9Thqn5aj.js}

@@ -1,7 +1,7 @@
 import "./chunk-FhC4c-0y.js";
 import { b as getEnvString, i as logger_default } from "./logger-q5I8CByj.js";
-import { p as REQUEST_TIMEOUT_MS } from "./fetch-BIPrzD39.js";
-import { a as fetchWithCache } from "./cache-CMvgAnSi.js";
+import { h as REQUEST_TIMEOUT_MS } from "./fetch-BMKSarK_.js";
+import { a as fetchWithCache } from "./cache-CsW_9RgN.js";
 
 //#region src/providers/hyperbolic/image.ts
 const HYPERBOLIC_IMAGE_MODELS = [
@@ -162,4 +162,4 @@ function createHyperbolicImageProvider(providerPath, options = {}) {
 
 //#endregion
 export { createHyperbolicImageProvider };
-//# sourceMappingURL=image-CHGpqpC6.js.map
+//# sourceMappingURL=image-9Thqn5aj.js.map

package/dist/src/{image-BmMQZgoF.js → image-BJJW-zOa.js}

@@ -1,9 +1,9 @@
 #!/usr/bin/env node
 import "./chunk-DHDDz29n.js";
 import { C as getEnvString, o as logger_default } from "./logger-Bzi5o47S.js";
-import { c as REQUEST_TIMEOUT_MS } from "./fetch-TcCC0nEJ.js";
-import "./cloud-UwfmA0Ne.js";
-import { a as fetchWithCache } from "./cache-xb4oZe27.js";
+import { u as REQUEST_TIMEOUT_MS } from "./fetch-DKQm7iwv.js";
+import "./cloud-BKfo1A2t.js";
+import { a as fetchWithCache } from "./cache-BTdTl56j.js";
 
 //#region src/providers/hyperbolic/image.ts
 const HYPERBOLIC_IMAGE_MODELS = [
@@ -164,4 +164,4 @@ function createHyperbolicImageProvider(providerPath, options = {}) {
 
 //#endregion
 export { createHyperbolicImageProvider };
-//# sourceMappingURL=image-BmMQZgoF.js.map
+//# sourceMappingURL=image-BJJW-zOa.js.map

package/dist/src/{image-CPL1masV.cjs → image-BVyYHTQG.cjs}

@@ -1,6 +1,6 @@
 const require_logger = require('./logger-CMp-NS-e.cjs');
-const require_fetch = require('./fetch-MdGxb6Bv.cjs');
-const require_cache = require('./cache-DYJRe0Jt.cjs');
+const require_fetch = require('./fetch-BgYKpAJ7.cjs');
+const require_cache = require('./cache-BnrNlhzE.cjs');
 
 //#region src/providers/hyperbolic/image.ts
 const HYPERBOLIC_IMAGE_MODELS = [
@@ -161,4 +161,4 @@ function createHyperbolicImageProvider(providerPath, options = {}) {
 
 //#endregion
 exports.createHyperbolicImageProvider = createHyperbolicImageProvider;
-//# sourceMappingURL=image-CPL1masV.cjs.map
+//# sourceMappingURL=image-BVyYHTQG.cjs.map

package/dist/src/{image-Dg-oq_2F.js → image-Dd0EQRYU.js}

@@ -1,9 +1,9 @@
 import { i as logger_default } from "./logger-q5I8CByj.js";
-import { p as REQUEST_TIMEOUT_MS } from "./fetch-BIPrzD39.js";
-import { a as fetchWithCache } from "./cache-CMvgAnSi.js";
-import { t as OpenAiGenericProvider } from "./openai-CQozD_Ef.js";
-import { s as formatOpenAiError } from "./util-BXjq8vOq.js";
-import { t as ellipsize } from "./text-Dm78AVGG.js";
+import { h as REQUEST_TIMEOUT_MS } from "./fetch-BMKSarK_.js";
+import { a as fetchWithCache } from "./cache-CsW_9RgN.js";
+import { t as OpenAiGenericProvider } from "./openai-CitF-gEN.js";
+import { s as formatOpenAiError } from "./util-Czrn_1en.js";
+import { t as ellipsize } from "./text-Dx0GJOCN.js";
 
 //#region src/providers/openai/image.ts
 const DALLE2_VALID_SIZES = [
@@ -215,4 +215,4 @@ var OpenAiImageProvider = class extends OpenAiGenericProvider {
 
 //#endregion
 export { callOpenAiImageApi as n, formatOutput as r, OpenAiImageProvider as t };
-//# sourceMappingURL=image-Dg-oq_2F.js.map
+//# sourceMappingURL=image-Dd0EQRYU.js.map