promptfoo 0.121.5 → 0.121.8

package/dist/src/promptfoo-ClChwT74.cjs

@@ -0,0 +1,292 @@
+const require_rolldown_runtime = require("./rolldown-runtime-D_mwlA32.cjs");
+const require_logger = require("./logger-cfNpzI4o.cjs");
+const require_types = require("./types-CxJvaY2S.cjs");
+const require_fetch = require("./fetch-Cfm4EuOB.cjs");
+const require_version = require("./version-VzUqOBZk.cjs");
+const require_accounts = require("./accounts-BVz5gHLK.cjs");
+const require_remoteGeneration = require("./remoteGeneration-BpIYlb_O.cjs");
+let dedent = require("dedent");
+dedent = require_rolldown_runtime.__toESM(dedent, 1);
+//#region src/redteam/remoteMaterialization.ts
+const REMOTE_MATERIALIZATION_CONTEXT_VAR = "__promptfooRemoteMaterialization";
+function filterMaterializedVarsToInputs(vars, inputs, options) {
+	if (!inputs || Object.keys(inputs).length === 0) return vars;
+	return Object.fromEntries(Object.entries(vars).filter(([key]) => {
+		if (!Object.prototype.hasOwnProperty.call(inputs, key)) return false;
+		if (!options?.textOnly) return true;
+		return require_types.getInputType(inputs[key]) === "text";
+	}));
+}
+function buildRemoteMaterializedInputVariables(response, fallbackVars, inputs) {
+	const filteredFallbackVars = filterMaterializedVarsToInputs(fallbackVars, inputs, { textOnly: true });
+	const filteredMaterializedVars = filterMaterializedVarsToInputs(response.materializedVars ?? {}, inputs);
+	return {
+		...response.inputMaterialization ? { metadata: response.inputMaterialization } : {},
+		vars: {
+			...filteredFallbackVars,
+			...filteredMaterializedVars
+		}
+	};
+}
+function getRemoteMaterializationHost() {
+	const remoteUrl = require_remoteGeneration.getRemoteGenerationUrl();
+	try {
+		return new URL(remoteUrl).origin;
+	} catch {
+		return remoteUrl;
+	}
+}
+function buildRemoteMaterializationContextVars(context) {
+	return { [REMOTE_MATERIALIZATION_CONTEXT_VAR]: context };
+}
+function getRemoteMaterializationContextFromVars(vars) {
+	const raw = vars?.[REMOTE_MATERIALIZATION_CONTEXT_VAR];
+	if (!raw || typeof raw !== "object" || Array.isArray(raw)) return;
+	return raw;
+}
+function requiresRemoteMaterialization(inputs) {
+	return Boolean(inputs && Object.keys(inputs).length > 0);
+}
+function getRemoteMaterializationUpgradeError(operation) {
+	return `${operation} requires remote multi-input materialization support from a newer Promptfoo server. Current remote endpoint: ${getRemoteMaterializationHost()}. Upgrade the Promptfoo server you are connected to and try again.`;
+}
+function isRemoteMaterializationUpgradeError(error) {
+	return error instanceof Error && error.message.includes("requires remote multi-input materialization support from a newer");
+}
+function assertRemoteMaterializationHandled(response, operation) {
+	if (response?.materializationHandled === true) return;
+	throw new Error(getRemoteMaterializationUpgradeError(operation));
+}
+//#endregion
+//#region src/providers/promptfoo.ts
+/**
+* Provider for generating harmful/adversarial content using Promptfoo's unaligned models.
+* Used by red team plugins to generate test cases for harmful content categories.
+*/
+var PromptfooHarmfulCompletionProvider = class {
+	harmCategory;
+	n;
+	purpose;
+	config;
+	constructor(options) {
+		this.harmCategory = options.harmCategory;
+		this.n = options.n;
+		this.purpose = options.purpose;
+		this.config = options.config;
+	}
+	id() {
+		return `promptfoo:redteam:${this.harmCategory}`;
+	}
+	toString() {
+		return `[Promptfoo Harmful Completion Provider ${this.purpose} - ${this.harmCategory}]`;
+	}
+	async callApi(_prompt, _context, callApiOptions) {
+		if (require_remoteGeneration.neverGenerateRemote()) return { error: dedent.default`
+          Remote generation is disabled. Harmful content generation requires Promptfoo's unaligned models.
+
+          To enable:
+          - Remove PROMPTFOO_DISABLE_REMOTE_GENERATION (or PROMPTFOO_DISABLE_REDTEAM_REMOTE_GENERATION)
+          - Or configure an alternative unaligned model provider
+
+          Learn more: https://www.promptfoo.dev/docs/red-team/configuration#remote-generation
+        ` };
+		const body = {
+			email: require_accounts.getUserEmail(),
+			harmCategory: this.harmCategory,
+			n: this.n,
+			purpose: this.purpose,
+			version: require_version.VERSION,
+			config: this.config
+		};
+		try {
+			require_logger.logger.debug(`[HarmfulCompletionProvider] Calling generate harmful API (${require_remoteGeneration.getRemoteGenerationUrlForUnaligned()}) with body: ${JSON.stringify(body)}`);
+			const response = await require_fetch.fetchWithRetries(require_remoteGeneration.getRemoteGenerationUrlForUnaligned(), {
+				method: "POST",
+				headers: { "Content-Type": "application/json" },
+				body: JSON.stringify(body),
+				...callApiOptions?.abortSignal && { signal: callApiOptions.abortSignal }
+			}, 58e4, 2);
+			if (!response.ok) throw new Error(`API call failed with status ${response.status}: ${await response.text()}`);
+			const data = await response.json();
+			return { output: (Array.isArray(data.output) ? data.output : [data.output]).filter((item) => typeof item === "string" && item.length > 0) };
+		} catch (err) {
+			if (err instanceof Error && err.name === "AbortError") throw err;
+			require_logger.logger.info(`[HarmfulCompletionProvider] ${err}`);
+			return { error: `[HarmfulCompletionProvider] ${err}` };
+		}
+	}
+};
+/**
+* Provider for red team adversarial strategies using Promptfoo's task-specific models.
+* Supports multi-turn attack strategies like crescendo, goat, and iterative attacks.
+*/
+var PromptfooChatCompletionProvider = class {
+	options;
+	constructor(options) {
+		this.options = options;
+	}
+	id() {
+		return this.options.id || "promptfoo:chatcompletion";
+	}
+	toString() {
+		return `[Promptfoo Chat Completion Provider]`;
+	}
+	async callApi(prompt, context, callApiOptions) {
+		if (require_remoteGeneration.neverGenerateRemote()) return { error: dedent.default`
+          Remote generation is disabled. This red team strategy requires Promptfoo's task-specific models.
+
+          To enable:
+          - Remove PROMPTFOO_DISABLE_REMOTE_GENERATION (or PROMPTFOO_DISABLE_REDTEAM_REMOTE_GENERATION)
+          - Or provide OPENAI_API_KEY for local generation (may have lower quality)
+
+          Learn more: https://www.promptfoo.dev/docs/red-team/configuration#remote-generation
+        ` };
+		const materializationContext = getRemoteMaterializationContextFromVars(context?.vars);
+		const body = {
+			jsonOnly: this.options.jsonOnly,
+			preferSmallModel: this.options.preferSmallModel,
+			prompt,
+			step: context?.prompt.label,
+			task: this.options.task,
+			email: require_accounts.getUserEmail(),
+			...this.options.inputs && { inputs: this.options.inputs },
+			...materializationContext ? { materializationContext } : {}
+		};
+		try {
+			const response = await require_fetch.fetchWithRetries(require_remoteGeneration.getRemoteGenerationUrl(), {
+				method: "POST",
+				headers: { "Content-Type": "application/json" },
+				body: JSON.stringify(body),
+				...callApiOptions?.abortSignal && { signal: callApiOptions.abortSignal }
+			}, require_fetch.REQUEST_TIMEOUT_MS);
+			const data = await response.json();
+			if (!data.result) {
+				require_logger.logger.debug(`Error from promptfoo completion provider. Status: ${response.status} ${response.statusText} ${JSON.stringify(data)} `);
+				return { error: "LLM did not return a result, likely refusal" };
+			}
+			return {
+				inputMaterialization: data.inputMaterialization,
+				materializationHandled: data.materializationHandled,
+				materializedVars: data.materializedVars,
+				output: data.result,
+				tokenUsage: data.tokenUsage
+			};
+		} catch (err) {
+			if (err instanceof Error && err.name === "AbortError") throw err;
+			return { error: `API call error: ${String(err)}` };
+		}
+	}
+};
+const REDTEAM_SIMULATED_USER_TASK_ID = "mischievous-user-redteam";
+/**
+* Provider for simulating realistic user conversations using Promptfoo's conversation models.
+* Supports both regular simulated users and adversarial red team users.
+*/
+var PromptfooSimulatedUserProvider = class {
+	options;
+	taskId;
+	constructor(options = {}, taskId) {
+		this.options = options;
+		this.taskId = taskId;
+	}
+	id() {
+		return this.options.id || "promptfoo:agent";
+	}
+	toString() {
+		return "[Promptfoo Agent Provider]";
+	}
+	async callApi(prompt, _context, callApiOptions) {
+		const isRedteamTask = this.taskId === REDTEAM_SIMULATED_USER_TASK_ID;
+		if (isRedteamTask ? require_remoteGeneration.neverGenerateRemote() : require_remoteGeneration.neverGenerateRemoteForRegularEvals()) return { error: dedent.default`
+          Remote generation is disabled.
+
+          SimulatedUser requires Promptfoo's conversation simulation models.
+
+          To enable, remove ${isRedteamTask ? "PROMPTFOO_DISABLE_REMOTE_GENERATION or PROMPTFOO_DISABLE_REDTEAM_REMOTE_GENERATION" : "PROMPTFOO_DISABLE_REMOTE_GENERATION"}
+
+          Learn more: ${isRedteamTask ? "https://www.promptfoo.dev/docs/red-team/configuration#remote-generation" : "https://www.promptfoo.dev/docs/providers/simulated-user#remote-generation"}
+        ` };
+		const messages = JSON.parse(prompt);
+		const body = {
+			task: this.taskId,
+			instructions: this.options.instructions,
+			history: messages,
+			email: require_accounts.getUserEmail(),
+			version: require_version.VERSION
+		};
+		try {
+			const response = await require_fetch.fetchWithRetries(require_remoteGeneration.getRemoteGenerationUrl(), {
+				method: "POST",
+				headers: { "Content-Type": "application/json" },
+				body: JSON.stringify(body),
+				...callApiOptions?.abortSignal && { signal: callApiOptions.abortSignal }
+			}, require_fetch.REQUEST_TIMEOUT_MS);
+			if (!response.ok) throw new Error(`API call failed with status ${response.status}: ${await response.text()}`);
+			const data = await response.json();
+			return {
+				output: data.result,
+				tokenUsage: data.tokenUsage
+			};
+		} catch (err) {
+			if (err instanceof Error && err.name === "AbortError") throw err;
+			return { error: `API call error: ${String(err)}` };
+		}
+	}
+};
+//#endregion
+Object.defineProperty(exports, "PromptfooChatCompletionProvider", {
+	enumerable: true,
+	get: function() {
+		return PromptfooChatCompletionProvider;
+	}
+});
+Object.defineProperty(exports, "PromptfooHarmfulCompletionProvider", {
+	enumerable: true,
+	get: function() {
+		return PromptfooHarmfulCompletionProvider;
+	}
+});
+Object.defineProperty(exports, "PromptfooSimulatedUserProvider", {
+	enumerable: true,
+	get: function() {
+		return PromptfooSimulatedUserProvider;
+	}
+});
+Object.defineProperty(exports, "REDTEAM_SIMULATED_USER_TASK_ID", {
+	enumerable: true,
+	get: function() {
+		return REDTEAM_SIMULATED_USER_TASK_ID;
+	}
+});
+Object.defineProperty(exports, "assertRemoteMaterializationHandled", {
+	enumerable: true,
+	get: function() {
+		return assertRemoteMaterializationHandled;
+	}
+});
+Object.defineProperty(exports, "buildRemoteMaterializationContextVars", {
+	enumerable: true,
+	get: function() {
+		return buildRemoteMaterializationContextVars;
+	}
+});
+Object.defineProperty(exports, "buildRemoteMaterializedInputVariables", {
+	enumerable: true,
+	get: function() {
+		return buildRemoteMaterializedInputVariables;
+	}
+});
+Object.defineProperty(exports, "isRemoteMaterializationUpgradeError", {
+	enumerable: true,
+	get: function() {
+		return isRemoteMaterializationUpgradeError;
+	}
+});
+Object.defineProperty(exports, "requiresRemoteMaterialization", {
+	enumerable: true,
+	get: function() {
+		return requiresRemoteMaterialization;
+	}
+});
+
+//# sourceMappingURL=promptfoo-ClChwT74.cjs.map

package/dist/src/promptfoo-CqEpj6Sr.js

@@ -0,0 +1,237 @@
+import { a as logger } from "./logger-Ct2S6Yx-.js";
+import { Vt as getInputType } from "./types-D6glLbdF.js";
+import { n as fetchWithRetries, p as REQUEST_TIMEOUT_MS } from "./fetch-Doks14zQ.js";
+import { n as VERSION } from "./version-qVEN5qCm.js";
+import { i as getUserEmail } from "./accounts-DAv_0iE7.js";
+import { c as neverGenerateRemoteForRegularEvals, i as getRemoteGenerationUrlForUnaligned, r as getRemoteGenerationUrl, s as neverGenerateRemote } from "./remoteGeneration-CJC3E0aW.js";
+import dedent from "dedent";
+//#region src/redteam/remoteMaterialization.ts
+const REMOTE_MATERIALIZATION_CONTEXT_VAR = "__promptfooRemoteMaterialization";
+function filterMaterializedVarsToInputs(vars, inputs, options) {
+	if (!inputs || Object.keys(inputs).length === 0) return vars;
+	return Object.fromEntries(Object.entries(vars).filter(([key]) => {
+		if (!Object.prototype.hasOwnProperty.call(inputs, key)) return false;
+		if (!options?.textOnly) return true;
+		return getInputType(inputs[key]) === "text";
+	}));
+}
+function buildRemoteMaterializedInputVariables(response, fallbackVars, inputs) {
+	const filteredFallbackVars = filterMaterializedVarsToInputs(fallbackVars, inputs, { textOnly: true });
+	const filteredMaterializedVars = filterMaterializedVarsToInputs(response.materializedVars ?? {}, inputs);
+	return {
+		...response.inputMaterialization ? { metadata: response.inputMaterialization } : {},
+		vars: {
+			...filteredFallbackVars,
+			...filteredMaterializedVars
+		}
+	};
+}
+function getRemoteMaterializationHost() {
+	const remoteUrl = getRemoteGenerationUrl();
+	try {
+		return new URL(remoteUrl).origin;
+	} catch {
+		return remoteUrl;
+	}
+}
+function buildRemoteMaterializationContextVars(context) {
+	return { [REMOTE_MATERIALIZATION_CONTEXT_VAR]: context };
+}
+function getRemoteMaterializationContextFromVars(vars) {
+	const raw = vars?.[REMOTE_MATERIALIZATION_CONTEXT_VAR];
+	if (!raw || typeof raw !== "object" || Array.isArray(raw)) return;
+	return raw;
+}
+function requiresRemoteMaterialization(inputs) {
+	return Boolean(inputs && Object.keys(inputs).length > 0);
+}
+function getRemoteMaterializationUpgradeError(operation) {
+	return `${operation} requires remote multi-input materialization support from a newer Promptfoo server. Current remote endpoint: ${getRemoteMaterializationHost()}. Upgrade the Promptfoo server you are connected to and try again.`;
+}
+function isRemoteMaterializationUpgradeError(error) {
+	return error instanceof Error && error.message.includes("requires remote multi-input materialization support from a newer");
+}
+function assertRemoteMaterializationHandled(response, operation) {
+	if (response?.materializationHandled === true) return;
+	throw new Error(getRemoteMaterializationUpgradeError(operation));
+}
+//#endregion
+//#region src/providers/promptfoo.ts
+/**
+* Provider for generating harmful/adversarial content using Promptfoo's unaligned models.
+* Used by red team plugins to generate test cases for harmful content categories.
+*/
+var PromptfooHarmfulCompletionProvider = class {
+	harmCategory;
+	n;
+	purpose;
+	config;
+	constructor(options) {
+		this.harmCategory = options.harmCategory;
+		this.n = options.n;
+		this.purpose = options.purpose;
+		this.config = options.config;
+	}
+	id() {
+		return `promptfoo:redteam:${this.harmCategory}`;
+	}
+	toString() {
+		return `[Promptfoo Harmful Completion Provider ${this.purpose} - ${this.harmCategory}]`;
+	}
+	async callApi(_prompt, _context, callApiOptions) {
+		if (neverGenerateRemote()) return { error: dedent`
+          Remote generation is disabled. Harmful content generation requires Promptfoo's unaligned models.
+
+          To enable:
+          - Remove PROMPTFOO_DISABLE_REMOTE_GENERATION (or PROMPTFOO_DISABLE_REDTEAM_REMOTE_GENERATION)
+          - Or configure an alternative unaligned model provider
+
+          Learn more: https://www.promptfoo.dev/docs/red-team/configuration#remote-generation
+        ` };
+		const body = {
+			email: getUserEmail(),
+			harmCategory: this.harmCategory,
+			n: this.n,
+			purpose: this.purpose,
+			version: VERSION,
+			config: this.config
+		};
+		try {
+			logger.debug(`[HarmfulCompletionProvider] Calling generate harmful API (${getRemoteGenerationUrlForUnaligned()}) with body: ${JSON.stringify(body)}`);
+			const response = await fetchWithRetries(getRemoteGenerationUrlForUnaligned(), {
+				method: "POST",
+				headers: { "Content-Type": "application/json" },
+				body: JSON.stringify(body),
+				...callApiOptions?.abortSignal && { signal: callApiOptions.abortSignal }
+			}, 58e4, 2);
+			if (!response.ok) throw new Error(`API call failed with status ${response.status}: ${await response.text()}`);
+			const data = await response.json();
+			return { output: (Array.isArray(data.output) ? data.output : [data.output]).filter((item) => typeof item === "string" && item.length > 0) };
+		} catch (err) {
+			if (err instanceof Error && err.name === "AbortError") throw err;
+			logger.info(`[HarmfulCompletionProvider] ${err}`);
+			return { error: `[HarmfulCompletionProvider] ${err}` };
+		}
+	}
+};
+/**
+* Provider for red team adversarial strategies using Promptfoo's task-specific models.
+* Supports multi-turn attack strategies like crescendo, goat, and iterative attacks.
+*/
+var PromptfooChatCompletionProvider = class {
+	options;
+	constructor(options) {
+		this.options = options;
+	}
+	id() {
+		return this.options.id || "promptfoo:chatcompletion";
+	}
+	toString() {
+		return `[Promptfoo Chat Completion Provider]`;
+	}
+	async callApi(prompt, context, callApiOptions) {
+		if (neverGenerateRemote()) return { error: dedent`
+          Remote generation is disabled. This red team strategy requires Promptfoo's task-specific models.
+
+          To enable:
+          - Remove PROMPTFOO_DISABLE_REMOTE_GENERATION (or PROMPTFOO_DISABLE_REDTEAM_REMOTE_GENERATION)
+          - Or provide OPENAI_API_KEY for local generation (may have lower quality)
+
+          Learn more: https://www.promptfoo.dev/docs/red-team/configuration#remote-generation
+        ` };
+		const materializationContext = getRemoteMaterializationContextFromVars(context?.vars);
+		const body = {
+			jsonOnly: this.options.jsonOnly,
+			preferSmallModel: this.options.preferSmallModel,
+			prompt,
+			step: context?.prompt.label,
+			task: this.options.task,
+			email: getUserEmail(),
+			...this.options.inputs && { inputs: this.options.inputs },
+			...materializationContext ? { materializationContext } : {}
+		};
+		try {
+			const response = await fetchWithRetries(getRemoteGenerationUrl(), {
+				method: "POST",
+				headers: { "Content-Type": "application/json" },
+				body: JSON.stringify(body),
+				...callApiOptions?.abortSignal && { signal: callApiOptions.abortSignal }
+			}, REQUEST_TIMEOUT_MS);
+			const data = await response.json();
+			if (!data.result) {
+				logger.debug(`Error from promptfoo completion provider. Status: ${response.status} ${response.statusText} ${JSON.stringify(data)} `);
+				return { error: "LLM did not return a result, likely refusal" };
+			}
+			return {
+				inputMaterialization: data.inputMaterialization,
+				materializationHandled: data.materializationHandled,
+				materializedVars: data.materializedVars,
+				output: data.result,
+				tokenUsage: data.tokenUsage
+			};
+		} catch (err) {
+			if (err instanceof Error && err.name === "AbortError") throw err;
+			return { error: `API call error: ${String(err)}` };
+		}
+	}
+};
+const REDTEAM_SIMULATED_USER_TASK_ID = "mischievous-user-redteam";
+/**
+* Provider for simulating realistic user conversations using Promptfoo's conversation models.
+* Supports both regular simulated users and adversarial red team users.
+*/
+var PromptfooSimulatedUserProvider = class {
+	options;
+	taskId;
+	constructor(options = {}, taskId) {
+		this.options = options;
+		this.taskId = taskId;
+	}
+	id() {
+		return this.options.id || "promptfoo:agent";
+	}
+	toString() {
+		return "[Promptfoo Agent Provider]";
+	}
+	async callApi(prompt, _context, callApiOptions) {
+		const isRedteamTask = this.taskId === REDTEAM_SIMULATED_USER_TASK_ID;
+		if (isRedteamTask ? neverGenerateRemote() : neverGenerateRemoteForRegularEvals()) return { error: dedent`
+          Remote generation is disabled.
+
+          SimulatedUser requires Promptfoo's conversation simulation models.
+
+          To enable, remove ${isRedteamTask ? "PROMPTFOO_DISABLE_REMOTE_GENERATION or PROMPTFOO_DISABLE_REDTEAM_REMOTE_GENERATION" : "PROMPTFOO_DISABLE_REMOTE_GENERATION"}
+
+          Learn more: ${isRedteamTask ? "https://www.promptfoo.dev/docs/red-team/configuration#remote-generation" : "https://www.promptfoo.dev/docs/providers/simulated-user#remote-generation"}
+        ` };
+		const messages = JSON.parse(prompt);
+		const body = {
+			task: this.taskId,
+			instructions: this.options.instructions,
+			history: messages,
+			email: getUserEmail(),
+			version: VERSION
+		};
+		try {
+			const response = await fetchWithRetries(getRemoteGenerationUrl(), {
+				method: "POST",
+				headers: { "Content-Type": "application/json" },
+				body: JSON.stringify(body),
+				...callApiOptions?.abortSignal && { signal: callApiOptions.abortSignal }
+			}, REQUEST_TIMEOUT_MS);
+			if (!response.ok) throw new Error(`API call failed with status ${response.status}: ${await response.text()}`);
+			const data = await response.json();
+			return {
+				output: data.result,
+				tokenUsage: data.tokenUsage
+			};
+		} catch (err) {
+			if (err instanceof Error && err.name === "AbortError") throw err;
+			return { error: `API call error: ${String(err)}` };
+		}
+	}
+};
+//#endregion
+export { assertRemoteMaterializationHandled as a, isRemoteMaterializationUpgradeError as c, REDTEAM_SIMULATED_USER_TASK_ID as i, requiresRemoteMaterialization as l, PromptfooHarmfulCompletionProvider as n, buildRemoteMaterializationContextVars as o, PromptfooSimulatedUserProvider as r, buildRemoteMaterializedInputVariables as s, PromptfooChatCompletionProvider as t };
+
+//# sourceMappingURL=promptfoo-CqEpj6Sr.js.map