promptfoo 0.121.5 → 0.121.7

package/dist/src/simulatedUser-p3tACcmw.js

@@ -0,0 +1,223 @@
+#!/usr/bin/env node
+import { s as logger } from "./logger-BbY6ypFL.js";
+import { f as sleep } from "./fetch-DXUnXkVU.js";
+import { t as invariant } from "./invariant-B2Rf6avk.js";
+import { o as getNunjucksEngine } from "./render-CSP99NLm.js";
+import { j as maybeLoadConfigFromExternalFile } from "./util-DURocbYR.js";
+import { a as createEmptyTokenUsage, n as accumulateResponseTokenUsage } from "./tokenUsageUtils-CDet74yk.js";
+import { o as getSessionId } from "./util-kDURhgJW.js";
+import { r as PromptfooSimulatedUserProvider } from "./promptfoo-Rjp-MeBb.js";
+//#region src/providers/simulatedUser.ts
+/**
+* TODO(Will): Ideally this class is an Abstract Base Class that's implemented by the
+* Redteam and Non-Redteam SimulatedUser Providers. Address this in a follow-up PR.
+*/
+var SimulatedUser = class {
+	identifier;
+	maxTurns;
+	rawInstructions;
+	stateful;
+	configInitialMessages;
+	/**
+	* Because the SimulatedUser is inherited by the RedteamMischievousUserProvider, and different
+	* Cloud tasks are used for each, the taskId needs to be explicitly defined/scoped.
+	*/
+	taskId = "tau";
+	constructor({ id, label, config }) {
+		this.identifier = id ?? label ?? "agent-provider";
+		this.maxTurns = config.maxTurns ?? 10;
+		this.rawInstructions = config.instructions || "{{instructions}}";
+		this.stateful = config.stateful ?? false;
+		this.configInitialMessages = config.initialMessages;
+	}
+	id() {
+		return this.identifier;
+	}
+	/**
+	* Validates that a message has the required structure.
+	*/
+	isValidMessage(msg) {
+		return msg && typeof msg === "object" && typeof msg.content === "string" && (msg.role === "user" || msg.role === "assistant" || msg.role === "system");
+	}
+	/**
+	* Safely renders a Nunjucks template string, falling back to the original value on error.
+	*/
+	renderTemplate(template, vars) {
+		if (typeof template !== "string") return template;
+		try {
+			return getNunjucksEngine().renderString(template, vars || {});
+		} catch (err) {
+			logger.warn(`[SimulatedUser] Failed to render template: ${template.substring(0, 100)}. Error: ${err instanceof Error ? err.message : err}`);
+			return template;
+		}
+	}
+	/**
+	* Validates and filters an array of messages, logging warnings for invalid entries.
+	*/
+	validateMessages(messages) {
+		const validMessages = [];
+		for (let i = 0; i < messages.length; i++) if (this.isValidMessage(messages[i])) validMessages.push(messages[i]);
+		else logger.warn(`[SimulatedUser] Invalid message at index ${i}, skipping. Expected {role: 'user'|'assistant'|'system', content: string}, got: ${JSON.stringify(messages[i]).substring(0, 100)}`);
+		return validMessages;
+	}
+	/**
+	* Resolves initial messages from either an array or a file:// path.
+	* Supports loading messages from JSON and YAML files.
+	*/
+	resolveInitialMessages(initialMessages) {
+		if (!initialMessages) return [];
+		if (Array.isArray(initialMessages)) return initialMessages;
+		if (typeof initialMessages === "string") {
+			if (initialMessages.trim() === "") return [];
+			if (initialMessages.startsWith("file://")) {
+				try {
+					const resolved = maybeLoadConfigFromExternalFile(initialMessages);
+					if (Array.isArray(resolved)) return resolved;
+					logger.warn(`[SimulatedUser] Expected array of messages from file, got: ${typeof resolved}. Value: ${JSON.stringify(resolved).substring(0, 200)}`);
+				} catch (error) {
+					logger.warn(`[SimulatedUser] Failed to load initialMessages from file: ${error instanceof Error ? error.message : error}`);
+				}
+				return [];
+			}
+			if (initialMessages.trim().startsWith("[")) try {
+				const parsed = JSON.parse(initialMessages);
+				if (Array.isArray(parsed)) return parsed;
+				logger.warn(`[SimulatedUser] Parsed JSON but got ${typeof parsed} instead of array. Value: ${initialMessages.substring(0, 200)}`);
+			} catch (error) {
+				logger.warn(`[SimulatedUser] Failed to parse initialMessages as JSON: ${error}. Value: ${initialMessages.substring(0, 200)}`);
+			}
+			logger.warn(`[SimulatedUser] initialMessages is a string but could not be resolved: ${initialMessages.substring(0, 200)}`);
+			return [];
+		}
+		return [];
+	}
+	async sendMessageToUser(messages, userProvider) {
+		logger.debug("[SimulatedUser] Sending message to simulated user provider");
+		const flippedMessages = messages.map((message) => {
+			return {
+				role: message.role === "user" ? "assistant" : "user",
+				content: message.content
+			};
+		});
+		const response = await userProvider.callApi(JSON.stringify(flippedMessages));
+		if (response.error) return {
+			messages,
+			error: response.error
+		};
+		logger.debug(`User: ${response.output}`);
+		return {
+			messages: [...messages, {
+				role: "user",
+				content: String(response.output || "")
+			}],
+			tokenUsage: response.tokenUsage
+		};
+	}
+	async sendMessageToAgent(prompt, messages, targetProvider, context) {
+		invariant(context?.prompt?.raw, "Expected context.prompt.raw to be set");
+		const renderedPrompt = context.prompt.function ? prompt : getNunjucksEngine().renderString(context.prompt.raw, context.vars);
+		const targetPrompt = this.stateful ? context.vars?.sessionId ? JSON.stringify([{
+			role: "user",
+			content: messages[messages.length - 1].content
+		}]) : JSON.stringify([{
+			role: "system",
+			content: renderedPrompt
+		}, {
+			role: "user",
+			content: messages[messages.length - 1].content
+		}]) : JSON.stringify([{
+			role: "system",
+			content: renderedPrompt
+		}, ...messages]);
+		logger.debug(`[SimulatedUser] Sending message to target provider: ${targetPrompt}`);
+		const response = await targetProvider.callApi(targetPrompt, context);
+		if (response.sessionId) {
+			context = context ?? {
+				vars: {},
+				prompt: {
+					raw: "",
+					label: "target"
+				}
+			};
+			context.vars.sessionId = response.sessionId;
+		}
+		if (targetProvider.delay) {
+			logger.debug(`[SimulatedUser] Sleeping for ${targetProvider.delay}ms`);
+			await sleep(targetProvider.delay);
+		}
+		logger.debug(`[SimulatedUser] Agent: ${response.output}`);
+		return response;
+	}
+	async callApi(prompt, context, _callApiOptions) {
+		invariant(context?.originalProvider, "Expected originalProvider to be set");
+		const targetProvider = context.originalProvider;
+		const instructions = getNunjucksEngine().renderString(this.rawInstructions, context?.vars);
+		const userProvider = new PromptfooSimulatedUserProvider({ instructions }, this.taskId);
+		logger.debug(`[SimulatedUser] Formatted user instructions: ${instructions}`);
+		const varsInitialMessages = context?.vars?.initialMessages;
+		const templatedMessages = this.resolveInitialMessages(varsInitialMessages || this.configInitialMessages).map((msg) => ({
+			role: this.renderTemplate(msg.role, context?.vars),
+			content: this.renderTemplate(msg.content, context?.vars)
+		}));
+		const messages = this.validateMessages(templatedMessages);
+		if (messages.length > 0) logger.debug(`[SimulatedUser] Starting with ${messages.length} initial messages`);
+		const maxTurns = this.maxTurns;
+		const tokenUsage = createEmptyTokenUsage();
+		let agentResponse;
+		if ((messages.length > 0 ? messages[messages.length - 1].role : null) === "user") {
+			logger.debug("[SimulatedUser] Initial messages end with user message, getting agent response first");
+			agentResponse = await this.sendMessageToAgent(prompt, messages, targetProvider, context);
+			if (agentResponse.error) return {
+				error: agentResponse.error,
+				tokenUsage
+			};
+			messages.push({
+				role: "assistant",
+				content: String(agentResponse.output ?? "")
+			});
+			accumulateResponseTokenUsage(tokenUsage, agentResponse);
+		}
+		for (let i = 0; i < maxTurns; i++) {
+			logger.debug(`[SimulatedUser] Turn ${i + 1} of ${maxTurns}`);
+			const userResult = await this.sendMessageToUser(messages, userProvider);
+			if (userResult.error) return {
+				error: userResult.error,
+				tokenUsage
+			};
+			const { messages: messagesToUser, tokenUsage: userTokenUsage } = userResult;
+			accumulateResponseTokenUsage(tokenUsage, { tokenUsage: userTokenUsage });
+			const lastMessage = messagesToUser[messagesToUser.length - 1];
+			if (lastMessage.content && typeof lastMessage.content === "string" && lastMessage.content.includes("###STOP###")) break;
+			messages.push(lastMessage);
+			agentResponse = await this.sendMessageToAgent(prompt, messagesToUser, targetProvider, context);
+			if (agentResponse.error) return {
+				error: agentResponse.error,
+				tokenUsage
+			};
+			messages.push({
+				role: "assistant",
+				content: String(agentResponse.output ?? "")
+			});
+			accumulateResponseTokenUsage(tokenUsage, agentResponse);
+		}
+		return this.serializeOutput(messages, tokenUsage, agentResponse, getSessionId(agentResponse, context));
+	}
+	toString() {
+		return "AgentProvider";
+	}
+	serializeOutput(messages, tokenUsage, finalTargetResponse, sessionId) {
+		return {
+			output: messages.map((message) => `${message.role === "assistant" ? "Assistant" : "User"}: ${message.content}`).join("\n---\n"),
+			tokenUsage,
+			metadata: {
+				messages,
+				sessionId
+			},
+			guardrails: finalTargetResponse?.guardrails
+		};
+	}
+};
+//#endregion
+export { SimulatedUser as t };
+
+//# sourceMappingURL=simulatedUser-p3tACcmw.js.map

package/dist/src/{slack-Bamy_7te.js → slack-Bapo-7_8.js}

@@ -132,4 +132,4 @@ var SlackProvider = class {
 //#endregion
 export { SlackProvider };
 
-//# sourceMappingURL=slack-Bamy_7te.js.map
+//# sourceMappingURL=slack-Bapo-7_8.js.map

package/dist/src/{slack-BLlsDpfG.cjs → slack-DMC1QVEg.cjs}

@@ -1,4 +1,5 @@
-const require_logger = require("./logger-COuQb2xB.cjs");
+require("./rolldown-runtime-D_mwlA32.cjs");
+const require_logger = require("./logger-cfNpzI4o.cjs");
 let _slack_web_api = require("@slack/web-api");
 //#region src/providers/slack.ts
 var SlackProvider = class {
@@ -132,4 +133,4 @@ var SlackProvider = class {
 //#endregion
 exports.SlackProvider = SlackProvider;
 
-//# sourceMappingURL=slack-BLlsDpfG.cjs.map
+//# sourceMappingURL=slack-DMC1QVEg.cjs.map

package/dist/src/{slack-BPYLQLgb.js → slack-DTEFhrMn.js}

@@ -133,4 +133,4 @@ var SlackProvider = class {
 //#endregion
 export { SlackProvider };
 
-//# sourceMappingURL=slack-BPYLQLgb.js.map
+//# sourceMappingURL=slack-DTEFhrMn.js.map

package/dist/src/{slack-4zZX1OKP.js → slack-k-_CP84Q.js}

@@ -132,4 +132,4 @@ var SlackProvider = class {
 //#endregion
 export { SlackProvider };
 
-//# sourceMappingURL=slack-4zZX1OKP.js.map
+//# sourceMappingURL=slack-k-_CP84Q.js.map