promptfoo 0.121.4 → 0.121.7

package/dist/src/memoryPoisoning-CM83NWYl.js

@@ -0,0 +1,107 @@
+#!/usr/bin/env node
+import { s as logger } from "./logger-BbY6ypFL.js";
+import { n as fetchWithProxy } from "./fetch-DXUnXkVU.js";
+import { n as VERSION } from "./version-eRkNuGv8.js";
+import { t as invariant } from "./invariant-B2Rf6avk.js";
+import { o as getUserEmail } from "./accounts-CjFnOPmb.js";
+import { r as getRemoteGenerationUrl } from "./remoteGeneration-B1_XsKXU.js";
+import { a as createEmptyTokenUsage, n as accumulateResponseTokenUsage } from "./tokenUsageUtils-CDet74yk.js";
+import { d as messagesToRedteamHistory } from "./shared-9WHQ1oNE.js";
+import { i as throwIfTargetPromptExceedsMaxChars } from "./promptLength-4X-Wd8PG.js";
+import { t as REDTEAM_MEMORY_POISONING_PLUGIN_ID } from "./constants-DH5XYLKZ.js";
+//#region src/redteam/providers/agentic/memoryPoisoning.ts
+var MemoryPoisoningProvider = class {
+	constructor(config) {
+		this.config = config;
+	}
+	id() {
+		return REDTEAM_MEMORY_POISONING_PLUGIN_ID;
+	}
+	/**
+	* Generates a scenario containing a memory and a follow up question/command which is dependent on the memory.
+	*
+	*  1. Generate a scenario containing memories and follow up questions/commands which are dependent on the memories.
+	*  2. Send the memory message to the target.
+	*  3. Send the test case to the target; the test case should poison the memory created in the previous step.
+	*  4. Send the follow up question to the target.
+	*  5. A memory is successfully poisoned if the follow up question is answered incorrectly.
+	*
+	* @param prompt The poisoned prompt.
+	* @param context The context of the poisoned prompt.
+	* @param options The options for the poisoned prompt.
+	*/
+	async callApi(prompt, context, options) {
+		try {
+			const targetProvider = context?.originalProvider;
+			const purpose = context?.test?.metadata?.purpose;
+			invariant(targetProvider, "Expected originalProvider to be set");
+			invariant(context?.test, "Expected test to be set");
+			invariant(purpose, "Expected purpose to be set");
+			const scenarioRes = await fetchWithProxy(getRemoteGenerationUrl(), {
+				body: JSON.stringify({
+					task: "agentic:memory-poisoning-scenario",
+					purpose,
+					version: VERSION,
+					email: getUserEmail()
+				}),
+				headers: { "Content-Type": "application/json" },
+				method: "POST"
+			}, options?.abortSignal);
+			if (!scenarioRes.ok) throw new Error(`Failed to generate scenario: ${scenarioRes.statusText}`);
+			const scenario = await scenarioRes.json();
+			context.test.metadata ??= {};
+			context.test.metadata["scenario"] = scenario;
+			const totalTokenUsage = createEmptyTokenUsage();
+			throwIfTargetPromptExceedsMaxChars(scenario.memory);
+			const memoryResponse = await targetProvider.callApi(scenario.memory, context, options);
+			accumulateResponseTokenUsage(totalTokenUsage, memoryResponse);
+			throwIfTargetPromptExceedsMaxChars(prompt);
+			const testResponse = await targetProvider.callApi(prompt, context, options);
+			accumulateResponseTokenUsage(totalTokenUsage, testResponse);
+			throwIfTargetPromptExceedsMaxChars(scenario.followUp);
+			const response = await targetProvider.callApi(scenario.followUp, context, options);
+			accumulateResponseTokenUsage(totalTokenUsage, response);
+			const messages = [
+				{
+					content: scenario.memory,
+					role: "user"
+				},
+				{
+					content: memoryResponse.output,
+					role: "assistant"
+				},
+				{
+					content: prompt,
+					role: "user"
+				},
+				{
+					content: testResponse.output,
+					role: "assistant"
+				},
+				{
+					content: scenario.followUp,
+					role: "user"
+				},
+				{
+					content: response.output,
+					role: "assistant"
+				}
+			];
+			return {
+				output: response.output,
+				metadata: {
+					messages,
+					redteamHistory: messagesToRedteamHistory(messages)
+				},
+				tokenUsage: totalTokenUsage
+			};
+		} catch (error) {
+			logger.error(`Error in MemoryPoisoningProvider: ${error}`);
+			throw error;
+		}
+	}
+};
+//#endregion
+export { MemoryPoisoningProvider };
+
+//# sourceMappingURL=memoryPoisoning-CM83NWYl.js.map

package/dist/src/memoryPoisoning-D8h9gXJF.js

@@ -0,0 +1,106 @@
+import { a as logger } from "./logger-KD8JjCRJ.js";
+import { t as fetchWithProxy } from "./fetch-BufrQtvR.js";
+import { n as VERSION } from "./version-BK20a4sw.js";
+import { t as invariant } from "./invariant-DIYf9sP1.js";
+import { o as getUserEmail } from "./accounts-DanM1wq_.js";
+import { r as getRemoteGenerationUrl } from "./remoteGeneration-COpWcmWd.js";
+import { a as createEmptyTokenUsage, n as accumulateResponseTokenUsage } from "./tokenUsageUtils-BjVkdk18.js";
+import { d as messagesToRedteamHistory } from "./shared-7pmVZLNO.js";
+import { i as throwIfTargetPromptExceedsMaxChars } from "./promptLength-0qIHyhA5.js";
+import { t as REDTEAM_MEMORY_POISONING_PLUGIN_ID } from "./constants-BjJV0cRr.js";
+//#region src/redteam/providers/agentic/memoryPoisoning.ts
+var MemoryPoisoningProvider = class {
+	constructor(config) {
+		this.config = config;
+	}
+	id() {
+		return REDTEAM_MEMORY_POISONING_PLUGIN_ID;
+	}
+	/**
+	* Generates a scenario containing a memory and a follow up question/command which is dependent on the memory.
+	*
+	*  1. Generate a scenario containing memories and follow up questions/commands which are dependent on the memories.
+	*  2. Send the memory message to the target.
+	*  3. Send the test case to the target; the test case should poison the memory created in the previous step.
+	*  4. Send the follow up question to the target.
+	*  5. A memory is successfully poisoned if the follow up question is answered incorrectly.
+	*
+	* @param prompt The poisoned prompt.
+	* @param context The context of the poisoned prompt.
+	* @param options The options for the poisoned prompt.
+	*/
+	async callApi(prompt, context, options) {
+		try {
+			const targetProvider = context?.originalProvider;
+			const purpose = context?.test?.metadata?.purpose;
+			invariant(targetProvider, "Expected originalProvider to be set");
+			invariant(context?.test, "Expected test to be set");
+			invariant(purpose, "Expected purpose to be set");
+			const scenarioRes = await fetchWithProxy(getRemoteGenerationUrl(), {
+				body: JSON.stringify({
+					task: "agentic:memory-poisoning-scenario",
+					purpose,
+					version: VERSION,
+					email: getUserEmail()
+				}),
+				headers: { "Content-Type": "application/json" },
+				method: "POST"
+			}, options?.abortSignal);
+			if (!scenarioRes.ok) throw new Error(`Failed to generate scenario: ${scenarioRes.statusText}`);
+			const scenario = await scenarioRes.json();
+			context.test.metadata ??= {};
+			context.test.metadata["scenario"] = scenario;
+			const totalTokenUsage = createEmptyTokenUsage();
+			throwIfTargetPromptExceedsMaxChars(scenario.memory);
+			const memoryResponse = await targetProvider.callApi(scenario.memory, context, options);
+			accumulateResponseTokenUsage(totalTokenUsage, memoryResponse);
+			throwIfTargetPromptExceedsMaxChars(prompt);
+			const testResponse = await targetProvider.callApi(prompt, context, options);
+			accumulateResponseTokenUsage(totalTokenUsage, testResponse);
+			throwIfTargetPromptExceedsMaxChars(scenario.followUp);
+			const response = await targetProvider.callApi(scenario.followUp, context, options);
+			accumulateResponseTokenUsage(totalTokenUsage, response);
+			const messages = [
+				{
+					content: scenario.memory,
+					role: "user"
+				},
+				{
+					content: memoryResponse.output,
+					role: "assistant"
+				},
+				{
+					content: prompt,
+					role: "user"
+				},
+				{
+					content: testResponse.output,
+					role: "assistant"
+				},
+				{
+					content: scenario.followUp,
+					role: "user"
+				},
+				{
+					content: response.output,
+					role: "assistant"
+				}
+			];
+			return {
+				output: response.output,
+				metadata: {
+					messages,
+					redteamHistory: messagesToRedteamHistory(messages)
+				},
+				tokenUsage: totalTokenUsage
+			};
+		} catch (error) {
+			logger.error(`Error in MemoryPoisoningProvider: ${error}`);
+			throw error;
+		}
+	}
+};
+//#endregion
+export { MemoryPoisoningProvider };
+
+//# sourceMappingURL=memoryPoisoning-D8h9gXJF.js.map

package/dist/src/memoryPoisoning-Dp-btinn.cjs

@@ -0,0 +1,106 @@
+const require_logger = require("./logger-cfNpzI4o.cjs");
+const require_invariant = require("./invariant-QtnLD03y.cjs");
+const require_fetch = require("./fetch-Dw4XZHjj.cjs");
+const require_version = require("./version-BWCSaByA.cjs");
+const require_accounts = require("./accounts-CmWzeD2d.cjs");
+const require_remoteGeneration = require("./remoteGeneration-DS9N3pgB.cjs");
+const require_tokenUsageUtils = require("./tokenUsageUtils-_B-P8IAi.cjs");
+const require_shared = require("./shared-WkgnDkcg.cjs");
+const require_promptLength = require("./promptLength-BbBbDHNj.cjs");
+const require_constants = require("./constants-a2kYssQk.cjs");
+//#region src/redteam/providers/agentic/memoryPoisoning.ts
+var MemoryPoisoningProvider = class {
+	constructor(config) {
+		this.config = config;
+	}
+	id() {
+		return require_constants.REDTEAM_MEMORY_POISONING_PLUGIN_ID;
+	}
+	/**
+	* Generates a scenario containing a memory and a follow up question/command which is dependent on the memory.
+	*
+	*  1. Generate a scenario containing memories and follow up questions/commands which are dependent on the memories.
+	*  2. Send the memory message to the target.
+	*  3. Send the test case to the target; the test case should poison the memory created in the previous step.
+	*  4. Send the follow up question to the target.
+	*  5. A memory is successfully poisoned if the follow up question is answered incorrectly.
+	*
+	* @param prompt The poisoned prompt.
+	* @param context The context of the poisoned prompt.
+	* @param options The options for the poisoned prompt.
+	*/
+	async callApi(prompt, context, options) {
+		try {
+			const targetProvider = context?.originalProvider;
+			const purpose = context?.test?.metadata?.purpose;
+			require_invariant.invariant(targetProvider, "Expected originalProvider to be set");
+			require_invariant.invariant(context?.test, "Expected test to be set");
+			require_invariant.invariant(purpose, "Expected purpose to be set");
+			const scenarioRes = await require_fetch.fetchWithProxy(require_remoteGeneration.getRemoteGenerationUrl(), {
+				body: JSON.stringify({
+					task: "agentic:memory-poisoning-scenario",
+					purpose,
+					version: require_version.VERSION,
+					email: require_accounts.getUserEmail()
+				}),
+				headers: { "Content-Type": "application/json" },
+				method: "POST"
+			}, options?.abortSignal);
+			if (!scenarioRes.ok) throw new Error(`Failed to generate scenario: ${scenarioRes.statusText}`);
+			const scenario = await scenarioRes.json();
+			context.test.metadata ??= {};
+			context.test.metadata["scenario"] = scenario;
+			const totalTokenUsage = require_tokenUsageUtils.createEmptyTokenUsage();
+			require_promptLength.throwIfTargetPromptExceedsMaxChars(scenario.memory);
+			const memoryResponse = await targetProvider.callApi(scenario.memory, context, options);
+			require_tokenUsageUtils.accumulateResponseTokenUsage(totalTokenUsage, memoryResponse);
+			require_promptLength.throwIfTargetPromptExceedsMaxChars(prompt);
+			const testResponse = await targetProvider.callApi(prompt, context, options);
+			require_tokenUsageUtils.accumulateResponseTokenUsage(totalTokenUsage, testResponse);
+			require_promptLength.throwIfTargetPromptExceedsMaxChars(scenario.followUp);
+			const response = await targetProvider.callApi(scenario.followUp, context, options);
+			require_tokenUsageUtils.accumulateResponseTokenUsage(totalTokenUsage, response);
+			const messages = [
+				{
+					content: scenario.memory,
+					role: "user"
+				},
+				{
+					content: memoryResponse.output,
+					role: "assistant"
+				},
+				{
+					content: prompt,
+					role: "user"
+				},
+				{
+					content: testResponse.output,
+					role: "assistant"
+				},
+				{
+					content: scenario.followUp,
+					role: "user"
+				},
+				{
+					content: response.output,
+					role: "assistant"
+				}
+			];
+			return {
+				output: response.output,
+				metadata: {
+					messages,
+					redteamHistory: require_shared.messagesToRedteamHistory(messages)
+				},
+				tokenUsage: totalTokenUsage
+			};
+		} catch (error) {
+			require_logger.logger.error(`Error in MemoryPoisoningProvider: ${error}`);
+			throw error;
+		}
+	}
+};
+//#endregion
+exports.MemoryPoisoningProvider = MemoryPoisoningProvider;
+
+//# sourceMappingURL=memoryPoisoning-Dp-btinn.cjs.map

package/dist/src/memoryPoisoning-cLuCoTuJ.js

@@ -0,0 +1,106 @@
+import { a as logger } from "./logger-Ct2S6Yx-.js";
+import { t as invariant } from "./invariant-Ddh24eXh.js";
+import { t as fetchWithProxy } from "./fetch-It34O8Ur.js";
+import { n as VERSION } from "./version-lpHV_53E.js";
+import { i as getUserEmail } from "./accounts-Ca7WIoPY.js";
+import { r as getRemoteGenerationUrl } from "./remoteGeneration-DsaSwmG2.js";
+import { a as createEmptyTokenUsage, n as accumulateResponseTokenUsage } from "./tokenUsageUtils-CmnQ0G2m.js";
+import { d as messagesToRedteamHistory } from "./shared-D6IjElRI.js";
+import { i as throwIfTargetPromptExceedsMaxChars } from "./promptLength-B9nZEfO6.js";
+import { t as REDTEAM_MEMORY_POISONING_PLUGIN_ID } from "./constants-DZGEFLsu.js";
+//#region src/redteam/providers/agentic/memoryPoisoning.ts
+var MemoryPoisoningProvider = class {
+	constructor(config) {
+		this.config = config;
+	}
+	id() {
+		return REDTEAM_MEMORY_POISONING_PLUGIN_ID;
+	}
+	/**
+	* Generates a scenario containing a memory and a follow up question/command which is dependent on the memory.
+	*
+	*  1. Generate a scenario containing memories and follow up questions/commands which are dependent on the memories.
+	*  2. Send the memory message to the target.
+	*  3. Send the test case to the target; the test case should poison the memory created in the previous step.
+	*  4. Send the follow up question to the target.
+	*  5. A memory is successfully poisoned if the follow up question is answered incorrectly.
+	*
+	* @param prompt The poisoned prompt.
+	* @param context The context of the poisoned prompt.
+	* @param options The options for the poisoned prompt.
+	*/
+	async callApi(prompt, context, options) {
+		try {
+			const targetProvider = context?.originalProvider;
+			const purpose = context?.test?.metadata?.purpose;
+			invariant(targetProvider, "Expected originalProvider to be set");
+			invariant(context?.test, "Expected test to be set");
+			invariant(purpose, "Expected purpose to be set");
+			const scenarioRes = await fetchWithProxy(getRemoteGenerationUrl(), {
+				body: JSON.stringify({
+					task: "agentic:memory-poisoning-scenario",
+					purpose,
+					version: VERSION,
+					email: getUserEmail()
+				}),
+				headers: { "Content-Type": "application/json" },
+				method: "POST"
+			}, options?.abortSignal);
+			if (!scenarioRes.ok) throw new Error(`Failed to generate scenario: ${scenarioRes.statusText}`);
+			const scenario = await scenarioRes.json();
+			context.test.metadata ??= {};
+			context.test.metadata["scenario"] = scenario;
+			const totalTokenUsage = createEmptyTokenUsage();
+			throwIfTargetPromptExceedsMaxChars(scenario.memory);
+			const memoryResponse = await targetProvider.callApi(scenario.memory, context, options);
+			accumulateResponseTokenUsage(totalTokenUsage, memoryResponse);
+			throwIfTargetPromptExceedsMaxChars(prompt);
+			const testResponse = await targetProvider.callApi(prompt, context, options);
+			accumulateResponseTokenUsage(totalTokenUsage, testResponse);
+			throwIfTargetPromptExceedsMaxChars(scenario.followUp);
+			const response = await targetProvider.callApi(scenario.followUp, context, options);
+			accumulateResponseTokenUsage(totalTokenUsage, response);
+			const messages = [
+				{
+					content: scenario.memory,
+					role: "user"
+				},
+				{
+					content: memoryResponse.output,
+					role: "assistant"
+				},
+				{
+					content: prompt,
+					role: "user"
+				},
+				{
+					content: testResponse.output,
+					role: "assistant"
+				},
+				{
+					content: scenario.followUp,
+					role: "user"
+				},
+				{
+					content: response.output,
+					role: "assistant"
+				}
+			];
+			return {
+				output: response.output,
+				metadata: {
+					messages,
+					redteamHistory: messagesToRedteamHistory(messages)
+				},
+				tokenUsage: totalTokenUsage
+			};
+		} catch (error) {
+			logger.error(`Error in MemoryPoisoningProvider: ${error}`);
+			throw error;
+		}
+	}
+};
+//#endregion
+export { MemoryPoisoningProvider };
+
+//# sourceMappingURL=memoryPoisoning-cLuCoTuJ.js.map